From 1a2c0ec6fb7570641674cbc531eb637ea3aee040 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 21 Aug 2024 15:25:14 +0800 Subject: [PATCH 001/120] add cpp related code (#1) * add git lfs and gitignore * add cpp related code --- .gitattributes | 4 + .gitignore | 145 + cpp/CMakeLists.txt | 141 + cpp/cmake/BuildInfo.cmake | 44 + cpp/cmake/BuildInfoGenerator.cmake | 3 + cpp/cmake/CompilerSettings.cmake | 166 + cpp/cmake/FindGSasl.cmake | 27 + cpp/cmake/IncludeDirectories.cmake | 9 + cpp/cmake/InstallBcosUtilities.cmake | 3 + cpp/cmake/Installlibhdfs3.cmake | 6 + cpp/cmake/Options.cmake | 151 + cpp/cmake/ProjectAPSI.cmake | 28 + cpp/cmake/ProjectTCMalloc.cmake | 41 + cpp/cmake/ProjectWedprCrypto.cmake | 35 + cpp/cmake/TargetSettings.cmake | 122 + cpp/cmake/Version.cmake | 1 + cpp/cmake/config.cmake | 74 + cpp/cmake/scripts/buildinfo.cmake | 70 + cpp/cmake/templates/BuildInfo.h.in | 29 + cpp/libhelper/CMakeLists.txt | 8 + cpp/libhelper/CommandHelper.cpp | 111 + cpp/libhelper/CommandHelper.h | 33 + cpp/libhelper/ExitHandler.h | 50 + cpp/libinitializer/CMakeLists.txt | 4 + cpp/libinitializer/Common.h | 79 + cpp/libinitializer/FrontInitializer.cpp | 74 + cpp/libinitializer/FrontInitializer.h | 104 + cpp/libinitializer/Initializer.cpp | 544 + cpp/libinitializer/Initializer.h | 97 + cpp/libinitializer/ProFrontInitializer.h | 79 + cpp/libinitializer/ProtocolInitializer.h | 118 + cpp/ppc-cem/CMakeLists.txt | 8 + cpp/ppc-cem/src/CEMService.cpp | 371 + cpp/ppc-cem/src/CEMService.h | 63 + cpp/ppc-cem/src/CMakeLists.txt | 6 + cpp/ppc-cem/src/Common.h | 31 + cpp/ppc-cem/src/WedprUtilities.h | 28 + cpp/ppc-cem/src/wedpr_ffi_c_equality.h | 25 + cpp/ppc-cem/tests/CMakeLists.txt | 11 + cpp/ppc-cem/tests/TestCEMService.cpp | 136 + cpp/ppc-cem/tests/data/config.ini | 70 + cpp/ppc-cem/tests/data/d1 | 101 + cpp/ppc-cem/tests/data/d2 | 101 + cpp/ppc-cem/tests/data/d3 | 101 + cpp/ppc-cem/tests/main.cpp | 2 + cpp/ppc-crypto-c-sdk/CMakeLists.txt | 23 + .../bindings/java/build.gradle | 199 + .../java/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + cpp/ppc-crypto-c-sdk/bindings/java/gradlew | 185 + .../bindings/java/gradlew.bat | 89 + .../bindings/java/settings.gradle | 2 + .../wedpr/sdk/jni/jmh/BenchmarkMain.java | 27 + .../wedpr/sdk/jni/jmh/PaillierBenchmark.java | 101 + .../bindings/java/src/main/c/CMakeLists.txt | 15 + .../bindings/java/src/main/c/jni/Common.cpp | 114 + .../bindings/java/src/main/c/jni/Common.h | 33 + .../java/src/main/c/jni/JNIException.cpp | 49 + .../java/src/main/c/jni/JNIException.h | 38 + ...om_webank_wedpr_sdk_jni_crypto_FastOre.cpp | 251 + .../com_webank_wedpr_sdk_jni_crypto_FastOre.h | 100 + ...dpr_sdk_jni_crypto_SymmetricEncryption.cpp | 110 + ...wedpr_sdk_jni_crypto_SymmetricEncryption.h | 45 + ...k_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp | 142 + ...ank_wedpr_sdk_jni_homo_NativeFloatingIhc.h | 61 + ...k_jni_homo_NativeFloatingPointPaillier.cpp | 445 + ...sdk_jni_homo_NativeFloatingPointPaillier.h | 126 + .../com_webank_wedpr_sdk_jni_homo_NativeIhc.h | 61 + ...bank_wedpr_sdk_jni_homo_NativePaillier.cpp | 629 + ...webank_wedpr_sdk_jni_homo_NativePaillier.h | 235 + .../webank/wedpr/sdk/demo/GenerateKeys.java | 64 + .../sdk/jni/codec/FloatingPointNumber.java | 111 + .../wedpr/sdk/jni/codec/NumberCodec.java | 43 + .../sdk/jni/codec/NumberCodecException.java | 21 + .../wedpr/sdk/jni/codec/NumberCodecImpl.java | 149 + .../wedpr/sdk/jni/common/JniException.java | 20 + .../wedpr/sdk/jni/common/JniLibLoader.java | 278 + .../wedpr/sdk/jni/common/Utilities.java | 28 + .../webank/wedpr/sdk/jni/crypto/FastOre.java | 51 + .../sdk/jni/crypto/SymmetricEncryption.java | 49 + .../wedpr/sdk/jni/homo/NativeFloatingIhc.java | 31 + .../jni/homo/NativeFloatingPointPaillier.java | 60 + .../wedpr/sdk/jni/homo/NativePaillier.java | 241 + .../resources/META-INF/native/win/file.list | 7 + .../sdk/jni/test/NativeFloatingIhcTest.java | 86 + .../test/NativeFloatingPointPaillierTest.java | 237 + .../sdk/jni/test/NativePaillierTest.java | 144 + .../wedpr/sdk/jni/test/NumberCodecTest.java | 86 + .../sdk/jni/test/SymmetricEncryptionTest.java | 194 + .../ppc-crypto-c-sdk/fast_ore.cpp | 348 + .../ppc-crypto-c-sdk/fast_ore.h | 113 + .../ppc-crypto-c-sdk/floating_point_ihc.cpp | 172 + .../ppc-crypto-c-sdk/floating_point_ihc.h | 45 + .../floating_point_paillier.cpp | 264 + .../floating_point_paillier.h | 111 + .../ppc-crypto-c-sdk/homo_ihc.cpp | 192 + .../ppc-crypto-c-sdk/homo_ihc.h | 46 + .../ppc-crypto-c-sdk/homo_paillier.cpp | 555 + .../ppc-crypto-c-sdk/homo_paillier.h | 265 + .../ppc-crypto-c-sdk/symmetric_encryption.cpp | 187 + .../ppc-crypto-c-sdk/symmetric_encryption.h | 81 + .../ppc-crypto-c-sdk/utils/error.cpp | 73 + .../ppc-crypto-c-sdk/utils/error.h | 67 + .../ppc-crypto-c-sdk/utils/utilities.cpp | 62 + .../ppc-crypto-c-sdk/utils/utilities.h | 34 + cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt | 10 + cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp | 150 + .../tests/TestHomoPaillier.cpp | 461 + cpp/ppc-crypto-c-sdk/tests/main.cpp | 2 + cpp/ppc-crypto-core/CMakeLists.txt | 8 + cpp/ppc-crypto-core/src/CMakeLists.txt | 8 + cpp/ppc-crypto-core/src/Common.h | 34 + cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h | 136 + .../src/hash/BitMixMurmurHash.h | 283 + .../src/hash/HashFactoryImpl.h | 58 + cpp/ppc-crypto-core/src/hash/MD5Hash.h | 102 + cpp/ppc-crypto-core/src/hash/OpenSSLHash.h | 0 cpp/ppc-crypto-core/src/hash/SM3Hash.h | 119 + cpp/ppc-crypto-core/src/hash/Sha256Hash.h | 107 + cpp/ppc-crypto-core/src/hash/Sha512Hash.h | 107 + cpp/ppc-crypto-core/src/ore/FastOre.cpp | 230 + cpp/ppc-crypto-core/src/ore/FastOre.h | 177 + .../src/sym-crypto/OpenSSL3DES.cpp | 42 + .../src/sym-crypto/OpenSSL3DES.h | 50 + .../src/sym-crypto/OpenSSLAES.cpp | 89 + .../src/sym-crypto/OpenSSLAES.h | 59 + .../src/sym-crypto/OpenSSLCipher.h | 218 + .../src/sym-crypto/OpenSSLSM4.cpp | 44 + .../src/sym-crypto/OpenSSLSM4.h | 44 + cpp/ppc-crypto-core/src/tools/BitVector.cpp | 88 + cpp/ppc-crypto-core/src/tools/BitVector.h | 83 + cpp/ppc-crypto-core/tests/CMakeLists.txt | 10 + cpp/ppc-crypto-core/tests/TestBitVector.cpp | 60 + cpp/ppc-crypto-core/tests/TestHash.cpp | 132 + cpp/ppc-crypto-core/tests/TestOre4Float.cpp | 214 + cpp/ppc-crypto-core/tests/TestOre4Integer.cpp | 179 + cpp/ppc-crypto-core/tests/TestOre4String.cpp | 193 + cpp/ppc-crypto-core/tests/TestSymCrypto.cpp | 318 + cpp/ppc-crypto-core/tests/main.cpp | 2 + cpp/ppc-crypto/CMakeLists.txt | 11 + cpp/ppc-crypto/demo/CMakeLists.txt | 7 + cpp/ppc-crypto/demo/crypto_bench.cpp | 439 + cpp/ppc-crypto/src/CMakeLists.txt | 14 + cpp/ppc-crypto/src/Common.h | 51 + .../src/ecc/ECDHCryptoFactoryImpl.h | 72 + cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h | 74 + cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h | 52 + cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp | 207 + cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h | 59 + cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h | 121 + cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp | 309 + cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h | 115 + cpp/ppc-crypto/src/ecc/core/EcPoint.h | 191 + cpp/ppc-crypto/src/oprf/EcdhOprf.cpp | 183 + cpp/ppc-crypto/src/oprf/EcdhOprf.h | 115 + cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp | 78 + cpp/ppc-crypto/src/oprf/RA2018Oprf.h | 88 + cpp/ppc-crypto/src/prng/AESPRNG.cpp | 76 + cpp/ppc-crypto/src/prng/AESPRNG.h | 61 + cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp | 74 + cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h | 61 + cpp/ppc-crypto/src/randomot/SimplestOT.cpp | 100 + cpp/ppc-crypto/src/randomot/SimplestOT.h | 65 + cpp/ppc-crypto/tests/CMakeLists.txt | 10 + cpp/ppc-crypto/tests/TestEccCrypto.cpp | 264 + cpp/ppc-crypto/tests/TestOprf.cpp | 110 + cpp/ppc-crypto/tests/TestPRNG.cpp | 112 + cpp/ppc-crypto/tests/TestRandomot.cpp | 102 + cpp/ppc-crypto/tests/main.cpp | 2 + cpp/ppc-framework/Common.h | 74 + cpp/ppc-framework/crypto/CryptoBox.h | 41 + cpp/ppc-framework/crypto/ECDHCrypto.h | 51 + cpp/ppc-framework/crypto/EccCrypto.h | 68 + cpp/ppc-framework/crypto/Hash.h | 66 + cpp/ppc-framework/crypto/Ihc.h | 74 + cpp/ppc-framework/crypto/KeyPair.h | 42 + cpp/ppc-framework/crypto/Oprf.h | 115 + cpp/ppc-framework/crypto/Ore.h | 77 + cpp/ppc-framework/crypto/PRNG.h | 80 + cpp/ppc-framework/crypto/Paillier.h | 68 + .../crypto/RA2018OprfInterface.h | 46 + cpp/ppc-framework/crypto/SymCrypto.h | 73 + cpp/ppc-framework/front/Channel.h | 79 + cpp/ppc-framework/front/FrontInterface.h | 97 + cpp/ppc-framework/gateway/GatewayInterface.h | 81 + cpp/ppc-framework/http/HttpInterface.h | 55 + cpp/ppc-framework/http/HttpTypeDef.h | 32 + cpp/ppc-framework/io/DataBatch.h | 220 + cpp/ppc-framework/io/DataResourceLoader.h | 54 + cpp/ppc-framework/io/LineReader.h | 53 + cpp/ppc-framework/io/LineWriter.h | 55 + cpp/ppc-framework/libwrapper/BigNum.h | 479 + cpp/ppc-framework/libwrapper/Buffer.h | 44 + .../libwrapper/FloatingPointNumber.h | 41 + .../libwrapper/OreFloatingNumber.h | 80 + cpp/ppc-framework/protocol/DataResource.h | 140 + cpp/ppc-framework/protocol/GlobalConfig.h | 92 + cpp/ppc-framework/protocol/PPCMessageFace.h | 109 + cpp/ppc-framework/protocol/PartyResource.h | 81 + cpp/ppc-framework/protocol/Protocol.h | 505 + cpp/ppc-framework/protocol/Task.h | 173 + .../psi/TaskFrameworkInterface.h | 47 + cpp/ppc-framework/rpc/RpcInterface.h | 59 + cpp/ppc-framework/rpc/RpcStatusInterface.h | 53 + cpp/ppc-framework/rpc/RpcTypeDef.h | 48 + cpp/ppc-framework/storage/CacheStorage.h | 114 + cpp/ppc-framework/storage/FileStorage.h | 72 + cpp/ppc-framework/storage/RemoteStorage.h | 48 + cpp/ppc-framework/storage/SQLStorage.h | 116 + .../task/TaskFrameworkInterface.h | 47 + cpp/ppc-front/CMakeLists.txt | 13 + cpp/ppc-front/ppc-front/Common.h | 41 + cpp/ppc-front/ppc-front/Front.cpp | 365 + cpp/ppc-front/ppc-front/Front.h | 190 + cpp/ppc-front/ppc-front/PPCChannel.cpp | 183 + cpp/ppc-front/ppc-front/PPCChannel.h | 128 + cpp/ppc-front/ppc-front/PPCChannelManager.cpp | 138 + cpp/ppc-front/ppc-front/PPCChannelManager.h | 90 + cpp/ppc-front/test/CMakeLists.txt | 13 + .../test/unittests/PPCChannelTest.cpp | 118 + cpp/ppc-front/test/unittests/main.cpp | 2 + cpp/ppc-gateway/CMakeLists.txt | 24 + cpp/ppc-gateway/ppc-gateway/Common.h | 55 + .../ppc-gateway/FrontNodeManager.cpp | 62 + .../ppc-gateway/FrontNodeManager.h | 54 + cpp/ppc-gateway/ppc-gateway/Gateway.cpp | 774 ++ cpp/ppc-gateway/ppc-gateway/Gateway.h | 210 + .../ppc-gateway/GatewayConfigContext.cpp | 55 + .../ppc-gateway/GatewayConfigContext.h | 62 + cpp/ppc-gateway/ppc-gateway/ProTaskManager.h | 96 + cpp/ppc-gateway/ppc-gateway/TaskManager.cpp | 113 + cpp/ppc-gateway/ppc-gateway/TaskManager.h | 69 + .../ppc-gateway/WebSocketService.cpp | 387 + .../ppc-gateway/WebSocketService.h | 147 + cpp/ppc-gateway/test/CMakeLists.txt | 17 + cpp/ppc-gateway/test/data/config0.ini | 32 + cpp/ppc-gateway/test/demo/CMakeLists.txt | 8 + cpp/ppc-gateway/test/demo/gateway_demo.cpp | 136 + .../test/unittests/GatewayTest.cpp | 95 + cpp/ppc-gateway/test/unittests/MockCache.h | 86 + cpp/ppc-gateway/test/unittests/main.cpp | 2 + cpp/ppc-homo/CMakeLists.txt | 14 + cpp/ppc-homo/codec/CMakeLists.txt | 5 + cpp/ppc-homo/codec/Common.h | 58 + cpp/ppc-homo/codec/FloatingPointCipher.h | 72 + cpp/ppc-homo/codec/FloatingPointCodec.cpp | 95 + cpp/ppc-homo/codec/FloatingPointCodec.h | 47 + cpp/ppc-homo/codec/SignedNumberCodec.h | 109 + cpp/ppc-homo/demo/CMakeLists.txt | 7 + cpp/ppc-homo/demo/homo_perf_demo.cpp | 311 + cpp/ppc-homo/fahe/CMakeLists.txt | 5 + cpp/ppc-homo/fahe/Fahe.cpp | 125 + cpp/ppc-homo/fahe/Fahe.h | 55 + cpp/ppc-homo/ihc/CMakeLists.txt | 6 + cpp/ppc-homo/ihc/FloatingPointIhc.cpp | 71 + cpp/ppc-homo/ihc/FloatingPointIhc.h | 109 + cpp/ppc-homo/ihc/IhcImpl.cpp | 179 + cpp/ppc-homo/ihc/IhcImpl.h | 132 + cpp/ppc-homo/paillier/CMakeLists.txt | 5 + cpp/ppc-homo/paillier/Common.h | 34 + .../paillier/FloatingPointPaillier.cpp | 114 + cpp/ppc-homo/paillier/FloatingPointPaillier.h | 67 + cpp/ppc-homo/paillier/OpenSSLPaillier.cpp | 315 + cpp/ppc-homo/paillier/OpenSSLPaillier.h | 170 + .../paillier/OpenSSLPaillierKeyPair.cpp | 76 + .../paillier/OpenSSLPaillierKeyPair.h | 190 + cpp/ppc-homo/tests/CMakeLists.txt | 10 + cpp/ppc-homo/tests/TestFahe.cpp | 108 + cpp/ppc-homo/tests/TestFloatingPointCodec.cpp | 258 + .../tests/TestFloatingPointPaillier.cpp | 160 + cpp/ppc-homo/tests/TestIhc.cpp | 121 + cpp/ppc-homo/tests/TestIhcFloating.cpp | 127 + cpp/ppc-homo/tests/TestPaillier.cpp | 267 + cpp/ppc-homo/tests/main.cpp | 2 + cpp/ppc-http/CMakeLists.txt | 6 + cpp/ppc-http/demo/CMakeLists.txt | 16 + cpp/ppc-http/demo/config.ini | 18 + cpp/ppc-http/demo/http_client_demo.cpp | 140 + cpp/ppc-http/demo/http_demo.cpp | 77 + cpp/ppc-http/src/CMakeLists.txt | 4 + cpp/ppc-http/src/Http.cpp | 66 + cpp/ppc-http/src/Http.h | 132 + cpp/ppc-http/src/HttpClient.cpp | 122 + cpp/ppc-http/src/HttpClient.h | 81 + cpp/ppc-http/src/HttpFactory.cpp | 92 + cpp/ppc-http/src/HttpFactory.h | 48 + cpp/ppc-io/CMakeLists.txt | 12 + cpp/ppc-io/demo/CMakeLists.txt | 11 + cpp/ppc-io/demo/io_demo.cpp | 101 + cpp/ppc-io/src/BaseFileLineReader.cpp | 93 + cpp/ppc-io/src/BaseFileLineReader.h | 130 + cpp/ppc-io/src/CMakeLists.txt | 3 + cpp/ppc-io/src/Common.h | 66 + cpp/ppc-io/src/DataResourceLoaderImpl.cpp | 321 + cpp/ppc-io/src/DataResourceLoaderImpl.h | 82 + cpp/ppc-io/src/FileLineReader.cpp | 101 + cpp/ppc-io/src/FileLineReader.h | 68 + cpp/ppc-io/src/FileLineWriter.cpp | 94 + cpp/ppc-io/src/FileLineWriter.h | 84 + cpp/ppc-io/src/HDFSReader.cpp | 120 + cpp/ppc-io/src/HDFSReader.h | 101 + cpp/ppc-io/src/HDFSWriter.cpp | 93 + cpp/ppc-io/src/HDFSWriter.h | 103 + cpp/ppc-io/src/SQLResultReader.h | 72 + cpp/ppc-io/src/parser/BaseBufferParser.h | 96 + cpp/ppc-io/src/parser/BufferParser.h | 39 + cpp/ppc-io/src/parser/MmapParser.h | 64 + cpp/ppc-io/tests/CMakeLists.txt | 10 + cpp/ppc-io/tests/TestLineReader.cpp | 190 + cpp/ppc-io/tests/TestLineWriter.cpp | 55 + cpp/ppc-io/tests/data/linux_file.txt | 10 + cpp/ppc-io/tests/data/testData | 10002 ++++++++++++++++ cpp/ppc-io/tests/data/windows_file.txt | 10 + cpp/ppc-io/tests/main.cpp | 2 + cpp/ppc-main/CMakeLists.txt | 34 + cpp/ppc-main/air-node/AirNodeInitializer.cpp | 125 + cpp/ppc-main/air-node/AirNodeInitializer.h | 52 + cpp/ppc-main/air-node/CMakeLists.txt | 4 + cpp/ppc-main/air-node/main.cpp | 77 + cpp/ppc-main/cem-node/CEMInitializer.cpp | 76 + cpp/ppc-main/cem-node/CEMInitializer.h | 46 + cpp/ppc-main/cem-node/CMakeLists.txt | 4 + cpp/ppc-main/cem-node/main.cpp | 78 + cpp/ppc-main/gateway/CMakeLists.txt | 12 + cpp/ppc-main/gateway/GatewayServiceApp.cpp | 133 + cpp/ppc-main/gateway/GatewayServiceApp.h | 54 + cpp/ppc-main/gateway/main.cpp | 46 + cpp/ppc-main/mpc-node/CMakeLists.txt | 4 + cpp/ppc-main/mpc-node/MPCInitializer.cpp | 77 + cpp/ppc-main/mpc-node/MPCInitializer.h | 46 + cpp/ppc-main/mpc-node/main.cpp | 78 + cpp/ppc-main/pro-node/CMakeLists.txt | 11 + cpp/ppc-main/pro-node/ProNodeServiceApp.cpp | 147 + cpp/ppc-main/pro-node/ProNodeServiceApp.h | 52 + cpp/ppc-main/pro-node/main.cpp | 44 + cpp/ppc-mpc/CMakeLists.txt | 9 + cpp/ppc-mpc/src/CMakeLists.txt | 4 + cpp/ppc-mpc/src/Common.h | 66 + cpp/ppc-mpc/src/MPCService.cpp | 426 + cpp/ppc-mpc/src/MPCService.h | 68 + cpp/ppc-mpc/tests/CMakeLists.txt | 9 + cpp/ppc-mpc/tests/TestMPCService.cpp | 124 + cpp/ppc-mpc/tests/data/config.ini | 56 + cpp/ppc-mpc/tests/main.cpp | 2 + cpp/ppc-pir/CMakeLists.txt | 8 + cpp/ppc-pir/src/BaseOT.cpp | 288 + cpp/ppc-pir/src/BaseOT.h | 105 + cpp/ppc-pir/src/CMakeLists.txt | 31 + cpp/ppc-pir/src/Common.h | 196 + cpp/ppc-pir/src/OtPIRConfig.h | 70 + cpp/ppc-pir/src/OtPIRFactory.h | 48 + cpp/ppc-pir/src/OtPIRImpl.cpp | 562 + cpp/ppc-pir/src/OtPIRImpl.h | 215 + cpp/ppc-pir/src/protocol/OtPIR.tars | 15 + cpp/ppc-pir/tests/CMakeLists.txt | 12 + cpp/ppc-pir/tests/FakeOtPIRFactory.h | 92 + cpp/ppc-pir/tests/TestBaseOT.cpp | 250 + cpp/ppc-pir/tests/main.cpp | 2 + cpp/ppc-pir/tests/test.json | 6 + cpp/ppc-protocol/CMakeLists.txt | 8 + cpp/ppc-protocol/src/CMakeLists.txt | 4 + cpp/ppc-protocol/src/JsonTaskImpl.cpp | 431 + cpp/ppc-protocol/src/JsonTaskImpl.h | 178 + cpp/ppc-protocol/src/PPCMessage.cpp | 204 + cpp/ppc-protocol/src/PPCMessage.h | 158 + cpp/ppc-protocol/tests/CMakeLists.txt | 10 + cpp/ppc-protocol/tests/PPCMessageTest.cpp | 81 + cpp/ppc-protocol/tests/TestTaskImpl.cpp | 85 + cpp/ppc-protocol/tests/main.cpp | 2 + cpp/ppc-psi/CMakeLists.txt | 19 + cpp/ppc-psi/demo/CMakeLists.txt | 7 + cpp/ppc-psi/demo/hash_collision_bench.cpp | 103 + cpp/ppc-psi/src/Common.h | 152 + cpp/ppc-psi/src/PSIConfig.h | 171 + .../src/bs-ecdh-psi/BsEcdhPSIFactory.h | 45 + cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp | 384 + cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h | 165 + .../src/bs-ecdh-psi/BsEcdhPSIInterface.h | 59 + cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt | 4 + cpp/ppc-psi/src/bs-ecdh-psi/Common.h | 41 + .../src/bs-ecdh-psi/core/BsEcdhCache.cpp | 555 + .../src/bs-ecdh-psi/core/BsEcdhCache.h | 190 + .../src/bs-ecdh-psi/core/BsEcdhIoHandler.h | 202 + .../src/bs-ecdh-psi/core/BsEcdhTaskState.h | 184 + .../bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h | 49 + .../src/bs-ecdh-psi/ffi/wedpr_utilities.h | 51 + .../src/bs-ecdh-psi/protocol/BsEcdhResult.h | 70 + .../src/bs-ecdh-psi/protocol/Message.h | 409 + cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h | 68 + cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h | 46 + cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp | 588 + cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h | 187 + cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt | 31 + cpp/ppc-psi/src/cm2020-psi/Common.h | 69 + .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 981 ++ .../src/cm2020-psi/core/CM2020PSIReceiver.h | 156 + .../src/cm2020-psi/core/CM2020PSISender.cpp | 890 ++ .../src/cm2020-psi/core/CM2020PSISender.h | 152 + cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h | 95 + .../src/cm2020-psi/protocol/CM2020PSI.tars | 8 + .../src/cm2020-psi/protocol/CM2020PSIResult.h | 112 + cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt | 31 + cpp/ppc-psi/src/ecdh-conn-psi/Common.h | 84 + .../src/ecdh-conn-psi/EcdhConnPSIConfig.h | 74 + .../src/ecdh-conn-psi/EcdhConnPSIFactory.h | 50 + .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp | 311 + .../src/ecdh-conn-psi/EcdhConnPSIImpl.h | 160 + .../ecdh-conn-psi/EcdhConnPSIMessageFactory.h | 109 + .../src/ecdh-conn-psi/EcdhConnTaskState.h | 87 + .../ecdh-conn-psi/core/EcdhConnPSIClient.cpp | 325 + .../ecdh-conn-psi/core/EcdhConnPSIClient.h | 76 + .../ecdh-conn-psi/core/EcdhConnPSIServer.cpp | 349 + .../ecdh-conn-psi/core/EcdhConnPSIServer.h | 77 + .../ecdh-conn-psi/protocol/PSIConnMessage.h | 202 + .../protocol/PSIConnMessageFactory.cpp | 253 + .../protocol/PSIConnMessageFactory.h | 79 + cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt | 31 + cpp/ppc-psi/src/ecdh-multi-psi/Common.h | 19 + .../src/ecdh-multi-psi/EcdhMultiCache.h | 247 + .../src/ecdh-multi-psi/EcdhMultiPSIConfig.h | 51 + .../src/ecdh-multi-psi/EcdhMultiPSIFactory.h | 31 + .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp | 379 + .../src/ecdh-multi-psi/EcdhMultiPSIImpl.h | 167 + .../EcdhMultiPSIMessageFactory.h | 35 + .../core/EcdhMultiPSICalculator.cpp | 433 + .../core/EcdhMultiPSICalculator.h | 101 + .../core/EcdhMultiPSIMaster.cpp | 463 + .../ecdh-multi-psi/core/EcdhMultiPSIMaster.h | 116 + .../core/EcdhMultiPSIPartner.cpp | 209 + .../ecdh-multi-psi/core/EcdhMultiPSIPartner.h | 76 + cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt | 3 + cpp/ppc-psi/src/ecdh-psi/Common.h | 24 + cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp | 304 + cpp/ppc-psi/src/ecdh-psi/EcdhCache.h | 206 + cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h | 77 + cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h | 48 + cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp | 780 ++ cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h | 92 + .../src/ecdh-psi/EcdhPSIMessageFactory.h | 49 + cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h | 68 + cpp/ppc-psi/src/labeled-psi/CMakeLists.txt | 30 + cpp/ppc-psi/src/labeled-psi/Common.h | 103 + .../src/labeled-psi/LabeledPSIConfig.h | 71 + .../src/labeled-psi/LabeledPSIFactory.h | 46 + .../src/labeled-psi/LabeledPSIImpl.cpp | 794 ++ cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h | 150 + .../src/labeled-psi/core/BinBundle.cpp | 1287 ++ cpp/ppc-psi/src/labeled-psi/core/BinBundle.h | 430 + .../src/labeled-psi/core/LabeledPSIParams.h | 170 + .../labeled-psi/core/LabeledPSIReceiver.cpp | 657 + .../src/labeled-psi/core/LabeledPSIReceiver.h | 143 + .../src/labeled-psi/core/LabeledPSISender.cpp | 461 + .../src/labeled-psi/core/LabeledPSISender.h | 93 + .../src/labeled-psi/core/QueryPackage.cpp | 128 + .../src/labeled-psi/core/QueryPackage.h | 64 + .../src/labeled-psi/core/ResultPackage.cpp | 61 + .../src/labeled-psi/core/ResultPackage.h | 69 + cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp | 1184 ++ cpp/ppc-psi/src/labeled-psi/core/SenderDB.h | 374 + .../src/labeled-psi/core/TaskCommand.h | 95 + .../src/labeled-psi/protocol/LabeledPSI.tars | 45 + .../labeled-psi/protocol/LabeledPSIResult.h | 93 + .../src/labeled-psi/protocol/Protocol.h | 102 + .../src/labeled-psi/protocol/SenderCache.tars | 42 + cpp/ppc-psi/src/psi-framework/CMakeLists.txt | 29 + .../src/psi-framework/PSIFramework.cpp | 757 ++ cpp/ppc-psi/src/psi-framework/PSIFramework.h | 250 + cpp/ppc-psi/src/psi-framework/TaskGuarder.h | 336 + cpp/ppc-psi/src/psi-framework/TaskState.h | 460 + .../interfaces/PSIMessageFactory.h | 41 + .../interfaces/PSIMessageInterface.h | 161 + .../src/psi-framework/protocol/PSI.tars | 28 + .../src/psi-framework/protocol/PSIMessage.cpp | 42 + .../src/psi-framework/protocol/PSIMessage.h | 298 + cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt | 3 + cpp/ppc-psi/src/ra2018-psi/Common.h | 84 + cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h | 123 + cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h | 58 + cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp | 1077 ++ cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h | 186 + .../src/ra2018-psi/core/CuckooFilterInfo.h | 94 + .../src/ra2018-psi/core/CuckooFilterState.h | 106 + .../ra2018-psi/core/CuckooFiltersCache.cpp | 223 + .../src/ra2018-psi/core/CuckooFiltersCache.h | 87 + .../ra2018-psi/core/CuckoofilterAllocator.cpp | 150 + .../ra2018-psi/core/CuckoofilterAllocator.h | 67 + .../src/ra2018-psi/core/RA2018PSIClient.cpp | 212 + .../src/ra2018-psi/core/RA2018PSIClient.h | 100 + .../ra2018-psi/core/RA2018PSIClientCache.cpp | 194 + .../ra2018-psi/core/RA2018PSIClientCache.h | 114 + .../src/ra2018-psi/core/RA2018PSIServer.cpp | 48 + .../src/ra2018-psi/core/RA2018PSIServer.h | 41 + .../src/ra2018-psi/core/Ra2018TaskParam.h | 73 + .../src/ra2018-psi/protocol/RA2018Message.h | 154 + .../ra2018-psi/storage/RA2018PSIStorage.cpp | 423 + .../src/ra2018-psi/storage/RA2018PSIStorage.h | 163 + cpp/ppc-psi/tests/CMakeLists.txt | 11 + .../tests/bs-ecdh-psi/TestBsEcdhCache.cpp | 192 + .../tests/cm2020-psi/FakeCM2020PSIFactory.h | 88 + .../tests/cm2020-psi/TestCM2020Impl.cpp | 326 + cpp/ppc-psi/tests/labeled-psi/DataTools.h | 44 + .../tests/labeled-psi/FakeLabeledPSIFactory.h | 87 + .../tests/labeled-psi/TestLabeledPSIImpl.cpp | 321 + .../tests/labeled-psi/TestSenderDB.cpp | 116 + cpp/ppc-psi/tests/main.cpp | 2 + .../tests/ra2018-psi/TestEcdhPSIImpl.cpp | 155 + .../tests/ra2018-psi/TestRA2018Impl.cpp | 261 + .../tests/ra2018-psi/TestRA2018Message.cpp | 102 + .../mock-data/append-full-evaluate.txt | 16 + .../ra2018-psi/mock-data/fullevaluate.txt | 10002 ++++++++++++++++ .../tests/ra2018-psi/mock-data/psi.txt | 50 + .../tests/ra2018-psi/mock-data/psi2.txt | 672 ++ cpp/ppc-psi/tests/ra2018-psi/mock/Common.h | 120 + .../tests/ra2018-psi/mock/EcdhPSIFixture.h | 110 + .../ra2018-psi/mock/FakeRA2018PSIStorage.h | 146 + .../ra2018-psi/mock/RA2018MessageFixture.h | 185 + .../tests/ra2018-psi/mock/RA2018PSIFixture.h | 123 + cpp/ppc-rpc/CMakeLists.txt | 12 + cpp/ppc-rpc/demo/CMakeLists.txt | 11 + cpp/ppc-rpc/demo/config.ini | 14 + cpp/ppc-rpc/demo/rpc_demo.cpp | 72 + cpp/ppc-rpc/src/CMakeLists.txt | 4 + cpp/ppc-rpc/src/JsonRequest.h | 101 + cpp/ppc-rpc/src/JsonResponse.h | 87 + cpp/ppc-rpc/src/Rpc.cpp | 458 + cpp/ppc-rpc/src/Rpc.h | 159 + cpp/ppc-rpc/src/RpcFactory.cpp | 94 + cpp/ppc-rpc/src/RpcFactory.h | 47 + cpp/ppc-rpc/src/RpcMemory.cpp | 137 + cpp/ppc-rpc/src/RpcMemory.h | 65 + cpp/ppc-rpc/tests/CMakeLists.txt | 10 + cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp | 75 + cpp/ppc-rpc/tests/main.cpp | 2 + cpp/ppc-storage/CMakeLists.txt | 6 + cpp/ppc-storage/demo/CMakeLists.txt | 10 + cpp/ppc-storage/demo/storage_demo.cpp | 122 + cpp/ppc-storage/src/CMakeLists.txt | 7 + cpp/ppc-storage/src/CacheStorageFactoryImpl.h | 75 + cpp/ppc-storage/src/Common.h | 31 + cpp/ppc-storage/src/FileStorageFactoryImpl.h | 49 + cpp/ppc-storage/src/SQLStorageFactoryImpl.h | 52 + cpp/ppc-storage/src/hdfs/Common.h | 32 + cpp/ppc-storage/src/hdfs/HDFSStorage.cpp | 197 + cpp/ppc-storage/src/hdfs/HDFSStorage.h | 114 + cpp/ppc-storage/src/mysql/Common.h | 31 + cpp/ppc-storage/src/mysql/MySQLStorage.cpp | 403 + cpp/ppc-storage/src/mysql/MySQLStorage.h | 271 + cpp/ppc-storage/src/redis/RedisStorage.cpp | 62 + cpp/ppc-storage/src/redis/RedisStorage.h | 80 + cpp/ppc-tars-protocol/CMakeLists.txt | 53 + .../ppc-tars-protocol/Common.h | 396 + .../ppc-tars-protocol/TarsSerialize.h | 43 + .../TarsServantProxyCallback.h | 242 + .../ppc-tars-protocol/TarsStruct.h | 39 + .../client/FrontServiceClient.h | 139 + .../client/GatewayServiceClient.cpp | 24 + .../client/GatewayServiceClient.h | 298 + .../ppc-tars-protocol/tars/Error.tars | 6 + .../ppc-tars-protocol/tars/FrontService.tars | 7 + .../tars/GatewayService.tars | 19 + .../ppc-tars-protocol/tars/TaskInfo.tars | 7 + cpp/ppc-tars-protocol/test/CMakeLists.txt | 13 + cpp/ppc-tars-protocol/test/SerializeTest.cpp | 70 + cpp/ppc-tars-protocol/test/main.cpp | 2 + .../FrontService/FrontServiceServer.cpp | 55 + .../FrontService/FrontServiceServer.h | 53 + .../GatewayService/GatewayInitializer.cpp | 56 + .../GatewayService/GatewayInitializer.h | 75 + .../GatewayService/GatewayServiceServer.cpp | 141 + .../GatewayService/GatewayServiceServer.h | 74 + cpp/ppc-tools/CMakeLists.txt | 14 + cpp/ppc-tools/src/codec/CodecUtility.h | 122 + cpp/ppc-tools/src/common/ConcurrentPool.h | 79 + cpp/ppc-tools/src/common/MemInfo.h | 58 + cpp/ppc-tools/src/common/Progress.h | 105 + cpp/ppc-tools/src/common/TransTools.h | 191 + cpp/ppc-tools/src/config/Common.h | 30 + cpp/ppc-tools/src/config/NetworkConfig.h | 68 + cpp/ppc-tools/src/config/PPCConfig.cpp | 664 + cpp/ppc-tools/src/config/PPCConfig.h | 366 + cpp/ppc-tools/src/config/ParamChecker.h | 54 + cpp/ppc-tools/src/cuckoo/Common.h | 85 + cpp/ppc-tools/src/cuckoo/Cuckoofilter.h | 339 + cpp/ppc-tools/src/cuckoo/HashTable.cpp | 163 + cpp/ppc-tools/src/cuckoo/HashTable.h | 147 + cpp/ppc-tools/tests/CMakeLists.txt | 10 + cpp/ppc-tools/tests/TestCuckoofilter.cpp | 220 + cpp/ppc-tools/tests/TestTransTools.cpp | 69 + cpp/ppc-tools/tests/main.cpp | 2 + cpp/ppc-udf/CMakeLists.txt | 35 + cpp/ppc-udf/hive/build.gradle | 159 + .../hive/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + cpp/ppc-udf/hive/gradlew | 185 + cpp/ppc-udf/hive/gradlew.bat | 89 + cpp/ppc-udf/hive/settings.gradle | 2 + .../webank/wedpr/hive/udf/config/Config.java | 55 + .../wedpr/hive/udf/config/ConfigLoader.java | 46 + .../hive/udf/config/model/ConfigProperty.java | 59 + .../hive/udf/config/model/KeyConfig.java | 131 + .../hive/udf/demo/GeneratePaillierMain.java | 61 + .../hive/udf/exceptions/ConfigException.java | 11 + .../udf/exceptions/DecryptionException.java | 11 + .../udf/exceptions/EncryptionException.java | 24 + .../hive/udf/exceptions/KeyException.java | 11 + .../hive/udf/impl/ihc/IhcDecryptionUDF.java | 74 + .../hive/udf/impl/ihc/IhcEncryptionUDF.java | 74 + .../wedpr/hive/udf/impl/ihc/IhcSumUDAF.java | 211 + .../hive/udf/impl/ore/OreDecryptionUDF.java | 61 + .../hive/udf/impl/ore/OreEncryptionUDF.java | 57 + .../udf/impl/ore/OreNumberDecryptionUDF.java | 48 + .../udf/impl/ore/OreNumberEncryptionUDF.java | 72 + .../impl/paillier/PaillierDecryptionUDF.java | 83 + .../impl/paillier/PaillierEncryptionUDF.java | 86 + .../udf/impl/paillier/PaillierSumUDAF.java | 219 + .../udf/impl/symmetric/SymDecryptionUDF.java | 73 + .../udf/impl/symmetric/SymEncryptionUDF.java | 83 + .../hive/src/main/resources/config.toml | 7 + .../hive/udf/impl/IhcEncAndDecUDFTest.java | 60 + .../wedpr/hive/udf/impl/IhcSumUDAFTest.java | 78 + .../wedpr/hive/udf/impl/OreDecUDFTest.java | 138 + .../hive/udf/impl/PaillierDecUDFTest.java | 41 + .../hive/udf/impl/PaillierEncUDFTest.java | 46 + .../hive/udf/impl/PaillierSumUDAFTest.java | 61 + .../wedpr/hive/udf/impl/SymDecUDFTest.java | 68 + .../hive/udf/impl/SymEncryptionUDFTest.java | 172 + .../hive/src/test/resources/config.toml | 7 + cpp/ppc-udf/mysql/ore_dec.cpp | 119 + cpp/ppc-udf/mysql/ore_dec.h | 42 + cpp/ppc-udf/mysql/ore_enc.cpp | 119 + cpp/ppc-udf/mysql/ore_enc.h | 42 + cpp/ppc-udf/mysql/paillier_dec.cpp | 160 + cpp/ppc-udf/mysql/paillier_dec.h | 41 + cpp/ppc-udf/mysql/paillier_enc.cpp | 150 + cpp/ppc-udf/mysql/paillier_enc.h | 41 + cpp/ppc-udf/mysql/paillier_sum.cpp | 184 + cpp/ppc-udf/mysql/paillier_sum.h | 44 + cpp/ppc-udf/mysql/sym_dec.cpp | 178 + cpp/ppc-udf/mysql/sym_dec.h | 42 + cpp/ppc-udf/mysql/sym_enc.cpp | 171 + cpp/ppc-udf/mysql/sym_enc.h | 42 + cpp/ppc-udf/tests/CMakeLists.txt | 10 + cpp/ppc-udf/tests/OreDecUDFTest.cpp | 153 + cpp/ppc-udf/tests/OreEncUDFTest.cpp | 154 + cpp/ppc-udf/tests/PaillierDecUDFTest.cpp | 299 + cpp/ppc-udf/tests/PaillierEncUDFTest.cpp | 178 + cpp/ppc-udf/tests/PaillierSumUDFTest.cpp | 200 + cpp/ppc-udf/tests/SymDecUDFTest.cpp | 252 + cpp/ppc-udf/tests/SymEncUDFTest.cpp | 228 + cpp/ppc-udf/tests/main.cpp | 2 + cpp/test-utils/FakeFront.h | 240 + cpp/test-utils/FakePPCMessage.h | 117 + cpp/test-utils/FileTool.h | 134 + cpp/test-utils/TaskMock.h | 94 + cpp/tools/build_ppc.sh | 926 ++ cpp/tools/build_wedpr_cem.sh | 729 ++ cpp/tools/build_wedpr_mpc.sh | 716 ++ cpp/tools/generate_psi.py | 75 + cpp/tools/install_depends.sh | 153 + cpp/tools/ppc-builder/build_ppc.py | 20 + .../ppc-builder/conf/config-example.toml | 208 + cpp/tools/ppc-builder/requirements.txt | 4 + cpp/tools/ppc-builder/src/common/utilities.py | 189 + .../src/config/binary_generator.py | 26 + .../ppc-builder/src/config/cert_generator.py | 57 + .../src/config/ppc_deploy_config.py | 370 + .../config/ppc_gateway_config_generator.py | 207 + .../src/config/ppc_node_config_generator.py | 259 + .../src/config/tars_config_generator.py | 123 + .../src/controller/commandline_helper.py | 88 + cpp/tools/ppc-builder/src/scripts/gen_cert.sh | 465 + .../ppc-builder/src/tpl/config.ini.gateway | 58 + cpp/tools/ppc-builder/src/tpl/config.ini.node | 110 + cpp/tools/ppc-builder/src/tpl/tars.conf | 31 + cpp/tools/ppc-builder/src/tpl/tars_start.sh | 33 + .../ppc-builder/src/tpl/tars_start_all.sh | 14 + cpp/tools/ppc-builder/src/tpl/tars_stop.sh | 31 + .../ppc-builder/src/tpl/tars_stop_all.sh | 14 + cpp/vcpkg-configuration.json | 16 + cpp/vcpkg.json | 79 + 680 files changed, 104344 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 cpp/CMakeLists.txt create mode 100644 cpp/cmake/BuildInfo.cmake create mode 100644 cpp/cmake/BuildInfoGenerator.cmake create mode 100644 cpp/cmake/CompilerSettings.cmake create mode 100644 cpp/cmake/FindGSasl.cmake create mode 100644 cpp/cmake/IncludeDirectories.cmake create mode 100644 cpp/cmake/InstallBcosUtilities.cmake create mode 100644 cpp/cmake/Installlibhdfs3.cmake create mode 100644 cpp/cmake/Options.cmake create mode 100644 cpp/cmake/ProjectAPSI.cmake create mode 100644 cpp/cmake/ProjectTCMalloc.cmake create mode 100644 cpp/cmake/ProjectWedprCrypto.cmake create mode 100644 cpp/cmake/TargetSettings.cmake create mode 100644 cpp/cmake/Version.cmake create mode 100644 cpp/cmake/config.cmake create mode 100644 cpp/cmake/scripts/buildinfo.cmake create mode 100644 cpp/cmake/templates/BuildInfo.h.in create mode 100644 cpp/libhelper/CMakeLists.txt create mode 100644 cpp/libhelper/CommandHelper.cpp create mode 100644 cpp/libhelper/CommandHelper.h create mode 100644 cpp/libhelper/ExitHandler.h create mode 100644 cpp/libinitializer/CMakeLists.txt create mode 100644 cpp/libinitializer/Common.h create mode 100644 cpp/libinitializer/FrontInitializer.cpp create mode 100644 cpp/libinitializer/FrontInitializer.h create mode 100644 cpp/libinitializer/Initializer.cpp create mode 100644 cpp/libinitializer/Initializer.h create mode 100644 cpp/libinitializer/ProFrontInitializer.h create mode 100644 cpp/libinitializer/ProtocolInitializer.h create mode 100644 cpp/ppc-cem/CMakeLists.txt create mode 100644 cpp/ppc-cem/src/CEMService.cpp create mode 100644 cpp/ppc-cem/src/CEMService.h create mode 100644 cpp/ppc-cem/src/CMakeLists.txt create mode 100644 cpp/ppc-cem/src/Common.h create mode 100644 cpp/ppc-cem/src/WedprUtilities.h create mode 100644 cpp/ppc-cem/src/wedpr_ffi_c_equality.h create mode 100644 cpp/ppc-cem/tests/CMakeLists.txt create mode 100644 cpp/ppc-cem/tests/TestCEMService.cpp create mode 100644 cpp/ppc-cem/tests/data/config.ini create mode 100644 cpp/ppc-cem/tests/data/d1 create mode 100644 cpp/ppc-cem/tests/data/d2 create mode 100644 cpp/ppc-cem/tests/data/d3 create mode 100644 cpp/ppc-cem/tests/main.cpp create mode 100644 cpp/ppc-crypto-c-sdk/CMakeLists.txt create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/build.gradle create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/gradlew create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java create mode 100644 cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp create mode 100644 cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h create mode 100644 cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt create mode 100644 cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp create mode 100644 cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp create mode 100644 cpp/ppc-crypto-c-sdk/tests/main.cpp create mode 100644 cpp/ppc-crypto-core/CMakeLists.txt create mode 100644 cpp/ppc-crypto-core/src/CMakeLists.txt create mode 100644 cpp/ppc-crypto-core/src/Common.h create mode 100644 cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h create mode 100644 cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h create mode 100644 cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h create mode 100644 cpp/ppc-crypto-core/src/hash/MD5Hash.h create mode 100644 cpp/ppc-crypto-core/src/hash/OpenSSLHash.h create mode 100644 cpp/ppc-crypto-core/src/hash/SM3Hash.h create mode 100644 cpp/ppc-crypto-core/src/hash/Sha256Hash.h create mode 100644 cpp/ppc-crypto-core/src/hash/Sha512Hash.h create mode 100644 cpp/ppc-crypto-core/src/ore/FastOre.cpp create mode 100644 cpp/ppc-crypto-core/src/ore/FastOre.h create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp create mode 100644 cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h create mode 100644 cpp/ppc-crypto-core/src/tools/BitVector.cpp create mode 100644 cpp/ppc-crypto-core/src/tools/BitVector.h create mode 100644 cpp/ppc-crypto-core/tests/CMakeLists.txt create mode 100644 cpp/ppc-crypto-core/tests/TestBitVector.cpp create mode 100644 cpp/ppc-crypto-core/tests/TestHash.cpp create mode 100644 cpp/ppc-crypto-core/tests/TestOre4Float.cpp create mode 100644 cpp/ppc-crypto-core/tests/TestOre4Integer.cpp create mode 100644 cpp/ppc-crypto-core/tests/TestOre4String.cpp create mode 100644 cpp/ppc-crypto-core/tests/TestSymCrypto.cpp create mode 100644 cpp/ppc-crypto-core/tests/main.cpp create mode 100644 cpp/ppc-crypto/CMakeLists.txt create mode 100644 cpp/ppc-crypto/demo/CMakeLists.txt create mode 100644 cpp/ppc-crypto/demo/crypto_bench.cpp create mode 100644 cpp/ppc-crypto/src/CMakeLists.txt create mode 100644 cpp/ppc-crypto/src/Common.h create mode 100644 cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h create mode 100644 cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h create mode 100644 cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h create mode 100644 cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp create mode 100644 cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h create mode 100644 cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h create mode 100644 cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp create mode 100644 cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h create mode 100644 cpp/ppc-crypto/src/ecc/core/EcPoint.h create mode 100644 cpp/ppc-crypto/src/oprf/EcdhOprf.cpp create mode 100644 cpp/ppc-crypto/src/oprf/EcdhOprf.h create mode 100644 cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp create mode 100644 cpp/ppc-crypto/src/oprf/RA2018Oprf.h create mode 100644 cpp/ppc-crypto/src/prng/AESPRNG.cpp create mode 100644 cpp/ppc-crypto/src/prng/AESPRNG.h create mode 100644 cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp create mode 100644 cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h create mode 100644 cpp/ppc-crypto/src/randomot/SimplestOT.cpp create mode 100644 cpp/ppc-crypto/src/randomot/SimplestOT.h create mode 100644 cpp/ppc-crypto/tests/CMakeLists.txt create mode 100644 cpp/ppc-crypto/tests/TestEccCrypto.cpp create mode 100644 cpp/ppc-crypto/tests/TestOprf.cpp create mode 100644 cpp/ppc-crypto/tests/TestPRNG.cpp create mode 100644 cpp/ppc-crypto/tests/TestRandomot.cpp create mode 100644 cpp/ppc-crypto/tests/main.cpp create mode 100644 cpp/ppc-framework/Common.h create mode 100644 cpp/ppc-framework/crypto/CryptoBox.h create mode 100644 cpp/ppc-framework/crypto/ECDHCrypto.h create mode 100644 cpp/ppc-framework/crypto/EccCrypto.h create mode 100644 cpp/ppc-framework/crypto/Hash.h create mode 100644 cpp/ppc-framework/crypto/Ihc.h create mode 100644 cpp/ppc-framework/crypto/KeyPair.h create mode 100644 cpp/ppc-framework/crypto/Oprf.h create mode 100644 cpp/ppc-framework/crypto/Ore.h create mode 100644 cpp/ppc-framework/crypto/PRNG.h create mode 100644 cpp/ppc-framework/crypto/Paillier.h create mode 100644 cpp/ppc-framework/crypto/RA2018OprfInterface.h create mode 100644 cpp/ppc-framework/crypto/SymCrypto.h create mode 100644 cpp/ppc-framework/front/Channel.h create mode 100644 cpp/ppc-framework/front/FrontInterface.h create mode 100644 cpp/ppc-framework/gateway/GatewayInterface.h create mode 100644 cpp/ppc-framework/http/HttpInterface.h create mode 100644 cpp/ppc-framework/http/HttpTypeDef.h create mode 100644 cpp/ppc-framework/io/DataBatch.h create mode 100644 cpp/ppc-framework/io/DataResourceLoader.h create mode 100644 cpp/ppc-framework/io/LineReader.h create mode 100644 cpp/ppc-framework/io/LineWriter.h create mode 100644 cpp/ppc-framework/libwrapper/BigNum.h create mode 100644 cpp/ppc-framework/libwrapper/Buffer.h create mode 100644 cpp/ppc-framework/libwrapper/FloatingPointNumber.h create mode 100644 cpp/ppc-framework/libwrapper/OreFloatingNumber.h create mode 100644 cpp/ppc-framework/protocol/DataResource.h create mode 100644 cpp/ppc-framework/protocol/GlobalConfig.h create mode 100644 cpp/ppc-framework/protocol/PPCMessageFace.h create mode 100644 cpp/ppc-framework/protocol/PartyResource.h create mode 100644 cpp/ppc-framework/protocol/Protocol.h create mode 100644 cpp/ppc-framework/protocol/Task.h create mode 100644 cpp/ppc-framework/psi/TaskFrameworkInterface.h create mode 100644 cpp/ppc-framework/rpc/RpcInterface.h create mode 100644 cpp/ppc-framework/rpc/RpcStatusInterface.h create mode 100644 cpp/ppc-framework/rpc/RpcTypeDef.h create mode 100644 cpp/ppc-framework/storage/CacheStorage.h create mode 100644 cpp/ppc-framework/storage/FileStorage.h create mode 100644 cpp/ppc-framework/storage/RemoteStorage.h create mode 100644 cpp/ppc-framework/storage/SQLStorage.h create mode 100644 cpp/ppc-framework/task/TaskFrameworkInterface.h create mode 100644 cpp/ppc-front/CMakeLists.txt create mode 100644 cpp/ppc-front/ppc-front/Common.h create mode 100644 cpp/ppc-front/ppc-front/Front.cpp create mode 100644 cpp/ppc-front/ppc-front/Front.h create mode 100644 cpp/ppc-front/ppc-front/PPCChannel.cpp create mode 100644 cpp/ppc-front/ppc-front/PPCChannel.h create mode 100644 cpp/ppc-front/ppc-front/PPCChannelManager.cpp create mode 100644 cpp/ppc-front/ppc-front/PPCChannelManager.h create mode 100644 cpp/ppc-front/test/CMakeLists.txt create mode 100644 cpp/ppc-front/test/unittests/PPCChannelTest.cpp create mode 100644 cpp/ppc-front/test/unittests/main.cpp create mode 100644 cpp/ppc-gateway/CMakeLists.txt create mode 100644 cpp/ppc-gateway/ppc-gateway/Common.h create mode 100644 cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h create mode 100644 cpp/ppc-gateway/ppc-gateway/Gateway.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/Gateway.h create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h create mode 100644 cpp/ppc-gateway/ppc-gateway/ProTaskManager.h create mode 100644 cpp/ppc-gateway/ppc-gateway/TaskManager.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/TaskManager.h create mode 100644 cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/WebSocketService.h create mode 100644 cpp/ppc-gateway/test/CMakeLists.txt create mode 100644 cpp/ppc-gateway/test/data/config0.ini create mode 100644 cpp/ppc-gateway/test/demo/CMakeLists.txt create mode 100644 cpp/ppc-gateway/test/demo/gateway_demo.cpp create mode 100644 cpp/ppc-gateway/test/unittests/GatewayTest.cpp create mode 100644 cpp/ppc-gateway/test/unittests/MockCache.h create mode 100644 cpp/ppc-gateway/test/unittests/main.cpp create mode 100644 cpp/ppc-homo/CMakeLists.txt create mode 100644 cpp/ppc-homo/codec/CMakeLists.txt create mode 100644 cpp/ppc-homo/codec/Common.h create mode 100644 cpp/ppc-homo/codec/FloatingPointCipher.h create mode 100644 cpp/ppc-homo/codec/FloatingPointCodec.cpp create mode 100644 cpp/ppc-homo/codec/FloatingPointCodec.h create mode 100644 cpp/ppc-homo/codec/SignedNumberCodec.h create mode 100644 cpp/ppc-homo/demo/CMakeLists.txt create mode 100644 cpp/ppc-homo/demo/homo_perf_demo.cpp create mode 100644 cpp/ppc-homo/fahe/CMakeLists.txt create mode 100644 cpp/ppc-homo/fahe/Fahe.cpp create mode 100644 cpp/ppc-homo/fahe/Fahe.h create mode 100644 cpp/ppc-homo/ihc/CMakeLists.txt create mode 100644 cpp/ppc-homo/ihc/FloatingPointIhc.cpp create mode 100644 cpp/ppc-homo/ihc/FloatingPointIhc.h create mode 100644 cpp/ppc-homo/ihc/IhcImpl.cpp create mode 100644 cpp/ppc-homo/ihc/IhcImpl.h create mode 100644 cpp/ppc-homo/paillier/CMakeLists.txt create mode 100644 cpp/ppc-homo/paillier/Common.h create mode 100644 cpp/ppc-homo/paillier/FloatingPointPaillier.cpp create mode 100644 cpp/ppc-homo/paillier/FloatingPointPaillier.h create mode 100644 cpp/ppc-homo/paillier/OpenSSLPaillier.cpp create mode 100644 cpp/ppc-homo/paillier/OpenSSLPaillier.h create mode 100644 cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp create mode 100644 cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h create mode 100644 cpp/ppc-homo/tests/CMakeLists.txt create mode 100644 cpp/ppc-homo/tests/TestFahe.cpp create mode 100644 cpp/ppc-homo/tests/TestFloatingPointCodec.cpp create mode 100644 cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp create mode 100644 cpp/ppc-homo/tests/TestIhc.cpp create mode 100644 cpp/ppc-homo/tests/TestIhcFloating.cpp create mode 100644 cpp/ppc-homo/tests/TestPaillier.cpp create mode 100644 cpp/ppc-homo/tests/main.cpp create mode 100644 cpp/ppc-http/CMakeLists.txt create mode 100644 cpp/ppc-http/demo/CMakeLists.txt create mode 100644 cpp/ppc-http/demo/config.ini create mode 100644 cpp/ppc-http/demo/http_client_demo.cpp create mode 100644 cpp/ppc-http/demo/http_demo.cpp create mode 100644 cpp/ppc-http/src/CMakeLists.txt create mode 100644 cpp/ppc-http/src/Http.cpp create mode 100644 cpp/ppc-http/src/Http.h create mode 100644 cpp/ppc-http/src/HttpClient.cpp create mode 100644 cpp/ppc-http/src/HttpClient.h create mode 100644 cpp/ppc-http/src/HttpFactory.cpp create mode 100644 cpp/ppc-http/src/HttpFactory.h create mode 100644 cpp/ppc-io/CMakeLists.txt create mode 100644 cpp/ppc-io/demo/CMakeLists.txt create mode 100644 cpp/ppc-io/demo/io_demo.cpp create mode 100644 cpp/ppc-io/src/BaseFileLineReader.cpp create mode 100644 cpp/ppc-io/src/BaseFileLineReader.h create mode 100644 cpp/ppc-io/src/CMakeLists.txt create mode 100644 cpp/ppc-io/src/Common.h create mode 100644 cpp/ppc-io/src/DataResourceLoaderImpl.cpp create mode 100644 cpp/ppc-io/src/DataResourceLoaderImpl.h create mode 100644 cpp/ppc-io/src/FileLineReader.cpp create mode 100644 cpp/ppc-io/src/FileLineReader.h create mode 100644 cpp/ppc-io/src/FileLineWriter.cpp create mode 100644 cpp/ppc-io/src/FileLineWriter.h create mode 100644 cpp/ppc-io/src/HDFSReader.cpp create mode 100644 cpp/ppc-io/src/HDFSReader.h create mode 100644 cpp/ppc-io/src/HDFSWriter.cpp create mode 100644 cpp/ppc-io/src/HDFSWriter.h create mode 100644 cpp/ppc-io/src/SQLResultReader.h create mode 100644 cpp/ppc-io/src/parser/BaseBufferParser.h create mode 100644 cpp/ppc-io/src/parser/BufferParser.h create mode 100644 cpp/ppc-io/src/parser/MmapParser.h create mode 100644 cpp/ppc-io/tests/CMakeLists.txt create mode 100644 cpp/ppc-io/tests/TestLineReader.cpp create mode 100644 cpp/ppc-io/tests/TestLineWriter.cpp create mode 100644 cpp/ppc-io/tests/data/linux_file.txt create mode 100644 cpp/ppc-io/tests/data/testData create mode 100644 cpp/ppc-io/tests/data/windows_file.txt create mode 100644 cpp/ppc-io/tests/main.cpp create mode 100644 cpp/ppc-main/CMakeLists.txt create mode 100644 cpp/ppc-main/air-node/AirNodeInitializer.cpp create mode 100644 cpp/ppc-main/air-node/AirNodeInitializer.h create mode 100644 cpp/ppc-main/air-node/CMakeLists.txt create mode 100644 cpp/ppc-main/air-node/main.cpp create mode 100644 cpp/ppc-main/cem-node/CEMInitializer.cpp create mode 100644 cpp/ppc-main/cem-node/CEMInitializer.h create mode 100644 cpp/ppc-main/cem-node/CMakeLists.txt create mode 100644 cpp/ppc-main/cem-node/main.cpp create mode 100644 cpp/ppc-main/gateway/CMakeLists.txt create mode 100644 cpp/ppc-main/gateway/GatewayServiceApp.cpp create mode 100644 cpp/ppc-main/gateway/GatewayServiceApp.h create mode 100644 cpp/ppc-main/gateway/main.cpp create mode 100644 cpp/ppc-main/mpc-node/CMakeLists.txt create mode 100644 cpp/ppc-main/mpc-node/MPCInitializer.cpp create mode 100644 cpp/ppc-main/mpc-node/MPCInitializer.h create mode 100644 cpp/ppc-main/mpc-node/main.cpp create mode 100644 cpp/ppc-main/pro-node/CMakeLists.txt create mode 100644 cpp/ppc-main/pro-node/ProNodeServiceApp.cpp create mode 100644 cpp/ppc-main/pro-node/ProNodeServiceApp.h create mode 100644 cpp/ppc-main/pro-node/main.cpp create mode 100644 cpp/ppc-mpc/CMakeLists.txt create mode 100644 cpp/ppc-mpc/src/CMakeLists.txt create mode 100644 cpp/ppc-mpc/src/Common.h create mode 100644 cpp/ppc-mpc/src/MPCService.cpp create mode 100644 cpp/ppc-mpc/src/MPCService.h create mode 100644 cpp/ppc-mpc/tests/CMakeLists.txt create mode 100644 cpp/ppc-mpc/tests/TestMPCService.cpp create mode 100644 cpp/ppc-mpc/tests/data/config.ini create mode 100644 cpp/ppc-mpc/tests/main.cpp create mode 100644 cpp/ppc-pir/CMakeLists.txt create mode 100644 cpp/ppc-pir/src/BaseOT.cpp create mode 100644 cpp/ppc-pir/src/BaseOT.h create mode 100644 cpp/ppc-pir/src/CMakeLists.txt create mode 100644 cpp/ppc-pir/src/Common.h create mode 100644 cpp/ppc-pir/src/OtPIRConfig.h create mode 100644 cpp/ppc-pir/src/OtPIRFactory.h create mode 100644 cpp/ppc-pir/src/OtPIRImpl.cpp create mode 100644 cpp/ppc-pir/src/OtPIRImpl.h create mode 100644 cpp/ppc-pir/src/protocol/OtPIR.tars create mode 100644 cpp/ppc-pir/tests/CMakeLists.txt create mode 100644 cpp/ppc-pir/tests/FakeOtPIRFactory.h create mode 100644 cpp/ppc-pir/tests/TestBaseOT.cpp create mode 100644 cpp/ppc-pir/tests/main.cpp create mode 100644 cpp/ppc-pir/tests/test.json create mode 100644 cpp/ppc-protocol/CMakeLists.txt create mode 100644 cpp/ppc-protocol/src/CMakeLists.txt create mode 100644 cpp/ppc-protocol/src/JsonTaskImpl.cpp create mode 100644 cpp/ppc-protocol/src/JsonTaskImpl.h create mode 100644 cpp/ppc-protocol/src/PPCMessage.cpp create mode 100644 cpp/ppc-protocol/src/PPCMessage.h create mode 100644 cpp/ppc-protocol/tests/CMakeLists.txt create mode 100644 cpp/ppc-protocol/tests/PPCMessageTest.cpp create mode 100644 cpp/ppc-protocol/tests/TestTaskImpl.cpp create mode 100644 cpp/ppc-protocol/tests/main.cpp create mode 100644 cpp/ppc-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/demo/CMakeLists.txt create mode 100644 cpp/ppc-psi/demo/hash_collision_bench.cpp create mode 100644 cpp/ppc-psi/src/Common.h create mode 100644 cpp/ppc-psi/src/PSIConfig.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/Common.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h create mode 100644 cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp create mode 100644 cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/cm2020-psi/Common.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp create mode 100644 cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp create mode 100644 cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h create mode 100644 cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars create mode 100644 cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/Common.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp create mode 100644 cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/Common.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp create mode 100644 cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/ecdh-psi/Common.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhCache.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h create mode 100644 cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h create mode 100644 cpp/ppc-psi/src/labeled-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/labeled-psi/Common.h create mode 100644 cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h create mode 100644 cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h create mode 100644 cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/BinBundle.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp create mode 100644 cpp/ppc-psi/src/labeled-psi/core/SenderDB.h create mode 100644 cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h create mode 100644 cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars create mode 100644 cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h create mode 100644 cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h create mode 100644 cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars create mode 100644 cpp/ppc-psi/src/psi-framework/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/psi-framework/PSIFramework.cpp create mode 100644 cpp/ppc-psi/src/psi-framework/PSIFramework.h create mode 100644 cpp/ppc-psi/src/psi-framework/TaskGuarder.h create mode 100644 cpp/ppc-psi/src/psi-framework/TaskState.h create mode 100644 cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h create mode 100644 cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h create mode 100644 cpp/ppc-psi/src/psi-framework/protocol/PSI.tars create mode 100644 cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp create mode 100644 cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt create mode 100644 cpp/ppc-psi/src/ra2018-psi/Common.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h create mode 100644 cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp create mode 100644 cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h create mode 100644 cpp/ppc-psi/tests/CMakeLists.txt create mode 100644 cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp create mode 100644 cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h create mode 100644 cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp create mode 100644 cpp/ppc-psi/tests/labeled-psi/DataTools.h create mode 100644 cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h create mode 100644 cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp create mode 100644 cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp create mode 100644 cpp/ppc-psi/tests/main.cpp create mode 100644 cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp create mode 100644 cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp create mode 100644 cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock/Common.h create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h create mode 100644 cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h create mode 100644 cpp/ppc-rpc/CMakeLists.txt create mode 100644 cpp/ppc-rpc/demo/CMakeLists.txt create mode 100644 cpp/ppc-rpc/demo/config.ini create mode 100644 cpp/ppc-rpc/demo/rpc_demo.cpp create mode 100644 cpp/ppc-rpc/src/CMakeLists.txt create mode 100644 cpp/ppc-rpc/src/JsonRequest.h create mode 100644 cpp/ppc-rpc/src/JsonResponse.h create mode 100644 cpp/ppc-rpc/src/Rpc.cpp create mode 100644 cpp/ppc-rpc/src/Rpc.h create mode 100644 cpp/ppc-rpc/src/RpcFactory.cpp create mode 100644 cpp/ppc-rpc/src/RpcFactory.h create mode 100644 cpp/ppc-rpc/src/RpcMemory.cpp create mode 100644 cpp/ppc-rpc/src/RpcMemory.h create mode 100644 cpp/ppc-rpc/tests/CMakeLists.txt create mode 100644 cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp create mode 100644 cpp/ppc-rpc/tests/main.cpp create mode 100644 cpp/ppc-storage/CMakeLists.txt create mode 100644 cpp/ppc-storage/demo/CMakeLists.txt create mode 100644 cpp/ppc-storage/demo/storage_demo.cpp create mode 100644 cpp/ppc-storage/src/CMakeLists.txt create mode 100644 cpp/ppc-storage/src/CacheStorageFactoryImpl.h create mode 100644 cpp/ppc-storage/src/Common.h create mode 100644 cpp/ppc-storage/src/FileStorageFactoryImpl.h create mode 100644 cpp/ppc-storage/src/SQLStorageFactoryImpl.h create mode 100644 cpp/ppc-storage/src/hdfs/Common.h create mode 100644 cpp/ppc-storage/src/hdfs/HDFSStorage.cpp create mode 100644 cpp/ppc-storage/src/hdfs/HDFSStorage.h create mode 100644 cpp/ppc-storage/src/mysql/Common.h create mode 100644 cpp/ppc-storage/src/mysql/MySQLStorage.cpp create mode 100644 cpp/ppc-storage/src/mysql/MySQLStorage.h create mode 100644 cpp/ppc-storage/src/redis/RedisStorage.cpp create mode 100644 cpp/ppc-storage/src/redis/RedisStorage.h create mode 100644 cpp/ppc-tars-protocol/CMakeLists.txt create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars create mode 100644 cpp/ppc-tars-protocol/test/CMakeLists.txt create mode 100644 cpp/ppc-tars-protocol/test/SerializeTest.cpp create mode 100644 cpp/ppc-tars-protocol/test/main.cpp create mode 100644 cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp create mode 100644 cpp/ppc-tars-service/FrontService/FrontServiceServer.h create mode 100644 cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp create mode 100644 cpp/ppc-tars-service/GatewayService/GatewayInitializer.h create mode 100644 cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp create mode 100644 cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h create mode 100644 cpp/ppc-tools/CMakeLists.txt create mode 100644 cpp/ppc-tools/src/codec/CodecUtility.h create mode 100644 cpp/ppc-tools/src/common/ConcurrentPool.h create mode 100644 cpp/ppc-tools/src/common/MemInfo.h create mode 100644 cpp/ppc-tools/src/common/Progress.h create mode 100644 cpp/ppc-tools/src/common/TransTools.h create mode 100644 cpp/ppc-tools/src/config/Common.h create mode 100644 cpp/ppc-tools/src/config/NetworkConfig.h create mode 100644 cpp/ppc-tools/src/config/PPCConfig.cpp create mode 100644 cpp/ppc-tools/src/config/PPCConfig.h create mode 100644 cpp/ppc-tools/src/config/ParamChecker.h create mode 100644 cpp/ppc-tools/src/cuckoo/Common.h create mode 100644 cpp/ppc-tools/src/cuckoo/Cuckoofilter.h create mode 100644 cpp/ppc-tools/src/cuckoo/HashTable.cpp create mode 100644 cpp/ppc-tools/src/cuckoo/HashTable.h create mode 100644 cpp/ppc-tools/tests/CMakeLists.txt create mode 100644 cpp/ppc-tools/tests/TestCuckoofilter.cpp create mode 100644 cpp/ppc-tools/tests/TestTransTools.cpp create mode 100644 cpp/ppc-tools/tests/main.cpp create mode 100644 cpp/ppc-udf/CMakeLists.txt create mode 100644 cpp/ppc-udf/hive/build.gradle create mode 100644 cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar create mode 100644 cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties create mode 100644 cpp/ppc-udf/hive/gradlew create mode 100644 cpp/ppc-udf/hive/gradlew.bat create mode 100644 cpp/ppc-udf/hive/settings.gradle create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java create mode 100644 cpp/ppc-udf/hive/src/main/resources/config.toml create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java create mode 100644 cpp/ppc-udf/hive/src/test/resources/config.toml create mode 100644 cpp/ppc-udf/mysql/ore_dec.cpp create mode 100644 cpp/ppc-udf/mysql/ore_dec.h create mode 100644 cpp/ppc-udf/mysql/ore_enc.cpp create mode 100644 cpp/ppc-udf/mysql/ore_enc.h create mode 100644 cpp/ppc-udf/mysql/paillier_dec.cpp create mode 100644 cpp/ppc-udf/mysql/paillier_dec.h create mode 100644 cpp/ppc-udf/mysql/paillier_enc.cpp create mode 100644 cpp/ppc-udf/mysql/paillier_enc.h create mode 100644 cpp/ppc-udf/mysql/paillier_sum.cpp create mode 100644 cpp/ppc-udf/mysql/paillier_sum.h create mode 100644 cpp/ppc-udf/mysql/sym_dec.cpp create mode 100644 cpp/ppc-udf/mysql/sym_dec.h create mode 100644 cpp/ppc-udf/mysql/sym_enc.cpp create mode 100644 cpp/ppc-udf/mysql/sym_enc.h create mode 100644 cpp/ppc-udf/tests/CMakeLists.txt create mode 100644 cpp/ppc-udf/tests/OreDecUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/OreEncUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/PaillierDecUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/PaillierEncUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/PaillierSumUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/SymDecUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/SymEncUDFTest.cpp create mode 100644 cpp/ppc-udf/tests/main.cpp create mode 100644 cpp/test-utils/FakeFront.h create mode 100644 cpp/test-utils/FakePPCMessage.h create mode 100644 cpp/test-utils/FileTool.h create mode 100644 cpp/test-utils/TaskMock.h create mode 100644 cpp/tools/build_ppc.sh create mode 100644 cpp/tools/build_wedpr_cem.sh create mode 100644 cpp/tools/build_wedpr_mpc.sh create mode 100644 cpp/tools/generate_psi.py create mode 100644 cpp/tools/install_depends.sh create mode 100644 cpp/tools/ppc-builder/build_ppc.py create mode 100644 cpp/tools/ppc-builder/conf/config-example.toml create mode 100644 cpp/tools/ppc-builder/requirements.txt create mode 100644 cpp/tools/ppc-builder/src/common/utilities.py create mode 100644 cpp/tools/ppc-builder/src/config/binary_generator.py create mode 100644 cpp/tools/ppc-builder/src/config/cert_generator.py create mode 100644 cpp/tools/ppc-builder/src/config/ppc_deploy_config.py create mode 100644 cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py create mode 100644 cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py create mode 100644 cpp/tools/ppc-builder/src/config/tars_config_generator.py create mode 100644 cpp/tools/ppc-builder/src/controller/commandline_helper.py create mode 100644 cpp/tools/ppc-builder/src/scripts/gen_cert.sh create mode 100644 cpp/tools/ppc-builder/src/tpl/config.ini.gateway create mode 100644 cpp/tools/ppc-builder/src/tpl/config.ini.node create mode 100644 cpp/tools/ppc-builder/src/tpl/tars.conf create mode 100644 cpp/tools/ppc-builder/src/tpl/tars_start.sh create mode 100644 cpp/tools/ppc-builder/src/tpl/tars_start_all.sh create mode 100644 cpp/tools/ppc-builder/src/tpl/tars_stop.sh create mode 100644 cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh create mode 100644 cpp/vcpkg-configuration.json create mode 100644 cpp/vcpkg.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..78629f50 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.gif filter=lfs diff=lfs merge=lfs -text +*.otf filter=lfs diff=lfs merge=lfs -text +*.svg filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..bb7e592f --- /dev/null +++ b/.gitignore @@ -0,0 +1,145 @@ +# Prerequisites +*.d + +.DS_Store +.vscode +.idea +.cache + +cmake-build-debug +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +build/** +GPATH +GRTAGS +GTAGS +deps +deps/** + +d1-encrypted +d2-encrypted +d3-encrypted +__pycache__ + +ppc-psi/tests/cm2020-psi/data +ppc-psi/tests/bs-ecdh-psi/data +ppc-psi/tests/labeled-psi/data + +tools/*-node +ppc-crypto-c-sdk/bindings/java/.gradle/** +ppc-udf/hive/bin/** +ppc-udf/hive/build/** +ppc-udf/hive/dist/** +ppc-udf/hive/.gradle/** +ppc-udf/hive/libs/ppc-crypto-sdk-jni-1* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +logs/ +*/service/PSI-DATA/ +static/ +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# pyenv +.python-version + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Editors +.idea/ +*.crt +*/bin +*/logs +*.csv +*/*/logs +*/start.out +*/application.yml +*/client_config.py + +# Mac cache file +.DS_Store + diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 00000000..42aef4a7 --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,141 @@ +#------------------------------------------------------------------------------ +# Top-level CMake file for WeDPR-Component +# ------------------------------------------------------------------------------ +# Copyright (C) 2022 WeDPR +# SPDX-License-Identifier: Apache-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#------------------------------------------------------------------------------ + +cmake_minimum_required(VERSION 3.14) + +set(PROJECT_CMAKE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake" CACHE PATH "The cmake path for the library") +list(APPEND CMAKE_MODULE_PATH ${PROJECT_CMAKE_DIR}) + +add_definitions(-w) + +include(FetchContent) +if (NOT DEFINED URL_BASE) + set(URL_BASE "github.com") + #set(URL_BASE "github.com.cnpmjs.org") +endif () + +# export windows dll symbol +if(WIN32) + message(STATUS "Compile on Windows") + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") +endif() + +# vcpkg init +if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) + find_package(Git REQUIRED) + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive -- vcpkg WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cmake" + CACHE STRING "Vcpkg toolchain file") +endif() + +include(Version) +project(WeDPR-Component VERSION ${VERSION}) + +# basic settings +include(Options) +configure_project() +include(CompilerSettings) +include(BuildInfoGenerator) + + +find_package(OpenSSL REQUIRED) +include(InstallBcosUtilities) +if(BUILD_SDK) + ##### the sdk-dependencies ##### + # find JNI + set(JAVA_AWT_LIBRARY NotNeeded) + set(JAVA_JVM_LIBRARY NotNeeded) + find_package(JNI REQUIRED) + include_directories(${JNI_INCLUDE_DIRS}) +endif() + +# ipp-crypto +#if(ENABLE_IPP_CRYPTO) +# hunter_add_package(ipp-crypto) +#endif() + +if(ENABLE_CPU_FEATURES) + find_package(CpuFeatures REQUIRED) +endif() + +include(IncludeDirectories) +include(TargetSettings) + +if(BUILD_SDK) + add_subdirectory(ppc-crypto-c-sdk) + set(JNI_SOURCE_PATH ppc-crypto-c-sdk/bindings/java/src/main/c) + add_subdirectory(${JNI_SOURCE_PATH}) +endif() +add_subdirectory(ppc-crypto-core) +add_subdirectory(ppc-homo) + +# when BUILD_SDK, the following modules no need to compile +if(NOT BUILD_SDK AND NOT BUILD_UDF) + ##### the full-dependencies ##### + find_package(TBB REQUIRED) + find_package(jsoncpp REQUIRED) + find_package(Boost REQUIRED unit_test_framework) + + find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) + # tcmalloc + include(ProjectTCMalloc) + + find_package(SEAL REQUIRED) + find_package(Kuku REQUIRED) + + # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 + include(ProjectAPSI) + # Wedpr Crypto + include(ProjectWedprCrypto) + include(FindGSasl) + include(Installlibhdfs3) + + add_subdirectory(ppc-crypto) + add_subdirectory(libhelper) + add_subdirectory(libinitializer) + add_subdirectory(ppc-io) + add_subdirectory(ppc-protocol) + add_subdirectory(ppc-gateway) + add_subdirectory(ppc-front) + add_subdirectory(ppc-tars-protocol) + add_subdirectory(ppc-tools) + add_subdirectory(ppc-storage) + add_subdirectory(ppc-psi) + add_subdirectory(ppc-rpc) + add_subdirectory(ppc-http) + add_subdirectory(ppc-mpc) + add_subdirectory(ppc-pir) + if(BUILD_CEM) + add_subdirectory(ppc-cem) + endif () + add_subdirectory(ppc-main) +endif() +if(BUILD_UDF) + add_subdirectory(ppc-udf) +endif() + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) +endif () + +# for code coverage +if (COVERAGE) + include(Coverage) + config_coverage("coverage" "'/usr*' 'boost/*'") +endif () diff --git a/cpp/cmake/BuildInfo.cmake b/cpp/cmake/BuildInfo.cmake new file mode 100644 index 00000000..628b7c23 --- /dev/null +++ b/cpp/cmake/BuildInfo.cmake @@ -0,0 +1,44 @@ +function(create_build_info) + # Set build platform; to be written to BuildInfo.h + set(PPC_BUILD_OS "${CMAKE_SYSTEM_NAME}") + + if (CMAKE_COMPILER_IS_MINGW) + set(PPC_BUILD_COMPILER "mingw") + elseif (CMAKE_COMPILER_IS_MSYS) + set(PPC_BUILD_COMPILER "msys") + elseif (CMAKE_COMPILER_IS_GNUCXX) + set(PPC_BUILD_COMPILER "g++") + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(PPC_BUILD_COMPILER "clang") + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + set(PPC_BUILD_COMPILER "appleclang") + else () + set(PPC_BUILD_COMPILER "unknown") + endif () + + set(PPC_BUILD_PLATFORM "${PPC_BUILD_OS}/${PPC_BUILD_COMPILER}") + + + if (CMAKE_BUILD_TYPE) + set(_cmake_build_type ${CMAKE_BUILD_TYPE}) + else() + set(_cmake_build_type "${CMAKE_CFG_INTDIR}") + endif() + # Generate header file containing useful build information + add_custom_target(BuildInfo.h ALL + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -DPPC_SOURCE_DIR="${PROJECT_SOURCE_DIR}" + -DPPC_BUILDINFO_IN="${CMAKE_CURRENT_SOURCE_DIR}/cmake/templates/BuildInfo.h.in" + -DPPC_DST_DIR="${PROJECT_BINARY_DIR}/include" + -DPPC_CMAKE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/cmake" + -DPPC_BUILD_TYPE="${_cmake_build_type}" + -DPPC_BUILD_OS="${PPC_BUILD_OS}" + -DPPC_BUILD_COMPILER="${PPC_BUILD_COMPILER}" + -DPPC_BUILD_PLATFORM="${PPC_BUILD_PLATFORM}" + -DPPC_BUILD_NUMBER="${PPC_BUILD_NUMBER}" + -DPPC_VERSION_SUFFIX="${VERSION_SUFFIX}" + -DPROJECT_VERSION="${PROJECT_VERSION}" + -P "${PPC_SCRIPTS_DIR}/buildinfo.cmake" + ) + include_directories(BEFORE ${PROJECT_BINARY_DIR}) +endfunction() diff --git a/cpp/cmake/BuildInfoGenerator.cmake b/cpp/cmake/BuildInfoGenerator.cmake new file mode 100644 index 00000000..00871500 --- /dev/null +++ b/cpp/cmake/BuildInfoGenerator.cmake @@ -0,0 +1,3 @@ +set(PPC_SCRIPTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cmake/scripts) +include(BuildInfo) +create_build_info() diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake new file mode 100644 index 00000000..005d130a --- /dev/null +++ b/cpp/cmake/CompilerSettings.cmake @@ -0,0 +1,166 @@ +if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")) + find_program(CCACHE_PROGRAM ccache) + if(CCACHE_PROGRAM) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE_PROGRAM}") + endif() + set(CMAKE_CXX_STANDARD 20) + # set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "/usr/bin/time") + # set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "/usr/bin/time") + # Use ISO C++17 standard language. + set(CMAKE_CXX_FLAGS "-pthread -fPIC -fexceptions") + # set(CMAKE_CXX_VISIBILITY_PRESET hidden) + # Enables all the warnings about constructions that some users consider questionable, + # and that are easy to avoid. Also enable some extra warning flags that are not + # enabled by -Wall. Finally, treat at warnings-as-errors, which forces developers + # to fix warnings as they arise, so they don't accumulate "to be fixed later". + add_compile_options(-Werror) + add_compile_options(-Wall) + add_compile_options(-pedantic) + add_compile_options(-Wextra) + add_compile_options(-Wno-deprecated-declarations) + add_compile_options(-Wno-unused-variable) + add_compile_options(-Wno-unused-parameter) + add_compile_options(-Wno-unused-function) + add_compile_options(-Wno-unused-private-field) + add_compile_options(-Wno-variadic-macros) + add_compile_options(-Wno-deprecated-declarations) + add_compile_options(-Wno-deprecated-copy) + # add_compile_options(-Wno-missing-field-initializers) + # Disable warnings about unknown pragmas (which is enabled by -Wall). + add_compile_options(-Wno-unknown-pragmas) + add_compile_options(-fno-omit-frame-pointer) + # for boost json spirit + add_compile_options(-DBOOST_SPIRIT_THREADSAFE) + # for tbb, TODO: https://software.intel.com/sites/default/files/managed/b2/d2/TBBRevamp.pdf + add_compile_options(-DTBB_SUPPRESS_DEPRECATED_MESSAGES=1) + # build deps lib Release + set(_only_release_configuration "-DCMAKE_BUILD_TYPE=Release") + + if(BUILD_STATIC) + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + SET(BUILD_SHARED_LIBRARIES OFF) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") + # Note: If bring the -static option, apple will fail to link + if (NOT APPLE) + SET(CMAKE_EXE_LINKER_FLAGS "-static") + endif() + # SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Bdynamic -ldl -lpthread -Wl,-Bstatic -static-libstdc++ ") + endif () + + if(TESTS) + add_compile_options(-DBOOST_TEST_THREAD_SAFE) + endif () + + if(PROF) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pg") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg") + endif () + + # Configuration-specific compiler settings. + set(CMAKE_CXX_FLAGS_DEBUG "-Og -g") + set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG") + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") + + option(USE_LD_GOLD "Use GNU gold linker" ON) + if (USE_LD_GOLD) + execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION) + if ("${LD_VERSION}" MATCHES "GNU gold") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold") + endif () + endif () + + # Additional GCC-specific compiler settings. + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") + # Check that we've got GCC 7.0 or newer. + set(GCC_MIN_VERSION "7.0") + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) + if (NOT (GCC_VERSION VERSION_GREATER ${GCC_MIN_VERSION} OR GCC_VERSION VERSION_EQUAL ${GCC_MIN_VERSION})) + message(FATAL_ERROR "${PROJECT_NAME} requires g++ ${GCC_MIN_VERSION} or greater. Current is ${GCC_VERSION}") + endif () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MARCH_TYPE}") + set(CMAKE_C_FLAGS "-std=c99 ${CMAKE_C_FLAGS} ${MARCH_TYPE}") + + # Strong stack protection was only added in GCC 4.9. + # Use it if we have the option to do so. + # See https://lwn.net/Articles/584225/ + if (GCC_VERSION VERSION_GREATER 4.9 OR GCC_VERSION VERSION_EQUAL 4.9) + add_compile_options(-fstack-protector-strong) + add_compile_options(-fstack-protector) + endif() + # Additional Clang-specific compiler settings. + elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.0) + set(CMAKE_CXX_FLAGS_DEBUG "-O -g") + endif() + # set(CMAKE_CXX_FLAGS "-stdlib=libc++ ${CMAKE_CXX_FLAGS}") + add_compile_options(-fstack-protector) + add_compile_options(-Winconsistent-missing-override) + # Some Linux-specific Clang settings. We don't want these for OS X. + if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") + # Tell Boost that we're using Clang's libc++. Not sure exactly why we need to do. + add_definitions(-DBOOST_ASIO_HAS_CLANG_LIBCXX) + # Use fancy colors in the compiler diagnostics + add_compile_options(-fcolor-diagnostics) + endif() + endif() + + if (COVERAGE) + set(TESTS ON) + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") + set(CMAKE_CXX_FLAGS "-g --coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-g --coverage ${CMAKE_C_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "--coverage ${CMAKE_EXE_LINKER_FLAGS}") + elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + add_compile_options(-Wno-unused-command-line-argument) + set(CMAKE_CXX_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_C_FLAGS}") + endif() + endif () +elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC") + + # Only support visual studio 2017 and visual studio 2019 + set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support + + message(STATUS "Compile On Windows, MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}") + + if (MSVC_TOOLSET_VERSION EQUAL 141) + message(STATUS "Compile On Visual Studio 2017") + elseif(MSVC_TOOLSET_VERSION EQUAL 142) + message(STATUS "Compile On Visual Studio 2019") + else() + message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}") + endif() + + add_definitions(-DUSE_STD_RANGES) + add_compile_options(/std:c++latest) + add_compile_options(-bigobj) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") + # set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG") + # set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os") + # set(CMAKE_CXX_FLAGS_RELEASE "/MT") + # set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG") + link_libraries(ws2_32 Crypt32 userenv) +else () + message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.") +endif () + +if (SANITIZE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=${SANITIZE}") + if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/sanitizer-blacklist.txt") + endif() +endif() + +# rust static library linking requirements for macos +if(APPLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Security") +else() + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ldl") +endif() +set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY ON) diff --git a/cpp/cmake/FindGSasl.cmake b/cpp/cmake/FindGSasl.cmake new file mode 100644 index 00000000..caefeea1 --- /dev/null +++ b/cpp/cmake/FindGSasl.cmake @@ -0,0 +1,27 @@ +# - Try to find the GNU sasl library (gsasl) +# +# Once done this will define +# +# GSASL_FOUND - System has gnutls +# GSASL_INCLUDE_DIR - The gnutls include directory +# GSASL_LIBRARIES - The libraries needed to use gnutls +# GSASL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + # in cache already + SET(GSasl_FIND_QUIETLY TRUE) +ENDIF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + +FIND_PATH(GSASL_INCLUDE_DIR gsasl.h) + +FIND_LIBRARY(GSASL_LIBRARIES gsasl) +FIND_LIBRARY(GSASL_STATIC_LIBRARIES NAMES "libgsasl.a") + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set GSASL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSASL DEFAULT_MSG GSASL_LIBRARIES GSASL_INCLUDE_DIR) + +MARK_AS_ADVANCED(GSASL_INCLUDE_DIR GSASL_LIBRARIES) \ No newline at end of file diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake new file mode 100644 index 00000000..c22d3cce --- /dev/null +++ b/cpp/cmake/IncludeDirectories.cmake @@ -0,0 +1,9 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-crypto-c-sdk) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-tars-protocol) + +set(VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include") +include_directories(${VCPKG_INCLUDE_PATH}) \ No newline at end of file diff --git a/cpp/cmake/InstallBcosUtilities.cmake b/cpp/cmake/InstallBcosUtilities.cmake new file mode 100644 index 00000000..5fc51ace --- /dev/null +++ b/cpp/cmake/InstallBcosUtilities.cmake @@ -0,0 +1,3 @@ +find_package(Boost COMPONENTS log filesystem chrono thread serialization iostreams system) +find_package(ZLIB REQUIRED) +find_package(bcos-utilities REQUIRED) \ No newline at end of file diff --git a/cpp/cmake/Installlibhdfs3.cmake b/cpp/cmake/Installlibhdfs3.cmake new file mode 100644 index 00000000..bcfc5d81 --- /dev/null +++ b/cpp/cmake/Installlibhdfs3.cmake @@ -0,0 +1,6 @@ +find_package(libxml2 REQUIRED) +find_package(protobuf REQUIRED) +if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") + find_package(unofficial-libuuid REQUIRED) +endif() +find_package(libhdfs3 REQUIRED) \ No newline at end of file diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake new file mode 100644 index 00000000..ab407a53 --- /dev/null +++ b/cpp/cmake/Options.cmake @@ -0,0 +1,151 @@ +macro(default_option O DEF) + if (DEFINED ${O}) + if (${${O}}) + set(${O} ON) + else () + set(${O} OFF) + endif () + else () + set(${O} ${DEF}) + endif () +endmacro() + +# common settings +if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "4") + message(FATAL "The ${PROJECT_NAME} does not support compiling on 32-bit systems") +endif () + +EXECUTE_PROCESS(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE) + +macro(configure_project) + set(NAME ${PROJECT_NAME}) + + # Default to RelWithDebInfo configuration if no configuration is explicitly specified. + if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) + endif () + + default_option(BUILD_STATIC OFF) + + # unit tests + default_option(TESTS OFF) + # code coverage + default_option(COVERAGE OFF) + + #debug + default_option(DEBUG OFF) + + #cem + default_option(BUILD_CEM OFF) + + #conn_psi + default_option(ENABLE_CONN OFF) + + default_option(ENABLE_DEMO OFF) + # sdk + default_option(BUILD_SDK OFF) + default_option(BUILD_UDF OFF) + if(BUILD_UDF) + set(BUILD_SDK ON) + endif() + + # Suffix like "-rc1" e.t.c. to append to versions wherever needed. + if (NOT DEFINED VERSION_SUFFIX) + set(VERSION_SUFFIX "") + endif () + print_config(${NAME}) +endmacro() + +# for boost-ssl enable/disable native +set(ARCH_NATIVE OFF) +if ("${ARCHITECTURE}" MATCHES "aarch64" OR "${ARCHITECTURE}" MATCHES "arm64") + set(ARCH_NATIVE ON) +endif () + +set(VISIBILITY_FLAG " -fvisibility=hidden -fvisibility-inlines-hidden") +if (BUILD_UDF) + set(VISIBILITY_FLAG "") +endif() +if (BUILD_SDK) + set(VISIBILITY_FLAG "") +endif() +set(MARCH_TYPE "-march=x86-64 -mtune=generic ${VISIBILITY_FLAG}") +if (ARCH_NATIVE) + set(MARCH_TYPE "-march=native -mtune=native ${VISIBILITY_FLAG}") +endif () + +# for enable sse4.2(hdfs used) +set(ENABLE_SSE OFF) +# for enable/disable ipp-crypto +if (APPLE) + EXECUTE_PROCESS(COMMAND sysctl -a COMMAND grep "machdep.cpu.*features" COMMAND tr -d '\n' OUTPUT_VARIABLE SUPPORTED_INSTRUCTIONS) + message("* SUPPORTED_INSTRUCTIONS: ${SUPPORTED_INSTRUCTIONS}") + # detect sse4.2 + if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*SSE4.2.*") + set(ENABLE_SSE ON) + endif () +elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # detect sse4_2 + FILE(READ "/proc/cpuinfo" SUPPORTED_INSTRUCTIONS) + if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*sse4_2.*") + set(ENABLE_SSE ON) + endif () +endif () + +set(ENABLE_CPU_FEATURES OFF) +# only ENABLE_CPU_FEATURES for aarch64 and x86 +if ("${ARCHITECTURE}" MATCHES "aarch64") + add_definitions(-DARCH) + set(ENABLE_CPU_FEATURES ON) +endif () + +if ("${ARCHITECTURE}" MATCHES "x86_64") + add_definitions(-DX86) + set(ENABLE_CPU_FEATURES ON) +endif () + +if (ENABLE_CPU_FEATURES) + add_definitions(-DENABLE_CPU_FEATURES) +endif () + +# Enable CONN_PSI Joint Running With Ant Company +if (ENABLE_CONN) + add_definitions(-DENABLE_CONN) +endif () + +set(ENABLE_IPP_CRYPTO OFF) +# Note: only ENABLE_CRYPTO_MB for x86_64 +# if ("${ARCHITECTURE}" MATCHES "x86_64") +# set(ENABLE_IPP_CRYPTO ON) +# add_definitions(-DENABLE_CRYPTO_MB) +# endif () + +# fix the boost beast build failed for [call to 'async_teardown' is ambiguous] +add_definitions(-DBOOST_ASIO_DISABLE_CONCEPTS) + +macro(print_config NAME) + message("") + message("------------------------------------------------------------------------") + message("-- Configuring ${NAME} ${PROJECT_VERSION}${VERSION_SUFFIX}") + message("------------------------------------------------------------------------") + message("-- CMake Cmake version and location ${CMAKE_VERSION} (${CMAKE_COMMAND})") + message("-- Compiler C++ compiler version ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") + message("-- CMAKE_BUILD_TYPE Build type ${CMAKE_BUILD_TYPE}") + message("-- TARGET_PLATFORM Target platform ${CMAKE_SYSTEM_NAME} ${ARCHITECTURE}") + message("-- BUILD_STATIC Build static ${BUILD_STATIC}") + message("-- COVERAGE Build code coverage ${COVERAGE}") + message("-- TESTS Build tests ${TESTS}") + message("-- ARCH_NATIVE Enable native code ${ARCH_NATIVE}") + message("-- ENABLE_CONN Enable conn_psi ant company ${ENABLE_CONN}") + message("-- ENABLE_IPP_CRYPTO Enable ipp-crypto ${ENABLE_IPP_CRYPTO}") + message("-- ENABLE_CPU_FEATURES Enable cpu-features ${ENABLE_CPU_FEATURES}") + message("-- ENABLE_SSE Enable SSE ${ENABLE_SSE}") + message("-- BUILD_CEM Enable CEM ${BUILD_CEM}") + message("-- DEMO Enable DEMO ${ENABLE_DEMO}") + message("-- BUILD_SDK BUILD SDK ${BUILD_SDK}") + message("-- BUILD_UDF BUILD UDF ${BUILD_UDF}") + message("-- DEBUG ${DEBUG}") + message("------------------------------------------------------------------------") + message("") +endmacro() \ No newline at end of file diff --git a/cpp/cmake/ProjectAPSI.cmake b/cpp/cmake/ProjectAPSI.cmake new file mode 100644 index 00000000..7dfadd01 --- /dev/null +++ b/cpp/cmake/ProjectAPSI.cmake @@ -0,0 +1,28 @@ +include(ExternalProject) +include(GNUInstallDirs) + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deps/) +ExternalProject_Add(APSI + PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/deps + GIT_REPOSITORY https://${URL_BASE}/WeDPR-Team/APSI.git + GIT_TAG 7626026ec925e865a8e722251a71c9f167eb5fd2 + GIT_SHALLOW false + BUILD_IN_SOURCE 1 + CMAKE_ARGS -DCMAKE_PREFIX_PATH=${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} -DCMAKE_INSTALL_PREFIX=${CMAKE_SOURCE_DIR}/deps + LOG_DOWNLOAD 0 + LOG_CONFIGURE 0 + LOG_BUILD 0 + LOG_INSTALL 0 + ) + +ExternalProject_Get_Property(APSI SOURCE_DIR) + +set(APSI_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/deps/include/APSI-0.8) +set(APSI_LIBRARY_DIR ${CMAKE_SOURCE_DIR}/deps/${CMAKE_INSTALL_LIBDIR}/libapsi-0.8.a) +file(MAKE_DIRECTORY ${APSI_INCLUDE_DIR}) # Must exist. + +add_library(apsi INTERFACE IMPORTED) +set_property(TARGET apsi PROPERTY INTERFACE_LINK_LIBRARIES ${APSI_LIBRARY_DIR}) +set_property(TARGET apsi PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${APSI_INCLUDE_DIR}) +add_dependencies(apsi APSI) +unset(SOURCE_DIR) diff --git a/cpp/cmake/ProjectTCMalloc.cmake b/cpp/cmake/ProjectTCMalloc.cmake new file mode 100644 index 00000000..dcdb29c4 --- /dev/null +++ b/cpp/cmake/ProjectTCMalloc.cmake @@ -0,0 +1,41 @@ +include(ExternalProject) + +set(GPERFTOOLS_OPTIONS --disable-shared --disable-cpu-profiler --disable-heap-profiler --disable-heap-checker --disable-debugalloc --enable-minimal) +set(TCMALLOC_LIB_NAME "libtcmalloc_minimal.a") +if (DEBUG) + set(GPERFTOOLS_OPTIONS "") + set(TCMALLOC_LIB_NAME "libtcmalloc${CMAKE_STATIC_LIBRARY_SUFFIX}") +endif() + +set(TCMALLOC_CONFIG ./configure CXXFLAGS=-DHAVE_POSIX_MEMALIGN_SYMBOL=1 --enable-frame-pointers ${GPERFTOOLS_OPTIONS} --prefix=${CMAKE_SOURCE_DIR}/deps/src/gperftools) + +set(TCMALLOC_MAKE make install) + +ExternalProject_Add(gperftools + PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/deps/ + DOWNLOAD_NAME gperftools-2.10.tar.gz + DOWNLOAD_NO_PROGRESS 1 + URL https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz + URL_HASH SHA256=83e3bfdd28b8bcf53222c3798d4d395d52dadbbae59e8730c4a6d31a9c3732d8 + BUILD_IN_SOURCE 1 + LOG_CONFIGURE 1 + LOG_BUILD 1 + LOG_INSTALL 1 + CONFIGURE_COMMAND ${TCMALLOC_CONFIG} + BUILD_COMMAND ${TCMALLOC_MAKE} + INSTALL_COMMAND "" + BUILD_BYPRODUCTS /.libs/${TCMALLOC_LIB_NAME} +) + +ExternalProject_Get_Property(gperftools SOURCE_DIR) +add_library(TCMalloc INTERFACE IMPORTED) + +set(TCMALLOC_INCLUDE_DIR ${SOURCE_DIR}/include/) +set(TCMALLOC_LIBRARY ${SOURCE_DIR}/.libs/${TCMALLOC_LIB_NAME}) +file(MAKE_DIRECTORY ${TCMALLOC_INCLUDE_DIR}) # Must exist. + +set_property(TARGET TCMalloc PROPERTY INTERFACE_LINK_LIBRARIES ${TCMALLOC_LIBRARY}) +set_property(TARGET TCMalloc PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TCMALLOC_INCLUDE_DIR}) + +add_dependencies(TCMalloc gperftools) +unset(SOURCE_DIR) diff --git a/cpp/cmake/ProjectWedprCrypto.cmake b/cpp/cmake/ProjectWedprCrypto.cmake new file mode 100644 index 00000000..641c6fa3 --- /dev/null +++ b/cpp/cmake/ProjectWedprCrypto.cmake @@ -0,0 +1,35 @@ +include(ExternalProject) +include(GNUInstallDirs) + +find_program(CARGO_COMMAND NAMES cargo REQUIRED PATHS "${USER_HOME}\\.cargo\\bin") + +if(NOT CARGO_COMMAND) + message(FATAL_ERROR "cargo is not installed") +endif() + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deps/) + +set(ECC_DIR "ffi/ffi_c/ffi_c_ecc_edwards25519") +ExternalProject_Add(WEDPR_CRYPTO_ECC + PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/deps + GIT_REPOSITORY https://${URL_BASE}/WeBankBlockchain/WeDPR-Lab-Crypto.git + GIT_TAG c569186dd4bb73c5348012e15b2db18f7f20c103 + GIT_SHALLOW false + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND cd ${ECC_DIR} && ${CARGO_COMMAND} build --release + INSTALL_COMMAND cp /target/release/libffi_c_edwards25519.a ${CMAKE_SOURCE_DIR}/deps/${CMAKE_INSTALL_LIBDIR}/libffi_c_edwards25519.a + LOG_DOWNLOAD 1 + LOG_CONFIGURE 1 + LOG_BUILD 0 + LOG_INSTALL 1 + ) + +ExternalProject_Get_Property(WEDPR_CRYPTO_ECC SOURCE_DIR) + +set(WEDPR_LIBRARY ${CMAKE_SOURCE_DIR}/deps/${CMAKE_INSTALL_LIBDIR}/libffi_c_edwards25519.a) + +add_library(wedpr_ecc INTERFACE IMPORTED) +set_property(TARGET wedpr_ecc PROPERTY INTERFACE_LINK_LIBRARIES ${WEDPR_LIBRARY}) +add_dependencies(wedpr_ecc WEDPR_CRYPTO_ECC) +unset(SOURCE_DIR) diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake new file mode 100644 index 00000000..61b17f44 --- /dev/null +++ b/cpp/cmake/TargetSettings.cmake @@ -0,0 +1,122 @@ +set(BCOS_UTILITIES_TARGET "bcos-utilities") +set(PROTOBUF_TARGET "protobuf::libprotobuf") +# ppc-crypto-core +set(CRYPTO_CORE_TARGET "ppc-crypto-core") +set(BCOS_BOOSTSSL_TARGET "bcos-boostssl") + +# ppc-crypto +set(CRYPTO_TARGET "ppc-crypto") +set(PAILLIER_TARGET "ppc-paillier") +set(FAHE_TARGET "ppc-fahe") +set(IHC_TARGET "ppc-ihc") +set(CRYPTO_CODEC_TARGET "ppc-crypto-codec") + +# ppc-tools +set(TOOLS_TARGET "ppc-tools") + +# ppc-protocol +set(PROTOCOL_TARGET "ppc-protocol") + +# ppc-tars-protocol +set(TARS_PROTOCOL_TARGET "ppc-protocol-tars") + +# ppc-front +SET(FRONT_TARGET "ppc-front") + +# ppc-gateway +set(GATEWAY_TARGET "ppc-gateway") + +# ppc-io +set(IO_TARGET "ppc-io") + +# ppc-storage +set(STORAGE_TARGET "ppc-storage") + +# ppc-psi: labeled_ps +set(PSI_FRAMEWORK_TARGET "psi-framework") +set(LABELED_PSI_TARGET "ppc-labeled-psi") +set(RA2018_PSI_TARGET "ppc-ra2018-psi") +set(CM2020_PSI_TARGET "ppc-cm2020-psi") +set(ECDH_2PC_PSI_TARGET "ecdh-2pc-psi") +set(ECDH_MULTI_PSI_TARGET "ecdh-multi-psi") +# TODO: optimize here +#set(ECDH_CONN_PSI_TARGET "ecdh-conn-psi") +set(ECDH_CONN_PSI_TARGET "") + +# ppc-http +set(HTTP_TARGET "ppc-http") +set(BS_ECDH_PSI_TARGET "bs-ecdh-psi") + +# ppc-rpc +set(RPC_TARGET "ppc-rpc") + +# libhelper +set(HELPER_TARGET "ppc-helper") + +# libinitializer +set(INIT_LIB init) + +# ppc-cem +set(CEM_TARGET "ppc-cem") + +# ppc-mpc +set(MPC_TARGET "ppc-mpc") + +# ppc-pir +set(PIR_TARGET "ppc-pir") + +# the binary-name +set(GATEWAY_BINARY_NAME ppc-gateway-service) +set(GATEWAY_PKG_NAME PPCGatewayService) + +# current only include the psi function +set(PRO_BINARY_NAME ppc-pro-node) +set(PPC_NODE_PKG_NAME PPCProNodeService) + +set(AIR_BINARY_NAME ppc-air-node) +set(CEM_BINARY_NAME wedpr-cem) +set(MPC_BINARY_NAME wedpr-mpc) + +set(HDFS_LIB libhdfs3-static) + +# set cpu-info +set(CPU_FEATURES_LIB "") +if (ENABLE_CPU_FEATURES) + set(CPU_FEATURES_LIB CpuFeatures::cpu_features) +endif () + +#====== ppc-crypto-c-sdk =========== +set(PPC_CRYPTO_C_SDK_STATIC_TARGET ppc-crypto-c-sdk-static) +set(PPC_CRYPTO_C_SDK_TARGET ppc-crypto-c-sdk) + +# add suffix for arm +if(ARCH_NATIVE) + message(STATUS "Building arm architecture, CMAKE_HOST_SYSTEM_PROCESSOR => ${CMAKE_HOST_SYSTEM_PROCESSOR}") + set(PPC_CRYPTO_C_SDK_STATIC_TARGET "ppc-crypto-c-sdk-aarch64") + set(PPC_CRYPTO_C_SDK_TARGET "ppc-crypto-c-sdk-static-aarch64") +endif() +#====== ppc-crypto-c-sdk =========== + + +#====== ppc-crypto-sdk-jni =========== +set(PPC_CRYPTO_SDK_JNI_TARGET ppc-crypto-sdk-jni) +set(PPC_CRYPTO_SDK_JNI_STATIC_TARGET ppc-crypto-sdk-jni-static) +# add suffix for arm +if(ARCH_NATIVE) + set(PPC_CRYPTO_SDK_JNI_TARGET "ppc-crypto-sdk-jni-aarch64") + set(PPC_CRYPTO_SDK_JNI_STATIC_TARGET "ppc-crypto-sdk-jni-static-aarch64") +endif() +#====== ppc-crypto-sdk-jni =========== + +# ========== ppc-udf =========== +set(PPC_UDF_TARGET ppc-udf) +set(PPC_UDF_STATIC_TARGET ppc-udf-static) + +# add suffix for arm +if(ARCH_NATIVE) + set(PPC_UDF_TARGET ppc-udf-aarch64) + set(PPC_UDF_STATIC_TARGET ppc-udf-static-aarch64) +endif() +# ========== ppc-udf =========== + +set(BOOST_UNIT_TEST Boost::unit_test_framework) \ No newline at end of file diff --git a/cpp/cmake/Version.cmake b/cpp/cmake/Version.cmake new file mode 100644 index 00000000..84833fb8 --- /dev/null +++ b/cpp/cmake/Version.cmake @@ -0,0 +1 @@ +set(VERSION "1.0.0") diff --git a/cpp/cmake/config.cmake b/cpp/cmake/config.cmake new file mode 100644 index 00000000..47e003f5 --- /dev/null +++ b/cpp/cmake/config.cmake @@ -0,0 +1,74 @@ +hunter_config(tarscpp VERSION 3.0.4-local + URL https://${URL_BASE}/FISCO-BCOS/TarsCpp/archive/e1a1df43e42a636d26c59f7fc12f62ae756d9ffc.tar.gz + SHA1 8632a852c0dbf367d8d4c4d2957fde2ff713abe0 + ) +hunter_config(cryptopp VERSION 8.6.0 + URL "https://${URL_BASE}/FISCO-BCOS/cryptopp/archive/1887f2f05679debf5d5b9553b8d1a686049eb8c0.tar.gz" + SHA1 062e044d15415151ddebe67cce63fbaeba4ba596 + ) + +# Note: mysql-connector-cpp default use the tassl +hunter_config(mysql-connector-cpp VERSION 8.0.31-local + URL "https://github.com/mysql/mysql-connector-cpp/archive/8.0.31.tar.gz" + SHA1 082c58faa4fbf61b796ccdb0b075b4cb3c331b9d + CMAKE_ARGS WITH_HUNTER_OpenSSl=ON WITH_HUNTER_PROTOBUF=ON + BUILDTYPE_DOCSTRING=${CMAKE_BUILD_TYPE} + ) + +hunter_config(OpenSSL VERSION tassl_1.1.1b_v1.4-local + URL https://${URL_BASE}/FISCO-BCOS/TASSL-1.1.1b/archive/5144cc460da7b7de36e27780aba0fd928da222ed.tar.gz + SHA1 e0d6de3a5abb2ea09cdf3fd22e50bd8b866125d6 + ) + +hunter_config(hiredis VERSION ${HUNTER_hiredis_VERSION}) +hunter_config(redis-plus-plus VERSION 1.3.6-local + URL https://${URL_BASE}/sewenew/redis-plus-plus/archive/1.3.6.tar.gz + SHA1 650a9fc65c958119f5360ae7dc2341487a29ac6d + CMAKE_ARGS REDIS_PLUS_PLUS_BUILD_STATIC=ON) + +hunter_config(SEAL VERSION 4.0.0 + URL https://${URL_BASE}/microsoft/SEAL/archive/4.0.0.tar.gz + SHA1 1ed568161c784d41a485590b08f841d213726e76 + CMAKE_ARGS SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF + ) + +hunter_config(Kuku VERSION 2.1.0 + URL https://${URL_BASE}/microsoft/Kuku/archive/2.1.0.tar.gz + SHA1 9dc206b885bd8da17877d78849c1b17a29bef59f + ) +hunter_config(tbb VERSION 2021.3.0 CMAKE_ARGS BUILD_SHARED_LIBS=OFF) + + +# the dependencies for libhdfs3 +hunter_config(Protobuf VERSION "3.19.4-p0" + URL https://github.com/cpp-pm/protobuf/archive/v3.19.4-p0.tar.gz + SHA1 e5b797dbc4e6ad92d0924ae86c130be4354c35b6 + CMAKE_ARGS CMAKE_POSITION_INDEPENDENT_CODE=TRUE + ) +hunter_config(libxml2 VERSION "2.9.7-p0" CMAKE_ARGS CMAKE_POSITION_INDEPENDENT_CODE=TRUE) +hunter_config(uuid VERSION 1.0.3 CMAKE_ARGS CMAKE_POSITION_INDEPENDENT_CODE=TRUE) + +hunter_config(cpu_features VERSION 0.7.0 + URL https://${URL_BASE}/google/cpu_features/archive/v0.7.0.tar.gz + SHA1 dacec18454c542ef450c883f45be7b2deaba7a91 + CMAKE_ARGS + CMAKE_POSITION_INDEPENDENT_CODE=TRUE + BUILD_TESTING=OFF + ) + +hunter_config(libhdfs3 VERSION 3.0.0-local + URL https://${URL_BASE}/FISCO-BCOS/libhdfs3/archive/f63e08d0c45f76591e448b56eb3f1b4e7de09c13.tar.gz + SHA1 88fdd418500478ea35fb09911d088430f0835893 + CMAKE_ARGS + ENABLE_BOOST=OFF + ENABLE_COVERAGE=OFF + ENABLE_SSE=${ENABLE_SSE} + ) + +hunter_config(ipp-crypto VERSION 202104-local + URL https://${URL_BASE}/intel/ipp-crypto/archive/ippcp_2021.4.tar.gz + SHA1 72edc2d16c30ec1612f45793befbc5689f72970f + CMAKE_ARGS + WITH_HUNTER_OpenSSl=ON + ) + \ No newline at end of file diff --git a/cpp/cmake/scripts/buildinfo.cmake b/cpp/cmake/scripts/buildinfo.cmake new file mode 100644 index 00000000..338156d0 --- /dev/null +++ b/cpp/cmake/scripts/buildinfo.cmake @@ -0,0 +1,70 @@ +macro(replace_if_different SOURCE DST) + set(extra_macro_args ${ARGN}) + set(options CREATE) + set(one_value_args) + set(multi_value_args) + cmake_parse_arguments(REPLACE_IF_DIFFERENT "${options}" "${one_value_args}" "${multi_value_args}" "${extra_macro_args}") + + if (REPLACE_IF_DIFFERENT_CREATE AND (NOT (EXISTS "${DST}"))) + file(WRITE "${DST}" "") + endif() + + execute_process(COMMAND ${CMAKE_COMMAND} -E compare_files "${SOURCE}" "${DST}" RESULT_VARIABLE DIFFERENT OUTPUT_QUIET ERROR_QUIET) + + if (DIFFERENT) + execute_process(COMMAND ${CMAKE_COMMAND} -E rename "${SOURCE}" "${DST}") + else() + execute_process(COMMAND ${CMAKE_COMMAND} -E remove "${SOURCE}") + endif() +endmacro() + +if (NOT PPC_BUILD_TYPE) + set(PPC_BUILD_TYPE "unknown") +endif() + +if (NOT PPC_BUILD_PLATFORM) + set(PPC_BUILD_PLATFORM "unknown") +endif() + +execute_process( + COMMAND git --git-dir=${PPC_SOURCE_DIR}/.git --work-tree=${PPC_SOURCE_DIR} rev-parse HEAD + OUTPUT_VARIABLE PPC_COMMIT_HASH OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET +) + +if (NOT PPC_COMMIT_HASH) + set(PPC_COMMIT_HASH 0) +endif() + +execute_process( + COMMAND git --git-dir=${PPC_SOURCE_DIR}/.git --work-tree=${PPC_SOURCE_DIR} diff HEAD --shortstat + OUTPUT_VARIABLE PPC_LOCAL_CHANGES OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET +) + +execute_process( + COMMAND date "+%Y%m%d %H:%M:%S" + OUTPUT_VARIABLE PPC_BUILD_TIME OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + ) + +execute_process( + COMMAND date "+%Y%m%d" + OUTPUT_VARIABLE PPC_BUILD_NUMBER OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + ) + +execute_process( + COMMAND git rev-parse --abbrev-ref HEAD + OUTPUT_VARIABLE PPC_BUILD_BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + ) + +if (PPC_LOCAL_CHANGES) + set(PPC_CLEAN_REPO 0) +else() + set(PPC_CLEAN_REPO 1) +endif() + +set(TMPFILE "${PPC_DST_DIR}/BuildInfo.h.tmp") +set(OUTFILE "${PPC_DST_DIR}/BuildInfo.h") + +configure_file("${PPC_BUILDINFO_IN}" "${TMPFILE}") + +replace_if_different("${TMPFILE}" "${OUTFILE}" CREATE) + diff --git a/cpp/cmake/templates/BuildInfo.h.in b/cpp/cmake/templates/BuildInfo.h.in new file mode 100644 index 00000000..4bb5187b --- /dev/null +++ b/cpp/cmake/templates/BuildInfo.h.in @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BuildInfo.h + */ +#pragma once +#define PPC_COMMIT_HASH "@PPC_COMMIT_HASH@" +#define PPC_CLEAN_REPO "@PPC_CLEAN_REPO@" +#define PPC_BUILD_TYPE "@PPC_BUILD_TYPE@" +#define PPC_BUILD_OS "@PPC_BUILD_OS@" +#define PPC_BUILD_COMPILER "@PPC_BUILD_COMPILER@" +#define PPC_BUILD_PLATFORM "@PPC_BUILD_PLATFORM@" +#define PPC_BUILD_NUMBER "@PPC_BUILD_NUMBER@" +#define PPC_VERSION_SUFFIX "@PPC_VERSION_SUFFIX@" +#define PPC_BUILD_TIME "@PPC_BUILD_TIME@" +#define PPC_BUILD_BRANCH "@PPC_BUILD_BRANCH@" +#define PPC_PROJECT_VERSION "@PROJECT_VERSION@@PPC_VERSION_SUFFIX@" \ No newline at end of file diff --git a/cpp/libhelper/CMakeLists.txt b/cpp/libhelper/CMakeLists.txt new file mode 100644 index 00000000..77e19f19 --- /dev/null +++ b/cpp/libhelper/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE SRCS *.cpp) + +include_directories(${CMAKE_BINARY_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +add_library(${HELPER_TARGET} ${SRCS}) +find_package(Boost COMPONENTS filesystem program_options) +target_link_libraries(${HELPER_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} Boost::filesystem Boost::program_options) \ No newline at end of file diff --git a/cpp/libhelper/CommandHelper.cpp b/cpp/libhelper/CommandHelper.cpp new file mode 100644 index 00000000..49c94fcd --- /dev/null +++ b/cpp/libhelper/CommandHelper.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CommandHelper.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "CommandHelper.h" +#include +#include +#include +#include +#include + +using namespace ppc; + +void ppc::printVersion() +{ + std::cout << "PPCS-Core Version : " << PPC_PROJECT_VERSION << std::endl; + std::cout << "Build Time : " << PPC_BUILD_TIME << std::endl; + std::cout << "Build Type : " << PPC_BUILD_PLATFORM << "/" << PPC_BUILD_TYPE + << std::endl; + std::cout << "Git Branch : " << PPC_BUILD_BRANCH << std::endl; + std::cout << "Git Commit : " << PPC_COMMIT_HASH << std::endl; +} + +CommandLineParam ppc::initCommandLine(int argc, const char* argv[]) +{ + boost::program_options::options_description main_options("Usage of PPC"); + main_options.add_options()("help,h", "print help information")("version,v", "version of PPC")( + "config,c", boost::program_options::value(), "./config.ini"); + boost::program_options::variables_map vm; + try + { + boost::program_options::store( + boost::program_options::parse_command_line(argc, argv, main_options), vm); + } + catch (...) + { + printVersion(); + } + /// help information + if (vm.count("help") || vm.count("h")) + { + std::cout << main_options << std::endl; + exit(0); + } + /// version information + if (vm.count("version") || vm.count("v")) + { + printVersion(); + exit(0); + } + std::string configPath("./config.ini"); + if (vm.count("config")) + { + configPath = vm["config"].as(); + } + if (vm.count("c")) + { + configPath = vm["c"].as(); + } + // std::ifstream configFile(configPath); + // if (!configFile) + if (!boost::filesystem::exists(configPath)) + { + std::cout << "config \'" << configPath << "\' not found!"; + exit(0); + } + return ppc::CommandLineParam{configPath}; +} + +void ppc::initAppCommandLine(int argc, char* argv[]) +{ + boost::program_options::options_description main_options("Usage of PPC"); + main_options.add_options()("help,h", "print help information")("version,v", "version of PPC"); + boost::program_options::variables_map vm; + try + { + boost::program_options::store( + boost::program_options::parse_command_line(argc, argv, main_options), vm); + } + catch (...) + { + printVersion(); + } + /// help information + if (vm.count("help") || vm.count("h")) + { + std::cout << main_options << std::endl; + exit(0); + } + /// version information + if (vm.count("version") || vm.count("v")) + { + printVersion(); + exit(0); + } +} diff --git a/cpp/libhelper/CommandHelper.h b/cpp/libhelper/CommandHelper.h new file mode 100644 index 00000000..7724d0b1 --- /dev/null +++ b/cpp/libhelper/CommandHelper.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CommandHelper.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include +#include + +namespace ppc +{ +struct CommandLineParam +{ + std::string configFilePath; +}; +void printVersion(); +CommandLineParam initCommandLine(int argc, const char* argv[]); +void initAppCommandLine(int argc, char* argv[]); +} // namespace ppc \ No newline at end of file diff --git a/cpp/libhelper/ExitHandler.h b/cpp/libhelper/ExitHandler.h new file mode 100644 index 00000000..93f66195 --- /dev/null +++ b/cpp/libhelper/ExitHandler.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ExitHandler.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include + +namespace ppc +{ +class ExitHandler +{ +public: + void exit() { exitHandler(0); } + static void exitHandler(int signal) + { + std::cout << "[" << bcos::getCurrentDateTime() << "] " + << "exit because receive signal " << signal << std::endl; + ExitHandler::c_shouldExit.store(true); + } + bool shouldExit() const { return ExitHandler::c_shouldExit.load(); } + + static std::atomic_bool c_shouldExit; +}; +std::atomic_bool ExitHandler::c_shouldExit = {false}; + +void setDefaultOrCLocale() +{ +#if __unix__ + if (!std::setlocale(LC_ALL, "")) + { + setenv("LC_ALL", "C", 1); + } +#endif +} +} // namespace ppc \ No newline at end of file diff --git a/cpp/libinitializer/CMakeLists.txt b/cpp/libinitializer/CMakeLists.txt new file mode 100644 index 00000000..7fec7d12 --- /dev/null +++ b/cpp/libinitializer/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB SRC_LIST "*.cpp") + +add_library(${INIT_LIB} ${SRC_LIST}) +target_link_libraries(${INIT_LIB} PUBLIC ${PROTOCOL_TARGET} ${CRYPTO_TARGET} ${FRONT_TARGET} ${LABELED_PSI_TARGET} ${RA2018_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${ECDH_MULTI_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${BS_ECDH_PSI_TARGET} ${PIR_TARGET} ${IO_TARGET} ${STORAGE_TARGET}) diff --git a/cpp/libinitializer/Common.h b/cpp/libinitializer/Common.h new file mode 100644 index 00000000..93427dfc --- /dev/null +++ b/cpp/libinitializer/Common.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-11-14 + */ + +#pragma once +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "ppc-tools/src/config/Common.h" +#include +#include +#include +#include +#include + +#define INIT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("Initializer") + +namespace ppc::initializer +{ +inline bcos::bytes loadPrivateKey(std::string const& _keyPath, unsigned _hexedPrivateKeySize) +{ + auto keyContent = bcos::readContents(boost::filesystem::path(_keyPath)); + std::shared_ptr ecKey; + try + { + std::shared_ptr bioMem(BIO_new(BIO_s_mem()), [&](BIO* p) { BIO_free(p); }); + BIO_write(bioMem.get(), keyContent->data(), keyContent->size()); + + std::shared_ptr evpPKey(PEM_read_bio_PrivateKey(bioMem.get(), NULL, NULL, NULL), + [](EVP_PKEY* p) { EVP_PKEY_free(p); }); + if (!evpPKey) + { + BOOST_THROW_EXCEPTION( + ppc::tools::InvalidConfig() << bcos::errinfo_comment( + "SecureInitializer: parse privateKey failed for empty EVP_PKEY!")); + } + ecKey.reset(EVP_PKEY_get1_EC_KEY(evpPKey.get()), [](EC_KEY* p) { EC_KEY_free(p); }); + } + catch (std::exception const& e) + { + INIT_LOG(ERROR) << LOG_BADGE("SecureInitializer") << LOG_DESC("parse privateKey failed") + << LOG_KV("EINFO", boost::diagnostic_information(e)); + BOOST_THROW_EXCEPTION(ppc::tools::InvalidConfig() << bcos::errinfo_comment( + "SecureInitializer: parse privateKey failed, error: " + + std::string(boost::diagnostic_information(e)))); + } + std::shared_ptr ecPrivateKey( + EC_KEY_get0_private_key(ecKey.get()), [](const BIGNUM*) {}); + + std::shared_ptr privateKeyData( + BN_bn2hex(ecPrivateKey.get()), [](char* p) { OPENSSL_free(p); }); + std::string keyHex(privateKeyData.get()); + if (keyHex.size() >= _hexedPrivateKeySize) + { + return *(bcos::fromHexString(keyHex)); + } + for (size_t i = keyHex.size(); i < _hexedPrivateKeySize; i++) + { + keyHex = '0' + keyHex; + } + return *(bcos::fromHexString(keyHex)); +} +} // namespace ppc::initializer +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/cpp/libinitializer/FrontInitializer.cpp b/cpp/libinitializer/FrontInitializer.cpp new file mode 100644 index 00000000..6cbd6702 --- /dev/null +++ b/cpp/libinitializer/FrontInitializer.cpp @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontInitializer.cpp + * @author: shawnhe + * @date 2022-10-20 + */ + +#include "FrontInitializer.h" + +using namespace ppctars; +using namespace ppc; +using namespace ppc::front; +using namespace ppc::protocol; +using namespace ppc::initializer; + +void FrontInitializer::start() +{ + if (!m_running.exchange(true)) + { + FRONT_LOG(INFO) << LOG_DESC("start the front"); + if (m_front) + { + m_front->start(); + } + if (m_gatewayReporter) + { + m_gatewayReporter->registerTimeoutHandler( + boost::bind(&FrontInitializer::reportOtherGateway, this)); + m_gatewayReporter->start(); + } + FRONT_LOG(INFO) << LOG_DESC("start the front success"); + } +} + +void FrontInitializer::stop() +{ + if (m_running.exchange(false)) + { + FRONT_LOG(INFO) << LOG_DESC("stop the front"); + if (m_front) + { + m_front->stop(); + } + if (m_gatewayReporter) + { + m_gatewayReporter->stop(); + } + FRONT_LOG(INFO) << LOG_DESC("stop the front success"); + } +} + +void FrontInitializer::init() +{ + FRONT_LOG(INFO) << LOG_BADGE("init front"); + + auto ioService = std::make_shared(); + // init front + m_frontFactory = std::make_shared(m_selfAgencyId, m_threadPool); + m_front = m_frontFactory->buildFront(ioService); + FRONT_LOG(INFO) << LOG_BADGE("init front success"); +} diff --git a/cpp/libinitializer/FrontInitializer.h b/cpp/libinitializer/FrontInitializer.h new file mode 100644 index 00000000..f00491e2 --- /dev/null +++ b/cpp/libinitializer/FrontInitializer.h @@ -0,0 +1,104 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontInitializer.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "ppc-framework/front/Channel.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include "ppc-framework/rpc/RpcStatusInterface.h" +#include "ppc-front/ppc-front/Front.h" +#include "ppc-front/ppc-front/PPCChannelManager.h" +#include +#include + +namespace ppc +{ +namespace initializer +{ +class FrontInitializer +{ +public: + using Ptr = std::shared_ptr; + + explicit FrontInitializer(const std::string& _selfAgencyId, + std::shared_ptr _threadPool, + front::PPCMessageFaceFactory::Ptr _messageFactory) + : m_selfAgencyId(_selfAgencyId), + m_threadPool(std::move(_threadPool)), + m_messageFactory(std::move(_messageFactory)), + m_gatewayReporter(std::make_shared(3000, "gatewayReporter")) + { + init(); + } + + virtual ~FrontInitializer() { stop(); } + + virtual void start(); + virtual void stop(); + + const std::string& selfAgencyId() const { return m_selfAgencyId; } + std::shared_ptr threadPool() { return m_threadPool; } + + front::Front::Ptr front() { return m_front; } + + front::FrontFactory::Ptr frontFactory() { return m_frontFactory; } + + front::PPCMessageFaceFactory::Ptr messageFactory() { return m_messageFactory; } + + void setRpcStatus(rpc::RpcStatusInterface::Ptr _status) { m_rpcStatus = std::move(_status); } + +protected: + void init(); + // report the gateway endpoints of other parties to the gateway periodically + // Note: must set gatewayInterface into the front before call this function + // Note: must set RpcStorageInterface into the FrontInitializer before call this function + virtual void reportOtherGateway() + { + try + { + auto gatewayList = m_rpcStatus->listGateway(); + if (!gatewayList.empty()) + { + m_front->gatewayInterface()->registerGateway(gatewayList); + } + } + catch (std::exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("reportOtherGateway exception") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + m_gatewayReporter->restart(); + } + +protected: + std::string m_selfAgencyId; + std::shared_ptr m_threadPool; + front::PPCMessageFaceFactory::Ptr m_messageFactory; + + front::Front::Ptr m_front; + front::FrontFactory::Ptr m_frontFactory; + + std::atomic_bool m_running = {false}; + std::shared_ptr m_gatewayReporter; + rpc::RpcStatusInterface::Ptr m_rpcStatus; +}; +} // namespace initializer +} // namespace ppc \ No newline at end of file diff --git a/cpp/libinitializer/Initializer.cpp b/cpp/libinitializer/Initializer.cpp new file mode 100644 index 00000000..cff5c1a3 --- /dev/null +++ b/cpp/libinitializer/Initializer.cpp @@ -0,0 +1,544 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Initializer.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "Initializer.h" +#include "Common.h" +#include "ProFrontInitializer.h" +#include "ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h" +#include "ppc-crypto/src/oprf/RA2018Oprf.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-pir/src/OtPIRFactory.h" +#include "ppc-pir/src/OtPIRImpl.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h" +#include "ppc-psi/src/cm2020-psi/CM2020PSIFactory.h" +#if 0 +//TODO: optimize here +#include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h" +#endif +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h" +#include "ppc-psi/src/ecdh-psi/EcdhPSIFactory.h" +#include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h" +#include "ppc-psi/src/ra2018-psi/RA2018PSIFactory.h" +#include +#include + +using namespace ppc::initializer; +using namespace ppc::storage; +using namespace ppc::front; +using namespace ppc::psi; +using namespace ppc::pir; +using namespace ppc::tools; +using namespace ppc::crypto; + +Initializer::Initializer(std::string const& _configPath) : m_configPath(_configPath) +{ + // load the config + m_config = std::make_shared(); + m_config->loadConfig(_configPath); +} + +void Initializer::init(ppc::protocol::NodeArch _arch) +{ + INIT_LOG(INFO) << LOG_DESC("init the wedpr-component") << LOG_KV("arch", _arch); + // load the protocol + m_protocolInitializer = std::make_shared(); + m_protocolInitializer->init(m_config); + + auto ppcMessageFactory = std::make_shared(); + // init the frontService + INIT_LOG(INFO) << LOG_DESC("init the frontService") << LOG_KV("agency", m_config->agencyID()); + auto frontThreadPool = std::make_shared("front", m_config->threadPoolSize()); + + // Note: must set the m_holdingMessageMinutes before init the node + if (_arch == ppc::protocol::NodeArch::AIR) + { + m_frontInitializer = std::make_shared( + m_config->agencyID(), frontThreadPool, m_protocolInitializer->ppcMsgFactory()); + // load the gateway config + m_config->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, m_configPath); + } + else + { + m_frontInitializer = std::make_shared( + m_config->agencyID(), frontThreadPool, m_protocolInitializer->ppcMsgFactory()); + m_agencyInfoFetcher = std::make_shared(3000, "agencyInfoFetcher"); + // Note: the timer start work only after calling start + auto self = weak_from_this(); + m_agencyInfoFetcher->registerTimeoutHandler([self]() { + auto init = self.lock(); + if (!init) + { + return; + } + init->fetchAgencyListPeriodically(); + }); + // load the tars config + m_config->loadTarsConfig(m_configPath); + } + INIT_LOG(INFO) << LOG_DESC("init the frontService success") + << LOG_KV("agency", m_config->agencyID()) << LOG_KV("arch", _arch); + + auto cryptoBox = m_protocolInitializer->cryptoBox(); + SQLStorage::Ptr sqlStorage = nullptr; + if (!m_config->disableRA2018()) + { + // init the SQLStorage(without set the dbName) + auto opt = std::make_shared( + *(m_config->storageConfig().sqlConnectionOpt)); + opt->database = ""; + INIT_LOG(INFO) << LOG_DESC("init the sqlStorage for ra2018-psi") << opt->desc(); + sqlStorage = m_protocolInitializer->sqlStorageFactory()->createSQLStorage( + ppc::protocol::DataResourceType::MySQL, opt); + INIT_LOG(INFO) << LOG_DESC("init the sqlStorage for ra2018-psi success"); + } + // init the fileStorage for ra2018 + FileStorage::Ptr fileStorage = nullptr; + if (!m_config->disableRA2018() && m_config->ra2018PSIConfig().useHDFS) + { + auto const& fileStorageOption = m_config->storageConfig().fileStorageConnectionOpt; + INIT_LOG(INFO) << LOG_DESC("init the hdfs-storage for ra2018-psi") + << fileStorageOption->desc(); + fileStorage = m_protocolInitializer->fileStorageFactory()->createFileStorage( + ppc::protocol::DataResourceType::HDFS, fileStorageOption); + INIT_LOG(INFO) << LOG_DESC("init the hdfs-storage for ra2018-psi success"); + } + + // init the ra2018-psi + INIT_LOG(INFO) << LOG_DESC("init the ra2018-psi"); + auto ra2018PSIFactory = std::make_shared(); + auto oprf = std::make_shared( + m_config->privateKey(), cryptoBox->eccCrypto(), cryptoBox->hashImpl()); + m_ra2018PSI = ra2018PSIFactory->createRA2018PSI(m_config->agencyID(), + m_frontInitializer->front(), m_config, oprf, m_protocolInitializer->binHashImpl(), + m_protocolInitializer->ppcMsgFactory(), sqlStorage, fileStorage, + std::make_shared("ra2018", m_config->threadPoolSize()), + m_protocolInitializer->dataResourceLoader()); + INIT_LOG(INFO) << LOG_DESC("init the ra2018-psi success"); + + // init the labeled-psi + INIT_LOG(INFO) << LOG_DESC("init the labeled-psi"); + auto labeledPSIFactory = std::make_shared(); + m_labeledPSI = labeledPSIFactory->buildLabeledPSI(m_config->agencyID(), + m_frontInitializer->front(), cryptoBox, + std::make_shared("t_labeled-psi", m_config->threadPoolSize()), + m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes()); + INIT_LOG(INFO) << LOG_DESC("init the labeled-psi success"); + + // init the cm2020-psi + INIT_LOG(INFO) << LOG_DESC("init the cm2020-psi"); + auto cm2020PSIFactory = std::make_shared(); + m_cm2020PSI = cm2020PSIFactory->buildCM2020PSI(m_config->agencyID(), + m_frontInitializer->front(), cryptoBox, + std::make_shared("t_cm2020-psi", m_config->threadPoolSize()), + m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes(), + m_config->cm2020PSIConfig().parallelism); + INIT_LOG(INFO) << LOG_DESC("init the cm2020-psi success"); + + // init the ecdh-psi + INIT_LOG(INFO) << LOG_DESC("create ecdh-psi"); + auto ecdhPSIFactory = std::make_shared(); + auto ecdhCryptoFactory = std::make_shared(m_config->privateKey()); + m_ecdhPSI = ecdhPSIFactory->createEcdhPSI(m_config, ecdhCryptoFactory, + m_frontInitializer->front(), m_protocolInitializer->ppcMsgFactory(), nullptr, + m_protocolInitializer->dataResourceLoader()); + INIT_LOG(INFO) << LOG_DESC("create ecdh-psi success"); + +#if 0 +// TODO: optimize here + // init the ecdh-conn-psi + INIT_LOG(INFO) << LOG_DESC("create ecdh-conn-psi"); + auto ecdhConnPSIFactory = std::make_shared(); + m_ecdhConnPSI = ecdhConnPSIFactory->createEcdhConnPSI(m_config, ecdhCryptoFactory, + m_frontInitializer->front(), m_protocolInitializer->ppcMsgFactory(), + std::make_shared("t_ecdh-conn-psi", std::thread::hardware_concurrency()), + m_protocolInitializer->dataResourceLoader()); + INIT_LOG(INFO) << LOG_DESC("create ecdh-conn-psi success"); +#endif + // init the ecdh-multi-psi + INIT_LOG(INFO) << LOG_DESC("create ecdh-multi-psi"); + auto ecdhMultiPSIFactory = std::make_shared(); + m_ecdhMultiPSI = ecdhMultiPSIFactory->createEcdhMultiPSI(m_config, m_frontInitializer->front(), + cryptoBox, + std::make_shared("t_ecdh-multi-psi", std::thread::hardware_concurrency()), + m_protocolInitializer->dataResourceLoader()); + INIT_LOG(INFO) << LOG_DESC("create ecdh-multi-psi success"); + + // init the ot-pir + INIT_LOG(INFO) << LOG_DESC("create ot-pir"); + auto otPIRFactory = std::make_shared(); + m_otPIR = otPIRFactory->buildOtPIR(m_config->agencyID(), m_frontInitializer->front(), cryptoBox, + std::make_shared("t_ot-pir", std::thread::hardware_concurrency()), + m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes()); + + // auto otPIRImpl = std::make_shared(); + // m_otPIR = otPIRImpl; + // init the bs mode ecdh psi + INIT_LOG(INFO) << LOG_DESC("create bs mode ecdh psi"); + auto bsEcdhPSIFactory = std::make_shared(); + m_bsEcdhPSI = bsEcdhPSIFactory->buildBsEcdhPSI( + std::make_shared("t_bs-ecdh-psi", m_config->threadPoolSize()), + m_protocolInitializer->dataResourceLoader(), m_config->taskTimeoutMinutes()); + INIT_LOG(INFO) << LOG_DESC("create bs mode ecdh psi success"); + + initMsgHandlers(); + + // add parallelism control for tbb + tbb::global_control gc( + tbb::global_control::max_allowed_parallelism, m_config->threadPoolSize()); +} + +// init the msg-handlers +void Initializer::initMsgHandlers() +{ + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ra2018PSI"); + // register msg-handlers for ra2018-psi + auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::RA_PSI_2PC, + [weakRA2018PSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakRA2018PSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + }); + + // register msg-handlers for labeled-psi + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for labeledPSI"); + auto weakLabeledPSI = std::weak_ptr(m_labeledPSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC, + [weakLabeledPSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakLabeledPSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + }); + + // register msg-handlers for cm2020-psi + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for CM2020PSI"); + auto weakCM2020PSI = std::weak_ptr(m_cm2020PSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::CM_PSI_2PC, + [weakCM2020PSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakCM2020PSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + }); + + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-psi"); + // register msg-handlers for ecdh-psi + auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC, + [weakEcdhPSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakEcdhPSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + }); + + // register msg-handlers for ecdh-conn-psi + /*INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-conn-psi"); + auto weakEcdhConnPSI = std::weak_ptr(m_ecdhConnPSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN, + [weakEcdhConnPSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakEcdhConnPSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + });*/ + + // register msg-handlers for ecdh-multi-psi + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-multi-psi"); + auto weakEcdhMultiPSI = std::weak_ptr(m_ecdhMultiPSI); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI, + [weakEcdhMultiPSI](ppc::front::PPCMessageFace::Ptr _msg) { + auto psi = weakEcdhMultiPSI.lock(); + if (!psi) + { + return; + } + psi->onReceiveMessage(_msg); + }); + + INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ot-pir"); + // register msg-handlers for ecdh-psi + auto weakOtPIR = std::weak_ptr(m_otPIR); + m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PIR, + (uint8_t)ppc::protocol::PSIAlgorithmType::OT_PIR_2PC, + [weakOtPIR](ppc::front::PPCMessageFace::Ptr _msg) { + auto pir = weakOtPIR.lock(); + if (!pir) + { + return; + } + pir->onReceiveMessage(_msg); + }); +} + +void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc) +{ + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler"); + + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ra2018PSI"); + // register task handler for ra2018-psi + auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::RA_PSI_2PC, + [weakRA2018PSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto ra2018Impl = weakRA2018PSI.lock(); + if (!ra2018Impl) + { + return; + } + ra2018Impl->asyncRunTask(_task, std::move(_handler)); + }); + + // register task handler for labeled-psi + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for labeledPSI"); + auto weakLabeledPSI = std::weak_ptr(m_labeledPSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC, + [weakLabeledPSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto labeledPSI = weakLabeledPSI.lock(); + if (!labeledPSI) + { + return; + } + labeledPSI->asyncRunTask(_task, std::move(_handler)); + }); + + // register task handler for cm2020-psi + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for cm2020PSI"); + auto weakCM2020PSI = std::weak_ptr(m_cm2020PSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::CM_PSI_2PC, + [weakCM2020PSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto cm2020PSI = weakCM2020PSI.lock(); + if (!cm2020PSI) + { + return; + } + cm2020PSI->asyncRunTask(_task, std::move(_handler)); + }); + + // register task handler for ecdh-psi + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhPSI"); + auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC, + [weakEcdhPSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto psi = weakEcdhPSI.lock(); + if (!psi) + { + return; + } + psi->asyncRunTask(_task, std::move(_handler)); + }); + + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhMultiPSI"); + auto weakEcdhMultiPSI = std::weak_ptr(m_ecdhMultiPSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI, + [weakEcdhMultiPSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto psi = weakEcdhMultiPSI.lock(); + if (!psi) + { + return; + } + psi->asyncRunTask(_task, std::move(_handler)); + }); + + /*INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhConnPSI"); + auto weakEcdhConnPSI = std::weak_ptr(m_ecdhConnPSI); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI, + (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN, + [weakEcdhConnPSI](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto psi = weakEcdhConnPSI.lock(); + if (!psi) + { + return; + } + psi->asyncRunTask(_task, std::move(_handler)); + });*/ + + // register task handler for ot-pir + INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for otPIR"); + auto weakOtPIR = std::weak_ptr(m_otPIR); + _rpc->registerTaskHandler(ppc::protocol::TaskType::PIR, + (uint8_t)ppc::protocol::PSIAlgorithmType::OT_PIR_2PC, + [weakOtPIR](ppc::protocol::Task::ConstPtr _task, + std::function _handler) { + auto pir = weakOtPIR.lock(); + if (!pir) + { + return; + } + pir->asyncRunTask(_task, std::move(_handler)); + }); +} + +void Initializer::start() +{ + if (m_agencyInfoFetcher) + { + m_agencyInfoFetcher->start(); + } + if (m_frontInitializer) + { + m_frontInitializer->start(); + } + /*if (m_ecdhConnPSI) + { + m_ecdhConnPSI->start(); + }*/ + if (m_ecdhMultiPSI) + { + m_ecdhMultiPSI->start(); + } + if (m_cm2020PSI) + { + m_cm2020PSI->start(); + } + if (m_ra2018PSI) + { + m_ra2018PSI->start(); + } + if (m_labeledPSI) + { + m_labeledPSI->start(); + } + // start the ecdh-psi + if (m_ecdhPSI) + { + m_ecdhPSI->start(); + } + if (m_otPIR) + { + m_otPIR->start(); + } + + if (m_bsEcdhPSI) + { + m_bsEcdhPSI->start(); + } +} + +void Initializer::stop() +{ + if (m_agencyInfoFetcher) + { + m_agencyInfoFetcher->stop(); + } + // stop the network firstly + if (m_frontInitializer) + { + m_frontInitializer->stop(); + } + /*if (m_ecdhConnPSI) + { + m_ecdhConnPSI->stop(); + }*/ + if (m_ecdhMultiPSI) + { + m_ecdhMultiPSI->stop(); + } + if (m_cm2020PSI) + { + m_cm2020PSI->stop(); + } + // if (m_ra2018PSI) + // { + // m_ra2018PSI->stop(); + // } + // if (m_labeledPSI) + // { + // m_labeledPSI->stop(); + // } + // stop the ecdh-psi + if (m_ecdhPSI) + { + m_ecdhPSI->stop(); + } + if (m_otPIR) + { + m_otPIR->stop(); + } + if (m_bsEcdhPSI) + { + m_bsEcdhPSI->stop(); + } +} + +void Initializer::fetchAgencyListPeriodically() +{ + if (!m_agencyInfoFetcher) + { + return; + } + fetchAgencyList(); + m_agencyInfoFetcher->restart(); +} + +// fetch the agency-list for ecdh and ra2018 periodically in pro-mode +void Initializer::fetchAgencyList() +{ + auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI); + auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI); + m_frontInitializer->front()->asyncGetAgencyList( + [weakEcdhPSI, weakRA2018PSI]( + bcos::Error::Ptr _error, std::vector&& _agencyList) { + if (_error) + { + INIT_LOG(INFO) << LOG_DESC("asyncGetAgencyList failed") + << LOG_KV("code", _error->errorCode()); + return; + } + auto ecdhPsi = weakEcdhPSI.lock(); + if (ecdhPsi) + { + ecdhPsi->psiConfig()->updateAgenyList(_agencyList); + } + auto ra2018Psi = weakRA2018PSI.lock(); + if (ra2018Psi) + { + ra2018Psi->psiConfig()->updateAgenyList(_agencyList); + } + }); +} \ No newline at end of file diff --git a/cpp/libinitializer/Initializer.h b/cpp/libinitializer/Initializer.h new file mode 100644 index 00000000..e5424733 --- /dev/null +++ b/cpp/libinitializer/Initializer.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Initializer.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "FrontInitializer.h" +#include "ProtocolInitializer.h" +#include "ppc-framework/rpc/RpcInterface.h" +#include "ppc-framework/rpc/RpcTypeDef.h" +#include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include + +namespace ppc::psi +{ +class RA2018PSIImpl; +class LabeledPSIImpl; +class CM2020PSIImpl; +class EcdhPSIImpl; +class EcdhMultiPSIImpl; +// class EcdhConnPSIImpl; +} // namespace ppc::psi + +namespace ppc::pir +{ +class OtPIRImpl; +} // namespace ppc::pir +namespace ppc::initializer +{ +class Initializer : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + Initializer(std::string const& _configPath); + virtual ~Initializer() { stop(); } + + // init the service + virtual void init(ppc::protocol::NodeArch _arch); + virtual void stop(); + virtual void start(); + + ppc::tools::PPCConfig::Ptr config() { return m_config; } + FrontInitializer::Ptr const& frontInitializer() const { return m_frontInitializer; } + ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } + ProtocolInitializer::Ptr const& protocolInitializer() const { return m_protocolInitializer; } + ppc::psi::BsEcdhPSIImpl::Ptr const& bsEcdhPsi() const { return m_bsEcdhPSI; } + + void registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc); + + virtual void fetchAgencyList(); + +protected: + virtual void initMsgHandlers(); + void fetchAgencyListPeriodically(); + +private: + std::string m_configPath; + ppc::tools::PPCConfig::Ptr m_config; + ProtocolInitializer::Ptr m_protocolInitializer; + FrontInitializer::Ptr m_frontInitializer; + // timer to fetch all agency information + std::shared_ptr m_agencyInfoFetcher; + + // the ra2018-psi + std::shared_ptr m_ra2018PSI; + // the labeled-psi + std::shared_ptr m_labeledPSI; + // the cm2020-psi + std::shared_ptr m_cm2020PSI; + // the ecdh-psi + std::shared_ptr m_ecdhPSI; + // the ecdh-multi-psi + std::shared_ptr m_ecdhMultiPSI; + // the ecdh-conn-psi + // std::shared_ptr m_ecdhConnPSI; + + std::shared_ptr m_otPIR; + + std::shared_ptr m_bsEcdhPSI; +}; +} // namespace ppc::initializer \ No newline at end of file diff --git a/cpp/libinitializer/ProFrontInitializer.h b/cpp/libinitializer/ProFrontInitializer.h new file mode 100644 index 00000000..8355d968 --- /dev/null +++ b/cpp/libinitializer/ProFrontInitializer.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProFrontInitializer.h + * @author: yujiechen + * @date 2022-11-25 + */ +#pragma once +#include "FrontInitializer.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include + +namespace ppc::initializer +{ +class ProFrontInitializer : public FrontInitializer +{ +public: + using Ptr = std::shared_ptr; + ProFrontInitializer(const std::string& _selfAgencyId, + std::shared_ptr _threadPool, + front::PPCMessageFaceFactory::Ptr _messageFactory) + : FrontInitializer(_selfAgencyId, _threadPool, _messageFactory), + m_statusReporter(std::make_shared(3000, "frontReporter")) + {} + ~ProFrontInitializer() override { stop(); } + + void start() override + { + FrontInitializer::start(); + if (m_statusReporter) + { + m_statusReporter->registerTimeoutHandler( + boost::bind(&ProFrontInitializer::reportStatusToGateway, this)); + m_statusReporter->start(); + } + } + + void stop() override + { + if (m_statusReporter) + { + m_statusReporter->stop(); + } + FrontInitializer::stop(); + } + +protected: + // report the endPoint to the gateway periodically + // Note: must set gatewayInterface into the front before call this function + virtual void reportStatusToGateway() + { + try + { + m_front->gatewayInterface()->registerFront(m_front->selfEndPoint(), nullptr); + } + catch (std::exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("reportStatusToGateway exception") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + m_statusReporter->restart(); + } + +private: + std::shared_ptr m_statusReporter; +}; +} // namespace ppc::initializer diff --git a/cpp/libinitializer/ProtocolInitializer.h b/cpp/libinitializer/ProtocolInitializer.h new file mode 100644 index 00000000..4d9546d4 --- /dev/null +++ b/cpp/libinitializer/ProtocolInitializer.h @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProtocolInitializer.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "Common.h" +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-crypto-core/src/hash/SM3Hash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/protocol/GlobalConfig.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-storage/src/FileStorageFactoryImpl.h" +#include "ppc-storage/src/SQLStorageFactoryImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include + +namespace ppc::initializer +{ +class ProtocolInitializer +{ +public: + using Ptr = std::shared_ptr; + ProtocolInitializer() + : m_ppcMsgFactory(std::make_shared()), + m_sqlStorageFactory(std::make_shared()), + m_fileStorageFactory(std::make_shared()) + {} + + virtual ~ProtocolInitializer() = default; + + virtual void init(ppc::tools::PPCConfig::Ptr _config) + { + m_dataResourceLoader = std::make_shared( + _config->storageConfig().sqlConnectionOpt, + _config->storageConfig().fileStorageConnectionOpt, + _config->storageConfig().remoteConnectionOpt, m_sqlStorageFactory, m_fileStorageFactory, + m_remoteStorageFactory); + + g_PPCConfig.setSMCrypto(_config->smCrypto()); + if (!_config->smCrypto()) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared( + hashImpl, ppc::protocol::ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + m_binHashImpl = std::make_shared(); + } + else + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared( + hashImpl, ppc::protocol::ECCCurve::SM2); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + m_binHashImpl = hashImpl; + } + INIT_LOG(INFO) << LOG_DESC("loadPrivateKey") + << LOG_KV("privateKeyPath", _config->privateKeyPath()); + auto privateKey = loadPrivateKey(_config->privateKeyPath(), PRIVATE_KEY_HEX_LEN); + _config->setPrivateKey(privateKey); + INIT_LOG(INFO) << LOG_DESC("loadPrivateKey success"); + } + + ppc::front::PPCMessageFaceFactory::Ptr const& ppcMsgFactory() const { return m_ppcMsgFactory; } + ppc::crypto::CryptoBox::Ptr const& cryptoBox() const { return m_cryptoBox; }; + ppc::crypto::Hash::Ptr const& binHashImpl() const { return m_binHashImpl; }; + + ppc::storage::SQLStorageFactory::Ptr const& sqlStorageFactory() const + { + return m_sqlStorageFactory; + } + ppc::io::DataResourceLoader::Ptr const& dataResourceLoader() const + { + return m_dataResourceLoader; + } + + ppc::storage::FileStorageFactory::Ptr const& fileStorageFactory() const + { + return m_fileStorageFactory; + } + + ppc::storage::RemoteStorageFactory::Ptr const& remoteStorageFactory() const + { + return m_remoteStorageFactory; + } + +private: + ppc::front::PPCMessageFaceFactory::Ptr m_ppcMsgFactory; + ppc::storage::SQLStorageFactory::Ptr m_sqlStorageFactory; + ppc::storage::FileStorageFactory::Ptr m_fileStorageFactory; + ppc::storage::RemoteStorageFactory::Ptr m_remoteStorageFactory; + ppc::io::DataResourceLoader::Ptr m_dataResourceLoader; + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + ppc::crypto::Hash::Ptr m_binHashImpl; + + constexpr static unsigned PRIVATE_KEY_HEX_LEN = 64; +}; +} // namespace ppc::initializer \ No newline at end of file diff --git a/cpp/ppc-cem/CMakeLists.txt b/cpp/ppc-cem/CMakeLists.txt new file mode 100644 index 00000000..c846ec95 --- /dev/null +++ b/cpp/ppc-cem/CMakeLists.txt @@ -0,0 +1,8 @@ +project(ppc-cem VERSION ${VERSION}) +add_subdirectory(src) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() diff --git a/cpp/ppc-cem/src/CEMService.cpp b/cpp/ppc-cem/src/CEMService.cpp new file mode 100644 index 00000000..696ed196 --- /dev/null +++ b/cpp/ppc-cem/src/CEMService.cpp @@ -0,0 +1,371 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CEMService.cpp + * @author: caryliao + * @date 2022-11-04 + */ + +#include "CEMService.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include "ppc-storage/src/FileStorageFactoryImpl.h" +#include "wedpr_ffi_c_equality.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ppc; +using namespace ppc::cem; +using namespace bcos; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::tools; +using namespace ppc::storage; + +void CEMService::doCipherTextEqualityMatch(const Json::Value::Members& fieldNames, + const std::vector& fieldValues, LineReader::Ptr lineReader, + Json::Value& matchCount) +{ + auto fieldNameLine = (lineReader->next(1))->get(0); + std::vector fileFieldNames; + // obtain the fieldName + boost::split(fileFieldNames, fieldNameLine, boost::is_any_of(",")); + std::vector matchFieldColumnIndexs; + // obtain the field that need to be matched, put it into matchFieldColumnIndexs + for (size_t i = 0; i < fileFieldNames.size(); i++) + { + for (auto fieldName : fieldNames) + { + auto fileFieldName = fileFieldNames.at(i); + boost::algorithm::trim(fieldName); + boost::algorithm::trim(fileFieldName); + if (fieldName == fileFieldName) + { + matchFieldColumnIndexs.emplace_back(i); + } + } + } + uint64_t lineSize = 0; + for (auto fieldName : fieldNames) + { + matchCount[fieldName] = 0; + } + // record the match-result + std::vector> matchCountResult(matchFieldColumnIndexs.size()); + int64_t readPerBatchLines = m_cemConfig.readPerBatchLines; + while (true) + { + // batch read dataset line + auto dataBatch = lineReader->next(readPerBatchLines); + if (!dataBatch) + { + break; + } + lineSize += dataBatch->size(); + // paraller for handle ciphtertext match in dataset + tbb::parallel_for( + tbb::blocked_range(0U, dataBatch->size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto fileLine = dataBatch->get(i); + // erase windows \r character + fileLine.erase( + std::remove(fileLine.begin(), fileLine.end(), '\r'), fileLine.end()); + std::vector fileFieldValues; + boost::split(fileFieldValues, fileLine, boost::is_any_of(",")); + tbb::parallel_for(tbb::blocked_range(0U, matchFieldColumnIndexs.size()), + [&](auto const& range) { + for (auto j = range.begin(); j < range.end(); j++) + { + // auto startT = utcSteadyTime(); + auto decodedRequestValue = fromHexString(fieldValues.at(j)); + auto decodedFileValue = + fromHexString(fileFieldValues.at(matchFieldColumnIndexs.at(j))); + CInputBuffer inputBuffer1{ + (const char*)decodedRequestValue->data(), CIPHERTEXT_LEN}; + CInputBuffer inputBuffer2{ + (const char*)decodedFileValue->data(), CIPHERTEXT_LEN}; + auto matchResult = wedpr_pairing_bls128_equality_test( + &inputBuffer1, &inputBuffer2); + // CEM_LOG(INFO) << LOG_DESC("match once ok") + // << LOG_KV("timecost(ms)", utcSteadyTime() - startT); + if (matchResult == WEDPR_SUCCESS) + { + (matchCountResult[j]).fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + }); + } + // assign the result + for (uint64_t i = 0; i < matchFieldColumnIndexs.size(); i++) + { + auto fieldName = fileFieldNames.at(matchFieldColumnIndexs.at(i)); + boost::algorithm::trim(fieldName); + matchCount[fieldName] = matchCountResult[i].load(); + } + CEM_LOG(INFO) << LOG_DESC("match dataset file ok") << LOG_KV("file lines", lineSize); +} + +void CEMService::makeCiphertextEqualityMatch( + Json::Value const& request, Json::Value& response, DataResourceType _type) +{ + auto startT = utcSteadyTime(); + tbb::parallel_for(tbb::blocked_range(0U, request.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto datasetId = request[(Json::ArrayIndex)i]["dataset_id"].asString(); + auto matchField = request[(Json::ArrayIndex)i]["match_field"]; + Json::Value result; + result["dataset_id"] = datasetId; + auto fieldNames = matchField.getMemberNames(); + std::vector fieldValues; + for (auto fieldName : fieldNames) + { + const std::string fieldValue = matchField[fieldName].asString(); + fieldValues.emplace_back(fieldValue); + } + Json::Value matchCount; + auto linereader = initialize_lineReader(datasetId, _type); + doCipherTextEqualityMatch(fieldNames, fieldValues, linereader, matchCount); + result["match_count"] = matchCount; + response[(Json::ArrayIndex)i] = result; + } + }); + CEM_LOG(INFO) << LOG_DESC("ciphertext equality match request handle ok") + << LOG_KV("timecost(ms)", utcSteadyTime() - startT); +} + +void CEMService::makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func) +{ + Json::Value response(Json::arrayValue); + response.resize((Json::ArrayIndex)request.size()); + makeCiphertextEqualityMatch(request, response, DataResourceType::HDFS); + func(nullptr, std::move(response)); +} + +void CEMService::doEncryptDataset(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter) +{ + try + { + auto fieldNameLine = (lineReader->next(1))->get(0); + auto dataBatchFieldName = std::make_shared(); + // erase windows \r character + fieldNameLine.erase( + std::remove(fieldNameLine.begin(), fieldNameLine.end(), '\r'), fieldNameLine.end()); + dataBatchFieldName->append(fieldNameLine); + lineWriter->writeLine(dataBatchFieldName, DataSchema::String, "\n"); + uint64_t lineSize = 0; + int64_t readPerBatchLines = m_cemConfig.readPerBatchLines; + while (true) + { + // batch read dataset line + auto dataBatch = lineReader->next(readPerBatchLines); + if (!dataBatch) + { + break; + } + lineSize += dataBatch->size(); + auto ciphertextFileLines = std::make_shared(); + ciphertextFileLines->resize(dataBatch->size()); + // parallel for handle encrypt dataset + tbb::parallel_for(tbb::blocked_range(0U, dataBatch->size()), [&](auto const& + range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto fileLine = dataBatch->get(i); + // erase windows \r character + fileLine.erase( + std::remove(fileLine.begin(), fileLine.end(), '\r'), fileLine.end()); + std::vector fileFieldValues; + boost::split(fileFieldValues, fileLine, boost::is_any_of(",")); + std::ostringstream ciphertextFileLine; + for (uint64_t i = 0; i < fileFieldValues.size(); i++) + { + // id in index 0, need not be encrypted + auto fieldValue = fileFieldValues.at(i); + boost::algorithm::trim(fieldValue); + if (i == 0) + { + ciphertextFileLine << fieldValue; + } + else + { + // auto startT = utcSteadyTime(); + const char* plaintext = fieldValue.data(); + CInputBuffer inputBuffer{plaintext, fieldValue.size()}; + auto ciphertext = std::make_shared(); + ciphertext->resize(CIPHERTEXT_LEN); + COutputBuffer outputBuffer{(char*)ciphertext->data(), CIPHERTEXT_LEN}; + auto encryptResult = + wedpr_pairing_bls128_encrypt_message(&inputBuffer, &outputBuffer); + // << LOG_KV("timecost(ms)", utcSteadyTime() - startT); + if (encryptResult == WEDPR_SUCCESS) + { + auto ciphertextHexStr = *toHexString(*ciphertext); + ciphertextFileLine << "," << ciphertextHexStr; + } + else + { + BOOST_THROW_EXCEPTION( + OpenDatasetFileFailException() + << errinfo_comment("encrypt dataset file fail")); + } + } + } + ciphertextFileLines->set(i, ciphertextFileLine.str()); + } + }); + lineWriter->writeLine(ciphertextFileLines, DataSchema::String, "\n"); + } + lineWriter->close(); + CEM_LOG(INFO) << LOG_DESC("encrypt dataset file ok") << LOG_KV("file lines", lineSize); + } + catch (const char*& e) + { + CEM_LOG(INFO) << LOG_KV("doEncryptDataset", e); + } +} + +void CEMService::encryptDataset(Json::Value const& request, Json::Value& response) +{ + // parallel for handle dataset + auto startT = utcSteadyTime(); + tbb::parallel_for(tbb::blocked_range(0U, request.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + const std::string datasetId = request[(Json::ArrayIndex)i].asString(); + auto linereader = initialize_lineReader(datasetId, DataResourceType::HDFS); + auto linewriter = initialize_lineWriter(datasetId, DataResourceType::HDFS); + doEncryptDataset(linereader, linewriter); + renameSource(datasetId, DataResourceType::HDFS); + response[(Json::ArrayIndex)i] = datasetId + m_cemConfig.ciphertextSuffix; + } + }); + CEM_LOG(INFO) << LOG_DESC("encrypt dataset request handle ok") + << LOG_KV("timecost(ms)", utcSteadyTime() - startT); +} + +void CEMService::encryptDatasetRpc(Json::Value const& request, RespFunc func) +{ + Json::Value response; + encryptDataset(request, response); + func(nullptr, std::move(response)); +} + +LineReader::Ptr CEMService::initialize_lineReader( + const std::string& _datasetId, DataResourceType _type) +{ + std::string file_to = m_cemConfig.datasetFilePath + "/" + _datasetId; + auto factory = std::make_shared(); + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, factory, nullptr); + const auto dataResource = std::make_shared(); + dataResource->setResourceID(_datasetId); + const auto dataResourceDesc = std::make_shared(); + dataResourceDesc->setType((int)_type); + if ((int)_type == (int)DataResourceType::HDFS) + { + file_to = m_cemConfig.datasetHDFSPath + "/" + _datasetId; + dataResourceDesc->setFileStorageConnectOption(m_storageConfig.fileStorageConnectionOpt); + } + else if ((int)_type == (int)DataResourceType::FILE) + { + file_to = m_cemConfig.datasetFilePath + "/" + _datasetId; + } + dataResourceDesc->setPath(file_to); + dataResource->setDesc(dataResourceDesc); + return dataResourceLoader->loadReader(dataResource->desc()); +} + +LineWriter::Ptr CEMService::initialize_lineWriter( + const std::string& _datasetId, DataResourceType _type) +{ + std::string file_to = + m_cemConfig.datasetFilePath + "/" + _datasetId + m_cemConfig.ciphertextSuffix; + auto factory = std::make_shared(); + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, factory, nullptr); + const auto dataResource = std::make_shared(); + dataResource->setResourceID(_datasetId); + const auto dataResourceDesc = std::make_shared(); + dataResourceDesc->setType((int)_type); + if ((int)_type == (int)DataResourceType::HDFS) + { + file_to = m_cemConfig.datasetHDFSPath + "/" + _datasetId + m_cemConfig.ciphertextSuffix; + dataResourceDesc->setFileStorageConnectOption(m_storageConfig.fileStorageConnectionOpt); + } + else if ((int)_type == (int)DataResourceType::FILE) + { + file_to = m_cemConfig.datasetFilePath + "/" + _datasetId + m_cemConfig.ciphertextSuffix; + } + dataResourceDesc->setPath(file_to); + dataResource->setOutputDesc(dataResourceDesc); + return dataResourceLoader->loadWriter(dataResource->outputDesc()); +} + +void CEMService::renameSource(const std::string& _datasetId, DataResourceType _type) +{ + if ((int)_type == (int)DataResourceType::HDFS) + { + std::string from_file = + m_cemConfig.datasetHDFSPath + "/" + _datasetId + m_cemConfig.ciphertextSuffix; + std::string to_file = m_cemConfig.datasetHDFSPath + "/" + _datasetId; + auto factory = std::make_shared(); + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, factory, nullptr); + const auto dataResource = std::make_shared(); + dataResource->setResourceID(_datasetId); + const auto dataResourceDesc = std::make_shared(); + dataResourceDesc->setType((int)_type); + dataResourceDesc->setFileStorageConnectOption(m_storageConfig.fileStorageConnectionOpt); + dataResourceDesc->setPath(from_file); + dataResource->setOutputDesc(dataResourceDesc); + dataResourceLoader->renameResource(dataResource->outputDesc(), to_file); + } + else if ((int)_type == (int)DataResourceType::FILE) + { + std::string from_file = + m_cemConfig.datasetFilePath + "/" + _datasetId + m_cemConfig.ciphertextSuffix; + std::string to_file = m_cemConfig.datasetFilePath + "/" + _datasetId; + if (boost::filesystem::exists(from_file)) + { + boost::filesystem::remove(from_file); + } + boost::filesystem::rename(from_file, to_file); + } +} + + +void CEMService::setCEMConfig(CEMConfig const& cemConfig) +{ + m_cemConfig = cemConfig; +} + +void CEMService::setStorageConfig(StorageConfig const& storageConfig) +{ + m_storageConfig = storageConfig; +} \ No newline at end of file diff --git a/cpp/ppc-cem/src/CEMService.h b/cpp/ppc-cem/src/CEMService.h new file mode 100644 index 00000000..394a4c4c --- /dev/null +++ b/cpp/ppc-cem/src/CEMService.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CEMService.h + * @author: caryliao + * @date 2022-11-04 + */ +#pragma once +#include "Common.h" +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::io; +using namespace ppc::rpc; +using namespace ppc::tools; + +namespace ppc::cem +{ +class CEMService +{ +public: + using Ptr = std::shared_ptr; + CEMService() = default; + virtual ~CEMService() = default; + + void makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func); + void makeCiphertextEqualityMatch(Json::Value const& request, Json::Value& response, ppc::protocol::DataResourceType _type); + void encryptDatasetRpc(Json::Value const& request, RespFunc func); + void encryptDataset(Json::Value const& request, Json::Value& response); + + void setCEMConfig(CEMConfig const& cemConfig); + void setStorageConfig(StorageConfig const& storageConfig); + void doCipherTextEqualityMatch(const Json::Value::Members& fieldNames, + const std::vector& fieldValues, LineReader::Ptr lineReader, + Json::Value& matchCount); + void doEncryptDataset(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter); + LineReader::Ptr initialize_lineReader(const std::string& _datasetId, ppc::protocol::DataResourceType _type); + LineWriter::Ptr initialize_lineWriter(const std::string& _datasetId, ppc::protocol::DataResourceType _type); + void renameSource(const std::string& _datasetId, ppc::protocol::DataResourceType _type); + +private: + CEMConfig m_cemConfig; + StorageConfig m_storageConfig; +}; +} // namespace ppc::cem \ No newline at end of file diff --git a/cpp/ppc-cem/src/CMakeLists.txt b/cpp/ppc-cem/src/CMakeLists.txt new file mode 100644 index 00000000..ccb4451b --- /dev/null +++ b/cpp/ppc-cem/src/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +set(FFI_CEM_LIBRARY ${CMAKE_SOURCE_DIR}/deps/lib/libffi_c_equality.a) + +add_library(${CEM_TARGET} ${SOURCES}) +target_link_libraries(${CEM_TARGET} PUBLIC ${FFI_CEM_LIBRARY} ${IO_TARGET} TBB::tbb) diff --git a/cpp/ppc-cem/src/Common.h b/cpp/ppc-cem/src/Common.h new file mode 100644 index 00000000..c1f9bc59 --- /dev/null +++ b/cpp/ppc-cem/src/Common.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: caryliao + * @date 2022-11-03 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +#define CEM_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("CM") + +namespace ppc::cem +{ +const size_t CIPHERTEXT_LEN = 144; +DERIVE_PPC_EXCEPTION(OpenDatasetFileFailException); +DERIVE_PPC_EXCEPTION(CipherMatchFailException); +} // namespace ppc::cem \ No newline at end of file diff --git a/cpp/ppc-cem/src/WedprUtilities.h b/cpp/ppc-cem/src/WedprUtilities.h new file mode 100644 index 00000000..856d4769 --- /dev/null +++ b/cpp/ppc-cem/src/WedprUtilities.h @@ -0,0 +1,28 @@ +#ifndef _WEDPR_UTILITIES_H_ +#define _WEDPR_UTILITIES_H_ + +#include +#include +#include +#include +#include + +extern "C" { + +struct CInputBuffer +{ + const char* data; + uintptr_t len; +}; + +struct COutputBuffer +{ + char* data; + uintptr_t len; +}; + +const int8_t WEDPR_ERROR = -1; +const int8_t WEDPR_SUCCESS = 0; +} // extern "C" + +#endif diff --git a/cpp/ppc-cem/src/wedpr_ffi_c_equality.h b/cpp/ppc-cem/src/wedpr_ffi_c_equality.h new file mode 100644 index 00000000..eae1e03c --- /dev/null +++ b/cpp/ppc-cem/src/wedpr_ffi_c_equality.h @@ -0,0 +1,25 @@ +#ifndef _WEDPR_CRYPTO_H_ +#define _WEDPR_CRYPTO_H_ + +#include +#include +#include +#include + +#include "WedprUtilities.h" + +extern "C" { +/** + * C interface for 'encrypt_message'. + */ +int8_t wedpr_pairing_bls128_encrypt_message(const CInputBuffer *raw_plaintext, + COutputBuffer *output_ciphertext); + +/** + * C interface for 'equality_test'. + */ +int8_t wedpr_pairing_bls128_equality_test(const CInputBuffer *raw_cipher1, + const CInputBuffer *raw_cipher2); +} + +#endif diff --git a/cpp/ppc-cem/tests/CMakeLists.txt b/cpp/ppc-cem/tests/CMakeLists.txt new file mode 100644 index 00000000..ae572c6c --- /dev/null +++ b/cpp/ppc-cem/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-cem) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +# target_link_libraries(${TEST_BINARY_NAME} ${CEM_TARGET} ${BCOS_UTILITIES_TARGET} jsoncpp_static ${BOOST_UNIT_TEST}) +target_link_libraries(${TEST_BINARY_NAME} ${CEM_TARGET} ${RPC_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-cem WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) diff --git a/cpp/ppc-cem/tests/TestCEMService.cpp b/cpp/ppc-cem/tests/TestCEMService.cpp new file mode 100644 index 00000000..cbcfc668 --- /dev/null +++ b/cpp/ppc-cem/tests/TestCEMService.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestCEMService.cpp + * @author: caryliao + * @date 2022-11-19 + */ +#include "ppc-cem/src/CEMService.h" +#include "ppc-cem/src/Common.h" +#include +#include + +using namespace ppc::cem; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(cMServiceTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(testCEMService) +{ + auto cemService = std::make_shared(); + std::string configPath{"../../../ppc-cem/tests/data/config.ini"}; + boost::property_tree::ptree pt; + boost::property_tree::read_ini(configPath, pt); + // load the cm config + auto ppcConfig = std::make_shared(); + ppcConfig->loadCEMConfig(pt); + auto cemConfig = ppcConfig->cemConfig(); + auto storageConfig = ppcConfig->storageConfig(); + cemService->setCEMConfig(cemConfig); + cemService->setStorageConfig(storageConfig); + + // d1 encrypted + std::string all_datasets[3] = {"d1", "d2", "d3"}; + for (int i = 0; i < 3; i++) + { + std::string datasetId = all_datasets[i]; + std::string encrypted_file = + cemConfig.datasetFilePath + "/" + datasetId + cemConfig.ciphertextSuffix; + if (boost::filesystem::exists(encrypted_file)) + { + boost::filesystem::remove(encrypted_file); + } + auto linereader = + cemService->initialize_lineReader(datasetId, ppc::protocol::DataResourceType::FILE); + auto linewriter = + cemService->initialize_lineWriter(datasetId, ppc::protocol::DataResourceType::FILE); + cemService->doEncryptDataset(linereader, linewriter); + } + + // d1 matched + Json::Value request2(Json::arrayValue); + + Json::Value param1; + param1["dataset_id"] = "d1-encrypted"; + Json::Value matchField1; + matchField1["x1"] = + "b53d3973005bd84da8367d4d247946b00ed269454d8799608631262862b3ee2a912fa42e1e71af77236ad851ea" + "e44f0c91a674f59469a47c0d8cac7385009d22f012f44f3477ad254a91f3ef07b7a2fc2c78012b2538a9052ce0" + "53dd884ffe2914f7027ae7d545b050426f12f8f225a058a4c609773d0ae7cc73e103954790b7d52a398a88e42e" + "dd8a006cf1325828b9"; + matchField1["x2"] = + "8357af78f88dfbe857bc4eb6801b83714d3f3127bdd4e68bbc17444fe8783686dd36ab44f9b80fb6c173ef0fb1" + "8c22e093216884966cd61c3723c987d91471c76a449d18070358862ebdc50c5d326035c1c188b5642baba5f3f6" + "37a7b5aec5fc054fe607cd0fad22e75468c47dd30a74af05f42a84ba73adf7f277fe94a8f4cd42b23563bbbe42" + "a1e493f44a65f1c52c"; + param1["match_field"] = matchField1; + + Json::Value param2; + param2["dataset_id"] = "d2-encrypted"; + Json::Value matchField2; + matchField2["x1"] = + "971a80d65f1983e14a5bd33347237adadd8e8586ac766d1e06e68bde9957032e009cab0c3501cb3d620aefab4c" + "063fc9b43d2bedd7b0ddc9c5b24d53fa2e35cd41b689350e417df8cb24ce9e585501c04c9535bd2525d02f5397" + "c60ed353358916b585666d0f124df64269151c1f218df4becc79b237124ddda84b64bfa0230b5b012a9a87d28c" + "629d635b0ae84467ab"; + matchField2["x2"] = + "a8374c5e1ca1ef7944ac904491b94d0dafc3c8bb81feaf4f7bff24d9083f8cb459a3181aa454556558ba80ffb9" + "7d91cdaaa17e4e27a75d3835ab69963429aee031b54aa5ecdadbea45566452da568b51aed0b188d1d0ac86ed5e" + "5f665239094e156c1eec61d1d3aa66d98eb004ef3cb870a79cb1fe9ac7c698dcf360d85bfd066b9fd1c9642dd9" + "6960c9bd41027e368b"; + param2["match_field"] = matchField2; + + Json::Value param3; + param3["dataset_id"] = "d3-encrypted"; + Json::Value matchField3; + matchField3["x1"] = + "acf497223d82cf3347d19f00ba4e5e1139c1897e05667186d9c766fa482a4aacaf66b6aecd335bc437e637f8d9" + "b4327f8ed030aec4e047d32cde0b00421a3f31c1857dac3bdfb99700fea524fe3873bfb66809a7ea48f3df251e" + "41dbe27685f10fd525dd18b4e3eabb5e89bba75593439cc73ff19b5afd9f45a2a89407353b0d2b5af875f270dd" + "3d865776216ce194b2"; + matchField3["x2"] = + "a5e604a12f2f06848b626e16cd13d8f74cf866c3963074715b8251ff77fe1c797e40c12785ac872432d814e8df" + "7cd6ad8383c2605111479ddb47941ae27e5e3eb29d1bfd2f2f92b6a7f3c312ddefacf12fc016776d96fbd9998d" + "99399359b11d10641c1d724af6a42659ce0f254a2258af388e8a89c7e7b420a5daa9efb304cdae0ec1c328c6f8" + "5ff3c9fdcbbe29cce8"; + param3["match_field"] = matchField3; + + request2.append(param1); + request2.append(param2); + request2.append(param3); + + Json::Value response2(Json::arrayValue); + response2.resize((Json::ArrayIndex)request2.size()); + cemService->makeCiphertextEqualityMatch( + request2, response2, ppc::protocol::DataResourceType::FILE); + BOOST_CHECK(response2.size() == 3); + uint64_t matchResult = 50; + BOOST_CHECK(response2[(Json::ArrayIndex)0]["dataset_id"].asString() == "d1-encrypted"); + BOOST_CHECK(response2[(Json::ArrayIndex)0]["match_count"]["x1"].asUInt64() == matchResult); + BOOST_CHECK(response2[(Json::ArrayIndex)0]["match_count"]["x2"].asUInt64() == matchResult); + + BOOST_CHECK(response2[(Json::ArrayIndex)1]["dataset_id"].asString() == "d2-encrypted"); + BOOST_CHECK(response2[(Json::ArrayIndex)1]["match_count"]["x1"].asUInt64() == matchResult); + BOOST_CHECK(response2[(Json::ArrayIndex)1]["match_count"]["x2"].asUInt64() == matchResult); + + BOOST_CHECK(response2[(Json::ArrayIndex)2]["dataset_id"].asString() == "d3-encrypted"); + BOOST_CHECK(response2[(Json::ArrayIndex)2]["match_count"]["x1"].asUInt64() == matchResult); + BOOST_CHECK(response2[(Json::ArrayIndex)2]["match_count"]["x2"].asUInt64() == matchResult); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-cem/tests/data/config.ini b/cpp/ppc-cem/tests/data/config.ini new file mode 100644 index 00000000..c5a93e32 --- /dev/null +++ b/cpp/ppc-cem/tests/data/config.ini @@ -0,0 +1,70 @@ +[agency] + ; the agency-id of self-party + id = agency + ; the agency info + +[cem] + use_mysql = false + dataset_hdfs_path = /user/ppc/admin + dataset_file_path = ../../../ppc-cem/tests/data + ciphertext_suffix = -encrypted + read_per_batch_lines = 100000 + +[crypto] + sm_crypto = false + +[rpc] + listen_ip=0.0.0.0 + listen_port=5002 + thread_count=4 + ; ssl or sm ssl + sm_ssl=false + ; ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + +; [storage] +; host = 127.0.0.1 +; ; the mysqlx_port +; port = 3306 +; user = root +; password = 12345678 +; database = ppc15 + +[hdfs_storage] + ; the hdfs configuration + user = ppc + name_node = 127.0.0.1 + name_node_port = 9900 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 2000 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[log] + enable=true + ; print the log to std::cout or not, default print to the log files + enable_console_output = false + log_path=./log + ; info debug trace + level=info + ; MB + max_log_file_size=200 + ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message + #format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + #enable_rotate_by_hour=false + #log_name_pattern=ppcs-psi4ef.log + ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log + #rotate_name_pattern=log_%Y%m%d.%H%M.log + ; if archive_path is empty, the archive function will be disabled + ; archive_path=./log/ + #compress_archive_file=true + ; ; 0: no limit, in MB + ; max_archive_files=10 + ; ; 0: no limit, in MB + ; max_archive_size=0 + ; min_free_space=0 diff --git a/cpp/ppc-cem/tests/data/d1 b/cpp/ppc-cem/tests/data/d1 new file mode 100644 index 00000000..ff848899 --- /dev/null +++ b/cpp/ppc-cem/tests/data/d1 @@ -0,0 +1,101 @@ +id,x1,x2 +1,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +2,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +3,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +4,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +5,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +6,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +7,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +8,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +9,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +10,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +11,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +12,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +13,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +14,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +15,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +16,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +17,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +18,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +19,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +20,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +21,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +22,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +23,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +24,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +25,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +26,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +27,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +28,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +29,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +30,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +31,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +32,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +33,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +34,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +35,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +36,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +37,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +38,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +39,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +40,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +41,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +42,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +43,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +44,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +45,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +46,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +47,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +48,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +49,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +50,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +51,c127b40c158f404aa432656b013bad12,424637e2531f4ddfad8f43d8d3d45f3e +52,f35367150f904fa3ab3442e77355e43a,600d3627245341d580fe078b19450d6d +53,29627c5f37e94779a8ec95322e5afbe1,63533be072f8427ea9ba6d0369163862 +54,a41df047bf2f4e919033441f61d1b7fd,e2382ff5ba01431d9f48e085419ec4ef +55,65db7c7b020d4b0d9229a2ad0ea4c7c5,e775c8b1f7154d119ec8ddbea66cd4bc +56,e749e17a71a2447fac63bca47b6846a7,0045251d24b9466695d5d0c38929ea90 +57,14c697e5bce841878ebf89bf08a9ba14,04317b907cdc4043b8200db5110831c1 +58,38047064e02a4ca4af076ae26aa679f7,9bb62566a45c4d229a4fd795f8777a75 +59,5fb6f18de87043a497f0677537c189bc,2724e3acf012450e8355d84b8b3ece86 +60,b44e260738b343629faaa7d9f549bbcb,94b80d51709d4c5faaee2d78a762bbc4 +61,808af13908c14cd89092219dc37b29cf,1085ef4476aa4635b7d3e78ea67f83c3 +62,af18d05a4e2548468d61ccad12a0340e,2024c88fcc03414a81b64da8409dc8af +63,ba426827999c462b9df669d7e932144c,2b2d0515df814e4f94c1110d3b584ba4 +64,525584d2c08f439eb7bee2db6b91ac37,8188e4aee9354bb2b9fc8536f34b1c24 +65,f1d40891ad90463cb530bcd4fce3b733,1c6bb273297846dbbb920e5409c6e5ff +66,f9f22de01edd4644b6c55ef7a993b184,55adc457cd124274a27ebb830ad3dc66 +67,e4ff7eaa9dd04f149049aa12bf21e68a,0c3fb3b958b14243a3edcd1bc9a51283 +68,71d984a3e0ad42698f3186cfbfdcbd87,0a520b68a33c499b9809ae269a55c394 +69,96d145bc82ff4e2a84551c20ef6f1cc5,affad8d9f1284a5dbdf97ca825288654 +70,fcece0ef256c47fdaa5678e20df0f4a1,623b626acc0b494d90f64879575c51e3 +71,87800bc6aeb5461b869afaba3d7430b3,0825eac882c0486b8316723e722e29cf +72,dd943a94e743432592c68e38352e83c8,3aeb080d0f234cc198b3165baab8d8f5 +73,8f7f6922e81246a680f931c44ecc0d1a,42f4992a390a440a9aa1b440a02e78f5 +74,42d3d9010a7548faa386e83ae61f4802,f4d3dbad28004d759be6229bc2bbff9a +75,01dc4d7631f94ba987e435c87d632684,44bde70d206b4450bde0991d405ab0fa +76,ccea7c284fce4d3988d8948f79b2fef6,04fecfe64b874cc8b2cad3e2bc205317 +77,272d7020324f4fa9a36a31b64ef798fb,c803c1ef9eab459387bb0be0fe6e1e93 +78,c92ee2e67c22477482eb6a9b3492283a,07cf6bbaea12454ba9f8693486df6282 +79,afba2c31d5fb4e658fa16b269bd48179,c55973e7e2f04e96976e99218bf2a8d5 +80,7066d9b40a994cce8b5eb555a3a265c8,6bb406d7c2de40ff828bc6e2a4e97ee9 +81,ce88a3d29e574780a649399794d793bf,b5ac277d1e7242089a30a48d78033ac2 +82,2514054bd2fd42f79ecdbb4b4fac3301,af552fac31434e3b94bdb2dd7087c6e7 +83,1466f1d054284e78b1844dd343bdc095,970e1fd372ee4dacbc5d911a0f556efe +84,21d7c25b139845ea84b8a04dc45a1fc9,d28447da4e5843adadb9845fab94a278 +85,3e451bbe00b34590b0d627d0d17d4aab,8aa5643bab1342ffa4229bef214807d1 +86,d2e635d343a440478e8ae39d5f07c87d,adf9ab5d926c4d48ac33fa301f2e5cd4 +87,2ec3545d3ce84b4290481b4b3d5c9fd2,6c0a1f2b9d124cd1a2e27a419ecfce58 +88,42d37153df1c4e7f9eacf0628f07d22a,695082c3bd6a48a69e74c7435a642b1b +89,403c2018e9b64e7caba38b12ecdb8358,470671bd2e7640c8851426c9ebe4c924 +90,ccb7cefb3efd4a49a6e5e400caf239da,2d73f2af749d4c04990f6464d8d1350d +91,a39164ae87d84d8a8faecdfecfa537f3,1d82ac0af6e1484dbc16b1fe2fde368f +92,908045cc3301407ca0c8818e01eebf5f,92dc6cd19357410e96b60d96238e4f03 +93,ac388e933ba6482fbb5e3505d1a1e18a,4711abe2a1774467901f76dcd4d590b8 +94,5978cdb261304e13ab299f7c4a8bf3ff,d9ea4748b79842888038c71e72061abd +95,7808218f4ea74e57acbb0a913b967d29,d7fc8e21ae6f4d6c8dc0ffdf4e376aab +96,c0cd4c990efc43c18e46c246a1e3382e,8e1a87e86a5d48e593935786aa0cbc38 +97,5c0aaf7f191c4720a343f6d8b8c4704c,2ad4bd183efa49e984302df6cba20956 +98,15d996bfe59a4cc8ab4899e6cce426e2,fd4d20fb799c41a9afdf64b206e08a4e +99,d4a93966765b4a42b1697c8d14881211,36727725990f49a6bd4d92b1ec6424d2 +100,ebf75e2646914039803a59fd2b2e8fd9,e0ae4ddcf5d9424cb949b03882606219 diff --git a/cpp/ppc-cem/tests/data/d2 b/cpp/ppc-cem/tests/data/d2 new file mode 100644 index 00000000..ff848899 --- /dev/null +++ b/cpp/ppc-cem/tests/data/d2 @@ -0,0 +1,101 @@ +id,x1,x2 +1,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +2,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +3,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +4,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +5,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +6,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +7,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +8,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +9,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +10,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +11,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +12,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +13,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +14,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +15,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +16,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +17,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +18,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +19,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +20,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +21,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +22,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +23,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +24,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +25,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +26,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +27,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +28,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +29,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +30,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +31,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +32,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +33,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +34,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +35,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +36,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +37,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +38,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +39,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +40,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +41,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +42,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +43,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +44,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +45,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +46,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +47,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +48,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +49,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +50,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +51,c127b40c158f404aa432656b013bad12,424637e2531f4ddfad8f43d8d3d45f3e +52,f35367150f904fa3ab3442e77355e43a,600d3627245341d580fe078b19450d6d +53,29627c5f37e94779a8ec95322e5afbe1,63533be072f8427ea9ba6d0369163862 +54,a41df047bf2f4e919033441f61d1b7fd,e2382ff5ba01431d9f48e085419ec4ef +55,65db7c7b020d4b0d9229a2ad0ea4c7c5,e775c8b1f7154d119ec8ddbea66cd4bc +56,e749e17a71a2447fac63bca47b6846a7,0045251d24b9466695d5d0c38929ea90 +57,14c697e5bce841878ebf89bf08a9ba14,04317b907cdc4043b8200db5110831c1 +58,38047064e02a4ca4af076ae26aa679f7,9bb62566a45c4d229a4fd795f8777a75 +59,5fb6f18de87043a497f0677537c189bc,2724e3acf012450e8355d84b8b3ece86 +60,b44e260738b343629faaa7d9f549bbcb,94b80d51709d4c5faaee2d78a762bbc4 +61,808af13908c14cd89092219dc37b29cf,1085ef4476aa4635b7d3e78ea67f83c3 +62,af18d05a4e2548468d61ccad12a0340e,2024c88fcc03414a81b64da8409dc8af +63,ba426827999c462b9df669d7e932144c,2b2d0515df814e4f94c1110d3b584ba4 +64,525584d2c08f439eb7bee2db6b91ac37,8188e4aee9354bb2b9fc8536f34b1c24 +65,f1d40891ad90463cb530bcd4fce3b733,1c6bb273297846dbbb920e5409c6e5ff +66,f9f22de01edd4644b6c55ef7a993b184,55adc457cd124274a27ebb830ad3dc66 +67,e4ff7eaa9dd04f149049aa12bf21e68a,0c3fb3b958b14243a3edcd1bc9a51283 +68,71d984a3e0ad42698f3186cfbfdcbd87,0a520b68a33c499b9809ae269a55c394 +69,96d145bc82ff4e2a84551c20ef6f1cc5,affad8d9f1284a5dbdf97ca825288654 +70,fcece0ef256c47fdaa5678e20df0f4a1,623b626acc0b494d90f64879575c51e3 +71,87800bc6aeb5461b869afaba3d7430b3,0825eac882c0486b8316723e722e29cf +72,dd943a94e743432592c68e38352e83c8,3aeb080d0f234cc198b3165baab8d8f5 +73,8f7f6922e81246a680f931c44ecc0d1a,42f4992a390a440a9aa1b440a02e78f5 +74,42d3d9010a7548faa386e83ae61f4802,f4d3dbad28004d759be6229bc2bbff9a +75,01dc4d7631f94ba987e435c87d632684,44bde70d206b4450bde0991d405ab0fa +76,ccea7c284fce4d3988d8948f79b2fef6,04fecfe64b874cc8b2cad3e2bc205317 +77,272d7020324f4fa9a36a31b64ef798fb,c803c1ef9eab459387bb0be0fe6e1e93 +78,c92ee2e67c22477482eb6a9b3492283a,07cf6bbaea12454ba9f8693486df6282 +79,afba2c31d5fb4e658fa16b269bd48179,c55973e7e2f04e96976e99218bf2a8d5 +80,7066d9b40a994cce8b5eb555a3a265c8,6bb406d7c2de40ff828bc6e2a4e97ee9 +81,ce88a3d29e574780a649399794d793bf,b5ac277d1e7242089a30a48d78033ac2 +82,2514054bd2fd42f79ecdbb4b4fac3301,af552fac31434e3b94bdb2dd7087c6e7 +83,1466f1d054284e78b1844dd343bdc095,970e1fd372ee4dacbc5d911a0f556efe +84,21d7c25b139845ea84b8a04dc45a1fc9,d28447da4e5843adadb9845fab94a278 +85,3e451bbe00b34590b0d627d0d17d4aab,8aa5643bab1342ffa4229bef214807d1 +86,d2e635d343a440478e8ae39d5f07c87d,adf9ab5d926c4d48ac33fa301f2e5cd4 +87,2ec3545d3ce84b4290481b4b3d5c9fd2,6c0a1f2b9d124cd1a2e27a419ecfce58 +88,42d37153df1c4e7f9eacf0628f07d22a,695082c3bd6a48a69e74c7435a642b1b +89,403c2018e9b64e7caba38b12ecdb8358,470671bd2e7640c8851426c9ebe4c924 +90,ccb7cefb3efd4a49a6e5e400caf239da,2d73f2af749d4c04990f6464d8d1350d +91,a39164ae87d84d8a8faecdfecfa537f3,1d82ac0af6e1484dbc16b1fe2fde368f +92,908045cc3301407ca0c8818e01eebf5f,92dc6cd19357410e96b60d96238e4f03 +93,ac388e933ba6482fbb5e3505d1a1e18a,4711abe2a1774467901f76dcd4d590b8 +94,5978cdb261304e13ab299f7c4a8bf3ff,d9ea4748b79842888038c71e72061abd +95,7808218f4ea74e57acbb0a913b967d29,d7fc8e21ae6f4d6c8dc0ffdf4e376aab +96,c0cd4c990efc43c18e46c246a1e3382e,8e1a87e86a5d48e593935786aa0cbc38 +97,5c0aaf7f191c4720a343f6d8b8c4704c,2ad4bd183efa49e984302df6cba20956 +98,15d996bfe59a4cc8ab4899e6cce426e2,fd4d20fb799c41a9afdf64b206e08a4e +99,d4a93966765b4a42b1697c8d14881211,36727725990f49a6bd4d92b1ec6424d2 +100,ebf75e2646914039803a59fd2b2e8fd9,e0ae4ddcf5d9424cb949b03882606219 diff --git a/cpp/ppc-cem/tests/data/d3 b/cpp/ppc-cem/tests/data/d3 new file mode 100644 index 00000000..ff848899 --- /dev/null +++ b/cpp/ppc-cem/tests/data/d3 @@ -0,0 +1,101 @@ +id,x1,x2 +1,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +2,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +3,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +4,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +5,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +6,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +7,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +8,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +9,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +10,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +11,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +12,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +13,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +14,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +15,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +16,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +17,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +18,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +19,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +20,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +21,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +22,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +23,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +24,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +25,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +26,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +27,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +28,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +29,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +30,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +31,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +32,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +33,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +34,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +35,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +36,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +37,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +38,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +39,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +40,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +41,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +42,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +43,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +44,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +45,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +46,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +47,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +48,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +49,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +50,b01fd1269fc544b4b54ad067a5a96cc8,68ef58e92d6a4e62b8725f80f161cc04 +51,c127b40c158f404aa432656b013bad12,424637e2531f4ddfad8f43d8d3d45f3e +52,f35367150f904fa3ab3442e77355e43a,600d3627245341d580fe078b19450d6d +53,29627c5f37e94779a8ec95322e5afbe1,63533be072f8427ea9ba6d0369163862 +54,a41df047bf2f4e919033441f61d1b7fd,e2382ff5ba01431d9f48e085419ec4ef +55,65db7c7b020d4b0d9229a2ad0ea4c7c5,e775c8b1f7154d119ec8ddbea66cd4bc +56,e749e17a71a2447fac63bca47b6846a7,0045251d24b9466695d5d0c38929ea90 +57,14c697e5bce841878ebf89bf08a9ba14,04317b907cdc4043b8200db5110831c1 +58,38047064e02a4ca4af076ae26aa679f7,9bb62566a45c4d229a4fd795f8777a75 +59,5fb6f18de87043a497f0677537c189bc,2724e3acf012450e8355d84b8b3ece86 +60,b44e260738b343629faaa7d9f549bbcb,94b80d51709d4c5faaee2d78a762bbc4 +61,808af13908c14cd89092219dc37b29cf,1085ef4476aa4635b7d3e78ea67f83c3 +62,af18d05a4e2548468d61ccad12a0340e,2024c88fcc03414a81b64da8409dc8af +63,ba426827999c462b9df669d7e932144c,2b2d0515df814e4f94c1110d3b584ba4 +64,525584d2c08f439eb7bee2db6b91ac37,8188e4aee9354bb2b9fc8536f34b1c24 +65,f1d40891ad90463cb530bcd4fce3b733,1c6bb273297846dbbb920e5409c6e5ff +66,f9f22de01edd4644b6c55ef7a993b184,55adc457cd124274a27ebb830ad3dc66 +67,e4ff7eaa9dd04f149049aa12bf21e68a,0c3fb3b958b14243a3edcd1bc9a51283 +68,71d984a3e0ad42698f3186cfbfdcbd87,0a520b68a33c499b9809ae269a55c394 +69,96d145bc82ff4e2a84551c20ef6f1cc5,affad8d9f1284a5dbdf97ca825288654 +70,fcece0ef256c47fdaa5678e20df0f4a1,623b626acc0b494d90f64879575c51e3 +71,87800bc6aeb5461b869afaba3d7430b3,0825eac882c0486b8316723e722e29cf +72,dd943a94e743432592c68e38352e83c8,3aeb080d0f234cc198b3165baab8d8f5 +73,8f7f6922e81246a680f931c44ecc0d1a,42f4992a390a440a9aa1b440a02e78f5 +74,42d3d9010a7548faa386e83ae61f4802,f4d3dbad28004d759be6229bc2bbff9a +75,01dc4d7631f94ba987e435c87d632684,44bde70d206b4450bde0991d405ab0fa +76,ccea7c284fce4d3988d8948f79b2fef6,04fecfe64b874cc8b2cad3e2bc205317 +77,272d7020324f4fa9a36a31b64ef798fb,c803c1ef9eab459387bb0be0fe6e1e93 +78,c92ee2e67c22477482eb6a9b3492283a,07cf6bbaea12454ba9f8693486df6282 +79,afba2c31d5fb4e658fa16b269bd48179,c55973e7e2f04e96976e99218bf2a8d5 +80,7066d9b40a994cce8b5eb555a3a265c8,6bb406d7c2de40ff828bc6e2a4e97ee9 +81,ce88a3d29e574780a649399794d793bf,b5ac277d1e7242089a30a48d78033ac2 +82,2514054bd2fd42f79ecdbb4b4fac3301,af552fac31434e3b94bdb2dd7087c6e7 +83,1466f1d054284e78b1844dd343bdc095,970e1fd372ee4dacbc5d911a0f556efe +84,21d7c25b139845ea84b8a04dc45a1fc9,d28447da4e5843adadb9845fab94a278 +85,3e451bbe00b34590b0d627d0d17d4aab,8aa5643bab1342ffa4229bef214807d1 +86,d2e635d343a440478e8ae39d5f07c87d,adf9ab5d926c4d48ac33fa301f2e5cd4 +87,2ec3545d3ce84b4290481b4b3d5c9fd2,6c0a1f2b9d124cd1a2e27a419ecfce58 +88,42d37153df1c4e7f9eacf0628f07d22a,695082c3bd6a48a69e74c7435a642b1b +89,403c2018e9b64e7caba38b12ecdb8358,470671bd2e7640c8851426c9ebe4c924 +90,ccb7cefb3efd4a49a6e5e400caf239da,2d73f2af749d4c04990f6464d8d1350d +91,a39164ae87d84d8a8faecdfecfa537f3,1d82ac0af6e1484dbc16b1fe2fde368f +92,908045cc3301407ca0c8818e01eebf5f,92dc6cd19357410e96b60d96238e4f03 +93,ac388e933ba6482fbb5e3505d1a1e18a,4711abe2a1774467901f76dcd4d590b8 +94,5978cdb261304e13ab299f7c4a8bf3ff,d9ea4748b79842888038c71e72061abd +95,7808218f4ea74e57acbb0a913b967d29,d7fc8e21ae6f4d6c8dc0ffdf4e376aab +96,c0cd4c990efc43c18e46c246a1e3382e,8e1a87e86a5d48e593935786aa0cbc38 +97,5c0aaf7f191c4720a343f6d8b8c4704c,2ad4bd183efa49e984302df6cba20956 +98,15d996bfe59a4cc8ab4899e6cce426e2,fd4d20fb799c41a9afdf64b206e08a4e +99,d4a93966765b4a42b1697c8d14881211,36727725990f49a6bd4d92b1ec6424d2 +100,ebf75e2646914039803a59fd2b2e8fd9,e0ae4ddcf5d9424cb949b03882606219 diff --git a/cpp/ppc-cem/tests/main.cpp b/cpp/ppc-cem/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-cem/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/CMakeLists.txt b/cpp/ppc-crypto-c-sdk/CMakeLists.txt new file mode 100644 index 00000000..e55eb59d --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/CMakeLists.txt @@ -0,0 +1,23 @@ +project(ppc-crypto-c-sdk VERSION ${VERSION}) + +# export windows dll symbol +if(WIN32) + message(STATUS "Compile on Windows") + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") +endif() + +file(GLOB_RECURSE SRCS ppc-crypto-c-sdk/*.cpp ppc-crypto-c-sdk/*.c) + +# generate the static lib +add_library(${PPC_CRYPTO_C_SDK_STATIC_TARGET} ${SRCS}) +target_link_libraries(${PPC_CRYPTO_C_SDK_STATIC_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) + +# generate the shared lib +add_library(${PPC_CRYPTO_C_SDK_TARGET} SHARED ${SRCS}) +target_link_libraries(${PPC_CRYPTO_C_SDK_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/build.gradle b/cpp/ppc-crypto-c-sdk/bindings/java/build.gradle new file mode 100644 index 00000000..b379d40b --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/build.gradle @@ -0,0 +1,199 @@ +plugins { + id 'java' + id 'idea' + id 'java-library' + id 'maven-publish' + //id 'org.ajoberstar.grgit' version '4.1.1' + //id "de.undercouch.download" version "4.1.2" +// id 'com.github.sherter.google-java-format' version '0.9' +} + +println("Notice: current gradle version is " + gradle.gradleVersion) +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +[compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8' + +repositories { + maven { url "https://maven.aliyun.com/nexus/content/groups/public/" } + maven { url "https://oss.sonatype.org/service/local/staging/deploy/maven2"} + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } + maven {url "https://plugins.gradle.org/m2/"} + mavenCentral() + mavenLocal() +} + +// googleJavaFormat { +// toolVersion = '1.7' +// options style: 'AOSP' +// source = sourceSets*.allJava +// include '**/*.java' +// exclude '**/*Test.java' +// exclude '**/Test*.java' +// exclude '**/Mock*.java' +// } + +sourceSets { + main { + java { + srcDir 'src/main/java' + } + resources { + srcDir 'src/main/resources' + } + } + + jmh { + java { + srcDir 'src/jmh/java' + } + resources { + srcDir 'src/jmh/resources' + } + + runtimeClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.main.compileClasspath + + compileClasspath += main.output + runtimeClasspath += main.output + } +} +dependencies { + api ('org.slf4j:slf4j-api:1.7.36') + + jmhImplementation 'org.openjdk.jmh:jmh-core:1.36' + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.36' + + testImplementation ('org.slf4j:slf4j-log4j12:1.7.36') + testImplementation ('junit:junit:4.13.2') +} + +archivesBaseName = 'ppc-crypto-sdk-jni' +group = 'com.webank.wedpr' +version = '1.0.0.20240325' + +// Additional attribute definition +ext { + if (!project.hasProperty("ossrhUsername")) { + ossrhUsername="xxx" + } + + if (!project.hasProperty("ossrhPassword")) { + ossrhPassword="xxx" + } +} + +jar { + exclude '**/*.xml' + exclude '**/*.properties' + /*manifest { + try { + def repo = grgit.open(currentDir: project.rootDir) + if (repo != null) { + + def date = new Date().format("yyyy-MM-dd'T'HH:mm:ssZ") + def branch = repo.branch.getCurrent().getName() + def commit = repo.head().getAbbreviatedId(40) + + attributes(["Implementation-Timestamp": date, + "Git-Branch" : branch, + "Git-Commit" : commit]) + } + } catch (Exception e) { + e.printStackTrace() + } + } from sourceSets.main.output*/ + + doLast { + copy { + from destinationDirectory + into 'dist/apps' + } + + copy { + from configurations.runtimeClasspath + into 'dist/lib' + } + } +} + +javadoc { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +task sourcesJar(type: Jar) { + from sourceSets.main.allJava + archiveClassifier = 'sources' + duplicatesStrategy 'exclude' +} +tasks.withType(Copy).all { + duplicatesStrategy 'exclude' +} + + +task javadocJar(type: Jar) { + from javadoc + archiveClassifier = 'javadoc' +} + +task jmh(type: JavaExec, dependsOn: jmhClasses) { + main = 'com.webank.wedpr.sdk.jni.jmh.BenchmarkMain' + classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath +} + +publishing { + publications { + mavenJava(MavenPublication) { + + artifactId project.name + groupId project.group + version project.version + + from components.java + artifact sourcesJar + artifact javadocJar + + pom { + name = 'wedpr' + description = 'wedpr ppc-crypto-sdk-jni' + url = 'http://www.fisco-bcos.org' + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + scm { + connection = 'http://github.com/WeBankBlockchain/wedpr-components.git' + url = 'http://github.com/WeBankBlockchain/wedpr-components.git' + } + + developers { + developer { + id = 'zhangsan' + name = 'zhangsan' + email = 'zhangsan@example.com' + } + } + } + } + } + repositories { + maven { + def releasesRepoURL = "https://oss.sonatype.org/service/local/staging/deploy/maven2" + def snapshotsRepoURL = "https://oss.sonatype.org/content/repositories/snapshots" + allowInsecureProtocol = true + url = !version.endsWith("SNAPSHOT") ? releasesRepoURL : snapshotsRepoURL + + credentials { + username 'deployment' + password 'deployment123' + } + } + } + +} + diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar b/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q
Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties b/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a33e7343 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +# distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip +distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradlew b/cpp/ppc-crypto-c-sdk/bindings/java/gradlew new file mode 100644 index 00000000..4f906e0c --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat b/cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat new file mode 100644 index 00000000..ac1b06f9 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle b/cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle new file mode 100644 index 00000000..4f80dfb3 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'ppc-crypto-sdk-jni' + diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java new file mode 100644 index 00000000..4100ca99 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java @@ -0,0 +1,27 @@ +package com.webank.wedpr.sdk.jni.jmh; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +public class BenchmarkMain { + public static void main(String[] args) throws Exception { + Options opt = + new OptionsBuilder() + .include("com.webank.wedpr.sdk.jni.jmh.*") + .forks(1) + .timeUnit(TimeUnit.NANOSECONDS) + .warmupIterations(5) + .warmupTime(TimeValue.valueOf("1s")) + .measurementIterations(60) + .measurementTime(TimeValue.valueOf("1s")) + // .resultFormat(ResultFormatType.JSON) + .resultFormat(ResultFormatType.TEXT) + .build(); + + new Runner(opt).run(); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java new file mode 100644 index 00000000..d9e2c34f --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java @@ -0,0 +1,101 @@ +package com.webank.wedpr.sdk.jni.jmh; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.Paillier; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.math.BigInteger; +import java.util.concurrent.TimeUnit; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 120, time = 1, timeUnit = TimeUnit.SECONDS) +public class PaillierBenchmark +{ + private Paillier paillier = new Paillier(); + // the keypair + private long keypair; + // the public key + private long publickey; + // m1 + private byte[] m1; + // m2 + private byte[] m2; + // v + private byte[] v; + int keyBits; + // cipher1 + public byte[] c1; + public byte[] c2; + + @Setup + public void setup() throws JniException { + m1 = BigInteger.valueOf(System.currentTimeMillis()).toByteArray(); + m2 = BigInteger.valueOf(System.currentTimeMillis() + 100000).toByteArray(); + v = BigInteger.valueOf(2134234).toByteArray(); + keyBits = 2048; + keypair = paillier.generateKeyPair(2048); + publickey = paillier.getPublicKeyJniObject(keypair); + c1 = paillier.encryptFast(m1, keypair); + c2 = paillier.encryptFast(m2, keypair); + } + + @TearDown + public void tearDown() { + // Clean up resources if needed + } + + @Benchmark + public void generateKeyPair() throws JniException { + paillier.generateKeyPair(keyBits); + } + + @Benchmark + public void encryptionFast() throws JniException { + paillier.encryptFast(m1, keypair); + } + + @Benchmark + public void encrypt() throws JniException { + paillier.encrypt(m1, publickey); + } + + @Benchmark + public void decryption() throws JniException { + paillier.decrypt(c1, keypair); + } + + @Benchmark + public void add() throws JniException { + paillier.add(c1, c2, publickey); + } + + @Benchmark + public void sub() throws JniException { + paillier.sub(c1, c2, publickey); + } + + @Benchmark + public void scalaMul() throws JniException { + paillier.scalaMul(v, c1, publickey); + } + + public static void main(String[] args) throws Exception { + Options opt = new OptionsBuilder().include(PaillierBenchmark.class.getSimpleName()).forks(1).build(); + new Runner(opt).run(); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt new file mode 100644 index 00000000..3741b8c1 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt @@ -0,0 +1,15 @@ +# source +file(GLOB_RECURSE SOURCES "jni/*.h" "jni/*.cpp" "jni/*.c") + +add_library(${PPC_CRYPTO_SDK_JNI_TARGET} SHARED ${SOURCES}) +target_link_libraries(${PPC_CRYPTO_SDK_JNI_TARGET} PUBLIC ${PPC_CRYPTO_C_SDK_STATIC_TARGET}) + +add_library(${PPC_CRYPTO_SDK_JNI_STATIC_TARGET} ${SOURCES}) +target_link_libraries(${PPC_CRYPTO_SDK_JNI_STATIC_TARGET} PUBLIC ${PPC_CRYPTO_C_SDK_STATIC_TARGET}) + + +SET(LIBRARY_OUTPUT_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../resources/META-INF/native/) + +message(STATUS "CMAKE_INSTALL_INCLUDEDIR => ${CMAKE_INSTALL_INCLUDEDIR}") +message(STATUS "CMAKE_CURRENT_SOURCE_DIR => ${CMAKE_CURRENT_SOURCE_DIR}") +message(STATUS "LIB_DIR_PATH => ${CMAKE_CURRENT_SOURCE_DIR}/../resources/META-INF/native/") \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp new file mode 100644 index 00000000..62dc727f --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.cpp + * @author: yujiechen + * @date 2023-08-30 + */ +#include "Common.h" +#include "JNIException.h" +#include "openssl/err.h" +#include + +// convert java bytes to InputBuffer +InputBuffer convertToInputBuffer(bcos::bytes& resultBuffer, JNIEnv* env, jbyteArray arrayData) +{ + resultBuffer.resize(env->GetArrayLength(arrayData)); + env->GetByteArrayRegion(arrayData, 0, resultBuffer.size(), (jbyte*)resultBuffer.data()); + return InputBuffer{resultBuffer.data(), resultBuffer.size()}; +} + +jbyteArray BigNumToJavaBigIntegerBytes(JNIEnv* env, BIGNUM* value) +{ + if (!value) + { + THROW_JNI_EXCEPTION(env, "Not support convert null big-num to big-integer!"); + return NULL; + } + auto resultLen = BN_num_bytes(value) + 1; + bcos::bytes valueBin(resultLen); + if (BN_bn2bin(value, valueBin.data() + 1) < 0) + { + THROW_JNI_EXCEPTION( + env, "BN_bn2bin error: " + std::string(ERR_error_string(ERR_get_error(), NULL))); + return NULL; + } + // negative case + if (BN_is_negative(value)) + { + bool carry = true; + for (int64_t i = resultLen - 1; i >= 0; i--) + { + valueBin[i] ^= 0xff; + if (carry) + { + carry = (++(valueBin[i])) == 0; + } + } + valueBin[0] |= 0x80; + } + else + { + valueBin[0] = 0x00; + } + jbyteArray ret = env->NewByteArray(resultLen); + env->SetByteArrayRegion(ret, 0, resultLen, (jbyte*)valueBin.data()); + return ret; +} + +// convert java big-integer to openssl BIGNUM +ppc::crypto::BigNum JavaBigIntegerToBigNum(JNIEnv* env, jbyteArray bigIntegerData) +{ + bcos::bytes javaBigIntegerBytes; + convertToInputBuffer(javaBigIntegerBytes, env, bigIntegerData); + unsigned char* data = javaBigIntegerBytes.data(); + unsigned char* pData = data; + auto len = javaBigIntegerBytes.size(); + ppc::crypto::BigNum result; + // zero + if (0 == len) + { + BN_zero(result.bn().get()); + return result; + } + bool negative = (data[0] & 0x80) != 0; + // convert to two's complement if negative + bcos::bytes twosComplement(len, 0); + if (negative) + { + bool carry = true; + for (int i = len - 1; i >= 0; i--) + { + twosComplement[i] = (data[i] ^ 0xFF); + if (carry) + { + carry = (++twosComplement[i]) == 0; + } + } + pData = twosComplement.data(); + } + // convert to BIGNUM + if (!BN_bin2bn(pData, len, result.bn().get())) + { + BN_clear_free(result.bn().get()); + THROW_JNI_EXCEPTION(env, "convert java bigInteger data to BigNum error" + + std::string(ERR_error_string(ERR_get_error(), NULL))); + } + if (negative) + { + BN_set_negative(result.bn().get(), 1); + } + return result; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h new file mode 100644 index 00000000..e26fad72 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2023-08-14 + */ + +#pragma once + +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include + +// convert java bytes to InputBuffer +InputBuffer convertToInputBuffer(bcos::bytes& resultBuffer, JNIEnv* env, jbyteArray arrayData); +jbyteArray BigNumToJavaBigIntegerBytes(JNIEnv* env, BIGNUM* value); +// convert java big-integer to openssl BIGNUM +ppc::crypto::BigNum JavaBigIntegerToBigNum(JNIEnv* env, jbyteArray bigIntegerData); \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp new file mode 100644 index 00000000..e9a907de --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JNIException.cpp + * @author: yujiechen + * @date 2023-08-14 + */ +#include "JNIException.h" +#include +#include + +void ThrowJNIException(JNIEnv* env, const char* kpFile, int iLine, const std::string& message) +{ + const char* className = "com/webank/wedpr/sdk/jni/common/JniException"; + (void)kpFile; + (void)iLine; + + // Find the exception class. + jclass tClass = env->FindClass(className); + if (env->ExceptionCheck()) + { + env->ExceptionDescribe(); + env->ExceptionClear(); + tClass = env->FindClass("java/lang/Exception"); + if (tClass == NULL) + { + std::cerr << " Not found exception class: " + << "java/lang/Exception" << std::endl; + env->DeleteLocalRef(tClass); + return; + } + } + + // Throw the exception with error info + env->ThrowNew(tClass, message.c_str()); + env->DeleteLocalRef(tClass); +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h new file mode 100644 index 00000000..6bf6dc22 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JNIException.h + * @author: yujiechen + * @date 2023-08-14 + */ +#include +#include +/* Header for class common */ + +#ifndef __JNI_EXCEPTION_H__ +#define __JNI_EXCEPTION_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define THROW_JNI_EXCEPTION(_ENV_, _INFO_) ThrowJNIException(_ENV_, __FILE__, __LINE__, _INFO_); + +void ThrowJNIException(JNIEnv* env, const char* kpFile, int iLine, const std::string& message); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp new file mode 100644 index 00000000..26218da3 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp @@ -0,0 +1,251 @@ +#include "com_webank_wedpr_sdk_jni_crypto_FastOre.h" +#include "Common.h" +#include "JNIException.h" +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: keyBytes + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_keyBytes(JNIEnv* env, jclass) +{ + auto keyBytes = fast_ore_key_bytes(); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + return keyBytes; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: cipherSize + * Signature: (JZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_cipherSize( + JNIEnv* env, jclass, jlong plainSize, jboolean hex) +{ + auto cipherSize = fast_ore_get_cipher_size(plainSize, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + return cipherSize; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: plainSize + * Signature: (JZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_plainSize( + JNIEnv* env, jclass, jlong cipherSize, jboolean hex) +{ + auto plainSize = fast_ore_get_plain_size(cipherSize, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + return plainSize; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: generateKey + * Signature: ()[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_generateKey( + JNIEnv* env, jclass cls) +{ + auto keyBytes = Java_com_webank_wedpr_sdk_jni_crypto_FastOre_keyBytes(env, cls); + bcos::bytes keyData(keyBytes); + OutputBuffer keyBuffer{keyData.data(), keyData.size()}; + fast_ore_generate_key(&keyBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(keyBuffer.len); + env->SetByteArrayRegion(jResult, 0, keyBuffer.len, (jbyte*)keyBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4String + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4String( + JNIEnv* env, jclass cls, jbyteArray sk, jbyteArray plainData, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes plainBytes; + auto plainBuffer = convertToInputBuffer(plainBytes, env, plainData); + bcos::bytes cipherBytes( + Java_com_webank_wedpr_sdk_jni_crypto_FastOre_cipherSize(env, cls, plainBuffer.len, hex)); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + string_fast_ore_encrypt(&cipherBuffer, &skBuffer, &plainBuffer, hex); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4String + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4String( + JNIEnv* env, jclass cls, jbyteArray sk, jbyteArray cipher, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + bcos::bytes plainBytes( + Java_com_webank_wedpr_sdk_jni_crypto_FastOre_plainSize(env, cls, cipherBuffer.len, hex)); + OutputBuffer plainBuffer{plainBytes.data(), plainBytes.size()}; + string_fast_ore_decrypt(&plainBuffer, &skBuffer, &cipherBuffer, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(plainBuffer.len); + env->SetByteArrayRegion(jResult, 0, plainBuffer.len, (jbyte*)plainBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4Integer + * Signature: ([BJZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4Integer( + JNIEnv* env, jclass cls, jbyteArray sk, jlong plain, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes cipherBytes( + Java_com_webank_wedpr_sdk_jni_crypto_FastOre_cipherSize(env, cls, sizeof(plain), hex)); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + integer_fast_ore_encrypt(&cipherBuffer, &skBuffer, static_cast(plain), hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4Integer + * Signature: ([B[BZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4Integer( + JNIEnv* env, jclass cls, jbyteArray sk, jbyteArray cipher, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + int64_t plain = 0; + integer_fast_ore_decrypt(&plain, &skBuffer, &cipherBuffer, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return 0; + } + return plain; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4Float + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4Float( + JNIEnv* env, jclass, jbyteArray sk, jbyteArray plainData, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes plainBytes; + auto plainBuffer = convertToInputBuffer(plainBytes, env, plainData); + + bcos::bytes cipherBytes(fast_ore_get_float_cipher_size(plainBuffer.len, hex)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + float_fast_ore_encrypt(&cipherBuffer, &skBuffer, &plainBuffer, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4Float + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4Float( + JNIEnv* env, jclass, jbyteArray sk, jbyteArray cipherData, jboolean hex) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipherData); + + bcos::bytes plainBytes(64); + OutputBuffer plainBuffer{plainBytes.data(), plainBytes.size()}; + float_fast_ore_decrypt(&plainBuffer, &skBuffer, &cipherBuffer, hex); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(plainBuffer.len); + env->SetByteArrayRegion(jResult, 0, plainBuffer.len, (jbyte*)plainBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: compare + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_compare( + JNIEnv* env, jclass, jbyteArray c1, jbyteArray c2) +{ + bcos::bytes c1Bytes; + auto c1Buffer = convertToInputBuffer(c1Bytes, env, c1); + bcos::bytes c2Bytes; + auto c2Buffer = convertToInputBuffer(c2Bytes, env, c2); + auto ret = fast_ore_compare(&c1Buffer, &c2Buffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return -1; + } + return ret; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h new file mode 100644 index 00000000..8e332da7 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h @@ -0,0 +1,100 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_crypto_FastOre */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_crypto_FastOre +#define _Included_com_webank_wedpr_sdk_jni_crypto_FastOre +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: keyBytes + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_keyBytes(JNIEnv*, jclass); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: cipherSize + * Signature: (JZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_cipherSize( + JNIEnv*, jclass, jlong, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: plainSize + * Signature: (JZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_plainSize( + JNIEnv*, jclass, jlong, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: generateKey + * Signature: ()[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_generateKey( + JNIEnv*, jclass); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4String + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4String( + JNIEnv*, jclass, jbyteArray, jbyteArray, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4String + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4String( + JNIEnv*, jclass, jbyteArray, jbyteArray, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4Integer + * Signature: ([BJZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4Integer( + JNIEnv*, jclass, jbyteArray, jlong, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4Integer + * Signature: ([B[BZ)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4Integer( + JNIEnv*, jclass, jbyteArray, jbyteArray, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: encrypt4Float + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_encrypt4Float( + JNIEnv*, jclass, jbyteArray, jbyteArray, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: decrypt4Float + * Signature: ([B[BZ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_decrypt4Float( + JNIEnv*, jclass, jbyteArray, jbyteArray, jboolean); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_FastOre + * Method: compare + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_FastOre_compare( + JNIEnv*, jclass, jbyteArray, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp new file mode 100644 index 00000000..c02b9be1 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp @@ -0,0 +1,110 @@ +#include "com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h" +#include "Common.h" +#include "JNIException.h" +#include "bcos-utilities/Common.h" +#include "ppc-crypto-c-sdk/symmetric_encryption.h" +#include "ppc-crypto-c-sdk/utils/error.h" +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: keyBytes + * Signature: (II)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_keyBytes( + JNIEnv* env, jclass, jint algorithmType, jint mode) +{ + auto keyBytes = symmetric_key_bytes(algorithmType, mode); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + return keyBytes; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: generateKey + * Signature: (II)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_generateKey( + JNIEnv* env, jclass, jint algorithmType, jint mode) +{ + bcos::bytes keyData(symmetric_key_bytes(algorithmType, mode)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + OutputBuffer keyBuffer{keyData.data(), keyData.size()}; + symmetric_generate_key(&keyBuffer, algorithmType, mode); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + auto jResult = env->NewByteArray(keyBuffer.len); + env->SetByteArrayRegion(jResult, 0, keyBuffer.len, (jbyte*)keyBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: encrypt + * Signature: (II[B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_encrypt( + JNIEnv* env, jclass, jint algorithmType, jint mode, jbyteArray sk, jbyteArray iv, + jbyteArray plainData) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes ivBytes; + auto ivBuffer = convertToInputBuffer(ivBytes, env, iv); + bcos::bytes plainBytes; + auto plainBuffer = convertToInputBuffer(plainBytes, env, plainData); + bcos::bytes cipherData(symmetric_block_size(algorithmType) + plainBuffer.len); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + symmetric_encrypt(&cipherBuffer, algorithmType, mode, &skBuffer, &ivBuffer, &plainBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: decrypt + * Signature: (II[B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_decrypt( + JNIEnv* env, jclass, jint algorithm, jint mode, jbyteArray sk, jbyteArray iv, jbyteArray cipher) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes ivBytes; + auto ivBuffer = convertToInputBuffer(ivBytes, env, iv); + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + bcos::bytes plainData(symmetric_block_size(algorithm) + cipherBuffer.len); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + OutputBuffer plainBuffer{plainData.data(), plainData.size()}; + symmetric_decrypt(&plainBuffer, algorithm, mode, &skBuffer, &ivBuffer, &cipherBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(plainBuffer.len); + env->SetByteArrayRegion(jResult, 0, plainBuffer.len, (jbyte*)plainBuffer.data); + return jResult; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h new file mode 100644 index 00000000..a220d023 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption +#define _Included_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: keyBytes + * Signature: (II)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_keyBytes( + JNIEnv*, jclass, jint, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: generateKey + * Signature: (II)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_generateKey( + JNIEnv*, jclass, jint, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: encrypt + * Signature: (II[B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_encrypt( + JNIEnv*, jclass, jint, jint, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption + * Method: decrypt + * Signature: (II[B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption_decrypt( + JNIEnv*, jclass, jint, jint, jbyteArray, jbyteArray, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp new file mode 100644 index 00000000..cc8677ef --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp @@ -0,0 +1,142 @@ +#include "com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h" +#include "Common.h" +#include "JNIException.h" +#include "ppc-crypto-c-sdk/floating_point_ihc.h" +#include "ppc-crypto-c-sdk/homo_ihc.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include +#include + +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_generateKey( + JNIEnv* env, jclass, jint mode) +{ + bcos::bytes keyBytes(ihc_key_bytes(mode)); + OutputBuffer key{keyBytes.data(), keyBytes.size()}; + ihc_generate_key(&key, mode); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + } + auto jResult = env->NewByteArray(key.len); + env->SetByteArrayRegion(jResult, 0, key.len, (jbyte*)key.data); + return jResult; +} + +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_encrypt( + JNIEnv* env, jclass, jint mode, jbyteArray key, jbyteArray significant, jint exponent) +{ + bcos::bytes cipherBytes(ihc_floating_cipher_bytes(mode)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + auto bigNumValue = JavaBigIntegerToBigNum(env, significant); + + bcos::bytes keyBytes; + auto keyBuffer = convertToInputBuffer(keyBytes, env, key); + ihc_floating_encrypt(&cipherBuffer, mode, &keyBuffer, bigNumValue.bn().get(), exponent); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +JNIEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_decrypt( + JNIEnv* env, jclass, jint mode, jbyteArray key, jbyteArray cipher) +{ + bcos::bytes cipherCBytes; + auto cipherBuffer = convertToInputBuffer(cipherCBytes, env, cipher); + bcos::bytes keyBytes; + auto keyBuffer = convertToInputBuffer(keyBytes, env, key); + + ppc::FloatingPointNumber resultFp; + ihc_floating_decrypt( + resultFp.value.bn().get(), &resultFp.exponent, mode, &keyBuffer, &cipherBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // convert to FloatingPointNumber + jclass fpNumberClass = env->FindClass("com/webank/wedpr/sdk/jni/codec/FloatingPointNumber"); + // FloatingPointNumber(byte[] significantBytes, int exponent) + jmethodID initFunID = env->GetMethodID(fpNumberClass, "", "([BI)V"); + jbyteArray cipherBytes = BigNumToJavaBigIntegerBytes(env, resultFp.value.bn().get()); + jobject result = env->NewObject(fpNumberClass, initFunID, cipherBytes, resultFp.exponent); + // release the used jbyteArray + env->DeleteLocalRef(cipherBytes); + return result; +} + + +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_add( + JNIEnv* env, jclass, jint mode, jbyteArray cipher1, jbyteArray cipher2) +{ + bcos::bytes cipherBytes1; + auto cipherBuffer1 = convertToInputBuffer(cipherBytes1, env, cipher1); + bcos::bytes cipherBytes2; + auto cipherBuffer2 = convertToInputBuffer(cipherBytes2, env, cipher2); + bcos::bytes cipherBytes(ihc_floating_cipher_bytes(mode)); + OutputBuffer resultBuffer{cipherBytes.data(), cipherBytes.size()}; + ihc_floating_add(&resultBuffer, mode, &cipherBuffer1, &cipherBuffer2); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} + + +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_sub( + JNIEnv* env, jclass, jint mode, jbyteArray cipher1, jbyteArray cipher2) +{ + bcos::bytes cipherBytes1; + auto cipherBuffer1 = convertToInputBuffer(cipherBytes1, env, cipher1); + bcos::bytes cipherBytes2; + auto cipherBuffer2 = convertToInputBuffer(cipherBytes2, env, cipher2); + bcos::bytes cipherBytes(ihc_floating_cipher_bytes(mode)); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + ihc_floating_sub(&cipherBuffer, mode, &cipherBuffer1, &cipherBuffer2); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + + +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_scalaMul( + JNIEnv* env, jclass, jint mode, jbyteArray vSignificant, jint vExponent, jbyteArray cipher) +{ + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + auto bigNumValue = JavaBigIntegerToBigNum(env, vSignificant); + bcos::bytes resultBytes(ihc_floating_cipher_bytes(mode)); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + ihc_floating_scalaMul(&resultBuffer, mode, bigNumValue.bn().get(), vExponent, &cipherBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h new file mode 100644 index 00000000..7780e9ca --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc +#define _Included_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: generateKey + * Signature: (I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_generateKey( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: encrypt + * Signature: (I[B[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_encrypt( + JNIEnv*, jclass, jint, jbyteArray, jbyteArray, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: decrypt + * Signature: (I[B[B)Lcom/webank/wedpr/sdk/jni/codec/FloatingPointNumber; + */ +JNIEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_decrypt( + JNIEnv*, jclass, jint, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: add + * Signature: (I[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_add( + JNIEnv*, jclass, jint, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: sub + * Signature: (I[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_sub( + JNIEnv*, jclass, jint, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc + * Method: scalaMul + * Signature: (I[B[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc_scalaMul( + JNIEnv*, jclass, jint, jbyteArray, jint, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp new file mode 100644 index 00000000..ac0907e3 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp @@ -0,0 +1,445 @@ +#include "com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h" +#include "Common.h" +#include "JNIException.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: maxCipherBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_maxCipherBytes( + JNIEnv*, jclass, jint keyBits) +{ + return floating_point_paillier_cipher_bytes(keyBits); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptFast + * Signature: ([BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptFast( + JNIEnv* env, jclass, jbyteArray significantBytes, jint exponent, jlong jkeypair) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + + auto keypair = reinterpret_cast(jkeypair); + auto keyBits = paillier_key_bits_from_keypair(keypair); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto cipherLen = floating_point_paillier_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + + floating_point_paillier_encrypt_fast(&cipherBuffer, significant.bn().get(), exponent, keypair); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptFastWithoutPrecompute + * Signature: ([BI[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptFastWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray significantBytes, jint exponent, jbyteArray sk, jbyteArray pk) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, pk); + auto cipherLen = floating_point_paillier_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + + floating_point_paillier_encrypt_fast_without_precompute( + &cipherBuffer, significant.bn().get(), exponent, &skBuffer, &pkBuffer); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encrypt + * Signature: ([BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encrypt( + JNIEnv* env, jclass, jbyteArray significantBytes, jint exponent, jlong jpublicKey) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + + auto pk = reinterpret_cast(jpublicKey); + auto keyBits = paillier_key_bits_from_public_key(pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto cipherLen = floating_point_paillier_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + floating_point_paillier_encrypt(&cipherBuffer, significant.bn().get(), exponent, pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptWithoutPrecompute + * Signature: ([BI[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray significantBytes, jint exponent, jbyteArray pkBytes) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + + bcos::bytes pkCBytes; + auto pkBuffer = convertToInputBuffer(pkCBytes, env, pkBytes); + auto cipherLen = floating_point_paillier_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + floating_point_paillier_encrypt_without_precompute( + &cipherBuffer, significant.bn().get(), exponent, &pkBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: decrypt + * Signature: ([BJ)Ljava/math/BigInteger; + */ +JNIEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_decrypt( + JNIEnv* env, jclass, jbyteArray cipher, jlong jkeypair) +{ + ppc::crypto::BigNum decrypted_significant; + int16_t exponent; + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + floating_point_paillier_decrypt(decrypted_significant.bn().get(), &exponent, &cipherBuffer, + reinterpret_cast(jkeypair)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // convert to FloatingPointNumber + jclass fpNumberClass = env->FindClass("com/webank/wedpr/sdk/jni/codec/FloatingPointNumber"); + // FloatingPointNumber(byte[] significantBytes, int exponent) + jmethodID initFunID = env->GetMethodID(fpNumberClass, "", "([BI)V"); + jbyteArray plainBytes = BigNumToJavaBigIntegerBytes(env, decrypted_significant.bn().get()); + jobject result = env->NewObject(fpNumberClass, initFunID, plainBytes, exponent); + env->DeleteLocalRef(plainBytes); + return result; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: decryptWithoutPrecompute + * Signature: ([B[B[B)Lcom/webank/wedpr/sdk/jni/codec/FloatingPointNumber; + */ +JNIEXPORT jobject JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_decryptWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray cipher, jbyteArray sk, jbyteArray pk) +{ + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, pk); + ppc::crypto::BigNum decrypted_significant; + int16_t exponent; + bcos::bytes cipherCBytes; + auto cipherBuffer = convertToInputBuffer(cipherCBytes, env, cipher); + floating_point_paillier_decrypt_without_precompute( + decrypted_significant.bn().get(), &exponent, &cipherBuffer, &skBuffer, &pkBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // convert to FloatingPointNumber + jclass fpNumberClass = env->FindClass("com/webank/wedpr/sdk/jni/codec/FloatingPointNumber"); + // FloatingPointNumber(byte[] significantBytes, int exponent) + jmethodID initFunID = env->GetMethodID(fpNumberClass, "", "([BI)V"); + jbyteArray plainBytes = BigNumToJavaBigIntegerBytes(env, decrypted_significant.bn().get()); + jobject result = env->NewObject(fpNumberClass, initFunID, plainBytes, exponent); + env->DeleteLocalRef(plainBytes); + return result; +} + +jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_impl( + JNIEnv* env, jbyteArray c1, jbyteArray c2, jlong jpublic_key, bool add) +{ + auto pk = reinterpret_cast(jpublic_key); + auto keyBits = paillier_key_bits_from_public_key(pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto cipherLen = floating_point_paillier_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + + bcos::bytes c1CBytes; + auto c1Buffer = convertToInputBuffer(c1CBytes, env, c1); + bcos::bytes c2CBytes; + auto c2Buffer = convertToInputBuffer(c2CBytes, env, c2); + if (add) + { + floating_point_paillier_add(&cipherBuffer, &c1Buffer, &c2Buffer, pk); + } + else + { + floating_point_paillier_sub(&cipherBuffer, &c1Buffer, &c2Buffer, pk); + } + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: add + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add( + JNIEnv* env, jclass, jbyteArray c1, jbyteArray c2, jlong jpublic_key) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_impl( + env, c1, c2, jpublic_key, true); +} + + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: sub + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_sub( + JNIEnv* env, jclass, jbyteArray c1, jbyteArray c2, jlong jpublic_key) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_impl( + env, c1, c2, jpublic_key, false); +} + +jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_without_precompute_impl( + JNIEnv* env, jbyteArray c1, jbyteArray c2, jbyteArray pkBytes, bool add) +{ + bcos::bytes pkCBytes; + auto pkBuffer = convertToInputBuffer(pkCBytes, env, pkBytes); + auto cipherLen = floating_point_paillier_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + bcos::bytes c1Bytes; + auto c1Buffer = convertToInputBuffer(c1Bytes, env, c1); + bcos::bytes c2Bytes; + auto c2Buffer = convertToInputBuffer(c2Bytes, env, c2); + if (add) + { + floating_point_paillier_add_without_precompute( + &cipherBuffer, &c1Buffer, &c2Buffer, &pkBuffer); + } + else + { + floating_point_paillier_sub_without_precompute( + &cipherBuffer, &c1Buffer, &c2Buffer, &pkBuffer); + } + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: decryptWithoutPrecompute + * Signature: ([B[B[B)Lcom/webank/wedpr/sdk/jni/codec/FloatingPointNumber; + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_addWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray c1, jbyteArray c2, jbyteArray pkBytes) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_without_precompute_impl( + env, c1, c2, pkBytes, true); +} +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: addWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_subWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray c1, jbyteArray c2, jbyteArray pkBytes) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add_without_precompute_impl( + env, c1, c2, pkBytes, false); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: scalaMul + * Signature: ([BI[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_scalaMul(JNIEnv* env, jclass, + jbyteArray significantBytes, jint exponent, jbyteArray cipherBytes, jlong jpublic_key) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + + auto pk = reinterpret_cast(jpublic_key); + auto keyBits = paillier_key_bits_from_public_key(pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + auto cipherLen = floating_point_paillier_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherResult(cipherLen); + OutputBuffer cipherResultBuffer{cipherResult.data(), cipherResult.size()}; + // get the cipher buffer + bcos::bytes cipherCBytes; + auto cipher = convertToInputBuffer(cipherCBytes, env, cipherBytes); + + floating_point_paillier_scalaMul( + &cipherResultBuffer, significant.bn().get(), exponent, &cipher, pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherResultBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherResultBuffer.len, (jbyte*)cipherResultBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: scalaMulWithoutPrecompute + * Signature: ([BI[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_scalaMulWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray significantBytes, jint exponent, jbyteArray cipherBytes, + jbyteArray pkBytes) +{ + auto significant = JavaBigIntegerToBigNum(env, significantBytes); + bcos::bytes pkCBytes; + auto pkBuffer = convertToInputBuffer(pkCBytes, env, pkBytes); + auto cipherLen = floating_point_paillier_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherResult(cipherLen); + OutputBuffer cipherResultBuffer{cipherResult.data(), cipherResult.size()}; + // get the cipher buffer + bcos::bytes cipherCBytes; + auto cipher = convertToInputBuffer(cipherCBytes, env, cipherBytes); + + floating_point_paillier_scalaMul_without_precompute( + &cipherResultBuffer, significant.bn().get(), exponent, &cipher, &pkBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto jResult = env->NewByteArray(cipherResultBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherResultBuffer.len, (jbyte*)cipherResultBuffer.data); + return jResult; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h new file mode 100644 index 00000000..015d8043 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h @@ -0,0 +1,126 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier +#define _Included_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: maxCipherBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_maxCipherBytes( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptFast + * Signature: ([BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptFast( + JNIEnv*, jclass, jbyteArray, jint, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptFastWithoutPrecompute + * Signature: ([BI[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptFastWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jint, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encrypt + * Signature: ([BIJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encrypt( + JNIEnv*, jclass, jbyteArray, jint, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: encryptWithoutPrecompute + * Signature: ([BI[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_encryptWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jint, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: decrypt + * Signature: ([BJ)Lcom/webank/wedpr/sdk/jni/codec/FloatingPointNumber; + */ +JNIEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_decrypt( + JNIEnv*, jclass, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: decryptWithoutPrecompute + * Signature: ([B[B[B)Lcom/webank/wedpr/sdk/jni/codec/FloatingPointNumber; + */ +JNIEXPORT jobject JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_decryptWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: add + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_add( + JNIEnv*, jclass, jbyteArray, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: addWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_addWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: sub + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_sub( + JNIEnv*, jclass, jbyteArray, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: subWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_subWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: scalaMul + * Signature: ([BI[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_scalaMul( + JNIEnv*, jclass, jbyteArray, jint, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier + * Method: scalaMulWithoutPrecompute + * Signature: ([BI[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier_scalaMulWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jint, jbyteArray, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h new file mode 100644 index 00000000..fa254ce0 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h @@ -0,0 +1,61 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_homo_NativeIhc */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_homo_NativeIhc +#define _Included_com_webank_wedpr_sdk_jni_homo_NativeIhc +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: generateKey + * Signature: (I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_generateKey( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: encrypt + * Signature: ([B[BII)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_encrypt( + JNIEnv*, jclass, jbyteArray, jbyteArray, jint, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: decrypt + * Signature: ([B[BII)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_decrypt( + JNIEnv*, jclass, jbyteArray, jbyteArray, jint, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: add + * Signature: ([B[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_add( + JNIEnv*, jclass, jbyteArray, jbyteArray, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: sub + * Signature: ([B[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_sub( + JNIEnv*, jclass, jbyteArray, jbyteArray, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativeIhc + * Method: scalaMul + * Signature: ([B[BI)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativeIhc_scalaMul( + JNIEnv*, jclass, jbyteArray, jbyteArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp new file mode 100644 index 00000000..c5973629 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp @@ -0,0 +1,629 @@ +#include "com_webank_wedpr_sdk_jni_homo_NativePaillier.h" +#include "Common.h" +#include "JNIException.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxPublicKeyBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxPublicKeyBytes( + JNIEnv*, jclass, jint keyBits) +{ + return paillier_max_public_key_bytes(keyBits); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxPrivateKeyBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxPrivateKeyBytes( + JNIEnv*, jclass, jint keyBits) +{ + return paillier_max_private_key_bytes(keyBits); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxCipherBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxCipherBytes( + JNIEnv*, jclass, jint keyBits) +{ + return paillier_max_cipher_bytes(keyBits); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: keyBitsFromKeyPair + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_keyBitsFromKeyPair( + JNIEnv*, jclass, jlong keypair) +{ + return paillier_key_bits_from_keypair(reinterpret_cast(keypair)); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: keyBitsFromPublicKey + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_keyBitsFromPublicKey( + JNIEnv*, jclass, jlong public_key) +{ + return paillier_key_bits_from_public_key(reinterpret_cast(public_key)); +} +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: generateKeyPair + * Signature: (I)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_generateKeyPair( + JNIEnv* env, jclass, jint keyBits) +{ + auto keyPair = paillier_generate_keypair(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return 0; + } + return reinterpret_cast(keyPair); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadKeyPair + * Signature: ([B[B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadKeyPair( + JNIEnv* env, jclass, jbyteArray privateKey, jbyteArray publicKey) +{ + // load key-pair from given private key and public key + bcos::bytes skBytes; + auto privateKeyBuffer = convertToInputBuffer(skBytes, env, privateKey); + bcos::bytes pkBytes; + auto publicKeyBuffer = convertToInputBuffer(pkBytes, env, publicKey); + auto keyPair = paillier_load_keypair(&privateKeyBuffer, &publicKeyBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return 0; + } + return reinterpret_cast(keyPair); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freeKeyPair + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freeKeyPair( + JNIEnv*, jclass, jlong key_pair) +{ + paillier_free_key_pair(reinterpret_cast(key_pair)); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadPublicKey + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadPublicKey( + JNIEnv* env, jclass, jbyteArray publicKeyBytes) +{ + bcos::bytes pkBytes; + auto publicKeyBuffer = convertToInputBuffer(pkBytes, env, publicKeyBytes); + auto publicKeyObj = paillier_load_public_key(&publicKeyBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return 0; + } + return reinterpret_cast(publicKeyObj); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freePublicKey + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freePublicKey( + JNIEnv*, jclass, jlong public_key) +{ + paillier_free_public_key(reinterpret_cast(public_key)); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPublicKeyJniObject + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPublicKeyJniObject( + JNIEnv*, jclass, jlong jkeyPair) +{ + auto keyPair = reinterpret_cast(jkeyPair); + return reinterpret_cast(paillier_get_public_key(keyPair)); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freePrivateKey + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freePrivateKey( + JNIEnv*, jclass, jlong private_key) +{ + paillier_free_public_key(reinterpret_cast(private_key)); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPublicKeyBytesFromKeyPair + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPublicKeyBytesFromKeyPair( + JNIEnv* env, jclass, jlong keypair) +{ + auto publicKeyBuffer = + paillier_get_public_key_bytes_from_keyPair(reinterpret_cast(keypair)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(publicKeyBuffer.len); + env->SetByteArrayRegion(jResult, 0, publicKeyBuffer.len, (jbyte*)publicKeyBuffer.data); + // release the allocated buffer + free(publicKeyBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPrivateKeyBytesFromKeyPair + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPrivateKeyBytesFromKeyPair( + JNIEnv* env, jclass, jlong keypair) +{ + auto skBuffer = paillier_get_private_key_bytes_from_keypair(reinterpret_cast(keypair)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(skBuffer.len); + env->SetByteArrayRegion(jResult, 0, skBuffer.len, (jbyte*)skBuffer.data); + // release the allocated buffer + free(skBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encryptFast + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptFast( + JNIEnv* env, jclass, jbyteArray value, jlong jkeypair) +{ + auto bigNumValue = JavaBigIntegerToBigNum(env, value); + auto keypair = reinterpret_cast(jkeypair); + // get the keyBits + auto keyBits = paillier_key_bits_from_keypair(keypair); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + // encrypt + auto cipherLen = paillier_max_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + paillier_encryt_fast(&cipherBuffer, nullptr, bigNumValue.bn().get(), keypair); + // copy the cipher into jbyteArray + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encryptFast + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptFastWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray value, jbyteArray sk, jbyteArray pk) +{ + auto bigNumValue = JavaBigIntegerToBigNum(env, value); + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, pk); + // Note: pkBuffer.len is not the real keyBytes + auto cipherLen = paillier_max_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + paillier_encryt_fast_without_precompute( + &cipherBuffer, nullptr, bigNumValue.bn().get(), &skBuffer, &pkBuffer); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encrypt + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encrypt( + JNIEnv* env, jclass, jbyteArray value, jlong public_key) +{ + auto bigNumValue = JavaBigIntegerToBigNum(env, value); + auto pk = reinterpret_cast(public_key); + // get the keyBits + auto keyBits = paillier_key_bits_from_public_key(pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // encrypt + auto cipherLen = paillier_max_cipher_bytes(keyBits); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + paillier_encryt(&cipherBuffer, nullptr, bigNumValue.bn().get(), pk); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encrypt + * Signature: ([B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray value, jbyteArray pk) +{ + auto bigNumValue = JavaBigIntegerToBigNum(env, value); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, pk); + // Note: pkBuffer.len is not the real keyBytes + auto cipherLen = paillier_max_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes cipherBytes(cipherLen); + OutputBuffer cipherBuffer{cipherBytes.data(), cipherBytes.size()}; + paillier_encryt_without_precompute(&cipherBuffer, nullptr, bigNumValue.bn().get(), &pkBuffer); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(cipherBuffer.len); + env->SetByteArrayRegion(jResult, 0, cipherBuffer.len, (jbyte*)cipherBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: decrypt + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_decrypt( + JNIEnv* env, jclass, jbyteArray cipher, jlong keypair) +{ + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + auto result = paillier_decrypt(&cipherBuffer, reinterpret_cast(keypair)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto bigIntegerBytes = BigNumToJavaBigIntegerBytes(env, result); + + if (result) + { + BN_clear_free(result); + } + return bigIntegerBytes; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: decrypt + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_decryptWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray cipher, jbyteArray sk, jbyteArray pk) +{ + bcos::bytes cipherBytes; + auto cipherBuffer = convertToInputBuffer(cipherBytes, env, cipher); + bcos::bytes skBytes; + auto skBuffer = convertToInputBuffer(skBytes, env, sk); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, pk); + auto result = paillier_decrypt_without_precompute(&cipherBuffer, &skBuffer, &pkBuffer); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + auto bigIntegerBytes = BigNumToJavaBigIntegerBytes(env, result); + if (result) + { + BN_clear_free(result); + } + return bigIntegerBytes; +} + + +jbyteArray Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_impl( + JNIEnv* env, jbyteArray cipher1, jbyteArray cipher2, jlong public_key, bool add) +{ + bcos::bytes c1Bytes; + auto cipher1Buffer = convertToInputBuffer(c1Bytes, env, cipher1); + bcos::bytes c2Bytes; + auto cipher2Buffer = convertToInputBuffer(c2Bytes, env, cipher2); + auto pk = reinterpret_cast(public_key); + + auto cipherLen = paillier_max_cipher_bytes(paillier_key_bits_from_public_key(pk)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes resultBytes(cipherLen); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + if (add) + { + paillier_add(&resultBuffer, &cipher1Buffer, &cipher2Buffer, pk); + } + else + { + paillier_sub(&resultBuffer, &cipher1Buffer, &cipher2Buffer, pk); + } + + // call error + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: add + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_add( + JNIEnv* env, jclass, jbyteArray cipher1, jbyteArray cipher2, jlong public_key) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_impl( + env, cipher1, cipher2, public_key, true); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: sub + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub( + JNIEnv* env, jclass, jbyteArray cipher1, jbyteArray cipher2, jlong public_key) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_impl( + env, cipher1, cipher2, public_key, false); +} + + +jbyteArray Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_without_precompute_impl( + JNIEnv* env, jbyteArray cipher1, jbyteArray cipher2, jbyteArray public_key, bool add) +{ + bcos::bytes c1Bytes; + auto cipher1Buffer = convertToInputBuffer(c1Bytes, env, cipher1); + bcos::bytes c2Bytes; + auto cipher2Buffer = convertToInputBuffer(c2Bytes, env, cipher2); + bcos::bytes pkBytes; + auto pkBuffer = convertToInputBuffer(pkBytes, env, public_key); + auto cipherLen = paillier_max_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes resultBytes(cipherLen); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + if (add) + { + paillier_add_without_precompute(&resultBuffer, &cipher1Buffer, &cipher2Buffer, &pkBuffer); + } + else + { + paillier_add_without_precompute(&resultBuffer, &cipher1Buffer, &cipher2Buffer, &pkBuffer); + } + // call error + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result to jResult + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: add + * Signature: ([B[B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_addWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray cipher1, jbyteArray cipher2, jbyteArray pkBytes) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_without_precompute_impl( + env, cipher1, cipher2, pkBytes, true); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: sub + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_subWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray cipher1, jbyteArray cipher2, jbyteArray pkBytes) +{ + return Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub_or_add_without_precompute_impl( + env, cipher1, cipher2, pkBytes, false); +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: scalaMul + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_scalaMul( + JNIEnv* env, jclass, jbyteArray value, jbyteArray cipherBytes, jlong public_key) +{ + auto v = JavaBigIntegerToBigNum(env, value); + bcos::bytes cipherCBytes; + auto cipherBuffer = convertToInputBuffer(cipherCBytes, env, cipherBytes); + auto pk = reinterpret_cast(public_key); + + // allocate the result buffer + auto cipherLen = paillier_max_cipher_bytes(paillier_key_bits_from_public_key(pk)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes resultBytes(cipherLen); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + // paillier scala_mul + paillier_scala_mul(&resultBuffer, v.bn().get(), &cipherBuffer, pk); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: scalMul + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_scalaMulWithoutPrecompute( + JNIEnv* env, jclass, jbyteArray value, jbyteArray cipherBytes, jbyteArray pkBytes) +{ + auto v = JavaBigIntegerToBigNum(env, value); + bcos::bytes cipherCBytes; + auto cipherBuffer = convertToInputBuffer(cipherCBytes, env, cipherBytes); + bcos::bytes pkCBytes; + auto pkBuffer = convertToInputBuffer(pkCBytes, env, pkBytes); + auto cipherLen = paillier_max_cipher_bytes(pkBuffer.len * 8); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + bcos::bytes resultBytes(cipherLen); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + // paillier scala_mul + paillier_scala_mul_without_precompute(&resultBuffer, v.bn().get(), &cipherBuffer, &pkBuffer); + + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return nullptr; + } + // copy the result + auto jResult = env->NewByteArray(resultBuffer.len); + env->SetByteArrayRegion(jResult, 0, resultBuffer.len, (jbyte*)resultBuffer.data); + return jResult; +} + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadPrivateKey + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadPrivateKey( + JNIEnv* env, jclass, jbyteArray privateBytes) +{ + bcos::bytes skBytes; + auto privateKeyBuffer = convertToInputBuffer(skBytes, env, privateBytes); + jlong result = reinterpret_cast(paillier_load_private_key(&privateKeyBuffer)); + if (!last_call_success()) + { + THROW_JNI_EXCEPTION(env, get_last_error_msg()); + return -1; + } + return result; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h new file mode 100644 index 00000000..ebb82aa0 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h @@ -0,0 +1,235 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_webank_wedpr_sdk_jni_homo_NativePaillier */ + +#ifndef _Included_com_webank_wedpr_sdk_jni_homo_NativePaillier +#define _Included_com_webank_wedpr_sdk_jni_homo_NativePaillier +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxPublicKeyBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxPublicKeyBytes( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxPrivateKeyBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxPrivateKeyBytes( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: maxCipherBytes + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_maxCipherBytes( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: keyBitsFromKeyPair + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_keyBitsFromKeyPair( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: keyBitsFromPublicKey + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_keyBitsFromPublicKey( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: generateKeyPair + * Signature: (I)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_generateKeyPair( + JNIEnv*, jclass, jint); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadKeyPair + * Signature: ([B[B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadKeyPair( + JNIEnv*, jclass, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freeKeyPair + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freeKeyPair( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadPublicKey + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadPublicKey( + JNIEnv*, jclass, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freePublicKey + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freePublicKey( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: loadPrivateKey + * Signature: ([B)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_loadPrivateKey( + JNIEnv*, jclass, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: freePrivateKey + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_freePrivateKey( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPublicKeyJniObject + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPublicKeyJniObject( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPublicKeyBytesFromKeyPair + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPublicKeyBytesFromKeyPair( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: getPrivateKeyBytesFromKeyPair + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_getPrivateKeyBytesFromKeyPair( + JNIEnv*, jclass, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encryptFast + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptFast( + JNIEnv*, jclass, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encryptFastWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptFastWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encrypt + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encrypt( + JNIEnv*, jclass, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: encryptWithoutPrecompute + * Signature: ([B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_encryptWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: decrypt + * Signature: ([BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_decrypt( + JNIEnv*, jclass, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: decryptWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_decryptWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: add + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_add( + JNIEnv*, jclass, jbyteArray, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: addWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_addWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: sub + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_sub( + JNIEnv*, jclass, jbyteArray, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: subWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_subWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: scalaMul + * Signature: ([B[BJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_scalaMul( + JNIEnv*, jclass, jbyteArray, jbyteArray, jlong); + +/* + * Class: com_webank_wedpr_sdk_jni_homo_NativePaillier + * Method: scalMulWithoutPrecompute + * Signature: ([B[B[B)[B + */ +JNIEXPORT jbyteArray JNICALL +Java_com_webank_wedpr_sdk_jni_homo_NativePaillier_scalaMulWithoutPrecompute( + JNIEnv*, jclass, jbyteArray, jbyteArray, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java new file mode 100644 index 00000000..ef18128d --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java @@ -0,0 +1,64 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.demo; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.Utilities; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; + +public class GenerateKeys { + public static void main(String[] args) throws JniException { + // AES-128-CBC + System.out.println("Generate key for AES-128-CBC:"); + byte[] sk = + SymmetricEncryption.generateKey( + SymmetricEncryption.AlgorithmType.AES_128.ordinal(), + SymmetricEncryption.OperationMode.CBC.ordinal()); + System.out.println(Utilities.bytesToHex(sk)); + // AES-256-CBC + System.out.println("Generate key for AES-256-CBC:"); + sk = + SymmetricEncryption.generateKey( + SymmetricEncryption.AlgorithmType.AES_256.ordinal(), + SymmetricEncryption.OperationMode.CBC.ordinal()); + System.out.println(Utilities.bytesToHex(sk)); + // ore + System.out.println("Generate key for ORE:"); + sk = FastOre.generateKey(); + System.out.println(Utilities.bytesToHex(sk)); + // ihc-128 + System.out.println("Generate key for IHC-128:"); + sk = NativeFloatingIhc.generateKey(NativeFloatingIhc.mode.IHC_128.ordinal()); + System.out.println(Utilities.bytesToHex(sk)); + // ihc-256 + System.out.println("Generate key for IHC-256:"); + sk = NativeFloatingIhc.generateKey(NativeFloatingIhc.mode.IHC_256.ordinal()); + System.out.println(Utilities.bytesToHex(sk)); + // paillier + System.out.println("Generate key for paillier-2048:"); + long keyPair = NativePaillier.generateKeyPair(2048); + System.out.println( + "sk: " + + Utilities.bytesToHex( + NativePaillier.getPrivateKeyBytesFromKeyPair(keyPair))); + System.out.println( + "pk: " + + Utilities.bytesToHex( + NativePaillier.getPublicKeyBytesFromKeyPair(keyPair))); + NativePaillier.freeKeyPair(keyPair); + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java new file mode 100644 index 00000000..8df71e0c --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java @@ -0,0 +1,111 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.codec; + +import java.math.BigInteger; +import java.util.Objects; + +// the floating-point-value +public class FloatingPointNumber { + private final BigInteger significant; + private final int exponent; + + FloatingPointNumber(byte[] significantBytes, int exponent) { + this.significant = new BigInteger(significantBytes); + this.exponent = exponent; + } + + public FloatingPointNumber(BigInteger significant, int exponent) { + this.significant = significant; + this.exponent = exponent; + } + + public BigInteger getSignificant() { + return significant; + } + + public byte[] getSignificantBytes() { + return significant.toByteArray(); + } + + public int getExponent() { + return exponent; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FloatingPointNumber that = (FloatingPointNumber) o; + return exponent == that.exponent && Objects.equals(significant, that.significant); + } + + // @Override + public FloatingPointNumber add(FloatingPointNumber other) { + if (this.exponent == other.exponent) { + return new FloatingPointNumber( + this.getSignificant().add(other.getSignificant()), this.exponent); + } + if (this.exponent < other.exponent) { + int diffExp = other.exponent - this.exponent; + BigInteger baseC = BigInteger.valueOf(10).pow(diffExp); + FloatingPointNumber otherNew = + new FloatingPointNumber(other.getSignificant().multiply(baseC), this.exponent); + return new FloatingPointNumber( + otherNew.getSignificant().add(this.getSignificant()), otherNew.exponent); + } + int diffExp = this.exponent - other.exponent; + BigInteger baseC = BigInteger.valueOf(10).pow(diffExp); + FloatingPointNumber thisNew = + new FloatingPointNumber(this.getSignificant().multiply(baseC), other.exponent); + return new FloatingPointNumber( + thisNew.getSignificant().add(other.getSignificant()), thisNew.exponent); + } + + public FloatingPointNumber sub(FloatingPointNumber other) { + if (this.exponent == other.exponent) { + return new FloatingPointNumber( + this.getSignificant().add(other.getSignificant()), this.exponent); + } + if (this.exponent < other.exponent) { + int diffExp = other.exponent - this.exponent; + BigInteger baseC = BigInteger.valueOf(10).pow(diffExp); + FloatingPointNumber otherNew = + new FloatingPointNumber(other.getSignificant().multiply(baseC), this.exponent); + return new FloatingPointNumber( + otherNew.getSignificant().subtract(this.getSignificant()), otherNew.exponent); + } + int diffExp = this.exponent - other.exponent; + BigInteger baseC = BigInteger.valueOf(10).pow(diffExp); + FloatingPointNumber thisNew = + new FloatingPointNumber(this.getSignificant().multiply(baseC), other.exponent); + return new FloatingPointNumber( + thisNew.getSignificant().subtract(other.getSignificant()), thisNew.exponent); + } + + @Override + public int hashCode() { + return Objects.hash(significant, exponent); + } + + @Override + public String toString() { + return "FloatingPointNumber{" + + "significant=" + + significant + + ", exponent=" + + exponent + + '}'; + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java new file mode 100644 index 00000000..6f7388e7 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java @@ -0,0 +1,43 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.codec; + +import java.math.BigDecimal; +import java.math.BigInteger; + +public interface NumberCodec { + FloatingPointNumber encode(BigInteger value) throws NumberCodecException; + + FloatingPointNumber encode(long value) throws NumberCodecException; + + FloatingPointNumber encode(double value) throws NumberCodecException; + + FloatingPointNumber encode(BigDecimal value) throws NumberCodecException; + + BigInteger decodeBigInteger(FloatingPointNumber encodedValue) throws NumberCodecException; + + long decodeLong(FloatingPointNumber encodedValue) throws NumberCodecException; + + double decodeDouble(FloatingPointNumber encodedValue) throws NumberCodecException; + + BigDecimal decodeBigDecimal(FloatingPointNumber encodedValue) throws NumberCodecException; + + public byte[] toBytes(BigDecimal value); + + public byte[] toBytes(Double value); + + double toDouble(byte[] value); + + BigDecimal toBigDecimal(byte[] value); +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java new file mode 100644 index 00000000..a288901d --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java @@ -0,0 +1,21 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.codec; + +public class NumberCodecException extends Exception { + + public NumberCodecException(String message) { + super(message); + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java new file mode 100644 index 00000000..f1841709 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java @@ -0,0 +1,149 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.codec; + +import java.math.BigDecimal; +import java.math.BigInteger; + +public class NumberCodecImpl implements NumberCodec { + private static final int BASE = 10; + /** + * encode the given BigInteger into EncodedValue + * + * @param value + * @return + */ + public FloatingPointNumber encode(BigInteger value) throws NumberCodecException { + if (value == null) { + throw new NumberCodecException("Not support encode null BigInteger!"); + } + return new FloatingPointNumber(value, 0); + } + + /** + * encode the given long value into EncodedValue + * + * @param value + * @return + * @throws NumberCodecException + */ + public FloatingPointNumber encode(long value) throws NumberCodecException { + return encode(BigInteger.valueOf(value)); + } + + /** + * encode the given big-decimal into EncodedValue + * + * @param value + * @return + * @throws NumberCodecException + */ + public FloatingPointNumber encode(BigDecimal value) throws NumberCodecException { + if (value == null) { + throw new NumberCodecException("Not support encode null BigDecimal!"); + } + BigInteger significant; + int exp = -value.scale(); + if (value.scale() > 0) { + significant = value.scaleByPowerOfTen(value.scale()).toBigInteger(); + } else { + significant = value.unscaledValue(); + } + return new FloatingPointNumber(significant, exp); + } + + public FloatingPointNumber encode(double value) throws NumberCodecException { + if (Double.isInfinite(value) || Double.isNaN(value)) { + throw new NumberCodecException("Not support encode null/infinite double!"); + } + // convert double to BigDecimal + BigDecimal decimalValue = BigDecimal.valueOf(value); + return encode(decimalValue); + } + + /** + * decode the encodedValue into BigInteger + * + * @param encodedValue + * @return + * @throws NumberCodecException + */ + public BigInteger decodeBigInteger(FloatingPointNumber encodedValue) + throws NumberCodecException { + return encodedValue.getSignificant(); + } + + /** + * decode the encodedValue into long + * + * @param encodedValue + * @return + * @throws NumberCodecException + */ + public long decodeLong(FloatingPointNumber encodedValue) throws NumberCodecException { + BigInteger value = decodeBigInteger(encodedValue); + // check overflow or not + if (value.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0 + || value.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { + throw new NumberCodecException( + "the Decoded value can't be represented as long, decoded value: " + + value.toString()); + } + return value.longValue(); + } + + /** + * decode the given encodedValue into BigDecimal + * + * @param encodedValue + * @return + * @throws NumberCodecException + */ + public BigDecimal decodeBigDecimal(FloatingPointNumber encodedValue) + throws NumberCodecException { + BigInteger significant = encodedValue.getSignificant(); + return new BigDecimal(significant, -encodedValue.getExponent()); + } + + /** + * decode the encodedValue into double + * + * @param encodedValue + * @return + * @throws NumberCodecException + */ + public double decodeDouble(FloatingPointNumber encodedValue) throws NumberCodecException { + return decodeBigDecimal(encodedValue).doubleValue(); + } + + @Override + public byte[] toBytes(BigDecimal value) { + return value.toPlainString().getBytes(); + } + + @Override + public byte[] toBytes(Double value) { + return String.valueOf(value).getBytes(); + } + + @Override + public double toDouble(byte[] value) { + return Double.parseDouble(new String(value)); + } + + @Override + public BigDecimal toBigDecimal(byte[] value) { + return new BigDecimal(new String(value)); + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java new file mode 100644 index 00000000..2bf87ce9 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java @@ -0,0 +1,20 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.common; + +public class JniException extends Exception { + public JniException(String message) { + super(message); + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java new file mode 100644 index 00000000..e2e7cf55 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java @@ -0,0 +1,278 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.Objects; +import java.util.Scanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper class to load ppc-crypto-sdk jni resources. + * + * @author octopus + */ +public final class JniLibLoader { + private static final Logger logger = LoggerFactory.getLogger(JniLibLoader.class); + + public static final String LINUX = "linux"; + public static final String MAC = "mac"; + public static final String WIN = "win"; + + public static final String NATIVE_RESOURCE_LIB_NAME = "ppc-crypto-sdk-jni"; + public static final String NATIVE_RESOURCE_HOME = "/META-INF/native"; + public static final String NATIVE_WIN_DEPS_DIR = NATIVE_RESOURCE_HOME + "/win/"; + public static final String WIN_DEPS_FILE_LIST = "file.list"; + + public static final String OS_NAME = getOs(); + public static final String ARCH_NAME = getArch(); + + private static File WORKDIR; + + static { + // 1. initialize workdir + + // -Dcom.webank.wedpr.workdir + String workdir = System.getProperty("com.webank.wedpr.workdir"); + if (workdir != null) { + try { + File f = new File(workdir); + f.mkdirs(); + WORKDIR = f; + logger.info("initialize workdir, -Dcom.webank.wedpr.workdir: {}", WORKDIR); + } catch (Exception e) { + logger.debug("initialize workdir, cannot mkdir workdir: {}, e: ", workdir, e); + workdir = null; + } + } + + // default -Djava.io.tmpdir + if (workdir == null) { + workdir = System.getProperty("java.io.tmpdir"); + File f = new File(workdir); + f.mkdirs(); + WORKDIR = f; + + logger.info("initialize workdir, -Djava.io.tmpdirs: {}", WORKDIR); + } + } + + public static String getArch() { + String archName = System.getProperty("os.arch", ""); + if (archName.contains("aarch64") || archName.contains("arm64")) { + return "arm"; + } else { + return ""; + } + } + + public static String getLibName(String baseName) { + String osName = OS_NAME; + String archName = ARCH_NAME; + + logger.debug("osName: {}, archName: {}", osName, archName); + + if (osName.contains(WIN)) { + return baseName + ".dll"; + } else if (osName.contains(LINUX)) { + String arch = getArch(); + if ("arm".equals(arch)) { + return "lib" + baseName + "-aarch64" + ".so"; + } + return "lib" + baseName + ".so"; + } else if (osName.contains(MAC)) { + String arch = getArch(); + if ("arm".equals(arch)) { + return "lib" + baseName + "-aarch64" + ".dylib"; + } + return "lib" + baseName + ".dylib"; + } else { + throw new RuntimeException("unrecognized OS: " + osName); + } + } + + public static String getOs() { + String osName = System.getProperty("os.name").toLowerCase(); + if (osName.contains(WIN)) { + return WIN; + } else if (osName.contains(LINUX)) { + return LINUX; + } else if (osName.contains(MAC)) { + return MAC; + } else { + throw new RuntimeException("unrecognized OS: " + osName); + } + } + + public static void loadJniLibrary() { + String libPath = null; + boolean loadLibFromFsOk = false; + try { + // 1. try to load library from filesystem + // -Dcom.webank.wedpr.jni.library.path + libPath = System.getProperty("com.webank.wedpr.jni.library.path"); + if (libPath != null) { + logger.info( + "try to load library -Dcom.webank.wedpr.jni.library.path, libPath: {}", + libPath); + // try to load library from filesystem first + loadLibraryFromFs( + libPath, NATIVE_RESOURCE_LIB_NAME, new File(libPath).isAbsolute()); + loadLibFromFsOk = true; + } + } catch (Exception e0) { + logger.error("unable to load library from fs, libDir: {}, e: ", libPath, e0); + } + + // 2. try to load library from jar + if (!loadLibFromFsOk) { + try { + logger.info("try to load library from jar"); + if (Objects.equals(getOs(), WIN)) { + loadWinDepsLibraryFromJar(); + } + loadLibraryFromJar(NATIVE_RESOURCE_LIB_NAME); + } catch (Exception e1) { + logger.error("unable to load library from fs, e: ", e1); + throw new UnsatisfiedLinkError(e1.toString()); + } + } + } + + public static void loadLibrary(String lib, boolean absolute) { + if (absolute) { + System.load(lib); + } else { + System.loadLibrary(lib); + } + } + + public static void addLibraryDir(String libraryPath) + throws NoSuchFieldException, IllegalAccessException { + Field userPathsField = ClassLoader.class.getDeclaredField("usr_paths"); + userPathsField.setAccessible(true); + String[] paths = (String[]) userPathsField.get(null); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.length; i++) { + if (libraryPath.equals(paths[i])) { + continue; + } + sb.append(paths[i]).append(File.pathSeparator); + } + sb.append(libraryPath); + System.setProperty("java.library.path", sb.toString()); + logger.info("modify, java.library.path: " + sb.toString()); + final Field sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); + sysPathsField.setAccessible(true); + sysPathsField.set(null, null); + } + + public static void loadLibraryFromFs(String dir, String libName, boolean absolute) { + String resource = dir + "/" + getLibName(libName); + loadLibrary(resource, absolute); + } + + public static void loadLibraryFromJar(String baseName) + throws IOException, NoSuchFieldException, IllegalAccessException { + String libName = getLibName(baseName); + File tempDir = new File(WORKDIR, String.valueOf(System.nanoTime())); + // create temp dir + tempDir.mkdirs(); + File tempFile = new File(tempDir, libName); + logger.info("tempDir: {}, tempFile: {}", tempDir, tempFile); + + String resource = NATIVE_RESOURCE_HOME + "/" + libName; + try (InputStream is = JniLibLoader.class.getResourceAsStream(resource)) { + Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + try { + tempFile.delete(); + } catch (Exception e0) { + logger.debug("delete temp file error, e: ", e0); + } + logger.error("copy dynamic lib from jar failed, resource: {}, e: ", resource, e); + throw e; + } catch (NullPointerException e) { + throw new FileNotFoundException("Cannot found " + resource + " inside the JAR."); + } + + try { + loadLibrary(tempFile.getAbsolutePath(), true); + } catch (Exception e) { + logger.error("loadLibrary error, resource: {}, e: ", resource, e); + throw e; + } finally { + try { + tempFile.delete(); + tempDir.delete(); + logger.debug("remove temp dir and temp file, dir: {}, file: {}", tempDir, tempFile); + } catch (Exception ignored) { + } + } + } + + public static void loadWinDepsLibraryFromJar() throws IOException, NoSuchFieldException { + File tempDir = new File(WORKDIR, String.valueOf(System.nanoTime())); + tempDir.mkdirs(); + + try (InputStream is = + JniLibLoader.class.getResourceAsStream(NATIVE_WIN_DEPS_DIR + WIN_DEPS_FILE_LIST)) { + if (is == null) { + throw new FileNotFoundException( + "Cannot found " + WIN_DEPS_FILE_LIST + " inside the JAR."); + } + try (Scanner scanner = new Scanner(is)) { + while (scanner.hasNext()) { + String lib = scanner.next(); + File targetFile = new File(tempDir, lib); + logger.info( + "loadWinDepsLibraryFromJar tempDir: {}, targetFile: {}", + tempDir, + targetFile); + try (InputStream fis = + JniLibLoader.class.getResourceAsStream(NATIVE_WIN_DEPS_DIR + lib)) { + if (fis == null) { + throw new FileNotFoundException( + "Cannot found " + lib + " inside the JAR."); + } + Files.copy(fis, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + loadLibrary(targetFile.getAbsolutePath(), true); + } catch (IOException e) { + targetFile.delete(); + } catch (Exception e) { + logger.error( + "loadLibrary error, resource: {}, e: ", + targetFile.getAbsolutePath(), + e); + throw e; + } finally { + try { + targetFile.delete(); + tempDir.delete(); + logger.debug("remove temp dir and temp file, {}", targetFile); + } catch (Exception ignored) { + } + } + } + } + } + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java new file mode 100644 index 00000000..75ef1695 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java @@ -0,0 +1,28 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.common; + +public class Utilities { + private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray(); + + public static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 2]; + for (int j = 0; j < bytes.length; j++) { + int v = bytes[j] & 0xFF; + hexChars[j * 2] = HEX_ARRAY[v >>> 4]; + hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F]; + } + return new String(hexChars); + } +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java new file mode 100644 index 00000000..72bb9239 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java @@ -0,0 +1,51 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.crypto; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.JniLibLoader; + +public class FastOre { + static { + JniLibLoader.loadJniLibrary(); + } + + public static native int keyBytes() throws JniException; + + public static native long cipherSize(long plainDataSize, boolean hex) throws JniException; + + public static native long plainSize(long cipherDataSize, boolean hex) throws JniException; + + public static native byte[] generateKey() throws JniException; + + public static native byte[] encrypt4String(byte[] sk, byte[] plainData, boolean hexEncode) + throws JniException; + + public static native byte[] decrypt4String(byte[] sk, byte[] cipherData, boolean hexEncode) + throws JniException; + + public static native byte[] encrypt4Integer(byte[] sk, long plain, boolean hexEncode) + throws JniException; + + public static native long decrypt4Integer(byte[] sk, byte[] cipherData, boolean hexEncode) + throws JniException; + + public static native byte[] encrypt4Float(byte[] sk, byte[] plainData, boolean hexEncode) + throws JniException; + + public static native byte[] decrypt4Float(byte[] sk, byte[] cipherData, boolean hexEncode) + throws JniException; + + public static native int compare(byte[] cipher1, byte[] cipher2) throws JniException; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java new file mode 100644 index 00000000..614a7bfb --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java @@ -0,0 +1,49 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.crypto; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.JniLibLoader; + +public class SymmetricEncryption { + static { + JniLibLoader.loadJniLibrary(); + } + + public enum AlgorithmType { + AES_128, + AES_192, + AES_256, + TrippleDES, + SM4 + } + + public enum OperationMode { + ECB, + CBC, + CFB, + OFB, + CTR + } + + public static native int keyBytes(int algorithmType, int mode) throws JniException; + + public static native byte[] generateKey(int algorithmType, int mode) throws JniException; + + public static native byte[] encrypt( + int algorithmType, int mode, byte[] sk, byte[] iv, byte[] plain) throws JniException; + + public static native byte[] decrypt( + int algorithmType, int mode, byte[] sk, byte[] iv, byte[] cipher) throws JniException; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java new file mode 100644 index 00000000..c6f5c9c3 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java @@ -0,0 +1,31 @@ +package com.webank.wedpr.sdk.jni.homo; + +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.JniLibLoader; + +public class NativeFloatingIhc { + public enum mode { + IHC_128, + IHC_256 + } + + static { + JniLibLoader.loadJniLibrary(); + } + + public static native byte[] generateKey(int mode) throws JniException; + + public static native byte[] encrypt(int mode, byte[] key, byte[] significant, int exponent) + throws JniException; + + public static native FloatingPointNumber decrypt(int mode, byte[] key, byte[] cipher) + throws JniException; + + public static native byte[] add(int mode, byte[] cipher1, byte[] cipher2) throws JniException; + + public static native byte[] sub(int mode, byte[] cipher1, byte[] cipher2) throws JniException; + + public static native byte[] scalaMul(int mode, byte[] significant, int exponent, byte[] cipher) + throws JniException; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java new file mode 100644 index 00000000..4b002980 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java @@ -0,0 +1,60 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.homo; + +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.JniLibLoader; + +public class NativeFloatingPointPaillier { + static { + JniLibLoader.loadJniLibrary(); + } + + public static native int maxCipherBytes(int keyBits) throws JniException; + + public static native byte[] encryptFast(byte[] significant, int exponent, long keyPair) + throws JniException; + + public static native byte[] encryptFastWithoutPrecompute( + byte[] significant, int exponent, byte[] sk, byte[] pk) throws JniException; + + public static native byte[] encrypt(byte[] significant, int exponent, long publicKey) + throws JniException; + + public static native byte[] encryptWithoutPrecompute( + byte[] significant, int exponent, byte[] pk) throws JniException; + + public static native FloatingPointNumber decrypt(byte[] cipher, long keyPair) + throws JniException; + + public static native FloatingPointNumber decryptWithoutPrecompute( + byte[] cipher, byte[] sk, byte[] pk) throws JniException; + + public static native byte[] add(byte[] c1, byte[] c2, long publicKey) throws JniException; + + public static native byte[] addWithoutPrecompute(byte[] c1, byte[] c2, byte[] pk) + throws JniException; + + public static native byte[] sub(byte[] c1, byte[] c2, long publicKey) throws JniException; + + public static native byte[] subWithoutPrecompute(byte[] c1, byte[] c2, byte[] pkBytes) + throws JniException; + + public static native byte[] scalaMul( + byte[] significant, int exponent, byte[] cipher, long publicKey) throws JniException; + + public static native byte[] scalaMulWithoutPrecompute( + byte[] significant, int exponent, byte[] cipher, byte[] pkBytes) throws JniException; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java new file mode 100644 index 00000000..25877e6e --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java @@ -0,0 +1,241 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.homo; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.JniLibLoader; + +public class NativePaillier { + static { + JniLibLoader.loadJniLibrary(); + } + + /** + * get the max length of the public key with given keyBits + * + * @param keyBits the keyBits, e.g. 1024/2048 + * @return the max length of the public key + * @throws JniException + */ + public static native int maxPublicKeyBytes(int keyBits) throws JniException; + + /** + * get the max length of the private key with given keyBits + * + * @param keyBits the keyBits, e.g. 1024/2048 + * @return the max length of the private key + * @throws JniException + */ + public static native int maxPrivateKeyBytes(int keyBits) throws JniException; + + /** + * get the max length of the cipher with given keyBits + * + * @param keyBits the keyBits, e.g. 1024/2048 + * @return the max length of the cipher + * @throws JniException + */ + public static native int maxCipherBytes(int keyBits) throws JniException; + + /** + * get the keyBits according to the given keyPair + * + * @param keyPair + * @return + * @throws JniException + */ + public static native int keyBitsFromKeyPair(long keyPair) throws JniException; + + /** + * get the keyBits according to the given public key + * + * @param publicKey + * @return + * @throws JniException + */ + public static native int keyBitsFromPublicKey(long publicKey) throws JniException; + + /** + * generate the paillier KeyPair + * + * @param keyBits the keyLength + * @return pointer to the generated paillier key-pair + * @throws JniException + */ + public static native long generateKeyPair(int keyBits) throws JniException; + + /** + * load the encoded privateKey and publicKey into key-pair object + * + * @param privateKeyBytes the encoded privateKey + * @param publicKeyBytes the encoded publicKey + * @return + * @throws JniException + */ + public static native long loadKeyPair(byte[] privateKeyBytes, byte[] publicKeyBytes) + throws JniException; + + /** + * free the allocated keypair + * + * @param keyPairPointer + * @return + */ + public static native void freeKeyPair(long keyPairPointer) throws JniException; + + /** + * load the publicKey object from the encoded publicKey + * + * @param publicKeyBytes the encoded publicKey + * @return the pointer to the publicKey object + * @throws JniException + */ + public static native long loadPublicKey(byte[] publicKeyBytes) throws JniException; + + /** + * free the public key object + * + * @param publicKey + * @throws JniException + */ + public static native void freePublicKey(long publicKey) throws JniException; + + /** + * load the private key object from the encoded privateKey + * + * @param privateKeyBytes + * @return + * @throws JniException + */ + public static native long loadPrivateKey(byte[] privateKeyBytes) throws JniException; + + /** + * free the private key object + * + * @param privateKey + * @throws JniException + */ + public static native void freePrivateKey(long privateKey) throws JniException; + + /** + * get the public key jni object according to the keypair + * + * @param keypair the keypair jni object + * @return the public key jni object + * @throws JniException + */ + public static native long getPublicKeyJniObject(long keypair) throws JniException; + + /** + * get the encoded publicKey from given keyPair object + * + * @param keyPair the pointer to the keyPair object + * @return + * @throws JniException + */ + public static native byte[] getPublicKeyBytesFromKeyPair(long keyPair) throws JniException; + + /** + * get the encoded privateKey from the given keyPair object + * + * @param keyPair the pointer to the keyPair object + * @return + * @throws JniException + */ + public static native byte[] getPrivateKeyBytesFromKeyPair(long keyPair) throws JniException; + + /** + * encrypt the value into paillier cipher using given keyPair(using CRT optimization) + * + * @param value the value to be encrypted + * @param keyPair the keyPair used to encrypt + * @return the cipher + * @throws JniException + */ + public static native byte[] encryptFast(byte[] value, long keyPair) throws JniException; + + public static native byte[] encryptFastWithoutPrecompute( + byte[] value, byte[] privateKey, byte[] publicKey) throws JniException; + + /** + * encrypt the value into paillier cipher using given keyPair(without CRT optimization) + * + * @param value the value to be encrypted + * @param publicKey the publicKey used to encrypt + * @return the cipher + * @throws JniException + */ + public static native byte[] encrypt(byte[] value, long publicKey) throws JniException; + + public static native byte[] encryptWithoutPrecompute(byte[] value, byte[] publicKey) + throws JniException; + + /** + * decrypt the cipher using given keyPair + * + * @param cipher the cipher to be decrypted + * @param keyPair the keyPair used to decrypt + * @return the decrypted result + * @throws JniException + */ + public static native byte[] decrypt(byte[] cipher, long keyPair) throws JniException; + + public static native byte[] decryptWithoutPrecompute(byte[] cipher, byte[] sk, byte[] pk) + throws JniException; + + /** + * ciphertext space addition, namely: cipher1 + cipher2 + * + * @param cipher1 + * @param cipher2 + * @param publicKey + * @return the addition result + * @throws JniException + */ + public static native byte[] add(byte[] cipher1, byte[] cipher2, long publicKey) + throws JniException; + + public static native byte[] addWithoutPrecompute(byte[] cipher1, byte[] cipher2, byte[] pk) + throws JniException; + + /** + * ciphertext space subtraction, namely: cipher1 - cipher2 + * + * @param cipher1 + * @param cipher2 + * @param publicKey + * @return the subtraction result + * @throws JniException + */ + public static native byte[] sub(byte[] cipher1, byte[] cipher2, long publicKey) + throws JniException; + + public static native byte[] subWithoutPrecompute(byte[] cipher1, byte[] cipher2, byte[] pk) + throws JniException; + + /** + * ciphertext space scala-multiply, namely: v * cipher + * + * @param v the scala-number + * @param cipher + * @param publicKey + * @return the scala-multiply result + * @throws JniException + */ + public static native byte[] scalaMul(byte[] v, byte[] cipher, long publicKey) + throws JniException; + + public static native byte[] scalaMulWithoutPrecompute(byte[] v, byte[] cipher, byte[] pk) + throws JniException; +} diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list new file mode 100644 index 00000000..2fe00a0f --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list @@ -0,0 +1,7 @@ +api-ms-win-crt-heap-l1-1-0.dll +api-ms-win-crt-stdio-l1-1-0.dll +api-ms-win-crt-utility-l1-1-0.dll +api-ms-win-crt-environment-l1-1-0.dll +api-ms-win-crt-filesystem-l1-1-0.dll +api-ms-win-crt-convert-l1-1-0.dll +vcruntime140_1.dll \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java new file mode 100644 index 00000000..272fb1d0 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java @@ -0,0 +1,86 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.test; + +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecException; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import org.junit.Assert; +import org.junit.Test; +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; + + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; + +public class NativeFloatingIhcTest +{ + private void testFloatingIhcBigDecimalCase(int mode, BigDecimal m1, BigDecimal m2, BigDecimal v) throws JniException, NumberCodecException { + NativeFloatingIhc ihc = new NativeFloatingIhc(); + // generate key + byte[] key = ihc.generateKey(mode); + NumberCodec codec = new NumberCodecImpl(); + FloatingPointNumber fp1 = codec.encode(m1); + FloatingPointNumber fp2 = codec.encode(m2); + FloatingPointNumber fv = codec.encode(v); + // encrypt + byte[] c1 = ihc.encrypt(mode, key, fp1.getSignificantBytes(),fp1.getExponent()); + // decrypt + FloatingPointNumber resultValue = ihc.decrypt(mode, key, c1); + System.out.println("###### resultValue: " + resultValue.getSignificant()); + System.out.println("###### resultValue exp: " + resultValue.getExponent()); + Assert.assertTrue(codec.decodeBigDecimal(resultValue).equals(m1)); + // encrypt c2 + byte[] c2 = ihc.encrypt(mode, key, fp2.getSignificantBytes(),fp2.getExponent()); + + /////// c1 + c2 + byte[] addResult = ihc.add(mode, c1, c2); + // decrypt and check + resultValue = ihc.decrypt(mode, key, addResult); + BigDecimal expectedAddResult = m1.add(m2); + Assert.assertTrue(expectedAddResult.equals(codec.decodeBigDecimal(resultValue))); + + /////// c1 - c2 + byte[] subResult = ihc.sub(mode, c1, c2); + // decrypt and check + resultValue = ihc.decrypt(mode, key, subResult); + BigDecimal expectedSubResult = m1.subtract(m2); + Assert.assertTrue(expectedSubResult.equals(codec.decodeBigDecimal(resultValue))); + + //// v * c1 + byte[] mulResult = ihc.scalaMul(mode, fv.getSignificantBytes(), fv.getExponent(), c1); + // decrypt and check + resultValue = ihc.decrypt(mode, key, mulResult); + BigDecimal expectedMulResult = v.multiply(m1); + System.out.println("###### "); + Assert.assertTrue(expectedMulResult.equals(codec.decodeBigDecimal(resultValue))); + + // invalid case + String invalidCipher = "invalidCipher"; + Assert.assertThrows(JniException.class, ()->ihc.decrypt(mode, key, invalidCipher.getBytes())); + } + +@Test +public void testFloatingIhc() throws JniException, NumberCodecException { + System.out.println("##### testFloatingIhc: "); + + BigDecimal m1 = BigDecimal.valueOf(234324.02343243423432434); + BigDecimal m2 = BigDecimal.valueOf(-324234.0000123423432); + BigDecimal v = BigDecimal.valueOf(-1232.000000234); + testFloatingIhcBigDecimalCase(NativeFloatingIhc.mode.IHC_128.ordinal(), m1, m2, v); +} +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java new file mode 100644 index 00000000..08ef13de --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java @@ -0,0 +1,237 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.test; + +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecException; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; + +public class NativeFloatingPointPaillierTest +{ + private NumberCodec codec = new NumberCodecImpl(); + private void intCase(BigInteger m1, BigInteger m2, BigInteger v, long keypair, long publicKey) throws NumberCodecException, JniException { + // encrypt + FloatingPointNumber m1Fp = codec.encode(m1); + byte[] c1 = NativeFloatingPointPaillier.encrypt(m1Fp.getSignificantBytes(), m1Fp.getExponent(), publicKey); + byte[] pkBytes = NativePaillier.getPublicKeyBytesFromKeyPair(keypair); + byte[] skBytes = NativePaillier.getPrivateKeyBytesFromKeyPair(keypair); + byte[] tmp = NativeFloatingPointPaillier.encryptWithoutPrecompute(m1Fp.getSignificantBytes(), m1Fp.getExponent(), pkBytes); + + FloatingPointNumber m2Fp = codec.encode(m2); + byte[] c2 = NativeFloatingPointPaillier.encryptFast(m2Fp.getSignificantBytes(), m2Fp.getExponent(), keypair); + // decrypt and check + FloatingPointNumber decryptedC1 = NativeFloatingPointPaillier.decrypt(c1, keypair); + FloatingPointNumber tmpResult = NativeFloatingPointPaillier.decryptWithoutPrecompute(tmp, skBytes, pkBytes); + Assert.assertTrue(codec.decodeBigInteger(decryptedC1).compareTo(m1) == 0); + Assert.assertTrue(codec.decodeBigInteger(tmpResult).compareTo(m1) == 0); + + FloatingPointNumber decryptedC2 = NativeFloatingPointPaillier.decrypt(c2, keypair); + Assert.assertTrue(codec.decodeBigInteger(decryptedC2).compareTo(m2) == 0); + + // add + byte[] addCipher = NativeFloatingPointPaillier.add(c1, c2, publicKey); + tmp = NativeFloatingPointPaillier.addWithoutPrecompute(c1, c2, pkBytes); + FloatingPointNumber addResult = NativeFloatingPointPaillier.decrypt(addCipher, keypair); + tmpResult = NativeFloatingPointPaillier.decryptWithoutPrecompute(tmp, skBytes, pkBytes); + + BigInteger expectedAddResult = m1.add(m2); + Assert.assertTrue(codec.decodeBigInteger(addResult).compareTo(expectedAddResult) == 0); + Assert.assertTrue(codec.decodeBigInteger(tmpResult).compareTo(expectedAddResult) == 0); + + // sub + byte[] subCipher = NativeFloatingPointPaillier.sub(c1, c2, publicKey); + tmp = NativeFloatingPointPaillier.subWithoutPrecompute(c1, c2, pkBytes); + FloatingPointNumber subResult = NativeFloatingPointPaillier.decrypt(subCipher, keypair); + tmpResult = NativeFloatingPointPaillier.decryptWithoutPrecompute(subCipher, skBytes, pkBytes); + BigInteger expectedSubResult = m1.subtract(m2); + Assert.assertTrue(codec.decodeBigInteger(subResult).compareTo(expectedSubResult) == 0); + Assert.assertTrue(codec.decodeBigInteger(tmpResult).compareTo(expectedSubResult) == 0); + + // scalaMul + FloatingPointNumber vFp = codec.encode(v); + byte[] mulCipher = NativeFloatingPointPaillier.scalaMul(vFp.getSignificantBytes(), vFp.getExponent(), c1, publicKey); + tmp = NativeFloatingPointPaillier.scalaMulWithoutPrecompute(vFp.getSignificantBytes(), vFp.getExponent(), c1, pkBytes); + FloatingPointNumber mulResult = NativeFloatingPointPaillier.decrypt(mulCipher, keypair); + tmpResult = NativeFloatingPointPaillier.decryptWithoutPrecompute(mulCipher, skBytes, pkBytes); + BigInteger expectedMulResult = v.multiply(m1); + Assert.assertTrue(codec.decodeBigInteger(mulResult).compareTo(expectedMulResult) == 0); + Assert.assertTrue(codec.decodeBigInteger(tmpResult).compareTo(expectedMulResult) == 0); + } + + private void doubleCase(double m1, double m2, double v, long keypair, long publicKey) throws NumberCodecException, JniException { + // encrypt + FloatingPointNumber m1Fp = codec.encode(m1); + System.out.println("#### m1Fp: " + m1Fp.toString()); + byte[] c1 = NativeFloatingPointPaillier.encrypt(m1Fp.getSignificantBytes(), m1Fp.getExponent(), publicKey); + + FloatingPointNumber m2Fp = codec.encode(m2); + System.out.println("#### m2Fp: " + m2Fp.toString()); + byte[] c2 = NativeFloatingPointPaillier.encryptFast(m2Fp.getSignificantBytes(), m2Fp.getExponent(), keypair); + // decrypt and check + FloatingPointNumber decryptedC1 = NativeFloatingPointPaillier.decrypt(c1, keypair); + Assert.assertTrue(Double.compare(codec.decodeDouble(decryptedC1), m1) == 0); + + FloatingPointNumber decryptedC2 = NativeFloatingPointPaillier.decrypt(c2, keypair); + System.out.println("#### m2: " + m2); + System.out.println("#### decryptedC2: " + codec.decodeDouble(decryptedC2)); + Assert.assertTrue(Double.compare(codec.decodeDouble(decryptedC2), m2) == 0); + + // add + byte[] addCipher = NativeFloatingPointPaillier.add(c1, c2, publicKey); + FloatingPointNumber addResult = NativeFloatingPointPaillier.decrypt(addCipher, keypair); + BigDecimal expectedAddResult = BigDecimal.valueOf(m1).add(BigDecimal.valueOf(m2)); + System.out.println("#### addResult: " + addResult.toString()); + System.out.println("#### expectedAddResult: " + expectedAddResult); + System.out.println("m1: " + m1); + System.out.println("m2: " + m2); + Assert.assertTrue(expectedAddResult.compareTo(BigDecimal.valueOf(codec.decodeDouble(addResult))) == 0); + + // sub + byte[] subCipher = NativeFloatingPointPaillier.sub(c1, c2, publicKey); + FloatingPointNumber subResult = NativeFloatingPointPaillier.decrypt(subCipher, keypair); + BigDecimal expectedSubResult = BigDecimal.valueOf(m1).subtract(BigDecimal.valueOf(m2)); + Assert.assertTrue(expectedSubResult.compareTo(BigDecimal.valueOf(codec.decodeDouble(subResult))) == 0); + + // scalaMul + FloatingPointNumber vFp = codec.encode(v); + byte[] mulCipher = NativeFloatingPointPaillier.scalaMul(vFp.getSignificantBytes(), vFp.getExponent(), c1, publicKey); + FloatingPointNumber mulResult = NativeFloatingPointPaillier.decrypt(mulCipher, keypair); + BigDecimal expectedMulResult = BigDecimal.valueOf(v).multiply(BigDecimal.valueOf(m1)); + // Note: BigDecimal not loss the precision + Assert.assertTrue(expectedMulResult.compareTo(codec.decodeBigDecimal(mulResult)) == 0); + } + + private void decimalCase(BigDecimal m1, BigDecimal m2, BigDecimal v, long keypair, long publicKey) throws NumberCodecException, JniException { + // encrypt + FloatingPointNumber m1Fp = codec.encode(m1); + byte[] c1 = NativeFloatingPointPaillier.encrypt(m1Fp.getSignificantBytes(), m1Fp.getExponent(), publicKey); + + FloatingPointNumber m2Fp = codec.encode(m2); + byte[] c2 = NativeFloatingPointPaillier.encryptFast(m2Fp.getSignificantBytes(), m2Fp.getExponent(), keypair); + // decrypt and check + FloatingPointNumber decryptedC1 = NativeFloatingPointPaillier.decrypt(c1, keypair); + Assert.assertTrue(codec.decodeBigDecimal(decryptedC1).compareTo(m1) == 0); + + FloatingPointNumber decryptedC2 = NativeFloatingPointPaillier.decrypt(c2, keypair); + Assert.assertTrue(codec.decodeBigDecimal(decryptedC2).compareTo(m2) == 0); + + // add + byte[] addCipher = NativeFloatingPointPaillier.add(c1, c2, publicKey); + FloatingPointNumber addResult = NativeFloatingPointPaillier.decrypt(addCipher, keypair); + BigDecimal expectedAddResult = m1.add(m2); + Assert.assertTrue(codec.decodeBigDecimal(addResult).compareTo(expectedAddResult) == 0); + + // sub + byte[] subCipher = NativeFloatingPointPaillier.sub(c1, c2, publicKey); + FloatingPointNumber subResult = NativeFloatingPointPaillier.decrypt(subCipher, keypair); + BigDecimal expectedSubResult = m1.subtract(m2); + Assert.assertTrue(codec.decodeBigDecimal(subResult).compareTo(expectedSubResult) == 0); + + // scalaMul + FloatingPointNumber vFp = codec.encode(v); + byte[] mulCipher = NativeFloatingPointPaillier.scalaMul(vFp.getSignificantBytes(), vFp.getExponent(), c1, publicKey); + FloatingPointNumber mulResult = NativeFloatingPointPaillier.decrypt(mulCipher, keypair); + BigDecimal expectedMulResult = v.multiply(m1); + Assert.assertTrue(codec.decodeBigDecimal(mulResult).compareTo(expectedMulResult) == 0); + } + + @Test + public void testIntCase() throws JniException, NumberCodecException { + /// the int case + BigInteger m1 = BigInteger.valueOf(234234234); + BigInteger m2 = BigInteger.valueOf(456566423); + BigInteger v = BigInteger.valueOf(123); + int keyBits = 2048; + long keypair = NativePaillier.generateKeyPair(keyBits); + long publicKey = NativePaillier.getPublicKeyJniObject(keypair); + + // the positive case + intCase(m1, m2, v, keypair, publicKey); + // the negative case + m2 = BigInteger.valueOf(-456566423); + intCase(m1, m2, v, keypair, publicKey); + m1 = BigInteger.valueOf(-234234234); + intCase(m1, m2, v, keypair,publicKey); + + // zero caase + m1 = BigInteger.valueOf(0); + m2 = BigInteger.valueOf(0); + intCase(m1, m2, v, keypair,publicKey); + } + + @Test + public void testDoubleCase() throws JniException, NumberCodecException { + int keyBits = 2048; + long keypair = NativePaillier.generateKeyPair(keyBits); + long publicKey = NativePaillier.getPublicKeyJniObject(keypair); + + /// positive case + double m1 = 45646.00; + double m2 = 0.454354; + double v = 2.00; + doubleCase(m1, m2, v, keypair, publicKey); + + // negative case + m1 = -234324.0234324; + m2 = -0.34543; + v = -23432.001; + doubleCase(m1, m2, v, keypair, publicKey); + + // zero case + //m1 = 0.000; + m2 = 0.00; + doubleCase(m1, m2, v, keypair, publicKey); + } + + @Test + public void testBigDecimalCase() throws JniException, NumberCodecException { + int keyBits = 2048; + long keypair = NativePaillier.generateKeyPair(keyBits); + long publicKey = NativePaillier.getPublicKeyJniObject(keypair); + + BigDecimal m1 = BigDecimal.valueOf(12321231.2342343); + BigDecimal m2 = BigDecimal.valueOf(0.0023434); + BigDecimal v = BigDecimal.valueOf(123.3); + decimalCase(m1, m2, v, keypair, publicKey); + m1 = new BigDecimal("2343243223423423423423.798034320080"); + m2 = new BigDecimal("956456547424236462412342345.0023423"); + decimalCase(m1, m2, v, keypair, publicKey); + + // the negative case + m1 = BigDecimal.valueOf(234324.023432434); + m2 = BigDecimal.valueOf(-324234.00001); + v = BigDecimal.valueOf(-1232.00); + decimalCase(m1, m2, v, keypair, publicKey); + v = BigDecimal.valueOf(0.23434); + m1 = BigDecimal.valueOf(-43543.000); + decimalCase(m1, m2, v, keypair, publicKey); + m2 = BigDecimal.valueOf(-2343.00); + decimalCase(m1, m2, v, keypair, publicKey); + + // zero case + m1 = BigDecimal.valueOf(0); + m2 = BigDecimal.valueOf(0); + decimalCase(m1, m2, v, keypair, publicKey); + } + +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java new file mode 100644 index 00000000..08248877 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java @@ -0,0 +1,144 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.test; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.common.Utilities; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.Arrays; + +public class NativePaillierTest +{ + private void testPaillierImpl(int keyBits, BigInteger m1, BigInteger m2, BigInteger v) throws JniException { + NativePaillier paillier = new NativePaillier(); + // generate keypair + long keyPair = paillier.generateKeyPair(keyBits); + // get the public key from keyPair + byte[] pk = paillier.getPublicKeyBytesFromKeyPair(keyPair); + byte[] sk = paillier.getPrivateKeyBytesFromKeyPair(keyPair); + System.out.println("#### pk: " + Utilities.bytesToHex(pk)); + System.out.println("##### sk: " + Utilities.bytesToHex(sk)); + + // load the public key + long keyPair2 = paillier.loadKeyPair(sk, pk); + + // encrypt + byte[] m1Bytes = m1.toByteArray(); + System.out.println("###### m1: " + Arrays.toString(m1Bytes)); + byte[] c1 = paillier.encryptFast(m1Bytes, keyPair); + // encryptFastWithoutPrecompute + byte[] tmp = paillier.encryptFastWithoutPrecompute(m1Bytes, sk, pk); + byte[] tmpResult = paillier.decryptWithoutPrecompute(tmp, sk, pk); + + // decrypt + byte[] resultValue = paillier.decrypt(c1, keyPair); + BigInteger resultV = new BigInteger(resultValue); + Assert.assertTrue(Arrays.equals(tmpResult, resultValue)); + tmpResult = paillier.decryptWithoutPrecompute(c1, sk, pk); + Assert.assertTrue(Arrays.equals(tmpResult, resultValue)); + + System.out.println("#### resultValue len: " + resultValue.length + ", resultValue: " + Arrays.toString(resultValue)); + System.out.println("### resultValue: " + resultV); + System.out.println("#### m1: " + m1); + + + byte[] c2 = paillier.encryptFast(m2.toByteArray(), keyPair2); + System.out.println("#### c1 len: " + c1.length + ", array: " + Arrays.toString(c1)); + System.out.println("#### c2 len: " + c2.length + ", array: " + Arrays.toString(c2)); + + /////// c1 + c2 + long pkObj = paillier.loadPublicKey(pk); + long pkObj2 = paillier.getPublicKeyJniObject(keyPair); + + byte[] result = paillier.add(c1, c2, pkObj); + tmp = paillier.addWithoutPrecompute(c1, c2, pk); + System.out.println("#### result len: " + result.length + ", result: " + Arrays.toString(result)); + // decrypt and check + resultValue = paillier.decrypt(result, keyPair); + tmpResult = paillier.decryptWithoutPrecompute(tmp, sk, pk); + Assert.assertTrue(Arrays.equals(resultValue, tmpResult)); + System.out.println("#### resultValue len: " + resultValue.length + ", resultValue: " + Arrays.toString(resultValue)); + BigInteger addResult = new BigInteger(resultValue); + BigInteger expectedAddResult = m1.add(m2); + System.out.println("####### m1: " + m1); + System.out.println("####### m2: " + m2); + System.out.println("####### expectedAddResult: " + expectedAddResult.toString()); + System.out.println("####### addResult: " + addResult.toString()); + Assert.assertTrue(expectedAddResult.equals(addResult)); + + /////// c1 - c2 + result = paillier.sub(c1, c2, pkObj2); + tmp = paillier.subWithoutPrecompute(c1, c2, pk); + // decrypt and check + resultValue = paillier.decrypt(result, keyPair); + tmpResult = paillier.decryptWithoutPrecompute(result, sk, pk); + Assert.assertTrue(Arrays.equals(resultValue, tmpResult)); + BigInteger subResult = new BigInteger(resultValue); + BigInteger expectedSubResult = m1.subtract(m2); + Assert.assertTrue(expectedSubResult.equals(subResult)); + + //// v * c1 + result = paillier.scalaMul(v.toByteArray(), c1, pkObj); + tmp = paillier.scalaMulWithoutPrecompute(v.toByteArray(), c1, pk); + // decrypt and check + resultValue = paillier.decrypt(result, keyPair); + tmpResult = paillier.decryptWithoutPrecompute(tmp, sk, pk); + Assert.assertTrue(Arrays.equals(resultValue, tmpResult)); + BigInteger mulResult = new BigInteger(resultValue); + BigInteger expectedMulResult = v.multiply(m1); + Assert.assertTrue(mulResult.equals(expectedMulResult)); + + /// exception case: + try { + c1[1] += 90; + result = paillier.add(c1, c2, pkObj); + resultValue = paillier.decrypt(result, keyPair); + addResult = new BigInteger(resultValue); + Assert.assertFalse(expectedAddResult.equals(addResult)); + }catch(Exception e) + { + System.out.println("#### exception case: " + e.getMessage()); + } + + // call loadPrivateKey + long skObj = paillier.loadPrivateKey(sk); + long skObj2 = paillier.loadPrivateKey(sk); + + NativePaillier.freeKeyPair(keyPair); + NativePaillier.freeKeyPair(keyPair2); + + // Note: pkObj2 is loaded from keypair, it has been released after keypair has been released + NativePaillier.freePublicKey(pkObj); + + NativePaillier.freePrivateKey(skObj); + NativePaillier.freePrivateKey(skObj2); + } + +@Test +public void testPaillier() throws JniException { + // the positive case + BigInteger m1 = BigInteger.valueOf(34234234); + BigInteger m2 = BigInteger.valueOf(23423423); + BigInteger v = BigInteger.valueOf(23423); + testPaillierImpl(2048, m1, m2, v); + + // the negative case + m1 = BigInteger.valueOf(-34234234); + testPaillierImpl(2048, m1, m2, v); +} +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java new file mode 100644 index 00000000..790c6a75 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java @@ -0,0 +1,86 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.test; + +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecException; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; + +public class NumberCodecTest +{ + private final NumberCodec codec = new NumberCodecImpl(); + + private void checkBigIntegerEncodeDecode(BigInteger value) throws NumberCodecException { + FloatingPointNumber encodedV = codec.encode(value); + BigInteger decodedV = codec.decodeBigInteger(encodedV); + Assert.assertTrue(decodedV.compareTo(value) == 0); + } + + private void checkLongEncodeDecode(long value) throws NumberCodecException { + FloatingPointNumber encodedV = codec.encode(value); + long decodedV = codec.decodeLong(encodedV); + Assert.assertTrue(decodedV == value); + } + + private void checkDoubleEncodeDecode(double value) throws NumberCodecException + { + FloatingPointNumber encodedV = codec.encode(value); + double decodedV = codec.decodeDouble(encodedV); + Assert.assertTrue(Double.compare(value, decodedV) == 0); + } + + private void checkBigDecimalEncodeDecode(BigDecimal value) throws NumberCodecException + { + FloatingPointNumber encodedV = codec.encode(value); + BigDecimal decodedV = codec.decodeBigDecimal(encodedV); + Assert.assertTrue(decodedV.compareTo(value) == 0); + } + + @Test + public void testNumberCodec() throws NumberCodecException { + + //// check BigInteger encode decode + BigInteger value = BigInteger.valueOf(123432432); + checkBigIntegerEncodeDecode(value); + // negative case + value = BigInteger.valueOf(-123432432); + checkBigIntegerEncodeDecode(value); + + /// check long encode/decode + long longV = 234232454; + checkLongEncodeDecode(longV); + /// the negative case + longV = -234232454; + checkLongEncodeDecode(longV); + + /// check double + double doubleV = 11341.234234234; + checkDoubleEncodeDecode(doubleV); + // negative case + doubleV = -1234.234234234; + checkDoubleEncodeDecode(doubleV); + + /// check BigDecimal + BigDecimal decimalV = new BigDecimal("2342343243243.2344235235"); + checkBigDecimalEncodeDecode(decimalV); + decimalV = new BigDecimal("-342343243243.2344235235"); + checkBigDecimalEncodeDecode(decimalV); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java new file mode 100644 index 00000000..48c3ae61 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java @@ -0,0 +1,194 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.test; +import com.webank.wedpr.sdk.jni.codec.NumberCodecException; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; +import org.junit.Assert; +import org.junit.Test; +import com.webank.wedpr.sdk.jni.common.JniException; + +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class SymmetricEncryptionTest +{ + private void checkEncDec(int algorithm, int mode, byte[] iv, byte[] plain) throws JniException + { + byte[] key = SymmetricEncryption.generateKey(algorithm, mode); + //System.out.println("key: "); + //String key = "123"; + // encrypt + byte[] cipher = SymmetricEncryption.encrypt(algorithm, mode, key, iv, plain); + byte[] decPlain = SymmetricEncryption.decrypt(algorithm, mode, key, iv, cipher); + // check + System.out.println("plain: " + new String(plain)); + System.out.println("decPlain: " + new String(decPlain)); + Assert.assertTrue(Arrays.toString(plain).equals(Arrays.toString(decPlain))); + } + private void testEncDecCases(int algorithm, int mode, byte[] iv) throws JniException + { + String plain = "231234324234"; + byte[] plainBytes = plain.getBytes(); + System.out.println("#### plainBytes: " + Arrays.toString(plainBytes)); + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "@3423sf12!"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "a1b2c3d4e5"; + + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "zhongwenzhongw中文"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "中文中文謇謇謇齉齉@#"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "鬱"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "齉"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "躞"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "....."; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "测试特殊符号*&（……%￥ 123abc 1 12345678 #$%#&$#$*^&(#%$@#()_()"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + plain = "测试生僻字謇鬱齉躞*&（……%￥ 123abc 12345678 123456782345678 12345678 12345678 12345678 12345678 12345678 12345678 #$%#&$#$*^&(#%$@#()_()"; + checkEncDec(algorithm, mode, iv, plain.getBytes()); + } + + private void testEncDescImpl(byte[] iv) throws JniException + { + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), iv); + + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), iv); + + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), iv); + + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), iv); + + testEncDecCases(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), iv); + + testEncDecCases(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), iv); + testEncDecCases(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), iv); + } + @Test + public void testSymmetricEncDec() throws JniException, UnsupportedEncodingException { + String iv = "123"; + testEncDescImpl(iv.getBytes(StandardCharsets.UTF_8)); + + iv = "234324234"; + testEncDescImpl(iv.getBytes(StandardCharsets.UTF_8)); + } + + private void testFastOre4StringImpl(String plain, boolean hex)throws JniException, UnsupportedEncodingException + { + byte[] sk = FastOre.generateKey(); + // encrypt + byte[] cipher = FastOre.encrypt4String(sk, plain.getBytes(), hex); + // decrypt + byte[] decryptedValue = FastOre.decrypt4String(sk, cipher, hex); + Assert.assertEquals(plain, new String(decryptedValue)); + } + + private void testFastOre4NumberImpl(long plain, boolean hex)throws JniException, UnsupportedEncodingException + { + byte[] sk = FastOre.generateKey(); + // encrypt + byte[] cipher = FastOre.encrypt4Integer(sk, plain, hex); + // decrypt + long decryptedValue = FastOre.decrypt4Integer(sk, cipher, hex); + Assert.assertEquals(plain, decryptedValue); + } + + private void testFastOre4Double(double plain, boolean hex) throws JniException, NumberCodecException { + NumberCodecImpl numberCodec = new NumberCodecImpl(); + byte[] sk = FastOre.generateKey(); + // encrypt + byte[] cipher = FastOre.encrypt4Float(sk, numberCodec.toBytes(plain), hex); + // decrypt + byte[] decryptedValue = FastOre.decrypt4Float(sk, cipher, hex); + + double EPSILON = 1e-10; + double decryptedPlain = numberCodec.toDouble(decryptedValue); + Assert.assertTrue(Math.abs(decryptedPlain - plain) < EPSILON); + } + + private void testFastOre4Decimal(BigDecimal plain, boolean hex) throws JniException, NumberCodecException { + NumberCodecImpl numberCodec = new NumberCodecImpl(); + byte[] sk = FastOre.generateKey(); + // encrypt + byte[] cipher = FastOre.encrypt4Float(sk, numberCodec.toBytes(plain), hex); + // decrypt + byte[] decryptedValue = FastOre.decrypt4Float(sk, cipher, hex); + Assert.assertEquals(0, numberCodec.toBigDecimal(decryptedValue).compareTo(plain)); + } + + @Test + public void testOre() throws JniException, UnsupportedEncodingException, NumberCodecException { + String plain = "abcdwer"; + testFastOre4StringImpl(plain, true); + testFastOre4StringImpl(plain, false); + plain = "中文中文"; + testFastOre4StringImpl(plain, true); + testFastOre4StringImpl(plain, false); + plain = "测试生僻字謇鬱齉躞*&（……%￥"; + testFastOre4StringImpl(plain, true); + testFastOre4StringImpl(plain, false); + + long number = 0; + testFastOre4NumberImpl(number, true); + testFastOre4NumberImpl(number, false); + + for(int i = 0; i < 10000; i++) + { + number = -234567; + testFastOre4NumberImpl(number * i, true); + testFastOre4NumberImpl(number * i, false); + number = 876543; + testFastOre4NumberImpl(number * i, true); + testFastOre4NumberImpl(number * i, false); + } + + BigDecimal bigDecimalValue = new BigDecimal("25347862354.00000000000000002345645645000000000000000234342"); + for(int i = 0; i < 10000; i++) + { + testFastOre4Double(13425.000001232345435000000012345678 * i, true); + testFastOre4Double(13425.12345678 * i, false); + testFastOre4Double(-13425.12345678 * i, true); + testFastOre4Double(-13425.12345678 * i, false); + testFastOre4Decimal(bigDecimalValue.multiply(BigDecimal.valueOf(i)), true); + testFastOre4Decimal(bigDecimalValue.multiply(BigDecimal.valueOf(i)), false); + testFastOre4Decimal(bigDecimalValue.multiply(BigDecimal.valueOf(-i)), true); + testFastOre4Decimal(bigDecimalValue.multiply(BigDecimal.valueOf(-i)), false); + } + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp new file mode 100644 index 00000000..eb4ccbe9 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp @@ -0,0 +1,348 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file fast_ore.cpp + * @author: yujiechen + * @date 2023-09-12 + */ +#include "fast_ore.h" +#include "ppc-crypto-core/src/ore/FastOre.h" +#include "utils/error.h" +#include "utils/utilities.h" +#include +#include + +using namespace ppc; +using namespace ppc::crypto; + +thread_local FastOre::Ptr g_ore_impl = std::make_shared(); + +int fast_ore_key_bytes() +{ + clear_last_error(); + try + { + return g_ore_impl->keyBytes(); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } + return -1; +} + +uint64_t fast_ore_get_cipher_size(uint64_t plainSize, bool hex) +{ + clear_last_error(); + return g_ore_impl->estimatedCipherSize(plainSize, hex); +} + +uint64_t fast_ore_get_float_cipher_size(uint64_t _plainSize, bool hex) +{ + clear_last_error(); + return g_ore_impl->estimatedFloatCipherSize(_plainSize, hex); +} + +uint64_t fast_ore_get_plain_size(uint64_t cipherSize, bool hex) +{ + clear_last_error(); + return g_ore_impl->estimatedPlainSize(cipherSize, hex); +} +/** + * @brief generate key for the fastORE + * + * @param key the generated key + */ +void fast_ore_generate_key(OutputBuffer* key) +{ + clear_last_error(); + try + { + g_ore_impl->generateKey(key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void string_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, InputBuffer const* plain, bool hexEncode) +{ + clear_last_error(); + try + { + auto refPlain = bcos::bytesConstRef((bcos::byte*)plain->data, plain->len); + if (!hexEncode) + { + g_ore_impl->encrypt4String( + cipher, bcos::bytesConstRef((bcos::byte*)key->data, key->len), refPlain); + return; + } + auto hexedBufferSize = g_ore_impl->estimatedCipherSize(refPlain.size(), true); + if (cipher->len < hexedBufferSize) + { + auto errorMsg = "string_fast_ore_encrypt error for unenough cipher buffer, at least: " + + std::to_string(hexedBufferSize); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + bcos::bytes bytesResult(g_ore_impl->estimatedCipherSize(refPlain.size(), false)); + OutputBuffer bytesResultBuffer{bytesResult.data(), bytesResult.size()}; + g_ore_impl->encrypt4String( + &bytesResultBuffer, bcos::bytesConstRef((bcos::byte*)key->data, key->len), refPlain); + InputBuffer buffer{bytesResultBuffer.data, bytesResultBuffer.len}; + to_hex(cipher, &buffer); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void string_fast_ore_decrypt( + OutputBuffer* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode) +{ + clear_last_error(); + try + { + if (!hexEncode) + { + return g_ore_impl->decrypt4String(plain, + bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)cipher->data, cipher->len)); + } + // convert the cipher from hex to binary + bcos::bytes binaryCipher((cipher->len) / 2); + OutputBuffer binaryBuffer{binaryCipher.data(), binaryCipher.size()}; + from_hex(&binaryBuffer, cipher); + return g_ore_impl->decrypt4String(plain, + bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)binaryBuffer.data, binaryBuffer.len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void integer_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, int64_t const& plain, bool hexEncode) +{ + clear_last_error(); + try + { + if (!hexEncode) + { + g_ore_impl->encrypt4Integer( + cipher, bcos::bytesConstRef((bcos::byte*)key->data, key->len), plain); + return; + } + + auto hexedBufferSize = g_ore_impl->estimatedCipherSize(sizeof(plain), true); + if (cipher->len < hexedBufferSize) + { + auto errorMsg = + "integer_fast_ore_encrypt error for unenough cipher buffer, at least: " + + std::to_string(hexedBufferSize); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + + bcos::bytes bytesResult(g_ore_impl->estimatedCipherSize(sizeof(plain), false)); + OutputBuffer bytesResultBuffer{bytesResult.data(), bytesResult.size()}; + g_ore_impl->encrypt4Integer( + &bytesResultBuffer, bcos::bytesConstRef((bcos::byte*)key->data, key->len), plain); + InputBuffer buffer{bytesResultBuffer.data, bytesResultBuffer.len}; + to_hex(cipher, &buffer); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void integer_fast_ore_decrypt( + int64_t* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode) +{ + clear_last_error(); + try + { + if (!hexEncode) + { + return g_ore_impl->decrypt4Integer(plain, + bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)cipher->data, cipher->len)); + } + + bcos::bytes binaryCipher((cipher->len) / 2); + OutputBuffer binaryBuffer{binaryCipher.data(), binaryCipher.size()}; + from_hex(&binaryBuffer, cipher); + return g_ore_impl->decrypt4Integer(plain, + bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)binaryBuffer.data, binaryBuffer.len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void float_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, InputBuffer const* plain, bool hexEncode) +{ + clear_last_error(); + try + { + if (!hexEncode) + { + g_ore_impl->encrypt4Float(cipher, bcos::bytesConstRef((bcos::byte*)key->data, key->len), + float50{std::string(plain->data, plain->data + plain->len)}); + return; + } + + auto hexedBufferSize = g_ore_impl->estimatedFloatCipherSize(plain->len, true); + if (cipher->len < hexedBufferSize) + { + auto errorMsg = + "integer_fast_ore_encrypt error for unenough cipher buffer, at least: " + + std::to_string(hexedBufferSize); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + + bcos::bytes bytesResult(g_ore_impl->estimatedFloatCipherSize(plain->len, false)); + OutputBuffer bytesResultBuffer{bytesResult.data(), bytesResult.size()}; + g_ore_impl->encrypt4Float(&bytesResultBuffer, + bcos::bytesConstRef((bcos::byte*)key->data, key->len), + float50{std::string(plain->data, plain->data + plain->len)}); + InputBuffer buffer{bytesResultBuffer.data, bytesResultBuffer.len}; + to_hex(cipher, &buffer); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void float_fast_ore_decrypt( + OutputBuffer* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode) +{ + clear_last_error(); + try + { + float50 res; + if (!hexEncode) + { + res = g_ore_impl->decrypt4Float(bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)cipher->data, cipher->len)); + } + else + { + bcos::bytes binaryCipher((cipher->len) / 2); + OutputBuffer binaryBuffer{binaryCipher.data(), binaryCipher.size()}; + from_hex(&binaryBuffer, cipher); + res = g_ore_impl->decrypt4Float(bcos::bytesConstRef((bcos::byte*)key->data, key->len), + bcos::bytesConstRef((bcos::byte*)binaryBuffer.data, binaryBuffer.len)); + } + + auto plainStr = res.str(); + if (plain->len < plainStr.size()) + { + auto errorMsg = "float_fast_ore_decrypt error for unenough plain buffer, at least: " + + std::to_string(plainStr.size()); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + plain->len = plainStr.size(); + std::copy(plainStr.begin(), plainStr.end(), plain->data); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief compare between cipher1 and cipher2 + * + * @param cipher1 + * @param cipher2 + * @return int + */ +int fast_ore_compare(InputBuffer const* cipher1, InputBuffer const* cipher2) +{ + clear_last_error(); + try + { + return g_ore_impl->compare(cipher1, cipher2); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } + return -1; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h new file mode 100644 index 00000000..062f34c6 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file fast_ore.h + * @author: yujiechen + * @date 2023-08-25 + */ +#ifndef __FAST_ORE_H__ +#define __FAST_ORE_H__ + +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int fast_ore_key_bytes(); + +uint64_t fast_ore_get_cipher_size(uint64_t plainSize, bool hex); +uint64_t fast_ore_get_float_cipher_size(uint64_t plainSize, bool hex); +uint64_t fast_ore_get_plain_size(uint64_t cipherSize, bool hex); + +/** + * @brief generate key for the fastORE + * + * @param key the generated key + */ +void fast_ore_generate_key(OutputBuffer* key); + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void string_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, InputBuffer const* plain, bool hexEncode); + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void string_fast_ore_decrypt( + OutputBuffer* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode); + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void integer_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, int64_t const& plain, bool hexEncode); + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void integer_fast_ore_decrypt( + int64_t* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode); + +/** + * @brief encrypt the plain data into cipher data using given key + * + * @param cipher the encrypted data + * @param key the key used to encrypt + * @param plain the plain-data + */ +void float_fast_ore_encrypt( + OutputBuffer* cipher, InputBuffer const* key, InputBuffer const* plain, bool hexEncode); + +/** + * @brief decrypt the cipher data into plain data using given key + * + * @param plain the decrypted plain data + * @param key the key used to decrypt + * @param cipher the cipher + */ +void float_fast_ore_decrypt( + OutputBuffer* plain, InputBuffer const* key, InputBuffer const* cipher, bool hexEncode); + +/** + * @brief compare between cipher1 and cipher2 + * + * @param cipher1 + * @param cipher2 + * @return int + */ +int fast_ore_compare(InputBuffer const* cipher1, InputBuffer const* cipher2); +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp new file mode 100644 index 00000000..baa55a27 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp @@ -0,0 +1,172 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file floating_point_ihc.cpp + * @author: asherli + * @date 2023-11-28 + */ +#include "floating_point_ihc.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include "ppc-homo/ihc/FloatingPointIhc.h" +#include "utils/error.h" + +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace bcos; + +FloatingPointIhc::Ptr g_fp_ihc_128 = + std::make_shared(std::make_shared((int)Ihc::IhcMode::IHC_128, 16)); +FloatingPointIhc::Ptr g_fp_ihc_256 = + std::make_shared(std::make_shared((int)Ihc::IhcMode::IHC_256, 16)); + +FloatingPointIhc::Ptr obtainFpIhcInstance(int mode) +{ + switch (mode) + { + case (int)Ihc::IhcMode::IHC_128: + return g_fp_ihc_128; + case (int)Ihc::IhcMode::IHC_256: + return g_fp_ihc_256; + default: + { + std::string errorMsg = "Unsupported ihc mode!"; + set_last_error_msg(-1, errorMsg.c_str()); + } + } + return nullptr; +} + +uint64_t ihc_floating_cipher_bytes(int mode) +{ + clear_last_error(); + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return 0; + } + return pIhc->cipherBytes(); +} + +void ihc_floating_encrypt(OutputBuffer* cipherBytes, int mode, InputBuffer const* _key, + BIGNUM const* value, int16_t _exponent) +{ + clear_last_error(); + try + { + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return; + } + BigNum v; + BN_copy(v.bn().get(), value); + FloatingPointNumber ffpNumber(std::move(v), _exponent); + pIhc->encrypt( + cipherBytes, bcos::bytesConstRef((bcos::byte const*)_key->data, _key->len), ffpNumber); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_floating_decrypt( + BIGNUM* value, int16_t* exponent, int mode, InputBuffer const* key, InputBuffer const* cipher) +{ + clear_last_error(); + try + { + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return; + } + auto result = pIhc->decrypt(bcos::bytesConstRef((bcos::byte const*)key->data, key->len), + bcos::bytesConstRef((bcos::byte const*)cipher->data, cipher->len)); + result.value.swap(value); + *exponent = result.exponent; + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void ihc_floating_add( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2) +{ + clear_last_error(); + try + { + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->add(cipherBytes, bcos::bytesConstRef((bcos::byte const*)cipher1->data, cipher1->len), + bcos::bytesConstRef((bcos::byte const*)cipher2->data, cipher2->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void ihc_floating_sub( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2) +{ + clear_last_error(); + try + { + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->sub(cipherBytes, bcos::bytesConstRef((bcos::byte const*)cipher1->data, cipher1->len), + bcos::bytesConstRef((bcos::byte const*)cipher2->data, cipher2->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_floating_scalaMul(OutputBuffer* cipherBytes, int mode, BIGNUM const* v, int16_t exponent, + InputBuffer const* cipher) +{ + clear_last_error(); + try + { + auto pIhc = obtainFpIhcInstance(mode); + if (!pIhc) + { + return; + } + BigNum value; + BN_copy(value.bn().get(), v); + FloatingPointNumber vFpNumber(std::move(value), exponent); + pIhc->scalaMul(cipherBytes, vFpNumber, + bcos::bytesConstRef((bcos::byte const*)cipher->data, cipher->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h new file mode 100644 index 00000000..f310faae --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file floating_point_ihc.h + * @author: asherli + * @date 2023-11-28 + */ + +#ifndef __FLOATING_POINT_IHC_H__ +#define __FLOATING_POINT_IHC_H__ +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif +void ihc_floating_encrypt(OutputBuffer* cipherBytes, int mode, InputBuffer const* _key, + BIGNUM const* value, int16_t _exponent); +void ihc_floating_decrypt( + BIGNUM* value, int16_t* exponent, int mode, InputBuffer const* key, InputBuffer const* cipher); +void ihc_floating_add( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2); +void ihc_floating_sub( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2); +void ihc_floating_scalaMul(OutputBuffer* cipherBytes, int mode, BIGNUM const* v, int16_t exponent, + InputBuffer const* cipher); + +uint64_t ihc_floating_cipher_bytes(int mode); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp new file mode 100644 index 00000000..419411ad --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp @@ -0,0 +1,264 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file floating_point_paillier.cpp + * @author: yujiechen + * @date 2023-08-20 + */ +#include "floating_point_paillier.h" +#include "homo_paillier.h" +#include "ppc-homo/paillier/FloatingPointPaillier.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include "utils/error.h" + +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace bcos; + +OpenSSLPaillier::Ptr g_paillier_ptr = std::make_shared(); +thread_local FloatingPointPaillier::Ptr g_floating_point_paillier = + std::make_shared(g_paillier_ptr); + +unsigned int floating_point_paillier_cipher_bytes(unsigned int keyBits) +{ + clear_last_error(); + return FloatingPointPaillier::maxCipherBytesLen(keyBits); +} +/** + * @brief encrypt {value*10^exponent} using paillier crt-optimization + * + * @param cipherBytes the encrypted result + * @param value the value to be encrypted + * @param exponent the exponent of the value to be encrypted + * @param keypair the keypair used to encrypt + */ +void floating_point_paillier_encrypt_fast( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t exponent, void* keypair) +{ + clear_last_error(); + try + { + BigNum v; + BN_copy(v.bn().get(), value); + FloatingPointNumber floatingPointV(std::move(v), exponent); + g_floating_point_paillier->encryptFast(cipherBytes, floatingPointV, keypair); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void floating_point_paillier_encrypt_fast_without_precompute(OutputBuffer* cipherBytes, + BIGNUM const* value, int16_t exponent, InputBuffer const* sk, InputBuffer const* pkBytes) +{ + auto keyPair = paillier_load_keypair(sk, pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_encrypt_fast(cipherBytes, value, exponent, keyPair); + paillier_free_key_pair(keyPair); +} + +/** + * @brief encrypt {value*10^exponent} withou using paillier crt-optimization + * + * @param cipherBytes the encrypted result + * @param value the value to be encrypted + * @param exponent the exponent of the value to be encrypted + * @param public_key the public_key used to encrypt + */ +void floating_point_paillier_encrypt( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t exponent, void* public_key) +{ + clear_last_error(); + try + { + BigNum v; + BN_copy(v.bn().get(), value); + FloatingPointNumber floatingPointV(std::move(v), exponent); + g_floating_point_paillier->encrypt(cipherBytes, floatingPointV, public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void floating_point_paillier_encrypt_without_precompute( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t _exponent, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_encrypt(cipherBytes, value, _exponent, pk); + paillier_free_public_key(pk); +} +/** + * @brief paillier decrypt + * + * @param value the decrypted value + * @param exponent the exponent of the decrypted value + * @param cipher the cipher to be decrypted + * @param keypair the keyPair used to decrypt + */ +void floating_point_paillier_decrypt( + BIGNUM* value, int16_t* exponent, InputBuffer const* cipher, void* keypair) +{ + clear_last_error(); + try + { + auto ret = g_floating_point_paillier->decrypt( + bcos::bytesConstRef(cipher->data, cipher->len), keypair); + // swap the result to value + ret.value.swap(value); + *exponent = ret.exponent; + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void floating_point_paillier_decrypt_without_precompute(BIGNUM* value, int16_t* exponent, + InputBuffer const* cipher, InputBuffer const* sk, InputBuffer const* pkBytes) +{ + auto keyPair = paillier_load_keypair(sk, pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_decrypt(value, exponent, cipher, keyPair); + paillier_free_key_pair(keyPair); +} + +/** + * @brief paillier add + * + * @param cipherBytes the paillier cipher add result + * @param cipher1 + * @param cipher2 + * @param public_key + */ +void floating_point_paillier_add(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, void* public_key) +{ + clear_last_error(); + try + { + g_floating_point_paillier->add(cipherBytes, + bcos::bytesConstRef(cipher1->data, cipher1->len), + bcos::bytesConstRef(cipher2->data, cipher2->len), public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void floating_point_paillier_add_without_precompute(OutputBuffer* cipherBytes, + InputBuffer const* cipher1, InputBuffer const* cipher2, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_add(cipherBytes, cipher1, cipher2, pk); + paillier_free_public_key(pk); +} + +/** + * @brief paillier sub + * + * @param cipherBytes the paillier cipher sub result + * @param cipher1 + * @param cipher2 + * @param public_key + */ +void floating_point_paillier_sub(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, void* public_key) +{ + clear_last_error(); + try + { + g_floating_point_paillier->sub(cipherBytes, + bcos::bytesConstRef(cipher1->data, cipher1->len), + bcos::bytesConstRef(cipher2->data, cipher2->len), public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void floating_point_paillier_sub_without_precompute(OutputBuffer* cipherBytes, + InputBuffer const* cipher1, InputBuffer const* cipher2, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_sub(cipherBytes, cipher1, cipher2, pk); + paillier_free_public_key(pk); +} + +/** + * @brief paillier-scalar-multipy + * + * @param cipherBytes the paillier cipher multipy result + * @param v the value + * @param exponent the exponent of the value + * @param cipher the cipher + * @param public_key + */ +void floating_point_paillier_scalaMul(OutputBuffer* cipherBytes, BIGNUM const* v, int16_t exponent, + InputBuffer const* cipher, void* public_key) +{ + clear_last_error(); + try + { + BigNum value; + BN_copy(value.bn().get(), v); + FloatingPointNumber floatingPointV(std::move(value), exponent); + g_floating_point_paillier->scalaMul(cipherBytes, floatingPointV, + bcos::bytesConstRef(cipher->data, cipher->len), public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void floating_point_paillier_scalaMul_without_precompute(OutputBuffer* cipherBytes, BIGNUM const* v, + int16_t exponent, InputBuffer const* cipher, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + floating_point_paillier_scalaMul(cipherBytes, v, exponent, cipher, pk); + paillier_free_public_key(pk); +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h new file mode 100644 index 00000000..8834c6bf --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h @@ -0,0 +1,111 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file floating_point_paillier.h + * @author: yujiechen + * @date 2023-08-20 + */ +#ifndef __FLOATING_POINT_PAILLIER_H__ +#define __FLOATING_POINT_PAILLIER_H__ +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/Buffer.h" +#ifdef __cplusplus +extern "C" { +#endif + +unsigned int floating_point_paillier_cipher_bytes(unsigned int keyBits); +/** + * @brief encrypt {value*10^exponent} using paillier crt-optimization + * + * @param cipherBytes the encrypted result + * @param value the value to be encrypted + * @param exponent the exponent of the value to be encrypted + * @param keypair the keypair used to encrypt + */ +void floating_point_paillier_encrypt_fast( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t exponent, void* keypair); + +void floating_point_paillier_encrypt_fast_without_precompute(OutputBuffer* cipherBytes, + BIGNUM const* value, int16_t exponent, InputBuffer const* sk, InputBuffer const* pk); + +/** + * @brief encrypt {value*10^exponent} withou using paillier crt-optimization + * + * @param cipherBytes the encrypted result + * @param value the value to be encrypted + * @param exponent the exponent of the value to be encrypted + * @param public_key the public_key used to encrypt + */ +void floating_point_paillier_encrypt( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t _exponent, void* public_key); +void floating_point_paillier_encrypt_without_precompute( + OutputBuffer* cipherBytes, BIGNUM const* value, int16_t _exponent, InputBuffer const* pkBytes); + +/** + * @brief paillier decrypt + * + * @param value the decrypted value + * @param exponent the exponent of the decrypted value + * @param cipher the cipher to be decrypted + * @param keypair the keyPair used to decrypt + */ +void floating_point_paillier_decrypt( + BIGNUM* value, int16_t* exponent, InputBuffer const* cipher, void* keypair); +void floating_point_paillier_decrypt_without_precompute(BIGNUM* value, int16_t* exponent, + InputBuffer const* cipher, InputBuffer const* sk, InputBuffer const* pk); + +/** + * @brief paillier add + * + * @param cipherBytes the paillier cipher add result + * @param cipher1 + * @param cipher2 + * @param public_key + */ +void floating_point_paillier_add(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, void* public_key); +void floating_point_paillier_add_without_precompute(OutputBuffer* cipherBytes, + InputBuffer const* cipher1, InputBuffer const* cipher2, InputBuffer const* pkBytes); + +/** + * @brief paillier sub + * + * @param cipherBytes the paillier cipher sub result + * @param cipher1 + * @param cipher2 + * @param public_key + */ +void floating_point_paillier_sub(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, void* public_key); +void floating_point_paillier_sub_without_precompute(OutputBuffer* cipherBytes, + InputBuffer const* cipher1, InputBuffer const* cipher2, InputBuffer const* pkBytes); + +/** + * @brief paillier-scalar-multipy + * + * @param cipherBytes the paillier cipher multipy result + * @param v the value + * @param exponent the exponent of the value + * @param cipher the cipher + * @param public_key + */ +void floating_point_paillier_scalaMul(OutputBuffer* cipherBytes, BIGNUM const* v, int16_t exponent, + InputBuffer const* cipher, void* public_key); +void floating_point_paillier_scalaMul_without_precompute(OutputBuffer* cipherBytes, BIGNUM const* v, + int16_t exponent, InputBuffer const* cipher, InputBuffer const* pkBytes); +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp new file mode 100644 index 00000000..3662de14 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file homo_ihc.cpp + * @author: asherli + * @date 2023-11-28 + */ +#include "homo_ihc.h" +#include "ppc-homo/ihc/IhcImpl.h" +#include "utils/error.h" + +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace bcos; + +Ihc::Ptr g_ihc_128 = std::make_shared((int)Ihc::IhcMode::IHC_128, 16); +Ihc::Ptr g_ihc_256 = std::make_shared((int)Ihc::IhcMode::IHC_256, 16); + +Ihc::Ptr obtainIhcInstance(int mode) +{ + switch (mode) + { + case (int)Ihc::IhcMode::IHC_128: + return g_ihc_128; + case (int)Ihc::IhcMode::IHC_256: + return g_ihc_256; + default: + { + std::string errorMsg = "Unsupported ihc mode!"; + set_last_error_msg(-1, errorMsg.c_str()); + } + } + return nullptr; +} + +unsigned int ihc_key_bytes(int mode) +{ + clear_last_error(); + auto pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return 0; + } + return pIhc->keyBytes(); +} + +uint64_t ihc_cipher_bytes(int mode) +{ + clear_last_error(); + auto pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return 0; + } + return pIhc->cipherBytes(); +} + +void ihc_generate_key(OutputBuffer* key, int mode) +{ + clear_last_error(); + try + { + Ihc::Ptr pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->generateKey(key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_encrypt( + OutputBuffer* cipherBytes, int _mode, InputBuffer const* _key, BIGNUM const* _plain) +{ + clear_last_error(); + try + { + auto pIhc = obtainIhcInstance(_mode); + if (!pIhc) + { + return; + } + pIhc->encrypt(cipherBytes, bcos::bytesConstRef((bcos::byte*)_key->data, _key->len), _plain); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_decrypt(BIGNUM* _result, int _mode, InputBuffer const* _key, InputBuffer const* cipher) +{ + clear_last_error(); + try + { + auto pIhc = obtainIhcInstance(_mode); + if (!pIhc) + { + return; + } + auto decryptedV = pIhc->decrypt(bcos::bytesConstRef((bcos::byte*)_key->data, _key->len), + bcos::bytesConstRef((bcos::byte*)cipher->data, cipher->len)); + BN_print_fp(stdout, decryptedV.bn().get()); + decryptedV.swap(_result); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_add( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2) +{ + clear_last_error(); + try + { + auto pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->add(cipherBytes, bcos::bytesConstRef((bcos::byte const*)cipher1->data, cipher1->len), + bcos::bytesConstRef((bcos::byte const*)cipher2->data, cipher2->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_sub( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2) +{ + clear_last_error(); + try + { + auto pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->sub(cipherBytes, bcos::bytesConstRef((bcos::byte const*)cipher1->data, cipher1->len), + bcos::bytesConstRef((bcos::byte const*)cipher2->data, cipher2->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void ihc_scalaMul(OutputBuffer* cipherBytes, int mode, BIGNUM const* v, InputBuffer const* cipher) +{ + clear_last_error(); + try + { + auto pIhc = obtainIhcInstance(mode); + if (!pIhc) + { + return; + } + pIhc->scalaMul( + cipherBytes, v, bcos::bytesConstRef((bcos::byte const*)cipher->data, cipher->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h new file mode 100644 index 00000000..734e822b --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file homo_ihc.h + * @author: asherli + * @date 2023-11-28 + */ + +#ifndef __HOMO_IHC_H__ +#define __HOMO_IHC_H__ +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void ihc_generate_key(OutputBuffer* key, int mode); +void ihc_encrypt( + OutputBuffer* cipherBytes, int mode, InputBuffer const* _key, BIGNUM const* _plain); +void ihc_decrypt(BIGNUM* _result, int _mode, InputBuffer const* _key, InputBuffer const* cipher); + +void ihc_add( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2); +void ihc_sub( + OutputBuffer* cipherBytes, int mode, InputBuffer const* cipher1, InputBuffer const* cipher2); +void ihc_scalaMul(OutputBuffer* cipherBytes, int mode, BIGNUM const* v, InputBuffer const* cipher); + +unsigned int ihc_key_bytes(int mode); +uint64_t ihc_cipher_bytes(int mode); +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp new file mode 100644 index 00000000..020c8f57 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp @@ -0,0 +1,555 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file homo_paillier.cpp + * @author: yujiechen + * @date 2023-08-11 + */ +#include "homo_paillier.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include "ppc-homo/paillier/OpenSSLPaillierKeyPair.h" +#include "utils/error.h" +#include +#include + +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace bcos; + +thread_local OpenSSLPaillier::Ptr g_paillier_impl = std::make_shared(); + +unsigned int paillier_max_public_key_bytes(int key_bits) +{ + clear_last_error(); + return ppc::homo::PaillierPublicKey::maxBytes(key_bits); +} + +unsigned int paillier_r_bytes_len(int key_bits) +{ + clear_last_error(); + return ppc::homo::OpenSSLPaillier::rBytesLen(key_bits); +} + +unsigned int paillier_max_private_key_bytes(int key_bits) +{ + clear_last_error(); + return ppc::homo::PaillierPrivateKey::maxBytes(key_bits); +} + +unsigned int paillier_max_cipher_bytes(int key_bits) +{ + clear_last_error(); + return ppc::homo::OpenSSLPaillier::maxCipherBytesLen(key_bits); +} + +unsigned int paillier_key_bits_from_keypair(void* key_pair) +{ + clear_last_error(); + auto keyPair = (ppc::homo::OpenSSLPaillierKeyPair*)(key_pair); + auto pk = (ppc::homo::PaillierPublicKey*)(keyPair->pk()); + return pk->keyBits; +} + +unsigned int paillier_key_bits_from_public_key(void* public_key) +{ + clear_last_error(); + auto pk = (ppc::homo::PaillierPublicKey*)(public_key); + return pk->keyBits; +} + +/** + * @brief obtain n from the given keyPair + * + * @param n + * @param pk + * @return void + */ +void paillier_n_from_pk(BIGNUM* n, void* pk) +{ + clear_last_error(); + auto pkPtr = (PaillierPublicKey*)pk; + pkPtr->n.copy(n); +} + +/** + * @brief generate the paillier KeyPair + * + * @return void* pointer to the generated paillier key-pair + */ +void* paillier_generate_keypair(int key_bits) +{ + clear_last_error(); + try + { + auto keypair = g_paillier_impl->generateKeyPair(key_bits); + return keypair.release(); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return nullptr; + } +} + +/** + * @brief load the encoded privateKey and publicKey into key-pair object + * + * @param sk the encoded privateKey + * @param pk the encoded publicKey + * @return void* void* pointer to the loaded paillier key-pair + */ +void* paillier_load_keypair(InputBuffer const* sk, InputBuffer const* pk) +{ + clear_last_error(); + try + { + auto keyPair = + std::make_unique(sk->data, sk->len, pk->data, pk->len); + return keyPair.release(); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return nullptr; + } +} + + +/** + * @brief free the allocated keypair + * + * @param keypair + */ +void paillier_free_key_pair(void* keypair) +{ + clear_last_error(); + if (keypair) + { + free(keypair); + } +} + +/** + * @brief get the public key object according to the keypair + * + * @param keypair + * @return void* + */ +void* paillier_get_public_key(void* keypair) +{ + clear_last_error(); + auto paillierKeyPair = (OpenSSLPaillierKeyPair*)keypair; + return paillierKeyPair->pk(); +} +/** + * @brief load the publicKey object from the encoded publicKey + * + * @param pk the encoded publicKey + * @return void* the pointer to the public key object + */ +void* paillier_load_public_key(InputBuffer const* pkBytes) +{ + clear_last_error(); + try + { + auto pk = std::make_unique(pkBytes->data, pkBytes->len); + return pk.release(); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return nullptr; + } +} + +/** + * @brief free the allocated public key + * + * @param public_key + */ +void paillier_free_public_key(void* public_key) +{ + clear_last_error(); + if (public_key) + { + free(public_key); + } +} + +/** + * @brief free the private key + * + * @param private_key + */ +void paillier_free_private_key(void* private_key) +{ + clear_last_error(); + if (private_key) + { + free(private_key); + } +} + +/** + * @brief load the private key according to the keyBytes + * + * @param sk + * @return void* + */ +void* paillier_load_private_key(InputBuffer const* skBytes) +{ + clear_last_error(); + try + { + auto sk = std::make_unique(skBytes->data, skBytes->len); + return sk.release(); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return nullptr; + } +} + +/** + * @brief get the encoded publicKey from given keyPair object + * + * @param pkBytes return the encoded public key + * Note: should allocate OutputBuffer before call this function, the lifecycle of OutputBuffer is + * managed by the caller(maybe cross-programming language) + * @param keypair the pointer to the keyPair object + */ +void paillier_set_public_key_bytes_from_keypair(OutputBuffer* pkBytes, void* keypair) +{ + clear_last_error(); + try + { + auto paillierKeyPair = (OpenSSLPaillierKeyPair*)(keypair); + auto pk = (PaillierPublicKey*)(paillierKeyPair->pk()); + pkBytes->len = pk->serialize((bcos::byte*)pkBytes->data, pkBytes->len); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +OutputBuffer paillier_get_public_key_bytes_from_keyPair(void* keypair) +{ + clear_last_error(); + unsigned char* publicKeyBytes = NULL; + try + { + auto paillierKeyPair = (OpenSSLPaillierKeyPair*)(keypair); + auto pk = (PaillierPublicKey*)(paillierKeyPair->pk()); + + auto bufferLen = paillier_max_public_key_bytes(pk->keyBits); + // Note: this should be released by the caller after used + publicKeyBytes = (unsigned char*)malloc(bufferLen); + OutputBuffer publicKeyBuffer{publicKeyBytes, bufferLen}; + paillier_set_public_key_bytes_from_keypair(&publicKeyBuffer, keypair); + return publicKeyBuffer; + } + catch (std::exception const& e) + { + if (!publicKeyBytes) + { + free(publicKeyBytes); + } + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return OutputBuffer{NULL, 0}; + } +} + +/** + * @brief Get the private key bytes object + * + * @param keypair the pointer to the keyPair object + * @return void* the encoded private key + */ +void paillier_set_private_key_bytes_from_keypair(OutputBuffer* skBytes, void* keypair) +{ + clear_last_error(); + try + { + auto paillierKeyPair = (OpenSSLPaillierKeyPair*)(keypair); + auto sk = (PaillierPrivateKey*)(paillierKeyPair->sk()); + skBytes->len = sk->serialize((bcos::byte*)skBytes->data, skBytes->len); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief Get the private key bytes + * + * @param keypair the pointer to the keyPair object + * @return OutputBuffer the private key-bytes + * Note: the caller should release the allocated buffer + */ +OutputBuffer paillier_get_private_key_bytes_from_keypair(void* keypair) +{ + clear_last_error(); + unsigned char* skBytes = NULL; + try + { + auto paillierKeyPair = (OpenSSLPaillierKeyPair*)(keypair); + auto sk = (PaillierPrivateKey*)(paillierKeyPair->sk()); + auto bufferLen = paillier_max_private_key_bytes(sk->keyBits); + + // Note: this should be released by the caller after used + skBytes = (unsigned char*)malloc(bufferLen); + OutputBuffer skBuffer{skBytes, bufferLen}; + paillier_set_private_key_bytes_from_keypair(&skBuffer, keypair); + return skBuffer; + } + catch (std::exception const& e) + { + if (skBytes) + { + free(skBytes); + } + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return OutputBuffer{NULL, 0}; + } +} + +/** + * @brief encrypt the value into paillier cipher using given keyPair with CRT optimization + * + * @param cipherBytes return the encrypted cipher + * @param value the value to be encrypted + * @param keypair the keyPair used to encrypt + */ +void paillier_encryt_fast( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, void* keypair) +{ + clear_last_error(); + try + { + g_paillier_impl->encrypt_with_crt(cipherBytes, rBytes, value, keypair); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void paillier_encryt_fast_without_precompute(OutputBuffer* cipherBytes, OutputBuffer* rBytes, + BIGNUM* value, InputBuffer const* sk, InputBuffer const* pk) +{ + auto keyPair = paillier_load_keypair(sk, pk); + if (!last_call_success()) + { + return; + } + + paillier_encryt_fast(cipherBytes, rBytes, value, keyPair); + paillier_free_key_pair(keyPair); +} + +/** + * @brief encrypt the value into paillier cipher using given keyPair without CRT optimization + * + * @param cipherBytes return the encrypted cipher + * @param value the value to be encrypted + * @param keypair the keyPair used to encrypt + */ +void paillier_encryt( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, void* publicKey) +{ + clear_last_error(); + try + { + g_paillier_impl->encrypt(cipherBytes, rBytes, value, publicKey); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void paillier_encryt_without_precompute( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + paillier_encryt(cipherBytes, rBytes, value, pk); + paillier_free_public_key(pk); +} + +/** + * @brief decrypt the cipher using given keyPair + * + * @param cipher the cipher to be decrypted + * @param keypair the keyPair used to decrypt + * @return BIGNUM* the decrypted result + * Note: the caller should release the allocated BIGNUM + */ +BIGNUM* paillier_decrypt(InputBuffer const* cipher, void* keypair) +{ + clear_last_error(); + try + { + auto decryptResult = g_paillier_impl->decrypt( + bcos::bytesConstRef((bcos::byte const*)cipher->data, cipher->len), keypair); + BIGNUM* ret = BN_new(); + // swap the result + BN_swap(ret, decryptResult.bn().get()); + return ret; + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + return NULL; + } +} + +BIGNUM* paillier_decrypt_without_precompute( + InputBuffer const* cipher, InputBuffer const* sk, InputBuffer const* pk) +{ + auto keyPair = paillier_load_keypair(sk, pk); + if (!last_call_success()) + { + return NULL; + } + auto result = paillier_decrypt(cipher, keyPair); + paillier_free_key_pair(keyPair); + return result; +} + +/** + * @brief paillier ciphertext space addition, namely: cipher1 + cipher2 + * + * @param cipher1 + * @param cipher2 + * @param public_key + * @return void* the addition result + */ +void paillier_add(OutputBuffer* cipherBytes, InputBuffer const* cipher1, InputBuffer const* cipher2, + void* public_key) +{ + clear_last_error(); + try + { + auto cipher1Ref = bcos::bytesConstRef(cipher1->data, cipher1->len); + auto cipher2Ref = bcos::bytesConstRef(cipher2->data, cipher2->len); + g_paillier_impl->add(cipherBytes, cipher1Ref, cipher2Ref, public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} +void paillier_add_without_precompute(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + paillier_add(cipherBytes, cipher1, cipher2, pk); + paillier_free_public_key(pk); +} +/** + * @brief paillier ciphertext space addition, namely: cipher1 - cipher2 + * + * @param cipher1 + * @param cipher2 + * @param public_key + * @return void* the subtraction result + */ +void paillier_sub(OutputBuffer* cipherBytes, InputBuffer const* cipher1, InputBuffer const* cipher2, + void* public_key) +{ + clear_last_error(); + try + { + auto cipher1Ref = bcos::bytesConstRef(cipher1->data, cipher1->len); + auto cipher2Ref = bcos::bytesConstRef(cipher2->data, cipher2->len); + g_paillier_impl->sub(cipherBytes, cipher1Ref, cipher2Ref, public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void paillier_sub_without_precompute(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + paillier_sub(cipherBytes, cipher1, cipher2, pk); + paillier_free_public_key(pk); +} +/** + * @brief paillier ciphertext space scala-multiply, namely: v * cipher + * + * @param v the scala-number + * Note: the lifecycle of value is managed by the caller + * @param cipher + * @param public_key + * @return void* the scala-multiply result + */ +void paillier_scala_mul( + OutputBuffer* cipherBytes, BIGNUM* v, InputBuffer const* cipher, void* public_key) +{ + clear_last_error(); + try + { + auto cipherRef = bcos::bytesConstRef(cipher->data, cipher->len); + g_paillier_impl->scalaMul(cipherBytes, v, cipherRef, public_key); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +void paillier_scala_mul_without_precompute( + OutputBuffer* cipherBytes, BIGNUM* v, InputBuffer const* cipher, InputBuffer const* pkBytes) +{ + auto pk = paillier_load_public_key(pkBytes); + if (!last_call_success()) + { + return; + } + paillier_scala_mul(cipherBytes, v, cipher, pk); + paillier_free_public_key(pk); +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h new file mode 100644 index 00000000..0c70c294 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h @@ -0,0 +1,265 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file homo_paillier.h + * @author: yujiechen + * @date 2023-08-11 + */ +#ifndef __HOMO_PAILLIER_H__ +#define __HOMO_PAILLIER_H__ + +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief get the paillier max public key bytes according to the keyBits + * + * @param key_bits e.g. 512/1024/2048/3096 + * @return unsigned int the public key bytes + */ +unsigned int paillier_max_public_key_bytes(int key_bits); + +/** + * @brief get the paillier random r bytes according to the keyBits + * @param key_bits e.g. 512/1024/2048/3096 + */ +unsigned int paillier_r_bytes_len(int key_bits); + +/** + * @brief get the paillier max private key bytes according to the keyBits + * + * @param key_bits e.g. 512/1024/2048/3096 + * @return unsigned int the private key bytes + */ +unsigned int paillier_max_private_key_bytes(int key_bits); + +/** + * @brief get the paillier max cipher bytes according to the keyBits + * + * @param key_bits e.g. 512/1024/2048/3096 + * @return unsigned int the cipher bytes + */ +unsigned int paillier_max_cipher_bytes(int key_bits); + + +/** + * @brief get the paillier max cipher bytes according to the keypair + * + * @param key_pair the key pair object + * @return unsigned int + */ +unsigned int paillier_key_bits_from_keypair(void* key_pair); + +/** + * @brief obtain n from the given keyPair + * + * @param n + * @param pk + */ +void paillier_n_from_pk(BIGNUM* n, void* pk); + +/** + * @brief get the paillier max cipher bytes according to the public key + * + * @param public_key the public key object + * @return unsigned int + */ +unsigned int paillier_key_bits_from_public_key(void* public_key); + +/** + * @brief generate the paillier KeyPair + * + * @return void* pointer to the generated paillier key-pair + */ +void* paillier_generate_keypair(int key_bits); + +/** + * @brief load the encoded privateKey and publicKey into key-pair object + * + * @param sk the encoded privateKey + * @param pk the encoded publicKey + * @return void* void* pointer to the loaded paillier key-pair + */ +void* paillier_load_keypair(InputBuffer const* sk, InputBuffer const* pk); + +/** + * @brief free the allocated keypair + * + * @param keypair + */ +void paillier_free_key_pair(void* keypair); + +/** + * @brief get the public key object according to the keypair + * + * @param keypair + * @return void* + */ +void* paillier_get_public_key(void* keypair); + +/** + * @brief load the publicKey object from the encoded publicKey + * + * @param pk the encoded publicKey + * @return void* the pointer to the public key object + */ +void* paillier_load_public_key(InputBuffer const* pk); + + +/** + * @brief free the allocated public key + * + * @param public_key + */ +void paillier_free_public_key(void* public_key); + +/** + * @brief load the private key according to the keyBytes + * + * @param sk + * @return void* + */ +void* paillier_load_private_key(InputBuffer const* sk); + +/** + * @brief free the private key + * + * @param private_key + */ +void paillier_free_private_key(void* private_key); + +/** + * @brief get the encoded publicKey from given keyPair object + * + * @param pkBytes return the encoded public key + * Note: should allocate OutputBuffer before call this function, the lifecycle of OutputBuffer is + * managed by the caller(maybe cross-programming language) + * @param keypair the pointer to the keyPair object + */ +void paillier_set_public_key_bytes_from_keypair(OutputBuffer* pkBytes, void* keypair); + +/** + * @brief get the encoded publicKey from given keyPair object + * + * @param keypair the pointer to the keyPair object + * @return OutputBuffer the public key-bytes + * Note: the caller should release the allocated buffer + */ +OutputBuffer paillier_get_public_key_bytes_from_keyPair(void* keypair); + + +/** + * @brief Get the private key bytes object + * + * @param pkBytes return the encoded private key + * Note: should allocate OutputBuffer before call this function, the lifecycle of OutputBuffer is + * managed by the caller(maybe cross-programming language) + * @param keypair the pointer to the keyPair object + */ +void paillier_set_private_key_bytes_from_keypair(OutputBuffer* skBytes, void* keypair); + +/** + * @brief Get the private key bytes + * + * @param keypair the pointer to the keyPair object + * @return OutputBuffer the private key-bytes + * Note: the caller should release the allocated buffer + */ +OutputBuffer paillier_get_private_key_bytes_from_keypair(void* keypair); + +/** + * @brief encrypt the value into paillier cipher using given keyPair with CRT optimization + * + * @param cipherBytes return the encrypted cipher + * @param value the value to be encrypted + * @param keypair the keyPair used to encrypt + */ +void paillier_encryt_fast( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, void* keypair); +void paillier_encryt_fast_without_precompute(OutputBuffer* cipherBytes, OutputBuffer* rBytes, + BIGNUM* value, InputBuffer const* sk, InputBuffer const* pk); + +/** + * @brief encrypt the value into paillier cipher using given keyPair without CRT optimization + * + * @param cipherBytes return the encrypted cipher + * @param value the value to be encrypted + * @param publicKey the keyPair used to encrypt + */ +void paillier_encryt( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, void* publicKey); +void paillier_encryt_without_precompute( + OutputBuffer* cipherBytes, OutputBuffer* rBytes, BIGNUM* value, InputBuffer const* pk); + +/** + * @brief decrypt the cipher using given keyPair + * + * @param cipher the cipher to be decrypted + * @param keypair the keyPair used to decrypt + * @return BIGNUM* the decrypted result + * Note: the caller should release the allocated BIGNUM + */ +BIGNUM* paillier_decrypt(InputBuffer const* cipher, void* keypair); +BIGNUM* paillier_decrypt_without_precompute( + InputBuffer const* cipher, InputBuffer const* sk, InputBuffer const* pk); + +/** + * @brief paillier ciphertext space addition, namely: cipher1 + cipher2 + * + * @param cipher1 + * @param cipher2 + * @param public_key + * @return void* the addition result + */ +void paillier_add(OutputBuffer* cipherBytes, InputBuffer const* cipher1, InputBuffer const* cipher2, + void* public_key); +void paillier_add_without_precompute(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, InputBuffer const* pk); + +/** + * @brief paillier ciphertext space addition, namely: cipher1 - cipher2 + * + * @param cipher1 + * @param cipher2 + * @param public_key + * @return void* the subtraction result + */ +void paillier_sub(OutputBuffer* cipherBytes, InputBuffer const* cipher1, InputBuffer const* cipher2, + void* public_key); +void paillier_sub_without_precompute(OutputBuffer* cipherBytes, InputBuffer const* cipher1, + InputBuffer const* cipher2, InputBuffer const* pkBytes); + +/** + * @brief paillier ciphertext space scala-multiply, namely: v * cipher + * + * @param v the scala-number + * Note: the lifecycle of value is managed by the caller + * @param cipher + * @param public_key + * @return void* the scala-multiply result + */ +void paillier_scala_mul( + OutputBuffer* cipherBytes, BIGNUM* v, InputBuffer const* cipher, void* public_key); +void paillier_scala_mul_without_precompute( + OutputBuffer* cipherBytes, BIGNUM* v, InputBuffer const* cipher, InputBuffer const* pkBytes); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp new file mode 100644 index 00000000..fd5c4686 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp @@ -0,0 +1,187 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file symmetric_encryption.cpp + * @author: yujiechen + * @date 2023-09-08 + */ +#include "symmetric_encryption.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h" +#include "utils/error.h" +#include +#include + +using namespace ppc; +using namespace ppc::crypto; + +thread_local OpenSSLSM4::Ptr g_sm4_impl = std::make_shared(); +thread_local OpenSSL3DES::Ptr g_3des_impl = std::make_shared(); +thread_local OpenSSLAES::Ptr g_aes128_impl = + std::make_shared(OpenSSLAES::AESType::AES128); +thread_local OpenSSLAES::Ptr g_aes192_impl = + std::make_shared(OpenSSLAES::AESType::AES192); +thread_local OpenSSLAES::Ptr g_aes256_impl = + std::make_shared(OpenSSLAES::AESType::AES256); + + +SymCrypto::Ptr obtainSymCryptoImpl(int algorithm) +{ + switch (algorithm) + { + case AlgorithmType::AES_128: + return g_aes128_impl; + case AlgorithmType::AES_192: + return g_aes192_impl; + case AlgorithmType::AES_256: + return g_aes256_impl; + case AlgorithmType::TrippleDES: + return g_3des_impl; + case AlgorithmType::SM4: + return g_sm4_impl; + default: + return nullptr; + } +} + +unsigned int symmetric_block_size(int algorithm) +{ + clear_last_error(); + auto impl = obtainSymCryptoImpl(algorithm); + if (!impl) + { + auto errorMsg = + "symmetric_block_size error for unsupported algorithm: " + std::to_string(algorithm); + set_last_error_msg(-1, errorMsg.c_str()); + return -1; + } + return impl->blockSize(); +} + +int symmetric_key_bytes(int algorithm, int mode) +{ + clear_last_error(); + try + { + auto impl = obtainSymCryptoImpl(algorithm); + if (!impl) + { + auto errorMsg = + "symmetric_key_bytes error for unsupported algorithm: " + std::to_string(algorithm); + set_last_error_msg(-1, errorMsg.c_str()); + return -1; + } + return impl->keyBytes((SymCrypto::OperationMode)mode); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } + return -1; +} +/** + * @brief generate key for symmetric encryption + * + * @param _sk the generated key + */ +void symmetric_generate_key(OutputBuffer* _sk, int algorithm, int mode) +{ + clear_last_error(); + try + { + auto impl = obtainSymCryptoImpl(algorithm); + if (!impl) + { + auto errorMsg = "Unsupported algorithm: " + std::to_string(algorithm); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + impl->generateKey(_sk, (SymCrypto::OperationMode)mode); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief encrypt the plainData into cipher using given algorithm and mode + * + * @param _cipher the encrypted cipher + * @param algorithm the symmetric encryption algorithm, e.g. AES/DES/SM4 + * @param mode the encryption mode + * @param sk the sk used to encrypt + * @param plainData the plainData + */ +void symmetric_encrypt(OutputBuffer* cipher, int algorithm, int mode, InputBuffer const* sk, + InputBuffer const* iv, InputBuffer const* plainData) +{ + clear_last_error(); + try + { + auto impl = obtainSymCryptoImpl(algorithm); + if (!impl) + { + auto errorMsg = + "symmetric_encrypt error: unsupported algorithm " + std::to_string(algorithm); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + impl->encrypt(cipher, (SymCrypto::OperationMode)mode, + bcos::bytesConstRef(sk->data, sk->len), bcos::bytesConstRef(iv->data, iv->len), + bcos::bytesConstRef(plainData->data, plainData->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} + +/** + * @brief decrypt the given cipher into plain using given sk/algorithm and mode + * + * @param plain the decrypted plainData + * @param algorithm the symmetric encryption algorithm, e.g. AES/DES/SM4 + * @param mode the encryption mode + * @param sk the sk used to decrypt + * @param cipher the cipher + */ +void symmetric_decrypt(OutputBuffer* plain, int algorithm, int mode, InputBuffer const* sk, + InputBuffer const* iv, InputBuffer const* cipher) +{ + clear_last_error(); + try + { + auto impl = obtainSymCryptoImpl(algorithm); + if (!impl) + { + auto errorMsg = + "symmetric_decrypt failed for unsupported algorithm: " + std::to_string(algorithm); + set_last_error_msg(-1, errorMsg.c_str()); + return; + } + impl->decrypt(plain, (SymCrypto::OperationMode)mode, bcos::bytesConstRef(sk->data, sk->len), + bcos::bytesConstRef(iv->data, iv->len), bcos::bytesConstRef(cipher->data, cipher->len)); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + set_last_error_msg(-1, errorMsg.c_str()); + } +} diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h new file mode 100644 index 00000000..a23c1e55 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file symmetric_encryption.h + * @author: yujiechen + * @date 2023-08-25 + */ +#ifndef __SYMMETRIC_ENCRYPTION_H__ +#define __SYMMETRIC_ENCRYPTION_H__ + +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif +enum AlgorithmType +{ + AES_128, + AES_192, + AES_256, + TrippleDES, + SM4 +}; +unsigned int symmetric_block_size(int algorithm); +int symmetric_key_bytes(int algorithm, int mode); +/** + * @brief generate key for symmetric encryption + * + * @param _sk the generated key + */ +void symmetric_generate_key(OutputBuffer* _sk, int algorithm, int mode); + +/** + * @brief encrypt the plainData into cipher using given algorithm and mode + * + * @param _cipher the encrypted cipher + * @param algorithm the symmetric encryption algorithm, e.g. AES/DES/SM4 + * @param mode the encryption mode + * @param sk the sk used to encrypt + * @param plainData the plainData + */ +void symmetric_encrypt(OutputBuffer* cipher, int algorithm, int mode, InputBuffer const* sk, + InputBuffer const* iv, InputBuffer const* plainData); + +/** + * @brief decrypt the cipher + * + * @param algorithm + * @param mode + * @param sk + * @param cipher + */ + +/** + * @brief decrypt the given cipher into plain using given sk/algorithm and mode + * + * @param plain the decrypted plainData + * @param algorithm the symmetric encryption algorithm, e.g. AES/DES/SM4 + * @param mode the encryption mode + * @param sk the sk used to decrypt + * @param cipher the cipher + */ +void symmetric_decrypt(OutputBuffer* plain, int algorithm, int mode, InputBuffer const* sk, + InputBuffer const* iv, InputBuffer const* cipher); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp new file mode 100644 index 00000000..67907256 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file error.cpp + * @author: yujiechen + * @date 2023-08-11 + */ + + +#include "error.h" + +#include +#include +#include + +// record c error +thread_local int g_global_error = 0; +thread_local std::string g_global_msg; + +/** + * @brief the last sync operation success or not + * + * @return int + */ +int last_call_success() +{ + return g_global_error == PPC_CRYPTO_C_SDK_SUCCESS; +} + +/** + * @brief gets status of the most recent sync operation + * + * @return int + */ +int get_last_error() +{ + return g_global_error; +} + +/** + * @brief gets error message of the most recent sync operation, effect if get_last_error + * return not zero + * + * @return const char* + */ +const char* get_last_error_msg() +{ + return g_global_msg.c_str(); +} + +void clear_last_error() +{ + g_global_error = 0; + g_global_msg.clear(); +} + +void set_last_error_msg(int error, const char* msg) +{ + g_global_error = error; + g_global_msg = std::string(msg); +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h new file mode 100644 index 00000000..5355814b --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file error.h + * @author: yujiechen + * @date 2023-08-11 + */ + +#ifndef __INCLUDE_ERROR__ +#define __INCLUDE_ERROR__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define PPC_CRYPTO_C_SDK_SUCCESS (0) + +/** + * @brief the last sync operation success or not + * + * @return int + */ +int last_call_success(); + +/** + * @brief gets status of the recent sync operation + * Note: thread safe operation + * + * @return int + */ +int get_last_error(); + +/** + * @brief gets error message of the recent sync operation, effect if get_last_error, + * return not zero + * Note: thread safe operation + * + * @return const char* + */ +const char* get_last_error_msg(); + +/** + * @brief clear the last error + */ +void clear_last_error(); + +/** + * @brief set the last error and error message + */ +void set_last_error_msg(int error, const char* msg); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp new file mode 100644 index 00000000..fdb58790 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file utilities.cpp + * @author: yujiechen + * @date 2023-08-22 + */ +#include "utilities.h" +#include "error.h" +#include + +using namespace bcos; + +int max_msg_size = 255; + +void to_hex(OutputBuffer* hexData, InputBuffer const* bytesData) +{ + clear_last_error(); + try + { + auto hexStr = *(bcos::toHexString(bytesData->data, bytesData->data + bytesData->len)); + hexData->len = hexStr.size(); + memcpy((void*)hexData->data, (const void*)hexStr.data(), hexData->len); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + BCOS_LOG(ERROR) << LOG_DESC("to_hex error") << errorMsg; + set_last_error_msg(-1, errorMsg.data()); + } +} + +void from_hex(OutputBuffer* bytesData, InputBuffer const* hexData) +{ + clear_last_error(); + try + { + // TODO: decrease the copy overhead + auto hexString = std::string(hexData->data, hexData->data + hexData->len); + auto bytesResult = *(bcos::fromHexString(hexString)); + bytesData->len = bytesResult.size(); + memcpy((void*)bytesData->data, (const void*)bytesResult.data(), bytesResult.size()); + } + catch (std::exception const& e) + { + auto errorMsg = boost::diagnostic_information(e); + BCOS_LOG(ERROR) << LOG_DESC("from_hex error") << errorMsg; + set_last_error_msg(-1, errorMsg.data()); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h new file mode 100644 index 00000000..0f1bb6f1 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h @@ -0,0 +1,34 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file utilities.h + * @author: yujiechen + * @date 2023-08-22 + */ + +#ifndef __UTILITIES_H__ +#define __UTILITIES_H__ +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif +extern int max_msg_size; +void to_hex(OutputBuffer* hexData, InputBuffer const* bytesData); +void from_hex(OutputBuffer* bytesData, InputBuffer const* hexData); +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt b/cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt new file mode 100644 index 00000000..214f1ced --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-crypto-c-sdk) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${PPC_CRYPTO_C_SDK_STATIC_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-ppc-crypto-c-sdk WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp b/cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp new file mode 100644 index 00000000..3218c3dc --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestFastOre.cpp + * @author: shawnhe + * @date 2023-11-24 + */ + +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-framework/Common.h" +#include +#include + +using namespace bcos; +using namespace bcos::test; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(fastOreTest, TestPromptFixture) + + +void testStringOreImpl(bcos::bytes const& plain, bool hex) +{ + // generate sk + bcos::bytes skBytes(fast_ore_key_bytes()); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + fast_ore_generate_key(&skBuffer); + + // encrypt + InputBuffer sk{skBytes.data(), skBytes.size()}; + InputBuffer plainBuffer{plain.data(), plain.size()}; + + bcos::bytes cipher(fast_ore_get_cipher_size(plain.size(), hex)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + string_fast_ore_encrypt(&cipherBuffer, &sk, &plainBuffer, hex); + // decrypt + InputBuffer cipherBuffer2{cipher.data(), cipher.size()}; + bcos::bytes decryptedPlain(fast_ore_get_plain_size(cipher.size(), hex)); + OutputBuffer decryptedPlainBuffer{decryptedPlain.data(), decryptedPlain.size()}; + string_fast_ore_decrypt(&decryptedPlainBuffer, &sk, &cipherBuffer2, hex); + std::cout << "##### plain: " << std::string(plain.begin(), plain.end()) << std::endl; + std::cout << "##### decryptedPlain: " + << std::string(decryptedPlain.begin(), decryptedPlain.end()) << std::endl; + // check + BOOST_CHECK(decryptedPlain == plain); +} + +void testIntegerOreImpl(const int64_t& plain, bool hex) +{ + // generate sk + bcos::bytes skBytes(fast_ore_key_bytes()); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + fast_ore_generate_key(&skBuffer); + + // encrypt + InputBuffer sk{skBytes.data(), skBytes.size()}; + + bcos::bytes cipher(fast_ore_get_cipher_size(sizeof(plain), hex)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + integer_fast_ore_encrypt(&cipherBuffer, &sk, plain, hex); + // decrypt + InputBuffer cipherBuffer2{cipher.data(), cipher.size()}; + + int64_t decryptedPlain; + integer_fast_ore_decrypt(&decryptedPlain, &sk, &cipherBuffer2, hex); + + // check + BOOST_CHECK(decryptedPlain == plain); +} + +void testFloatOreImpl(const float50& plain, bool hex) +{ + // generate sk + bcos::bytes skBytes(fast_ore_key_bytes()); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + fast_ore_generate_key(&skBuffer); + + // encrypt + InputBuffer sk{skBytes.data(), skBytes.size()}; + + auto str = plain.str(); + bcos::bytes cipher(fast_ore_get_float_cipher_size(str.size(), hex)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + + + InputBuffer inputBuffer{(bcos::byte*)str.data(), str.size()}; + float_fast_ore_encrypt(&cipherBuffer, &sk, &inputBuffer, hex); + + // decrypt + InputBuffer cipherBuffer2{cipherBuffer.data, cipherBuffer.len}; + + std::string decryptedPlain; + decryptedPlain.resize(32); + OutputBuffer decryptedPlainBuffer{(bcos::byte*)decryptedPlain.data(), decryptedPlain.size()}; + float_fast_ore_decrypt(&decryptedPlainBuffer, &sk, &cipherBuffer2, hex); + decryptedPlain.resize(decryptedPlainBuffer.len); + + // check + BOOST_CHECK(float50{decryptedPlain} == plain); +} + +BOOST_AUTO_TEST_CASE(testStringOreOps) +{ + std::string plain = "adbwerwerwe"; + bcos::bytes plainBytes(plain.begin(), plain.end()); + testStringOreImpl(plainBytes, true); + testStringOreImpl(plainBytes, false); + + plain = "中文中文"; + bcos::bytes plainBytes2(plain.begin(), plain.end()); + testStringOreImpl(plainBytes2, true); + testStringOreImpl(plainBytes2, false); +} + +BOOST_AUTO_TEST_CASE(testIntegerOreOps) +{ + for (int i = 0; i < 1000; i++) + { + testIntegerOreImpl(i * 123456, true); + testIntegerOreImpl(i * 123456, false); + testIntegerOreImpl(i * -123456, true); + testIntegerOreImpl(i * -123456, false); + } +} + +BOOST_AUTO_TEST_CASE(testFloatOreOps) +{ + for (int i = 0; i < 1000; i++) + { + auto plain = float50{"-134335849.23449798"}; + testFloatOreImpl(plain * i, true); + testFloatOreImpl(plain * i, false); + testFloatOreImpl(plain * -i, true); + testFloatOreImpl(plain * -i, false); + } +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp b/cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp new file mode 100644 index 00000000..7a8df98c --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp @@ -0,0 +1,461 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestHomoPaillier.cpp + * @author: yujiechen + * @date 2023-08-15 + */ +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-crypto-c-sdk/floating_point_ihc.h" +#include "ppc-crypto-c-sdk/homo_ihc.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/crypto/Ihc.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-homo/paillier/OpenSSLPaillierKeyPair.h" +#include +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc::homo; +using namespace ppc::crypto; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(homoPaillierTest, TestPromptFixture) + +void checkPrivateKey(PaillierPrivateKey* sk1, PaillierPrivateKey* sk2) +{ + BOOST_CHECK(sk1->keyBits == sk2->keyBits); + BOOST_CHECK(sk1->lambda.cmp(sk2->lambda.bn().get()) == 0); + BOOST_CHECK(sk1->p.cmp(sk2->p.bn().get()) == 0); + BOOST_CHECK(sk1->q.cmp(sk2->q.bn().get()) == 0); + BOOST_CHECK(sk1->pSqrt.cmp(sk2->pSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qSqrt.cmp(sk2->qSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qSqrtInverse.cmp(sk2->qSqrtInverse.bn().get()) == 0); + BOOST_CHECK(sk1->pOrderSqrt.cmp(sk2->pOrderSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qOrderSqrt.cmp(sk2->qOrderSqrt.bn().get()) == 0); +} +void checkPublicKey(PaillierPublicKey* pk1, PaillierPublicKey* pk2) +{ + BOOST_CHECK(pk1->keyBits == pk2->keyBits); + BOOST_CHECK(pk1->n.cmp(pk2->n.bn().get()) == 0); + BOOST_CHECK(pk1->nSqrt.cmp(pk2->nSqrt.bn().get()) == 0); + BOOST_CHECK(pk1->h.cmp(pk2->h.bn().get()) == 0); + BOOST_CHECK(pk1->h_s.cmp(pk2->h_s.bn().get()) == 0); +} +void testPaillierKeyPairImpl(int _keyBits) +{ + auto keyPair = paillier_generate_keypair(_keyBits); + auto encodedPk = paillier_get_public_key_bytes_from_keyPair(keyPair); + auto encodedSk = paillier_get_private_key_bytes_from_keypair(keyPair); + // load the keypair + InputBuffer sk{(const unsigned char*)encodedSk.data, encodedSk.len}; + InputBuffer pk{(const unsigned char*)encodedPk.data, encodedPk.len}; + auto decodedKeyPair = paillier_load_keypair(&sk, &pk); + // check the key pair + auto pk1 = (PaillierPublicKey*)(((OpenSSLPaillierKeyPair*)keyPair)->pk()); + auto pk2 = (PaillierPublicKey*)(((OpenSSLPaillierKeyPair*)decodedKeyPair)->pk()); + checkPublicKey(pk1, pk2); + + auto sk1 = (PaillierPrivateKey*)(((OpenSSLPaillierKeyPair*)keyPair)->sk()); + auto sk2 = (PaillierPrivateKey*)(((OpenSSLPaillierKeyPair*)decodedKeyPair)->sk()); + checkPrivateKey(sk1, sk2); + + // load the public key + auto decodedPk = paillier_load_public_key(&pk); + checkPublicKey(pk1, (PaillierPublicKey*)decodedPk); +} +BOOST_AUTO_TEST_CASE(testPaillierKeyPair) +{ + testPaillierKeyPairImpl(2048); +} + +bcos::bytes testEncryptDecrypt(BIGNUM* _value, void* keypair, int _keyBits) +{ + auto pk = ((OpenSSLPaillierKeyPair*)keypair)->pk(); + BigNum n; + paillier_n_from_pk(n.bn().get(), (void*)pk); + std::cout << "#### n: " << BN_print_fp(stdout, n.bn().get()) << std::endl; + + bcos::bytes cipher(paillier_max_cipher_bytes(_keyBits), 0); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + // encrypt fast + bcos::bytes rBytes(paillier_r_bytes_len(_keyBits)); + OutputBuffer rBuffer{rBytes.data(), rBytes.size()}; + paillier_encryt_fast(&cipherBuffer, &rBuffer, _value, keypair); + if (!last_call_success()) + { + std::cout << "### paillier_encryt_fast error for: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + std::cout << "#### rBuffer: " << bcos::toHex(rBytes) << ", rBuffer size: " << rBuffer.len + << std::endl; + + // encrypt + bcos::bytes cipher2(paillier_max_cipher_bytes(_keyBits), 0); + OutputBuffer cipherBuffer2{cipher2.data(), cipher2.size()}; + paillier_encryt(&cipherBuffer2, nullptr, _value, pk); + if (!last_call_success()) + { + std::cout << "### paillier_encryt error for: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success() == 1); + // decrypt with keypair + InputBuffer inputBuffer{cipherBuffer.data, cipherBuffer.len}; + auto decodeV = paillier_decrypt(&inputBuffer, keypair); + if (!last_call_success()) + { + std::cout << "#### paillier decrypt error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success() == 1); + BOOST_CHECK(BN_cmp(_value, decodeV) == 0); + + InputBuffer inputBuffer2{cipherBuffer2.data, cipherBuffer2.len}; + decodeV = paillier_decrypt(&inputBuffer2, keypair); + if (!last_call_success()) + { + std::cout << "#### paillier decrypt error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + std::cout << "###### expectedValue: " << std::endl; + BN_print_fp(stdout, _value); + std::cout << "###### decodeV:" << std::endl; + BN_print_fp(stdout, decodeV); + BOOST_CHECK(BN_cmp(_value, decodeV) == 0); + cipher.resize(cipherBuffer.len); + return cipher; +} + +void testPaillierOpsImpl( + BigNum const& m1, BigNum const& m2, BigNum const& v, void* keypair, int keyBits) +{ + auto pk = ((OpenSSLPaillierKeyPair*)keypair)->pk(); + auto c1 = testEncryptDecrypt(m1.bn().get(), keypair, keyBits); + InputBuffer c1Buffer{c1.data(), c1.size()}; + + auto c2 = testEncryptDecrypt(m2.bn().get(), keypair, keyBits); + InputBuffer c2Buffer{c2.data(), c2.size()}; + + /// check m1 + m2 + bcos::bytes cipherResult(paillier_max_cipher_bytes(keyBits), 0); + OutputBuffer resultBuffer{cipherResult.data(), cipherResult.size()}; + paillier_add(&resultBuffer, &c1Buffer, &c2Buffer, pk); + if (!last_call_success()) + { + std::cout << "### paillier_add error for: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + // decrypt and check + InputBuffer encodedBuffer{resultBuffer.data, resultBuffer.len}; + auto decodeV = paillier_decrypt(&encodedBuffer, keypair); + if (!last_call_success()) + { + std::cout << "##### paillier_decrypt error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + auto addResult = m1.add(m2.bn().get()); + std::cout << "###### expectedAddResult: " << std::endl; + BN_print_fp(stdout, addResult.bn().get()); + std::cout << "###### addResult:" << std::endl; + BN_print_fp(stdout, decodeV); + BOOST_CHECK(BN_cmp(addResult.bn().get(), decodeV) == 0); + + /// check m1 - m2 + OutputBuffer subBuffer{cipherResult.data(), cipherResult.size()}; + paillier_sub(&subBuffer, &c1Buffer, &c2Buffer, pk); + if (!last_call_success()) + { + std::cout << "### paillier_sub error for: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + // decrypt and check + InputBuffer subResultBuffer{subBuffer.data, subBuffer.len}; + decodeV = paillier_decrypt(&subResultBuffer, keypair); + if (!last_call_success()) + { + std::cout << "#### paillier decrypt error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + auto subResult = m1.sub(m2.bn().get()); + std::cout << "###### m1:" << std::endl; + BN_print_fp(stdout, m1.bn().get()); + std::cout << "###### m2:" << std::endl; + BN_print_fp(stdout, m2.bn().get()); + + std::cout << "###### expectedSubResult: " << std::endl; + BN_print_fp(stdout, subResult.bn().get()); + std::cout << "###### subResult:" << std::endl; + BN_print_fp(stdout, decodeV); + BOOST_CHECK(BN_cmp(subResult.bn().get(), decodeV) == 0); + + /// check v * m1 + OutputBuffer mulBuffer{cipherResult.data(), cipherResult.size()}; + paillier_scala_mul(&mulBuffer, v.bn().get(), &c1Buffer, pk); + if (!last_call_success()) + { + std::cout << "### paillier_scala_mul error for: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + InputBuffer mulResultBuffer{mulBuffer.data, mulBuffer.len}; + decodeV = paillier_decrypt(&mulResultBuffer, keypair); + if (!last_call_success()) + { + std::cout << "#### paillier decrypt error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + auto ctx = createBNContext(); + BigNum mulResult; + v.mul(mulResult.bn().get(), m1.bn().get(), ctx.get()); + std::cout << "###### expectedMulResult: " << std::endl; + BN_print_fp(stdout, mulResult.bn().get()); + std::cout << "###### mulResult:" << std::endl; + BN_print_fp(stdout, decodeV); + BOOST_CHECK(BN_cmp(mulResult.bn().get(), decodeV) == 0); +} +// test paillier encrypt-decrypt +BOOST_AUTO_TEST_CASE(testPaillierEncryptDecrypt) +{ + BigNum value(bcos::utcSteadyTime()); + int keyBits = 2048; + auto keypair = paillier_generate_keypair(keyBits); + testEncryptDecrypt(value.bn().get(), keypair, keyBits); +} +// test paillier ops +BOOST_AUTO_TEST_CASE(testPaillierOps) +{ + int64_t m1 = 12323423434 + rand(); + int64_t m2 = 2342342344 + rand(); + int64_t v = 21334234234; + int keyBits = 2048; + srand(bcos::utcSteadyTime()); + + // positive case + auto keypair = paillier_generate_keypair(keyBits); + testPaillierOpsImpl(BigNum(m1), BigNum(m2), BigNum(v), keypair, keyBits); + + // negative case + m1 = -1234324 - rand(); + testPaillierOpsImpl(BigNum(m1), BigNum(m2), BigNum(v), keypair, keyBits); + + v = -34534 - rand(); + testPaillierOpsImpl(BigNum(m1), BigNum(m2), BigNum(v), keypair, keyBits); +} + +bcos::bytes testEncryptionDecryption(bcos::bytes const& skBytes, int mode, BigNum const& m1) +{ + // encrypt + InputBuffer sk{skBytes.data(), skBytes.size()}; + bcos::bytes cipher(ihc_cipher_bytes(mode)); + std::cout << "##### cipher len: " << cipher.size() << std::endl; + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + ihc_encrypt(&cipherBuffer, mode, &sk, m1.bn().get()); + std::cout << "##### cipher1: " << toHex(cipher) << std::endl; + // decrypt + InputBuffer cipherInput{cipher.data(), cipher.size()}; + BigNum resulttmp; + ihc_decrypt(resulttmp.bn().get(), mode, &sk, &cipherInput); + if (!last_call_success()) + { + std::cout << "##### testEncryptionDecryption error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + // check + std::cout << "#### m1: " << std::endl; + BN_print_fp(stdout, m1.bn().get()); + std::cout << "#### resulttmp: " << std::endl; + BN_print_fp(stdout, resulttmp.bn().get()); + BOOST_CHECK(resulttmp.cmp(m1.bn().get()) == 0); + return cipher; +} +void testIhcImpl(int mode, BigNum const& m1, BigNum const& m2, BigNum const& v) +{ + // generate sk + bcos::bytes skBytes(ihc_key_bytes(mode)); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + ihc_generate_key(&skBuffer, mode); + InputBuffer sk{skBytes.data(), skBytes.size()}; + // c1 + auto c1 = testEncryptionDecryption(skBytes, mode, m1); + // c2 + auto c2 = testEncryptionDecryption(skBytes, mode, m2); + // add test: m1 + m2 + std::cout << "##### testIhcImpl: add case" << std::endl; + InputBuffer cipher1Input{c1.data(), c1.size()}; + InputBuffer cipher2Input{c2.data(), c2.size()}; + bcos::bytes cipherSum(ihc_cipher_bytes(mode)); + OutputBuffer cipherBufferSum{cipherSum.data(), cipherSum.size()}; + ihc_add(&cipherBufferSum, mode, &cipher1Input, &cipher2Input); + std::cout << "##### cipherSum: " << toHex(cipherSum) << std::endl; + // decrypt + InputBuffer cipherSumBuffer{cipherSum.data(), cipherSum.size()}; + BigNum addResult; + ihc_decrypt(addResult.bn().get(), mode, &sk, &cipherSumBuffer); + // check + BigNum expected = m1.add(m2.bn().get()); + BOOST_CHECK(expected.cmp(addResult.bn().get()) == 0); + std::cout << "##### testIhcImpl: add case finish" << std::endl; +} + +bcos::bytes testIhcFloatingEncryptionDecryption( + int mode, bcos::bytes const& skBytes, FloatingPointNumber const& m1) + +{ + InputBuffer sk{skBytes.data(), skBytes.size()}; + // encryption + bcos::bytes cipher(ihc_floating_cipher_bytes(mode)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + ihc_floating_encrypt(&cipherBuffer, mode, &sk, m1.value.bn().get(), m1.exponent); + std::cout << "cipher: " << toHex(cipher) << std::endl; + InputBuffer cipherInput{cipher.data(), cipher.size()}; + + // decrypt + BigNum decryptedResult; + int16_t decryptedExponent = 0; + + ihc_floating_decrypt(decryptedResult.bn().get(), &decryptedExponent, mode, &sk, &cipherInput); + BOOST_CHECK(m1.exponent == decryptedExponent); + BOOST_CHECK(m1.value.cmp(decryptedResult.bn().get()) == 0); + return cipher; +} + +void testIhcFloatingImpl( + int mode, std::string const& s1, std::string const& s2, std::string const& s3) +{ + std::cout << "testIhcFloatingImpl" << std::endl; + // 生成密钥 + bcos::bytes skBytes(ihc_key_bytes(mode)); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + ihc_generate_key(&skBuffer, mode); + InputBuffer sk{skBytes.data(), skBytes.size()}; + + auto codec = std::make_shared(); + auto m1 = codec->toFloatingPoint(s1); + auto m2 = codec->toFloatingPoint(s2); + auto m3 = codec->toFloatingPoint(s3); + float50 f1(s1); + float50 f2(s2); + float50 f3(s3); + + auto c1 = testIhcFloatingEncryptionDecryption(mode, skBytes, m1); + auto c2 = testIhcFloatingEncryptionDecryption(mode, skBytes, m2); + auto c3 = testIhcFloatingEncryptionDecryption(mode, skBytes, m3); + // add test + std::cout << "#### testIhcFloatingImpl: add test" << std::endl; + bcos::bytes cipherSum(ihc_floating_cipher_bytes(mode)); + OutputBuffer cipherBufferSum{cipherSum.data(), cipherSum.size()}; + InputBuffer c1Buffer{c1.data(), c1.size()}; + InputBuffer c2Buffer{c2.data(), c2.size()}; + ihc_floating_add(&cipherBufferSum, mode, &c1Buffer, &c2Buffer); + InputBuffer cipherSumInput{cipherSum.data(), cipherSum.size()}; + FloatingPointNumber addResult; + ihc_floating_decrypt( + addResult.value.bn().get(), &addResult.exponent, mode, &sk, &cipherSumInput); + std::cout << std::fixed << std::setprecision(15); + std::cout << "#### f1 + f2: " << (f1 + f2) << std::endl; + std::cout << "#### addResult: " << codec->toFloat50(addResult) << std::endl; + float50 epsilon("0.0000000000001"); + BOOST_CHECK(fabs(codec->toFloat50(addResult) - f1 - f2) <= epsilon); + std::cout << "#### testIhcFloatingImpl: add test finished" << std::endl; + + // sub test + std::cout << "#### testIhcFloatingImpl: sub test" << std::endl; + bcos::bytes cipherSub(ihc_floating_cipher_bytes(mode)); + OutputBuffer cipherSubBuffer{cipherSub.data(), cipherSub.size()}; + ihc_floating_sub(&cipherSubBuffer, mode, &c1Buffer, &c2Buffer); + InputBuffer cipherSubInput{cipherSub.data(), cipherSub.size()}; + FloatingPointNumber subResult; + ihc_floating_decrypt( + subResult.value.bn().get(), &subResult.exponent, mode, &sk, &cipherSubInput); + std::cout << std::fixed << std::setprecision(15); + std::cout << "#### f1 - f2: " << (f1 - f2) << std::endl; + std::cout << "#### subResult: " << codec->toFloat50(subResult) << std::endl; + BOOST_CHECK(fabs(codec->toFloat50(subResult) - (f1 - f2)) <= epsilon); + std::cout << "#### testIhcFloatingImpl: sub test finished" << std::endl; + + // scalaMul test + std::cout << "#### testIhcFloatingImpl: scalaMul test" << std::endl; + bcos::bytes cipherMul(ihc_floating_cipher_bytes(mode)); + OutputBuffer cipherMulBuffer{cipherMul.data(), cipherMul.size()}; + ihc_floating_scalaMul(&cipherMulBuffer, mode, m3.value.bn().get(), m3.exponent, &c1Buffer); + InputBuffer cipherMulInput{cipherMul.data(), cipherMul.size()}; + FloatingPointNumber mulResult; + ihc_floating_decrypt( + mulResult.value.bn().get(), &mulResult.exponent, mode, &sk, &cipherMulInput); + BOOST_CHECK(fabs(codec->toFloat50(mulResult) - (f1 * f3)) <= epsilon); + std::cout << "#### testIhcFloatingImpl: scalaMul test finished" << std::endl; +} + +void testIhcItermImpl(int _mode) +{ + // generate sk + bcos::bytes skBytes(ihc_key_bytes(_mode)); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + ihc_generate_key(&skBuffer, _mode); + std::cout << "##### skBuffer: " << toHex(skBytes) << std::endl; + // encrypt + InputBuffer sk{skBytes.data(), skBytes.size()}; + BigNum m1(10000); + bcos::bytes cipherSum(ihc_cipher_bytes(_mode)); + OutputBuffer cipherBufferSum{cipherSum.data(), cipherSum.size()}; + BigNum expected(10000); + ihc_encrypt(&cipherBufferSum, _mode, &sk, m1.bn().get()); + for (int i = 0; i < 100; i++) + { + bcos::bytes cipher(ihc_cipher_bytes(_mode)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + ihc_encrypt(&cipherBuffer, _mode, &sk, m1.bn().get()); + InputBuffer cipherInput{cipher.data(), cipher.size()}; + InputBuffer cipherSumInput{cipherSum.data(), cipherSum.size()}; + ihc_add(&cipherBufferSum, _mode, &cipherSumInput, &cipherInput); + if (!last_call_success()) + { + std::cout << "#### ihc add error: " << get_last_error_msg() << std::endl; + } + BOOST_CHECK(last_call_success()); + BigNum result; + ihc_decrypt(result.bn().get(), _mode, &sk, &cipherSumInput); + expected = expected.add(m1.bn().get()); + BOOST_CHECK(result.cmp(expected.bn().get()) == 0); + } +} + +BOOST_AUTO_TEST_CASE(testIhcOps) +{ + int64_t m1 = 123213231; + int64_t m2 = 123213231; + int64_t v = 23; + int keyBits = 64; + testIhcImpl((int)Ihc::IhcMode::IHC_128, BigNum(m1), BigNum(m2), BigNum(v)); + // negative case + m1 = -1234324 - rand(); + testIhcImpl((int)Ihc::IhcMode::IHC_128, BigNum(m1), BigNum(m2), BigNum(v)); + v = -34534 - rand(); + testIhcImpl((int)Ihc::IhcMode::IHC_128, BigNum(m1), BigNum(m2), BigNum(v)); + + testIhcItermImpl((int)Ihc::IhcMode::IHC_128); + + std::string s1("-23423423.24534534"); + std::string s2("0.0000234688"); + std::string s3("-1233.002348"); + testIhcFloatingImpl((int)Ihc::IhcMode::IHC_128, s1, s2, s3); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-c-sdk/tests/main.cpp b/cpp/ppc-crypto-c-sdk/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-crypto-c-sdk/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-crypto-core/CMakeLists.txt b/cpp/ppc-crypto-core/CMakeLists.txt new file mode 100644 index 00000000..648242e1 --- /dev/null +++ b/cpp/ppc-crypto-core/CMakeLists.txt @@ -0,0 +1,8 @@ +project(ppc-crypto-core VERSION ${VERSION}) + +add_subdirectory(src) +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/CMakeLists.txt b/cpp/ppc-crypto-core/src/CMakeLists.txt new file mode 100644 index 00000000..ed0fa3f2 --- /dev/null +++ b/cpp/ppc-crypto-core/src/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${CRYPTO_CORE_TARGET} ${SRCS}) + +find_package(OpenSSL REQUIRED) +message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}") +message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}") + +target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/Common.h b/cpp/ppc-crypto-core/src/Common.h new file mode 100644 index 00000000..fce2f7ad --- /dev/null +++ b/cpp/ppc-crypto-core/src/Common.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "ppc-framework/Common.h" +#include +#include + +namespace ppc::crypto +{ +DERIVE_PPC_EXCEPTION(HashException); +DERIVE_PPC_EXCEPTION(SymCryptoException); +DERIVE_PPC_EXCEPTION(OreException); + +DERIVE_PPC_EXCEPTION(UnsupportedHashType); +DERIVE_PPC_EXCEPTION(BitVectorException); +DERIVE_PPC_EXCEPTION(FastOreException); +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h b/cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h new file mode 100644 index 00000000..985377ed --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BLAKE2bHash.h + * @author: shawnhe + * @date 2022-12-4 + */ + +#pragma once +#include "../Common.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +namespace ppc::crypto +{ +class BLAKE2bHashState : public HashState +{ +public: + using Ptr = std::shared_ptr; + BLAKE2bHashState() : m_state(std::make_shared()) {} + ~BLAKE2bHashState() override = default; + + void* state() override { return (void*)m_state.get(); } + +private: + std::shared_ptr m_state; +}; + +class BLAKE2bHash : public Hash +{ +public: + using Ptr = std::shared_ptr; + BLAKE2bHash() = default; + ~BLAKE2bHash() override = default; + + // the hashBytes length of given hash-algorithm + size_t hashLen() const override { return crypto_generichash_BYTES_MAX; } + + // the implementation of the hash-algorithm + ppc::protocol::HashImplName type() const override + { + return ppc::protocol::HashImplName::BLAKE2b; + } + + // calculate hash and output default length + bcos::bytes hash(bcos::bytesConstRef _input) const override + { + return hash(_input, crypto_generichash_BYTES_MAX); + } + + // calculate hash without key + bcos::bytes hash(bcos::bytesConstRef _input, size_t _outLen) const + { + return hash(_input, bcos::bytesConstRef(), _outLen); + } + + // calculate hash with a key + bcos::bytes hash(bcos::bytesConstRef _input, bcos::bytesConstRef _key, size_t _outLen) const + { + if (_outLen < crypto_generichash_BYTES_MIN || _outLen > crypto_generichash_BYTES_MAX) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "outLen of BLAKE2b should be between " + + std::to_string(crypto_generichash_BYTES_MIN) + " and " + + std::to_string(crypto_generichash_BYTES_MAX))); + } + + bcos::bytes result(_outLen); + auto ret = crypto_generichash(result.data(), _outLen, (const unsigned char*)_input.data(), + _input.size(), (const unsigned char*)_key.data(), _key.size()); + if (ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment( + "hash error for BLAKE2b_hash failed, code: " + std::to_string(ret))); + } + return result; + } + + // init the hash-state + HashState::Ptr init() const override { return init(bcos::bytesConstRef()); } + + // init the hash-state with a key + HashState::Ptr init(bcos::bytesConstRef _key) const + { + auto state = std::make_shared(); + auto ret = crypto_generichash_init((crypto_generichash_state*)state->state(), + (const unsigned char*)_key.data(), _key.size(), crypto_generichash_BYTES_MAX); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "BLAKE2b_init error: " + std::to_string(ret))); + } + return state; + } + + // update new message into the given hash-state + void update(HashState::Ptr _state, bcos::bytesConstRef _data) const override + { + auto ret = crypto_generichash_update((crypto_generichash_state*)_state->state(), + (const unsigned char*)_data.data(), _data.size()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "BLAKE2b_update error: " + std::to_string(ret))); + } + } + + // obtain the hash-result from the given hash-state + bcos::bytes final(HashState::Ptr _state) const override + { + bcos::bytes result(crypto_generichash_BYTES_MAX); + auto ret = crypto_generichash_final((crypto_generichash_state*)_state->state(), + result.data(), crypto_generichash_BYTES_MAX); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "BLAKE2b_final error: " + std::to_string(ret))); + } + return result; + } +}; +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h b/cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h new file mode 100644 index 00000000..785b60db --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h @@ -0,0 +1,283 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BitMixMurmurHash.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include +#include +namespace ppc::crypto +{ +class BitMixMurmurHash +{ +public: + BitMixMurmurHash() = default; + ~BitMixMurmurHash() = default; + + template + uint64_t hash(T const& _key, unsigned _hashBitsLen, uint32_t _seed = 32) const + { + // Note: _seed is unused when the key is integer + boost::ignore_unused(_seed); + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value, + "only support string/string_view/bytes/bytesConstRef/int8_t to int64_t/uint8_t to " + "uint64_t"); + // the bytes type + if constexpr (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) + { + return hashData((const unsigned char*)_key.data(), _key.size(), _hashBitsLen, _seed); + } + else + { + // the int type + return (_hashBitsLen <= 32 ? (hash32N(_key)) : hash64N(_key)); + } + } + +protected: + uint64_t hashData( + const unsigned char* key, uint32_t len, unsigned _hashBitsLen, uint32_t _seed) const + { + return (_hashBitsLen <= 32 ? (hash32(key, len, _seed)) : hash64(key, len, _seed)); + } + + inline uint64_t hash64N(uint64_t _value) const + { // Bit mix from MurmurHash64/CLHash + _value ^= _value >> 33; + _value *= 0xff51afd7ed558ccdULL; + _value ^= _value >> 33; + _value *= 0xc4ceb9fe1a85ec53ULL; + _value ^= _value >> 33; + return _value; + } + + inline uint32_t hash32N(uint32_t _value) const + { // Bit mix from MurmurHash32 + _value ^= _value >> 16; + _value *= 0x85ebca6b; + _value ^= _value >> 13; + _value *= 0xc2b2ae35; + _value ^= _value >> 16; + return _value; + } + + // Note: when generating 32bits hash, there is a high probability of a collision + // so we default to generate 64bits hash + uint32_t hash32(const unsigned char* key, uint32_t len, uint32_t seed) const + { + uint32_t c1 = 0xcc9e2d51; + uint32_t c2 = 0x1b873593; + uint32_t r1 = 15; + uint32_t r2 = 13; + uint32_t m = 5; + uint32_t n = 0xe6546b64; + uint32_t h = 0; + uint32_t k = 0; + uint8_t* d = (uint8_t*)key; // 32 bit extract from `key' + const uint32_t* chunks = NULL; + const uint8_t* tail = NULL; // tail - last 8 bytes + int i = 0; + int l = len / 4; // chunk length + + h = seed; + + chunks = (const uint32_t*)(d + l * 4); // body + tail = (const uint8_t*)(d + l * 4); // last 8 byte chunk of `key' + + // for each 4 byte chunk of `key' + for (i = -l; i != 0; ++i) + { + // next 4 byte chunk of `key' + k = chunks[i]; + + // encode next 4 byte chunk of `key' + k *= c1; + k = (k << r1) | (k >> (32 - r1)); + k *= c2; + + // append to hash + h ^= k; + h = (h << r2) | (h >> (32 - r2)); + h = h * m + n; + } + k = 0; + // remainder + switch (len & 3) + { // `len % 4' + case 3: + k ^= (tail[2] << 16); + break; + case 2: + k ^= (tail[1] << 8); + break; + case 1: + k ^= tail[0]; + k *= c1; + k = (k << r1) | (k >> (32 - r1)); + k *= c2; + h ^= k; + break; + default: // unreachable-branch + break; + } + + h ^= len; + + h ^= (h >> 16); + h *= 0x85ebca6b; + h ^= (h >> 13); + h *= 0xc2b2ae35; + h ^= (h >> 16); + + return h; + } + + uint64_t hash64(const unsigned char* key, const int len, const uint32_t seed) const + { + const uint8_t* data = (const uint8_t*)key; + const int nblocks = len / 16; + int i; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + uint64_t c1 = 0x87c37b91114253d5; + uint64_t c2 = 0x4cf5ad432745937f; + const uint64_t* blocks = (const uint64_t*)(data); + + for (i = 0; i < nblocks; i++) + { + uint64_t k1 = blocks[i * 2]; + uint64_t k2 = blocks[i * 2 + 1]; + + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = rotl64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = rotl64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) + { + case 15: + k2 ^= (uint64_t)(tail[14]) << 48; + break; + case 14: + k2 ^= (uint64_t)(tail[13]) << 40; + break; + case 13: + k2 ^= (uint64_t)(tail[12]) << 32; + break; + case 12: + k2 ^= (uint64_t)(tail[11]) << 24; + break; + case 11: + k2 ^= (uint64_t)(tail[10]) << 16; + break; + case 10: + k2 ^= (uint64_t)(tail[9]) << 8; + break; + case 9: + k2 ^= (uint64_t)(tail[8]) << 0; + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; + break; + case 8: + k1 ^= (uint64_t)(tail[7]) << 56; + break; + case 7: + k1 ^= (uint64_t)(tail[6]) << 48; + break; + case 6: + k1 ^= (uint64_t)(tail[5]) << 40; + break; + case 5: + k1 ^= (uint64_t)(tail[4]) << 32; + break; + case 4: + k1 ^= (uint64_t)(tail[3]) << 24; + break; + case 3: + k1 ^= (uint64_t)(tail[2]) << 16; + break; + case 2: + k1 ^= (uint64_t)(tail[1]) << 8; + break; + case 1: + k1 ^= (uint64_t)(tail[0]) << 0; + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; + break; + default: // unreachable-branch + break; + }; + h1 ^= len; + h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + return h1; + } + + inline uint64_t rotl64(uint64_t x, int8_t r) const { return (x << r) | (x >> (64 - r)); } + inline uint64_t fmix64(uint64_t k) const + { + k ^= k >> 33; + k *= 0xff51afd7ed558ccd; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53; + k ^= k >> 33; + return k; + } +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h b/cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h new file mode 100644 index 00000000..500713d3 --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HashFactoryImpl.h + * @author: yujiechen + * @date 2023-1-3 + */ +#pragma once +#include "../Common.h" +#include "BLAKE2bHash.h" +#include "MD5Hash.h" +#include "SM3Hash.h" +#include "Sha256Hash.h" +#include "Sha512Hash.h" +#include "ppc-framework/protocol/Protocol.h" + +namespace ppc::crypto +{ +class HashFactoryImpl : public HashFactory +{ +public: + using Ptr = std::shared_ptr; + HashFactoryImpl() = default; + ~HashFactoryImpl() override = default; + + Hash::Ptr createHashImpl(int8_t _hashType) const override + { + switch (_hashType) + { + case (int8_t)ppc::protocol::HashImplName::SHA256: + return std::make_shared(); + case (int8_t)ppc::protocol::HashImplName::SHA512: + return std::make_shared(); + case (int8_t)ppc::protocol::HashImplName::SM3: + return std::make_shared(); + case (int8_t)ppc::protocol::HashImplName::MD5: + return std::make_shared(); + case (int8_t)ppc::protocol::HashImplName::BLAKE2b: + return std::make_shared(); + default: + BOOST_THROW_EXCEPTION(UnsupportedHashType() << bcos::errinfo_comment( + "unsupported hashType: " + std::to_string(_hashType))); + } + } +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/hash/MD5Hash.h b/cpp/ppc-crypto-core/src/hash/MD5Hash.h new file mode 100644 index 00000000..b685ba5f --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/MD5Hash.h @@ -0,0 +1,102 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MD5Hash.h + * @author: yujiechen + * @date 2022-11-10 + */ +#pragma once +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "../Common.h" +#include "openssl/md5.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +namespace ppc::crypto +{ +class MD5HashState : public HashState +{ +public: + using Ptr = std::shared_ptr; + MD5HashState() : m_state(std::make_shared()) {} + ~MD5HashState() override = default; + + void* state() override { return (void*)m_state.get(); } + +private: + std::shared_ptr m_state; +}; + +class MD5Hash : public Hash +{ +public: + using Ptr = std::shared_ptr; + MD5Hash() = default; + ~MD5Hash() override = default; + + // the hashBytes length of given hash-algorithm + size_t hashLen() const override { return MD5_DIGEST_LENGTH; } + // the implementation of the hash-algorithm + ppc::protocol::HashImplName type() const override { return ppc::protocol::HashImplName::MD5; } + + // calculate-hash + bcos::bytes hash(bcos::bytesConstRef _input) const override + { + bcos::bytes result(MD5_DIGEST_LENGTH); + MD5(_input.data(), _input.size(), result.data()); + return result; + } + + // init the hash-state + HashState::Ptr init() const override + { + auto hashState = std::make_shared(); + auto ret = MD5_Init((MD5_CTX*)hashState->state()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment("md5_init error: " + std::to_string(ret))); + } + return hashState; + } + + // update new message into the given hash-state + void update(HashState::Ptr _state, bcos::bytesConstRef _data) const override + { + auto ret = MD5_Update((MD5_CTX*)_state->state(), (const void*)_data.data(), _data.size()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "md5_update error: " + std::to_string(ret))); + } + } + + // obtain the hash-result from the given hash-state + bcos::bytes final(HashState::Ptr _state) const override + { + bcos::bytes result(MD5_DIGEST_LENGTH); + auto ret = MD5_Final(result.data(), (MD5_CTX*)_state->state()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "md5_final error: " + std::to_string(ret))); + } + return result; + } +}; +} // namespace ppc::crypto +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/hash/OpenSSLHash.h b/cpp/ppc-crypto-core/src/hash/OpenSSLHash.h new file mode 100644 index 00000000..e69de29b diff --git a/cpp/ppc-crypto-core/src/hash/SM3Hash.h b/cpp/ppc-crypto-core/src/hash/SM3Hash.h new file mode 100644 index 00000000..9c742610 --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/SM3Hash.h @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SM3Hash.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "../Common.h" +#include "openssl/sm3.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" + +namespace ppc::crypto +{ +class SM3HashState : public HashState +{ +public: + using Ptr = std::shared_ptr; + SM3HashState() : m_state(std::make_shared()) {} + ~SM3HashState() override = default; + + void* state() override { return (void*)m_state.get(); } + +private: + std::shared_ptr m_state; +}; + +class SM3Hash : public Hash +{ +public: + using Ptr = std::shared_ptr; + SM3Hash() = default; + ~SM3Hash() override = default; + + // the hashBytes length of given hash-algorithm + size_t hashLen() const override { return 32; } + // the implementation of the hash-algorithm + ppc::protocol::HashImplName type() const override { return ppc::protocol::HashImplName::SM3; } + + // calculate-hash + bcos::bytes hash(bcos::bytesConstRef _input) const override + { + SM3_CTX hashState; + auto ret = sm3_init(&hashState); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment( + "hash error for sm3_init failed, code: " + std::to_string(ret))); + } + ret = sm3_update(&hashState, (const void*)_input.data(), _input.size()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment( + "hash error for sm3_update failed, code: " + std::to_string(ret))); + } + bcos::bytes result(32); + ret = sm3_final(result.data(), &hashState); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment( + "hash error for sm3_final failed, code: " + std::to_string(ret))); + } + return result; + } + + // init the hash-state + HashState::Ptr init() const override + { + auto hashState = std::make_shared(); + auto ret = sm3_init((SM3_CTX*)hashState->state()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + HashException() << bcos::errinfo_comment("sm3_init error: " + std::to_string(ret))); + } + return hashState; + } + + // update new message into the given hash-state + void update(HashState::Ptr _state, bcos::bytesConstRef _data) const override + { + auto ret = sm3_update((SM3_CTX*)_state->state(), (const void*)_data.data(), _data.size()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "sm3_update error: " + std::to_string(ret))); + } + } + + // obtain the hash-result from the given hash-state + bcos::bytes final(HashState::Ptr _state) const override + { + bcos::bytes result(32); + auto ret = sm3_final(result.data(), (SM3_CTX*)_state->state()); + if (1 != ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "sm3_final error: " + std::to_string(ret))); + } + return result; + } +}; +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/hash/Sha256Hash.h b/cpp/ppc-crypto-core/src/hash/Sha256Hash.h new file mode 100644 index 00000000..ecbcb8cb --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/Sha256Hash.h @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Sha256Hash.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "../Common.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +namespace ppc::crypto +{ +class Sha256HashState : public HashState +{ +public: + using Ptr = std::shared_ptr; + Sha256HashState() : m_state(std::make_shared()) {} + ~Sha256HashState() override = default; + void* state() override { return (void*)m_state.get(); } + +private: + std::shared_ptr m_state; +}; + +class Sha256Hash : public Hash +{ +public: + using Ptr = std::shared_ptr; + Sha256Hash() = default; + ~Sha256Hash() override = default; + + // the hashBytes length of given hash-algorithm + size_t hashLen() const override { return crypto_hash_sha256_BYTES; } + // the implementation of the hash-algorithm + ppc::protocol::HashImplName type() const override + { + return ppc::protocol::HashImplName::SHA256; + } + + // calculate-hash + bcos::bytes hash(bcos::bytesConstRef _input) const override + { + bcos::bytes result(crypto_hash_sha256_BYTES); + auto ret = crypto_hash_sha256(result.data(), _input.data(), _input.size()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha256 error: " + std::to_string(ret))); + } + return result; + } + + // init the hash-state + HashState::Ptr init() const override + { + auto state = std::make_shared(); + auto ret = crypto_hash_sha256_init((crypto_hash_sha256_state*)state->state()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha256_init error: " + std::to_string(ret))); + } + return state; + } + + // update new message into the given hash-state + void update(HashState::Ptr _state, bcos::bytesConstRef _data) const override + { + auto ret = crypto_hash_sha256_update( + (crypto_hash_sha256_state*)_state->state(), _data.data(), _data.size()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha256_update error: " + std::to_string(ret))); + } + } + + // obtain the hash-result from the given hash-state + bcos::bytes final(HashState::Ptr _state) const override + { + bcos::bytes result(crypto_hash_sha256_BYTES); + auto ret = + crypto_hash_sha256_final((crypto_hash_sha256_state*)_state->state(), result.data()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha256_final error: " + std::to_string(ret))); + } + return result; + } +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/hash/Sha512Hash.h b/cpp/ppc-crypto-core/src/hash/Sha512Hash.h new file mode 100644 index 00000000..f8a207b2 --- /dev/null +++ b/cpp/ppc-crypto-core/src/hash/Sha512Hash.h @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Sha512Hash.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "../Common.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +namespace ppc::crypto +{ +class Sha512HashState : public HashState +{ +public: + using Ptr = std::shared_ptr; + Sha512HashState() : m_state(std::make_shared()) {} + ~Sha512HashState() override = default; + void* state() override { return (void*)m_state.get(); } + +private: + std::shared_ptr m_state; +}; + +class Sha512Hash : public Hash +{ +public: + using Ptr = std::shared_ptr; + Sha512Hash() = default; + virtual ~Sha512Hash() = default; + + // the hashBytes length of given hash-algorithm + size_t hashLen() const override { return crypto_hash_sha512_BYTES; } + // the implementation of the hash-algorithm + ppc::protocol::HashImplName type() const override + { + return ppc::protocol::HashImplName::SHA512; + } + + // calculate-hash + bcos::bytes hash(bcos::bytesConstRef _input) const override + { + bcos::bytes result(crypto_hash_sha512_BYTES); + auto ret = crypto_hash_sha512(result.data(), _input.data(), _input.size()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha512 error: " + std::to_string(ret))); + } + return result; + } + + // init the hash-state + HashState::Ptr init() const override + { + auto hashState = std::make_shared(); + auto ret = crypto_hash_sha512_init((crypto_hash_sha512_state*)hashState->state()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha512_init error: " + std::to_string(ret))); + } + return hashState; + } + + // update new message into the given hash-state + void update(HashState::Ptr _state, bcos::bytesConstRef _data) const override + { + auto ret = crypto_hash_sha512_update( + (crypto_hash_sha512_state*)_state->state(), _data.data(), _data.size()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha512_state error: " + std::to_string(ret))); + } + } + + // obtain the hash-result from the given hash-state + bcos::bytes final(HashState::Ptr _state) const override + { + bcos::bytes result(crypto_hash_sha512_BYTES); + auto ret = + crypto_hash_sha512_final((crypto_hash_sha512_state*)_state->state(), result.data()); + if (ret) + { + BOOST_THROW_EXCEPTION(HashException() << bcos::errinfo_comment( + "crypto_hash_sha512_final error: " + std::to_string(ret))); + } + return result; + } +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/ore/FastOre.cpp b/cpp/ppc-crypto-core/src/ore/FastOre.cpp new file mode 100644 index 00000000..c62d63f2 --- /dev/null +++ b/cpp/ppc-crypto-core/src/ore/FastOre.cpp @@ -0,0 +1,230 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FastOre.cpp + * @author: shawnhe + * @date 2023-08-18 + */ +#include "FastOre.h" +#include "../Common.h" + +using namespace ppc; +using namespace ppc::crypto; + +// cipher encoded with hex +std::string FastOre::encrypt4String( + bcos::bytesConstRef const& _sk, const std::string& _plaintext) const +{ + bcos::bytes cipher(estimatedCipherSize(_plaintext.size(), false)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + encrypt4String( + &cipherBuffer, _sk, bcos::bytesConstRef((bcos::byte*)_plaintext.data(), _plaintext.size())); + return bcos::toHex(bcos::bytes{cipher.begin(), cipher.end()}); +} + +void FastOre::encrypt4String(OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _plaintext) const +{ + if (_plaintext.size() == 0) + { + return; + } + auto estimatedSize = estimatedCipherSize(_plaintext.size(), false); + if (_cipher->len < estimatedSize) + { + BOOST_THROW_EXCEPTION( + FastOreException() << bcos::errinfo_comment( + "FastOre4String encrypt failed for unenough cipher buffer, expect at least " + + std::to_string(estimatedSize))); + } + + // enc the first byte + bcos::bytes cipher0(CIPHER_BLOCK_SIZE); + auto tempCipher0 = encWithTruncation( + _sk, bcos::bytesConstRef((const unsigned char*)cipher0.data(), CIPHER_BLOCK_SIZE)); + uint16_t sum0 = (uint8_t)tempCipher0[1] + (uint8_t)_plaintext[0]; + _cipher->data[0] = (uint8_t)tempCipher0[0] + sum0 / 256; + _cipher->data[1] = sum0 % 256; + + // enc the other bytes + for (uint64_t i = 1; i < _plaintext.size(); i++) + { + auto tempCipher = encWithTruncation( + _sk, bcos::bytesConstRef((const unsigned char*)_cipher->data, i * CIPHER_BLOCK_SIZE)); + uint16_t sum = (uint8_t)tempCipher[1] + (uint8_t)_plaintext[i]; + _cipher->data[i * CIPHER_BLOCK_SIZE] = (uint8_t)tempCipher[0] + sum / 256; + _cipher->data[i * CIPHER_BLOCK_SIZE + 1] = sum % 256; + } +} + +// cipher encoded with hex +std::string FastOre::decrypt4String( + bcos::bytesConstRef const& _key, const std::string& _ciphertext) const +{ + bcos::bytes cipher = bcos::fromHex(_ciphertext); + std::string plain; + plain.resize(estimatedPlainSize(_ciphertext.size(), true)); + OutputBuffer resultBuffer{(bcos::byte*)plain.data(), plain.size()}; + decrypt4String(&resultBuffer, _key, bcos::ref(cipher)); + + return plain; +} + +void FastOre::decrypt4String( + OutputBuffer* _plain, bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const +{ + if (_cipher.size() < CIPHER_BLOCK_SIZE) + { + BOOST_THROW_EXCEPTION( + FastOreException() << bcos::errinfo_comment( + "FastOre4String decrypt failed for invalid cipher, min cipher size is: " + + std::to_string(CIPHER_BLOCK_SIZE))); + } + auto blockSize = estimatedPlainSize(_cipher.size(), false); + if (_plain->len < blockSize) + { + BOOST_THROW_EXCEPTION( + FastOreException() << bcos::errinfo_comment( + "FastOre4String decrypt failed for unenough plain buffer, expect at least: " + + std::to_string(blockSize))); + } + + // dec the first byte + bcos::bytes cipher0(CIPHER_BLOCK_SIZE); + auto tempCipher0 = encWithTruncation( + _sk, bcos::bytesConstRef((const unsigned char*)cipher0.data(), CIPHER_BLOCK_SIZE)); + int diff0 = (uint8_t)_cipher[1] - (uint8_t)tempCipher0[1]; + if (diff0 < 0) + { + diff0 += 256; + } + _plain->data[0] = diff0; + + // dec the other bytes + for (uint64_t i = 1; i < blockSize; i++) + { + auto tempCipher = encWithTruncation( + _sk, bcos::bytesConstRef((const unsigned char*)_cipher.data(), i * CIPHER_BLOCK_SIZE)); + int diff = (uint8_t)_cipher[i * CIPHER_BLOCK_SIZE + 1] - (uint8_t)tempCipher[1]; + if (diff < 0) + { + diff += 256; + } + _plain->data[i] = diff; + } +} + +// cipher encoded with base64 +std::string FastOre::encrypt4Integer(bcos::bytesConstRef const& _sk, const int64_t& _plain) const +{ + bcos::bytes cipher(estimatedCipherSize(sizeof(_plain), false)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + encrypt4Integer(&cipherBuffer, _sk, _plain); + return bcos::toHex(bcos::bytes{cipher.begin(), cipher.end()}); +} + +void FastOre::encrypt4Integer( + OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, const int64_t& _plain) const +{ + int64_t plain = 0; + OutputBuffer plainB{(bcos::byte*)&plain, sizeof(plain)}; + formatNumberPlain(&plainB, _plain); + encrypt4String(_cipher, _sk, bcos::bytesConstRef((bcos::byte*)plainB.data, plainB.len)); +} + +// cipher encoded with base64 +int64_t FastOre::decrypt4Integer(bcos::bytesConstRef const& _key, const std::string& _cipher) const +{ + bcos::bytes cipher = bcos::fromHex(_cipher); + int64_t plain = 0; + decrypt4Integer(&plain, _key, bcos::ref(cipher)); + return plain; +} + +void FastOre::decrypt4Integer( + int64_t* _plain, bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& cipher) const +{ + int64_t plain = 0; + OutputBuffer resultBuffer{(bcos::byte*)&plain, sizeof(plain)}; + decrypt4String(&resultBuffer, _sk, cipher); + *_plain = recoverNumberPlain(resultBuffer); +} + +// cipher encoded with hex +std::string FastOre::encrypt4Float(bcos::bytesConstRef const& _sk, const float50& _plain) const +{ + bcos::bytes cipher(estimatedFloatCipherSize(_plain.str().size(), false)); + OutputBuffer cipherBuffer{cipher.data(), cipher.size()}; + encrypt4Float(&cipherBuffer, _sk, _plain); + return bcos::toHex(bcos::bytes{cipher.begin(), cipher.begin() + cipherBuffer.len}); +} + +void FastOre::encrypt4Float( + OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, const float50& _plain) const +{ + OreFloatingNumber ofn(_plain); + encrypt4String( + _cipher, _sk, bcos::bytesConstRef((bcos::byte*)&ofn.integerPart, sizeof(ofn.integerPart))); + + if (!ofn.decimalPart.empty()) + { + OutputBuffer decimalCipherBuffer{ + _cipher->data + sizeof(ofn.integerPart) * CIPHER_BLOCK_SIZE, + ofn.decimalPart.size() * CIPHER_BLOCK_SIZE}; + encrypt4String(&decimalCipherBuffer, _sk, + bcos::bytesConstRef((bcos::byte*)ofn.decimalPart.data(), ofn.decimalPart.size())); + } + _cipher->len = (sizeof(ofn.integerPart) + ofn.decimalPart.size()) * CIPHER_BLOCK_SIZE; +} + +// cipher encoded with base64 +float50 FastOre::decrypt4Float(bcos::bytesConstRef const& _sk, const std::string& _cipher) const +{ + bcos::bytes cipher = bcos::fromHex(_cipher); + return decrypt4Float(_sk, bcos::ref(cipher)); +} + +float50 FastOre::decrypt4Float( + bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const +{ + OreFloatingNumber ofn; + OutputBuffer integerOut{(bcos::byte*)&ofn.integerPart, sizeof(ofn.integerPart)}; + auto integerCipherSize = sizeof(ofn.integerPart) * CIPHER_BLOCK_SIZE; + decrypt4String(&integerOut, _sk, bcos::bytesConstRef(_cipher.data(), integerCipherSize)); + + if (_cipher.size() > integerCipherSize) + { + auto decimalSize = (_cipher.size() - integerCipherSize) / CIPHER_BLOCK_SIZE; + ofn.decimalPart.resize(decimalSize); + OutputBuffer decimalOut{(bcos::byte*)ofn.decimalPart.data(), decimalSize}; + decrypt4String(&decimalOut, _sk, + bcos::bytesConstRef( + _cipher.data() + integerCipherSize, decimalSize * CIPHER_BLOCK_SIZE)); + } + return ofn.value(); +} + +int FastOre::compare(const std::string& _ciphertext0, const std::string& _ciphertext1) const +{ + return _ciphertext0.compare(_ciphertext1); +} + +int FastOre::compare(InputBuffer const* c1, InputBuffer const* c2) const +{ + std::string str1{c1->data, c1->data + c1->len}; + std::string str2{c2->data, c2->data + c2->len}; + + return str1.compare(str2); +} diff --git a/cpp/ppc-crypto-core/src/ore/FastOre.h b/cpp/ppc-crypto-core/src/ore/FastOre.h new file mode 100644 index 00000000..5ef10a0f --- /dev/null +++ b/cpp/ppc-crypto-core/src/ore/FastOre.h @@ -0,0 +1,177 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FastOre.h + * @author: shawnhe + * @date 2023-08-18 + */ + +#pragma once + +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-framework/crypto/Ore.h" +#include "ppc-framework/libwrapper/OreFloatingNumber.h" + +#define CIPHER_BLOCK_SIZE 2 + +namespace ppc::crypto +{ +class FastOre : public Ore +{ +public: + using Ptr = std::shared_ptr; + FastOre() { m_symCrypto = std::make_shared(OpenSSLAES::AESType::AES128); } + ~FastOre() override = default; + + // cipher encoded with base64 + void encrypt4String(OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _plaintext) const override; + std::string encrypt4String( + bcos::bytesConstRef const& _key, const std::string& _plaintext) const override; + + // cipher encoded with base64 + std::string decrypt4String( + bcos::bytesConstRef const& _key, const std::string& _ciphertext) const override; + void decrypt4String(OutputBuffer* plain, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& cipher) const override; + + // cipher encoded with base64 + std::string encrypt4Integer( + bcos::bytesConstRef const& _sk, const int64_t& _plain) const override; + void encrypt4Integer(OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, + const int64_t& _plain) const override; + + // cipher encoded with base64 + int64_t decrypt4Integer( + bcos::bytesConstRef const& _sk, const std::string& _cipher) const override; + void decrypt4Integer(int64_t* _plain, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _cipher) const override; + + // cipher encoded with base64 + std::string encrypt4Float(bcos::bytesConstRef const& _sk, const float50& _plain) const override; + void encrypt4Float(OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, + const float50& _plain) const override; + + // cipher encoded with base64 + float50 decrypt4Float( + bcos::bytesConstRef const& _sk, const std::string& _cipher) const override; + float50 decrypt4Float( + bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const override; + + int compare(const std::string& _ciphertext0, const std::string& _ciphertext1) const override; + int compare(InputBuffer const* c1, InputBuffer const* c2) const override; + + int keyBytes() const override { return m_symCrypto->keyBytes(SymCrypto::OperationMode::CBC); } + + bcos::bytes generateKey() const override + { + return m_symCrypto->generateKey(SymCrypto::OperationMode::CBC); + } + + void generateKey(OutputBuffer* sk) const override + { + return m_symCrypto->generateKey(sk, SymCrypto::OperationMode::CBC); + } + + + uint64_t estimatedCipherSize(uint64_t _plainSize, bool hex) const + { + if (!hex) + { + return _plainSize * CIPHER_BLOCK_SIZE; + } + return _plainSize * CIPHER_BLOCK_SIZE * 2; + } + + uint64_t estimatedFloatCipherSize(uint64_t _plainSize, bool hex) const + { + if (!hex) + { + return (sizeof(int64_t) + _plainSize) * CIPHER_BLOCK_SIZE; + } + return (sizeof(int64_t) + _plainSize) * CIPHER_BLOCK_SIZE * 2; + } + + uint64_t estimatedPlainSize(uint64_t cipherSize, bool hex) const + { + uint64_t plainSize = 0; + if (!hex) + { + plainSize = cipherSize / CIPHER_BLOCK_SIZE; + } + else + { + plainSize = cipherSize / CIPHER_BLOCK_SIZE / 2; + } + return plainSize; + } + + static void formatNumberPlain(OutputBuffer* _buffer, const int64_t& _plain) + { + // Note: Add brackets here to adapt to windows compilation + if (_plain > (std::numeric_limits::max)() / 2 || + _plain <= (std::numeric_limits::min)() / 2) + { + BOOST_THROW_EXCEPTION( + FastOreException() << bcos::errinfo_comment( + "plain is too large or too small, must be in range (-2^62, 2^62)")); + } + + int64_t plain = _plain + (std::numeric_limits::max)() / 2; + + for (int i = sizeof(plain) - 1; i >= 0; --i) + { + _buffer->data[i] = static_cast((plain >> (8 * (7 - i))) & 0xFF); + } + _buffer->len = sizeof(plain); + } + + static int64_t recoverNumberPlain(const OutputBuffer& outputBuffer) + { + int64_t result = 0; + + for (uint64_t i = 0; i < outputBuffer.len; ++i) + { + result = (result << 8) | outputBuffer.data[i]; + } + + return result - (std::numeric_limits::max)() / 2; + } + +private: + // keep only the first two bytes + bcos::bytes encWithTruncation( + bcos::bytesConstRef const& _key, bcos::bytesConstRef const& _plaintext) const + { + bcos::bytes iv(0); + bcos::bytes cipher = + m_symCrypto->encrypt(SymCrypto::OperationMode::CBC, _key, bcos::ref(iv), _plaintext); + uint64_t size = cipher.size(); + if (cipher[size - 2] == 0xFF) + { + cipher[size - 2] = 0xFE; + } + + bcos::bytes result(2); + result[0] = cipher[size - 2]; + result[1] = cipher[size - 1]; + return result; + } + +private: + SymCrypto::Ptr m_symCrypto; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp new file mode 100644 index 00000000..ea73cf47 --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSL3DES.cpp + * @author: shawnhe + * @date 2023-08-16 + */ + +#include "OpenSSL3DES.h" + +using namespace ppc::crypto; +using namespace ppc::protocol; + +EvpCipherPtr OpenSSL3DES::createCipherMeth(OperationMode _mode) const +{ + switch (_mode) + { + case SymCrypto::OperationMode::ECB: + return EvpCipherPtr(EVP_des_ede3_ecb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CBC: + return EvpCipherPtr(EVP_des_ede3_cbc(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CFB: + return EvpCipherPtr(EVP_des_ede3_cfb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::OFB: + return EvpCipherPtr(EVP_des_ede3_ofb(), EvpCipherDeleter()); + default: + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "unsupported crypto mode: " + std::to_string(int(_mode)))); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h new file mode 100644 index 00000000..7c0d355f --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSL3DES.h + * @author: shawnhe + * @date 2023-08-16 + */ + +#pragma once + +#include "../Common.h" +#include "OpenSSLCipher.h" + +namespace ppc::crypto +{ +class OpenSSL3DES : public OpenSSLCipher +{ +public: + using Ptr = std::shared_ptr; + + OpenSSL3DES(protocol::DataPaddingType _padding = protocol::DataPaddingType::PKCS7) + : OpenSSLCipher(_padding) + { + // the key size: 168 bits; the block size: 64 bits + m_blockSize = 8; + } + ~OpenSSL3DES() override = default; + + protocol::SymCryptoImplName type() const override + { + return protocol::SymCryptoImplName::DES_EDE3; + } + +public: + EvpCipherPtr createCipherMeth(OperationMode _mode) const override; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp new file mode 100644 index 00000000..71d63f69 --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLAES.cpp + * @author: shawnhe + * @date 2022-11-29 + */ + +#include "OpenSSLAES.h" + +using namespace ppc::crypto; +using namespace ppc::protocol; +EvpCipherPtr OpenSSLAES::createCipherMeth(OperationMode _mode) const +{ + if (m_aesType == AESType::AES128) + { + switch (_mode) + { + case SymCrypto::OperationMode::ECB: + return EvpCipherPtr(EVP_aes_128_ecb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CBC: + return EvpCipherPtr(EVP_aes_128_cbc(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CFB: + return EvpCipherPtr(EVP_aes_128_cfb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::OFB: + return EvpCipherPtr(EVP_aes_128_ofb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CTR: + return EvpCipherPtr(EVP_aes_128_ctr(), EvpCipherDeleter()); + default: + BOOST_THROW_EXCEPTION( + SymCryptoException() << bcos::errinfo_comment( + "unsupported mode for AES128: " + std::to_string(int(_mode)))); + } + } + else if (m_aesType == AESType::AES192) + { + switch (_mode) + { + case SymCrypto::OperationMode::ECB: + return EvpCipherPtr(EVP_aes_192_ecb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CBC: + return EvpCipherPtr(EVP_aes_192_cbc(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CFB: + return EvpCipherPtr(EVP_aes_192_cfb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::OFB: + return EvpCipherPtr(EVP_aes_192_ofb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CTR: + return EvpCipherPtr(EVP_aes_192_ctr(), EvpCipherDeleter()); + default: + BOOST_THROW_EXCEPTION( + SymCryptoException() << bcos::errinfo_comment( + "unsupported mode for AES192: " + std::to_string(int(_mode)))); + } + } + else if (m_aesType == AESType::AES256) + { + switch (_mode) + { + case SymCrypto::OperationMode::ECB: + return EvpCipherPtr(EVP_aes_256_ecb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CBC: + return EvpCipherPtr(EVP_aes_256_cbc(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CFB: + return EvpCipherPtr(EVP_aes_256_cfb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::OFB: + return EvpCipherPtr(EVP_aes_256_ofb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CTR: + return EvpCipherPtr(EVP_aes_256_ctr(), EvpCipherDeleter()); + default: + BOOST_THROW_EXCEPTION( + SymCryptoException() << bcos::errinfo_comment( + "unsupported mode for AES256: " + std::to_string(int(_mode)))); + } + } + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "Unsupported aes-algorithm: " + std::to_string((int)m_aesType))); +} diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h new file mode 100644 index 00000000..17b54bd2 --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLAES.h + * @author: shawnhe + * @date 2022-11-29 + */ + +#pragma once + +#include "../Common.h" +#include "OpenSSLCipher.h" + +namespace ppc::crypto +{ +class OpenSSLAES : public OpenSSLCipher +{ +public: + using Ptr = std::shared_ptr; + + enum class AESType : int + { + AES128 = 16, + AES192 = 24, + AES256 = 32, + }; + + OpenSSLAES() = delete; + + ~OpenSSLAES() override = default; + + OpenSSLAES( + AESType _aesType, protocol::DataPaddingType _padding = protocol::DataPaddingType::PKCS7) + : OpenSSLCipher(_padding), m_aesType(_aesType) + {} + + protocol::SymCryptoImplName type() const override { return protocol::SymCryptoImplName::AES; } + + AESType aesType() { return m_aesType; } + + EvpCipherPtr createCipherMeth(OperationMode _mode) const override; + +private: + AESType m_aesType; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h new file mode 100644 index 00000000..84e68590 --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h @@ -0,0 +1,218 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLCipher.h + * @author: shawnhe + * @date 2023-08-17 + */ + +#pragma once + +#include "../Common.h" +#include "ppc-framework/crypto/SymCrypto.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include + +namespace ppc::crypto +{ +struct EvpCipherDeleter +{ +public: + void operator()(EVP_CIPHER const* _cipher) { (void)_cipher; } +}; +using EvpCipherPtr = std::shared_ptr; + +struct EvpCipherCtxDeleter +{ +public: + void operator()(EVP_CIPHER_CTX* _ctx) { EVP_CIPHER_CTX_free(_ctx); } +}; +using EvpCipherCtxPtr = std::shared_ptr; + +class OpenSSLCipher : public SymCrypto +{ +public: + OpenSSLCipher() = delete; + + ~OpenSSLCipher() override = default; + + OpenSSLCipher(protocol::DataPaddingType _padding = protocol::DataPaddingType::PKCS7) + : SymCrypto() + { + m_padding = _padding; + if (int(m_padding) < 0 || int(m_padding) > EVP_PADDING_ZERO) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "undefined padding: " + std::to_string(int(_padding)))); + } + } + +private: + virtual EvpCipherPtr createCipherMeth(OperationMode _mode) const = 0; + + virtual void initCipherCtx(OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, EVP_CIPHER_CTX* _ctx, int _enc) const + { + auto cipher = createCipherMeth(_mode); + auto keyLength = EVP_CIPHER_key_length(cipher.get()); + auto key = _sk.toBytes(); + key.resize(keyLength); + auto iv = _iv.toBytes(); + iv.resize(m_blockSize); + if (_mode == SymCrypto::OperationMode::ECB) + { + if (1 != EVP_CipherInit_ex(_ctx, cipher.get(), nullptr, key.data(), nullptr, _enc)) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "initCipherCtx: EVP_CipherInit_ex failed: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return; + } + + if (1 != EVP_CipherInit_ex(_ctx, cipher.get(), nullptr, key.data(), iv.data(), _enc)) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "initCipherCtx: EVP_CipherInit_ex failed: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + virtual EvpCipherCtxPtr createCipherCtx() const + { + return EvpCipherCtxPtr(EVP_CIPHER_CTX_new(), EvpCipherCtxDeleter()); + } + +public: + protocol::DataPaddingType padding() { return m_padding; } + + void encrypt(OutputBuffer* _cipher, OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _plaintext) const override + { + auto minCipherSize = _plaintext.size() + m_blockSize; + if (_cipher->len < minCipherSize) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "The reserved cipher buffer is not enough, at least: " + + std::to_string(minCipherSize))); + } + auto ctx = createCipherCtx(); + initCipherCtx(_mode, _sk, _iv, ctx.get(), 1); + + if (_mode == SymCrypto::OperationMode::ECB || _mode == SymCrypto::OperationMode::CBC) + { + EVP_CIPHER_CTX_set_padding(ctx.get(), int(m_padding)); + } + EVP_CIPHER_CTX_set_padding(ctx.get(), int(m_padding)); + int outLen; + if (1 != EVP_CipherUpdate( + ctx.get(), _cipher->data, &outLen, _plaintext.data(), _plaintext.size())) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "encrypt failed for EVP_CipherUpdate exception: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + + int outLenFinal = 0; + if (1 != EVP_CipherFinal_ex(ctx.get(), _cipher->data + outLen, &outLenFinal)) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "encrypt failed for EVP_CipherFinal exception: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + _cipher->len = outLen + outLenFinal; + } + + bcos::bytes encrypt(OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _plaintext) const override + { + bcos::bytes cipherResult(_plaintext.size() + blockSize()); + OutputBuffer cipherBuffer{cipherResult.data(), cipherResult.size()}; + encrypt(&cipherBuffer, _mode, _sk, _iv, _plaintext); + cipherResult.resize(cipherBuffer.len); + return cipherResult; + } + + void decrypt(OutputBuffer* plain, OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _ciphertext) const override + { + auto minPlainDataSize = _ciphertext.size() + m_blockSize; + if (plain->len < minPlainDataSize) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "The reserved plainData buffer is not enough, at least: " + + std::to_string(minPlainDataSize))); + } + auto ctx = createCipherCtx(); + initCipherCtx(_mode, _sk, _iv, ctx.get(), 0); + if (_mode == SymCrypto::OperationMode::ECB || _mode == SymCrypto::OperationMode::CBC) + { + EVP_CIPHER_CTX_set_padding(ctx.get(), int(m_padding)); + } + EVP_CIPHER_CTX_set_padding(ctx.get(), int(m_padding)); + + int outLen; + if (1 != EVP_CipherUpdate( + ctx.get(), plain->data, &outLen, _ciphertext.data(), _ciphertext.size())) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "decrypt failed for EVP_CipherUpdate exception: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + + int outLenFinal = 0; + if (1 != EVP_CipherFinal_ex(ctx.get(), plain->data + outLen, &outLenFinal)) + { + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "decrypt failed for EVP_CipherFinal exception: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + plain->len = outLen + outLenFinal; + } + + bcos::bytes decrypt(OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _ciphertext) const override + { + bcos::bytes result(_ciphertext.size() + m_blockSize, 0); + OutputBuffer resultBuffer{result.data(), result.size()}; + decrypt(&resultBuffer, _mode, _sk, _iv, _ciphertext); + result.resize(resultBuffer.len); + return result; + } + + bcos::bytes generateKey(OperationMode _mode) const override + { + auto key = generateRand(keyBytes(_mode) * 8); + bcos::bytes keyBytes; + key.toBytes(keyBytes, false); + return keyBytes; + } + + void generateKey(OutputBuffer* _key, OperationMode _mode) const override + { + auto key = generateRand(keyBytes(_mode) * 8); + key.toOutputBuffer(_key, false); + } + + virtual unsigned int keyBytes(OperationMode _mode) const override + { + auto cipher = createCipherMeth(_mode); + return EVP_CIPHER_key_length(cipher.get()); + } + +protected: + protocol::DataPaddingType m_padding{protocol::DataPaddingType::PKCS7}; +}; +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp new file mode 100644 index 00000000..aa9452d7 --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLSM4.cpp + * @author: shawnhe + * @date 2022-11-29 + */ + +#include "OpenSSLSM4.h" + +using namespace ppc::crypto; +using namespace ppc::protocol; + +EvpCipherPtr OpenSSLSM4::createCipherMeth(OperationMode _mode) const +{ + switch (_mode) + { + case SymCrypto::OperationMode::ECB: + return EvpCipherPtr(EVP_sm4_ecb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CBC: + return EvpCipherPtr(EVP_sm4_cbc(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CFB: + return EvpCipherPtr(EVP_sm4_cfb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::OFB: + return EvpCipherPtr(EVP_sm4_ofb(), EvpCipherDeleter()); + case SymCrypto::OperationMode::CTR: + return EvpCipherPtr(EVP_sm4_ctr(), EvpCipherDeleter()); + default: + BOOST_THROW_EXCEPTION(SymCryptoException() << bcos::errinfo_comment( + "unsupported crypto type: " + std::to_string(int(_mode)))); + } +} diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h new file mode 100644 index 00000000..e04836bb --- /dev/null +++ b/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLSM4.h + * @author: shawnhe + * @date 2022-11-29 + */ + + +#pragma once + +#include "../Common.h" +#include "OpenSSLCipher.h" + +namespace ppc::crypto +{ +class OpenSSLSM4 : public OpenSSLCipher +{ +public: + using Ptr = std::shared_ptr; + OpenSSLSM4(protocol::DataPaddingType _padding = protocol::DataPaddingType::PKCS7) + : OpenSSLCipher(_padding) + {} + + ~OpenSSLSM4() override = default; + protocol::SymCryptoImplName type() const override { return protocol::SymCryptoImplName::SM4; } + +private: + EvpCipherPtr createCipherMeth(OperationMode _mode) const override; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/src/tools/BitVector.cpp b/cpp/ppc-crypto-core/src/tools/BitVector.cpp new file mode 100644 index 00000000..6f21fcf2 --- /dev/null +++ b/cpp/ppc-crypto-core/src/tools/BitVector.cpp @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BitVector.cpp + * @author: shawnhe + * @date 2022-12-5 + */ + +#include "BitVector.h" +#include + +using namespace ppc::crypto; + +void BitVector::randomize(PRNG::Ptr _prng, uint32_t _bytesLen) +{ + m_data.clear(); + auto randomBytes = _prng->generate(_bytesLen); + append(randomBytes); +} + +void BitVector::append(const bcos::bytes& _data) +{ + int start = _data.size() - 1; + for (int i = start; i >= 0; --i) + { + auto bits = std::bitset<8>(_data[i]); + for (int j = 0; j < 8; ++j) + { + m_data.push_back(bits[j]); + } + } +} + +void BitVector::append(const std::string& _binary) +{ + auto newBits = boost::dynamic_bitset<>(_binary); + for (uint32_t i = 0; i < newBits.size(); ++i) + { + m_data.push_back(newBits[i]); + } +} + +std::string BitVector::toString() +{ + std::string result; + boost::to_string(m_data, result); + return result; +} + +bcos::bytes BitVector::toBytes() +{ + if (m_data.size() % 8) + { + BOOST_THROW_EXCEPTION( + BitVectorException() << bcos::errinfo_comment("the size must be divisible by 8")); + } + + uint32_t len = m_data.size() / 8; + bcos::bytes result(len); + + for (uint32_t i = 0; i < len; ++i) + { + bcos::byte b = 0; + b |= m_data[i * 8 + 0] << 0; + b |= m_data[i * 8 + 1] << 1; + b |= m_data[i * 8 + 2] << 2; + b |= m_data[i * 8 + 3] << 3; + b |= m_data[i * 8 + 4] << 4; + b |= m_data[i * 8 + 5] << 5; + b |= m_data[i * 8 + 6] << 6; + b |= m_data[i * 8 + 7] << 7; + result[len - 1 - i] = b; + } + + return result; +} \ No newline at end of file diff --git a/cpp/ppc-crypto-core/src/tools/BitVector.h b/cpp/ppc-crypto-core/src/tools/BitVector.h new file mode 100644 index 00000000..7cbcc93c --- /dev/null +++ b/cpp/ppc-crypto-core/src/tools/BitVector.h @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BitVector.h + * @author: shawnhe + * @date 2022-12-5 + */ + + +#include "../Common.h" +#include "ppc-framework/crypto/PRNG.h" +#include +#include + +#pragma once + +namespace ppc::crypto +{ +class BitVector +{ +public: + using Ptr = std::shared_ptr; + + BitVector() = default; + ~BitVector() = default; + + // init '_len' bits by '_number' + BitVector(uint32_t _len, uint32_t _number) { m_data = boost::dynamic_bitset<>(_len, _number); } + + // init by binary string + BitVector(const std::string& _binary) { m_data = boost::dynamic_bitset<>(_binary); } + + // init by string + BitVector(const bcos::bytes& _data) { append(_data); } + + // get a bit by location + bool get(uint32_t _location) { return m_data[_location]; } + + // set a bit by location + void set(uint32_t _location, bool _bit) { m_data[_location] = _bit; } + + // generate random '_bytesLen * 8' bits + void randomize(PRNG::Ptr _prng, uint32_t _bytesLen); + + // append from the high position + void append(const std::string& _binary); + + // append from the high position + void append(const bcos::bytes& _data); + + bool equals(const BitVector& _bitVector) { return m_data == _bitVector.data(); } + + boost::dynamic_bitset<>& data() { return m_data; } + + [[nodiscard]] const boost::dynamic_bitset<>& data() const { return m_data; } + + uint32_t size() { return m_data.size(); } + + void resize(uint32_t _size) { m_data.resize(_size); } + + void clear() { m_data.clear(); } + + std::string toString(); + + bcos::bytes toBytes(); + +private: + boost::dynamic_bitset<> m_data; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto-core/tests/CMakeLists.txt b/cpp/ppc-crypto-core/tests/CMakeLists.txt new file mode 100644 index 00000000..f38a2d05 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-crypto-core) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${CRYPTO_CORE_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-crypto-core WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) diff --git a/cpp/ppc-crypto-core/tests/TestBitVector.cpp b/cpp/ppc-crypto-core/tests/TestBitVector.cpp new file mode 100644 index 00000000..7441f437 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestBitVector.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestBitvector.cpp + * @author: shawnhe + * @date 2022-12-6 + */ + +#include "ppc-crypto-core/src/tools/BitVector.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(BitvectorTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(testBitvector) +{ + auto str = "10011011101"; + BitVector bv(str); + BOOST_CHECK(str == bv.toString()); + + BitVector bv0("10011011101"); + BOOST_CHECK(bv.equals(bv0)); + + BitVector bv1(10, 12345); + BOOST_CHECK("0000111001" == bv1.toString()); + BOOST_CHECK(bv1.get(0) && !bv1.get(1) && bv1.get(3)); + bv1.set(0, 0), bv1.set(1, 1), bv1.set(2, 1); + BOOST_CHECK("0000111110" == bv1.toString()); + + BitVector bv2(bcos::bytes{'a'}); + bv2.append(bcos::bytes{'b'}); + bv2.append(bcos::bytes{'c'}); + BOOST_CHECK("011000110110001001100001" == bv2.toString()); + bcos::bytes bytes1{'c', 'b', 'a'}; + BOOST_CHECK(bytes1 == bv2.toBytes()); + bv2.append("111000111"); + BOOST_CHECK("111000111011000110110001001100001" == bv2.toString()); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-core/tests/TestHash.cpp b/cpp/ppc-crypto-core/tests/TestHash.cpp new file mode 100644 index 00000000..f520c444 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestHash.cpp @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestHash.cpp + * @author: yujiechen + * @date 2022-11-2 + */ +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-crypto-core/src/hash/SM3Hash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto-core/src/hash/Sha512Hash.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(hashTest, TestPromptFixture) + +void testHash(Hash::Ptr _hashImpl, uint64_t expectedHashLen) +{ + std::string input = "abcdwerwlerkwel" + std::to_string(utcSteadyTime()); + bcos::bytes inputData(input.begin(), input.end()); + // check hash + auto hashResult = _hashImpl->hash(ref(inputData)); + for (int i = 0; i < 1000; i++) + { + BOOST_CHECK(_hashImpl->hash(ref(inputData)) == hashResult); + BOOST_CHECK(hashResult.size() == expectedHashLen); + } + + // check update + auto state = _hashImpl->init(); + _hashImpl->update(state, ref(inputData)); + auto finalHashResult = _hashImpl->final(state); + BOOST_CHECK(finalHashResult == hashResult); + + // check update multiple input + std::string input2 = "werwelr234weweskdsj2342"; + bcos::bytes inputData2(input2.begin(), input2.end()); + state = _hashImpl->init(); + _hashImpl->update(state, ref(inputData)); + _hashImpl->update(state, ref(inputData2)); + finalHashResult = _hashImpl->final(state); + for (int i = 0; i < 1000; i++) + { + state = _hashImpl->init(); + _hashImpl->update(state, ref(inputData)); + _hashImpl->update(state, ref(inputData2)); + BOOST_CHECK(_hashImpl->final(state) == finalHashResult); + } +} + +void checkFixedInput( + Hash::Ptr _hashImpl, std::string const& _input, std::string const& _expectedHash) +{ + bytes inputData(_input.begin(), _input.end()); + auto result = _hashImpl->hash(ref(inputData)); + std::cout << *toHexString(result) << std::endl; + BOOST_CHECK(*toHexString(result) == _expectedHash); +} +BOOST_AUTO_TEST_CASE(testSha256Hash) +{ + testHash(std::make_shared(), 32); + std::string input = "abcdwerwlerkwel"; + std::string expectedSha256 = "d7a5cc8e0f256e5c3f7f1c551d9206a22983da73969eb54b40b111c378ea4b70"; + checkFixedInput(std::make_shared(), input, expectedSha256); +} + +BOOST_AUTO_TEST_CASE(testSha512Hash) +{ + testHash(std::make_shared(), 64); + std::string input = "abcdwerwlerkwel"; + std::string expectedHash = + "f8e36730b763de5699d111fb65aae0e73afaedb914f634ad8210891f4b5b88f8ffc3fcc557c4273f61cc5177a4" + "67481a8d7b91a86a4c2de9ce342e91e7c3144d"; + checkFixedInput(std::make_shared(), input, expectedHash); +} + +BOOST_AUTO_TEST_CASE(testSM3Hash) +{ + testHash(std::make_shared(), 32); + std::string input = "abcdwerwlerkwel"; + std::string expectedHash = "9e057883cc739dc1ee95ae65a0e86b946330c77e48123e9cea2b31a43cb69db1"; + checkFixedInput(std::make_shared(), input, expectedHash); +} + +BOOST_AUTO_TEST_CASE(testMD5Hash) +{ + testHash(std::make_shared(), 16); + std::string input = "abcdwerwlerkwel"; + std::string expectedHash = "a3eeda2c6b29a2518d0945baf3cfd220"; + checkFixedInput(std::make_shared(), input, expectedHash); +} + +BOOST_AUTO_TEST_CASE(testBLAKE2bHash) +{ + testHash(std::make_shared(), 64); + + std::string input = "abcdwerwlerkwel"; + std::string expected512Hash = + "fa693cb55a305997ed33839bf540d88fdcd7f343e6ff4aa34d31fbdfaf6567569e8dd46a535e7d44b77de5e784" + "d8ac2e1a6413b473be3790c9f407594f1ead06"; + checkFixedInput(std::make_shared(), input, expected512Hash); + + + bytes inputData(input.begin(), input.end()); + auto hashImpl = std::make_shared(); + auto result = hashImpl->hash(ref(inputData), 32); + std::string expected256Hash = + "869cf8d5848f78a3a9c076e5d36a0a05ef0f37c2837a952b6590981762b3c9b3"; + BOOST_CHECK(*toHexString(result) == expected256Hash); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-crypto-core/tests/TestOre4Float.cpp b/cpp/ppc-crypto-core/tests/TestOre4Float.cpp new file mode 100644 index 00000000..0469b8a2 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestOre4Float.cpp @@ -0,0 +1,214 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestOre4Float.cpp + * @author: shawnhe + * @date 2023-12-07 + */ +#pragma execution_character_set("utf-8") +#include "ppc-crypto-core/src/ore/FastOre.h" +#include "ppc-framework/libwrapper/OreFloatingNumber.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace ppc::protocol; +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; +namespace bmp = boost::multiprecision; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(Ore4FloatTest, TestPromptFixture) + +float50 generateRandomFloat50() +{ + std::random_device rd; + std::mt19937_64 gen0(rd()); + std::uniform_int_distribution distribution0(-(1LL << 62) + 2, (1LL << 62) - 2); + int64_t integerPart = distribution0(gen0); + std::mt19937_64 gen1(rd()); + std::uniform_int_distribution distribution1(0, 99999999); + int64_t decimalPart = distribution1(gen1); + float50 randomCppFloat(std::to_string(integerPart) + "." + std::to_string(decimalPart)); + return randomCppFloat; +} + +std::vector generateRandomFloatNumbers(int vectorSize) +{ + std::vector numberVector; + numberVector.reserve(vectorSize); + + for (int i = 0; i < vectorSize; ++i) + { + numberVector.push_back(generateRandomFloat50()); + } + + // Sort the vector of strings + std::sort(numberVector.begin(), numberVector.end()); + return numberVector; +} + +void runOre(bcos::bytes const& key, FastOre::Ptr _ore, const float50& _input) +{ + auto cipher = _ore->encrypt4Float(bcos::ref(key), _input); + auto plain = _ore->decrypt4Float(bcos::ref(key), cipher); + BOOST_CHECK(_input == plain); +} + +void testOreFunctionImpl(bcos::bytes const& key) +{ + auto fastOre = std::make_shared(); + runOre(key, fastOre, float50{"-4611686018427387901.0000000000000213433437899"}); + runOre(key, fastOre, float50{"4611686018427387901.3437899"}); + runOre(key, fastOre, 0); + + auto inputs = generateRandomFloatNumbers(1000); + for (auto input : inputs) + { + runOre(key, fastOre, input); + } +} + +BOOST_AUTO_TEST_CASE(testOreFunction) +{ + auto ore = std::make_shared(); + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreFunctionImpl(key); + key = ore->generateKey(); + testOreFunctionImpl(key); +} + +void runOreOrderTest(bcos::bytes const& key, const std::vector& inputs) +{ + auto fastOre = std::make_shared(); + auto preCipher = fastOre->encrypt4Float(bcos::ref(key), inputs[0]); + + for (uint64_t i = 1; i < inputs.size(); i++) + { + auto cipher = fastOre->encrypt4Float(bcos::ref(key), inputs[i]); + BOOST_CHECK(fastOre->compare(preCipher, cipher) <= 0); + preCipher = cipher; + auto plain = fastOre->decrypt4Float(bcos::ref(key), cipher); + BOOST_CHECK(plain == inputs[i]); + } +} + +void testOreOrderImpl(bcos::bytes const& key) +{ + auto inputs = generateRandomFloatNumbers(1000); + runOreOrderTest(key, inputs); +} + +BOOST_AUTO_TEST_CASE(testOreOrder) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreOrderImpl(key); + auto fastOre = std::make_shared(); + key = fastOre->generateKey(); + testOreOrderImpl(key); +} + +void testEnc(FastOre::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + auto start = bcos::utcTimeUs(); + float50 input("7121925678.23456789"); + for (auto i = 0; i < _count; i++) + { + _ore->encrypt4Float(bcos::ref(key), input); + } + std::cout << "Enc, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +void testDec(FastOre::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + float50 input("7121925678.23456789"); + auto cipher = _ore->encrypt4Float(bcos::ref(key), input); + auto start = bcos::utcTimeUs(); + + for (auto i = 0; i < _count; i++) + { + _ore->decrypt4Float(bcos::ref(key), cipher); + } + std::cout << "Dec, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +void testCompare(bcos::bytes const& key, FastOre::Ptr _ore, int _count) +{ + auto cipher1 = _ore->encrypt4Float(bcos::ref(key), float50{"7121925678.23456789"}); + auto cipher2 = _ore->encrypt4Float(bcos::ref(key), float50{"7121925678.23456789"}); + auto start = bcos::utcTimeUs(); + + for (auto i = 0; i < _count; i++) + { + _ore->compare(cipher1, cipher2); + } + std::cout << "Com, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +BOOST_AUTO_TEST_CASE(PerformanceTest) +{ + auto fastOre = std::make_shared(); + testEnc(fastOre, 100000); + testDec(fastOre, 100000); + auto key = fastOre->generateKey(); + testCompare(key, fastOre, 100000); +} + +void testOreFloatingNumber(std::string _value, std::string _target) +{ + float50 cppDecFloatValue(_value); + OreFloatingNumber ofn(cppDecFloatValue); + std::cout << "### valueStr: " << ofn.value().str() << ", target: " << _target << std::endl; + BOOST_CHECK(ofn.value().str() == _target); +} + +BOOST_AUTO_TEST_CASE(testOreFloatingNumberFunc) +{ + testOreFloatingNumber("0.00000", "0"); + testOreFloatingNumber("12374435346556789", "12374435346556789"); + testOreFloatingNumber("1233456789.0", "1233456789"); + testOreFloatingNumber("123456789.000000", "123456789"); + testOreFloatingNumber("12364456789.1234500", "12364456789.12345"); + testOreFloatingNumber("1234566789.000000019", "1234566789.000000019"); + testOreFloatingNumber("1234566789.999999999", "1234566789.999999999"); + testOreFloatingNumber("-1237456789", "-1237456789"); + testOreFloatingNumber("-123456789.000000", "-123456789"); + testOreFloatingNumber("-12364456789.1234500", "-12364456789.12345"); + testOreFloatingNumber("-1234566789.12345678999", "-1234566789.12345678999"); + testOreFloatingNumber("-1234566789.00000001", "-1234566789.00000001"); + testOreFloatingNumber("-12374435346556789.99999999", "-12374435346556789.99999999"); + testOreFloatingNumber("-12374435346556789.88888888", "-12374435346556789.88888888"); + testOreFloatingNumber("-1234566789.8888", "-1234566789.8888"); + + testOreFloatingNumber( + "-12374435346556789.000000000000088888888", "-12374435346556789.000000000000088888888"); + testOreFloatingNumber("12374435346556789.0000000000000000000000000000000000011118888888811111", + "12374435346556789.0000000000000000000000000000000000011118888888811111"); + // precision truncate case + testOreFloatingNumber( + "25347862354.0000000000000000234564564500000000000000023434200000000000456456", + "25347862354.00000000000000002345645645000000000000000234342"); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-core/tests/TestOre4Integer.cpp b/cpp/ppc-crypto-core/tests/TestOre4Integer.cpp new file mode 100644 index 00000000..6fcabad0 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestOre4Integer.cpp @@ -0,0 +1,179 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestOre4Integer.cpp + * @author: shawnhe + * @date 2023-11-22 + */ +#pragma execution_character_set("utf-8") +#include "ppc-crypto-core/src/ore/FastOre.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace ppc::protocol; +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(Ore4IntegerTest, TestPromptFixture) + +int64_t generateRandomInt64() +{ + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution distribution(-(1LL << 62) + 1, (1LL << 62) - 1); + return distribution(gen); +} + +std::vector generateRandomNumber(int vectorSize) +{ + std::vector numberVector; + numberVector.reserve(vectorSize); + + for (int i = 0; i < vectorSize; ++i) + { + numberVector.push_back(generateRandomInt64()); + } + + // Sort the vector of strings + std::sort(numberVector.begin(), numberVector.end()); + return numberVector; +} + + +void runOre(bcos::bytes const& key, FastOre::Ptr _ore, const int64_t& _input) +{ + auto cipher = _ore->encrypt4Integer(bcos::ref(key), _input); + auto plain = _ore->decrypt4Integer(bcos::ref(key), cipher); + BOOST_CHECK(_input == plain); +} + +void testOreFunctionImpl(bcos::bytes const& key) +{ + auto fastOre = std::make_shared(); + runOre(key, fastOre, -(1LL << 62) + 1); + runOre(key, fastOre, (1LL << 62) - 1); + runOre(key, fastOre, 0); + runOre(key, fastOre, 123456); + runOre(key, fastOre, -234567); + + auto inputs = generateRandomNumber(1000); + for (auto input : inputs) + { + runOre(key, fastOre, input); + } +} + +BOOST_AUTO_TEST_CASE(testOreFunction) +{ + auto ore = std::make_shared(); + int64_t _plain = -123456; + OutputBuffer plain{(bcos::byte*)&_plain, sizeof(_plain)}; + ore->formatNumberPlain(&plain, _plain); + BOOST_CHECK(ore->recoverNumberPlain(plain) == -123456); + + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreFunctionImpl(key); + key = ore->generateKey(); + testOreFunctionImpl(key); +} + +void runOreOrderTest(bcos::bytes const& key, const std::vector& inputs) +{ + auto fastOre = std::make_shared(); + auto preCipher = fastOre->encrypt4Integer(bcos::ref(key), inputs[0]); + + for (uint64_t i = 1; i < inputs.size(); i++) + { + auto cipher = fastOre->encrypt4Integer(bcos::ref(key), inputs[i]); + BOOST_CHECK(fastOre->compare(preCipher, cipher) <= 0); + preCipher = cipher; + auto plain = fastOre->decrypt4Integer(bcos::ref(key), cipher); + BOOST_CHECK(plain == inputs[i]); + } +} + +void testOreOrderImpl(bcos::bytes const& key) +{ + auto inputs = generateRandomNumber(1000); + runOreOrderTest(key, inputs); +} + +BOOST_AUTO_TEST_CASE(testOreOrder) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreOrderImpl(key); + auto fastOre = std::make_shared(); + key = fastOre->generateKey(); + testOreOrderImpl(key); +} + +void testEnc(FastOre::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + auto start = bcos::utcTimeUs(); + int64_t input = 712125678; + for (auto i = 0; i < _count; i++) + { + _ore->encrypt4Integer(bcos::ref(key), input); + } + std::cout << "Enc, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +void testDec(FastOre::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + int64_t input = 8712125678; + auto cipher = _ore->encrypt4Integer(bcos::ref(key), input); + auto start = bcos::utcTimeUs(); + + for (auto i = 0; i < _count; i++) + { + _ore->decrypt4Integer(bcos::ref(key), cipher); + } + std::cout << "Dec, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +void testCompare(bcos::bytes const& key, FastOre::Ptr _ore, int _count) +{ + auto cipher1 = _ore->encrypt4Integer(bcos::ref(key), 8712125678); + auto cipher2 = _ore->encrypt4Integer(bcos::ref(key), 8712125679); + auto start = bcos::utcTimeUs(); + + for (auto i = 0; i < _count; i++) + { + _ore->compare(cipher1, cipher2); + } + std::cout << "Com, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +BOOST_AUTO_TEST_CASE(PerformanceTest) +{ + auto fastOre = std::make_shared(); + testEnc(fastOre, 100000); + testDec(fastOre, 100000); + auto key = fastOre->generateKey(); + testCompare(key, fastOre, 100000); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-core/tests/TestOre4String.cpp b/cpp/ppc-crypto-core/tests/TestOre4String.cpp new file mode 100644 index 00000000..57cea7f5 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestOre4String.cpp @@ -0,0 +1,193 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestOre4String.cpp + * @author: shawnhe + * @date 2023-08-18 + */ +#pragma execution_character_set("utf-8") +#include "ppc-crypto-core/src/ore/FastOre.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ppc::protocol; +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(Ore4StringTest, TestPromptFixture) + +std::vector generateRandomString(int vectorSize, int stringLength) +{ + std::srand(static_cast(std::time(nullptr))); + std::vector stringVector; + stringVector.reserve(vectorSize); + + for (int i = 0; i < vectorSize; ++i) + { + std::string randomString; + randomString.reserve(stringLength); + + for (int j = 0; j < stringLength; ++j) + { + char randomChar = static_cast(std::rand() % 256); + randomString.push_back(randomChar); + } + + stringVector.push_back(randomString); + } + + // Sort the vector of strings + std::sort(stringVector.begin(), stringVector.end()); + return stringVector; +} + +void runOre(bcos::bytes const& key, Ore::Ptr _ore) +{ + std::string input1 = "123456781234567812345678"; + std::string input2 = "中文中文中文中文"; + + auto cipher1 = _ore->encrypt4String(bcos::ref(key), input1); + auto cipher2 = _ore->encrypt4String(bcos::ref(key), input2); + + auto plain2 = _ore->decrypt4String(bcos::ref(key), cipher2); + auto plain1 = _ore->decrypt4String(bcos::ref(key), cipher1); + BOOST_CHECK(input1 == plain1); + BOOST_CHECK(input2 == plain2); +} + +void runOre(bcos::bytes const& key, Ore::Ptr _ore, const std::string& _input) +{ + auto cipher = _ore->encrypt4String(bcos::ref(key), _input); + auto plain = _ore->decrypt4String(bcos::ref(key), cipher); + BOOST_CHECK(_input == plain); +} + +void testOreFunctionImpl(bcos::bytes const& key) +{ + auto fastOre = std::make_shared(); + runOre(key, fastOre); + runOre(key, fastOre, "测试生僻字謇鬱齉躞"); + runOre(key, fastOre, "测试标点符号！@#￥%……&*（）~"); + runOre(key, fastOre, "测试中文"); + runOre(key, fastOre, "abcd1234"); + + auto inputs = generateRandomString(1000, 24); + for (auto input : inputs) + { + runOre(key, fastOre, input); + } +} + +BOOST_AUTO_TEST_CASE(testOreFunction) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreFunctionImpl(key); + + auto ore = std::make_shared(); + key = ore->generateKey(); + testOreFunctionImpl(key); +} + +void runOreOrderTest(bcos::bytes const& key, const std::vector& inputs) +{ + auto fastOre = std::make_shared(); + auto preCipher = fastOre->encrypt4String(bcos::ref(key), inputs[0]); + + for (uint64_t i = 1; i < inputs.size(); i++) + { + auto cipher = fastOre->encrypt4String(bcos::ref(key), inputs[i]); + BOOST_CHECK(fastOre->compare(preCipher, cipher) <= 0); + preCipher = cipher; + auto plain = fastOre->decrypt4String(bcos::ref(key), cipher); + BOOST_CHECK(plain == inputs[i]); + } +} + +void testOreOrderImpl(bcos::bytes const& key) +{ + auto inputs = generateRandomString(1000, 24); + runOreOrderTest(key, inputs); +} + +BOOST_AUTO_TEST_CASE(testOreOrder) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + testOreOrderImpl(key); + auto fastOre = std::make_shared(); + key = fastOre->generateKey(); + testOreOrderImpl(key); +} + +void testEnc(Ore::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + auto start = bcos::utcSteadyTime(); + std::string input = "2024-10-29 12:19:40"; + for (auto i = 0; i < _count; i++) + { + _ore->encrypt4String(bcos::ref(key), input); + } + std::cout << "FastOre Enc, Costs: " << (bcos::utcSteadyTime() - start) << "ms" << std::endl; + std::cout << "FastOre Enc, us/op: " << (bcos::utcSteadyTime() - start)*1000 / _count << "us" << std::endl; +} + +void testDec(Ore::Ptr _ore, int _count) +{ + bcos::bytes key{'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'}; + std::string input = "2024-10-29 12:19:40"; + auto cipher = _ore->encrypt4String(bcos::ref(key), input); + auto start = bcos::utcSteadyTime(); + + for (auto i = 0; i < _count; i++) + { + _ore->decrypt4String(bcos::ref(key), cipher); + } + std::cout << "FastOre Dec, Costs: " << (bcos::utcSteadyTime() - start) << "ms" << std::endl; + std::cout << "FastOre Dec, us/op: " << (bcos::utcSteadyTime() - start)*1000 / _count << "us" << std::endl; +} + +void testCompare(bcos::bytes const& key, Ore::Ptr _ore, int _count) +{ + auto cipher1 = _ore->encrypt4String(bcos::ref(key), "500232188712125678"); + auto cipher2 = _ore->encrypt4String(bcos::ref(key), "500232188712125679"); + auto start = bcos::utcTimeUs(); + + for (auto i = 0; i < _count; i++) + { + _ore->compare(cipher1, cipher2); + } + std::cout << "Com, Costs: " << (bcos::utcTimeUs() - start) * 1000 / _count << "ns" << std::endl; +} + +BOOST_AUTO_TEST_CASE(PerformanceTest) +{ + auto fastOre = std::make_shared(); + testEnc(fastOre, 100000); + testDec(fastOre, 100000); + auto key = fastOre->generateKey(); + testCompare(key, fastOre, 100000); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-core/tests/TestSymCrypto.cpp b/cpp/ppc-crypto-core/tests/TestSymCrypto.cpp new file mode 100644 index 00000000..2d6c3ea6 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/TestSymCrypto.cpp @@ -0,0 +1,318 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestSymCrypto.cpp + * @author: shawnhe + * @date 2022-11-29 + */ +#pragma execution_character_set("utf-8") +#include "ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h" +#include +#include +#include + +using namespace ppc::protocol; +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(SymCryptoTest, TestPromptFixture) + +void runSymCrypto(SymCrypto::OperationMode _mode, SymCrypto::Ptr _symCryptoEnc, + std::string const& _input, bool _useSpecifiedKey = false) +{ + u128 key = 1234567, iv = 7654321; + bcos::bytes bkey; + bcos::toBigEndian(key, bkey); + bcos::bytes biv; + bcos::toBigEndian(iv, biv); + if (!_useSpecifiedKey) + { + bkey = _symCryptoEnc->generateKey(_mode); + } + auto cipher = _symCryptoEnc->encrypt(_mode, bcos::ref(bkey), bcos::ref(biv), + bcos::bytesConstRef((bcos::byte const*)_input.data(), _input.size())); + auto plain = _symCryptoEnc->decrypt(_mode, bcos::ref(bkey), bcos::ref(biv), bcos::ref(cipher)); + std::cout << "### _input: " << _input << std::endl; + std::cout << "### plain: " << std::string(plain.begin(), plain.end()) << std::endl; + BOOST_CHECK(_input == std::string(plain.begin(), plain.end())); +} + +void runSymCrypto( + SymCrypto::OperationMode _mode, SymCrypto::Ptr _symCryptoEnc, bool _useSpecifiedKey) +{ + runSymCrypto(_mode, _symCryptoEnc, "a1", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e5", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e5f6", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e5f6g7", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e5f6g7h", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "a1b2c3d4e5f6g7h8", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "謇", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "鬱", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "齉", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "躞", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "&", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "^", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "……", _useSpecifiedKey); + runSymCrypto(_mode, _symCryptoEnc, "@", _useSpecifiedKey); + + runSymCrypto(_mode, _symCryptoEnc, + "test symmetric encryption 12345678 12345678 12345678 12345678 12345678 12345678" + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 " + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 " + "12345678", + _useSpecifiedKey); + + runSymCrypto(_mode, _symCryptoEnc, + "测试特殊符号*&（……%￥ 123abc 12345678 12345678 12345678 12345678 12345678 12345678" + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 " + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 #$%#&$#$*^&(#%$@#()_()", + _useSpecifiedKey); + + runSymCrypto(_mode, _symCryptoEnc, + "测试生僻字謇鬱齉躞*&（……%￥ 123abc 12345678 12345678 12345678 12345678 12345678 12345678" + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 12345678 " + "12345678 12345678 12345678 12345678 12345678 12345678 12345678 #$%#&$#$*^&(#%$@#()_()", + _useSpecifiedKey); +} + +void runSymCryptoEnc(SymCrypto::OperationMode _mode, SymCrypto::Ptr _symCryptoEnc, + const std::string& _input, const std::string& _expected) +{ + std::string key = "0123456789abcdef0123456789abcdef"; + std::string iv = "0123456789abcdef"; + + bcos::bytes bkey(key.begin(), key.end()); + bcos::bytes biv(iv.begin(), iv.end()); + + auto cipher = _symCryptoEnc->encrypt( + _mode, ref(bkey), ref(biv), bcos::bytesConstRef((bcos::byte*)_input.data(), _input.size())); + std::cout << "cipher: " << *toHexString(cipher) << std::endl; + BOOST_CHECK(_expected == *toHexString(cipher)); +} + +void testAESEncDec(bool _useSpecifiedKey) +{ + // test ecb + std::cout << "====== testSM4EncDesc =====" << std::endl; + auto sm4Enc = std::make_shared(); + std::cout << "====== testSM4EncDesc ECB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::ECB, sm4Enc, _useSpecifiedKey); + // test cbc + std::cout << "====== testSM4EncDesc CBC=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CBC, sm4Enc, _useSpecifiedKey); + std::cout << "====== testSM4EncDesc CFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CFB, sm4Enc, _useSpecifiedKey); + std::cout << "====== testSM4EncDesc OFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::OFB, sm4Enc, _useSpecifiedKey); + std::cout << "====== testSM4EncDesc CTR=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CTR, sm4Enc, _useSpecifiedKey); + std::cout << "====== testSM4EncDesc finished =====" << std::endl; + + std::cout << "====== testAES128EncDec =====" << std::endl; + auto aes128Enc = std::make_shared(OpenSSLAES::AESType::AES128); + std::cout << "====== testAES128EncDec ECB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::ECB, aes128Enc, _useSpecifiedKey); + std::cout << "====== testAES128EncDec CBC=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CBC, aes128Enc, _useSpecifiedKey); + std::cout << "====== testAES128EncDec CFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CFB, aes128Enc, _useSpecifiedKey); + std::cout << "====== testAES128EncDec OFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::OFB, aes128Enc, _useSpecifiedKey); + std::cout << "====== testAES128EncDec CTR=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CTR, aes128Enc, _useSpecifiedKey); + std::cout << "====== testAES128EncDec finished =====" << std::endl; + + std::cout << "====== testAES192EncDec =====" << std::endl; + auto aes192Enc = std::make_shared(OpenSSLAES::AESType::AES192); + std::cout << "====== testAES192EncDec ECB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::ECB, aes192Enc, _useSpecifiedKey); + std::cout << "====== testAES192EncDec CBC=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CBC, aes192Enc, _useSpecifiedKey); + std::cout << "====== testAES192EncDec CFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CFB, aes192Enc, _useSpecifiedKey); + std::cout << "====== testAES192EncDec OFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::OFB, aes192Enc, _useSpecifiedKey); + std::cout << "====== testAES192EncDec CTR=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CTR, aes192Enc, _useSpecifiedKey); + std::cout << "====== testAES192EncDec finished=====" << std::endl; + + std::cout << "====== testAES256EncDec =====" << std::endl; + auto aes256Enc = std::make_shared(OpenSSLAES::AESType::AES256); + std::cout << "====== aes256Enc ECB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::ECB, aes256Enc, _useSpecifiedKey); + std::cout << "====== aes256Enc CBC=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CBC, aes256Enc, _useSpecifiedKey); + std::cout << "====== aes256Enc CFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CFB, aes256Enc, _useSpecifiedKey); + std::cout << "====== aes256Enc OFB=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::OFB, aes256Enc, _useSpecifiedKey); + std::cout << "====== aes256Enc CTR=====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CTR, aes256Enc, _useSpecifiedKey); + std::cout << "====== testAES256EncDec finished=====" << std::endl; + + auto desEnc = std::make_shared(); + std::cout << "====== test3DESEncDec =====" << std::endl; + std::cout << "====== test3DESEncDec: ECB =====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::ECB, desEnc, _useSpecifiedKey); + std::cout << "====== test3DESEncDec: CBC =====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CBC, desEnc, _useSpecifiedKey); + std::cout << "====== test3DESEncDec: CFB =====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::CFB, desEnc, _useSpecifiedKey); + std::cout << "====== test3DESEncDec: OFB =====" << std::endl; + runSymCrypto(SymCrypto::OperationMode::OFB, desEnc, _useSpecifiedKey); + std::cout << "====== test3DESEncDec finished=====" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testSymCryptoPKCS7) +{ + std::cout << "===== testAESEncDec ======" << std::endl; + testAESEncDec(true); + std::cout << "===== testAESEncDec finished ======" << std::endl; + + std::cout << "===== testAESEncDec and generateKey ======" << std::endl; + testAESEncDec(false); + std::cout << "===== testAESEncDec and generateKey finished ======" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testSymCryptoEncPKCS7) +{ + std::cout << "====== testSymCryptoEncPKCS7 =====" << std::endl; + std::string input = "Hello, World, Hello Tomorrow."; + + // test ecb + auto sm4Enc = std::make_shared(); + runSymCryptoEnc(SymCrypto::OperationMode::ECB, sm4Enc, input, + "b654e713da62ef2899233b2542c234461ef8aa28a7faef3a089bbffd4e7f708b"); + runSymCryptoEnc(SymCrypto::OperationMode::CBC, sm4Enc, input, + "5ac1fe687a1227eb8a03d593b2a9a6d4c2d24d6a5bb08c3f4a9427d917fc6643"); + runSymCryptoEnc(SymCrypto::OperationMode::OFB, sm4Enc, input, + "5b42e00fd81c7e6b7c6eb7dfcad5d3e39645db8c7da36770ebf35f7033"); + runSymCryptoEnc(SymCrypto::OperationMode::CTR, sm4Enc, input, + "5b42e00fd81c7e6b7c6eb7dfcad5d3e39288a332b76f342309f73b9940"); + runSymCryptoEnc(SymCrypto::OperationMode::CFB, sm4Enc, input, + "5b42e00fd81c7e6b7c6eb7dfcad5d3e34b9b7ccde57e48d4309e1aa33a"); + + + auto aes128Enc = std::make_shared(OpenSSLAES::AESType::AES128); + runSymCryptoEnc(SymCrypto::OperationMode::ECB, aes128Enc, input, + "2f491097abdda469351be5c2a86c56fc2804e80bd08ac540cf5c30d11658c7f2"); + runSymCryptoEnc(SymCrypto::OperationMode::CBC, aes128Enc, input, + "edd475c62aae977b0cf5a3e6f94c95dbd19955211fd2cbd3df5517a6888c42a7"); + runSymCryptoEnc(SymCrypto::OperationMode::OFB, aes128Enc, input, + "3a1712e471f0dd566fd5740c5529fd000819bd65e9f99987da63e1160e"); + runSymCryptoEnc(SymCrypto::OperationMode::CTR, aes128Enc, input, + "3a1712e471f0dd566fd5740c5529fd009da1a4302925f3fe44d7212132"); + runSymCryptoEnc(SymCrypto::OperationMode::CFB, aes128Enc, input, + "3a1712e471f0dd566fd5740c5529fd00bfdaf41af70405dfeb98b01730"); + + + auto aes192Enc = std::make_shared(OpenSSLAES::AESType::AES192); + runSymCryptoEnc(SymCrypto::OperationMode::ECB, aes192Enc, input, + "926dcc7c086c08663e4c98628c2cc1c55a1c991fc93c37ca1cf7dbc6c660ee60"); + runSymCryptoEnc(SymCrypto::OperationMode::CBC, aes192Enc, input, + "34d5d3dbd184f0f123c96e98880fe877a3dc3364206c062e1d1427c1570244f7"); + + runSymCryptoEnc(SymCrypto::OperationMode::CFB, aes192Enc, input, + "92096245a9be5272896d356998ca309251817976f6521bd51c4ac87ec7"); + runSymCryptoEnc(SymCrypto::OperationMode::OFB, aes192Enc, input, + "92096245a9be5272896d356998ca30929553d60e30f9d84ce15bbdf080"); + runSymCryptoEnc(SymCrypto::OperationMode::CTR, aes192Enc, input, + "92096245a9be5272896d356998ca3092b92fe5fcc9efeaf1e48a3c56b0"); + + + auto aes256Enc = std::make_shared(OpenSSLAES::AESType::AES256); + runSymCryptoEnc(SymCrypto::OperationMode::ECB, aes256Enc, input, + "11f963d26ca05c4a7551e810e692f122f4d57e90d658c7e1ff88dc1432ec170e"); + runSymCryptoEnc(SymCrypto::OperationMode::CBC, aes256Enc, input, + "8937a782e67fd1d56e6e4b8d614e4afe3b1c9084e60d304ffb6546576e397428"); + runSymCryptoEnc(SymCrypto::OperationMode::CFB, aes256Enc, input, + "b059f60cb320fbcf4eed15b2f9fb5e5050e7b2936fe1ff1be3387e5c35"); + runSymCryptoEnc(SymCrypto::OperationMode::OFB, aes256Enc, input, + "b059f60cb320fbcf4eed15b2f9fb5e501790fa5fbe89d8b1a16de8e49e"); + runSymCryptoEnc(SymCrypto::OperationMode::CTR, aes256Enc, input, + "b059f60cb320fbcf4eed15b2f9fb5e50d9f3075d1b0d4f080e5a633deb"); + + auto desEnc = std::make_shared(); + runSymCryptoEnc(SymCrypto::OperationMode::ECB, desEnc, input, + "0bb66fcb35528a9aae4240b9235cd5b403d4e7648bef133d8f145e5475462c26"); + runSymCryptoEnc(SymCrypto::OperationMode::CBC, desEnc, input, + "a8f91cfb2ab2f98f5f8cb92545a2b3112df69bd94c64e883a9ec84e832cb273d"); + runSymCryptoEnc(SymCrypto::OperationMode::CFB, desEnc, input, + "04c5b63ce04ad91512c7616c0845f64896a1ffdeb94bbc114b3d23d78b"); + runSymCryptoEnc(SymCrypto::OperationMode::OFB, desEnc, input, + "04c5b63ce04ad915d7cc3432e00d338268178b71b2ee5ca31433e57916"); + std::cout << "====== testSymCryptoEncPKCS7 finished =====" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testSm4) +{ + auto sm4Enc = std::make_shared(); + bcos::bytes bkey = sm4Enc->generateKey(SymCrypto::OperationMode::CBC); + bcos::bytes biv; + u128 iv = 7654321; + bcos::toBigEndian(iv, biv); + + bcos::bytes inputs(1024 * 10); + + int count = 10000; + std::vector ciphers(count); + + auto start = bcos::utcTimeUs(); + for(int i = 0; i < count; i++) + { + ciphers[i] = sm4Enc->encrypt(SymCrypto::OperationMode::CBC, bcos::ref(bkey), bcos::ref(biv), + bcos::bytesConstRef((bcos::byte const*)inputs.data(), inputs.size())); + } + std::cout << "CBC, ENC Costs: " << bcos::utcTimeUs() - start<< "us" << std::endl; + + start = bcos::utcTimeUs(); + for(int i = 0; i < count; i++) + { + sm4Enc->decrypt(SymCrypto::OperationMode::CBC, bcos::ref(bkey), bcos::ref(biv), + bcos::bytesConstRef((bcos::byte const*)ciphers[i].data(), ciphers[i].size())); + } + std::cout << "CBC, DEC Costs: " << bcos::utcTimeUs() - start<< "us" << std::endl; + + start = bcos::utcTimeUs(); + for(int i = 0; i < count; i++) + { + ciphers[i] = sm4Enc->encrypt(SymCrypto::OperationMode::CTR, bcos::ref(bkey), bcos::ref(biv), + bcos::bytesConstRef((bcos::byte const*)inputs.data(), inputs.size())); + } + std::cout << "CTR, ENC Costs: " << bcos::utcTimeUs() - start<< "us" << std::endl; + + start = bcos::utcTimeUs(); + for(int i = 0; i < count; i++) + { + sm4Enc->decrypt(SymCrypto::OperationMode::CTR, bcos::ref(bkey), bcos::ref(biv), + bcos::bytesConstRef((bcos::byte const*)ciphers[i].data(), ciphers[i].size())); + } + std::cout << "CTR, DEC Costs: " << bcos::utcTimeUs() - start<< "us" << std::endl; + +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto-core/tests/main.cpp b/cpp/ppc-crypto-core/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-crypto-core/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-crypto/CMakeLists.txt b/cpp/ppc-crypto/CMakeLists.txt new file mode 100644 index 00000000..c17b68b1 --- /dev/null +++ b/cpp/ppc-crypto/CMakeLists.txt @@ -0,0 +1,11 @@ +project(ppc-crypto VERSION ${VERSION}) + +add_subdirectory(src) +if(DEMO) + add_subdirectory(demo) +endif() +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/cpp/ppc-crypto/demo/CMakeLists.txt b/cpp/ppc-crypto/demo/CMakeLists.txt new file mode 100644 index 00000000..5a7678b8 --- /dev/null +++ b/cpp/ppc-crypto/demo/CMakeLists.txt @@ -0,0 +1,7 @@ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(BINARY_NAME crypto_bench) +add_executable(${BINARY_NAME} crypto_bench.cpp) +target_link_libraries(${BINARY_NAME} ${CRYPTO_TARGET}) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-crypto/demo/crypto_bench.cpp b/cpp/ppc-crypto/demo/crypto_bench.cpp new file mode 100644 index 00000000..53688fa7 --- /dev/null +++ b/cpp/ppc-crypto/demo/crypto_bench.cpp @@ -0,0 +1,439 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file crypto_bench.cpp + * @desc: bench for io + * @author: yujiechen + * @date 2022-12-1 + */ +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-crypto-core/src/hash/SM3Hash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto-core/src/hash/Sha512Hash.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h" +#include "ppc-crypto/src/ecc/ECDHCryptoImpl.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/IppECDHCryptoImpl.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-crypto/src/oprf/RA2018Oprf.h" +#include "ppc-framework/io/DataBatch.h" + +using namespace bcos; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::crypto; + +const std::string HASH_CMD = "hash"; +const std::string ECC_CMD = "ecc"; +const std::string ENCRYPT_CMD = "enc"; +const std::string OPRF_CMD = "oprf"; +const std::string ECDH_CMD = "ecdh"; + + +void Usage(std::string const& _appName) +{ + std::cout << _appName << " [" << HASH_CMD << "/" << ECC_CMD << "/" << ENCRYPT_CMD << "/" + << OPRF_CMD << "/" << ECDH_CMD << "] count" << std::endl; +} + +double getTPS(int64_t _endT, int64_t _startT, size_t _count) +{ + return (1000.0 * (double)_count) / (double)(_endT - _startT); +} + +void hashPerf( + Hash::Ptr _hash, std::string const& _hashName, std::string const& _inputData, size_t _count) +{ + std::cout << std::endl; + std::cout << "----------- " << _hashName << " perf start -----------" << std::endl; + auto startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _hash->hash(bytesConstRef((byte const*)_inputData.c_str(), _inputData.size())); + } + std::cout << "input data size: " << (double)_inputData.size() / 1000.0 + << "KB, loops: " << _count << ", timeCost: " << utcSteadyTime() - startT << " ms" + << std::endl; + std::cout << "TPS of " << _hashName << ": " + << getTPS(utcSteadyTime(), startT, _count) * (double)_inputData.size() / 1000.0 + << " KB/s" << std::endl; + std::cout << "----------- " << _hashName << " perf end -----------" << std::endl; + std::cout << std::endl; +} + +void hashPerf(size_t _count) +{ + std::string inputData = "abcdwer234q4@#2424wdf"; + std::string deltaData = inputData; + for (int i = 0; i < 50; i++) + { + inputData += deltaData; + } + // MD5Hash perf + Hash::Ptr hashImpl = std::make_shared(); + hashPerf(hashImpl, "MD5Hash", inputData, _count); + // Sha256Hash perf + hashImpl = std::make_shared(); + hashPerf(hashImpl, "Sha256Hash", inputData, _count); + // Sha512Hash perf + hashImpl = std::make_shared(); + hashPerf(hashImpl, "Sha512Hash", inputData, _count); + // SM3Hash perf + hashImpl = std::make_shared(); + hashPerf(hashImpl, "SM3Hash", inputData, _count); + // BLAKE2bHash + hashImpl = std::make_shared(); + hashPerf(hashImpl, "BLAKE2bHash", inputData, _count); +} + +void eccCryptoPerf(EccCrypto::Ptr _eccCrypto, std::string const& _eccName, + bcos::bytes const& _hashData, size_t _count) +{ + std::cout << std::endl; + std::cout << "----------- " << _eccName << " perf start -----------" << std::endl; + // generateRandomScalar + auto startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _eccCrypto->generateRandomScalar(); + } + std::cout << "TPS of " << _eccName + << ": generateRandomScalar: " << getTPS(utcSteadyTime(), startT, _count) + << ", timecost:" << (utcSteadyTime() - startT) << " ms" << std::endl; + // hashToCurve + startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _eccCrypto->hashToCurve(_hashData); + } + std::cout << "TPS of " << _eccName + << ": hashToCurve: " << getTPS(utcSteadyTime(), startT, _count) + << ", timecost:" << (utcSteadyTime() - startT) << " ms" << std::endl; + auto point = _eccCrypto->hashToCurve(_hashData); + auto scalar = _eccCrypto->generateRandomScalar(); + // ecMultiply + startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _eccCrypto->ecMultiply(point, scalar); + } + std::cout << "TPS of " << _eccName + << ": ecMultiply: " << getTPS(utcSteadyTime(), startT, _count) + << ", timecost:" << (utcSteadyTime() - startT) << " ms" << std::endl; + // scalarInvert + startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _eccCrypto->scalarInvert(scalar); + } + std::cout << "TPS of " << _eccName + << ": scalarInvert: " << getTPS(utcSteadyTime(), startT, _count) + << ", timecost:" << (utcSteadyTime() - startT) << " ms" << std::endl; + // mulGenerator + startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + _eccCrypto->mulGenerator(scalar); + } + std::cout << "TPS of " << _eccName + << ": mulGenerator: " << getTPS(utcSteadyTime(), startT, _count) + << ", timecost:" << (utcSteadyTime() - startT) << " ms" << std::endl; + std::cout << "----------- " << _eccName << " perf end -----------" << std::endl; + std::cout << std::endl; +} + +void eccCryptoPerf(size_t _count) +{ + auto hashImpl = std::make_shared(); + std::string inputStr = "abcdwer234q4@#2424wdf"; + bytes inputData(inputStr.begin(), inputStr.end()); + auto hashData = hashImpl->hash(ref(inputData)); + // ed25519 perf + EccCrypto::Ptr eccCrypto = std::make_shared(); + eccCryptoPerf(eccCrypto, "Ed25519EccCrypto", hashData, _count); + // sm2 perf + eccCrypto = std::make_shared(hashImpl, ECCCurve::SM2); + eccCryptoPerf(eccCrypto, "OpenSSLEccCrypto-SM2", hashData, _count); + // prime256v1 perf + eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + eccCryptoPerf(eccCrypto, "OpenSSLEccCrypto-P256", hashData, _count); + // secp256k1 perf + eccCrypto = std::make_shared(hashImpl, ECCCurve::SECP256K1); + eccCryptoPerf(eccCrypto, "OpenSSLEccCrypto-SECP256K1", hashData, _count); +} + + +void encryptPerf(SymCrypto::Ptr _encryptor, OpenSSLAES::OperationMode _mode, + std::string const& _inputData, const std::string& _encryptorName, size_t _count) +{ + std::string keyString = "abcdefgwerelkewrwerw"; + bcos::bytes key(keyString.begin(), keyString.end()); + std::string ivData = "12334324"; + bcos::bytes iv(ivData.begin(), ivData.end()); + + std::cout << std::endl; + std::cout << "----------- " << _encryptorName << " perf test start -----------" << std::endl; + // encrypt + bcos::bytes encryptedData; + auto startT = utcSteadyTime(); + for (size_t i = 0; i < _count; i++) + { + encryptedData = _encryptor->encrypt(_mode, ref(key), ref(iv), + bytesConstRef((byte const*)_inputData.c_str(), _inputData.size())); + } + std::cout << "PlainData size:" << (double)_inputData.size() / 1000.0 << " KB, loops: " << _count + << ", timeCost: " << utcSteadyTime() - startT << " ms" << std::endl; + std::cout << "TPS of " << _encryptorName << " encrypt:" + << (getTPS(utcSteadyTime(), startT, _count) * (double)(_inputData.size())) / 1000.0 + << "KB/s" << std::endl; + std::cout << std::endl; + // decrypt + startT = utcSteadyTime(); + bytes decryptedData; + for (size_t i = 0; i < _count; i++) + { + decryptedData = _encryptor->decrypt(_mode, ref(key), ref(iv), ref(encryptedData)); + } + std::cout << "CiperData size:" << (double)encryptedData.size() / 1000.0 + << " KB, loops: " << _count << ", timeCost:" << utcSteadyTime() - startT << " ms" + << std::endl; + std::cout << "TPS of " << _encryptorName << " decrypt:" + << (getTPS(utcSteadyTime(), startT, _count) * (double)_inputData.size()) / 1000.0 + << "KB/s" << std::endl; + bytes plainBytes(_inputData.begin(), _inputData.end()); + assert(plainBytes == decryptedData); + + std::cout << "----------- " << _encryptorName << " perf test end -----------" << std::endl; + std::cout << std::endl; +} + +void encryptPerf(size_t _count) +{ + std::string inputData = "w3rwerk2-304swlerkjewlrjoiur4kslfjsd,fmnsdlfjlwerlwerjw;erwe;rewrew"; + std::string deltaData = inputData; + for (int i = 0; i < 100; i++) + { + inputData += deltaData; + } + + // AES128-CBC + SymCrypto::Ptr encryptor = nullptr; + encryptor = std::make_shared(OpenSSLAES::AESType::AES128); + encryptPerf(encryptor, SymCrypto::OperationMode::CBC, inputData, "AES128-CBC", _count); + + // AES192-CBC + encryptor = std::make_shared(OpenSSLAES::AESType::AES192); + encryptPerf(encryptor, SymCrypto::OperationMode::CBC, inputData, "AES192-CBC", _count); + // AES256-CBC + encryptor = std::make_shared(OpenSSLAES::AESType::AES256); + encryptPerf(encryptor, SymCrypto::OperationMode::CBC, inputData, "AES256-CBC", _count); + // SM4-CBC + encryptor = std::make_shared(); + encryptPerf(encryptor, SymCrypto::OperationMode::CBC, inputData, "SM4-CBC", _count); +} + + +void RA2018OprfCryptoPerf( + RA2018Oprf::Ptr _oprf, std::vector const& _inputData, std::string const& _testCase) +{ + std::cout << std::endl; + std::cout << "----------- " << _testCase << " perf start -----------" << std::endl; + //// FullEvaluate + auto tmpData = _inputData; + auto dataBatch = std::make_shared(); + dataBatch->setData(std::move(tmpData)); + + auto startT = utcSteadyTime(); + std::vector fullEvaluateResult; + _oprf->fullEvaluate(dataBatch, fullEvaluateResult); + std::cout << "fullEvaluate: timecost of " << _testCase << " : " << (utcSteadyTime() - startT) + << " ms" << std::endl; + + /// blind + startT = utcSteadyTime(); + std::vector blindResult; + auto privateKey = _oprf->generatePrivateKey(); + _oprf->blind(dataBatch, privateKey, blindResult); + std::cout << "Blind: timecost of " << _testCase << " : " << (utcSteadyTime() - startT) << " ms" + << std::endl; + + /// evaluate + startT = utcSteadyTime(); + std::vector evaluateData; + _oprf->evaluate(blindResult, evaluateData); + std::cout << "Evaluate: timecost of " << _testCase << " : " << (utcSteadyTime() - startT) + << " ms" << std::endl; + + /// finalize + startT = utcSteadyTime(); + std::vector finalizeResult; + auto invPrivateKey = _oprf->inv(privateKey); + _oprf->finalize(evaluateData, invPrivateKey, finalizeResult); + std::cout << "Finalize: timecost of " << _testCase << " : " << (utcSteadyTime() - startT) + << " ms" << std::endl; + std::cout << "----------- " << _testCase << " perf end -----------" << std::endl; + std::cout << std::endl; +} + +void oprfCryptoPerf(size_t _count) +{ + // fake the input data + std::vector data; + for (size_t i = 0; i < _count; i++) + { + auto inputStr = std::to_string(i); + data.emplace_back(bcos::bytes(inputStr.begin(), inputStr.end())); + } + + EccCrypto::Ptr eccCrypto = std::make_shared(); + // generate the private-key + auto privateKey = eccCrypto->generateRandomScalar(); + auto hashImpl = std::make_shared(); + + /// ed25519 ra2018 oprf + auto oprf = std::make_shared(privateKey, eccCrypto, hashImpl); + RA2018OprfCryptoPerf(oprf, data, "RA2018-OPRF-ED25519"); + // P256 ra2018 oprf + eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + privateKey = eccCrypto->generateRandomScalar(); + oprf = std::make_shared(privateKey, eccCrypto, hashImpl); + RA2018OprfCryptoPerf(oprf, data, "RA2018-OPRF-P256"); + + /// sm2 ra2018 oprf + eccCrypto = std::make_shared(hashImpl, ECCCurve::SM2); + privateKey = eccCrypto->generateRandomScalar(); + oprf = std::make_shared(privateKey, eccCrypto, hashImpl); + RA2018OprfCryptoPerf(oprf, data, "RA2018-OPRF-SM2"); +} + +void ecdhCryptoPerf(ECDHCrypto::Ptr _ecdhCrypto, std::vector const& _inputData, + std::string const& _testCase) +{ + try + { + std::cout << std::endl; + std::cout << "----------- " << _testCase << " perf start -----------" << std::endl; + //// batchGetPublicKey + auto tmpData = _inputData; + auto dataBatch = std::make_shared(); + dataBatch->setData(std::move(tmpData)); + + auto startT = utcSteadyTime(); + auto publicKeyResult = _ecdhCrypto->batchGetPublicKey(dataBatch); + std::cout << "batchGetPublicKey: timecost of " << _testCase << " : " + << (utcSteadyTime() - startT) << " ms" << std::endl; + + /// batchGetSharedPublicKey + startT = utcSteadyTime(); + auto sharedKeys = _ecdhCrypto->batchGetSharedPublicKey(publicKeyResult); + std::cout << "batchGetSharedPublicKey: timecost of " << _testCase << " : " + << (utcSteadyTime() - startT) << " ms" << std::endl; + std::cout << "----------- " << _testCase << " perf end -----------" << std::endl; + } + catch (std::exception const& e) + { + std::cout << "ecdhCryptoPerf exception, error: " << boost::diagnostic_information(e) + << std::endl; + } + std::cout << std::endl; +} + +void ecdhCryptoPerf(size_t _count) +{ + // fake the input data + std::vector data; + for (size_t i = 0; i < _count; i++) + { + auto inputStr = std::to_string(i); + data.emplace_back(bcos::bytes(inputStr.begin(), inputStr.end())); + } + + EccCrypto::Ptr eccCrypto = std::make_shared(); + // generate the private-key + auto privateKey = eccCrypto->generateRandomScalar(); + auto hashImpl = std::make_shared(); + auto cryptoBox = std::make_shared(hashImpl, eccCrypto); + + /// ed25519 ecdh-crypto + auto ecdhCrypto = std::make_shared(privateKey, cryptoBox); + ecdhCryptoPerf(ecdhCrypto, data, "ECDH-CRYPTO-ED25519"); + // P256 ecdh-crypto + eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + privateKey = eccCrypto->generateRandomScalar(); + cryptoBox = std::make_shared(hashImpl, eccCrypto); + ecdhCrypto = std::make_shared(privateKey, cryptoBox); + ecdhCryptoPerf(ecdhCrypto, data, "ECDH-CRYPTO-P256"); + + /// sm2 ecdh-crypto + eccCrypto = std::make_shared(hashImpl, ECCCurve::SM2); + privateKey = eccCrypto->generateRandomScalar(); + cryptoBox = std::make_shared(hashImpl, eccCrypto); + ecdhCrypto = std::make_shared(privateKey, cryptoBox); + ecdhCryptoPerf(ecdhCrypto, data, "ECDH-CRYPTO-SM2"); + +// ipp x25519 crypto +#ifdef ENABLE_CRYPTO_MB + if (ppc::CPU_FEATURES.avx512ifma) + { + auto ippEdchCrypto = std::make_shared(privateKey, hashImpl); + ecdhCryptoPerf(ippEdchCrypto, data, "IPP-ECDH-CRYPTO-x25519"); + } + else + { + std::cout << "Ignore IPP-ECDH-CRYPTO-x25519 test for crypto_mb not enabled!" << std::endl; + } +#else + std::cout << "Ignore IPP-ECDH-CRYPTO-x25519 test for crypto_mb not enabled!" << std::endl; +#endif +} + +int main(int argc, char* argv[]) +{ + if (argc < 3) + { + Usage(argv[0]); + return -1; + } + auto cmd = argv[1]; + size_t count = atoi(argv[2]); + if (HASH_CMD == cmd) + { + hashPerf(count); + } + else if (ENCRYPT_CMD == cmd) + { + encryptPerf(count); + } + else if (ECC_CMD == cmd) + { + eccCryptoPerf(count); + } + else if (OPRF_CMD == cmd) + { + oprfCryptoPerf(count); + } + else if (ECDH_CMD == cmd) + { + ecdhCryptoPerf(count); + } + else + { + std::cout << "Invalid subcommand \"" << cmd << "\"" << std::endl; + Usage(argv[0]); + } + return 0; +} diff --git a/cpp/ppc-crypto/src/CMakeLists.txt b/cpp/ppc-crypto/src/CMakeLists.txt new file mode 100644 index 00000000..b699d88c --- /dev/null +++ b/cpp/ppc-crypto/src/CMakeLists.txt @@ -0,0 +1,14 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${CRYPTO_TARGET} ${SRCS}) + +find_package(unofficial-sodium REQUIRED) +find_package(OpenSSL REQUIRED) +message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}") +message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}") + +target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB}) + +if (ENABLE_IPP_CRYPTO) + find_package(ipp-crypto REQUIRED) + target_link_libraries(${CRYPTO_TARGET} PUBLIC ipp-crypto::crypto_mb) +endif () \ No newline at end of file diff --git a/cpp/ppc-crypto/src/Common.h b/cpp/ppc-crypto/src/Common.h new file mode 100644 index 00000000..de36dddc --- /dev/null +++ b/cpp/ppc-crypto/src/Common.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "ppc-framework/Common.h" +#include +#include + +#define CRYPTO_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("CRYPTO") + +namespace ppc::crypto +{ +DERIVE_PPC_EXCEPTION(OprfFinalizeException); +DERIVE_PPC_EXCEPTION(HashToCurveException); +DERIVE_PPC_EXCEPTION(HashToScalarException); +DERIVE_PPC_EXCEPTION(ScalarInvertException); +DERIVE_PPC_EXCEPTION(ScalarCalculateException); +DERIVE_PPC_EXCEPTION(EcMultipleException); +DERIVE_PPC_EXCEPTION(MulGeneratorError); +DERIVE_PPC_EXCEPTION(EcAddError); +DERIVE_PPC_EXCEPTION(EcSubError); + +DERIVE_PPC_EXCEPTION(ECGroupGetCurveError); +DERIVE_PPC_EXCEPTION(ECGroupGetOrderError); +DERIVE_PPC_EXCEPTION(ECPoint2BNError); +DERIVE_PPC_EXCEPTION(ECPointMulError); +DERIVE_PPC_EXCEPTION(ECPointAddError); +DERIVE_PPC_EXCEPTION(ECPointSubError); +DERIVE_PPC_EXCEPTION(ECPointBn2PointError); + +DERIVE_PPC_EXCEPTION(UnsupportedCurveType); +DERIVE_PPC_EXCEPTION(GenerateRandomScalarError); + +DERIVE_PPC_EXCEPTION(X25519GetSharedKeyError); +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h b/cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h new file mode 100644 index 00000000..9fcede1d --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ECDHCryptoFactoryImpl.h + * @author: yujiechen + * @date 2023-1-3 + */ +#pragma once +#include "../ecc/EccCryptoFactoryImpl.h" +#include "ECDHCryptoImpl.h" +#include "IppECDHCryptoImpl.h" +#include "ppc-crypto-core/src/hash/HashFactoryImpl.h" +#include "ppc-framework/crypto/CryptoBox.h" +namespace ppc::crypto +{ +class ECDHCryptoFactoryImpl : public ECDHCryptoFactory +{ +public: + using Ptr = std::shared_ptr; + ECDHCryptoFactoryImpl(bcos::bytes const& _privateKey) + : m_privateKey(_privateKey), + m_hashFactory(std::make_shared()), + m_eccCryptoFactory(std::make_shared()) + {} + + ~ECDHCryptoFactoryImpl() override = default; + + ECDHCrypto::Ptr createECDHCrypto(int _curveType, int _hashType) const override + { + auto hashImpl = m_hashFactory->createHashImpl(_hashType); +#ifdef ENABLE_CRYPTO_MB + if (ppc::CPU_FEATURES.avx512ifma && _curveType == (int)ppc::protocol::ECCCurve::IPP_X25519) + { + return std::make_shared(m_privateKey, hashImpl); + } +#endif + if (_curveType == (int)ppc::protocol::ECCCurve::IPP_X25519) + { + BOOST_THROW_EXCEPTION( + UnsupportedCurveType() << bcos::errinfo_comment( + "Not supported IPP_X25519 for missing avx512ifma instruction")); + } + auto eccCrypto = m_eccCryptoFactory->createEccCrypto(_curveType, hashImpl); + auto cryptoBox = std::make_shared(hashImpl, eccCrypto); + return std::make_shared(m_privateKey, cryptoBox); + } + + CryptoBox::Ptr createCryptoBox(int8_t _curveType, int8_t _hashType) const override + { + auto hashImpl = m_hashFactory->createHashImpl(_hashType); + auto eccCrypto = m_eccCryptoFactory->createEccCrypto(_curveType, hashImpl); + return std::make_shared(hashImpl, eccCrypto); + } + +private: + bcos::bytes m_privateKey; + HashFactory::Ptr m_hashFactory; + EccCryptoFactory::Ptr m_eccCryptoFactory; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h b/cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h new file mode 100644 index 00000000..5a21c5f3 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ECDHCryptoImpl.h + * @author: yujiechen + * @date 2022-12-29 + */ +#pragma once +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/crypto/ECDHCrypto.h" +#include + +namespace ppc::crypto +{ +class ECDHCryptoImpl : public ECDHCrypto +{ +public: + using Ptr = std::shared_ptr; + ECDHCryptoImpl(bcos::bytes const& _privateKey, CryptoBox::Ptr const& _cryptoBox) + : m_privateKey(_privateKey), m_cryptoBox(_cryptoBox) + {} + + // calculate the ecdh public-key according to privateKey and input + std::vector batchGetPublicKey(ppc::io::DataBatch::Ptr const& _input) override + { + std::vector result; + result.resize(_input->size()); + tbb::parallel_for(tbb::blocked_range(0U, _input->size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto const& data = _input->getBytes(i); + auto hashData = + m_cryptoBox->hashImpl()->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_cryptoBox->eccCrypto()->hashToCurve(hashData); + result[i] = m_cryptoBox->eccCrypto()->ecMultiply(point, m_privateKey); + } + }); + return result; + } + + // calculate the ecdh shared-publicKey according to the publicKey and privateKey + std::vector batchGetSharedPublicKey( + std::vector const& _publicKeys) override + { + std::vector result; + result.resize(_publicKeys.size()); + tbb::parallel_for( + tbb::blocked_range(0U, _publicKeys.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + result[i] = + m_cryptoBox->eccCrypto()->ecMultiply(_publicKeys.at(i), m_privateKey); + } + }); + return result; + } + +private: + bcos::bytes m_privateKey; + CryptoBox::Ptr m_cryptoBox; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h b/cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h new file mode 100644 index 00000000..ae428660 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EccCryptoFactoryImpl.h + * @author: yujiechen + * @date 2023-1-3 + */ +#pragma once +#include "../Common.h" +#include "Ed25519EccCrypto.h" +#include "OpenSSLEccCrypto.h" +#include "ppc-framework/protocol/Protocol.h" + +namespace ppc::crypto +{ +class EccCryptoFactoryImpl : public EccCryptoFactory +{ +public: + using Ptr = std::shared_ptr; + EccCryptoFactoryImpl() = default; + ~EccCryptoFactoryImpl() override = default; + + EccCrypto::Ptr createEccCrypto(int8_t _curveType, Hash::Ptr const& _hashImpl) const override + { + switch (_curveType) + { + case (int8_t)ppc::protocol::ECCCurve::ED25519: + return std::make_shared(); + case (int8_t)ppc::protocol::ECCCurve::SM2: + case (int8_t)ppc::protocol::ECCCurve::SECP256K1: + case (int8_t)ppc::protocol::ECCCurve::P256: + return std::make_shared( + _hashImpl, (ppc::protocol::ECCCurve)_curveType); + default: + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << bcos::errinfo_comment( + "unsupported curveType: " + std::to_string(_curveType))); + } + } +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp b/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp new file mode 100644 index 00000000..0619c17a --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Ed25519EccCrypto.cpp + * @author: yujiechen + * @date 2022-11-2 + */ +#include "Ed25519EccCrypto.h" +#include "../Common.h" +#include +using namespace ppc::crypto; +using namespace bcos; + +uint32_t Ed25519EccCrypto::pointSizeInBytes() const +{ + return crypto_core_ed25519_BYTES; +} + +bcos::bytes Ed25519EccCrypto::generateRandomScalar() const +{ + bcos::bytes scalar(crypto_core_ed25519_SCALARBYTES); + crypto_core_ed25519_random(scalar.data()); + bcos::bytes reducedResult(crypto_core_ed25519_SCALARBYTES); + // Note: must reduce here to mod L + crypto_core_ed25519_scalar_reduce(reducedResult.data(), scalar.data()); + return reducedResult; +} + +bcos::bytes Ed25519EccCrypto::hashToCurve(bcos::bytes&& _hashData) const +{ + if (_hashData.size() < crypto_core_ed25519_HASHBYTES) + { + _hashData.resize(crypto_core_ed25519_HASHBYTES); + } + return convertHashToCurve(_hashData); +} + +bcos::bytes Ed25519EccCrypto::hashToCurve(bcos::bytes const& _hashData) const +{ + if (_hashData.size() < crypto_core_ed25519_HASHBYTES) + { + bcos::bytes tmpData(_hashData.begin(), _hashData.end()); + tmpData.resize(crypto_core_ed25519_HASHBYTES); + return convertHashToCurve(tmpData); + } + return convertHashToCurve(_hashData); +} + +// convert the hashData into ec-point over the given curve +bcos::bytes Ed25519EccCrypto::convertHashToCurve(bcos::bytes const& _hashData) const +{ + bcos::bytes ecPoint(crypto_core_ed25519_BYTES); + auto ret = crypto_core_ed25519_from_hash(ecPoint.data(), _hashData.data()); + if (ret) + { + BOOST_THROW_EXCEPTION( + HashToCurveException() << errinfo_comment( + "call crypto_core_ed25519_from_hash error, code: " + std::to_string(ret))); + } + return ecPoint; +} + +// get the multiplicative inverse of _data over the given curve +bcos::bytes Ed25519EccCrypto::scalarInvert(bcos::bytes const& _data) const +{ + if (_data.size() < crypto_core_ed25519_SCALARBYTES) + { + BOOST_THROW_EXCEPTION(ScalarInvertException() << errinfo_comment( + "the length of scalar-data must be no smaller than: " + + std::to_string(crypto_core_ed25519_SCALARBYTES))); + } + bcos::bytes invertedScalar(crypto_core_ed25519_SCALARBYTES); + auto ret = crypto_core_ed25519_scalar_invert(invertedScalar.data(), _data.data()); + if (ret) + { + BOOST_THROW_EXCEPTION( + HashToCurveException() << errinfo_comment( + "call crypto_core_ed25519_scalar_invert error, code: " + std::to_string(ret))); + } + return invertedScalar; +} + +// multiply the _ecPoint by _scalar +bcos::bytes Ed25519EccCrypto::ecMultiply( + bcos::bytes const& _ecPoint, bcos::bytes const& _scalar) const +{ + if (_ecPoint.size() < crypto_scalarmult_ed25519_BYTES) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "the ecpoint for ed25519 must be no smaller than: " + + std::to_string(crypto_scalarmult_ed25519_BYTES))); + } + if (_scalar.size() < crypto_scalarmult_ed25519_SCALARBYTES) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "the scalar for ed25519 must be no smaller than: " + + std::to_string(crypto_scalarmult_ed25519_SCALARBYTES))); + } + bcos::bytes result(crypto_scalarmult_ed25519_BYTES); + auto ret = crypto_scalarmult_ed25519_noclamp(result.data(), _scalar.data(), _ecPoint.data()); + if (ret) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "crypto_scalarmult_ed25519 error: " + std::to_string(ret))); + } + return result; +} + +bcos::bytes Ed25519EccCrypto::ecAdd( + bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const +{ + if (_ecPointA.size() < crypto_scalarmult_ed25519_BYTES || + _ecPointB.size() < crypto_scalarmult_ed25519_BYTES) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "the ecpoint for ed25519 must be no smaller than: " + + std::to_string(crypto_scalarmult_ed25519_BYTES))); + } + + bcos::bytes result(crypto_scalarmult_ed25519_BYTES); + auto ret = crypto_core_ed25519_add(result.data(), _ecPointA.data(), _ecPointB.data()); + if (ret) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "crypto_core_ed25519_add error: " + std::to_string(ret))); + } + return result; +} + +bcos::bytes Ed25519EccCrypto::ecSub( + bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const +{ + if (_ecPointA.size() < crypto_scalarmult_ed25519_BYTES || + _ecPointB.size() < crypto_scalarmult_ed25519_BYTES) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "the ecpoint for ed25519 must be no smaller than: " + + std::to_string(crypto_scalarmult_ed25519_BYTES))); + } + + bcos::bytes result(crypto_scalarmult_ed25519_BYTES); + auto ret = crypto_core_ed25519_sub(result.data(), _ecPointA.data(), _ecPointB.data()); + if (ret) + { + BOOST_THROW_EXCEPTION(EcMultipleException() << errinfo_comment( + "crypto_core_ed25519_sub error: " + std::to_string(ret))); + } + return result; +} + +bool Ed25519EccCrypto::isValidEcPoint(bcos::bytes const& _ecPoint) const +{ + if (_ecPoint.size() < crypto_core_ed25519_SCALARBYTES) + { + return false; + } + // Note: if _ecPoint is not a valid point, return 0 + return (crypto_core_ed25519_is_valid_point(_ecPoint.data()) != 0); +} + +bcos::bytes Ed25519EccCrypto::mulGenerator(bcos::bytes const& _scalar) const +{ + bcos::bytes result(crypto_scalarmult_ed25519_SCALARBYTES); + auto ret = crypto_scalarmult_ed25519_base_noclamp(result.data(), _scalar.data()); + if (ret) + { + BOOST_THROW_EXCEPTION( + EcMultipleException() << errinfo_comment( + "crypto_scalarmult_ed25519_base_noclamp error: " + std::to_string(ret))); + } + return result; +} + +bcos::bytes Ed25519EccCrypto::hashToScalar(bcos::bytes const& _hashData) const +{ + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << errinfo_comment("method not support: ")); +} + +bcos::bytes Ed25519EccCrypto::scalarAdd( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << errinfo_comment("method not support: ")); +} + +bcos::bytes Ed25519EccCrypto::scalarSub( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << errinfo_comment("method not support: ")); +} + +bcos::bytes Ed25519EccCrypto::scalarMul( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << errinfo_comment("method not support: ")); +} diff --git a/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h b/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h new file mode 100644 index 00000000..baa39675 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Ed25519EccCrypto.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/protocol/Protocol.h" + +namespace ppc::crypto +{ +class Ed25519EccCrypto : public EccCrypto +{ +public: + using Ptr = std::shared_ptr; + Ed25519EccCrypto() = default; + ~Ed25519EccCrypto() override = default; + + uint32_t pointSizeInBytes() const override; + // get the curve + ppc::protocol::ECCCurve getCurve() const override { return ppc::protocol::ECCCurve::ED25519; } + + bcos::bytes generateRandomScalar() const override; + // convert the hashData into ec-point over the given curve + bcos::bytes hashToCurve(bcos::bytes&& _hashData) const override; + bcos::bytes hashToCurve(bcos::bytes const& _hashData) const override; + // get the multiplicative inverse of _data over the given curve + bcos::bytes scalarInvert(bcos::bytes const& _data) const override; + // multiply the _ecPoint by _scalar + bcos::bytes ecMultiply(bcos::bytes const& _ecPoint, bcos::bytes const& _scalar) const override; + bcos::bytes ecAdd(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const override; + bcos::bytes ecSub(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const override; + bool isValidEcPoint(bcos::bytes const& _ecPoint) const override; + bcos::bytes mulGenerator(bcos::bytes const& _scalar) const override; + + bcos::bytes hashToScalar(bcos::bytes const& _hashData) const override; + bcos::bytes scalarAdd(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + bcos::bytes scalarSub(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + bcos::bytes scalarMul(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + + +private: + bcos::bytes convertHashToCurve(bcos::bytes const& _hashData) const; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h b/cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h new file mode 100644 index 00000000..5b4579a5 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IppECDHCryptoImpl.h + * @author: yujiechen + * @date 2023-1-3 + */ +#ifdef ENABLE_CRYPTO_MB +#pragma once +#include "../Common.h" +#include "ppc-framework/crypto/ECDHCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include +#include + +namespace ppc::crypto +{ +class IppECDHCryptoImpl : public ECDHCrypto +{ +public: + IppECDHCryptoImpl(bcos::bytes const& _privateKey, Hash::Ptr const& _hashImpl) + : m_privateKeyData(_privateKey), m_hashImpl(_hashImpl) + { + std::fill(m_privateKey.begin(), m_privateKey.end(), + static_cast(&m_privateKeyData[0])); + } + ~IppECDHCryptoImpl() override = default; + + // calculate the ecdh public-key according to privateKey and input + std::vector batchGetPublicKey(ppc::io::DataBatch::Ptr const& _input) override + { + // calculate hash for the input + std::vector hashResult; + hashResult.resize(_input->size()); + tbb::parallel_for(tbb::blocked_range(0U, _input->size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto const& data = _input->get(i); + hashResult[i] = m_hashImpl->hash(bcos::bytesConstRef(data.data(), data.size())); + } + }); + return batchGetSharedPublicKeyImpl(hashResult); + } + + // calculate the ecdh shared-publicKey according to the publicKey and privateKey + std::vector batchGetSharedPublicKey( + std::vector const& _publicKey) override + { + return batchGetSharedPublicKeyImpl(_publicKey); + } + +protected: + virtual std::vector batchGetSharedPublicKeyImpl( + std::vector const& _inputPoints) + { + std::vector result; + result.resize(_inputPoints.size()); + uint64_t dataBatchSize = + (_inputPoints.size() + CRYPTO_MB_BATCH_SIZE - 1) / CRYPTO_MB_BATCH_SIZE; + tbb::parallel_for(tbb::blocked_range(0U, dataBatchSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + std::array pk; + std::array sharedKey; + auto startOffset = i * CRYPTO_MB_BATCH_SIZE; + uint64_t k = 0; + int8u junkBuffer[CRYPTO_MB_BATCH_SIZE][X25519_ELEMENT_SIZE]; // Junk buffer + for (uint64_t j = startOffset; j < startOffset + 8; j++) + { + if (j < result.size()) + { + result[j].resize(X25519_ELEMENT_SIZE); + pk[k] = static_cast(_inputPoints[j].data()); + sharedKey[k] = static_cast(result[j].data()); + } + else + { + pk[k] = static_cast(_inputPoints[startOffset].data()); + sharedKey[k] = (int8u*)junkBuffer[k]; + } + k++; + } + // call mbx_x25519_mb8 + auto status = mbx_x25519_mb8(sharedKey.data(), m_privateKey.data(), pk.data()); + if (status != 0) + { + BOOST_THROW_EXCEPTION( + X25519GetSharedKeyError() << bcos::errinfo_comment( + "mbx_x25519_mb8 error, status: " + std::to_string(status))); + } + } + }); + return result; + } + +private: + // enforce the life-time of the privateKey + bcos::bytes m_privateKeyData; + // store the address of the privateKey + std::array m_privateKey; + Hash::Ptr m_hashImpl; + + // 32 bytes + constexpr static int X25519_ELEMENT_SIZE = 32; + // 8 elements per crypto_mb calls + constexpr static int CRYPTO_MB_BATCH_SIZE = 8; +}; +} // namespace ppc::crypto +#endif \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp b/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp new file mode 100644 index 00000000..b640f6d7 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp @@ -0,0 +1,309 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLEccCrypto.cpp + * @author: yujiechen + * @date 2022-12-5 + */ +#include "OpenSSLEccCrypto.h" +#include "openssl/obj_mac.h" +#include "ppc-framework/libwrapper/BigNum.h" +using namespace ppc::crypto; +using namespace ppc::protocol; +using namespace bcos; + +OpenSSLEccCrypto::OpenSSLEccCrypto(Hash::Ptr _hashImpl, ppc::protocol::ECCCurve const& _curve) + : m_hashImpl(std::move(_hashImpl)), m_curve(_curve) +{ + switch (_curve) + { + case ECCCurve::SM2: + { + m_group = EcGroup(SM2_POINT_BYTES_LEN, NID_sm2); + m_keySize = SM2_KEY_SIZE; + break; + } + case ECCCurve::SECP256K1: + { + m_group = EcGroup(SECP256K1_POINT_BYTES_LEN, NID_secp256k1); + m_keySize = SECP256K1_KEY_SIZE; + break; + } + case ECCCurve::P256: + { + m_group = EcGroup(P256_POINT_BYTES_LEN, NID_X9_62_prime256v1); + m_keySize = P256_KEY_SIZE; + break; + } + default: + { + BOOST_THROW_EXCEPTION(UnsupportedCurveType() << errinfo_comment( + "Not supported curve type " + std::to_string((int)_curve))); + } + } +} + +// generate a random group-element +bcos::bytes OpenSSLEccCrypto::generateRandomScalar() const +{ + bcos::bytes result; + generateRandomScalarImpl().toBytes(result); + return result; +} + + +// convert the hashData into ec-point over the given curve +bcos::bytes OpenSSLEccCrypto::hashToCurve(bcos::bytes&& _hashData) const +{ + bcos::bytes result; + hashToCurveImpl(_hashData).toBytes(result); + return result; +} + +bcos::bytes OpenSSLEccCrypto::hashToCurve(bcos::bytes const& _hashData) const +{ + bcos::bytes result; + hashToCurveImpl(_hashData).toBytes(result); + return result; +} + +// get the multiplicative inverse of _data over the given curve +bcos::bytes OpenSSLEccCrypto::scalarInvert(bcos::bytes const& _data) const +{ + try + { + bcos::bytes result; + invertImpl(_data).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(ScalarInvertException() + << errinfo_comment("OpenSSLEccCrypto scalarInvert error: " + + std::string(boost::diagnostic_information(e)))); + } +} + + +bcos::bytes OpenSSLEccCrypto::scalarAdd( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + try + { + bcos::bytes result; + result.resize(_scalarA.size()); + addImpl(_scalarA, _scalarB).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(ScalarCalculateException() + << errinfo_comment("OpenSSLEccCrypto scalarAdd error: " + + std::string(boost::diagnostic_information(e)))); + } +} + +bcos::bytes OpenSSLEccCrypto::scalarSub( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + try + { + bcos::bytes result; + result.resize(_scalarA.size()); + subImpl(_scalarA, _scalarB).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(ScalarCalculateException() + << errinfo_comment("OpenSSLEccCrypto scalarSub error: " + + std::string(boost::diagnostic_information(e)))); + } +} + + +bcos::bytes OpenSSLEccCrypto::scalarMul( + bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const +{ + try + { + bcos::bytes result; + result.resize(_scalarA.size()); + mulImpl(_scalarA, _scalarB).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(ScalarCalculateException() + << errinfo_comment("OpenSSLEccCrypto scalscalarMularSub error: " + + std::string(boost::diagnostic_information(e)))); + } +} + +bcos::bytes OpenSSLEccCrypto::hashToScalar(const bcos::bytes& _hashData) const +{ + try + { + BigNum x; + auto hashResult = m_hashImpl->hash(ref(_hashData)); + x.fromBytesModP(ref(hashResult), m_group.p()); + bcos::bytes result; + result.resize(hashResult.size()); + x.toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(HashToScalarException() + << errinfo_comment("OpenSSLEccCrypto hashToScalar error: " + + std::string(boost::diagnostic_information(e)))); + } +} + + +EcPoint OpenSSLEccCrypto::hashToCurveImpl(bcos::bytes const& _hashData) const +{ + try + { + auto bnContext = createBNContext(); + BigNum x; + // Note: After actual measurement, sm2 will always retry once, so for the sm2 elliptic + // curve, will hash the hashData again + if (m_curve == ECCCurve::SM2) + { + auto hashResult = m_hashImpl->hash(ref(_hashData)); + x.fromBytesModP(ref(hashResult), m_group.p()); + } + else + { + x.fromBytesModP(ref(_hashData), m_group.p()); + } + int retryCount = 0; + EcPoint point(m_group); + while (true) + { + if (EC_POINT_set_compressed_coordinates(m_group.ecGroup().get(), point.point().get(), + x.bn().get(), 0, bnContext.get()) == 1) + { + break; + } + // re-hash the x + bcos::bytes xBytes; + x.toBytes(xBytes); + auto hashResult = m_hashImpl->hash(ref(xBytes)); + x.fromBytesModP(ref(hashResult), m_group.p()); + retryCount++; + } + return point; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(HashToCurveException() + << errinfo_comment("OpenSSLEccCrypto hashToCurve error: " + + std::string(boost::diagnostic_information(e)))); + } +} + +// multiply the _ecPoint by _scalar +bcos::bytes OpenSSLEccCrypto::ecMultiply( + bcos::bytes const& _ecPoint, bcos::bytes const& _scalar) const +{ + try + { + // convert bytes to EcPoint + EcPoint point(m_group, ref(_ecPoint)); + // convert bytes to BigNumber + BigNum sk(ref(_scalar)); + bcos::bytes result; + point.ecMultiply(sk).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + EcMultipleException() << errinfo_comment( + "ecMultiply error:" + std::string(boost::diagnostic_information(e)))); + } +} + +bool OpenSSLEccCrypto::isValidEcPoint(bcos::bytes const& _ecPoint) const +{ + // try to convert to EcPoint + try + { + EcPoint point(m_group, ref(_ecPoint)); + return true; + } + catch (std::exception const& e) + { + CRYPTO_LOG(WARNING) << LOG_DESC("isValidEcPoint: convert bytes to ecPoint error") + << LOG_KV("exception", boost::diagnostic_information(e)); + return false; + } +} + +bcos::bytes OpenSSLEccCrypto::mulGenerator(bcos::bytes const& _scalar) const +{ + try + { + BigNum sk(ref(_scalar)); + bcos::bytes result; + basePointMultiply(m_group, sk).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + MulGeneratorError() << errinfo_comment("OpenSSLEccCrypto mulGenerator error: " + + std::string(boost::diagnostic_information(e)))); + } +} + +bcos::bytes OpenSSLEccCrypto::ecAdd( + bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const +{ + try + { + EcPoint pointA(m_group, ref(_ecPointA)); + EcPoint pointB(m_group, ref(_ecPointB)); + bcos::bytes result; + pointA.add(pointB).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + EcAddError() << errinfo_comment( + "OpenSSLEccCrypto ecAdd error: " + std::string(boost::diagnostic_information(e)))); + } +} +bcos::bytes OpenSSLEccCrypto::ecSub( + bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const +{ + try + { + EcPoint pointA(m_group, ref(_ecPointA)); + EcPoint pointB(m_group, ref(_ecPointB)); + bcos::bytes result; + pointA.sub(pointB).toBytes(result); + return result; + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + EcSubError() << errinfo_comment( + "OpenSSLEccCrypto ecSub error: " + std::string(boost::diagnostic_information(e)))); + } +} \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h b/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h new file mode 100644 index 00000000..c20f1077 --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLEccCrypto.h + * @author: yujiechen + * @date 2022-12-5 + */ +#pragma once +#include "core/EcPoint.h" +#include "openssl/rand.h" +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/libwrapper/BigNum.h" +namespace ppc::crypto +{ +class OpenSSLEccCrypto : public EccCrypto +{ +public: + using Ptr = std::shared_ptr; + OpenSSLEccCrypto(Hash::Ptr _hashImpl, ppc::protocol::ECCCurve const& _curve); + ~OpenSSLEccCrypto() override = default; + + uint32_t pointSizeInBytes() const override { return m_group.pointBytesLen(); } + // get the curve + ppc::protocol::ECCCurve getCurve() const override { return m_curve; } + + // generate a random group-element + bcos::bytes generateRandomScalar() const override; + // convert the hashData into ec-point over the given curve + bcos::bytes hashToCurve(bcos::bytes&& _hashData) const override; + bcos::bytes hashToCurve(bcos::bytes const& _hashData) const override; + + // get the multiplicative inverse of _data over the given curve + bcos::bytes scalarInvert(bcos::bytes const& _data) const override; + // multiply the _ecPoint by _scalar + bcos::bytes ecMultiply(bcos::bytes const& _ecPoint, bcos::bytes const& _scalar) const override; + bool isValidEcPoint(bcos::bytes const& _ecPoint) const override; + bcos::bytes mulGenerator(bcos::bytes const& _scalar) const override; + + bcos::bytes ecAdd(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const override; + bcos::bytes ecSub(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const override; + + bcos::bytes hashToScalar(bcos::bytes const& _hashData) const override; + bcos::bytes scalarAdd(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + bcos::bytes scalarSub(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + bcos::bytes scalarMul(bcos::bytes const& _scalarA, bcos::bytes const& _scalarB) const override; + + EcGroup const& group() const { return m_group; } + EcPoint hashToCurveImpl(bcos::bytes const& _hashData) const; + BigNum generateRandomScalarImpl() const + { + bcos::bytes result; + result.resize(m_keySize); + if (RAND_bytes(result.data(), m_keySize) != 1) + { + BOOST_THROW_EXCEPTION(GenerateRandomScalarError()); + } + return BigNum(bcos::bytesConstRef(result.data(), result.size()), m_group.n(), false); + } + + // get the multiplicative inverse of _data over the given curve + BigNum invertImpl(bcos::bytes const& _data) const + { + BigNum value(bcos::bytesConstRef(_data.data(), _data.size())); + return value.Invert(m_group.n()); + } + + BigNum addImpl(bcos::bytes const& _data1, bcos::bytes const& _data2) const + { + BigNum value1(bcos::bytesConstRef(_data1.data(), _data1.size())); + BigNum value2(bcos::bytesConstRef(_data2.data(), _data2.size())); + return value1.add(value2, m_group.n()); + } + + BigNum subImpl(bcos::bytes const& _data1, bcos::bytes const& _data2) const + { + BigNum value1(bcos::bytesConstRef(_data1.data(), _data1.size())); + BigNum value2(bcos::bytesConstRef(_data2.data(), _data2.size())); + return value1.sub(value2, m_group.n()); + } + + BigNum mulImpl(bcos::bytes const& _data1, bcos::bytes const& _data2) const + { + BigNum value1(bcos::bytesConstRef(_data1.data(), _data1.size())); + BigNum value2(bcos::bytesConstRef(_data2.data(), _data2.size())); + return value1.mul(value2, m_group.n()); + } + +private: + Hash::Ptr m_hashImpl; + ppc::protocol::ECCCurve m_curve; + int m_keySize; + EcGroup m_group; + + constexpr static int SM2_KEY_SIZE = 32; + constexpr static int SECP256K1_KEY_SIZE = 32; + constexpr static int P256_KEY_SIZE = 32; + + constexpr static int SM2_POINT_BYTES_LEN = 33; + constexpr static int SECP256K1_POINT_BYTES_LEN = 33; + constexpr static int P256_POINT_BYTES_LEN = 33; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/ecc/core/EcPoint.h b/cpp/ppc-crypto/src/ecc/core/EcPoint.h new file mode 100644 index 00000000..2085f83d --- /dev/null +++ b/cpp/ppc-crypto/src/ecc/core/EcPoint.h @@ -0,0 +1,191 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcPoint.h + * @author: yujiechen + * @date 2022-12-5 + */ +#pragma once +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "../../Common.h" +#include "openssl/ec.h" +#include "openssl/obj_mac.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include +#include + +namespace ppc::crypto +{ +struct EcGroupDeleter +{ +public: + void operator()(EC_GROUP* _group) { EC_GROUP_free(_group); } +}; +using EcGroupPtr = std::shared_ptr; + +///// The ec-group +class EcGroup +{ +public: + EcGroup() = default; + EcGroup(int _pointBytesLen, EC_GROUP* _group) + : m_pointBytesLen(_pointBytesLen), m_ecGroup(EcGroupPtr(_group, EcGroupDeleter())) + { + auto bnContext = createBNContext(); + // obtain the curve params + if (EC_GROUP_get_curve(m_ecGroup.get(), m_p.bn().get(), m_a.bn().get(), m_b.bn().get(), + bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ECGroupGetCurveError()); + } + // get the order + if (EC_GROUP_get_order(m_ecGroup.get(), m_n.bn().get(), bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ECGroupGetOrderError()); + } + } + EcGroup(int _pointBytesLen, int _nid) + : EcGroup(_pointBytesLen, EC_GROUP_new_by_curve_name(_nid)) + {} + BigNum const& p() const { return m_p; } + + BigNum const& a() const { return m_a; } + BigNum const& b() const { return m_b; } + BigNum const& n() const { return m_n; } + + EcGroupPtr const& ecGroup() const { return m_ecGroup; } + int pointBytesLen() const { return m_pointBytesLen; } + +private: + int m_pointBytesLen; + BigNum m_p; + BigNum m_a; + BigNum m_b; + + BigNum m_n; + EcGroupPtr m_ecGroup; +}; + +///// The ec-point +struct ECPointDeleter +{ +public: + void operator()(EC_POINT* point) { EC_POINT_clear_free(point); } +}; +using ECPointPtr = std::shared_ptr; + +class EcPoint +{ +public: + EcPoint(EcGroup _group) + : m_group(std::move(_group)), + m_point(ECPointPtr(EC_POINT_new(m_group.ecGroup().get()), ECPointDeleter())) + {} + EcPoint(EcGroup _group, bcos::bytesConstRef const& _data) : EcPoint(std::move(_group)) + { + fromBytes(_data); + } + + ~EcPoint() = default; + + // convert bytes to the point + void fromBytes(bcos::bytesConstRef const& _data) + { + auto bnContext = createBNContext(); + BigNum pointNum; + pointNum.fromBigEndianBytes((bcos::byte*)(_data.data()), _data.size(), false); + auto bn = EC_POINT_bn2point( + m_group.ecGroup().get(), pointNum.bn().get(), m_point.get(), bnContext.get()); + if (!bn) + { + BOOST_THROW_EXCEPTION(ECPointBn2PointError()); + } + } + + // convert the point to bytes + void toBytes(bcos::bytes& _data) + { + auto bnContext = createBNContext(); + unsigned char* buf = NULL; + auto bufferLen = EC_POINT_point2buf(m_group.ecGroup().get(), m_point.get(), + POINT_CONVERSION_COMPRESSED, &buf, bnContext.get()); + if (bufferLen == 0) + { + BOOST_THROW_EXCEPTION(ECPoint2BNError()); + } + _data.resize(bufferLen); + memcpy(_data.data(), buf, bufferLen); + OPENSSL_free(buf); + } + + // ecMultiply + EcPoint ecMultiply(BigNum const& _sk) const + { + auto bnContext = createBNContext(); + EcPoint result(m_group); + if (EC_POINT_mul(m_group.ecGroup().get(), result.point().get(), NULL, m_point.get(), + _sk.bn().get(), bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ECPointMulError()); + } + return result; + } + + EcPoint add(EcPoint const& _point) const + { + auto bnContext = createBNContext(); + EcPoint result(m_group); + if (EC_POINT_add(m_group.ecGroup().get(), result.point().get(), m_point.get(), + _point.point().get(), bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ECPointAddError()); + } + return result; + } + + EcPoint sub(EcPoint& _point) const + { + auto bnContext = createBNContext(); + // invert the point + if (EC_POINT_invert(m_group.ecGroup().get(), _point.point().get(), bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION( + ECPointSubError() << bcos::errinfo_comment("EcPoint sub error for invert failed")); + } + return add(_point); + } + + EcGroup const& group() const { return m_group; } + ECPointPtr const& point() const { return m_point; } + +private: + EcGroup m_group; + ECPointPtr m_point; +}; + +// basePointMultiply +inline EcPoint basePointMultiply(EcGroup const& _group, BigNum const& _sk) +{ + auto bnContext = createBNContext(); + EcPoint result(_group); + if (EC_POINT_mul(_group.ecGroup().get(), result.point().get(), _sk.bn().get(), NULL, NULL, + bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ECPointMulError()); + } + return result; +} + +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/oprf/EcdhOprf.cpp b/cpp/ppc-crypto/src/oprf/EcdhOprf.cpp new file mode 100644 index 00000000..dd791aae --- /dev/null +++ b/cpp/ppc-crypto/src/oprf/EcdhOprf.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhOprf.cpp + * @author: shawnhe + * @date 2022-11-3 + */ + +#include "EcdhOprf.h" +#include "../Common.h" +#include + +using namespace ppc::protocol; +using namespace ppc::crypto; + +// H1(_input)^r +bcos::bytes ppc::crypto::EcdhOprfClient::blind(std::string_view _input) const +{ + bcos::bytesConstRef inputData((const unsigned char*)_input.data(), _input.size()); + auto hashResult = m_hash->hash(inputData); + auto ecPoint = m_eccCrypto->hashToCurve(hashResult); + + return m_eccCrypto->ecMultiply(ecPoint, m_privateKey); +} + +bcos::bytes ppc::crypto::EcdhOprfClient::blind(const bcos::bytes& _input) const +{ + bcos::bytesConstRef inputData(_input.data(), _input.size()); + auto hashResult = m_hash->hash(inputData); + auto ecPoint = m_eccCrypto->hashToCurve(hashResult); + + return m_eccCrypto->ecMultiply(ecPoint, m_privateKey); +} + +std::vector ppc::crypto::EcdhOprfClient::blind( + const std::vector& _inputs) const +{ + auto inputsSize = _inputs.size(); + + std::vector results; + results.reserve(inputsSize); + results.resize(inputsSize); + + tbb::parallel_for(tbb::blocked_range(0U, inputsSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + results[i] = blind(_inputs[i]); + } + }); + return results; +} + +// H2(_input,_evaluatedItem^(1/r)) +bcos::bytes ppc::crypto::EcdhOprfClient::finalize( + std::string_view _input, const bcos::bytes& _evaluatedItem) const +{ + auto invertR = m_eccCrypto->scalarInvert(m_privateKey); + auto unblindedItem = m_eccCrypto->ecMultiply(_evaluatedItem, invertR); + + auto hashState = m_hash->init(); + bcos::bytesConstRef inputData((const unsigned char*)_input.data(), _input.size()); + m_hash->update(hashState, inputData); + m_hash->update(hashState, + bcos::bytesConstRef((const unsigned char*)unblindedItem.data(), unblindedItem.size())); + + auto finalH = m_hash->final(hashState); + finalH.resize(m_outputSize); + return finalH; +} + +std::vector ppc::crypto::EcdhOprfClient::finalize( + const std::vector& _inputs, const std::vector& _evaluatedItems) const +{ + auto inputsSize = _inputs.size(); + if (inputsSize != _evaluatedItems.size()) + { + BOOST_THROW_EXCEPTION(OprfFinalizeException() << bcos::errinfo_comment( + "The number of input does not match the evaluated element")); + } + + std::vector results; + results.reserve(inputsSize); + results.resize(inputsSize); + + tbb::parallel_for(tbb::blocked_range(0U, inputsSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + results[i] = finalize(_inputs[i], _evaluatedItems[i]); + } + }); + + return results; +} + +// (_blindedItem)^sk +bcos::bytes EcdhOprfServer::evaluate(const bcos::bytes& _blindedItem) const +{ + return m_eccCrypto->ecMultiply(_blindedItem, m_privateKey); +} + +std::vector ppc::crypto::EcdhOprfServer::evaluate( + const std::vector& _blindedItems) const +{ + auto inputsSize = _blindedItems.size(); + + std::vector results; + results.reserve(inputsSize); + results.resize(inputsSize); + + tbb::parallel_for(tbb::blocked_range(0U, inputsSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + results[i] = evaluate(_blindedItems[i]); + } + }); + + return results; +} + +// OPRF(_input, sk) = H2(x, H1(_input)^sk) +bcos::bytes ppc::crypto::EcdhOprfServer::fullEvaluate(const bcos::bytes& _input) const +{ + bcos::bytesConstRef inputData(_input.data(), _input.size()); + auto hashResult = m_hash->hash(inputData); + auto ecPoint = m_eccCrypto->hashToCurve(hashResult); + auto blindedItem = m_eccCrypto->ecMultiply(ecPoint, m_privateKey); + + auto hashState = m_hash->init(); + m_hash->update(hashState, inputData); + m_hash->update(hashState, + bcos::bytesConstRef((const unsigned char*)blindedItem.data(), blindedItem.size())); + + auto finalH = m_hash->final(hashState); + finalH.resize(m_outputSize); + return finalH; +} + +bcos::bytes ppc::crypto::EcdhOprfServer::fullEvaluate(const std::string_view _input) const +{ + bcos::bytesConstRef inputData((const unsigned char*)_input.data(), _input.size()); + auto hashResult = m_hash->hash(inputData); + auto ecPoint = m_eccCrypto->hashToCurve(hashResult); + auto blindedItem = m_eccCrypto->ecMultiply(ecPoint, m_privateKey); + + auto hashState = m_hash->init(); + m_hash->update(hashState, inputData); + m_hash->update(hashState, + bcos::bytesConstRef((const unsigned char*)blindedItem.data(), blindedItem.size())); + + auto finalH = m_hash->final(hashState); + finalH.resize(m_outputSize); + return finalH; +} + +std::vector ppc::crypto::EcdhOprfServer::fullEvaluate( + const std::vector& _inputs) const +{ + auto inputsSize = _inputs.size(); + + std::vector results; + results.reserve(inputsSize); + results.resize(inputsSize); + + tbb::parallel_for(tbb::blocked_range(0U, inputsSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + results[i] = fullEvaluate(_inputs[i]); + } + }); + return results; +} diff --git a/cpp/ppc-crypto/src/oprf/EcdhOprf.h b/cpp/ppc-crypto/src/oprf/EcdhOprf.h new file mode 100644 index 00000000..44dedbd0 --- /dev/null +++ b/cpp/ppc-crypto/src/oprf/EcdhOprf.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhOprf.h + * @author: shawnhe + * @date 2022-11-3 + */ + +#pragma once +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/crypto/Oprf.h" + +namespace ppc::crypto +{ +/** + * + * reference: JKK14 + * title: Round-optimal password-protected secret sharing and T-PAKE in the password-only model + * url: https://eprint.iacr.org/2014/650.pdf + * + * OPRF(x, sk) = H2(x, H1(x)^sk) + * + * Client: y, r Server: x, sk + * ------------------------------------------------------------------------------------- + * blindedElement = H1(y)^r + * + * blindedElement + * ----------> + * + * evaluatedElement = H1(y)^r)^sk + * + * evaluatedElement + * <---------- + * + * output = H2(y,(H1(y)^r)^sk^(1/r))=H2(y,H1(y)^sk) + */ + +class EcdhOprfClient : public OprfClient +{ +public: + using Ptr = std::shared_ptr; + EcdhOprfClient() = delete; + EcdhOprfClient(uint16_t _outputSize, Hash::Ptr _hash, EccCrypto::Ptr _eccCrypto) + : OprfClient(ppc::protocol::OprfType::EcdhOprf, _outputSize), + m_hash(std::move(_hash)), + m_eccCrypto(std::move(_eccCrypto)) + { + m_type = protocol::OprfType::EcdhOprf; + + // init private key + m_privateKey = m_eccCrypto->generateRandomScalar(); + } + + virtual ~EcdhOprfClient() = default; + + // blind input by private key + bcos::bytes blind(const bcos::bytes& _input) const override; + bcos::bytes blind(std::string_view _input) const override; + std::vector blind(const std::vector& _inputs) const override; + + // unblind evaluated item, and hash with input + bcos::bytes finalize(std::string_view _input, const bcos::bytes& _evaluatedItem) const override; + std::vector finalize(const std::vector& _inputs, + const std::vector& _evaluatedItems) const override; + +private: + Hash::Ptr m_hash; + EccCrypto::Ptr m_eccCrypto; +}; + +class EcdhOprfServer : public OprfServer +{ +public: + using Ptr = std::shared_ptr; + EcdhOprfServer() = delete; + EcdhOprfServer(uint16_t _outputSize, Hash::Ptr _hash, EccCrypto::Ptr _eccCrypto) + : OprfServer(ppc::protocol::OprfType::EcdhOprf, _outputSize), + m_hash(std::move(_hash)), + m_eccCrypto(std::move(_eccCrypto)) + { + m_type = protocol::OprfType::EcdhOprf; + + // init private key + m_privateKey = m_eccCrypto->generateRandomScalar(); + } + + virtual ~EcdhOprfServer() = default; + + // evaluate blinded item by private key + bcos::bytes evaluate(const bcos::bytes& _blindedItem) const override; + std::vector evaluate(const std::vector& _blindedItems) const override; + + // compute the OPRF output with server's private key + bcos::bytes fullEvaluate(const bcos::bytes& _input) const override; + bcos::bytes fullEvaluate(const std::string_view _input) const override; + std::vector fullEvaluate(const std::vector& _inputs) const override; + +private: + Hash::Ptr m_hash; + EccCrypto::Ptr m_eccCrypto; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp b/cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp new file mode 100644 index 00000000..66d9513d --- /dev/null +++ b/cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018Oprf.cpp + * @author: yujiechen + * @date 2022-12-6 + */ +#include "RA2018Oprf.h" + +using namespace ppc::crypto; +using namespace ppc::io; +using namespace bcos; + +void RA2018Oprf::blind(ppc::io::DataBatch::Ptr const& _plainData, bcos::bytes const& _privateKey, + std::vector& _blindResult) +{ + // H1(yj)^Bj + _blindResult.resize(_plainData->size()); + tbb::parallel_for(tbb::blocked_range(0U, _plainData->size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + blindImpl(_plainData->getBytes(i), _privateKey, _blindResult[i]); + } + }); +} + + +void RA2018Oprf::finalize(std::vector const& _evaluatedData, + bcos::bytes const& _invPrivateKey, std::vector& _finalizedResult) +{ + _finalizedResult.resize(_evaluatedData.size()); + // (aj)^(1/Bj) + tbb::parallel_for( + tbb::blocked_range(0U, _evaluatedData.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + finalizeImpl(_invPrivateKey, _evaluatedData.at(i), _finalizedResult[i]); + } + }); +} + +void RA2018Oprf::fullEvaluate( + ppc::io::DataBatch::Ptr const& _input, std::vector& _result) +{ + _result.resize(_input->size()); + // H1(xi)^aj + tbb::parallel_for(tbb::blocked_range(0U, _input->size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + fullEvaluateImpl(_input->get(i), _result[i]); + } + }); +} + +void RA2018Oprf::evaluate( + std::vector const& _blindData, std::vector& _result) +{ + _result.resize(_blindData.size()); + // (aj)^a + tbb::parallel_for(tbb::blocked_range(0U, _blindData.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + evaluateImpl(_blindData.at(i), _result[i]); + } + }); +} diff --git a/cpp/ppc-crypto/src/oprf/RA2018Oprf.h b/cpp/ppc-crypto/src/oprf/RA2018Oprf.h new file mode 100644 index 00000000..6b70ecda --- /dev/null +++ b/cpp/ppc-crypto/src/oprf/RA2018Oprf.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018Oprf.h + * @author: yujiechen + * @date 2022-12-6 + */ +#pragma once +#include "../ecc/OpenSSLEccCrypto.h" +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/crypto/RA2018OprfInterface.h" +#include "ppc-framework/io/DataBatch.h" +#include +#include + +namespace ppc::crypto +{ +class RA2018Oprf : public RA2018OprfInterface +{ +public: + using Ptr = std::shared_ptr; + RA2018Oprf(bcos::bytes _privateKey, EccCrypto::Ptr _eccCrypto, Hash::Ptr _hashImpl) + : m_privateKey(std::move(_privateKey)), m_eccCrypto(_eccCrypto), m_hashImpl(_hashImpl) + {} + ~RA2018Oprf() override = default; + + bcos::bytes generatePrivateKey() override { return m_eccCrypto->generateRandomScalar(); } + bcos::bytes inv(bcos::bytes const& _data) override { return m_eccCrypto->scalarInvert(_data); } + + void blind(ppc::io::DataBatch::Ptr const& _plainData, bcos::bytes const& _privateKey, + std::vector& _blindResult) override; + + void finalize(std::vector const& _evaluatedData, bcos::bytes const& _invPrivateKey, + std::vector& _finalizedResult) override; + + void fullEvaluate( + ppc::io::DataBatch::Ptr const& _input, std::vector& _result) override; + void evaluate( + std::vector const& _blindData, std::vector& _result) override; + + +protected: + virtual void blindImpl( + bcos::bytes const& _input, bcos::bytes const& _privateKey, bcos::bytes& _blindResult) + { + auto hashResult = m_hashImpl->hash(bcos::bytesConstRef(_input.data(), _input.size())); + auto ecPoint = m_eccCrypto->hashToCurve(std::move(hashResult)); + _blindResult = m_eccCrypto->ecMultiply(ecPoint, _privateKey); + } + + virtual void finalizeImpl(bcos::bytes const& _invPrivateKey, bcos::bytes const& _evaluatedData, + bcos::bytes& _finalizedResult) + { + // recover the plainData + _finalizedResult = m_eccCrypto->ecMultiply(_evaluatedData, _invPrivateKey); + } + virtual void fullEvaluateImpl(bcos::bytes const& _input, bcos::bytes& _result) + { + auto hashResult = m_hashImpl->hash( + bcos::bytesConstRef((const unsigned char*)_input.data(), _input.size())); + auto ecPoint = m_eccCrypto->hashToCurve(std::move(hashResult)); + _result = m_eccCrypto->ecMultiply(ecPoint, m_privateKey); + } + + virtual void evaluateImpl(bcos::bytes const& _blindData, bcos::bytes& _result) + { + _result = m_eccCrypto->ecMultiply(_blindData, m_privateKey); + } + +protected: + bcos::bytes m_privateKey; + EccCrypto::Ptr m_eccCrypto; + Hash::Ptr m_hashImpl; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-crypto/src/prng/AESPRNG.cpp b/cpp/ppc-crypto/src/prng/AESPRNG.cpp new file mode 100644 index 00000000..145ca91f --- /dev/null +++ b/cpp/ppc-crypto/src/prng/AESPRNG.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AESPRNG.cpp + * @author: shawnhe + * @date 2022-11-29 + */ + +#include "AESPRNG.h" +#include + +namespace ppc::crypto +{ +// generate `_bytesLen` bytes random numbers +void AESPRNG::generate(bcos::byte* _dest, uint64_t _bytesLen) +{ + uint64_t bytesIdx = m_totalOutputs % BUFFER_CAPACITY; + + uint64_t step = std::min(_bytesLen, BUFFER_CAPACITY - bytesIdx); + memcpy(_dest, m_buffer.data() + bytesIdx, step); + _dest += step; + m_totalOutputs += step; + + _bytesLen -= step; + uint64_t round = _bytesLen / BUFFER_CAPACITY; + uint64_t remain = _bytesLen % BUFFER_CAPACITY; + + if (round) + { + parallelGen(_dest, m_totalOutputs / BUFFER_CAPACITY, round); + _dest += round * BUFFER_CAPACITY; + m_totalOutputs += round * BUFFER_CAPACITY; + } + + uint64_t currentIndex = m_totalOutputs / BUFFER_CAPACITY; + bcos::bytes indexData( + (bcos::byte*)(¤tIndex), (bcos::byte*)(¤tIndex) + sizeof(uint64_t)); + indexData.resize(BUFFER_CAPACITY); + m_buffer = m_aes->encrypt( + SymCrypto::OperationMode::CBC, bcos::ref(m_key), bcos::ref(m_iv), bcos::ref(indexData)); + + if (remain) + { + memcpy(_dest, m_buffer.data(), remain); + m_totalOutputs += remain; + } +} + +void AESPRNG::parallelGen(bcos::byte* _dest, uint64_t _currentRound, uint64_t _remainRound) +{ + tbb::parallel_for(tbb::blocked_range(0U, _remainRound), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + uint64_t currentIndex = _currentRound + i; + bcos::bytes indexData(BUFFER_CAPACITY); + memcpy(indexData.data(), (bcos::byte*)(¤tIndex), sizeof(uint64_t)); + auto temp = m_aes->encrypt(SymCrypto::OperationMode::CBC, bcos::ref(m_key), + bcos::ref(m_iv), bcos::ref(indexData)); + memcpy(_dest + i * BUFFER_CAPACITY, temp.data(), BUFFER_CAPACITY); + } + }); +} + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/src/prng/AESPRNG.h b/cpp/ppc-crypto/src/prng/AESPRNG.h new file mode 100644 index 00000000..c69b960a --- /dev/null +++ b/cpp/ppc-crypto/src/prng/AESPRNG.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AESPRNG.h + * @author: shawnhe + * @date 2022-11-29 + */ + +#pragma once +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-framework/crypto/PRNG.h" + +namespace ppc::crypto +{ +class AESPRNG : public PRNG +{ +public: + using Ptr = std::shared_ptr; + static constexpr uint64_t BUFFER_CAPACITY = 1024; + + // default constructor is not allowed + AESPRNG() = delete; + virtual ~AESPRNG() = default; + + AESPRNG(const bcos::bytes& _seed) : PRNG(_seed), m_key(_seed), m_iv(bcos::byte()) + { + m_aes = std::make_shared(OpenSSLAES::AESType::AES128); + bcos::bytes index(BUFFER_CAPACITY); + m_buffer = m_aes->encrypt( + SymCrypto::OperationMode::CBC, bcos::ref(m_key), bcos::ref(m_iv), bcos::ref(index)); + } + + ppc::protocol::PRNGImplName type() const override { return ppc::protocol::PRNGImplName::AES; } + + // generate `_bytesLen` bytes random numbers, saved in _dest + void generate(bcos::byte* _dest, uint64_t _bytesLen) override; + +private: + void parallelGen(bcos::byte* _dest, uint64_t _currentRound, uint64_t _remainRound); + + OpenSSLAES::Ptr m_aes; + bcos::bytes m_key; + bcos::bytes m_iv; + + // record current buffer + bcos::bytes m_buffer; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp b/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp new file mode 100644 index 00000000..a8907360 --- /dev/null +++ b/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BLAKE2bPRNG.cpp + * @author: shawnhe + * @date 2022-12-4 + */ + +#include "BLAKE2bPRNG.h" +#include + +namespace ppc::crypto +{ + +// generate `_bytesLen` bytes random numbers +void BLAKE2bPRNG::generate(bcos::byte* _dest, uint64_t _bytesLen) +{ + uint64_t bytesIdx = m_totalOutputs % BUFFER_CAPACITY; + + uint64_t step = std::min(_bytesLen, BUFFER_CAPACITY - bytesIdx); + memcpy(_dest, m_buffer.data() + bytesIdx, step); + _dest += step; + m_totalOutputs += step; + + _bytesLen -= step; + uint64_t round = _bytesLen / BUFFER_CAPACITY; + uint64_t remain = _bytesLen % BUFFER_CAPACITY; + + if (round) + { + parallelGen(_dest, m_totalOutputs / BUFFER_CAPACITY, round); + _dest += round * BUFFER_CAPACITY; + m_totalOutputs += round * BUFFER_CAPACITY; + } + + uint64_t currentIndex = m_totalOutputs / BUFFER_CAPACITY; + bcos::bytes indexData( + (bcos::byte*)(¤tIndex), (bcos::byte*)(¤tIndex) + sizeof(uint64_t)); + m_buffer = m_blake2Hash->hash(bcos::ref(indexData), bcos::ref(m_seeds), BUFFER_CAPACITY); + + if (remain) + { + memcpy(_dest, m_buffer.data(), remain); + m_totalOutputs += remain; + } +} + +void BLAKE2bPRNG::parallelGen(bcos::byte* _dest, uint64_t _currentRound, uint64_t _remainRound) +{ + tbb::parallel_for(tbb::blocked_range(0U, _remainRound), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + uint64_t currentIndex = _currentRound + i; + bcos::bytes indexData( + (bcos::byte*)(¤tIndex), (bcos::byte*)(¤tIndex) + sizeof(uint64_t)); + auto temp = m_blake2Hash->hash(bcos::ref(indexData), bcos::ref(m_seeds), BUFFER_CAPACITY); + memcpy(_dest + i * BUFFER_CAPACITY, temp.data(), BUFFER_CAPACITY); + } + }); +} + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h b/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h new file mode 100644 index 00000000..62904e0f --- /dev/null +++ b/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BLAKE2bPRNG.h + * @author: shawnhe + * @date 2022-12-4 + */ + +#pragma once +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-framework/crypto/PRNG.h" + +namespace ppc::crypto +{ +class BLAKE2bPRNG : public PRNG +{ +public: + using Ptr = std::shared_ptr; + static constexpr uint32_t BUFFER_CAPACITY = 64; + + // default constructor is not allowed + BLAKE2bPRNG() = delete; + virtual ~BLAKE2bPRNG() = default; + + BLAKE2bPRNG(const bcos::bytes& _seed) : PRNG(_seed) + { + m_blake2Hash = std::make_shared(); + bcos::bytes index(BUFFER_CAPACITY); + m_buffer = m_blake2Hash->hash(bcos::ref(index), bcos::ref(m_seeds), BUFFER_CAPACITY); + } + + ppc::protocol::PRNGImplName type() const override + { + return ppc::protocol::PRNGImplName::BLAKE2b; + } + + // generate `_bytesLen` bytes random numbers, saved in _dest + void generate(bcos::byte* _dest, uint64_t _bytesLen) override; + +private: + void parallelGen(bcos::byte* _dest, uint64_t _currentRound, uint64_t _remainRound); + + BLAKE2bHash::Ptr m_blake2Hash; + + // record current buffer + bcos::bytes m_buffer; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/src/randomot/SimplestOT.cpp b/cpp/ppc-crypto/src/randomot/SimplestOT.cpp new file mode 100644 index 00000000..ed330a8d --- /dev/null +++ b/cpp/ppc-crypto/src/randomot/SimplestOT.cpp @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2021 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SimplestOT.cpp + * @author: shawnhe + * @date 2022-07-06 + */ + +#include "SimplestOT.h" +#include + +using namespace ppc::crypto; +using namespace ppc::protocol; + +std::pair> SimplestOT::receiverGeneratePointsB( + BitVector::Ptr _choices, bcos::bytesPointer _pointA) +{ + auto otNumber = _choices->size(); + bcos::bytesPointer bytesB = std::make_shared(otNumber * m_ecc->pointSizeInBytes()); + std::vector scalars(otNumber); + + for (uint32_t i = 0; i < otNumber; ++i) + { + scalars[i] = m_ecc->generateRandomScalar(); + std::array pairB; + pairB[0] = m_ecc->mulGenerator(scalars[i]); + pairB[1] = m_ecc->ecAdd(*_pointA, pairB[0]); + memcpy(bytesB->data() + i * m_ecc->pointSizeInBytes(), pairB[_choices->get(i)].data(), + m_ecc->pointSizeInBytes()); + } + + return {bytesB, scalars}; +} + +std::vector SimplestOT::finishReceiver( + bcos::bytesPointer _pointA, std::vector _bScalars) +{ + auto otNumber = _bScalars.size(); + std::vector keys(otNumber); + + for (uint32_t i = 0; i < otNumber; ++i) + { + auto point = m_ecc->ecMultiply(*_pointA, _bScalars[i]); + auto state = m_hash->init(); + m_hash->update(state, bcos::ref(point)); + bcos::bytes indexData((bcos::byte*)(&i), (bcos::byte*)(&i) + sizeof(uint32_t)); + m_hash->update(state, bcos::ref(indexData)); + keys[i] = m_hash->final(state); + } + return keys; +} + +std::pair SimplestOT::senderGeneratePointA() +{ + auto a = m_ecc->generateRandomScalar(); + auto A = m_ecc->mulGenerator(a); + return {a, std::make_shared(A)}; +} + +std::vector> SimplestOT::finishSender( + bcos::bytes _aScalar, bcos::bytesPointer _pointA, bcos::bytesPointer _pointsB) +{ + uint32_t otNumber = _pointsB->size() / m_ecc->pointSizeInBytes(); + std::vector> keys(otNumber); + + auto A = m_ecc->ecMultiply(*_pointA, _aScalar); + + for (uint32_t i = 0; i < otNumber; ++i) + { + bcos::bytes indexData((bcos::byte*)(&i), (bcos::byte*)(&i) + sizeof(uint32_t)); + + bcos::bytes B(_pointsB->begin() + i * m_ecc->pointSizeInBytes(), + _pointsB->begin() + (i + 1) * m_ecc->pointSizeInBytes()); + auto point0 = m_ecc->ecMultiply(B, _aScalar); + auto state0 = m_hash->init(); + m_hash->update(state0, bcos::ref(point0)); + m_hash->update(state0, bcos::ref(indexData)); + keys[i][0] = m_hash->final(state0); + + auto point1 = m_ecc->ecSub(point0, A); + auto state1 = m_hash->init(); + m_hash->update(state1, bcos::ref(point1)); + m_hash->update(state1, bcos::ref(indexData)); + keys[i][1] = m_hash->final(state1); + } + + return keys; +} diff --git a/cpp/ppc-crypto/src/randomot/SimplestOT.h b/cpp/ppc-crypto/src/randomot/SimplestOT.h new file mode 100644 index 00000000..425ca330 --- /dev/null +++ b/cpp/ppc-crypto/src/randomot/SimplestOT.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SimplestOT.h + * @author: shawnhe + * @date 2022-12-5 + */ + +#pragma once + +#include "ppc-crypto-core/src/tools/BitVector.h" +#include "ppc-framework/Common.h" +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/protocol/Protocol.h" +#include +#include + +namespace ppc::crypto +{ +class SimplestOT : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + SimplestOT() = delete; + ~SimplestOT() = default; + + SimplestOT(EccCrypto::Ptr _ecc, Hash::Ptr _hash) + : m_ecc(std::move(_ecc)), m_hash(std::move(_hash)) + {} + + // receive point A and evaluate multi points B, return Bs bytes and multi scalar bs bytes + std::pair> receiverGeneratePointsB( + BitVector::Ptr _choices, bcos::bytesPointer _pointA); + + // return keys + std::vector finishReceiver( + bcos::bytesPointer _pointA, std::vector _bScalars); + + // generate a random point A + std::pair senderGeneratePointA(); + + // receive multi points B and generate keys + std::vector> finishSender( + bcos::bytes _aScalar, bcos::bytesPointer _pointA, bcos::bytesPointer _pointsB); + +private: + EccCrypto::Ptr m_ecc; + Hash::Ptr m_hash; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-crypto/tests/CMakeLists.txt b/cpp/ppc-crypto/tests/CMakeLists.txt new file mode 100644 index 00000000..cb482575 --- /dev/null +++ b/cpp/ppc-crypto/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-crypto) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-crypto WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-crypto/tests/TestEccCrypto.cpp b/cpp/ppc-crypto/tests/TestEccCrypto.cpp new file mode 100644 index 00000000..7b5c561d --- /dev/null +++ b/cpp/ppc-crypto/tests/TestEccCrypto.cpp @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestEccCrypto.cpp + * @author: yujiechen + * @date 2022-11-2 + */ +#include "ppc-crypto-core/src/hash/SM3Hash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto-core/src/hash/Sha512Hash.h" +#include "ppc-crypto/src/Common.h" +#include "ppc-crypto/src/ecc/ECDHCryptoImpl.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/IppECDHCryptoImpl.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-framework/io/DataBatch.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::protocol; +using namespace bcos; +using namespace ppc::io; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(eccCryptoTest, TestPromptFixture) + +void testOpenSSLEccCrypto(OpenSSLEccCrypto::Ptr _eccCryptoImpl, Hash::Ptr _hashImpl) +{ + std::string inputStr = "wer234lkejskdf"; + bcos::bytes inputData(inputStr.begin(), inputStr.end()); + auto hashResult = _hashImpl->hash(ref(inputData)); + + auto ecPointBytes = _eccCryptoImpl->hashToCurve(hashResult); + + EcPoint point(_eccCryptoImpl->group(), ref(ecPointBytes)); + bcos::bytes result; + point.toBytes(result); + BOOST_CHECK(ecPointBytes == result); + + EcPoint decodedPoint(_eccCryptoImpl->group(), ref(result)); + bcos::bytes result2; + decodedPoint.toBytes(result2); + BOOST_CHECK(result == result2); +} + +void testEccCrypto( + EccCrypto::Ptr _eccCryptoImpl, Hash::Ptr _hashImpl, bool _ed25519, bool _testIppCrypto) +{ + std::string inputStr = "wer234lkejskdf"; + bcos::bytes inputData(inputStr.begin(), inputStr.end()); + auto hashResult = _hashImpl->hash(ref(inputData)); + + // test hashToCurve + auto randomScalar = _eccCryptoImpl->generateRandomScalar(); + auto invRandomScalar = _eccCryptoImpl->scalarInvert(randomScalar); + // check the scalarInvert value + for (int i = 0; i < 1000; i++) + { + BOOST_CHECK(invRandomScalar == _eccCryptoImpl->scalarInvert(randomScalar)); + } + + // scalarInvert the invRandomScalar + std::cout << "#### randomScalar: " << *toHexString(randomScalar) << std::endl; + std::cout << "#### invRandomScalar : " << *toHexString(invRandomScalar) << std::endl; + std::cout << "#### invinvRandomScalar : " + << *toHexString(_eccCryptoImpl->scalarInvert(invRandomScalar)) << std::endl; + + auto ecPoint = _eccCryptoImpl->hashToCurve(hashResult); + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(ecPoint)); + // check (ecPoint^randomScalar)^invRandomScalar == ecPoint + auto multiPoint = _eccCryptoImpl->ecMultiply(ecPoint, randomScalar); + for (int i = 0; i < 1000; i++) + { + BOOST_CHECK(_eccCryptoImpl->ecMultiply(ecPoint, randomScalar) == multiPoint); + } + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(multiPoint)); + auto invPoint = _eccCryptoImpl->ecMultiply(multiPoint, invRandomScalar); + for (int i = 0; i < 1000; i++) + { + BOOST_CHECK(_eccCryptoImpl->ecMultiply(multiPoint, invRandomScalar) == invPoint); + } + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(invPoint)); + std::cout << "#### invPoint: " << *toHexString(invPoint) << std::endl; + std::cout << "#### eccPoint : " << *toHexString(ecPoint) << std::endl; + BOOST_CHECK(invPoint == ecPoint); + // check the point size + BOOST_CHECK(invPoint.size() == _eccCryptoImpl->pointSizeInBytes()); + BOOST_CHECK(ecPoint.size() == _eccCryptoImpl->pointSizeInBytes()); + + // test ecPointAdd and Sub + auto addResult = _eccCryptoImpl->ecAdd(invPoint, ecPoint); + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(addResult)); + auto subResult = _eccCryptoImpl->ecSub(addResult, invPoint); + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(subResult)); + BOOST_CHECK(subResult == ecPoint); + + + bytes invalidPoint; + BOOST_CHECK(_eccCryptoImpl->isValidEcPoint(invalidPoint) == false); + + // invalid point multipy + ecPoint.resize(16); + BOOST_CHECK_THROW(_eccCryptoImpl->ecMultiply(ecPoint, randomScalar), EcMultipleException); + // mul G + std::string scalar = "abbc1415ad291c4c2d3a6f26a2a6c7dc0da2690418e55e9fe73342b9fed69a03"; + std::string mulGPoint = "5efd75e128f35f24158920bfa597a720883d876698c52d9a02d916e46599fc11"; + if (_ed25519) + { + BOOST_CHECK( + _eccCryptoImpl->mulGenerator(*fromHexString(scalar)) == *fromHexString(mulGPoint)); + } + + // batch for mul G + for (int i = 0; i < 1000; i++) + { + BOOST_CHECK(_eccCryptoImpl->mulGenerator(randomScalar) == + _eccCryptoImpl->mulGenerator(randomScalar)); + } + // test ECDHCryptoImpl + auto cryptoBox = std::make_shared(_hashImpl, _eccCryptoImpl); + ECDHCrypto::Ptr serverEcdhCrypto = nullptr; + ECDHCrypto::Ptr clientEcdhCrypto = nullptr; + auto serverPrivateKey = _eccCryptoImpl->generateRandomScalar(); + auto clientPrivateKey = _eccCryptoImpl->generateRandomScalar(); + if (!_testIppCrypto) + { + serverEcdhCrypto = std::make_shared(serverPrivateKey, cryptoBox); + clientEcdhCrypto = std::make_shared(clientPrivateKey, cryptoBox); + } + else + { +#ifdef ENABLE_CRYPTO_MB + if (ppc::CPU_FEATURES.avx512ifma) + { + std::cout << "#### test IppECDHCryptoImpl" << std::endl; + serverEcdhCrypto = std::make_shared(serverPrivateKey, _hashImpl); + clientEcdhCrypto = std::make_shared(clientPrivateKey, _hashImpl); + } + else + { + std::cout << "#### return directly without test IppECDHCryptoImpl for not support " + "avx512ifma instruction" + << std::endl; + return; + } +#else + return; +#endif + } + // batchGetPublicKey + auto dataBatch = std::make_shared(); + dataBatch->setDataSchema(DataSchema::Bytes); + std::vector inputDatas; + for (int i = 0; i < 100; i++) + { + std::string item = std::to_string(i); + inputDatas.emplace_back(bcos::bytes(item.begin(), item.end())); + } + dataBatch->setData(std::move(inputDatas)); + + auto publicKeys = serverEcdhCrypto->batchGetPublicKey(dataBatch); + auto sharedPublicKeys1 = clientEcdhCrypto->batchGetSharedPublicKey(publicKeys); + + publicKeys = clientEcdhCrypto->batchGetPublicKey(dataBatch); + auto sharedPublicKeys2 = serverEcdhCrypto->batchGetSharedPublicKey(publicKeys); + + // check the result + BOOST_CHECK(sharedPublicKeys1.size() == sharedPublicKeys2.size()); + for (uint64_t i = 0; i < sharedPublicKeys1.size(); i++) + { + BOOST_CHECK(sharedPublicKeys1.at(i) == sharedPublicKeys2.at(i)); + } +} + +BOOST_AUTO_TEST_CASE(testEd25519EccCrypto) +{ + Hash::Ptr hashImpl = std::make_shared(); + testEccCrypto(std::make_shared(), hashImpl, true, false); + testEccCrypto(std::make_shared(), hashImpl, true, true); + + hashImpl = std::make_shared(); + testEccCrypto(std::make_shared(), hashImpl, true, false); + testEccCrypto(std::make_shared(), hashImpl, true, true); + + hashImpl = std::make_shared(); + testEccCrypto(std::make_shared(), hashImpl, true, false); + testEccCrypto(std::make_shared(), hashImpl, true, true); +} +BOOST_AUTO_TEST_CASE(testSM2EccCrypto) +{ + Hash::Ptr hashImpl = std::make_shared(); + OpenSSLEccCrypto::Ptr eccCrypto = std::make_shared(hashImpl, ECCCurve::SM2); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); +} + +BOOST_AUTO_TEST_CASE(testSecp256K1EccCrypto) +{ + Hash::Ptr hashImpl = std::make_shared(); + OpenSSLEccCrypto::Ptr eccCrypto = + std::make_shared(hashImpl, ECCCurve::SECP256K1); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); +} + +BOOST_AUTO_TEST_CASE(testP256EccCrypto) +{ + Hash::Ptr hashImpl = std::make_shared(); + OpenSSLEccCrypto::Ptr eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); + + hashImpl = std::make_shared(); + testOpenSSLEccCrypto(eccCrypto, hashImpl); + testEccCrypto(eccCrypto, hashImpl, false, false); + testEccCrypto(eccCrypto, hashImpl, false, true); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto/tests/TestOprf.cpp b/cpp/ppc-crypto/tests/TestOprf.cpp new file mode 100644 index 00000000..08dbf180 --- /dev/null +++ b/cpp/ppc-crypto/tests/TestOprf.cpp @@ -0,0 +1,110 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestOprf.cpp + * @author: shawnhe + * @date 2022-11-3 + */ + +#include "ppc-crypto-core/src/hash/HashFactoryImpl.h" +#include "ppc-crypto/src/ecc/EccCryptoFactoryImpl.h" +#include +#include +#include +#include + +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(EcdhOprfTest, TestPromptFixture) + +void runEcdhOprf(EccCrypto::Ptr _clientEcc, EccCrypto::Ptr _serverEcc, Hash::Ptr _clientHash, + Hash::Ptr _serverHash) +{ + uint16_t outputSize = 32; + auto client = std::make_shared(outputSize, _clientHash, _clientEcc); + auto server = std::make_shared(outputSize, _serverHash, _serverEcc); + + // test one item + std::string inputStr = "wer23dfddd4lkejskdf"; + bcos::bytes inputData(inputStr.begin(), inputStr.end()); + + auto blindedItem = client->blind(inputStr); + auto evaluatedItem = server->evaluate(blindedItem); + auto outputC = client->finalize(inputStr, evaluatedItem); + auto outputS = server->fullEvaluate(inputStr); + + std::cout << "#### oprf client output: " << *toHexString(outputC) << std::endl; + std::cout << "#### oprf server output: " << *toHexString(outputS) << std::endl; + BOOST_CHECK(outputC == outputS); + + // test batch + std::vector inputs; + for (uint64_t i = 0; i < 1024; i++) + { + inputs.emplace_back(inputStr + std::to_string(i)); + } + + auto blindedItems = client->blind(inputs); + auto evaluatedItems = server->evaluate(blindedItems); + auto outputCs = client->finalize(inputs, evaluatedItems); + auto outputSs = server->fullEvaluate(inputs); + + for (uint64_t i = 0; i < 1024; i++) + { + BOOST_CHECK(outputCs[i] == outputSs[i]); + } +} + + +BOOST_AUTO_TEST_CASE(testEcdhOprf) +{ + auto eccFactory = std::make_shared(); + auto hashFactory = std::make_shared(); + + // test sm + auto sm3Hash = hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SM3); + auto sm2Ecc0 = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SM2, sm3Hash); + auto sm2Ecc1 = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SM2, sm3Hash); + runEcdhOprf(sm2Ecc0, sm2Ecc1, sm3Hash, sm3Hash); + + // test others + std::vector hashSet = { + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA256), + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512), + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::BLAKE2b)}; + + for (auto& hashImpl : hashSet) + { + std::vector eccSet0 = { + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::ED25519, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SECP256K1, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hashImpl)}; + std::vector eccSet1 = { + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::ED25519, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SECP256K1, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hashImpl)}; + for (uint64_t i = 0; i < eccSet0.size(); i++) + { + runEcdhOprf(eccSet0[i], eccSet1[i], hashImpl, hashImpl); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto/tests/TestPRNG.cpp b/cpp/ppc-crypto/tests/TestPRNG.cpp new file mode 100644 index 00000000..f4ce8a5c --- /dev/null +++ b/cpp/ppc-crypto/tests/TestPRNG.cpp @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestPRNG.cpp + * @author: shawnhe + * @date 2022-11-29 + */ +#include "ppc-crypto-core/src/tools/BitVector.h" +#include "ppc-crypto/src/prng/AESPRNG.h" +#include "ppc-crypto/src/prng/BLAKE2bPRNG.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(PRNGTest, TestPromptFixture) + +void runPRNG(PRNG::Ptr _prng1, PRNG::Ptr _prng2) +{ + auto output11 = _prng1->generate(4511); + auto output12 = _prng1->generate(3275122); + auto output13 = _prng1->generate(25480233); + + auto output21 = _prng2->generate(4511); + auto output22 = _prng2->generate(3275122); + auto output23 = _prng2->generate(25480233); + + BOOST_CHECK(output11 == output21 && output11.size() == 4511); + BOOST_CHECK(output12 == output22 && output12.size() == 3275122); + BOOST_CHECK(output13 == output23 && output13.size() == 25480233); +} + +void runPRNGFunc(PRNG::Ptr _prng1, PRNG::Ptr _prng2) +{ + auto output11 = _prng1->generate(1234); + auto output12 = _prng1->generate(23456); + auto output13 = _prng1->generate(345678); + + auto output2 = _prng2->generate(370368); + + BOOST_CHECK(output11 + output12 + output13 == output2 && output2.size() == 370368); + + int intOut1 = _prng1->generate(); + int intOut2 = _prng2->generate(); + BOOST_CHECK(intOut1 == intOut2); + + bool boolOut1 = _prng1->generate(); + bool boolOut2 = _prng2->generate(); + BOOST_CHECK(boolOut1 == boolOut2); + + std::vector dest1(1024); + std::vector dest2(1024); + _prng1->generate(dest1.data(), 1024); + _prng2->generate(dest2.data(), 1024); + BOOST_CHECK(dest1 == dest2); +} + + +BOOST_AUTO_TEST_CASE(testAESPRNG) +{ + bcos::bytes seed1{'a', 'b', 'c'}; + auto prng1 = std::make_shared(seed1); + auto prng2 = std::make_shared(seed1); + runPRNG(prng1, prng2); + + bcos::bytes seed2{'c', 'd', '3'}; + prng1 = std::make_shared(seed2); + prng2 = std::make_shared(seed2); + runPRNGFunc(prng1, prng2); +} + +BOOST_AUTO_TEST_CASE(testBLAKE2bPRNG) +{ + bcos::bytes seed1{'a', 'b', 'c'}; + auto prng1 = std::make_shared(seed1); + auto prng2 = std::make_shared(seed1); + runPRNG(prng1, prng2); + + bcos::bytes seed2{'c', 'd', '3'}; + prng1 = std::make_shared(seed2); + prng2 = std::make_shared(seed2); + runPRNGFunc(prng1, prng2); + + BitVector bv0("10011011101"); + BitVector bv1(10, 12345); + bcos::bytes seed{'a', 'b', 'c'}; + auto prng0 = std::make_shared(seed); + auto prng4 = std::make_shared(seed); + bv0.randomize(prng0, 16); + bv1.randomize(prng4, 16); + BOOST_CHECK(bv1.equals(bv0)); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto/tests/TestRandomot.cpp b/cpp/ppc-crypto/tests/TestRandomot.cpp new file mode 100644 index 00000000..cea3a056 --- /dev/null +++ b/cpp/ppc-crypto/tests/TestRandomot.cpp @@ -0,0 +1,102 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestRandomot.cpp + * @author: shawnhe + * @date 2022-12-6 + */ + + +#include "ppc-crypto-core/src/hash/HashFactoryImpl.h" +#include "ppc-crypto-core/src/tools/BitVector.h" +#include "ppc-crypto/src/ecc/EccCryptoFactoryImpl.h" +#include "ppc-crypto/src/prng/AESPRNG.h" +#include "ppc-crypto/src/randomot/SimplestOT.h" +#include +#include + +using namespace ppc::crypto; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(RandomotTest, TestPromptFixture) + +void runRandomOT(EccCrypto::Ptr _clientEcc, EccCrypto::Ptr _serverEcc, Hash::Ptr _clientHash, + Hash::Ptr _serverHash) +{ + auto simplestOt0 = std::make_shared(_clientEcc, _clientHash); + auto simplestOt1 = std::make_shared(_serverEcc, _serverHash); + + auto chooses = std::make_shared(); + bcos::bytes seed{'b', '1', 'c', '0'}; + auto prng = std::make_shared(seed); + + uint32_t otNumber = 1024; + chooses->randomize(prng, otNumber / 8); + + std::string ch = chooses->toString(); + + std::pair pairA = simplestOt0->senderGeneratePointA(); + std::pair> parsBs = + simplestOt1->receiverGeneratePointsB(chooses, pairA.second); + std::vector receiverKeys = + simplestOt0->finishReceiver(pairA.second, parsBs.second); + std::vector> senderKeys = + simplestOt1->finishSender(pairA.first, pairA.second, parsBs.first); + + for (uint32_t i = 0; i < otNumber; ++i) + { + BOOST_CHECK(receiverKeys[i] == senderKeys[i][chooses->get(i)]); + } +} + +BOOST_AUTO_TEST_CASE(testRandomot) +{ + auto eccFactory = std::make_shared(); + auto hashFactory = std::make_shared(); + + // test sm + auto sm3Hash = hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SM3); + auto sm2Ecc0 = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SM2, sm3Hash); + auto sm2Ecc1 = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SM2, sm3Hash); + runRandomOT(sm2Ecc0, sm2Ecc1, sm3Hash, sm3Hash); + + // test others + std::vector hashSet = { + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA256), + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512), + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::BLAKE2b)}; + + for (auto& hashImpl : hashSet) + { + std::vector eccSet0 = { + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::ED25519, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SECP256K1, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hashImpl)}; + std::vector eccSet1 = { + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::ED25519, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::SECP256K1, hashImpl), + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hashImpl)}; + for (uint64_t i = 0; i < eccSet0.size(); i++) + { + runRandomOT(eccSet0[i], eccSet1[i], hashImpl, hashImpl); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-crypto/tests/main.cpp b/cpp/ppc-crypto/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-crypto/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-framework/Common.h b/cpp/ppc-framework/Common.h new file mode 100644 index 00000000..ea80b504 --- /dev/null +++ b/cpp/ppc-framework/Common.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-20 + */ +#pragma once +#include +#if ENABLE_CPU_FEATURES +#if X86 +#include +#endif +#if ARCH +#include +#endif +#endif + +#if defined(WIN32) || defined(WIN64) || defined(_WIN32) || defined(_WIN32_) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include + +namespace ppc +{ +/// construct a new exception class overriding Exception +#define DERIVE_PPC_EXCEPTION(X) \ + struct X : virtual bcos::Exception \ + { \ + } + +DERIVE_PPC_EXCEPTION(OpenFileFailed); +DERIVE_PPC_EXCEPTION(DataSchemaNotSetted); +DERIVE_PPC_EXCEPTION(UnsupportedDataSchema); + +constexpr static int MAX_PORT = 65535; +constexpr static int DEFAULT_SECURITY_PARAM = 128; + +#if ENABLE_CPU_FEATURES +#if X86 +static const cpu_features::X86Features CPU_FEATURES = cpu_features::GetX86Info().features; +#endif +#if ARCH +static const cpu_features::Aarch64Features CPU_FEATURES = cpu_features::GetAarch64Info().features; +#endif +#endif + +using u128 = boost::multiprecision::number>; + +// for double support +using u1024 = boost::multiprecision::number>; + +using s1024 = boost::multiprecision::number>; + +using float50 = boost::multiprecision::cpp_dec_float_50; +} // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/CryptoBox.h b/cpp/ppc-framework/crypto/CryptoBox.h new file mode 100644 index 00000000..b57c18d1 --- /dev/null +++ b/cpp/ppc-framework/crypto/CryptoBox.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CryptoBox.h + * @author: yujiechen + * @date 2022-11-7 + */ +#pragma once +#include "EccCrypto.h" +#include "Hash.h" +namespace ppc::crypto +{ +class CryptoBox +{ +public: + using Ptr = std::shared_ptr; + CryptoBox() = delete; + CryptoBox(Hash::Ptr const& _hashImpl, EccCrypto::Ptr const& _eccCrypto) + : m_hashImpl(_hashImpl), m_eccCrypto(_eccCrypto) + {} + + Hash::Ptr const& hashImpl() const { return m_hashImpl; } + EccCrypto::Ptr const& eccCrypto() const { return m_eccCrypto; } + +private: + Hash::Ptr m_hashImpl; + EccCrypto::Ptr m_eccCrypto; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/ECDHCrypto.h b/cpp/ppc-framework/crypto/ECDHCrypto.h new file mode 100644 index 00000000..ac634836 --- /dev/null +++ b/cpp/ppc-framework/crypto/ECDHCrypto.h @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ECDHCrypto.h + * @author: yujiechen + * @date 2022-12-28 + */ +#pragma once +#include "../io/DataBatch.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include +namespace ppc::crypto +{ +class ECDHCrypto +{ +public: + using Ptr = std::shared_ptr; + ECDHCrypto() = default; + virtual ~ECDHCrypto() = default; + + // calculate the ecdh public-key according to privateKey and input + virtual std::vector batchGetPublicKey(ppc::io::DataBatch::Ptr const& _input) = 0; + // calculate the ecdh shared-publicKey according to the publicKey and privateKey + virtual std::vector batchGetSharedPublicKey( + std::vector const& _publicKey) = 0; +}; + +class ECDHCryptoFactory +{ +public: + using Ptr = std::shared_ptr; + ECDHCryptoFactory() = default; + virtual ~ECDHCryptoFactory() = default; + + virtual ECDHCrypto::Ptr createECDHCrypto(int _curveType, int _hashType) const = 0; + + virtual CryptoBox::Ptr createCryptoBox(int8_t _curveType, int8_t _hashType) const = 0; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/EccCrypto.h b/cpp/ppc-framework/crypto/EccCrypto.h new file mode 100644 index 00000000..1e1942b4 --- /dev/null +++ b/cpp/ppc-framework/crypto/EccCrypto.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EccCrypto.h + * @author: yujiechen + * @date 2022-11-1 + */ +#pragma once +#include "../protocol/Protocol.h" +#include "Hash.h" +#include +#include + +namespace ppc::crypto +{ +class EccCrypto +{ +public: + using Ptr = std::shared_ptr; + EccCrypto() = default; + virtual ~EccCrypto() {} + + virtual uint32_t pointSizeInBytes() const = 0; + // get the curve + virtual ppc::protocol::ECCCurve getCurve() const = 0; + // generate a random group-element + virtual bcos::bytes generateRandomScalar() const = 0; + // convert the hashData into ec-point over the given curve + virtual bcos::bytes hashToCurve(bcos::bytes&& _hashData) const = 0; + virtual bcos::bytes hashToCurve(bcos::bytes const& _hashData) const = 0; + virtual bcos::bytes hashToScalar(bcos::bytes const& _hashData) const = 0; + + // get the multiplicative inverse of _data over the given curve + virtual bcos::bytes scalarInvert(bcos::bytes const& _data) const = 0; + virtual bcos::bytes scalarAdd(bcos::bytes const& _data1, bcos::bytes const& _data2) const = 0; + virtual bcos::bytes scalarSub(bcos::bytes const& _data1, bcos::bytes const& _data2) const = 0; + virtual bcos::bytes scalarMul(bcos::bytes const& _data1, bcos::bytes const& _data2) const = 0; + // multiply the _ecPoint by _scalar + virtual bcos::bytes ecMultiply( + bcos::bytes const& _ecPoint, bcos::bytes const& _scalar) const = 0; + virtual bcos::bytes ecAdd(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const = 0; + virtual bcos::bytes ecSub(bcos::bytes const& _ecPointA, bcos::bytes const& _ecPointB) const = 0; + virtual bool isValidEcPoint(bcos::bytes const& _ecPoint) const = 0; + virtual bcos::bytes mulGenerator(bcos::bytes const& _scalar) const = 0; +}; + +class EccCryptoFactory +{ +public: + using Ptr = std::shared_ptr; + EccCryptoFactory() = default; + virtual ~EccCryptoFactory() = default; + + virtual EccCrypto::Ptr createEccCrypto(int8_t _curveType, Hash::Ptr const& _hashImpl) const = 0; +}; +} // namespace ppc::crypto diff --git a/cpp/ppc-framework/crypto/Hash.h b/cpp/ppc-framework/crypto/Hash.h new file mode 100644 index 00000000..66509cf0 --- /dev/null +++ b/cpp/ppc-framework/crypto/Hash.h @@ -0,0 +1,66 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Hash.h + * @author: yujiechen + * @date 2022-11-2 + */ +#pragma once +#include "../protocol/Protocol.h" +#include +#include +namespace ppc::crypto +{ +class HashState +{ +public: + using Ptr = std::shared_ptr; + HashState() = default; + virtual ~HashState() {} + virtual void* state() = 0; +}; + +class Hash +{ +public: + using Ptr = std::shared_ptr; + Hash() = default; + virtual ~Hash() = default; + + // the hashBytes length of given hash-algorithm + virtual size_t hashLen() const = 0; + // the implementation of the hash-algorithm + virtual ppc::protocol::HashImplName type() const = 0; + + // calculate-hash + virtual bcos::bytes hash(bcos::bytesConstRef _input) const = 0; + // init the hash-state + virtual HashState::Ptr init() const = 0; + // update new message into the given hash-state + virtual void update(HashState::Ptr _state, bcos::bytesConstRef _data) const = 0; + // obtain the hash-result from the given hash-state + virtual bcos::bytes final(HashState::Ptr _state) const = 0; +}; + +class HashFactory +{ +public: + using Ptr = std::shared_ptr; + HashFactory() = default; + virtual ~HashFactory() = default; + + virtual Hash::Ptr createHashImpl(int8_t _hashType) const = 0; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/Ihc.h b/cpp/ppc-framework/crypto/Ihc.h new file mode 100644 index 00000000..4372d2c6 --- /dev/null +++ b/cpp/ppc-framework/crypto/Ihc.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Ihc.h + * @author: asherli + * @date 2023-11-23 + */ + +#pragma once +#include "../libwrapper/BigNum.h" +#include "../libwrapper/Buffer.h" +#include +#include + +namespace ppc::homo +{ +class Ihc +{ +public: + enum class IhcMode + { + IHC_128, + IHC_256, + }; + + using Ptr = std::shared_ptr; + Ihc() = default; + virtual ~Ihc() = default; + + // generate key according to given mode + virtual bcos::bytes generateKey() const = 0; + virtual void generateKey(OutputBuffer* _result) const = 0; + + // encrypt the given value to cipher using given key + virtual void encrypt( + OutputBuffer* _cipher, bcos::bytesConstRef const& _key, BIGNUM const* _value) const = 0; + virtual bcos::bytes encrypt(bcos::bytesConstRef const& _key, BIGNUM const* _value) const = 0; + // decrypt the given cipher to value using given key + virtual ppc::crypto::BigNum decrypt( + bcos::bytesConstRef const& _key, bcos::bytesConstRef const& _cipher) const = 0; + // ihc add + virtual void add(OutputBuffer* _addResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const = 0; + virtual bcos::bytes add( + bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const = 0; + // ihc sub + virtual void sub(OutputBuffer* _subResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const = 0; + virtual bcos::bytes sub( + bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const = 0; + + // ihc scalaMul + virtual void scalaMul(OutputBuffer* _mulResult, BIGNUM const* _value, + bcos::bytesConstRef const& _cipher) const = 0; + virtual bcos::bytes scalaMul( + BIGNUM const* _value, bcos::bytesConstRef const& _cipher) const = 0; + + virtual unsigned int keyBits() const = 0; + virtual unsigned int keyBytes() const = 0; + virtual uint64_t cipherBytes() const = 0; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/KeyPair.h b/cpp/ppc-framework/crypto/KeyPair.h new file mode 100644 index 00000000..384feca6 --- /dev/null +++ b/cpp/ppc-framework/crypto/KeyPair.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file KeyPair.h + * @author: yujiechen + * @date 2023-08-04 + */ +#pragma once +#include +#include + +namespace ppc::crypto +{ +class KeyPair +{ +public: + using Ptr = std::shared_ptr; + using UniquePtr = std::unique_ptr; + KeyPair() = default; + virtual ~KeyPair() = default; + + virtual void* sk() const = 0; + virtual void* pk() const = 0; + + // serialize the sk + virtual bcos::bytes serializeSK() const = 0; + // serialize the pk + virtual bcos::bytes serializePK() const = 0; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/Oprf.h b/cpp/ppc-framework/crypto/Oprf.h new file mode 100644 index 00000000..dab7832d --- /dev/null +++ b/cpp/ppc-framework/crypto/Oprf.h @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Oprf.h + * @author: shawnhe + * @date 2022-11-3 + */ + +#pragma once +#include "ppc-framework/protocol/Protocol.h" +#include +#include + +namespace ppc::crypto +{ +/** + * reference: draft-irtf-cfrg-voprf-14 + * title: Oblivious Pseudorandom Functions (OPRFs) using Prime-Order Groups + * url: https://datatracker.ietf.org/doc/draft-irtf-cfrg-voprf/ + * + * In the OPRF mode, a client and server interact to compute output = F(skS, input), where input is + * the client's private input, skS is the server's private key, and output is the OPRF output. After + * the execution of the protocol, the client learns output and the server learns nothing. This + * interaction is shown below. + * + * Client Server(sk) + * ------------------------------------------------------------------------------------- + * blindedElement = Blind(input) + * + * blindedElement + * ----------> + * + * evaluatedElement = BlindEvaluate(sk, blindedElement) + * + * evaluatedElement + * <---------- + * + * output = Finalize(input, blind, evaluatedElement) + */ +class Oprf +{ +public: + using Ptr = std::shared_ptr; + Oprf() = default; + Oprf(ppc::protocol::OprfType _type, uint16_t _outputSize) + : m_type(_type), m_outputSize(_outputSize) + {} + virtual ~Oprf() = default; + + ppc::protocol::OprfType type() { return m_type; } + uint16_t outputSize() { return m_outputSize; } + void setOutputSize(uint16_t _size) { m_outputSize = _size; } + bcos::bytes privateKey() { return m_privateKey; } + void setPrivateKey(bcos::bytes _key) { m_privateKey = std::move(_key); } + +protected: + ppc::protocol::OprfType m_type; + uint16_t m_outputSize; + bcos::bytes m_privateKey; +}; + +class OprfClient : public Oprf +{ +public: + using Ptr = std::shared_ptr; + OprfClient() = default; + OprfClient(ppc::protocol::OprfType _type, uint16_t _outputSize) : Oprf(_type, _outputSize) {} + virtual ~OprfClient() = default; + + // blind input by private key + virtual bcos::bytes blind(const bcos::bytes& _input) const = 0; + virtual bcos::bytes blind(std::string_view _input) const = 0; + virtual std::vector blind(const std::vector& _inputs) const = 0; + + // unblind evaluated item, and hash with input + virtual bcos::bytes finalize( + std::string_view _input, const bcos::bytes& _evaluatedItem) const = 0; + virtual std::vector finalize(const std::vector& _inputs, + const std::vector& _evaluatedItems) const = 0; +}; + + +class OprfServer : public Oprf +{ +public: + using Ptr = std::shared_ptr; + OprfServer() = default; + OprfServer(ppc::protocol::OprfType _type, uint16_t _outputSize) : Oprf(_type, _outputSize) {} + virtual ~OprfServer() = default; + + // evaluate blinded item by private key + virtual bcos::bytes evaluate(const bcos::bytes& _blindedItem) const = 0; + virtual std::vector evaluate( + const std::vector& _blindedItems) const = 0; + + // compute the OPRF output with server's private key + virtual bcos::bytes fullEvaluate(const bcos::bytes& _input) const = 0; + virtual bcos::bytes fullEvaluate(const std::string_view _input) const = 0; + virtual std::vector fullEvaluate( + const std::vector& _inputs) const = 0; +}; + +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/Ore.h b/cpp/ppc-framework/crypto/Ore.h new file mode 100644 index 00000000..1100c3be --- /dev/null +++ b/cpp/ppc-framework/crypto/Ore.h @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Ore.h + * @author: shawnhe + * @date 2023-08-18 + */ +#pragma once + +#include +#include + +namespace ppc::crypto +{ +class Ore +{ +public: + using Ptr = std::shared_ptr; + Ore() = default; + virtual ~Ore() = default; + + virtual bcos::bytes generateKey() const = 0; + virtual void generateKey(OutputBuffer* sk) const = 0; + virtual int keyBytes() const = 0; + + // cipher encoded with hex + virtual std::string encrypt4String( + bcos::bytesConstRef const& _sk, const std::string& _plaintext) const = 0; + virtual void encrypt4String(OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _plaintext) const = 0; + + // cipher encoded with hex + virtual std::string decrypt4String( + bcos::bytesConstRef const& _sk, const std::string& _ciphertext) const = 0; + virtual void decrypt4String(OutputBuffer* plain, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& cipher) const = 0; + + // cipher encoded with base64 + virtual std::string encrypt4Integer( + bcos::bytesConstRef const& _sk, const int64_t& _plain) const = 0; + virtual void encrypt4Integer( + OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, const int64_t& _plain) const = 0; + + // cipher encoded with base64 + virtual int64_t decrypt4Integer( + bcos::bytesConstRef const& _sk, const std::string& _cipher) const = 0; + virtual void decrypt4Integer(int64_t* _plain, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _cipher) const = 0; + + // cipher encoded with base64 + virtual std::string encrypt4Float( + bcos::bytesConstRef const& _sk, const float50& _plain) const = 0; + virtual void encrypt4Float( + OutputBuffer* _cipher, bcos::bytesConstRef const& _sk, const float50& _plain) const = 0; + + // cipher encoded with base64 + virtual float50 decrypt4Float( + bcos::bytesConstRef const& _sk, const std::string& _cipher) const = 0; + virtual float50 decrypt4Float( + bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const = 0; + + virtual int compare(const std::string& _ciphertext0, const std::string& _ciphertext1) const = 0; + virtual int compare(InputBuffer const* c1, InputBuffer const* c2) const = 0; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/PRNG.h b/cpp/ppc-framework/crypto/PRNG.h new file mode 100644 index 00000000..54c2e49a --- /dev/null +++ b/cpp/ppc-framework/crypto/PRNG.h @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PRNG.h + * @author: shawnhe + * @date 2022-11-29 + */ + +#pragma once +#include "../protocol/Protocol.h" +#include +#include + +namespace ppc::crypto +{ +class PRNG +{ +public: + using Ptr = std::shared_ptr; + + // default constructor is not allowed + PRNG() = delete; + virtual ~PRNG() = default; + + PRNG(const bcos::bytes& _seed) : m_seeds(_seed) {} + + virtual ppc::protocol::PRNGImplName type() const = 0; + + // generate `_bytesLen` bytes random numbers, saved in _dest + virtual void generate(bcos::byte* _dest, uint64_t _bytesLen) = 0; + bcos::bytes generate(uint64_t _bytesLen) + { + bcos::bytes res(_bytesLen); + generate(res.data(), _bytesLen); + return res; + } + + // generate one T type random + template + typename std::enable_if::value && std::is_trivial::value, T>::type + generate() + { + T ret; + generate((bcos::byte*)&ret, sizeof(T)); + return ret; + } + + // generate '_size' T type randoms + template + typename std::enable_if::value && std::is_trivial::value, + void>::type + generate(T* dest, uint64_t _size) + { + uint64_t totalLen = _size * sizeof(T); + generate((bcos::byte*)dest, totalLen); + } + + const bcos::bytes& seeds() const { return m_seeds; } + uint64_t totalOutputs() const { return m_totalOutputs; } + +protected: + bcos::bytes m_seeds; + + // the length of the random number that has been output + uint64_t m_totalOutputs{0}; +}; + +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/Paillier.h b/cpp/ppc-framework/crypto/Paillier.h new file mode 100644 index 00000000..d593e37b --- /dev/null +++ b/cpp/ppc-framework/crypto/Paillier.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Paillier.h + * @author: yujiechen + * @date 2023-08-04 + */ +#pragma once +#include "../libwrapper/BigNum.h" +#include "../libwrapper/Buffer.h" +#include "KeyPair.h" + +namespace ppc::homo +{ +class Paillier +{ +public: + using Ptr = std::shared_ptr; + Paillier() = default; + virtual ~Paillier() = default; + + // generate the keypair + virtual ppc::crypto::KeyPair::UniquePtr generateKeyPair(unsigned const _keyLength) const = 0; + // encrypt the _value + virtual bcos::bytes encrypt_with_crt(BIGNUM const* _value, void* _keyPair) const = 0; + // encrypt with crt optimization + virtual void encrypt_with_crt(OutputBuffer* _result, OutputBuffer* _rBuffer, + BIGNUM const* _value, void* _keyPair) const = 0; + // encrypt without crt optimization + virtual void encrypt( + OutputBuffer* _result, OutputBuffer* _rBuffer, BIGNUM const* _value, void* _pk) const = 0; + virtual bcos::bytes encrypt(BIGNUM const* _value, void* _pk) const = 0; + + // decrypt the cipher data + virtual ppc::crypto::BigNum decrypt( + bcos::bytesConstRef const& _cipherData, void* _keyPair) const = 0; + + // _cipher1 add _cipher2 + virtual bcos::bytes add(bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const = 0; + virtual void add(OutputBuffer* _result, bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const = 0; + + // _cipher1 sub _cipher2 + virtual bcos::bytes sub(bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const = 0; + virtual void sub(OutputBuffer* _result, bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const = 0; + + // v * _cipher + virtual bcos::bytes scalaMul( + BIGNUM const* v, bcos::bytesConstRef const& _cipher, void* _publicKey) const = 0; + virtual void scalaMul(OutputBuffer* _result, BIGNUM const* v, + bcos::bytesConstRef const& _cipher, void* _publicKey) const = 0; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/RA2018OprfInterface.h b/cpp/ppc-framework/crypto/RA2018OprfInterface.h new file mode 100644 index 00000000..49ceb853 --- /dev/null +++ b/cpp/ppc-framework/crypto/RA2018OprfInterface.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018OprfInterface.h + * @author: yujiechen + * @date 2022-12-6 + */ +#pragma once +#include "ppc-framework/io/DataBatch.h" +#include +namespace ppc::crypto +{ +class RA2018OprfInterface +{ +public: + using Ptr = std::shared_ptr; + RA2018OprfInterface() = default; + virtual ~RA2018OprfInterface() = default; + + virtual bcos::bytes generatePrivateKey() = 0; + virtual bcos::bytes inv(bcos::bytes const& _data) = 0; + + virtual void blind(ppc::io::DataBatch::Ptr const& _plainData, bcos::bytes const& _privateKey, + std::vector& _blindResult) = 0; + + virtual void finalize(std::vector const& _evaluatedData, + bcos::bytes const& _invPrivateKey, std::vector& _finalizedResult) = 0; + + virtual void fullEvaluate( + ppc::io::DataBatch::Ptr const& _input, std::vector& _result) = 0; + virtual void evaluate( + std::vector const& _blindData, std::vector& _result) = 0; +}; +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/crypto/SymCrypto.h b/cpp/ppc-framework/crypto/SymCrypto.h new file mode 100644 index 00000000..b073c38f --- /dev/null +++ b/cpp/ppc-framework/crypto/SymCrypto.h @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SymCrypto.h + * @author: shawnhe + * @date 2022-11-29 + */ + +#pragma once + +#include "../Common.h" +#include "../libwrapper/Buffer.h" +#include "../protocol/Protocol.h" +#include +#include +#include + +namespace ppc::crypto +{ +class SymCrypto +{ +public: + using Ptr = std::shared_ptr; + static constexpr uint32_t DEFAULT_BLOCK_SIZE_BYTE = 16; + enum class OperationMode : int + { + ECB, + CBC, + CFB, + OFB, + CTR, + }; + SymCrypto() : m_blockSize(16) {} + virtual ~SymCrypto() = default; + + + virtual ppc::protocol::SymCryptoImplName type() const = 0; + + virtual bcos::bytes generateKey(OperationMode _mode) const = 0; + virtual void generateKey(OutputBuffer* _key, OperationMode _mode) const = 0; + + virtual bcos::bytes encrypt(OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _plaintext) const = 0; + virtual void encrypt(OutputBuffer* _cipher, OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _plaintext) const = 0; + + virtual void decrypt(OutputBuffer* plain, OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _ciphertext) const = 0; + + virtual bcos::bytes decrypt(OperationMode _mode, bcos::bytesConstRef const& _sk, + bcos::bytesConstRef const& _iv, bcos::bytesConstRef const& _ciphertext) const = 0; + + virtual unsigned int keyBytes(OperationMode _mode) const = 0; + unsigned int blockSize() const { return m_blockSize; } + +protected: + // the length of the block + unsigned int m_blockSize = DEFAULT_BLOCK_SIZE_BYTE; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-framework/front/Channel.h b/cpp/ppc-framework/front/Channel.h new file mode 100644 index 00000000..784a06fa --- /dev/null +++ b/cpp/ppc-framework/front/Channel.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief interface for channel which is the abstract communication unit of the privacy service + * @file Channel.h + * @author: shawnhe + * @date 2022-10-27 + */ + +#pragma once +#include "FrontInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include "ppc-framework/protocol/Task.h" +#include +#include + +namespace ppc +{ +namespace front +{ +/** + * Each task hold a channel instance. + * The front can dispatch the message to the specific channel by task ID. + */ +class Channel +{ +public: + using Ptr = std::shared_ptr; + Channel() = default; + virtual ~Channel() {} + +public: + /** + * @brief notice task info to gateway by front + * @param _taskInfo the latest task information + */ + virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; + + /** + * @brief: send message + * @param _agencyID: message receiver + * @return void + */ + virtual void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) = 0; + + /** + * @brief: receive message, note that one message will consume one handler + * @param _messageType: the message type is defined by each crypto algorithm + * @param _seq: message seq + * @param _secondsTimeout: timeout by seconds + * @return void + */ + virtual void asyncReceiveMessage(uint8_t _messageType, uint32_t _seq, uint32_t _secondsTimeout, + std::function _handler) = 0; + + + /** + * @brief: used for front to dispatch message + * @param _messageType: the message type is defined by each crypto algorithm + * @return void + */ + virtual void onMessageArrived(uint8_t _messageType, front::PPCMessageFace::Ptr _message) = 0; +}; + +} // namespace front +} // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/front/FrontInterface.h b/cpp/ppc-framework/front/FrontInterface.h new file mode 100644 index 00000000..41e491d7 --- /dev/null +++ b/cpp/ppc-framework/front/FrontInterface.h @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief interface for front service module + * @file FrontInterface.h + * @author: shawnhe + * @date 2022-10-19 + */ + +#pragma once +#include "../protocol/PPCMessageFace.h" +#include "ppc-framework/protocol/Task.h" +#include + +namespace ppc +{ +namespace front +{ +using ErrorCallbackFunc = std::function; +using ResponseFunc = std::function; +using CallbackFunc = std::function; + +using GetAgencyListCallback = std::function&&)>; + +/** + * @brief: the interface provided by the front service + */ +class FrontInterface +{ +public: + using Ptr = std::shared_ptr; + FrontInterface() = default; + virtual ~FrontInterface() {} + /** + * @brief: start/stop service + */ + virtual void start() = 0; + virtual void stop() = 0; + + /** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ + virtual void onReceiveMessage( + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) = 0; + + + /** + * @brief: send message to other party by gateway + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @param _callback: callback called when the message sent successfully + * @param _respCallback: callback called when receive the response from peer + * @return void + */ + virtual void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) = 0; + + // send response when receiving message from given agencyID + virtual void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) = 0; + + /** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ + virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; + + // erase the task-info when task finished + virtual bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) = 0; + + // get the agencyList from the gateway + virtual void asyncGetAgencyList(GetAgencyListCallback _callback) = 0; + + virtual std::string const& selfEndPoint() const { return m_selfEndPoint; } + +protected: + // the selfEndPoint for the air-mode-node can be localhost + std::string m_selfEndPoint = "localhost"; +}; + +} // namespace front +} // namespace ppc diff --git a/cpp/ppc-framework/gateway/GatewayInterface.h b/cpp/ppc-framework/gateway/GatewayInterface.h new file mode 100644 index 00000000..52ceda88 --- /dev/null +++ b/cpp/ppc-framework/gateway/GatewayInterface.h @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief interface for Gateway module + * @file GatewayInterface.h + * @author: shawnhe + * @date 2022-10-19 + */ +#pragma once +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include "ppc-framework/protocol/Task.h" +#include + +namespace ppc +{ +namespace gateway +{ +using ErrorCallbackFunc = std::function; + +/** + * @brief: A list of interfaces provided by the gateway which are called by the front service. + */ +class GatewayInterface +{ +public: + using Ptr = std::shared_ptr; + GatewayInterface() = default; + virtual ~GatewayInterface() {} + + /** + * @brief: start/stop service + */ + virtual void start() = 0; + virtual void stop() = 0; + +public: + /** + * @brief: send message to gateway + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @param _callback: callback + * @return void + */ + virtual void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + ErrorCallbackFunc _callback) = 0; + + + /** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ + virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; + + // erase the task-info when task finished + virtual bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) = 0; + + // register the gateway info of other parties + virtual bcos::Error::Ptr registerGateway( + const std::vector& _gatewayList) = 0; + + virtual void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front) = 0; + virtual void unregisterFront(std::string const&) {} + + virtual void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) = 0; +}; + +} // namespace gateway +} // namespace ppc diff --git a/cpp/ppc-framework/http/HttpInterface.h b/cpp/ppc-framework/http/HttpInterface.h new file mode 100644 index 00000000..e99d7af4 --- /dev/null +++ b/cpp/ppc-framework/http/HttpInterface.h @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpInterface.h + * @author: zachma + * @date 2023-8-34 + */ + +#pragma once +#include "../protocol/Protocol.h" +#include "../protocol/Task.h" +#include +#include +#include +#include + +#define HTTP_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("HTTP") + +namespace bcos +{ +namespace boostssl +{ +class MessageFace; +class WsSession; +} // namespace boostssl +} // namespace bcos +namespace ppc::http +{ +using RespUrlFunc = std::function; +class HttpInterface +{ +public: + using Ptr = std::shared_ptr; + HttpInterface() = default; + virtual ~HttpInterface() = default; + + virtual void start() = 0; + virtual void stop() = 0; + + virtual void registerUrlHandler(std::string const& _urlName, + std::function _handler) = 0; +}; +} // namespace ppc::http \ No newline at end of file diff --git a/cpp/ppc-framework/http/HttpTypeDef.h b/cpp/ppc-framework/http/HttpTypeDef.h new file mode 100644 index 00000000..d8375e8f --- /dev/null +++ b/cpp/ppc-framework/http/HttpTypeDef.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpTypeDef.h + * @author: zachma + * @date 2023-8-23 + */ + +#pragma once +#include + +namespace ppc::http +{ + enum class HttpError : int32_t +{ + UriNotFound = -32605, +}; + +} // namespace ppc::http + diff --git a/cpp/ppc-framework/io/DataBatch.h b/cpp/ppc-framework/io/DataBatch.h new file mode 100644 index 00000000..46ace021 --- /dev/null +++ b/cpp/ppc-framework/io/DataBatch.h @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataBatch.h + * @author: yujiechen + * @date 2022-10-17 + */ +#pragma once +#include "../Common.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::io +{ +using DataType = std::variant; + +enum DataSchema +{ + String, + Bytes, + Sint, + Uint, + Double, + Float, + Null, // the data-schema not setted +}; + +// the DataBatch +class DataBatch +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + DataBatch() : m_data(std::make_shared>()) {} + DataBatch(std::vector _data) : DataBatch() { *m_data = std::move(_data); } + ~DataBatch() = default; + + template + void setData(std::vector&& _data) + { + m_data->clear(); + for (auto& it : _data) + { + append(std::move(it)); + } + } + + template + void setData(std::vector const& _data) + { + m_data->clear(); + for (auto const& it : _data) + { + append(it); + } + } + + // append to different line + template + void append(T _data) + { + if constexpr (std::is_same::value || std::is_same::value) + { + m_capacityBytes += _data.size(); + } + else + { + m_capacityBytes += sizeof(_data); + } + m_data->emplace_back(_data); + } + + // append to the last line + // Note: only bcos::bytes, std::string can use this function + template + void appendToLine(T&& _data) + { + auto dataIndex = m_data->size() - 1; + auto& data = std::get((*m_data)[dataIndex]); + data.insert(data.end(), _data.begin(), _data.end()); + if constexpr (std::is_same::value || std::is_same::value) + { + m_capacityBytes += _data.size(); + } + } + + // Note: can't pass-in a invalid index + template + T const& get(uint64_t _index) const + { + return std::get(m_data->at(_index)); + } + + // Note: This introduces additional copy overhead + // convert to bytes + bcos::bytes getBytes(uint64_t _index) + { + if (m_dataSchema == DataSchema::Null) + { + BOOST_THROW_EXCEPTION(DataSchemaNotSetted() << bcos::errinfo_comment( + "Must set DataSchema before calling getBytes!")); + } + switch (m_dataSchema) + { + case DataSchema::String: + { + auto const& item = get(_index); + return bcos::bytes(item.begin(), item.end()); + } + case DataSchema::Bytes: + { + return get(_index); + } + case DataSchema::Uint: + { + auto const& value = get(_index); + auto result = boost::lexical_cast(value); + return bcos::bytes(result.begin(), result.end()); + } + case DataSchema::Sint: + { + auto const& value = get(_index); + auto result = boost::lexical_cast(value); + return bcos::bytes(result.begin(), result.end()); + } + case DataSchema::Double: + { + auto const& value = get(_index); + auto result = boost::lexical_cast(value); + return bcos::bytes(result.begin(), result.end()); + } + case DataSchema::Float: + { + auto const& value = get(_index); + auto result = boost::lexical_cast(value); + return bcos::bytes(result.begin(), result.end()); + } + default: + { + BOOST_THROW_EXCEPTION( + UnsupportedDataSchema() << bcos::errinfo_comment("unsupported data schema")); + } + } + } + + uint64_t size() const { return m_data->size(); } + + template + uint64_t elementSize(uint64_t _offset) const + { + if constexpr (std::is_same::value || std::is_same::value) + { + return std::get((*m_data)[_offset]).size(); + } + else + { + return sizeof(T); + } + } + uint64_t capacityBytes() const { return m_capacityBytes; } + + void resize(uint64_t _size) { m_data->resize(_size); } + + template + void resizeElement(uint64_t _index, uint64_t _size) + { + std::get((*m_data)[_index]).resize(_size); + } + + template + void set(uint64_t _offset, T&& _data) + { + if constexpr (std::is_same::value || std::is_same::value) + { + auto orgDataSize = elementSize(_offset); + m_capacityBytes -= orgDataSize; + m_capacityBytes += _data.size(); + } + (*m_data)[_offset].emplace(std::move(_data)); + } + + void setDataSchema(DataSchema _dataSchema) { m_dataSchema = _dataSchema; } + + DataSchema getDataSchema() const { return m_dataSchema; } + + uint32_t dedup() + { + if (!m_data || m_data->empty()) + { + return 0; + } + + tbb::parallel_sort(m_data->begin(), m_data->end()); + auto unique_end = std::unique(m_data->begin(), m_data->end()); + m_data->erase(unique_end, m_data->end()); + return m_data->size(); + } + +private: + std::shared_ptr> m_data; + uint64_t m_capacityBytes = 0; + DataSchema m_dataSchema = DataSchema::Null; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-framework/io/DataResourceLoader.h b/cpp/ppc-framework/io/DataResourceLoader.h new file mode 100644 index 00000000..9f3db868 --- /dev/null +++ b/cpp/ppc-framework/io/DataResourceLoader.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataResourceLoader.h + * @author: yujiechen + * @date 2022-11-4 + */ + +#pragma once +#include "../protocol/DataResource.h" +#include "../protocol/Protocol.h" +#include "../storage/FileStorage.h" +#include "LineReader.h" +#include "LineWriter.h" + +namespace ppc::io +{ +class DataResourceLoader +{ +public: + using Ptr = std::shared_ptr; + DataResourceLoader() = default; + virtual ~DataResourceLoader() = default; + + // load resource according to the dataResource + // Note: the FILE type data resource only support parsed by line + // the FILE type data resource can be loaded by segment + // the SQL type data resource support parsed by both line and column + // all the data returned from the SQL will be loaded into the memory + virtual LineReader::Ptr loadReader(ppc::protocol::DataResourceDesc::ConstPtr _dataResourceDesc, + ppc::io::DataSchema _schema = ppc::io::DataSchema::String, bool _parseByColumn = true, + ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0; + + // create a LineWriter to store data into the data-resource specified by outputDesc + // Note: only support file/hdfs now + virtual LineWriter::Ptr loadWriter(ppc::protocol::DataResourceDesc::ConstPtr _dataResourceDesc, + bool _truncate = false, ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0; + + virtual void checkResourceExists(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-framework/io/LineReader.h b/cpp/ppc-framework/io/LineReader.h new file mode 100644 index 00000000..a1031a06 --- /dev/null +++ b/cpp/ppc-framework/io/LineReader.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Reader.h + * @author: yujiechen + * @date 2022-10-13 + */ +#pragma once +#include "../protocol/Protocol.h" +#include "DataBatch.h" + +namespace ppc::io +{ +class LineReader +{ +public: + using Ptr = std::shared_ptr; + LineReader() = default; + virtual ~LineReader() = default; + + // get the next _offset line data + virtual DataBatch::Ptr next(int64_t _size, DataSchema schema = DataSchema::String) = 0; + virtual bcos::bytes readBytes() = 0; + // the capacity of the file or memory-bytes + virtual uint64_t capacity() const = 0; + virtual uint64_t columnSize() const = 0; + virtual ppc::protocol::DataResourceType type() const = 0; + virtual void clean() {} +}; + +class LineReaderFactory +{ +public: + using Ptr = std::shared_ptr; + LineReaderFactory() = default; + virtual ~LineReaderFactory() = default; + + virtual LineReader::Ptr createLineReader(std::string const& _filePath, + uint64_t _mmapGranularity = 500 * 1024 * 1024, char _lineSpliter = '\n') = 0; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-framework/io/LineWriter.h b/cpp/ppc-framework/io/LineWriter.h new file mode 100644 index 00000000..ba4d1907 --- /dev/null +++ b/cpp/ppc-framework/io/LineWriter.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Writer.h + * @author: yujiechen + * @date 2022-10-17 + */ +#pragma once +#include "../protocol/Protocol.h" +#include "DataBatch.h" +namespace ppc::io +{ +class LineWriter +{ +public: + using Ptr = std::shared_ptr; + LineWriter() = default; + virtual ~LineWriter() = default; + + virtual ppc::protocol::DataResourceType type() const = 0; + + // write the data into the writer(Note: only support row-data) + virtual bool writeLine( + DataBatch::ConstPtr _data, DataSchema _schema, std::string _lineSplitter = "\n") = 0; + + virtual void writeBytes(bcos::bytesConstRef _data) = 0; + + // flush the data into the storage-backend + virtual void flush() = 0; + // close the storage-backend resource(Note: flush before close) + virtual void close() = 0; + + void setFileInfo(protocol::FileInfo::Ptr _fileInfo) { m_fileInfo = std::move(_fileInfo); } + protocol::FileInfo::Ptr const& fileInfo() const { return m_fileInfo; } + + virtual void upload() {} + + virtual void clean() {} + +private: + protocol::FileInfo::Ptr m_fileInfo; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-framework/libwrapper/BigNum.h b/cpp/ppc-framework/libwrapper/BigNum.h new file mode 100644 index 00000000..4a43de70 --- /dev/null +++ b/cpp/ppc-framework/libwrapper/BigNum.h @@ -0,0 +1,479 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BigNum.h + * @author: yujiechen + * @date 2022-12-5 + */ +#pragma once +#include "Buffer.h" +#include "openssl/bn.h" +#include "openssl/err.h" +#include "ppc-framework/Common.h" +#include +#include +#include + +namespace ppc::crypto +{ +DERIVE_PPC_EXCEPTION(ConvertBytesToBigNumError); +DERIVE_PPC_EXCEPTION(CreateBNContextError); +DERIVE_PPC_EXCEPTION(BigNumSubError); +DERIVE_PPC_EXCEPTION(BigNumAddError); +DERIVE_PPC_EXCEPTION(BigNumMultiError); +DERIVE_PPC_EXCEPTION(GeneratePrimeError); +DERIVE_PPC_EXCEPTION(BigNumGCDError); +DERIVE_PPC_EXCEPTION(BigNumDivError); +DERIVE_PPC_EXCEPTION(BigNumModExpError); +DERIVE_PPC_EXCEPTION(ConvertIntToBigNumError); +DERIVE_PPC_EXCEPTION(BigNumRandError); +DERIVE_PPC_EXCEPTION(BigNumSetWordError); +DERIVE_PPC_EXCEPTION(BigNumGetBytesLenError); +DERIVE_PPC_EXCEPTION(NotEnoughBufferSize); + +struct BNCtxDeleter +{ +public: + void operator()(BN_CTX* _bnctx) { BN_CTX_free(_bnctx); } +}; +using BNCtxPtr = std::unique_ptr; + +struct BNDeleter +{ + void operator()(BIGNUM* bn) { BN_clear_free(bn); } +}; +using BNPtr = std::shared_ptr; + +inline BNCtxPtr createBNContext() +{ + BNCtxPtr bnContext(BN_CTX_new()); + if (!bnContext) + { + BOOST_THROW_EXCEPTION(CreateBNContextError()); + } + return bnContext; +} + +class BigNum +{ +public: + BigNum() : m_bn(BNPtr(BN_new(), BNDeleter())) {} + BigNum(bcos::bytesConstRef const& _data, bool _signed = true) : BigNum() + { + fromBytes(_data, _signed); + } + BigNum(bcos::bytesConstRef const& _data, BigNum const& p, bool _signed) : BigNum() + { + fromBytesModP(_data, p, _signed); + } + + explicit BigNum(int64_t const _value) : BigNum() + { + auto absV = _value > 0 ? _value : (-_value); + auto ret = BN_set_word(m_bn.get(), absV); + if (1 != ret) + { + BOOST_THROW_EXCEPTION( + ConvertIntToBigNumError() << bcos::errinfo_comment( + "BN_hex2bn error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + BN_set_negative(m_bn.get(), (_value >= 0 ? 0 : 1)); + } + + BigNum(s1024 const& _value) : BigNum() + { + bcos::bytes encodedData(128, 0); + u1024 unsignedValue = u1024(_value); + bool negative = false; + if (_value < 0) + { + unsignedValue = u1024(-_value); + negative = true; + } + bcos::toBigEndian(unsignedValue, encodedData); + fromBigEndianBytes(encodedData.data(), encodedData.size(), negative); + } + + ~BigNum() = default; + + // swap _data with m_bn + void swap(BIGNUM* _data) { BN_swap(m_bn.get(), _data); } + void copy(BIGNUM* _data) { BN_copy(_data, m_bn.get()); } + int64_t getWord() const { return (int64_t)BN_get_word(m_bn.get()); } + + // convert bytes to BIGNUM + void fromBytes(bcos::bytesConstRef const& _data, bool _signed = true) + { + if (_data.size() == 0) + { + BOOST_THROW_EXCEPTION( + ConvertBytesToBigNumError() << bcos::errinfo_comment("Invalid BigNum data")); + } + if (_signed) + { + bool negative = false; + if (_data[0] == 0xff) + { + negative = true; + } + fromBigEndianBytes((bcos::byte*)(_data.data() + 1), _data.size() - 1, negative); + return; + } + fromBigEndianBytes((bcos::byte*)(_data.data()), _data.size(), false); + } + + void fromBigEndianBytes(bcos::byte* _start, int64_t _size, bool _negative) + { + if (!BN_bin2bn(_start, _size, m_bn.get())) + { + BOOST_THROW_EXCEPTION(ConvertBytesToBigNumError()); + } + if (_negative) + { + BN_set_negative(m_bn.get(), 1); + } + } + + void fromBytesModP(bcos::bytesConstRef const& _data, BigNum const& p, bool _signed = true) const + { + BigNum value(_data, _signed); + auto bnContext = createBNContext(); + if (BN_nnmod(m_bn.get(), value.bn().get(), p.bn().get(), bnContext.get()) != 1) + { + BOOST_THROW_EXCEPTION(ConvertBytesToBigNumError()); + } + } + + // convert BIGNUM to bytes + void toBytes(bcos::bytes& _output, bool _signed = true) const + { + auto bytesLen = BN_num_bytes(m_bn.get()); + if (bytesLen < 0) + { + BOOST_THROW_EXCEPTION( + BigNumGetBytesLenError() << bcos::errinfo_comment( + "BN_num_bytes error:" + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + auto bufferSize = _signed ? (bytesLen + 1) : bytesLen; + _output.resize(bufferSize); + OutputBuffer buffer{_output.data(), _output.size()}; + toOutputBuffer(&buffer, _signed); + } + + // convert to s1024 + s1024 toS1024() const + { + auto bytesLen = BN_num_bytes(m_bn.get()); + bcos::bytes bigEndianBytes(bytesLen, 0); + BN_bn2bin(m_bn.get(), (bcos::byte*)bigEndianBytes.data()); + auto result = bcos::fromBigEndian(bigEndianBytes); + if (BN_is_negative(m_bn.get())) + { + return s1024(0 - result); + } + return result; + } + + void toOutputBuffer(OutputBuffer* _output, bool _signed = true) const + { + auto bytesLen = BN_num_bytes(m_bn.get()); + if (bytesLen < 0) + { + BOOST_THROW_EXCEPTION( + BigNumGetBytesLenError() << bcos::errinfo_comment( + "BN_num_bytes error:" + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + auto bufferSize = _signed ? (bytesLen + 1) : bytesLen; + if (_output->len < (uint64_t)bufferSize) + { + BOOST_THROW_EXCEPTION( + NotEnoughBufferSize() << bcos::errinfo_comment( + "toOutputBuffer error for not enough buffer size, min buffer size is " + + std::to_string(bufferSize))); + } + _output->len = bufferSize; + if (_signed) + { + if (!BN_is_negative(m_bn.get())) + { + *(_output->data) = 0x00; + } + else + { + *(_output->data) = 0xff; + } + BN_bn2bin(m_bn.get(), (bcos::byte*)_output->data + 1); + return; + } + BN_bn2bin(m_bn.get(), (bcos::byte*)_output->data); + } + + // invert: (m_bn)^-1 mod _p + BigNum Invert(BigNum const& _p) const + { + auto bnContext = createBNContext(); + BigNum inversedNum; + BN_mod_inverse(inversedNum.bn().get(), m_bn.get(), _p.bn().get(), bnContext.get()); + return inversedNum; + } + + // m_bn + _value + BigNum add(BIGNUM const* _v) const + { + BigNum result; + add(result.bn().get(), _v); + return result; + } + + void add(BIGNUM* _result, BIGNUM const* _v) const + { + auto ret = BN_add(_result, m_bn.get(), _v); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumAddError() << bcos::errinfo_comment( + "BN_add error:" + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // (m_bn + _v) mod _m + BigNum modAdd(BIGNUM const* _v, BIGNUM const* _m) const + { + BigNum result; + auto ret = BN_mod_add_quick(result.bn().get(), m_bn.get(), _v, _m); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumAddError() << bcos::errinfo_comment( + "BN_mod_add_quick error:" + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return result; + } + + // sub + BigNum sub(BIGNUM const* _subValue) const + { + BigNum result; + auto ret = BN_sub(result.bn().get(), m_bn.get(), _subValue); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumSubError() << bcos::errinfo_comment( + "BN_sub error:" + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return result; + } + + // multiply + void mul(BIGNUM* _result, BIGNUM const* _value, BN_CTX* _ctx) const + { + auto ret = BN_mul(_result, m_bn.get(), _value, _ctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumMultiError() << bcos::errinfo_comment( + "BN_mul error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // mud sub + void modSub(BIGNUM* _result, BIGNUM const* _value, BIGNUM const* _p, BN_CTX* _ctx) const + { + BigNum result; + auto ret = BN_mod_sub(_result, m_bn.get(), _value, _p, _ctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumAddError() << bcos::errinfo_comment( + "BN_mod_sub error:" + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // mod multiply + void modMul(BIGNUM* _result, BIGNUM const* _value, BIGNUM const* _p, BN_CTX* _ctx) const + { + auto ret = BN_mod_mul(_result, m_bn.get(), _value, _p, _ctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumMultiError() << bcos::errinfo_comment( + "BN_mod_mul error: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // generate prime bn + void generatePrime(unsigned int const _keyLength) + { + // Note: here set safe to 0 to improve the performance + // if set safe to 1, generate prime will cost multiple ms + auto ret = BN_generate_prime_ex(m_bn.get(), _keyLength, 0, NULL, NULL, NULL); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(GeneratePrimeError() << bcos::errinfo_comment( + "BN_generate_prime_ex error: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // gcd(m_bn, _q) + void gcd(BIGNUM* _result, BIGNUM const* _q, BN_CTX* _bnctx) const + { + auto ret = BN_gcd(_result, m_bn.get(), _q, _bnctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumGCDError() << bcos::errinfo_comment( + "BN_gcd error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // m_bn/_q + void div(BIGNUM* _div, BIGNUM* _r, BIGNUM const* _q, BN_CTX* _bnctx) const + { + auto ret = BN_div(_div, _r, m_bn.get(), _q, _bnctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumDivError() << bcos::errinfo_comment( + "BN_div error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // (m_bn)^p mod m + BigNum modExp(const BIGNUM* _p, const BIGNUM* _m, BN_CTX* _ctx) const + { + BigNum result; + auto ret = BN_mod_exp(result.bn().get(), m_bn.get(), _p, _m, _ctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumModExpError() << bcos::errinfo_comment( + "BN_mod_exp error: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return result; + } + + // m_bn ^p + BigNum exp(const BIGNUM* _p, BN_CTX* _ctx) const + { + BigNum result; + auto ret = BN_exp(result.bn().get(), m_bn.get(), _p, _ctx); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumModExpError() << bcos::errinfo_comment( + "BN_exp error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return result; + } + + int isBitSet(int n) { return BN_is_bit_set(m_bn.get(), n); } + + int cmp(const BIGNUM* b) const { return BN_cmp(m_bn.get(), b); } + + // set m_bn to _v + void setWord(BN_ULONG _v) + { + auto ret = BN_set_word(m_bn.get(), _v); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumSetWordError() << bcos::errinfo_comment( + "BN_set_word error: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + // divide m_bn by _v + void divConst(BN_ULONG w) + { + auto ret = BN_div_word(m_bn.get(), w); + if ((BN_ULONG)-1 == ret) + { + BOOST_THROW_EXCEPTION( + BigNumDivError() << bcos::errinfo_comment( + "BN_div_word error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + } + + BigNum add(BigNum const& _num, BigNum const& _p) const + { + auto bnContext = createBNContext(); + BigNum addNum; + BN_mod_add(addNum.bn().get(), m_bn.get(), _num.bn().get(), _p.bn().get(), bnContext.get()); + return addNum; + } + + BigNum sub(BigNum const& _num, BigNum const& _p) const + { + auto bnContext = createBNContext(); + BigNum subNum; + BN_mod_sub(subNum.bn().get(), m_bn.get(), _num.bn().get(), _p.bn().get(), bnContext.get()); + return subNum; + } + + BigNum mul(BigNum const& _num, BigNum const& _p) const + { + auto bnContext = createBNContext(); + BigNum mulNum; + BN_mod_mul(mulNum.bn().get(), m_bn.get(), _num.bn().get(), _p.bn().get(), bnContext.get()); + return mulNum; + } + BNPtr const& bn() const { return m_bn; } + +private: + BNPtr m_bn; +}; + +// generate random in range of (0, _n) +inline void randRange(BIGNUM* _r, const BIGNUM* _n) +{ + auto ret = BN_rand_range(_r, _n); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION(BigNumRandError() << bcos::errinfo_comment( + "BN_rand_range error: " + + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } +} + +// generate random with bits size +inline BigNum generateRand(int _bits) +{ + BigNum r; + auto ret = BN_rand(r.bn().get(), _bits, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY); + if (1 != ret) + [[unlikely]] + { + BOOST_THROW_EXCEPTION( + BigNumRandError() << bcos::errinfo_comment( + "BN_rand error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return r; +} + +} // namespace ppc::crypto \ No newline at end of file diff --git a/cpp/ppc-framework/libwrapper/Buffer.h b/cpp/ppc-framework/libwrapper/Buffer.h new file mode 100644 index 00000000..15d984cc --- /dev/null +++ b/cpp/ppc-framework/libwrapper/Buffer.h @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file buffer.h + * @author: yujiechen + * @date 2023-08-11 + */ +#ifndef __BUFFER_H__ +#define __BUFFER_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct +{ + const unsigned char* data; + uint64_t len; +} InputBuffer; + +typedef struct +{ + unsigned char* data; + uint64_t len; +} OutputBuffer; + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-framework/libwrapper/FloatingPointNumber.h b/cpp/ppc-framework/libwrapper/FloatingPointNumber.h new file mode 100644 index 00000000..6c286ab2 --- /dev/null +++ b/cpp/ppc-framework/libwrapper/FloatingPointNumber.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointNumber.h + * @author: yujiechen + * @date 2023-08-16 + */ +#pragma once +#include "BigNum.h" +#include "openssl/bn.h" +#include +#include +#include +namespace ppc +{ +DERIVE_PPC_EXCEPTION(FloatingPointNumberError); +struct FloatingPointNumber +{ + ppc::crypto::BigNum value; + int16_t exponent; + + using Ptr = std::shared_ptr; + FloatingPointNumber() = default; + FloatingPointNumber(ppc::crypto::BigNum&& _value, int _exponent) + : value(std::move(_value)), exponent(_exponent) + {} + virtual ~FloatingPointNumber() = default; +}; +} // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/libwrapper/OreFloatingNumber.h b/cpp/ppc-framework/libwrapper/OreFloatingNumber.h new file mode 100644 index 00000000..14b03f01 --- /dev/null +++ b/cpp/ppc-framework/libwrapper/OreFloatingNumber.h @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OreFloatingNumber.h + * @author: shawnhe + * @date 2023-12-06 + */ +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-tools/src/common/TransTools.h" +#include +#include +#include +#include + +namespace ppc +{ +DERIVE_PPC_EXCEPTION(OreFloatingNumberError); + +struct OreFloatingNumber +{ + int64_t integerPart; + std::string decimalPart; + + OreFloatingNumber() = default; + virtual ~OreFloatingNumber() = default; + + OreFloatingNumber(const float50& _value) + { + if (_value > (std::numeric_limits::max)() / 2 - 1 || + _value <= (std::numeric_limits::min)() / 2 + 1) + { + BOOST_THROW_EXCEPTION( + OreFloatingNumberError() << bcos::errinfo_comment( + "plain is too large or too small, must be in range (-2^62 + 1, 2^62 - 1)")); + } + // set to positive + // Note: If there are too many decimal, there will be a problem of precision truncation. + float50 value = _value + (std::numeric_limits::max)() / 2; + std::string valueStr = value.str(); + auto dotPos = valueStr.find('.'); + int64_t intValue = std::stoll(valueStr.substr(0, dotPos)); + + integerPart = tools::toBigEndian(intValue); + + if (dotPos != std::string::npos) + { + decimalPart = valueStr.substr(dotPos + 1, valueStr.size()); + } + } + + float50 value() + { + std::string valueStr; + if (decimalPart.empty()) + { + valueStr = std::to_string(ppc::tools::fromBigEndian(integerPart)); + } + else + { + valueStr = std::to_string(ppc::tools::fromBigEndian(integerPart)) + "." + decimalPart; + } + + return float50(valueStr) - (std::numeric_limits::max)() / 2; + } +}; +} // namespace ppc diff --git a/cpp/ppc-framework/protocol/DataResource.h b/cpp/ppc-framework/protocol/DataResource.h new file mode 100644 index 00000000..3a095050 --- /dev/null +++ b/cpp/ppc-framework/protocol/DataResource.h @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataResource.h + * @author: yujiechen + * @date 2022-10-13 + */ +#pragma once +#include "Protocol.h" +#include +#include +#include + +namespace ppc::protocol +{ +class DataResourceDesc +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + DataResourceDesc() = default; + virtual ~DataResourceDesc() = default; + + virtual uint16_t type() const { return m_type; } + virtual void setType(uint16_t _type) { m_type = _type; } + + virtual std::string const& path() const { return m_path; } + virtual void setPath(std::string const& _path) { m_path = _path; } + + virtual std::string const& accessCommand() const { return m_accessCommand; } + virtual void setAccessCommand(std::string const& _command) { m_accessCommand = _command; } + + virtual std::string const& fileID() const { return m_fileID; } + virtual void setFileID(std::string const& _fileID) { m_fileID = _fileID; } + + virtual std::string const& fileMd5() const { return m_fileMd5; } + virtual void setFileMd5(std::string const& _fileMd5) { m_fileMd5 = _fileMd5; } + + virtual std::string const& bizSeqNo() const { return m_bizSeqNo; } + virtual void setBizSeqNo(std::string const& _bizSeqNo) { m_bizSeqNo = _bizSeqNo; } + + virtual SQLConnectionOption::Ptr const& sqlConnectionOption() const + { + return m_sqlConnectionOption; + } + virtual void setSQLConnectionOption(SQLConnectionOption::Ptr const& _option) + { + m_sqlConnectionOption = _option; + } + + // the hdfs connection + ppc::protocol::FileStorageConnectionOption::Ptr const& fileStorageConnectionOption() const + { + return m_fileStorageConnectionOption; + } + virtual void setFileStorageConnectOption( + ppc::protocol::FileStorageConnectionOption::Ptr const& _fileStorageConnectionOption) + { + m_fileStorageConnectionOption = _fileStorageConnectionOption; + } + + +private: + std::string m_path; + std::string m_accessCommand; + std::string m_fileID; + std::string m_fileMd5; + std::string m_bizSeqNo; + + // the sql-connection-option + SQLConnectionOption::Ptr m_sqlConnectionOption; + // the file-storage-connection-option + FileStorageConnectionOption::Ptr m_fileStorageConnectionOption; + + uint16_t m_type; +}; + +class DataResource +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + using OriginData = std::vector>; + DataResource() = default; + DataResource(std::string const& _resourceID) : m_resourceID(_resourceID) {} + + virtual ~DataResource() = default; + virtual std::string const& resourceID() const { return m_resourceID; } + + virtual DataResourceDesc::ConstPtr desc() const { return m_desc; } + virtual DataResourceDesc::Ptr mutableDesc() const { return m_desc; } + virtual DataResourceDesc::ConstPtr outputDesc() const { return m_outputDesc; } + virtual DataResourceDesc::Ptr mutableOutputDesc() const { return m_outputDesc; } + virtual OriginData const& rawData() const { return m_rawData; } + + virtual void setResourceID(std::string const& _id) { m_resourceID = _id; } + virtual void setDesc(DataResourceDesc::Ptr _desc) { m_desc = _desc; } + virtual void setOutputDesc(DataResourceDesc::Ptr _desc) { m_outputDesc = _desc; } + virtual void setRawData(OriginData const& _data) { m_rawData = _data; } + +private: + std::string m_resourceID; + DataResourceDesc::Ptr m_desc; + DataResourceDesc::Ptr m_outputDesc; + OriginData m_rawData; +}; + +inline std::string printDataResourceInfo(DataResource::ConstPtr _dataResource) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("dataResource", _dataResource->resourceID()); + if (_dataResource->desc()) + { + stringstream << LOG_KV("dataType", _dataResource->desc()->type()) + << LOG_KV("command", _dataResource->desc()->accessCommand()) + << LOG_KV("path", _dataResource->desc()->path()) + << LOG_KV("bizSeqNo", _dataResource->desc()->bizSeqNo()) + << LOG_KV("fileID", _dataResource->desc()->fileID()) + << LOG_KV("fileMd5", _dataResource->desc()->fileMd5()); + } + if (_dataResource->outputDesc()) + { + stringstream << LOG_KV("outputType", _dataResource->outputDesc()->type()) + << LOG_KV("outputPath", _dataResource->outputDesc()->path()); + } + return stringstream.str(); +} +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/GlobalConfig.h b/cpp/ppc-framework/protocol/GlobalConfig.h new file mode 100644 index 00000000..ab02443e --- /dev/null +++ b/cpp/ppc-framework/protocol/GlobalConfig.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GlobalConfig.h + * @author: yujiechen + * @date 2023-1-3 + */ +#pragma once +#include "../Common.h" +#include "Protocol.h" +namespace ppc::protocol +{ +class GlobalConfig +{ +public: + static GlobalConfig& instance() + { + static GlobalConfig ins; + return ins; + } + GlobalConfig() + { + // set the supported-curves for ecdh-psi + auto key = calculateKey((uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC); +#ifdef ENABLE_CRYPTO_MB +#ifdef ENABLE_CPU_FEATURES + if (ppc::CPU_FEATURES.avx512ifma) + { + c_supportedCurves[key] = {(int)ECCCurve::IPP_X25519}; + } +#endif +#endif + c_supportedCurves[key].emplace_back((int)ECCCurve::P256); + c_supportedCurves[key].emplace_back((int)ECCCurve::ED25519); + c_supportedCurves[key].emplace_back((int)ECCCurve::SM2); + c_supportedCurves[key].emplace_back((int)ECCCurve::SECP256K1); + + // set the supported-hash algorithm for ecdh-psi + c_supportedHashList[key] = {(int)HashImplName::BLAKE2b, (int)HashImplName::SHA256, + (int)HashImplName::SM3, (int)HashImplName::SHA512}; + } + + std::vector supportedCurves(uint8_t _taskType, uint8_t _algorithmType) + { + auto key = calculateKey(_taskType, _algorithmType); + auto it = c_supportedCurves.find(key); + if (it != c_supportedCurves.end()) + { + return it->second; + } + return std::vector(); + } + + std::vector supportedHashList(uint8_t _taskType, uint8_t _algorithmType) + { + auto key = calculateKey(_taskType, _algorithmType); + auto it = c_supportedHashList.find(key); + if (it != c_supportedHashList.end()) + { + return it->second; + } + return std::vector(); + } + + void setSMCrypto(bool _smCrypto) { m_smCrypto = _smCrypto; } + bool smCrypto() const { return m_smCrypto; } + +private: + uint16_t calculateKey(uint8_t _taskType, uint8_t _algorithmType) + { + return (uint16_t)_taskType << 8 | (uint16_t)(_algorithmType); + } + +private: + std::map> c_supportedCurves; + std::map> c_supportedHashList; + bool m_smCrypto = false; +}; +} // namespace ppc::protocol +#define g_PPCConfig ppc::protocol::GlobalConfig::instance() \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/PPCMessageFace.h b/cpp/ppc-framework/protocol/PPCMessageFace.h new file mode 100644 index 00000000..275a473d --- /dev/null +++ b/cpp/ppc-framework/protocol/PPCMessageFace.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief ppc network message interface + * @file PPCMessageFace.h + * @author: shawnhe + * @date 2022-10-19 + */ + +#pragma once +#include "Protocol.h" +#include +#include +#include +#include +#include +#include + +namespace ppc +{ +namespace front +{ +enum MessageExtFlag : uint16_t +{ + ResponseFlag = 0x0001, +}; +class PPCMessageFace +{ +public: + using Ptr = std::shared_ptr; + +public: + virtual ~PPCMessageFace() {} + + virtual uint8_t version() const = 0; + virtual void setVersion(uint8_t) = 0; + virtual uint8_t taskType() const = 0; + virtual void setTaskType(uint8_t) = 0; + virtual uint8_t algorithmType() const = 0; + virtual void setAlgorithmType(uint8_t) = 0; + virtual uint8_t messageType() const = 0; + virtual void setMessageType(uint8_t) = 0; + virtual uint32_t seq() const = 0; + virtual void setSeq(uint32_t) = 0; + virtual std::string const& taskID() const = 0; + virtual void setTaskID(std::string const&) = 0; + virtual std::string const& sender() const = 0; + virtual void setSender(std::string const&) = 0; + virtual uint16_t ext() const = 0; + virtual void setExt(uint16_t) = 0; + virtual std::shared_ptr data() const = 0; + virtual void setData(std::shared_ptr) = 0; + virtual std::map header() = 0; + virtual void setHeader(std::map) = 0; + virtual void encode(bcos::bytes& _buffer) = 0; + virtual int64_t decode(bcos::bytesPointer _buffer) = 0; + virtual int64_t decode(bcos::bytesConstRef _buffer) = 0; + + virtual uint32_t length() const = 0; + + virtual std::string uuid() const = 0; + virtual void setUuid(std::string const& _uuid) = 0; + + // determine the message is response or not + virtual bool response() const = 0; + // set the message to be response + virtual void setResponse() = 0; +}; + +class PPCMessageFaceFactory +{ +public: + using Ptr = std::shared_ptr; + +public: + virtual ~PPCMessageFaceFactory() {} + virtual PPCMessageFace::Ptr buildPPCMessage() = 0; + virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesConstRef _data) = 0; + virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesPointer _buffer) = 0; + virtual PPCMessageFace::Ptr buildPPCMessage(uint8_t _taskType, uint8_t _algorithmType, + std::string const& _taskID, bcos::bytesPointer _data) = 0; +}; + +inline std::string printPPCMsg(PPCMessageFace::Ptr _msg) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("sender", _msg->sender()) + << LOG_KV("taskType", (ppc::protocol::TaskType)(_msg->taskType())) + << LOG_KV("algorithm", int(_msg->algorithmType())) + << LOG_KV("messageType", int(_msg->messageType())) + << LOG_KV("task", _msg->taskID()) << LOG_KV("seq", _msg->seq()) + << LOG_KV("uuid", _msg->uuid()) << LOG_KV("response", _msg->response()) + << LOG_KV("size", _msg->data()->size()); + return stringstream.str(); +} +} // namespace front +} // namespace ppc diff --git a/cpp/ppc-framework/protocol/PartyResource.h b/cpp/ppc-framework/protocol/PartyResource.h new file mode 100644 index 00000000..98bb4984 --- /dev/null +++ b/cpp/ppc-framework/protocol/PartyResource.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PartyResource.h + * @author: yujiechen + * @date 2022-10-13 + */ +#pragma once +#include "DataResource.h" +#include "Protocol.h" +#include +#include +#include +#include + +namespace ppc::protocol +{ +class PartyResource +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + PartyResource() = default; + PartyResource(std::string const& _id, uint16_t _partyIndex) + : m_id(_id), m_partyIndex(_partyIndex) + {} + + virtual ~PartyResource() = default; + + // the party index that represents the role in the task + virtual uint16_t partyIndex() const { return m_partyIndex; } + // the party id + virtual std::string const& id() const { return m_id; } + virtual std::string const& desc() const { return m_desc; } + virtual DataResource::ConstPtr dataResource() const { return m_dataResource; } + virtual DataResource::Ptr mutableDataResource() const { return m_dataResource; } + + virtual void setId(std::string const& _id) { m_id = _id; } + virtual void setDesc(std::string _desc) { m_desc = _desc; } + virtual void setDataResource(DataResource::Ptr _dataResource) + { + m_dataResource = _dataResource; + } + // the mpc index + virtual void setPartyIndex(uint16_t _partyIndex) { m_partyIndex = _partyIndex; } + +private: + std::string m_id; + uint16_t m_partyIndex; + std::string m_desc; + DataResource::Ptr m_dataResource; +}; +using Parties = std::vector; +using ConstParties = std::vector; + +inline std::string printPartyInfo(PartyResource::ConstPtr _party) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("partyId", _party->id()) << LOG_KV("partyIndex", _party->partyIndex()) + << LOG_KV("desc", _party->desc()); + + if (_party->dataResource()) + { + auto const& dataResource = _party->dataResource(); + stringstream << printDataResourceInfo(dataResource); + } + return stringstream.str(); +} +} // namespace ppc::protocol diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h new file mode 100644 index 00000000..889b7104 --- /dev/null +++ b/cpp/ppc-framework/protocol/Protocol.h @@ -0,0 +1,505 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Protocol.h + * @author: yujiechen + * @date 2022-10-13 + */ +#pragma once + +#if defined(WIN32) || defined(WIN64) || defined(_WIN32) || defined(_WIN32_) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include +#include +#include +#include + +namespace ppc::protocol +{ +// the task type +enum class TaskType : uint8_t +{ + PSI = 0x00, + PIR = 0x01, +}; + +inline std::ostream& operator<<(std::ostream& _out, TaskType const& _type) +{ + switch (_type) + { + case TaskType::PSI: + _out << "PSI"; + break; + case TaskType::PIR: + _out << "PIR"; + break; + default: + _out << "UnknownTaskType"; + break; + } + return _out; +} + +// the PSIAlgorithm type +// TODO: rename PSIAlgorithmType to TaskAlgorithmType +enum class PSIAlgorithmType : uint8_t +{ + // PSI implementation for https://eprint.iacr.org/2020/729.pdf(Private Set Intersection in the + // Internet Setting from Lightweight Oblivious PRF) + CM_PSI_2PC = 0x00, + // PSI implementation for https://eprint.iacr.org/2017/677.pdf(Faster Unbalanced Private Set + // Intersection) + RA_PSI_2PC = 0x01, + // PSI implementation for https://eprint.iacr.org/2021/1116(Labeled PSI from Homomorphic + // Encryption with Reduced Computation and Communication) + LABELED_PSI_2PC = 0x02, + ECDH_PSI_2PC = 0x03, + ECDH_PSI_MULTI = 0x04, + ECDH_PSI_CONN = 0x05, + OT_PIR_2PC = 0x10, + BS_ECDH_PSI = 0x06, +}; + +// enum class PSIAlgorithmType : uint8_t +// { +// OT_PIR_2PC = 0x10, +// }; + +inline std::ostream& operator<<(std::ostream& _out, PSIAlgorithmType const& _type) +{ + switch (_type) + { + case PSIAlgorithmType::CM_PSI_2PC: + _out << "CM_PSI_2PC"; + break; + case PSIAlgorithmType::RA_PSI_2PC: + _out << "RA_PSI_2PC"; + break; + case PSIAlgorithmType::LABELED_PSI_2PC: + _out << "LABELED_PSI_2PC"; + break; + case PSIAlgorithmType::ECDH_PSI_2PC: + _out << "ECDH_PSI_2PC"; + break; + case PSIAlgorithmType::OT_PIR_2PC: + _out << "OT_PIR_2PC"; + break; + case PSIAlgorithmType::ECDH_PSI_MULTI: + _out << "ECDH_PSI_MULTI"; + break; + case PSIAlgorithmType::ECDH_PSI_CONN: + _out << "ECDH_PSI_CONN"; + break; + default: + _out << "UnknownPSIAlgorithm"; + break; + } + return _out; +} + +// the type of the data resource, now support FILE/HDFS, will support SQL in the future +enum class DataResourceType : uint16_t +{ + FILE, + MySQL, + HDFS +}; + +enum class CacheType : uint16_t +{ + Redis +}; + +enum class TaskStatus +{ + PENDING, + RUNNING, + PAUSING, + FAILED, + COMPLETED +}; + +inline bool isExecutable(TaskStatus _status) +{ + return _status == TaskStatus::PENDING || _status == TaskStatus::PAUSING || + _status == TaskStatus::RUNNING; +} + +inline bool isNotExecutable(TaskStatus _status) +{ + return _status == TaskStatus::FAILED || _status == TaskStatus::COMPLETED; +} + +inline std::string toString(TaskStatus _status) +{ + switch (_status) + { + case TaskStatus::PENDING: + return "PENDING"; + case TaskStatus::RUNNING: + return "RUNNING"; + case TaskStatus::PAUSING: + return "PAUSING"; + case TaskStatus::FAILED: + return "FAILED"; + case TaskStatus::COMPLETED: + return "COMPLETED"; + default: + return "UNKNOWN"; + } +} + +inline TaskStatus fromString(const std::string& _str) +{ + static const std::map statusMap = {{"PENDING", TaskStatus::PENDING}, + {"RUNNING", TaskStatus::RUNNING}, {"PAUSING", TaskStatus::PAUSING}, + {"FAILED", TaskStatus::FAILED}, {"COMPLETED", TaskStatus::COMPLETED}}; + + auto it = statusMap.find(_str); + if (it != statusMap.end()) + { + return it->second; + } + else + { + return TaskStatus::FAILED; + } +} + +inline std::ostream& operator<<(std::ostream& _out, DataResourceType const& _type) +{ + switch (_type) + { + case DataResourceType::FILE: + _out << "FILE"; + break; + case DataResourceType::MySQL: + _out << "MySQL"; + break; + case DataResourceType::HDFS: + _out << "HDFS"; + break; + default: + _out << "UnknownDataResourceType"; + break; + } + return _out; +} + +// the party type +// RA involves Server and Client +enum class PartyType : uint16_t +{ + Client, + Server, +}; +inline std::ostream& operator<<(std::ostream& _out, PartyType const& _type) +{ + switch (_type) + { + case PartyType::Client: + _out << "Client"; + break; + case PartyType::Server: + _out << "Server"; + break; + default: + _out << "UnknownPartyType"; + break; + } + return _out; +} + +enum class PartiesType : uint16_t +{ + Calculator, + Partner, + Master, +}; + +inline std::ostream& operator<<(std::ostream& _out, PartiesType const& _type) +{ + switch (_type) + { + case PartiesType::Calculator: + _out << "Calculator"; + break; + case PartiesType::Partner: + _out << "Partner"; + break; + case PartiesType::Master: + _out << "Master"; + break; + default: + _out << "UnknownPartiesType"; + break; + } + return _out; +} + +enum class NodeArch : uint8_t +{ + AIR, // all in one + PRO // gateway and front are independent +}; + +inline std::ostream& operator<<(std::ostream& _out, NodeArch const& _type) +{ + switch (_type) + { + case NodeArch::AIR: + _out << "AIR"; + break; + case NodeArch::PRO: + _out << "PRO"; + break; + default: + _out << "UnknownNodeArch"; + break; + } + return _out; +} + +// define the ppc ret-code +enum PPCRetCode : int +{ + SUCCESS = 0, + EXCEPTION = -100, + TIMEOUT = -101, + DECODE_PPC_MESSAGE_ERROR = -102, + NETWORK_ERROR = -103, + NOTIFY_TASK_ERROR = -104, + REGISTER_GATEWAY_URL_ERROR = -105, + + // the storage error + READ_RPC_STATUS_ERROR = -200, + WRITE_RPC_STATUS_ERROR = -201 +}; + +// the supported ecc-curve +enum class ECCCurve : int8_t +{ + ED25519, + SM2, + SECP256K1, // todo: FourQ + P256, + IPP_X25519, +}; + +// the supported oprf +enum class OprfType : int8_t +{ + EcdhOprf, +}; + +inline std::ostream& operator<<(std::ostream& _out, ECCCurve const& _type) +{ + switch (_type) + { + case ECCCurve::ED25519: + _out << "ED25519"; + break; + case ECCCurve::SM2: + _out << "SM2"; + break; + case ECCCurve::SECP256K1: + _out << "SECP256K1"; + break; + case ECCCurve::P256: + _out << "P256"; + break; + case ECCCurve::IPP_X25519: + _out << "IPP_X25519"; + break; + default: + _out << "UnknownEccCurve"; + break; + } + return _out; +} + +enum class HashImplName : int8_t +{ + SHA256, + SHA512, + SM3, + MD5, + BLAKE2b // TODO: sha3 + // TODO: blake3(https://github.com/BLAKE3-team/BLAKE3) +}; + +enum class PRNGImplName : int8_t +{ + AES, + BLAKE2b, +}; + +enum class SymCryptoImplName : int8_t +{ + AES, + DES_EDE3, + SM4, +}; + +enum class DataPaddingType : int +{ + NONE = 0, + PKCS7 = 1, + ISO7816_4 = 2, + ANSI923 = 3, + ISO10126 = 4, + ZERO = 5, +}; + + +inline std::ostream& operator<<(std::ostream& _out, HashImplName const& _type) +{ + switch (_type) + { + case HashImplName::SHA256: + _out << "SHA256"; + break; + case HashImplName::SHA512: + _out << "SHA512"; + break; + case HashImplName::SM3: + _out << "SM3"; + break; + case HashImplName::MD5: + _out << "MD5"; + break; + case HashImplName::BLAKE2b: + _out << "BLAKE2b"; + break; + default: + _out << "UnknownHashImpl"; + break; + } + return _out; +} + +enum class MessageType : uint16_t +{ + GatewayMessage = 0x0000, + RpcRequest = 0x1000, // the rpc request type +}; + +// option used for sql-connection +struct SQLConnectionOption +{ + using Ptr = std::shared_ptr; + std::string host; + int port; + std::string user; + std::string password; + std::string database; + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("host", host) << LOG_KV("port", port) << LOG_KV("user", user) + << LOG_KV("database", database); + return oss.str(); + } +}; + +//// the hdfs connection-option +struct FileStorageConnectionOption +{ + using Ptr = std::shared_ptr; + std::string nameNode; + uint16_t nameNodePort; + std::string userName; + std::string token; + // replace-datanode-on-failure + bool replaceDataNodeOnFailure = false; + // the default connection-timeout for the hdfs is 1000ms + uint16_t connectionTimeout = 1000; + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("nameNode", nameNode) << LOG_KV("nameNodePort", nameNodePort) + << LOG_KV("user", userName) << LOG_KV("token", token) + << LOG_KV("replace-datanode-on-failure", replaceDataNodeOnFailure) + << LOG_KV("connectionTimeout", connectionTimeout); + return oss.str(); + } +}; + +struct RemoteStorageConnectionOption +{ + using Ptr = std::shared_ptr; + std::string host; + uint16_t port; + std::string legalPerson; + std::string user; + std::string password; + std::string obInitServer; + std::string monitorServer; + uint32_t timeoutSec = 600; + int saveCount = 3; // month + std::string logPath; + + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("host", host) << LOG_KV("port", port) << LOG_KV("legalPerson", legalPerson) + << LOG_KV("user", user) << LOG_KV("password", password) + << LOG_KV("obInitServer", obInitServer) << LOG_KV("monitorServer", monitorServer) + << LOG_KV("timeoutSec", timeoutSec) << LOG_KV("saveCount", saveCount) + << LOG_KV("logPath", logPath); + return oss.str(); + } +}; + +struct GatewayInfo +{ + std::string agencyID; + std::string endpoint; +}; + +struct FileInfo +{ + using Ptr = std::shared_ptr; + std::string path; + std::string bizSeqNo; + std::string fileID; + std::string fileMd5; + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("path", path) << LOG_KV("bizSeqNo", bizSeqNo) << LOG_KV("fileID", fileID) + << LOG_KV("fileMd5", fileMd5); + return oss.str(); + } +}; + +const std::string FRONT_NAME = "Front"; +const std::string FRONT_SERVANT_NAME = "FrontServiceObj"; +const std::string FRONT_SERVICE_NAME = "FrontService"; + +const std::string GATEWAY_NAME = "Gateway"; +const std::string GATEWAY_SERVANT_NAME = "GatewayServiceObj"; +const std::string GATEWAY_SERVICE_NAME = "GatewayService"; + +inline std::string getPrxDesc(std::string const& _serviceName, std::string const& _objName) +{ + return _serviceName + "." + _objName; +} + +} // namespace ppc::protocol diff --git a/cpp/ppc-framework/protocol/Task.h b/cpp/ppc-framework/protocol/Task.h new file mode 100644 index 00000000..727205db --- /dev/null +++ b/cpp/ppc-framework/protocol/Task.h @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PartyResource.h + * @author: yujiechen + * @date 2022-10-13 + */ +#pragma once +#include "PartyResource.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::protocol +{ +struct GatewayTaskInfo +{ + using Ptr = std::shared_ptr; + GatewayTaskInfo(std::string const& _taskID) : taskID(_taskID) {} + GatewayTaskInfo() = default; + + std::string taskID; + // Note: optional for the app-level + std::string serviceEndpoint; +}; + + +// Note: can extend on demand +class TaskResult +{ +public: + using Ptr = std::shared_ptr; + TaskResult(std::string const& _taskID) : m_taskID(_taskID) {} + virtual ~TaskResult() = default; + + virtual std::string const& taskID() const { return m_taskID; } + + virtual std::string const& status() const { return m_status; } + virtual void setStatus(const std::string& _status) { m_status = _status; } + + virtual bcos::Error::Ptr error() const { return m_error; } + virtual void setError(bcos::Error::Ptr _error) { m_error = std::move(_error); } + + virtual uint64_t timeCost() const { return m_timeCost; } + virtual void setTimeCost(uint64_t _cost) { m_timeCost = _cost; } + + virtual FileInfo::Ptr fileInfo() const { return m_fileInfo; } + virtual void setFileInfo(FileInfo::Ptr _fileInfo) { m_fileInfo = std::move(_fileInfo); } + + // serialize the taskResult to json + virtual Json::Value serializeToJson() const + { + Json::Value response; + response["taskID"] = m_taskID; + if (!m_status.empty()) + { + response["status"] = m_status; + } + if (m_timeCost) + { + response["timeCost"] = std::to_string(m_timeCost) + "ms"; + } + if (m_fileInfo && !m_fileInfo->bizSeqNo.empty()) + { + response["bizSeqNo"] = m_fileInfo->bizSeqNo; + response["fileID"] = m_fileInfo->fileID; + response["fileMd5"] = m_fileInfo->fileMd5; + } + if (m_error) + { + response["code"] = m_error->errorCode(); + response["message"] = m_error->errorMessage(); + } + else + { + response["code"] = 0; + response["message"] = "success"; + } + return response; + } + +protected: + std::string m_taskID; + std::string m_status; + uint64_t m_timeCost = 0; + bcos::Error::Ptr m_error = nullptr; + FileInfo::Ptr m_fileInfo = nullptr; +}; + +class Task +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + Task() = default; + virtual ~Task() = default; + + // the taskID + virtual std::string const& id() const = 0; + // the taskType, e.g. PSI + virtual uint8_t type() const = 0; + // the algorithm + virtual uint8_t algorithm() const = 0; + // information of the local party + virtual PartyResource::ConstPtr selfParty() const = 0; + virtual PartyResource::Ptr mutableSelfParty() const = 0; + // information of the peer party + virtual PartyResource::ConstPtr getPartyByID(std::string const& _partyID) const = 0; + virtual PartyResource::ConstPtr getPartyByIndex(uint16_t _partyIndex) const = 0; + // get all peer-parties + virtual std::map const& getAllPeerParties() const = 0; + // params of the task, can be deserialized using json + virtual std::string const& param() const = 0; + // sync the psi result to peer or not + virtual bool syncResultToPeer() const = 0; + virtual bool lowBandwidth() const = 0; + virtual std::vector const& getReceiverLists() const = 0; + + // the setters + virtual void setId(std::string const& _id) = 0; + virtual void setType(uint8_t _type) = 0; + virtual void setAlgorithm(uint8_t _algorithm) = 0; + virtual void setSelf(PartyResource::Ptr _self) = 0; + virtual void addParty(PartyResource::Ptr _party) = 0; + virtual void setParam(std::string const& _param) = 0; + virtual void setSyncResultToPeer(bool _syncResultToPeer) = 0; + virtual void setLowBandwidth(bool _lowBandwidth) = 0; + // decode the task + virtual void decode(std::string_view _taskData) = 0; + virtual std::string encode() const = 0; +}; + +class TaskFactory +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + TaskFactory() = default; + virtual ~TaskFactory() = default; + + virtual Task::Ptr createTask(std::string_view _taskInfo) = 0; + virtual Task::Ptr createTask() = 0; +}; + +inline std::string printTaskInfo(Task::ConstPtr _task) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("id", _task->id()) + << LOG_KV("type", (ppc::protocol::TaskType)_task->type()) + << LOG_KV("algorithm", (ppc::protocol::PSIAlgorithmType)_task->algorithm()) + << LOG_KV("taskPtr", _task); + if (_task->selfParty()) + { + stringstream << printPartyInfo(_task->selfParty()); + } + return stringstream.str(); +} +} // namespace ppc::protocol diff --git a/cpp/ppc-framework/psi/TaskFrameworkInterface.h b/cpp/ppc-framework/psi/TaskFrameworkInterface.h new file mode 100644 index 00000000..12d461bb --- /dev/null +++ b/cpp/ppc-framework/psi/TaskFrameworkInterface.h @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief the PSIFrameworkInterface + * @file PSIFrameworkInterface.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "../protocol/PPCMessageFace.h" +#include "../protocol/Task.h" +#include +#include +namespace ppc::psi +{ +using TaskResponseCallback = std::function; + +class PSIFrameworkInterface +{ +public: + using Ptr = std::shared_ptr; + PSIFrameworkInterface() = default; + virtual ~PSIFrameworkInterface() = default; + + // run task + virtual void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) = 0; + + // register to the front to get the message related to ra2018 + virtual void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) = 0; + + virtual void start() = 0; + virtual void stop() = 0; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-framework/rpc/RpcInterface.h b/cpp/ppc-framework/rpc/RpcInterface.h new file mode 100644 index 00000000..05520e93 --- /dev/null +++ b/cpp/ppc-framework/rpc/RpcInterface.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcInterface.h + * @author: yujiechen + * @date 2022-11-3 + */ +#pragma once +#include "../protocol/Protocol.h" +#include "../protocol/Task.h" +#include +#include +#include +#include +#include + +#define RPC_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("RPC") + +namespace bcos +{ +namespace boostssl +{ +class MessageFace; +class WsSession; +} // namespace boostssl +} // namespace bcos +namespace ppc::rpc +{ +using RespFunc = std::function; +using TaskHandler = std::function)>; +class RpcInterface +{ +public: + using Ptr = std::shared_ptr; + RpcInterface() = default; + virtual ~RpcInterface() = default; + + virtual void start() = 0; + virtual void stop() = 0; + virtual void registerHandler(std::string const& _methodName, + std::function _handler) = 0; + + virtual void registerTaskHandler( + ppc::protocol::TaskType _type, uint8_t _algorithm, TaskHandler _handler) = 0; +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-framework/rpc/RpcStatusInterface.h b/cpp/ppc-framework/rpc/RpcStatusInterface.h new file mode 100644 index 00000000..441a08a2 --- /dev/null +++ b/cpp/ppc-framework/rpc/RpcStatusInterface.h @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcStatusInterface.h + * @author: shawnhe + * @date 2023-02-19 + */ + +#pragma once +#include "../protocol/Protocol.h" +#include "../protocol/Task.h" +#include "../storage/SQLStorage.h" +#include +#include +#include +#include + +#define RPC_STATUS_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("RpcStatus") + +namespace ppc::rpc +{ + +class RpcStatusInterface +{ +public: + using Ptr = std::shared_ptr; + RpcStatusInterface() = default; + virtual ~RpcStatusInterface() = default; + + virtual void start() = 0; + virtual void stop() = 0; + + virtual bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) = 0; + virtual bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) = 0; + virtual protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) = 0; + virtual bcos::Error::Ptr insertGateway( + const std::string& _agencyID, const std::string& _endpoint) = 0; + virtual bcos::Error::Ptr deleteGateway(const std::string& _agencyID) = 0; + virtual std::vector listGateway() = 0; +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-framework/rpc/RpcTypeDef.h b/cpp/ppc-framework/rpc/RpcTypeDef.h new file mode 100644 index 00000000..3be4ee05 --- /dev/null +++ b/cpp/ppc-framework/rpc/RpcTypeDef.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcTypeDef.h + * @author: yujiechen + * @date 2022-11-3 + */ +#pragma once +#include + +namespace ppc::rpc +{ +enum class RpcError : int32_t +{ + LackOfMemory = -33, + InvalidRequest = -32600, + MethodNotFound = -32601, + NotImplemented = -32602, + StorageNotSet = -32603, + NoPermission = -32604, + BsModePsiNotSet = -32605, +}; + +std::string const RUN_TASK_METHOD = "runTask"; +std::string const ASYNC_RUN_TASK_METHOD = "asyncRunTask"; +std::string const GET_TASK_STATUS = "getTaskStatus"; +std::string const REGISTER_GATEWAY_URL = "registerGatewayUrl"; + +std::string const ASYNC_RUN_BS_MODE_TASK = "asyncRunBsModeTask"; +std::string const FETCH_CIPHER = "fetchCipher"; +std::string const SEND_ECDH_CIPHER = "sendEcdhCipher"; +std::string const SEND_PARTNER_CIPHER = "sendPartnerCipher"; +std::string const GET_BS_MODE_TASK_STATUS = "getBsModeTaskStatus"; +std::string const KILL_BS_MODE_TASK = "killBsModeTask"; +std::string const UPDATE_BS_MODE_TASK_STATUS = "updateBsModeTaskStatus"; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-framework/storage/CacheStorage.h b/cpp/ppc-framework/storage/CacheStorage.h new file mode 100644 index 00000000..96da2e02 --- /dev/null +++ b/cpp/ppc-framework/storage/CacheStorage.h @@ -0,0 +1,114 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file StorageCache.h + * @author: shawnhe + * @date 2022-10-21 + */ + +#pragma once +#include "ppc-framework/protocol/Protocol.h" +#include +#include +#include +#include +#include + +namespace ppc::storage +{ +struct CacheStorageConfig +{ + ppc::protocol::CacheType type; + std::string proxy; + std::string obServer; + std::string cluster; + std::string user; + std::string host; + uint16_t port; + std::string password; + uint16_t database; + uint16_t pool; + // default connection-timeout is 500ms + uint16_t connectionTimeout = 500; + // default socket-timeout is 500ms + uint16_t socketTimeout = 500; + + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("type", (int)type) << LOG_KV("proxy", proxy) << LOG_KV("obServer", obServer) + << LOG_KV("cluster", cluster) << LOG_KV("host", host) << LOG_KV("port", port) + << LOG_KV("database", database) << LOG_KV("poolSize", pool) + << LOG_KV("connectionTimeout", connectionTimeout) + << LOG_KV("socketTimeout", socketTimeout); + return oss.str(); + } +}; + +class CacheStorage +{ +public: + using Ptr = std::shared_ptr; + CacheStorage() = default; + virtual ~CacheStorage() {} + + /** + * @brief: check whether the key exists + * @param _key: key + * @return whether the key exists + */ + virtual bool exists(const std::string& _key) = 0; + + /** + * @brief: set key value + * @param _expirationTime: timeout of key, in seconds, when set to -1, never expire + */ + virtual void setValue( + const std::string& _key, const std::string& _value, int32_t _expirationSeconds = -1) = 0; + + /** + * @brief: get value by key + * @param _key: key + * @return std::nullopt if the key doesn't exist; + */ + virtual std::optional getValue(const std::string& _key) = 0; + + /** + * @brief: set a timeout on key + * @param _expirationTime: timeout of key, seconds + * @return whether setting is successful + */ + virtual bool expireKey(const std::string& _key, uint32_t _expirationSeconds) = 0; + + /** + * @brief: delete key + * @param _key: key + * @return the number of key deleted + */ + virtual uint64_t deleteKey(const std::string& _key) = 0; +}; + + +class CacheStorageFactory +{ +public: + using Ptr = std::shared_ptr; + CacheStorageFactory() = default; + virtual ~CacheStorageFactory() = default; + + virtual CacheStorage::Ptr createCacheStorage(CacheStorageConfig const& _config) = 0; +}; + +} // namespace ppc::storage diff --git a/cpp/ppc-framework/storage/FileStorage.h b/cpp/ppc-framework/storage/FileStorage.h new file mode 100644 index 00000000..beccc85c --- /dev/null +++ b/cpp/ppc-framework/storage/FileStorage.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileStorage.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "../protocol/Protocol.h" +#include + +namespace ppc::storage +{ +class FileHandler +{ +public: + using Ptr = std::shared_ptr; + FileHandler() = default; + virtual ~FileHandler() = default; + + virtual void* fileInfoHandler() const = 0; + virtual void* storageHandler() const = 0; + virtual std::string const& path() const = 0; +}; + +class FileStorage +{ +public: + using Ptr = std::shared_ptr; + FileStorage() = default; + virtual ~FileStorage() = default; + + // create the directory + virtual void createDirectory(std::string const& _dirPath) const = 0; + // open the file + virtual FileHandler::Ptr openFile( + std::string const& _path, bool _createIfNotExists = false) const = 0; + // close the file + virtual void closeFile(FileHandler::Ptr _handler) const = 0; + + // delete the specified file + virtual void deleteFile(std::string const& _path) const = 0; + + // check the specified file exists or not + virtual bool fileExists(std::string const& _path) const = 0; + + virtual void renameFile(std::string const& _old_path, std::string const& _new_path) const = 0; +}; + +class FileStorageFactory +{ +public: + using Ptr = std::shared_ptr; + FileStorageFactory() = default; + virtual ~FileStorageFactory() = default; + + virtual FileStorage::Ptr createFileStorage(ppc::protocol::DataResourceType _type, + ppc::protocol::FileStorageConnectionOption::Ptr const& _option) = 0; +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-framework/storage/RemoteStorage.h b/cpp/ppc-framework/storage/RemoteStorage.h new file mode 100644 index 00000000..16906016 --- /dev/null +++ b/cpp/ppc-framework/storage/RemoteStorage.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RemoteStorage.h + * @author: shawnhe + * @date 2023-1-29 + */ +#pragma once +#include "../protocol/Protocol.h" +#include + +namespace ppc::storage +{ + +class RemoteStorage +{ +public: + using Ptr = std::shared_ptr; + RemoteStorage() = default; + virtual ~RemoteStorage() = default; + + virtual protocol::FileInfo::Ptr upload(std::string const& _path) = 0; + virtual void download(protocol::FileInfo::Ptr _fileInfo) = 0; +}; + +class RemoteStorageFactory +{ +public: + using Ptr = std::shared_ptr; + RemoteStorageFactory() = default; + virtual ~RemoteStorageFactory() = default; + + virtual RemoteStorage::Ptr createRemoteStorage(ppc::protocol::DataResourceType _type, + ppc::protocol::RemoteStorageConnectionOption::Ptr const& _option) = 0; +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-framework/storage/SQLStorage.h b/cpp/ppc-framework/storage/SQLStorage.h new file mode 100644 index 00000000..90f3cd60 --- /dev/null +++ b/cpp/ppc-framework/storage/SQLStorage.h @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SQLStorage.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "../io/DataBatch.h" +#include "ppc-framework/protocol/Protocol.h" +#include +#include +namespace ppc::storage +{ +enum FieldDataType : int +{ + SINT = 0, + UINT = 1, + DOUBLE = 2, + FLOAT = 3, + BYTES = 4, + STRING = 5, + TERMINATE, +}; + +using MetaFieldType = std::vector; +class QueryResult +{ +public: + using Ptr = std::shared_ptr; + QueryResult() = default; + virtual ~QueryResult() = default; + + MetaFieldType const& metaData() const { return m_metaData; } + std::vector const& data() const { return m_data; } + + MetaFieldType& mutableMetaData() { return m_metaData; } + std::vector& mutableData() { return m_data; } + + uint64_t size() const { return m_data.size(); } + +private: + // the meta-data + MetaFieldType m_metaData; + // the data + std::vector m_data; +}; + +class Statement +{ +public: + using Ptr = std::shared_ptr; + Statement() = default; + virtual ~Statement() = default; + // get the statementContext + virtual void* statementContext() = 0; + virtual void* takeSession() = 0; +}; + +class SQLResultContext +{ +public: + using Ptr = std::shared_ptr; + SQLResultContext() = default; + virtual ~SQLResultContext() = default; + + // get the sql-result + virtual void* result() = 0; +}; + +// Only the synchronous interface is provided here +class SQLStorage +{ +public: + using Ptr = std::shared_ptr; + + SQLStorage() = default; + virtual ~SQLStorage() = default; + + // use the database + virtual void useDataBase(const char* _database) = 0; + + // exec the sql-query-command + virtual QueryResult::Ptr execQuery(bool _parseByColumn, const char* _command, ...) const = 0; + // exec the sql-write-command: insert/update/delete + virtual void execCommit(const char* _command, ...) const = 0; + + virtual Statement::Ptr generateStatement(const char* _command, ...) const = 0; + virtual SQLResultContext::Ptr execStatement(Statement::Ptr _statement) const = 0; + virtual void appendStatement(Statement::Ptr _statement, ...) const = 0; + virtual std::string const& databaseName() const = 0; +}; + +class SQLStorageFactory +{ +public: + using Ptr = std::shared_ptr; + SQLStorageFactory() = default; + virtual ~SQLStorageFactory() = default; + + virtual SQLStorage::Ptr createSQLStorage(ppc::protocol::DataResourceType _type, + ppc::protocol::SQLConnectionOption::Ptr const& _option) = 0; +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-framework/task/TaskFrameworkInterface.h b/cpp/ppc-framework/task/TaskFrameworkInterface.h new file mode 100644 index 00000000..437d94bf --- /dev/null +++ b/cpp/ppc-framework/task/TaskFrameworkInterface.h @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief the TaskFrameworkInterface + * @file TaskFrameworkInterface.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "../protocol/PPCMessageFace.h" +#include "../protocol/Task.h" +#include +#include +namespace ppc::task +{ +using TaskResponseCallback = std::function; + +class TaskFrameworkInterface +{ +public: + using Ptr = std::shared_ptr; + TaskFrameworkInterface() = default; + virtual ~TaskFrameworkInterface() = default; + + // run task + virtual void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) = 0; + + // register to the front to get the message related to ra2018 + virtual void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) = 0; + + virtual void start() = 0; + virtual void stop() = 0; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-front/CMakeLists.txt b/cpp/ppc-front/CMakeLists.txt new file mode 100644 index 00000000..71e64d23 --- /dev/null +++ b/cpp/ppc-front/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.14) + +aux_source_directory(ppc-front SRCS) + + +add_library(${FRONT_TARGET} ${SRCS}) +target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TCMalloc) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(test) +endif () \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/Common.h b/cpp/ppc-front/ppc-front/Common.h new file mode 100644 index 00000000..677ed467 --- /dev/null +++ b/cpp/ppc-front/ppc-front/Common.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-10-25 + */ + +#pragma once + +#include "ppc-framework/Common.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ppc::front +{ + +#define FRONT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[FRONT]" + +#define HOLDING_MESSAGE_TIMEOUT_M 30 +#define CHANNEL_THREAD_POOL_MODULE "t_channel_manager" + +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/Front.cpp b/cpp/ppc-front/ppc-front/Front.cpp new file mode 100644 index 00000000..4fa711c0 --- /dev/null +++ b/cpp/ppc-front/ppc-front/Front.cpp @@ -0,0 +1,365 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Front.cpp + * @author: shawnhe + * @date 2022-10-20 + */ + +#include "Front.h" +#include +#include +#include +#include + +using namespace bcos; +using namespace ppc::front; +using namespace ppc::protocol; + +void Front::start() +{ + if (m_running) + { + FRONT_LOG(INFO) << LOG_DESC("Front has already been started"); + return; + } + m_running = true; + FRONT_LOG(INFO) << LOG_DESC("start the Front"); + m_thread = std::make_shared([&] { + bcos::pthread_setThreadName("front_io_service"); + while (m_running) + { + try + { + m_ioService->run(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (m_running && m_ioService->stopped()) + { + m_ioService->restart(); + } + } + catch (std::exception& e) + { + FRONT_LOG(WARNING) + << LOG_DESC("Exception in Front Thread:") << boost::diagnostic_information(e); + } + } + FRONT_LOG(INFO) << "Front exit"; + }); +} + + +/** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ +void Front::onReceiveMessage(PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) +{ + if (_callback) + { + if (m_threadPool) + { + m_threadPool->enqueue([_callback] { _callback(nullptr); }); + } + else + { + _callback(nullptr); + } + } + + FRONT_LOG(TRACE) << LOG_BADGE("onReceiveMessage") << printPPCMsg(_message); + // response package + if (_message->response()) + { + handleCallback(nullptr, _message->uuid(), _message, _message->sender()); + return; + } + + uint16_t type = ((uint16_t)_message->taskType() << 8) | _message->algorithmType(); + auto it = m_handlers.find(type); + if (it != m_handlers.end()) + { + if (m_threadPool) + { + auto handler = it->second; + m_threadPool->enqueue([handler, _message] { handler(_message); }); + } + else + { + it->second(std::move(_message)); + } + } + else + { + FRONT_LOG(WARNING) << LOG_BADGE("onReceiveMessage") << LOG_DESC("message handler not found") + << LOG_KV("taskType", unsigned(_message->taskType())) + << LOG_KV("algorithmType", unsigned(_message->algorithmType())) + << LOG_KV("messageType", unsigned(_message->messageType())) + << LOG_KV("seq", _message->seq()) << LOG_KV("taskID", _message->taskID()) + << LOG_KV("sender", _message->sender()); + } +} + + +/** + * @brief: send message to other party by gateway + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @return void + */ +void Front::asyncSendMessage(const std::string& _agencyID, PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) +{ + // generate uuid for the message + static thread_local auto uuid_gen = boost::uuids::basic_random_generator(); + std::string uuid = boost::uuids::to_string(uuid_gen()); + _message->setUuid(uuid); + // call gateway interface to send the message + _message->setSender(m_selfAgencyId); + auto taskID = _message->taskID(); + FRONT_LOG(TRACE) << LOG_BADGE("asyncSendMessage") + << LOG_KV("taskType", unsigned(_message->taskType())) + << LOG_KV("algorithmType", unsigned(_message->algorithmType())) + << LOG_KV("messageType", unsigned(_message->messageType())) + << LOG_KV("seq", _message->seq()) << LOG_KV("taskID", _message->taskID()) + << LOG_KV("receiver", _agencyID) << LOG_KV("uud", _message->uuid()); + // timeout logic + if (_respCallback) + { + auto callback = std::make_shared(_respCallback); + addCallback(uuid, callback); + if (_timeout > 0) + { + auto timeoutHandler = std::make_shared( + *m_ioService, boost::posix_time::milliseconds(_timeout)); + callback->timeoutHandler = timeoutHandler; + auto self = weak_from_this(); + timeoutHandler->async_wait( + [self, _agencyID, taskID, uuid](const boost::system::error_code& e) { + auto front = self.lock(); + if (front) + { + front->onMessageTimeout(e, _agencyID, taskID, uuid); + } + }); + } + } + auto self = weak_from_this(); + sendMessageToGateway(_agencyID, std::move(_message), uuid, false, + [self, uuid, _agencyID, taskID, _callback](const bcos::Error::Ptr& _error) { + auto front = self.lock(); + if (!front) + { + return; + } + // send message to gateway error, try to handleCallback + if (_error && (_error->errorCode() != 0)) + { + front->handleCallback(_error, uuid, nullptr, _agencyID); + } + if (_callback) + { + _callback(_error); + } + }); +} + +void Front::handleCallback(bcos::Error::Ptr const& _error, std::string const& _uuid, + PPCMessageFace::Ptr _message, std::string const& _agencyID) +{ + auto callback = getAndEraseCallback(_uuid); + if (!callback) + { + return; + } + // cancel the timer + if (callback->timeoutHandler) + { + callback->timeoutHandler->cancel(); + } + if (!_message) + { + return; + } + auto self = weak_from_this(); + auto respFunc = [self, _agencyID, _uuid](PPCMessageFace::Ptr _resp) { + auto front = self.lock(); + if (!front) + { + return; + } + front->sendMessageToGateway(front->m_selfAgencyId, std::move(_resp), _uuid, true, + [_agencyID](const bcos::Error::Ptr& _error) { + if (!_error) + { + return; + } + FRONT_LOG(WARNING) + << LOG_DESC("asyncSendResponse message error") << LOG_KV("agency", _agencyID) + << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); + }); + }; + if (m_threadPool) + { + m_threadPool->enqueue([_error, callback, _message, _agencyID, respFunc] { + callback->callback(_error, _agencyID, _message, respFunc); + }); + } + else + { + callback->callback(_error, _agencyID, std::move(_message), std::move(respFunc)); + } +} + +void Front::sendMessageToGateway(std::string const& _agencyID, PPCMessageFace::Ptr _message, + std::string const& _uuid, bool _response, ErrorCallbackFunc _callback) +{ + _message->setSender(m_selfAgencyId); + _message->setUuid(_uuid); + if (_response) + { + _message->setResponse(); + } + m_gatewayInterface->asyncSendMessage(_agencyID, std::move(_message), std::move(_callback)); +} + +// send response +void Front::asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) +{ + FRONT_LOG(TRACE) << LOG_DESC("asyncSendResponse") << printPPCMsg(_message); + sendMessageToGateway(_agencyID, std::move(_message), _uuid, true, std::move(_callback)); +} + +void Front::onMessageTimeout(const boost::system::error_code& e, std::string const& _agencyID, + std::string const& _taskID, std::string const& _uuid) +{ + // the timer has been canceled + if (e) + { + return; + } + + try + { + auto callback = getAndEraseCallback(_uuid); + if (!callback) + { + return; + } + if (callback->timeoutHandler) + { + callback->timeoutHandler->cancel(); + } + auto errorMsg = "send message with uuid=" + _uuid + ", agency = " + _agencyID + + ", task = " + _taskID + " timeout"; + auto error = std::make_shared(PPCRetCode::TIMEOUT, errorMsg); + if (m_threadPool) + { + m_threadPool->enqueue([callback, _agencyID, error]() { + callback->callback(error, _agencyID, nullptr, nullptr); + }); + } + else + { + callback->callback(std::move(error), _agencyID, nullptr, nullptr); + } + FRONT_LOG(WARNING) << LOG_BADGE("onMessageTimeout") << LOG_KV("uuid", _uuid) + << LOG_KV("agency", _agencyID) << LOG_KV("task", _taskID); + } + catch (std::exception& e) + { + FRONT_LOG(ERROR) << "onMessageTimeout" << LOG_KV("uuid", _uuid) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + + +/** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ +bcos::Error::Ptr Front::notifyTaskInfo(GatewayTaskInfo::Ptr _taskInfo) +{ + auto startT = bcos::utcSteadyTime(); + if (_taskInfo->serviceEndpoint.empty()) + { + _taskInfo->serviceEndpoint = m_selfEndPoint; + } + auto ret = m_gatewayInterface->notifyTaskInfo(_taskInfo); + FRONT_LOG(INFO) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", _taskInfo->taskID) + << LOG_KV("serviceEndpoint", _taskInfo->serviceEndpoint) + << LOG_KV("timecost", bcos::utcSteadyTime() - startT); + return ret; +} + +// erase the task-info when task finished +bcos::Error::Ptr Front::eraseTaskInfo(std::string const& _taskID) +{ + auto startT = bcos::utcSteadyTime(); + auto ret = m_gatewayInterface->eraseTaskInfo(_taskID); + FRONT_LOG(INFO) << LOG_BADGE("eraseTaskInfo") << LOG_KV("taskID", _taskID) + << LOG_KV("timecost", bcos::utcSteadyTime() - startT); + return ret; +} + +// get the agencyList from the gateway +void Front::asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) +{ + FRONT_LOG(TRACE) << LOG_BADGE("asyncGetAgencyList"); + if (!m_gatewayInterface) + { + std::vector emptyAgencies; + _callback(std::make_shared( + -1, "asyncGetAgencyList failed for the gateway not been inited into front!"), + std::move(emptyAgencies)); + return; + } + m_gatewayInterface->asyncGetAgencyList(std::move(_callback)); +} + +void Front::stop() +{ + if (!m_running) + { + FRONT_LOG(INFO) << LOG_DESC("Front has already been stopped"); + return; + } + m_running = false; + if (m_ioService) + { + m_ioService->stop(); + } + if (m_thread) + { + // stop the thread + if (m_thread->get_id() != std::this_thread::get_id()) + { + m_thread->join(); + } + else + { + m_thread->detach(); + } + } +} + + +Front::Ptr FrontFactory::buildFront(std::shared_ptr _ioService) +{ + FRONT_LOG(INFO) << LOG_BADGE("buildFront") << LOG_KV("agencyID", m_selfAgencyId); + return std::make_shared(std::move(_ioService), m_selfAgencyId, m_threadPool); +} \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/Front.h b/cpp/ppc-front/ppc-front/Front.h new file mode 100644 index 00000000..0796b195 --- /dev/null +++ b/cpp/ppc-front/ppc-front/Front.h @@ -0,0 +1,190 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Front.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "Common.h" +#include "FrontService.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" +#include "ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h" +#include +#include + + +#include + +namespace ppc +{ +namespace front +{ +struct Callback +{ + using Ptr = std::shared_ptr; + Callback(CallbackFunc _callback) : callback(std::move(_callback)) {} + std::shared_ptr timeoutHandler; + CallbackFunc callback; +}; +class Front : public front::FrontInterface, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + Front(std::shared_ptr _ioService, std::string const& _selfAgencyId, + bcos::ThreadPool::Ptr _threadPool, std::string const& _selfEndPoint = "localhost") + : m_ioService(std::move(_ioService)), + m_selfAgencyId(_selfAgencyId), + m_threadPool(std::move(_threadPool)) + { + m_selfEndPoint = _selfEndPoint; + } + + Front(const Front&) = delete; + Front(Front&&) = delete; + + Front& operator=(const Front&) = delete; + Front& operator=(Front&&) = delete; + + virtual ~Front() override = default; + + void start() override; + void stop() override; + + /** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ + void onReceiveMessage( + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override; + + + /** + * @brief: send message to other party by gateway + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @return void + */ + void asyncSendMessage(const std::string& _agencyID, PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) override; + + void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override; + + /** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ + bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) override; + // erase the task-info when task finished + bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override; + + // get the agencyList from the gateway + void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override; + + // register message handler for algorithm + void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, + std::function _handler) + { + uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; + m_handlers[type] = std::move(_handler); + } + + ppc::gateway::GatewayInterface::Ptr gatewayInterface() { return m_gatewayInterface; } + void setGatewayInterface(ppc::gateway::GatewayInterface::Ptr _gatewayInterface) + { + m_gatewayInterface = std::move(_gatewayInterface); + } + + std::shared_ptr threadPool() const { return m_threadPool; } + const std::string& selfAgencyId() const { return m_selfAgencyId; } + // Note: the selfEndPoint must be setted before start the front + virtual void setSelfEndPoint(std::string const& _selfEndPoint) + { + FRONT_LOG(INFO) << LOG_DESC("setSelfEndPoint: ") << _selfEndPoint; + m_selfEndPoint = _selfEndPoint; + } + +private: + void addCallback(std::string const& _uuid, Callback::Ptr _callback) + { + bcos::WriteGuard l(x_uuidToCallback); + m_uuidToCallback[_uuid] = std::move(_callback); + } + + Callback::Ptr getAndEraseCallback(std::string const& _uuid) + { + bcos::UpgradableGuard l(x_uuidToCallback); + auto it = m_uuidToCallback.find(_uuid); + if (it != m_uuidToCallback.end()) + { + auto callback = it->second; + bcos::UpgradeGuard ul(l); + m_uuidToCallback.erase(it); + return callback; + } + return nullptr; + } + + void onMessageTimeout(const boost::system::error_code& e, std::string const& _agencyID, + std::string const& _taskID, std::string const& _uuid); + void handleCallback(bcos::Error::Ptr const& _error, std::string const& _uuid, + PPCMessageFace::Ptr _message, std::string const& _agencyID); + void sendMessageToGateway(std::string const& _agencyID, PPCMessageFace::Ptr _msg, + std::string const& _uuid, bool _response, ErrorCallbackFunc _callback); + +private: + std::shared_ptr m_ioService; + std::string m_selfAgencyId; + bcos::ThreadPool::Ptr m_threadPool; + + // gatewayInterface + ppc::gateway::GatewayInterface::Ptr m_gatewayInterface; + std::unordered_map> m_handlers; + + // uuid->callback + std::unordered_map m_uuidToCallback; + bcos::SharedMutex x_uuidToCallback; + + bool m_running = false; + // the thread to run ioservice + std::shared_ptr m_thread; +}; + +class FrontFactory +{ +public: + using Ptr = std::shared_ptr; + FrontFactory(std::string _selfAgencyId, std::shared_ptr _threadPool) + : m_selfAgencyId(std::move(_selfAgencyId)), m_threadPool(std::move(_threadPool)) + {} + +public: + Front::Ptr buildFront(std::shared_ptr _ioService); + +private: + std::string m_selfAgencyId; + // thread pool + std::shared_ptr m_threadPool; +}; + +} // namespace front +} // namespace ppc diff --git a/cpp/ppc-front/ppc-front/PPCChannel.cpp b/cpp/ppc-front/ppc-front/PPCChannel.cpp new file mode 100644 index 00000000..445ac80f --- /dev/null +++ b/cpp/ppc-front/ppc-front/PPCChannel.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCChannel.cpp + * @author: shawnhe + * @date 2022-10-27 + */ + +#include "PPCChannel.h" + +using namespace bcos; +using namespace ppc::front; +using namespace ppc::protocol; + + +/** + * @brief: receive message, note that one message will consume one handler + * @param _messageType: the message type is defined by each crypto algorithm + * @param _seq: message seq + * @param _secondsTimeout: timeout by seconds + * @return void + */ +void PPCChannel::asyncReceiveMessage(uint8_t _messageType, uint32_t _seq, uint32_t _secondsTimeout, + std::function _handler) +{ + if (!_handler) + { + return; + } + + WriteGuard lock(x_handler_message); + + auto message = getAndRemoveMessage(messageKey(_messageType, _seq)); + if (message) + { + // trigger handler by message + handleCallback(nullptr, message, std::move(_handler)); + } + else + { + auto messageHandler = std::make_shared(); + messageHandler->handler = std::move(_handler); + + if (_secondsTimeout == 0) + { + _secondsTimeout = HOLDING_MESSAGE_TIMEOUT_M * 60; + } + + // create timer to handle timeout + messageHandler->timer = std::make_shared( + *m_ioService, boost::posix_time::seconds(_secondsTimeout)); + + messageHandler->timer->async_wait( + [self = weak_from_this(), _messageType, _seq](boost::system::error_code _error) { + if (!_error) + { + auto channel = self.lock(); + if (channel) + { + // remove timeout handler + auto timeoutHandler = + channel->getAndRemoveHandler(channel->messageKey(_messageType, _seq)); + + if (timeoutHandler) + { + FRONT_LOG(DEBUG) << LOG_BADGE("channelWaitingMsgTimeout") + << LOG_KV("messageType", (int)_messageType); + + // trigger handler by error + auto error = std::make_shared( + PPCRetCode::TIMEOUT, "timeout waiting for message"); + auto handler = timeoutHandler->handler; + channel->handleCallback(error, nullptr, std::move(handler)); + } + } + } + }); + + addHandler(messageKey(_messageType, _seq), std::move(messageHandler)); + } +} + + +/** + * @brief: used for front to dispatch message + * @param _messageType: the message type is defined by each crypto algorithm + * @return void + */ +void PPCChannel::onMessageArrived(uint8_t _messageType, PPCMessageFace::Ptr _message) +{ + WriteGuard lock(x_handler_message); + + auto seq = _message->seq(); + auto messageHandler = getAndRemoveHandler(messageKey(_messageType, seq)); + if (messageHandler) + { + // clear timer + if (messageHandler->timer) + { + messageHandler->timer->cancel(); + } + + // trigger event by buffer + handleCallback(nullptr, std::move(_message), std::move(messageHandler->handler)); + } + else + { + addMessage(messageKey(_messageType, seq), std::move(_message)); + } +} + + +void PPCChannel::addHandler(uint64_t _messageKey, MessageHandler::Ptr _handler) +{ + WriteGuard lock(x_handlers); + m_handlers[_messageKey] = std::move(_handler); +} + + +PPCChannel::MessageHandler::Ptr PPCChannel::getAndRemoveHandler(uint64_t _messageKey) +{ + WriteGuard lock(x_handlers); + auto it = m_handlers.find(_messageKey); + if (it == m_handlers.end()) + { + return nullptr; + } + + auto ret = std::move(it->second); + m_handlers.erase(_messageKey); + return ret; +} + + +void PPCChannel::addMessage(uint64_t _messageKey, PPCMessageFace::Ptr _message) +{ + m_messages[_messageKey] = std::move(_message); +} + + +PPCMessageFace::Ptr PPCChannel::getAndRemoveMessage(uint64_t _messageKey) +{ + auto it = m_messages.find(_messageKey); + if (it == m_messages.end()) + { + return nullptr; + } + + auto ret = std::move(it->second); + m_messages.erase(_messageKey); + return ret; +} + + +void PPCChannel::handleCallback(bcos::Error::Ptr _error, PPCMessageFace::Ptr _message, + std::function&& _callback) +{ + if (!_callback) + { + return; + } + + if (m_threadPool) + { + m_threadPool->enqueue([_error, _message, _callback]() { _callback(_error, _message); }); + } + else + { + _callback(std::move(_error), std::move(_message)); + } +} diff --git a/cpp/ppc-front/ppc-front/PPCChannel.h b/cpp/ppc-front/ppc-front/PPCChannel.h new file mode 100644 index 00000000..8b8ec506 --- /dev/null +++ b/cpp/ppc-front/ppc-front/PPCChannel.h @@ -0,0 +1,128 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCChannel.h + * @author: shawnhe + * @date 2022-10-27 + */ + +#pragma once + +#include "Common.h" +#include "Front.h" +#include "ppc-framework/front/Channel.h" +#include +#include + +namespace ppc::front +{ +class PPCChannel : public Channel, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + PPCChannel(std::shared_ptr _ioService, front::Front::Ptr _front, + std::shared_ptr _threadPool) + : m_ioService(std::move(_ioService)), + m_front(std::move(_front)), + m_threadPool(std::move(_threadPool)) + {} + + ~PPCChannel() override + { + std::unordered_map().swap(m_handlers); + std::unordered_map().swap(m_messages); + MallocExtension::instance()->ReleaseFreeMemory(); + FRONT_LOG(INFO) << LOG_DESC("the PPCChannel destroyed"); + } + +public: + /** + * @brief notice task info to gateway by front + * @param _taskInfo the latest task information + */ + bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) override + { + return m_front->notifyTaskInfo(std::move(_taskInfo)); + }; + + /** + * @brief: send message + * @param _agencyID: message receiver + * @return void + */ + void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) override + { + m_front->asyncSendMessage(_agencyID, std::move(_message), _timeout, std::move(_callback), + std::move(_respCallback)); + } + + /** + * @brief: receive message, note that one message will consume one handler + * @param _messageType: the message type is defined by each crypto algorithm + * @param _seq: message seq + * @param _secondsTimeout: timeout by seconds + * @return void + */ + void asyncReceiveMessage(uint8_t _messageType, uint32_t _seq, uint32_t _secondsTimeout, + std::function _handler) override; + + + /** + * @brief: used for front to dispatch message + * @param _messageType: the message type is defined by each crypto algorithm + * @return void + */ + void onMessageArrived(uint8_t _messageType, front::PPCMessageFace::Ptr _message) override; + + + static inline uint64_t messageKey(uint8_t _messageType, uint32_t _seq) + { + uint64_t key = ((uint64_t)_messageType << 32) | _seq; + return key; + } + +protected: + struct MessageHandler + { + using Ptr = std::shared_ptr; + std::function handler; + std::shared_ptr timer; + }; + +private: + void addHandler(uint64_t _messageKey, MessageHandler::Ptr _handler); + MessageHandler::Ptr getAndRemoveHandler(uint64_t _messageKey); + + void addMessage(uint64_t _messageKey, front::PPCMessageFace::Ptr _message); + front::PPCMessageFace::Ptr getAndRemoveMessage(uint64_t _messageKey); + + void handleCallback(bcos::Error::Ptr _error, front::PPCMessageFace::Ptr _message, + std::function&& _callback); + +private: + std::shared_ptr m_ioService; + Front::Ptr m_front; + std::shared_ptr m_threadPool; + + mutable boost::shared_mutex x_handlers; + // key: messageType || seq + std::unordered_map m_handlers; + std::unordered_map m_messages; + + // coordinate handler and message + mutable boost::shared_mutex x_handler_message; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/PPCChannelManager.cpp b/cpp/ppc-front/ppc-front/PPCChannelManager.cpp new file mode 100644 index 00000000..897e902d --- /dev/null +++ b/cpp/ppc-front/ppc-front/PPCChannelManager.cpp @@ -0,0 +1,138 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCChannelManager.cpp + * @author: shawnhe + * @date 2022-10-27 + */ + + +#include "PPCChannelManager.h" + +#include + +using namespace bcos; +using namespace ppc::front; +using namespace ppc::protocol; + +void PPCChannelManager::registerMsgHandlerForChannel(uint8_t _taskType, uint8_t _algorithmType) +{ + FRONT_LOG(INFO) << LOG_BADGE("registerMsgHandlerForChannel") << LOG_KV("taskType", _taskType) + << LOG_KV("algorithmType", _algorithmType); + m_front->registerMessageHandler( + _taskType, _algorithmType, [self = weak_from_this()](PPCMessageFace::Ptr _message) { + auto channelManager = self.lock(); + if (channelManager) + { + channelManager->onMessageArrived(std::move(_message)); + } + }); +} + + +Channel::Ptr PPCChannelManager::buildChannelForTask(const std::string& _taskID) +{ + FRONT_LOG(INFO) << LOG_BADGE("buildChannelForTask") << LOG_KV("taskID", _taskID); + + WriteGuard lock(x_message_channel); + + // new a channel + auto channel = std::make_shared(m_ioService, m_front, m_threadPool); + m_channels[_taskID] = channel; + + // check to see if any message has arrived + auto it = m_holdingMessages.find(_taskID); + if (it != m_holdingMessages.end()) + { + auto holdingMessages = it->second; + if (holdingMessages->timer) + { + holdingMessages->timer->cancel(); + } + for (auto& msg : holdingMessages->messages) + { + channel->onMessageArrived(msg->messageType(), msg); + } + + // remove holding message + m_holdingMessages.erase(_taskID); + } + + return std::static_pointer_cast(channel); +} + + +void PPCChannelManager::removeChannelByTask(const std::string& _taskID) +{ + FRONT_LOG(INFO) << LOG_BADGE("removeChannelByTask") << LOG_KV("taskID", _taskID); + + WriteGuard lock(x_message_channel); + m_channels.erase(_taskID); +} + + +void PPCChannelManager::onMessageArrived(PPCMessageFace::Ptr _message) +{ + WriteGuard lock(x_message_channel); + + auto taskID = _message->taskID(); + + auto itC = m_channels.find(taskID); + if (itC != m_channels.end()) + { + itC->second->onMessageArrived(_message->messageType(), _message); + } + else + { + // hold the message + auto itM = m_holdingMessages.find(taskID); + if (itM != m_holdingMessages.end()) + { + itM->second->messages.emplace_back(_message); + } + else + { + auto holdingMessages = std::make_shared(); + holdingMessages->messages = std::vector(); + holdingMessages->messages.emplace_back(_message); + + // create timer to handle timeout + holdingMessages->timer = std::make_shared( + *m_ioService, boost::posix_time::minutes(HOLDING_MESSAGE_TIMEOUT_M)); + + holdingMessages->timer->async_wait( + [self = weak_from_this(), taskID](boost::system::error_code _error) { + if (!_error) + { + auto channelManager = self.lock(); + if (channelManager) + { + // remove timeout message + channelManager->removeHoldingMessages(taskID); + } + } + }); + + m_holdingMessages[taskID] = holdingMessages; + } + } +} + + +void PPCChannelManager::removeHoldingMessages(const std::string& _taskID) +{ + WriteGuard lock(x_message_channel); + m_holdingMessages.erase(_taskID); +} diff --git a/cpp/ppc-front/ppc-front/PPCChannelManager.h b/cpp/ppc-front/ppc-front/PPCChannelManager.h new file mode 100644 index 00000000..3e4b6bd4 --- /dev/null +++ b/cpp/ppc-front/ppc-front/PPCChannelManager.h @@ -0,0 +1,90 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCChannelManager.h + * @author: shawnhe + * @date 2022-10-27 + */ + +#pragma once + +#include "PPCChannel.h" +#include +#include + +namespace ppc::front +{ +class PPCChannelManager : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + PPCChannelManager(std::shared_ptr _ioService, front::Front::Ptr _front) + : m_ioService(std::move(_ioService)), m_front(std::move(_front)){}; + + ~PPCChannelManager() + { + std::unordered_map().swap(m_holdingMessages); + std::unordered_map().swap(m_channels); + MallocExtension::instance()->ReleaseFreeMemory(); + FRONT_LOG(INFO) << LOG_DESC("the PPCChannelManager destroyed"); + } + + void setThreadPool(std::shared_ptr _threadPool) + { + m_threadPool = std::move(_threadPool); + } + + /** + * if the algorithm need to use channel, register the handler of the algorithm in this function. + */ + void registerMsgHandlerForChannel(uint8_t _taskType, uint8_t _algorithmType); + + /** + * build a channel for each task + */ + Channel::Ptr buildChannelForTask(const std::string& _taskID); + + /** + * clean up channel at the end of the task + */ + void removeChannelByTask(const std::string& _taskID); + +protected: + struct HoldingMessage + { + using Ptr = std::shared_ptr; + std::vector messages; + std::shared_ptr timer; + }; + +private: + void onMessageArrived(front::PPCMessageFace::Ptr _message); + void removeHoldingMessages(const std::string& _taskID); + + +private: + std::shared_ptr m_ioService; + front::Front::Ptr m_front; + std::shared_ptr m_threadPool; + + /** + * hold the message for the situation that + * one party receives message from the other side while the Channel has not been registered. + */ + mutable boost::shared_mutex x_message_channel; + std::unordered_map m_holdingMessages; + std::unordered_map m_channels; +}; +} // namespace ppc::front diff --git a/cpp/ppc-front/test/CMakeLists.txt b/cpp/ppc-front/test/CMakeLists.txt new file mode 100644 index 00000000..d1b76fd6 --- /dev/null +++ b/cpp/ppc-front/test/CMakeLists.txt @@ -0,0 +1,13 @@ +file(GLOB_RECURSE SOURCES "unittests/*.cpp") + +# cmake settings +set(FRONT_TEST_BINARY_NAME test-ppc-front) + +add_executable(${FRONT_TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${FRONT_TEST_BINARY_NAME} PRIVATE .) + +target_compile_options(${FRONT_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unused-variable) + +target_link_libraries(${FRONT_TEST_BINARY_NAME} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) + +add_test(NAME test-front WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ppc-front/test/unittests COMMAND ${FRONT_TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-front/test/unittests/PPCChannelTest.cpp b/cpp/ppc-front/test/unittests/PPCChannelTest.cpp new file mode 100644 index 00000000..b301bcfb --- /dev/null +++ b/cpp/ppc-front/test/unittests/PPCChannelTest.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCChannelTest.cpp + * @author: shawnhe + * @date 2022-10-29 + */ + +#include "ppc-front/ppc-front/PPCChannel.h" +#include "ppc-front/ppc-front/PPCChannelManager.h" +#include "ppc-protocol/src/PPCMessage.h" +#include +#include +#include +#include + +using namespace ppc; +using namespace ppc::front; + +using namespace bcos; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(PPCChannelTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(test_ppcChannel) +{ + auto threadPool = std::make_shared("TEST_POOL_MODULE", 4); + auto ioService = std::make_shared(); + auto front = std::make_shared(ioService, "1001", threadPool); + + auto channelManager = std::make_shared(ioService, front); + // Note: must start here, otherwise the ioservice will not work + front->start(); + + // register message handler + channelManager->registerMsgHandlerForChannel( + uint8_t(protocol::TaskType::PSI), uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC)); + + // receive message + auto messageFactory = std::make_shared(); + auto message = messageFactory->buildPPCMessage(); + message->setVersion(1); + message->setTaskType(uint8_t(protocol::TaskType::PSI)); + message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC)); + message->setMessageType(4); + message->setSeq(5); + message->setTaskID("T_123456"); + message->setSender("1001"); + message->setData(std::make_shared(10, 'a')); + front->onReceiveMessage(message, nullptr); + + auto payload = std::make_shared(); + message->encode(*payload); + auto newMsg = messageFactory->buildPPCMessage(payload); + newMsg->setSeq(6); + front->onReceiveMessage(newMsg, nullptr); + + // build channel + auto channel = channelManager->buildChannelForTask("T_123456"); + + std::atomic flag = 0; + + channel->asyncReceiveMessage( + 4, 5, 1, [&flag](bcos::Error::Ptr, const PPCMessageFace::Ptr& _message) { + BOOST_CHECK(_message->seq() == 5); + BOOST_CHECK(_message->sender() == "1001"); + BOOST_CHECK(_message->data()->size() == 10); + flag++; + }); + + newMsg = messageFactory->buildPPCMessage(payload); + newMsg->setSeq(8); + front->onReceiveMessage(newMsg, nullptr); + + channel->asyncReceiveMessage( + 4, 6, 1, [&flag](bcos::Error::Ptr, const PPCMessageFace::Ptr& _message) { + BOOST_CHECK(_message->seq() == 6); + BOOST_CHECK(_message->sender() == "1001"); + BOOST_CHECK(_message->data()->size() == 10); + flag++; + }); + + // timeout + channel->asyncReceiveMessage( + 4, 7, 1, [&flag](const bcos::Error::Ptr& _error, const PPCMessageFace::Ptr& _message) { + BOOST_CHECK(_error->errorCode() == protocol::PPCRetCode::TIMEOUT); + flag++; + }); + + channel->asyncReceiveMessage( + 4, 8, 1, [&flag](bcos::Error::Ptr, const PPCMessageFace::Ptr& _message) { + BOOST_CHECK(_message->seq() == 8); + BOOST_CHECK(_message->sender() == "1001"); + BOOST_CHECK(_message->data()->size() == 10); + flag++; + }); + + while (flag != 4) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + front->stop(); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/ppc-front/test/unittests/main.cpp b/cpp/ppc-front/test/unittests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-front/test/unittests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-gateway/CMakeLists.txt b/cpp/ppc-gateway/CMakeLists.txt new file mode 100644 index 00000000..730bbda6 --- /dev/null +++ b/cpp/ppc-gateway/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.14) + +include(Version) +project(ppc-gateway VERSION ${VERSION}) + +find_package(Boost COMPONENTS filesystem) + +file(GLOB_RECURSE SRCS ppc-gateway/*.cpp) + +find_package(tarscpp REQUIRED) + +add_library(${GATEWAY_TARGET} ${SRCS}) +target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} ${HTTP_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} tarscpp::tarsservant tarscpp::tarsutil TBB::tbb) + +if (APPLE) + # target_compile_options(${GATEWAY_TARGET} PRIVATE -faligned-allocation) +endif () + +# ut +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(test) +endif () diff --git a/cpp/ppc-gateway/ppc-gateway/Common.h b/cpp/ppc-gateway/ppc-gateway/Common.h new file mode 100644 index 00000000..02f1f2de --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/Common.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once + +#include "ppc-framework/Common.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::gateway +{ +#define GATEWAY_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY]" + +#define GATEWAY_WS_CLIENT_MODULE "m_gateway_websocket_client" +#define GATEWAY_WS_SERVER_MODULE "m_gateway_websocket_server" +#define GATEWAY_THREAD_POOL_MODULE "t_gateway" + +#define SEND_MESSAGE_TO_FRONT_SUCCESS "success" +#define SEND_MESSAGE_TO_FRONT_ERROR "error" +#define SEND_MESSAGE_TO_FRONT_TIMEOUT "timeout" + +#define SEND_MESSAGE_TO_FRONT_SUCCESS_CODE "E0000000000" +#define SEND_MESSAGE_TO_FRONT_ERROR_CODE "-1" + +//HTTP HEADER DEFINE +#define HEAD_TASK_ID "x-ptp-session-id" +#define HEAD_ALGO_TYPE "x-ptp-algorithm-type" +#define HEAD_TASK_TYPE "x-ptp-task-type" +#define HEAD_SENDER_ID "x-ptp-sender-id" +#define HEAD_MESSAGE_TYPE "x-ptp-message-type" +#define HEAD_IS_RESPONSE "x-ptp-is-response" +#define HEAD_SEQ "x-ptp-seq" +#define HEAD_UUID "x-ptp-uuid" +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp new file mode 100644 index 00000000..b4c2e5a9 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontNodeManager.cpp + * @author: shawnhe + * @date 2022-10-23 + */ + +#include "FrontNodeManager.h" + +using namespace bcos; +using namespace ppc::gateway; +using namespace ppc::front; + +FrontInterface::Ptr FrontNodeManager::getFront(const std::string& _serviceEndpoint) +{ + bcos::ReadGuard lock(x_frontNodes); + auto it = m_frontNodes.find(_serviceEndpoint); + if (it != m_frontNodes.end()) + { + return it->second; + } + return nullptr; +} + +void FrontNodeManager::registerFront( + std::string const& _endPoint, front::FrontInterface::Ptr _front) +{ + bcos::UpgradableGuard l; + if (m_frontNodes.count(_endPoint)) + { + return; + } + bcos::UpgradeGuard ul(l); + m_frontNodes[_endPoint] = _front; + GATEWAY_LOG(INFO) << LOG_DESC("registerFront success") << LOG_KV("endPoint", _endPoint); +} + +void FrontNodeManager::unregisterFront(std::string const& _endPoint) +{ + bcos::UpgradableGuard l; + auto it = m_frontNodes.find(_endPoint); + if (it == m_frontNodes.end()) + { + return; + } + bcos::UpgradeGuard ul(l); + m_frontNodes.erase(it); + GATEWAY_LOG(INFO) << LOG_DESC("unregisterFront success") << LOG_KV("endPoint", _endPoint); +} diff --git a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h new file mode 100644 index 00000000..d5bac0f1 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontNodeManager.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once +#include "Common.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-front/ppc-front/Front.h" +#include +#include + +namespace ppc::gateway +{ +class FrontNodeManager : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + FrontNodeManager() = default; + virtual ~FrontNodeManager() = default; + + front::FrontInterface::Ptr getFront(const std::string& _serviceEndpoint); + virtual void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front); + + virtual void unregisterFront(std::string const& _endPoint); + + virtual std::unordered_map getAllFront() const + { + bcos::ReadGuard l(x_frontNodes); + return m_frontNodes; + } + +private: + // key: serviceEndpoint, value: FrontInterface + std::unordered_map m_frontNodes; + mutable bcos::SharedMutex x_frontNodes; +}; + +} // namespace ppc::gateway diff --git a/cpp/ppc-gateway/ppc-gateway/Gateway.cpp b/cpp/ppc-gateway/ppc-gateway/Gateway.cpp new file mode 100644 index 00000000..f85f80f1 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/Gateway.cpp @@ -0,0 +1,774 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Gateway.cpp + * @author: shawnhe + * @date 2022-10-20 + */ + +#include "Gateway.h" +#include "ProTaskManager.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; +using namespace bcos::boostssl::context; +using namespace ppc; +using namespace ppc::gateway; +using namespace ppc::protocol; +using namespace ppc::front; + + +void Gateway::start() +{ + if (m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("Gateway already started"); + return; + } + m_running = true; + GATEWAY_LOG(INFO) << LOG_DESC("start the Gateway"); + // register handler when receiving message from other agencies + if (m_protocol == m_webSocketService->gatewayConfig() + ->config() + ->gatewayConfig() + .networkConfig.PROTOCOL_WEBSOCKET) + { + registerWebSocketMsgHandler(); + } + else if (m_protocol == m_webSocketService->gatewayConfig() + ->config() + ->gatewayConfig() + .networkConfig.PROTOCOL_HTTP) + { +#if 0 + // TODO: optimize here + registerUrlMsgHandler(); +#endif + } + m_webSocketService->start(); + m_gatewayThread = std::make_shared([&] { + bcos::pthread_setThreadName("gw_io_service"); + while (m_running.load()) + { + try + { + m_ioContext->run(); + m_ioService->run(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (m_running) + { + if (m_ioService->stopped()) + { + m_ioService->restart(); + } + if (m_ioContext->stopped()) + { + m_ioContext->restart(); + } + } + } + catch (std::exception& e) + { + GATEWAY_LOG(WARNING) + << LOG_DESC("Exception in Gateway Thread:") << boost::diagnostic_information(e); + } + } + GATEWAY_LOG(INFO) << "Gateway exit"; + }); + GATEWAY_LOG(INFO) << LOG_BADGE("start the gateway end"); +} + + +void Gateway::stop() +{ + if (!m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("Gateway already stopped"); + return; + } + m_running = false; + GATEWAY_LOG(INFO) << LOG_DESC("stop the Gateway"); + if (m_webSocketService) + { + m_webSocketService->stop(); + } + if (m_ioService) + { + m_ioService->stop(); + } + if (m_ioContext) + { + m_ioContext->stop(); + } + // stop the gateway-thread + if (m_gatewayThread) + { + if (m_gatewayThread->get_id() != std::this_thread::get_id()) + { + m_gatewayThread->join(); + } + else + { + m_gatewayThread->detach(); + } + } + GATEWAY_LOG(INFO) << LOG_BADGE("stop the gateway end"); +} + +#if 0 +// TODO: optimize here +void Gateway::registerUrlMsgHandler() +{ + auto _url = m_webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.url; + m_webSocketService->httpServer()->registerUrlHandler( + _url, [self = weak_from_this()](bcos::boostssl::http::HttpRequest const& _httpReq, + ppc::http::RespUrlFunc _handler) { + auto gateway = self.lock(); + if (!gateway) + { + return; + } + std::string code = SEND_MESSAGE_TO_FRONT_SUCCESS_CODE; + std::string _resMsg = SEND_MESSAGE_TO_FRONT_SUCCESS; + try + { + // handle message + auto senderStr = _httpReq[HEAD_SENDER_ID].to_string(); + auto taskIdStr = _httpReq[HEAD_TASK_ID].to_string(); + auto algoTypeStr = _httpReq[HEAD_ALGO_TYPE].to_string(); + auto taskTypeStr = _httpReq[HEAD_TASK_TYPE].to_string(); + auto messageTypeStr = _httpReq[HEAD_MESSAGE_TYPE].to_string(); + auto seqStr = _httpReq[HEAD_SEQ].to_string(); + auto uuidStr = _httpReq[HEAD_UUID].to_string(); + bool isResponse = _httpReq[HEAD_IS_RESPONSE].to_string() == "1"; + auto reqBodyBytes = bcos::bytes(_httpReq.body().begin(), _httpReq.body().end()); + + GATEWAY_LOG(TRACE) << LOG_BADGE("registerUrlMsgHandler") + << LOG_KV("req size: ", reqBodyBytes.size()) + << LOG_KV("taskId: ", taskIdStr) + << LOG_KV("algorithmType: ", algoTypeStr) + << LOG_KV("taskType: ", taskTypeStr) + << LOG_KV("messageType: ", messageTypeStr) + << LOG_KV("UUID: ", uuidStr) << LOG_KV("seq: ", seqStr) + << LOG_KV("isResponse: ", isResponse); + + auto message = gateway->messageFactory()->buildPPCMessage(); + message->setData(std::make_shared(reqBodyBytes)); + message->setTaskID(taskIdStr); +#ifdef ENABLE_CONN + message->setAlgorithmType(5); + message->setTaskType(0); +#else + message->setAlgorithmType(0x00 + atoi(algoTypeStr.c_str())); + message->setTaskType(0x00 + atoi(taskTypeStr.c_str())); +#endif + message->setMessageType(0x00 + atoi(messageTypeStr.c_str())); + message->setSeq(0x00 + atoi(seqStr.c_str())); + message->setSender(senderStr); + message->setUuid(uuidStr); + if (isResponse) + { + message->setResponse(); + } + gateway->onMessageArrived(std::move(message)); + } + catch (std::exception& e) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") + << LOG_KV("exception", boost::diagnostic_information(e)); + code = SEND_MESSAGE_TO_FRONT_ERROR_CODE; + _resMsg = SEND_MESSAGE_TO_FRONT_ERROR; + } + + org::interconnection::link::TransportOutbound transportOutbound; + transportOutbound.set_allocated_message(new std::string(_resMsg)); + transportOutbound.set_allocated_code(new std::string(code)); + std::string responseStr; + transportOutbound.SerializeToString(&responseStr); + _handler(nullptr, std::move(bcos::bytes(responseStr.begin(), responseStr.end()))); + }); +} +#endif + +void Gateway::registerWebSocketMsgHandler() +{ + GATEWAY_LOG(INFO) << LOG_BADGE("registerWebSocketMsgHandler"); + m_webSocketService->webSocketServer()->registerMsgHandler( + 0, [self = weak_from_this()](const std::shared_ptr& _wsMessage, + const std::shared_ptr& _session) { + auto gateway = self.lock(); + if (!gateway) + { + return; + } + try + { + // prepare ack + auto ack = gateway->wsMessageFactory()->buildMessage(); + ack->setRespPacket(); + ack->setPacketType(_wsMessage->packetType()); + ack->setSeq(_wsMessage->seq()); + gateway->addAckCallback(_wsMessage->seq(), ack, _session); + + // handle message + auto payload = _wsMessage->payload(); + auto message = gateway->messageFactory()->buildPPCMessage(payload); + if (!message) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") + << LOG_DESC("decode ppc message error"); + gateway->sendAck(_wsMessage->seq(), SEND_MESSAGE_TO_FRONT_ERROR); + return; + } + + GATEWAY_LOG(TRACE) + << LOG_BADGE("onReceiveMsgFromOtherGateway") + << LOG_KV("taskType", unsigned(message->taskType())) + << LOG_KV("algorithmType", unsigned(message->algorithmType())) + << LOG_KV("messageType", unsigned(message->messageType())) + << LOG_KV("seq", message->seq()) << LOG_KV("taskID", message->taskID()) + << LOG_KV("sender", message->sender()); + + gateway->onMessageArrived(std::move(message)); + } + catch (std::exception& e) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") + << LOG_KV("exception", boost::diagnostic_information(e)); + gateway->sendAck(_wsMessage->seq(), SEND_MESSAGE_TO_FRONT_ERROR); + } + }); +} + + +/** + * @brief: send message to other agency + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @return void + */ +void Gateway::asyncSendMessage( + const std::string& _agencyID, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) +{ + auto taskID = _message->taskID(); + GATEWAY_LOG(TRACE) << LOG_BADGE("asyncSendMessage") << printPPCMsg(_message); + try + { + if (m_protocol == m_webSocketService->gatewayConfig() + ->config() + ->gatewayConfig() + .networkConfig.PROTOCOL_WEBSOCKET) + { + auto wsClient = m_webSocketService->webSocketClient(_agencyID); + if (!wsClient) + { + onError("asyncSendMessage", taskID, + BCOS_ERROR_PTR((int)PPCRetCode::NETWORK_ERROR, + "WebSocket client not found for " + _agencyID), + _callback); + return; + } + + auto payload = std::make_shared(); + _message->encode(*payload); + auto wsMessage = m_wsMessageFactory->buildMessage(0, payload); + wsMessage->setSeq(_message->uuid()); + + // forward to other agency + auto self = weak_from_this(); + wsClient->asyncSendMessage(wsMessage, Options(m_holdingMessageMinutes * 60 * 1000), + [self, wsMessage, wsClient, taskID, _callback](const Error::Ptr& _error, + const std::shared_ptr& _msg, + const std::shared_ptr& _session) { + Error::Ptr error; + // send success + if (!_error || _error->errorCode() == 0) + { + // check ack + auto payload = _msg->payload(); + if (payload) + { + std::string status = std::string(payload->begin(), payload->end()); + if (SEND_MESSAGE_TO_FRONT_ERROR == status || + SEND_MESSAGE_TO_FRONT_TIMEOUT == status) + { + error = std::make_shared(PPCRetCode::NETWORK_ERROR, + "send message to target front error, status = " + status); + } + } + if (!error) + { + GATEWAY_LOG(TRACE) + << LOG_DESC("asyncSendMessage success") << LOG_KV("task", taskID); + // response to the client in-case of tars-error + _callback(nullptr); + return; + } + } + + if (!error) + { + error = _error; + } + + auto gateway = self.lock(); + if (!gateway) + { + return; + } + gateway->onError("asyncSendMessage", taskID, error, _callback); + }); + } +#if 0 + else if (m_protocol == m_webSocketService->gatewayConfig() + ->config() + ->gatewayConfig() + .networkConfig.PROTOCOL_HTTP) + { + Error::Ptr error; + auto httpClient = m_webSocketService->httpClient(_agencyID); + auto _url = + m_webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.url; + auto header = _message->header(); + appendHeader(header, _message); + auto body = _message->data(); + auto response = httpClient->post(_url, header, *body); + // parse response to TransportOutbound + std::string reponseStr(response.begin(), response.end()); + org::interconnection::link::TransportOutbound transportOutbound; + transportOutbound.ParseFromString(reponseStr); + if (transportOutbound.code() != SEND_MESSAGE_TO_FRONT_SUCCESS_CODE) + { + error = std::make_shared(PPCRetCode::NETWORK_ERROR, + "send message to target front error, code = " + transportOutbound.code()); + } + if (!error) + { + GATEWAY_LOG(TRACE) + << LOG_DESC("asyncSendMessage success") << LOG_KV("task", taskID); + _callback(nullptr); + return; + } + onError("asyncSendMessage", taskID, error, _callback); + } +#endif + } + catch (std::exception& e) + { + onError("asyncSendMessage", taskID, + BCOS_ERROR_PTR( + (int)PPCRetCode::EXCEPTION, std::string(boost::diagnostic_information(e))), + std::move(_callback)); + return; + } +} + +void Gateway::onError(std::string const& _desc, std::string const& _taskID, bcos::Error::Ptr _error, + ErrorCallbackFunc _callback) +{ + if (!_error || _error->errorCode() == 0) + { + return; + } + + if (_error->errorCode() == WsError::TimeOut) + { + // Lower the log level because the caller will type out the error message + GATEWAY_LOG(INFO) << LOG_BADGE(_desc) << LOG_KV("taskID", _taskID) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + } + else + { + GATEWAY_LOG(ERROR) << LOG_BADGE(_desc) << LOG_KV("taskID", _taskID) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + } + + if (!_callback) + { + return; + } + + if (m_threadPool) + { + m_threadPool->enqueue([_callback, _error]() { _callback(_error); }); + } + else + { + _callback(std::move(_error)); + } +} + + +/** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ +bcos::Error::Ptr Gateway::notifyTaskInfo(GatewayTaskInfo::Ptr _taskInfo) +{ + auto startT = bcos::utcSteadyTime(); + auto error = std::make_shared(); + auto taskID = _taskInfo->taskID; + auto serviceEndpoint = _taskInfo->serviceEndpoint; + try + { + m_taskManager->registerTaskInfo(taskID, serviceEndpoint); + // check to see if any message has arrived + handleHoldingMessageQueue(std::move(_taskInfo)); + } + catch (std::exception& e) + { + error->setErrorCode(PPCRetCode::EXCEPTION); + error->setErrorMessage(boost::diagnostic_information(e)); + } + + if (error->errorCode()) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", taskID) + << LOG_DESC(error->errorMessage()); + } + GATEWAY_LOG(INFO) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", taskID) + << LOG_KV("serviceEndpoint", serviceEndpoint) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + return error; +} + +bcos::Error::Ptr Gateway::eraseTaskInfo(std::string const& _taskID) +{ + auto startT = bcos::utcSteadyTime(); + try + { + // release held message + getAndRemoveHoldingMessages(_taskID); + m_taskManager->removeTaskInfo(_taskID); + GATEWAY_LOG(INFO) << LOG_BADGE("eraseTaskInfo") << LOG_KV("taskID", _taskID) + << LOG_KV("timecost", bcos::utcSteadyTime() - startT); + return nullptr; + } + catch (std::exception const& e) + { + GATEWAY_LOG(ERROR) << LOG_DESC("eraseTaskInfo error") + << LOG_KV("exception", boost::diagnostic_information(e)) + << LOG_KV("timecost", bcos::utcSteadyTime() - startT); + return BCOS_ERROR_PTR( + PPCRetCode::EXCEPTION, "eraseTaskInfo error: " + boost::diagnostic_information(e)); + } +} + +// register gateway url for other parties +bcos::Error::Ptr Gateway::registerGateway( + const std::vector& _gatewayList) +{ + try + { + for (const auto& gateway : _gatewayList) + { + m_webSocketService->registerGatewayUrl(gateway.agencyID, gateway.endpoint); + } + + return nullptr; + } + catch (std::exception const& e) + { + GATEWAY_LOG(ERROR) << LOG_DESC("registerGateway error") + << LOG_KV("exception", boost::diagnostic_information(e)); + return BCOS_ERROR_PTR( + PPCRetCode::EXCEPTION, "registerGateway error: " + boost::diagnostic_information(e)); + } +} + +void Gateway::asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) +{ + GATEWAY_LOG(TRACE) << LOG_BADGE("asyncGetAgencyList"); + auto const& agencies = m_webSocketService->gatewayConfig()->config()->gatewayConfig().agencies; + std::vector agencyList; + for (auto const& it : agencies) + { + agencyList.emplace_back(it.first); + } + if (!_callback) + { + return; + } + _callback(nullptr, std::move(agencyList)); +} + +void Gateway::handleHoldingMessageQueue(protocol::GatewayTaskInfo::Ptr _taskInfo) +{ + HoldingMessageQueue::Ptr queue; + { + WriteGuard l(x_holdingMessageQueue); + auto it = m_holdingMessageQueue.find(_taskInfo->taskID); + // not find the holding-queue related to the task-info + if (it == m_holdingMessageQueue.end()) + { + return; + } + queue = it->second; + // erase the queue + m_holdingMessageQueue.erase(it); + } + // cancel the timer + if (queue->timer) + { + queue->timer->cancel(); + } + + auto frontInterface = m_frontNodeManager->getFront(_taskInfo->serviceEndpoint); + if (!frontInterface) + { + GATEWAY_LOG(WARNING) << LOG_DESC( + "handleHoldingMessageQueue error for not find the corresponding front"); + return; + } + // dispatch the message + for (auto& msg : queue->messages) + { + if (!frontInterface) + { + GATEWAY_LOG(WARNING) << LOG_DESC("send message error for the target front not found"); + sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); + continue; + } + // forward to self node + frontInterface->onReceiveMessage(msg, [self = weak_from_this(), msg]( + const bcos::Error::Ptr& _error) { + auto gateway = self.lock(); + if (!gateway) + { + return; + } + + if (_error && _error->errorCode() != 0) + { + GATEWAY_LOG(WARNING) + << LOG_DESC("handleHoldingMessageQueue: dispatch the message error") + << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); + gateway->sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); + return; + } + + gateway->sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_SUCCESS); + }); + } +} + +// broadcast the message to all front when the task-id is not specified +void Gateway::broadcastMsgToAllFront(ppc::front::PPCMessageFace::Ptr const& _message) +{ + auto frontList = m_frontNodeManager->getAllFront(); + GATEWAY_LOG(TRACE) << LOG_DESC("broadcastMsgToAllFront") + << LOG_KV("frontSize", frontList.size()); + for (auto const& it : frontList) + { + auto const& front = it.second; + auto const& serviceEndpoint = it.first; + dispatchMessageToFront(front, _message, serviceEndpoint); + } +} + +void Gateway::dispatchMessageToFront(ppc::front::FrontInterface::Ptr const& _front, + ppc::front::PPCMessageFace::Ptr const& _message, std::string const& _serviceEndpoint) +{ + auto taskID = _message->taskID(); + // dispatch message to the given front + auto startT = utcSteadyTime(); + _front->onReceiveMessage(_message, [self = weak_from_this(), taskID, _serviceEndpoint, _message, + startT](const bcos::Error::Ptr& _error) { + auto gateway = self.lock(); + if (!gateway) + { + return; + } + + if (_error && _error->errorCode() != 0) + { + GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveMessage: dispatch message to front error") + << printPPCMsg(_message) << LOG_KV("task", taskID) + << LOG_KV("front", _serviceEndpoint) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()) + << LOG_KV("timecost", (utcSteadyTime() - startT)); + gateway->sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); + return; + } + GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveMessage success") << printPPCMsg(_message) + << LOG_KV("timecost", (utcSteadyTime() - startT)); + gateway->sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_SUCCESS); + }); +} + +void Gateway::onMessageArrived(PPCMessageFace::Ptr _message) +{ + GATEWAY_LOG(TRACE) << LOG_DESC("onMessageArrived") << printPPCMsg(_message); + auto taskID = _message->taskID(); + // broadcast the message to all front when the task-id is not specified + if (taskID.empty()) + { + broadcastMsgToAllFront(_message); + return; + } + + bcos::UpgradableGuard l(x_holdingMessageQueue); + auto serviceEndpoint = m_taskManager->getServiceEndpoint(taskID); + if (!serviceEndpoint.empty()) + { + auto frontInterface = m_frontNodeManager->getFront(serviceEndpoint); + if (!frontInterface) + { + GATEWAY_LOG(WARNING) + << LOG_DESC( + "onMessageArrived: can't find the front to dispatch the receive message") + << printPPCMsg(_message) << LOG_KV("task", taskID) + << LOG_KV("frontEndPoint", serviceEndpoint); + sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); + return; + } + dispatchMessageToFront(frontInterface, _message, serviceEndpoint); + return; + } + // hold the message + GATEWAY_LOG(INFO) << LOG_BADGE("holdMessage") << LOG_KV("taskID", taskID); + + bcos::UpgradeGuard ul(l); + auto it = m_holdingMessageQueue.find(taskID); + if (it != m_holdingMessageQueue.end()) + { + it->second->messages.emplace_back(_message); + return; + } + // insert new holding-queue + auto queue = std::make_shared(); + queue->messages.emplace_back(_message); + // create timer to handle timeout + queue->timer = std::make_shared( + *m_ioService, boost::posix_time::minutes(m_holdingMessageMinutes)); + queue->timer->async_wait([self = weak_from_this(), taskID](boost::system::error_code _error) { + if (!_error) + { + auto gateway = self.lock(); + if (gateway) + { + // remove timeout message + auto msgQueue = gateway->getAndRemoveHoldingMessages(taskID); + gateway->handleTimeoutHoldingMessage(msgQueue); + } + } + }); + m_holdingMessageQueue[taskID] = queue; +} + +HoldingMessageQueue::Ptr Gateway::getAndRemoveHoldingMessages(const std::string& _taskID) +{ + WriteGuard lock(x_holdingMessageQueue); + auto it = m_holdingMessageQueue.find(_taskID); + if (it == m_holdingMessageQueue.end()) + { + return nullptr; + } + + HoldingMessageQueue::Ptr ret = it->second; + m_holdingMessageQueue.erase(_taskID); + return ret; +} + +void Gateway::handleTimeoutHoldingMessage(HoldingMessageQueue::Ptr _queue) +{ + if (!_queue) + { + return; + } + // dispatch the ack + for (auto& msg : _queue->messages) + { + sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_TIMEOUT); + } +} + +void Gateway::addAckCallback( + std::string const& _uuid, MessageFace::Ptr _msg, boostssl::ws::WsSession::Ptr _session) +{ + WriteGuard lock(x_ackCallbacks); + m_ackCallbacks[_uuid] = {std::move(_msg), std::move(_session)}; +} + + +void Gateway::sendAck(std::string const& _uuid, std::string const& _status) +{ + WriteGuard lock(x_ackCallbacks); + auto it = m_ackCallbacks.find(_uuid); + if (it == m_ackCallbacks.end()) + { + return; + } + + auto payload = std::make_shared(_status.begin(), _status.end()); + auto& msg = it->second.first; + auto& session = it->second.second; + msg->setPayload(payload); + session->asyncSendMessage(msg, Options(), nullptr); + + m_ackCallbacks.erase(it); +} + +void Gateway::appendHeader( + std::map& origin_header, front::PPCMessageFace::Ptr _message) +{ + origin_header["Content-Type"] = "application/octet-stream;charset=utf-8"; + origin_header["has_uri"] = std::to_string(true); + origin_header[HEAD_TASK_ID] = _message->taskID(); + origin_header[HEAD_ALGO_TYPE] = std::to_string(_message->algorithmType()); + origin_header[HEAD_TASK_TYPE] = std::to_string(_message->taskType()); + origin_header[HEAD_SENDER_ID] = _message->sender(); + origin_header[HEAD_MESSAGE_TYPE] = std::to_string(_message->messageType()); + origin_header[HEAD_SEQ] = std::to_string(_message->seq()); + origin_header[HEAD_IS_RESPONSE] = std::to_string(_message->response()); + origin_header[HEAD_UUID] = _message->uuid(); +} + +Gateway::Ptr GatewayFactory::buildGateway(NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, + storage::CacheStorage::Ptr _cache, front::PPCMessageFaceFactory::Ptr _messageFactory, + std::shared_ptr _threadPool) +{ + auto wsMessageFactory = std::make_shared(); + auto webSocketServiceFactory = std::make_shared(); + auto ioService = std::make_shared(); + auto ioContext = std::make_shared(); + + auto webSocketService = webSocketServiceFactory->buildWebSocketService(_config, ioContext); + + TaskManager::Ptr taskManager = nullptr; + if (_arch == NodeArch::AIR || _cache == nullptr) + { + GATEWAY_LOG(INFO) << LOG_BADGE("buildGateway without cache"); + taskManager = std::make_shared(ioService); + } + else + { + GATEWAY_LOG(INFO) << LOG_BADGE("buildGateway with cache"); + taskManager = std::make_shared(_cache, ioService); + } + auto frontNodeManager = std::make_shared(); + + return std::make_shared(std::move(webSocketService), std::move(wsMessageFactory), + std::move(ioService), std::move(_messageFactory), std::move(frontNodeManager), + std::move(taskManager), std::move(_threadPool), _config->holdingMessageMinutes(), + std::move(ioContext)); +} diff --git a/cpp/ppc-gateway/ppc-gateway/Gateway.h b/cpp/ppc-gateway/ppc-gateway/Gateway.h new file mode 100644 index 00000000..4a0a8a77 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/Gateway.h @@ -0,0 +1,210 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Gateway.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "Common.h" +#include "FrontNodeManager.h" +#include "GatewayService.h" +#include "TaskManager.h" +#include "WebSocketService.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-protocol/src/PPCMessage.h" +#if 0 +//TODO: optimize here +#include "ppc-protocol/src/protobuf/transport.pb.h" +#endif + +#include "tbb/concurrent_vector.h" +#include +#include + +#include + +namespace ppc::gateway +{ +struct HoldingMessageQueue +{ + using Ptr = std::shared_ptr; + HoldingMessageQueue() = default; + + tbb::concurrent_vector messages; + std::shared_ptr timer; +}; + +class Gateway : public GatewayInterface, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + Gateway(WebSocketService::Ptr _webSocketService, + bcos::boostssl::ws::WsMessageFactory::Ptr _wsMessageFactory, + std::shared_ptr _ioService, + front::PPCMessageFaceFactory::Ptr _messageFactory, FrontNodeManager::Ptr _frontNodeManager, + TaskManager::Ptr _taskManager, std::shared_ptr _threadPool, + int _holdingMessageMinutes = 10, + std::shared_ptr _ioContext = nullptr) + : m_holdingMessageMinutes(_holdingMessageMinutes), + m_webSocketService(std::move(_webSocketService)), + m_wsMessageFactory(std::move(_wsMessageFactory)), + m_ioService(std::move(_ioService)), + m_messageFactory(std::move(_messageFactory)), + m_frontNodeManager(std::move(_frontNodeManager)), + m_taskManager(std::move(_taskManager)), + m_threadPool(std::move(_threadPool)), + m_ioContext(_ioContext), + m_protocol( + _webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.protocol) + { + GATEWAY_LOG(INFO) << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); + } + + Gateway(const Gateway&) = delete; + Gateway(Gateway&&) = delete; + + Gateway& operator=(const Gateway&) = delete; + Gateway& operator=(Gateway&&) = delete; + + virtual ~Gateway() override { stop(); } + + void start() override; + void stop() override; + + void registerWebSocketMsgHandler(); + +#if 0 + // TODO: optimize here + void registerUrlMsgHandler(); +#endif + + /** + * @brief: send message to other agency + * @param _agencyID: agency ID of receiver + * @param _message: ppc message data + * @return void + */ + void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + ErrorCallbackFunc _callback) override; + + /** + * @brief notice task info to gateway + * @param _taskInfo the latest task information + */ + bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) override; + + // erase the task info + bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override; + + // register gateway url for other parties + bcos::Error::Ptr registerGateway( + const std::vector& _gatewayList) override; + + // get the agency-list + void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override; + + FrontNodeManager::Ptr frontNodeManager() { return m_frontNodeManager; } + TaskManager::Ptr taskManager() { return m_taskManager; } + void setTaskManager(TaskManager::Ptr _taskManager) { m_taskManager = std::move(_taskManager); } + + WebSocketService::Ptr webSocketService() { return m_webSocketService; } + + front::PPCMessageFaceFactory::Ptr messageFactory() { return m_messageFactory; } + bcos::boostssl::ws::WsMessageFactory::Ptr wsMessageFactory() { return m_wsMessageFactory; } + std::shared_ptr threadPool() { return m_threadPool; } + + void addAckCallback(std::string const& _uuid, bcos::boostssl::MessageFace::Ptr _msg, + bcos::boostssl::ws::WsSession::Ptr _session); + + void sendAck(std::string const& _uuid, std::string const& _status); + + // Note: since the front will periodically register the status, no need to response message to + // the front + void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front) override + { + m_frontNodeManager->registerFront(_endPoint, _front); + } + + void unregisterFront(std::string const& _endPoint) override + { + m_frontNodeManager->unregisterFront(_endPoint); + } + +protected: + virtual void handleHoldingMessageQueue(protocol::GatewayTaskInfo::Ptr _taskInfo); + virtual void onMessageArrived(front::PPCMessageFace::Ptr _message); + virtual HoldingMessageQueue::Ptr getAndRemoveHoldingMessages(const std::string& _taskID); + virtual void handleTimeoutHoldingMessage(HoldingMessageQueue::Ptr _queue); + virtual void onError(std::string const& _desc, std::string const& _taskID, + bcos::Error::Ptr _error, ErrorCallbackFunc _callback); + + void broadcastMsgToAllFront(ppc::front::PPCMessageFace::Ptr const& _message); + void dispatchMessageToFront(ppc::front::FrontInterface::Ptr const& _front, + ppc::front::PPCMessageFace::Ptr const& _message, std::string const& _serviceEndpoint); + void appendHeader( + std::map& origin_header, front::PPCMessageFace::Ptr _message); + +private: + int m_holdingMessageMinutes = 30; + int m_protocol; + WebSocketService::Ptr m_webSocketService; + bcos::boostssl::ws::WsMessageFactory::Ptr m_wsMessageFactory; + std::shared_ptr m_ioService; + std::shared_ptr m_ioContext; + front::PPCMessageFaceFactory::Ptr m_messageFactory; + + FrontNodeManager::Ptr m_frontNodeManager; + TaskManager::Ptr m_taskManager; + + std::shared_ptr m_threadPool; + + // the thread to make ioservice run + std::shared_ptr m_gatewayThread; + + /** + * hold the message for the situation that + * gateway receives message from the other side while the task has not been registered. + */ + mutable boost::shared_mutex x_holdingMessageQueue; + std::unordered_map m_holdingMessageQueue; + + std::atomic_bool m_running = {false}; + + mutable boost::shared_mutex x_ackCallbacks; + std::unordered_map > + m_ackCallbacks; +}; + + +class GatewayFactory +{ +public: + using Ptr = std::shared_ptr; + +public: + GatewayFactory() = default; + ~GatewayFactory() = default; + + Gateway::Ptr buildGateway(ppc::protocol::NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, + storage::CacheStorage::Ptr _cache, front::PPCMessageFaceFactory::Ptr _messageFactory, + std::shared_ptr _threadPool); +}; + +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp new file mode 100644 index 00000000..9a1a37ba --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayConfigContext.cpp + * @author: shawnhe + * @date 2022-10-23 + */ + +#include "GatewayConfigContext.h" + +using namespace bcos; +using namespace ppc::gateway; +void GatewayConfigContext::initContextConfig() +{ + m_contextConfig = std::make_shared(); + auto const& gatewayConfig = m_config->gatewayConfig().networkConfig; + // non-sm-ssl + if (!gatewayConfig.enableSM) + { + boostssl::context::ContextConfig::CertConfig certConfig; + certConfig.caCert = gatewayConfig.caCertPath; + certConfig.nodeCert = gatewayConfig.sslCertPath; + certConfig.nodeKey = gatewayConfig.sslKeyPath; + m_contextConfig->setCertConfig(certConfig); + m_contextConfig->setSslType("ssl"); + GATEWAY_LOG(INFO) << LOG_DESC("initConfig: rpc work in non-sm-ssl model") + << LOG_KV("caCert", certConfig.caCert) + << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("nodeKey", certConfig.nodeKey); + GATEWAY_LOG(INFO) << LOG_DESC("initContextConfig: non-sm-ssl"); + return; + } + // sm-ssl + boostssl::context::ContextConfig::SMCertConfig certConfig; + certConfig.caCert = gatewayConfig.smCaCertPath; + certConfig.nodeCert = gatewayConfig.smSslCertPath; + certConfig.nodeKey = gatewayConfig.smSslKeyPath; + certConfig.enNodeCert = gatewayConfig.smEnSslCertPath; + certConfig.enNodeKey = gatewayConfig.smEnSslKeyPath; + m_contextConfig->setSmCertConfig(certConfig); + m_contextConfig->setSslType("sm_ssl"); + GATEWAY_LOG(INFO) << LOG_DESC("initContextConfig: sm-ssl"); +} diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h new file mode 100644 index 00000000..41320ebc --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayConfigContext.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once + +#include "Common.h" +#include "ppc-framework/storage/CacheStorage.h" +#include "ppc-storage/src/redis/RedisStorage.h" +#include +#include +#include +#include +#include + +namespace ppc::gateway +{ +class GatewayConfigContext +{ +public: + using Ptr = std::shared_ptr; + GatewayConfigContext(ppc::tools::PPCConfig::Ptr _config) : m_config(_config) + { + if (!m_config->gatewayConfig().networkConfig.disableSsl) + { + GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig"); + initContextConfig(); + GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig success"); + } + } + virtual ~GatewayConfigContext() = default; + [[nodiscard]] std::shared_ptr contextConfig() const + { + return m_contextConfig; + } + ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } + +private: + void initContextConfig(); + +private: + ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_contextConfig; +}; + +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h b/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h new file mode 100644 index 00000000..fc8b8066 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProTaskManager.h + * @author: shawnhe + * @date 2022-10-23 + */ +#pragma once +#include "TaskManager.h" + +#include "ppc-framework/storage/CacheStorage.h" +#include +namespace ppc::gateway +{ +class ProTaskManager : public TaskManager +{ +public: + using Ptr = std::shared_ptr; + ProTaskManager( + storage::CacheStorage::Ptr _cache, std::shared_ptr _ioService) + : TaskManager(std::move(_ioService)), m_cache(std::move(_cache)) + {} + ~ProTaskManager() override = default; + + void registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint) override + { + // throw exception if taskID existed + TaskManager::registerTaskInfo(_taskID, _serviceEndpoint); + try + { + // add task info to cache server + m_cache->setValue(_taskID, _serviceEndpoint, TASK_TIMEOUT_M * 60); + } + catch (std::exception const& e) + { + GATEWAY_LOG(WARNING) << LOG_DESC( + "set value failed: " + std::string(boost::diagnostic_information(e))); + } + } + + void removeTaskInfo(const std::string& _taskID) override + { + // Note: remove the memory-task-info in-case-of the redis exception + TaskManager::removeTaskInfo(_taskID); + m_cache->deleteKey(_taskID); + } + + std::string getServiceEndpoint(const std::string& _taskID) override + { + // find task info in memory first + try + { + auto endPoint = TaskManager::getServiceEndpoint(_taskID); + if (!endPoint.empty()) + { + return endPoint; + } + // Note: different node should not share the cache with same database + // find task info in cache service + auto serviceEndpoint = m_cache->getValue(_taskID); + if (serviceEndpoint == std::nullopt) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("keyNotFoundInCache") << LOG_KV("key", _taskID); + return ""; + } + GATEWAY_LOG(TRACE) << LOG_DESC("getServiceEndpoint: find the task from redis cache") + << LOG_KV("task", _taskID); + // add task info to memory + auto taskInfo = prepareTaskInfo(_taskID, *serviceEndpoint); + addTaskInfo(_taskID, taskInfo); + return *serviceEndpoint; + } + catch (std::exception const& e) + { + GATEWAY_LOG(ERROR) << LOG_DESC("getServiceEndpoint error") + << LOG_KV("exception", boost::diagnostic_information(e)); + return ""; + } + } + +private: + storage::CacheStorage::Ptr m_cache; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp b/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp new file mode 100644 index 00000000..51ad5f15 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskManager.cpp + * @author: shawnhe + * @date 2022-10-23 + */ + +#include "TaskManager.h" +#include "ppc-framework/protocol/Protocol.h" + +using namespace bcos; +using namespace ppc::gateway; +using namespace ppc::storage; + +void TaskManager::registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("registerTaskInfo") << LOG_KV("taskID", _taskID) + << LOG_KV("serviceEndpoint", _serviceEndpoint); + if (getTaskInfo(_taskID)) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR(protocol::PPCRetCode::EXCEPTION, "task id already exists")); + } + + // add task info to memory + auto taskInfo = prepareTaskInfo(_taskID, _serviceEndpoint); + addTaskInfo(_taskID, taskInfo); +} + + +std::string TaskManager::getServiceEndpoint(const std::string& _taskID) +{ + // find task info in memory first + auto taskInfo = getTaskInfo(_taskID); + if (taskInfo) + { + return taskInfo->serviceEndpoint; + } + return ""; +} + + +TaskManager::TaskInfo::Ptr TaskManager::prepareTaskInfo( + const std::string& _taskID, const std::string& _serviceEndpoint) +{ + auto taskInfo = std::make_shared(); + taskInfo->serviceEndpoint = _serviceEndpoint; + + // create timer to handle timeout + taskInfo->timer = std::make_shared( + *m_ioService, boost::posix_time::minutes(TASK_TIMEOUT_M)); + + taskInfo->timer->async_wait( + [self = weak_from_this(), _taskID](boost::system::error_code _error) { + if (!_error) + { + auto taskManager = self.lock(); + if (taskManager) + { + // remove timeout event + taskManager->removeTaskInfo(_taskID); + } + } + }); + + return taskInfo; +} + + +TaskManager::TaskInfo::Ptr TaskManager::getTaskInfo(const std::string& _taskID) +{ + ReadGuard lock(x_tasks); + auto it = m_tasks.find(_taskID); + if (it != m_tasks.end()) + { + return it->second; + } + else + { + return nullptr; + } +} + + +void TaskManager::addTaskInfo( + const std::string& _taskID, const TaskManager::TaskInfo::Ptr& _taskInfo) +{ + WriteGuard lock(x_tasks); + GATEWAY_LOG(INFO) << LOG_BADGE("addTaskInfo") << LOG_KV("taskID", _taskID) + << LOG_KV("serviceEndpoint", _taskInfo->serviceEndpoint); + m_tasks.emplace(_taskID, _taskInfo); +} + + +void TaskManager::removeTaskInfo(const std::string& _taskID) +{ + WriteGuard lock(x_tasks); + GATEWAY_LOG(INFO) << LOG_BADGE("removeTaskInfo") << LOG_KV("taskID", _taskID); + m_tasks.erase(_taskID); +} diff --git a/cpp/ppc-gateway/ppc-gateway/TaskManager.h b/cpp/ppc-gateway/ppc-gateway/TaskManager.h new file mode 100644 index 00000000..214888a0 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/TaskManager.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskManager.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once + +#include "Common.h" +#include "GatewayConfigContext.h" +#include +#include +#include +#include +#include + +namespace ppc::gateway +{ +class TaskManager : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + TaskManager(std::shared_ptr _ioService) + : m_ioService(std::move(_ioService)) + {} + virtual ~TaskManager() = default; + + virtual void registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint); + + virtual std::string getServiceEndpoint(const std::string& _taskID); + + virtual void removeTaskInfo(const std::string& _taskID); + +protected: + struct TaskInfo + { + using Ptr = std::shared_ptr; + std::string serviceEndpoint; + // timeout of the task + std::shared_ptr timer; + }; + + TaskInfo::Ptr prepareTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint); + TaskInfo::Ptr getTaskInfo(const std::string& _taskID); + void addTaskInfo(const std::string& _taskID, const TaskInfo::Ptr& _taskInfo); + +protected: + std::shared_ptr m_ioService; + // key: taskID, value: TaskInfo + std::unordered_map m_tasks; + mutable bcos::SharedMutex x_tasks; + + constexpr static uint32_t TASK_TIMEOUT_M = 24 * 60; // minutes +}; +} // namespace ppc::gateway diff --git a/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp b/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp new file mode 100644 index 00000000..1c644fa8 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp @@ -0,0 +1,387 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file WebSocketService.cpp + * @author: shawnhe + * @date 2022-10-23 + */ + +#include "WebSocketService.h" +#include "ppc-tools/src/config/ParamChecker.h" +#include + +using namespace bcos; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; +using namespace ppc::gateway; +using namespace ppc::tools; +using namespace ppc::http; + +void WebSocketService::start() +{ + if (m_timer) + { + m_timer->registerTimeoutHandler( + boost::bind(&WebSocketService::reconnectUnconnectedClient, this)); + } + if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + m_wsServer->start(); + GATEWAY_LOG(INFO) << LOG_BADGE("start the WebSocketService end"); + } + else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + m_httpServer->start(); + GATEWAY_LOG(INFO) << LOG_BADGE("start the HttpService end"); + } + startConnect(); + m_timer->start(); +} + +void WebSocketService::stop() +{ + GATEWAY_LOG(INFO) << LOG_BADGE("stop the WebSocketService"); + if (m_timer) + { + m_timer->stop(); + } + ReadGuard l(x_agencyClients); + if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + for (auto& client : m_agencyClients) + { + auto it = client.second; + if (it) + { + it->stop(); + } + } + if (m_wsServer) + { + m_wsServer->stop(); + } + GATEWAY_LOG(INFO) << LOG_BADGE("stop the WebSocketService success"); + } + else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + if (m_httpServer) + { + m_httpServer->stop(); + } + GATEWAY_LOG(INFO) << LOG_BADGE("stop the HttpService success"); + } +} + + +void WebSocketService::registerGatewayUrl( + const std::string& _agencyID, const std::string& _agencyUrl) +{ + if (!insertAgency(_agencyID, _agencyUrl)) + { + return; + } + GATEWAY_LOG(INFO) << LOG_BADGE("registerGatewayUrl") << LOG_KV("agencyID", _agencyID) + << LOG_KV("agencyUrl", _agencyUrl); + try + { + if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + auto client = buildWebSocketClient(_agencyID); + insertIntoMap(_agencyID, client, x_agencyClients, m_agencyClients); + } + else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + auto client = buildHttpClient(_agencyID, m_ioContext); + insertIntoMap(_agencyID, client, x_agencyClients, m_agencyHttpClients); + } + } + catch (std::exception const& e) + { + GATEWAY_LOG(WARNING) << LOG_DESC("connect to the agency failed") + << LOG_KV("agency", _agencyID) << LOG_KV("url", _agencyUrl); + { + bcos::WriteGuard ucl(x_unConnectedAgencies); + m_unConnectedAgencies.insert(_agencyID); + } + } +} + +bool WebSocketService::insertAgency(const std::string& _agencyID, const std::string& _agencyUrl) +{ + std::vector endpoints; + boost::split(endpoints, _agencyUrl, boost::is_any_of(",")); + + WriteGuard l(x_agencies); + + if (m_urls.find(_agencyID) != m_urls.end()) + { + if (m_urls[_agencyID] == _agencyUrl) + { + // no need update + return false; + } + } + + m_urls[_agencyID] = _agencyUrl; + m_agencies[_agencyID] = endpoints; + return true; +} + +void WebSocketService::reconnectUnconnectedClient() +{ + { + // print connected clients + bcos::ReadGuard rl(x_agencyClients); + if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + GATEWAY_LOG(INFO) << LOG_DESC("connectedWebsocketClient") + << LOG_KV("size", m_agencyClients.size()); + } + else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + GATEWAY_LOG(INFO) << LOG_DESC("connectedHttpClient") + << LOG_KV("size", m_agencyHttpClients.size()); + } + } + + // start reconnecting + bcos::UpgradableGuard l(x_unConnectedAgencies); + GATEWAY_LOG(INFO) << LOG_DESC("reconnectUnconnectedClient") + << LOG_KV("size", m_unConnectedAgencies.size()); + for (auto it = m_unConnectedAgencies.begin(); it != m_unConnectedAgencies.end();) + { + try + { + if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") + << LOG_DESC("reconnectUnconnectedClient: connect to peer") + << LOG_KV("agency", *it); + auto client = buildWebSocketClient(*it); + GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") + << LOG_DESC("reconnectUnconnectedClient: connect to peer success") + << LOG_KV("agency", *it); + // insert the successfully started client into the m_agencyClients + insertIntoMap(*it, client, x_agencyClients, m_agencyClients); + // erase the connected client from the m_unConnectedAgencies + bcos::UpgradeGuard ul(l); + it = m_unConnectedAgencies.erase(it); + } + else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + GATEWAY_LOG(INFO) << LOG_BADGE("HttpService") + << LOG_DESC("reconnectUnconnectedClient: connect to peer") + << LOG_KV("agency", *it); + auto client = buildHttpClient(*it, m_ioContext); + GATEWAY_LOG(INFO) << LOG_BADGE("HttpService") + << LOG_DESC("reconnectUnconnectedClient: connect to peer success") + << LOG_KV("agency", *it); + // insert the successfully started client into the m_agencyHttpClients + insertIntoMap(*it, client, x_agencyClients, m_agencyHttpClients); + // erase the connected client from the m_unConnectedAgencies + bcos::UpgradeGuard ul(l); + it = m_unConnectedAgencies.erase(it); + } + } + catch (std::exception const& e) + { + it++; + GATEWAY_LOG(INFO) << LOG_BADGE("reconnectUnconnectedClient failed"); + } + } + m_timer->restart(); +} + +WsService::Ptr WebSocketService::webSocketClient(const std::string& _agencyID) +{ + return getValueFromMap( + _agencyID, x_agencyClients, m_agencyClients); +} + +HttpClient::Ptr WebSocketService::httpClient(const std::string& _agencyID) +{ + return getValueFromMap( + _agencyID, x_agencyClients, m_agencyHttpClients); +} + +void WebSocketService::startConnect() +{ + GATEWAY_LOG(INFO) << LOG_DESC("WebSocketService: startConnect"); + auto const& agencyConfig = m_config->config()->gatewayConfig().agencies; + auto protocol = m_config->config()->gatewayConfig().networkConfig.protocol; + for (auto const& it : agencyConfig) + { + try + { + if (protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + auto client = buildWebSocketClient(it.first); + insertIntoMap(it.first, client, x_agencyClients, m_agencyClients); + } + else if (protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + auto client = buildHttpClient(it.first, m_ioContext); + insertIntoMap(it.first, client, x_agencyClients, m_agencyHttpClients); + } + } + catch (std::exception const& e) + { + { + bcos::WriteGuard l(x_unConnectedAgencies); + m_unConnectedAgencies.insert(it.first); + } + GATEWAY_LOG(WARNING) << LOG_BADGE("startConnect") + << LOG_DESC("connect to the agency failed") + << LOG_KV("agency", it.first) + << LOG_KV("exception", boost::diagnostic_information(e)); + } + } + GATEWAY_LOG(INFO) << LOG_DESC("WebSocketService: startConnect success"); +} + +HttpClient::Ptr WebSocketService::buildHttpClient( + const std::string& _agencyID, std::shared_ptr _ioContext) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService: buildHttpClient") + << LOG_KV("agency", _agencyID); + { + bcos::ReadGuard l(x_agencies); + // one agencyID => one httpClient + for (const auto& endpoint : m_agencies.at(_agencyID)) + { + if (!checkEndpoint(endpoint)) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); + } + std::vector url; + boost::split(url, endpoint, boost::is_any_of(":"), boost::token_compress_on); + auto client = std::make_shared(*_ioContext, url[0], std::stoi(url[1])); + return client; + } + } +} + +WsService::Ptr WebSocketService::buildWebSocketClient(std::string const& _agencyID) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketClient") << LOG_KV("agency", _agencyID); + auto peers = std::make_shared(); + { + bcos::ReadGuard l(x_agencies); + for (const auto& endpoint : m_agencies.at(_agencyID)) + { + if (!checkEndpoint(endpoint)) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); + } + + std::vector url; + boost::split(url, endpoint, boost::is_any_of(":"), boost::token_compress_on); + NodeIPEndpoint nodeIpEndpoint = NodeIPEndpoint(url[0], std::stoi(url[1])); + peers->insert(nodeIpEndpoint); + } + } + + auto const& gatewayConfig = m_config->config()->gatewayConfig(); + auto wsConfig = std::make_shared(); + wsConfig->setModel(WsModel::Client); + wsConfig->setConnectPeers(peers); + wsConfig->setThreadPoolSize(gatewayConfig.networkConfig.threadPoolSize); + wsConfig->setDisableSsl(gatewayConfig.networkConfig.disableSsl); + wsConfig->setMaxMsgSize(gatewayConfig.maxAllowedMsgSize); + if (!wsConfig->disableSsl()) + { + wsConfig->setContextConfig(m_config->contextConfig()); + } + + auto wsInitializer = std::make_shared(); + wsInitializer->setConfig(wsConfig); + auto wsClient = std::make_shared(GATEWAY_WS_CLIENT_MODULE); + wsClient->setTimerFactory(std::make_shared()); + wsInitializer->initWsService(wsClient); + + wsClient->start(); + GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") << LOG_DESC("connect to peer success") + << LOG_KV("agency", _agencyID); + return wsClient; +} + +WsService::Ptr WebSocketServiceFactory::buildWebSocketServer( + const GatewayConfigContext::Ptr& _config) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketServer"); + auto wsConfig = std::make_shared(); + wsConfig->setModel(WsModel::Server); + + auto const& gatewayConfig = _config->config()->gatewayConfig(); + wsConfig->setListenIP(gatewayConfig.networkConfig.listenIp); + wsConfig->setListenPort(gatewayConfig.networkConfig.listenPort); + wsConfig->setThreadPoolSize(gatewayConfig.networkConfig.threadPoolSize); + wsConfig->setDisableSsl(gatewayConfig.networkConfig.disableSsl); + if (!wsConfig->disableSsl()) + { + wsConfig->setContextConfig(_config->contextConfig()); + } + wsConfig->setMaxMsgSize(gatewayConfig.maxAllowedMsgSize); + auto wsInitializer = std::make_shared(); + wsInitializer->setConfig(wsConfig); + auto wsService = std::make_shared(GATEWAY_WS_SERVER_MODULE); + wsService->setTimerFactory(std::make_shared()); + wsInitializer->initWsService(wsService); + + return wsService; +} + +Http::Ptr WebSocketServiceFactory::buildHttpServer(const GatewayConfigContext::Ptr& _config) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("buildHttpServer"); + auto ppcConfig = _config->config(); + auto httpFactory = std::make_shared(ppcConfig->agencyID()); + return httpFactory->buildHttp(ppcConfig); +} + +WebSocketService::Ptr WebSocketServiceFactory::buildWebSocketService( + ppc::tools::PPCConfig::Ptr const& _config, std::shared_ptr _ioContext) +{ + try + { + auto gatewayConfig = std::make_shared(_config); + auto _protocol = _config->gatewayConfig().networkConfig.protocol; + // init websocket service + if (_protocol == _config->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) + { + GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketService"); + auto wsServer = buildWebSocketServer(gatewayConfig); + auto webSocketService = std::make_shared(gatewayConfig, wsServer); + return webSocketService; + } + else if (_protocol == _config->gatewayConfig().networkConfig.PROTOCOL_HTTP) + { + GATEWAY_LOG(INFO) << LOG_BADGE("buildHttpService"); + auto httpServer = buildHttpServer(gatewayConfig); + auto webSocketService = + std::make_shared(gatewayConfig, httpServer, _ioContext); + return webSocketService; + } + } + catch (std::exception const& e) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("buildWebSocketService") + << LOG_DESC("init gateway websocket service failed, error: " + + boost::diagnostic_information(e)); + throw e; + } +} diff --git a/cpp/ppc-gateway/ppc-gateway/WebSocketService.h b/cpp/ppc-gateway/ppc-gateway/WebSocketService.h new file mode 100644 index 00000000..73c03ad5 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/WebSocketService.h @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file WebSocketService.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once + +#include "Common.h" +#include "GatewayConfigContext.h" +#include "ppc-http/src/Http.h" +#include "ppc-http/src/HttpClient.h" +#include "ppc-http/src/HttpFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::gateway +{ +class WebSocketService +{ +public: + using Ptr = std::shared_ptr; + using WebSocketClientMap = std::unordered_map; + using HttpClientMap = std::unordered_map; + WebSocketService( + GatewayConfigContext::Ptr _config, bcos::boostssl::ws::WsService::Ptr _wsServer) + : m_config(_config), + m_agencies(_config->config()->gatewayConfig().agencies), + m_wsServer(std::move(_wsServer)), + m_protocol(_config->config()->gatewayConfig().networkConfig.protocol), + m_timer(std::make_shared( + _config->config()->gatewayConfig().reconnectTime, "connectTimer")) + {} + + WebSocketService(GatewayConfigContext::Ptr _config, ppc::http::Http::Ptr _httpServer, + std::shared_ptr _ioContext) + : m_config(_config), + m_agencies(_config->config()->gatewayConfig().agencies), + m_httpServer(std::move(_httpServer)), + m_protocol(_config->config()->gatewayConfig().networkConfig.protocol), + m_ioContext(_ioContext), + m_timer(std::make_shared( + _config->config()->gatewayConfig().reconnectTime, "connectTimer")) + {} + + virtual ~WebSocketService() = default; + + void start(); + void stop(); + + void registerGatewayUrl(const std::string& _agencyID, const std::string& _agencyUrl); + bool insertAgency(const std::string& _agencyID, const std::string& _agencyUrl); + bcos::boostssl::ws::WsService::Ptr const& webSocketServer() const { return m_wsServer; } + ppc::http::Http::Ptr httpServer() const { return m_httpServer; } + bcos::boostssl::ws::WsService::Ptr webSocketClient(const std::string& _agencyID); + ppc::http::HttpClient::Ptr httpClient(const std::string& _agencyID); + GatewayConfigContext::Ptr const& gatewayConfig() const { return m_config; } + + +protected: + virtual bcos::boostssl::ws::WsService::Ptr buildWebSocketClient(std::string const& _agencyID); + virtual ppc::http::HttpClient::Ptr buildHttpClient( + const std::string& _agencyID, std::shared_ptr _ioContext); + virtual void startConnect(); + virtual void reconnectUnconnectedClient(); + + template + void insertIntoMap(std::string const& _key, T const& _value, bcos::SharedMutex& lock, S& _map) + { + bcos::WriteGuard l(lock); + _map[_key] = _value; + } + + template + T getValueFromMap(std::string const& _key, bcos::SharedMutex& lock, S const& _map) + { + bcos::ReadGuard l(lock); + auto it = _map.find(_key); + if (it != _map.end()) + { + return it->second; + } + return nullptr; + } + +private: + int m_protocol; + GatewayConfigContext::Ptr m_config; + bcos::SharedMutex x_agencies; + std::map> m_agencies; + std::map m_urls; + + bcos::boostssl::ws::WsService::Ptr m_wsServer; + ppc::http::Http::Ptr m_httpServer; + // the timer used to try connecting to the un-connected-clients + std::shared_ptr m_timer; + + std::shared_ptr m_ioContext; + // key: agencyID, value: WebSocketClient + WebSocketClientMap m_agencyClients; + HttpClientMap m_agencyHttpClients; + bcos::SharedMutex x_agencyClients; + // connect failed for all the agency-nodes are offline + std::set m_unConnectedAgencies; + mutable bcos::SharedMutex x_unConnectedAgencies; +}; + + +class WebSocketServiceFactory +{ +public: + using Ptr = std::shared_ptr; + +public: + WebSocketServiceFactory() = default; + ~WebSocketServiceFactory() = default; + + WebSocketService::Ptr buildWebSocketService(const ppc::tools::PPCConfig::Ptr& _config, + std::shared_ptr _ioContext = nullptr); + +private: + bcos::boostssl::ws::WsService::Ptr buildWebSocketServer( + const GatewayConfigContext::Ptr& _config); + + ppc::http::Http::Ptr buildHttpServer(const GatewayConfigContext::Ptr& _config); +}; + +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/test/CMakeLists.txt b/cpp/ppc-gateway/test/CMakeLists.txt new file mode 100644 index 00000000..8a952e2f --- /dev/null +++ b/cpp/ppc-gateway/test/CMakeLists.txt @@ -0,0 +1,17 @@ +file(GLOB_RECURSE SOURCES "unittests/*.cpp" "unittests/*.h") + +# cmake settings +set(GATEWAY_TEST_BINARY_NAME test-ppc-gateway) + +if (DEMO) + add_subdirectory(demo) +endif () + +add_executable(${GATEWAY_TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${GATEWAY_TEST_BINARY_NAME} PRIVATE .) + +target_compile_options(${GATEWAY_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unused-variable) + +target_link_libraries(${GATEWAY_TEST_BINARY_NAME} ${GATEWAY_TARGET} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) + +add_test(NAME test-gateway WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ppc-gateway/test/unittests COMMAND ${GATEWAY_TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-gateway/test/data/config0.ini b/cpp/ppc-gateway/test/data/config0.ini new file mode 100644 index 00000000..a07e8157 --- /dev/null +++ b/cpp/ppc-gateway/test/data/config0.ini @@ -0,0 +1,32 @@ +[gateway] + ; host ip + listen_ip = 0.0.0.0 + ; Websocket server listening port + listen_port = 34745 + ; for thread pools + thread_count = 16 + ; disable encrypt communication + disable_ssl = true + ; ssl or sm_ssl + ssl_type = sm_ssl + +[cache] + type = 0 + proxy = 127.0.0.1:20002 + obServer = 127.0.0.1:10904 + cluster = REDIS_CLUSTER + user = 1194 + host = 127.0.0.1 + port = 6379 + password = 123456 + database = 1 + pool_size = 16 + +[agency] + ; the agency-id of self-party + id = "1001" + ; the agency info + agency.1001 = 127.0.0.1:34745,127.0.0.1:34746 + agency.1002 = 127.0.0.1:34755,127.0.0.1:34756 + protocol = 1 + url = /api/v1/invoke \ No newline at end of file diff --git a/cpp/ppc-gateway/test/demo/CMakeLists.txt b/cpp/ppc-gateway/test/demo/CMakeLists.txt new file mode 100644 index 00000000..939d6111 --- /dev/null +++ b/cpp/ppc-gateway/test/demo/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(GATEWAY_DEMO_BINARY_NAME gateway_demo) + +add_executable(${GATEWAY_DEMO_BINARY_NAME} gateway_demo.cpp) + +target_link_libraries(${GATEWAY_DEMO_BINARY_NAME} ${GATEWAY_TARGET} ${STORAGE_TARGET} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) \ No newline at end of file diff --git a/cpp/ppc-gateway/test/demo/gateway_demo.cpp b/cpp/ppc-gateway/test/demo/gateway_demo.cpp new file mode 100644 index 00000000..b5d6f6f1 --- /dev/null +++ b/cpp/ppc-gateway/test/demo/gateway_demo.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file gateway_demo.cpp + * @author: shawnhe + * @date 2022-10-28 + */ + +#include "ppc-gateway/ppc-gateway/Gateway.h" +#include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" +#include "ppc-tools/src/config/PPCConfig.h" + +using namespace ppc; +using namespace ppctars; +using namespace ppc::front; +using namespace ppc::gateway; +using namespace ppc::storage; +using namespace ppc::protocol; + +using namespace bcos; +using namespace ppc::tools; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; + + +inline static PPCMessageFace::Ptr buildMessage(std::string _taskID, uint32_t _seq) +{ + auto messageFactory = std::make_shared(); + auto message = messageFactory->buildPPCMessage(); + message->setVersion(1); + message->setTaskType(uint8_t(protocol::TaskType::PSI)); + message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC)); + message->setMessageType(4); + message->setSeq(_seq); + message->setTaskID(_taskID); + message->setSender("1001"); + message->setData(std::make_shared(10, 'a')); + return message; +} + + +int main(int argc, char* argv[]) +{ + // config + auto config = std::make_shared(); + // not specify the certPath in air-mode + config->loadGatewayConfig( + ppc::protocol::NodeArch::AIR, nullptr, "../../../../ppc-gateway/test/data/config0.ini"); + auto gatewayConfig = config->gatewayConfig(); + + // global thread pool + auto threadPoolSize = gatewayConfig.networkConfig.threadPoolSize; + auto threadPool = std::make_shared(GATEWAY_THREAD_POOL_MODULE, threadPoolSize); + + auto ioService = std::make_shared(); + + // front factory + auto frontFactory = std::make_shared("1001", threadPool); + auto front = frontFactory->buildFront(ioService); + + // message factory + auto messageFactory = std::make_shared(); + auto wsMessageFactory = std::make_shared(); + + // Note: no-need the redis as cache in air-mode + auto gatewayFactory = std::make_shared(); + auto gateway = gatewayFactory->buildGateway( + ppc::protocol::NodeArch::AIR, config, nullptr, messageFactory, threadPool); + gateway->registerFront(front->selfEndPoint(), front); + gateway->start(); + + front->setGatewayInterface(gateway); + front->start(); + + std::atomic flag = 0; + front->registerMessageHandler(uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), + [&flag](const PPCMessageFace::Ptr& _message) { + std::cout << "message received\n" + << "type: " << unsigned(_message->messageType()) << std::endl; + std::cout << "sender: " << _message->sender() << std::endl; + std::cout << "seq: " << _message->seq() << std::endl; + flag++; + }); + + auto taskID = "02345678"; + auto info = std::make_shared(); + info->taskID = taskID; + info->serviceEndpoint = "endpoint1001"; + front->notifyTaskInfo(info); + + auto message = buildMessage(taskID, 0); + std::cout << "send message\n" + << "type: " << unsigned(message->messageType()) << std::endl; + std::cout << "sender: " << message->sender() << std::endl; + std::cout << "seq: " << message->seq() << std::endl; + front->asyncSendMessage("1001", message, 0, nullptr, nullptr); + + auto taskID1 = "12345678"; + auto message1 = buildMessage(taskID1, 1); + std::cout << "send message1\n" + << "type: " << unsigned(message1->messageType()) << std::endl; + std::cout << "sender: " << message1->sender() << std::endl; + std::cout << "seq: " << message1->seq() << std::endl; + front->asyncSendMessage("1001", message1, 0, nullptr, nullptr); + + auto message2 = buildMessage(taskID1, 2); + std::cout << "send message2\n" + << "type: " << unsigned(message2->messageType()) << std::endl; + std::cout << "sender: " << message2->sender() << std::endl; + std::cout << "seq: " << message2->seq() << std::endl; + front->asyncSendMessage("1001", message2, 0, nullptr, nullptr); + + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + + auto info1 = std::make_shared(); + info->taskID = taskID1; + info->serviceEndpoint = "endpoint1001"; + front->notifyTaskInfo(info); + + while (flag != 3) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} diff --git a/cpp/ppc-gateway/test/unittests/GatewayTest.cpp b/cpp/ppc-gateway/test/unittests/GatewayTest.cpp new file mode 100644 index 00000000..50ea67c3 --- /dev/null +++ b/cpp/ppc-gateway/test/unittests/GatewayTest.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayTest.cpp + * @author: shawnhe + * @date 2022-10-28 + */ + +#include "ppc-gateway/ppc-gateway/Gateway.h" +#include "MockCache.h" +#include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" +#include "ppc-gateway/ppc-gateway/TaskManager.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include +#include +#include + +using namespace ppc; +using namespace ppctars; +using namespace ppc::front; +using namespace ppc::gateway; +using namespace ppc::storage; +using namespace ppc::protocol; +using namespace ppc::mock; + +using namespace bcos; +using namespace bcos::test; +using namespace ppc::tools; + + +BOOST_FIXTURE_TEST_SUITE(GatewayTest, TestPromptFixture) + + +BOOST_AUTO_TEST_CASE(test_readConfig) +{ + auto ppcConfig = std::make_shared(); + std::string configPath = "../../../ppc-gateway/test/data/config0.ini"; + ppcConfig->loadGatewayConfig(ppc::protocol::NodeArch::PRO, nullptr, configPath); + auto config = std::make_shared(ppcConfig); + auto const& gatewayConfig = ppcConfig->gatewayConfig(); + BOOST_CHECK(gatewayConfig.networkConfig.disableSsl == true); + BOOST_CHECK(config->contextConfig() == nullptr); + BOOST_CHECK(gatewayConfig.networkConfig.listenIp == "0.0.0.0"); + BOOST_CHECK(gatewayConfig.networkConfig.listenPort == 34745); + BOOST_CHECK(gatewayConfig.networkConfig.threadPoolSize == 16); + BOOST_CHECK(gatewayConfig.cacheStorageConfig.host == "127.0.0.1"); + BOOST_CHECK(gatewayConfig.cacheStorageConfig.port == 6379); + BOOST_CHECK(gatewayConfig.cacheStorageConfig.password == "123456"); + BOOST_CHECK(gatewayConfig.cacheStorageConfig.pool == 16); + BOOST_CHECK(gatewayConfig.cacheStorageConfig.database == 1); + BOOST_CHECK(gatewayConfig.agencies.at("1001").size() == 2); + BOOST_CHECK(gatewayConfig.agencies.at("1002").size() == 2); +} + +BOOST_AUTO_TEST_CASE(test_taskManager) +{ + auto cache = std::make_shared(); + auto taskManager = std::make_shared(std::make_shared()); + taskManager->registerTaskInfo("1001", "endpoint1001"); + BOOST_CHECK(taskManager->getServiceEndpoint("1001") == "endpoint1001"); + + taskManager->registerTaskInfo("1002", "endpoint1002"); + BOOST_CHECK(taskManager->getServiceEndpoint("1002") == "endpoint1002"); + + BOOST_CHECK(taskManager->getServiceEndpoint("1003").empty()); +} + +BOOST_AUTO_TEST_CASE(test_frontNodeManager) +{ + auto threadPool = std::make_shared("TEST_POOL_MODULE", 4); + auto frontFactory = std::make_shared("1001", threadPool); + + auto frontNodeManager = std::make_shared(); + + auto ioService = std::make_shared(); + frontNodeManager->registerFront("1001", frontFactory->buildFront(ioService)); + auto node1 = frontNodeManager->getFront("1001"); + auto node2 = frontNodeManager->getFront("1001"); + BOOST_CHECK(node1 && node2); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/cpp/ppc-gateway/test/unittests/MockCache.h b/cpp/ppc-gateway/test/unittests/MockCache.h new file mode 100644 index 00000000..f052b652 --- /dev/null +++ b/cpp/ppc-gateway/test/unittests/MockCache.h @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MockCache.h + * @author: shawnhe + * @date 2022-10-31 + */ +#pragma once +#include "ppc-framework/storage/CacheStorage.h" +#include + +namespace ppc::mock +{ +class MockCache : public storage::CacheStorage +{ +public: + using Ptr = std::shared_ptr; + MockCache() = default; + ~MockCache() override {} + + /// Note: all these interfaces throws exception when error happened + /** + * @brief: check whether the key exists + * @param _key: key + * @return whether the key exists + */ + bool exists(const std::string& _key) override { return m_kv.find(_key) != m_kv.end(); } + + /** + * @brief: set key value + * @param _expirationTime: timeout of key, seconds + */ + void setValue(const std::string& _key, const std::string& _value, + int32_t _expirationSeconds = -1) override + { + m_kv.emplace(_key, _value); + } + + + /** + * @brief: get value by key + * @param _key: key + * @return value + */ + Optional getValue(const std::string& _key) override + { + auto it = m_kv.find(_key); + if (it == m_kv.end()) + { + return std::nullopt; + } + + return it->second; + } + + /** + * @brief: set a timeout on key + * @param _expirationTime: timeout of key, ms + * @return whether setting is successful + */ + bool expireKey(const std::string& _key, uint32_t _expirationTime) override { return true; } + + /** + * @brief: delete key + * @param _key: key + * @return the number of key deleted + */ + uint64_t deleteKey(const std::string& _key) override { return m_kv.erase(_key); } + +private: + std::unordered_map> m_kv; +}; + +} // namespace ppc::mock \ No newline at end of file diff --git a/cpp/ppc-gateway/test/unittests/main.cpp b/cpp/ppc-gateway/test/unittests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-gateway/test/unittests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-homo/CMakeLists.txt b/cpp/ppc-homo/CMakeLists.txt new file mode 100644 index 00000000..fda32930 --- /dev/null +++ b/cpp/ppc-homo/CMakeLists.txt @@ -0,0 +1,14 @@ +project(ppc-homo VERSION ${VERSION}) + +add_subdirectory(codec) +add_subdirectory(paillier) +add_subdirectory(fahe) +add_subdirectory(ihc) +if(DEMO) + add_subdirectory(demo) +endif() +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() diff --git a/cpp/ppc-homo/codec/CMakeLists.txt b/cpp/ppc-homo/codec/CMakeLists.txt new file mode 100644 index 00000000..c623ef00 --- /dev/null +++ b/cpp/ppc-homo/codec/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${CRYPTO_CODEC_TARGET} ${SRCS}) + +find_package(OpenSSL REQUIRED) +target_link_libraries(${CRYPTO_CODEC_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-homo/codec/Common.h b/cpp/ppc-homo/codec/Common.h new file mode 100644 index 00000000..e89aeddf --- /dev/null +++ b/cpp/ppc-homo/codec/Common.h @@ -0,0 +1,58 @@ + + + +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: asherli + * @date 2023-12-29 + */ +#pragma once +#include "FloatingPointCipher.h" +#include "openssl/bn.h" +namespace ppc::homo +{ +inline void precisionAlign(FloatingPointCipher& _c1, FloatingPointCipher& _c2, + std::function const& + _alignFunc) +{ + if (_c1.exponent() == _c2.exponent()) + { + return; + } + auto alignedCipher = &_c1; + int exp = 0; + // c1.exp < c2.exp, align exponent of c2 to the exponent of c1 + if (_c1.exponent() < _c2.exponent()) + { + alignedCipher = &_c2; + exp = _c2.exponent() - _c1.exponent(); + alignedCipher->setExponent(_c1.exponent()); + } + else + { + // c1 > c2, align exponent of c1 to the exponent of c2 + exp = _c1.exponent() - _c2.exponent(); + alignedCipher->setExponent(_c2.exponent()); + } + ppc::crypto::BigNum expBN(exp); + auto ctx = ppc::crypto::createBNContext(); + ppc::crypto::BigNum c_base = ppc::crypto::BigNum(10); + auto v = c_base.exp(expBN.bn().get(), ctx.get()); + auto cipher = _alignFunc(v.bn().get(), bcos::ref(alignedCipher->cipher())); + alignedCipher->setCipher(std::move(cipher)); +} +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/codec/FloatingPointCipher.h b/cpp/ppc-homo/codec/FloatingPointCipher.h new file mode 100644 index 00000000..2f1ae330 --- /dev/null +++ b/cpp/ppc-homo/codec/FloatingPointCipher.h @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointCipher.h + * @author: yujiechen + * @date 2023-08-17 + */ +#pragma once +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-tools/src/codec/CodecUtility.h" +#include +#include +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(FloatingPointCipherException); +class FloatingPointCipher +{ +public: + using Ptr = std::shared_ptr; + FloatingPointCipher(bcos::bytes&& _cipher, int16_t _exponent) + : m_cipher(std::move(_cipher)), m_exponent(_exponent) + {} + FloatingPointCipher(bcos::bytesConstRef const& _data) { decode(_data); } + + virtual ~FloatingPointCipher() = default; + + virtual void decode(bcos::bytesConstRef const& _data) + { + auto bufferLen = _data.size(); + // exponent + auto offset = decodeInteger(m_exponent, _data.data(), bufferLen, 0); + // cipher + decodeBuffer(m_cipher, _data.data(), bufferLen, offset); + } + + virtual void encode(OutputBuffer* _output) const + { + auto pEnd = _output->data + _output->len; + // exponent + auto p = encodeInteger(_output->data, pEnd, m_exponent); + // cipher + p = encodeBuffer(p, pEnd, bcos::ref(m_cipher)); + _output->len = (p - _output->data); + } + + int maxEncodedSize() const { return m_cipher.size() + sizeof(uint16_t) + sizeof(int16_t); } + + bcos::bytes const& cipher() const { return m_cipher; } + int16_t exponent() const { return m_exponent; } + + void setCipher(bcos::bytes&& _cipher) { m_cipher = std::move(_cipher); } + void setExponent(int16_t _exponent) { m_exponent = _exponent; } + +private: + // the cipher + bcos::bytes m_cipher; + // the exponent + int16_t m_exponent; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/codec/FloatingPointCodec.cpp b/cpp/ppc-homo/codec/FloatingPointCodec.cpp new file mode 100644 index 00000000..e6812537 --- /dev/null +++ b/cpp/ppc-homo/codec/FloatingPointCodec.cpp @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointCodec.cpp + * @author: yujiechen + * @date 2023-08-23 + */ +#include "FloatingPointCodec.h" +#include +#include +#include + +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace ppc; +using namespace bcos; + +s1024 FloatingPointCodec::fpToS1024(ppc::FloatingPointNumber const& _value) +{ + auto ctx = createBNContext(); + BigNum exponent(_value.exponent); + auto result = C_BASE.exp(exponent.bn().get(), ctx.get()); + _value.value.mul(result.bn().get(), result.bn().get(), ctx.get()); + return result.toS1024(); +} + +float50 FloatingPointCodec::toFloat50(ppc::FloatingPointNumber const& _value) +{ + if (_value.exponent >= 0) + { + return float50(fpToS1024(_value)); + } + return toDecimal(_value); +} + +float50 FloatingPointCodec::toDecimal(ppc::FloatingPointNumber const& _value) +{ + auto ctx = createBNContext(); + BigNum exponent(-_value.exponent); + auto divider = C_BASE.exp(exponent.bn().get(), ctx.get()); + return (float50)_value.value.toS1024() / (float50)divider.toS1024(); +} + +s1024 FloatingPointCodec::toInt(ppc::FloatingPointNumber const& _value) +{ + if (_value.exponent >= 0) + { + return fpToS1024(_value); + } + else + { + return s1024(toDecimal(_value)); + } +} +ppc::FloatingPointNumber FloatingPointCodec::toFloatingPoint(std::string const& _valueStr) +{ + float50 value(_valueStr); + auto pos = _valueStr.find("."); + auto exp = 0; + if (pos != std::string::npos) + { + exp = -(_valueStr.size() - 1 - pos); + } + // 10^exp + auto baseExp = boost::multiprecision::pow(s1024(10), -exp); + // convert the value to int + auto significant = (s1024)(float50(baseExp) * float50(value)); + FloatingPointNumber result; + result.exponent = exp; + // convert the significant to BigNum + BigNum significantBn(significant); + result.value = significantBn; + return result; +} + +ppc::FloatingPointNumber FloatingPointCodec::toFloatingPoint(s1024 const& _value) +{ + FloatingPointNumber result; + BigNum value(_value); + result.value = value; + result.exponent = 0; + return result; +} diff --git a/cpp/ppc-homo/codec/FloatingPointCodec.h b/cpp/ppc-homo/codec/FloatingPointCodec.h new file mode 100644 index 00000000..54f5b9ce --- /dev/null +++ b/cpp/ppc-homo/codec/FloatingPointCodec.h @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointCodec.h + * @author: yujiechen + * @date 2023-08-23 + */ +#pragma once +#include +#include +#include +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(FloatingPointCodecException); + +class FloatingPointCodec +{ +public: + using Ptr = std::shared_ptr; + FloatingPointCodec() = default; + virtual ~FloatingPointCodec() = default; + s1024 toInt(ppc::FloatingPointNumber const& _value); + float50 toFloat50(ppc::FloatingPointNumber const& _value); + + ppc::FloatingPointNumber toFloatingPoint(std::string const& _value); + ppc::FloatingPointNumber toFloatingPoint(s1024 const& _value); + +private: + s1024 fpToS1024(ppc::FloatingPointNumber const& _value); + float50 toDecimal(ppc::FloatingPointNumber const& _value); + +private: + const ppc::crypto::BigNum C_BASE = ppc::crypto::BigNum(10); +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/codec/SignedNumberCodec.h b/cpp/ppc-homo/codec/SignedNumberCodec.h new file mode 100644 index 00000000..7cddcf21 --- /dev/null +++ b/cpp/ppc-homo/codec/SignedNumberCodec.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SignedNumberCodec.h + * @author: yujiechen + * @date 2023-08-09 + */ +#pragma once +#include "ppc-framework/Common.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include + +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(BigNumOutOfRange); +DERIVE_PPC_EXCEPTION(SignedNumberCodecException); +class SignedNumberCodec +{ +public: + using Ptr = std::shared_ptr; + SignedNumberCodec(unsigned _maxBits) + { + ppc::crypto::BigNum one(1); + if (BN_lshift(m_n.bn().get(), one.bn().get(), _maxBits) != 1) + { + BOOST_THROW_EXCEPTION( + SignedNumberCodecException() << bcos::errinfo_comment( + "BN_lshift error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + init(); + } + // n is the public key + SignedNumberCodec(ppc::crypto::BigNum const& _n) + { + m_n = _n; + init(); + } + + // _v mod n + ppc::crypto::BigNum encode(BIGNUM const* v) const + { + check(v); + auto ctx = ppc::crypto::createBNContext(); + ppc::crypto::BigNum result; + BN_nnmod(result.bn().get(), v, m_n.bn().get(), ctx.get()); + // for negative case check + check(result.bn().get()); + return result; + } + + // _v - ( v >= N/3) * _v + ppc::crypto::BigNum decode(ppc::crypto::BigNum const& _v) const + { + check(_v.bn().get()); + if (BN_cmp(_v.bn().get(), m_maxPositive.bn().get()) < 0) + { + return _v; + } + return _v.sub(m_n.bn().get()); + } + ppc::crypto::BigNum const& n() const { return m_n; } + ppc::crypto::BigNum const& maxPositive() const { return m_maxPositive; } + ppc::crypto::BigNum const& negativeZero() const { return m_negativeZero; } + +private: + void check(BIGNUM const* v) const + { + if (BN_cmp(v, m_n.bn().get()) >= 0) + { + BOOST_THROW_EXCEPTION( + SignedNumberCodecException() << bcos::errinfo_comment("Invalid number for over N")); + } + if (BN_cmp(v, m_maxPositive.bn().get()) > 0 && BN_cmp(v, m_negativeZero.bn().get()) < 0) + { + BOOST_THROW_EXCEPTION(SignedNumberCodecException() + << bcos::errinfo_comment("Invalid number for overflow")); + } + } + + void init() + { + ppc::crypto::BigNum three(3); + auto ctx = ppc::crypto::createBNContext(); + // calculate m_maxPositive + m_n.div(m_maxPositive.bn().get(), NULL, three.bn().get(), ctx.get()); + // calculate m_negativeZero + ppc::crypto::BigNum two(2); + m_n.mul(m_negativeZero.bn().get(), two.bn().get(), ctx.get()); + m_negativeZero.div(m_negativeZero.bn().get(), NULL, three.bn().get(), ctx.get()); + } + +private: + ppc::crypto::BigNum m_n; + ppc::crypto::BigNum m_maxPositive; // N/3 + ppc::crypto::BigNum m_negativeZero; // 2*N/3 +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/demo/CMakeLists.txt b/cpp/ppc-homo/demo/CMakeLists.txt new file mode 100644 index 00000000..452de552 --- /dev/null +++ b/cpp/ppc-homo/demo/CMakeLists.txt @@ -0,0 +1,7 @@ +# file(GLOB SRC_LIST "*.cpp") +# file(GLOB HEADERS "*.h") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +add_executable(homo_perf_demo homo_perf_demo.cpp) +# add_executable(homo_perf_demo ${SRC_LIST} ${HEADERS}) +target_link_libraries(homo_perf_demo PUBLIC ${PAILLIER_TARGET} ${FAHE_TARGET}) diff --git a/cpp/ppc-homo/demo/homo_perf_demo.cpp b/cpp/ppc-homo/demo/homo_perf_demo.cpp new file mode 100644 index 00000000..4ce504ea --- /dev/null +++ b/cpp/ppc-homo/demo/homo_perf_demo.cpp @@ -0,0 +1,311 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file homo_perf_demo.cpp + * @author: yujiechen + * @date 2023-08-09 + */ +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-homo/codec/SignedNumberCodec.h" +#include "ppc-homo/fahe/Fahe.h" +#include "ppc-homo/paillier/FloatingPointPaillier.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" + +using namespace bcos; +using namespace ppc; +using namespace ppc::crypto; +using namespace ppc::homo; + +std::pair getTPS(int64_t _endT, int64_t _startT, size_t _count) +{ + auto tps = (1000000.0 * (double)_count) / (double)(_endT - _startT); + auto timeCostUs = (1000000.0) / tps; + return std::make_pair(tps, timeCostUs); +} + +void paillierPerf(int const _keyBits, size_t _loopCount) +{ + auto paillier = std::make_shared(); + + std::cout << "----------- paillier perf start -----------" << std::endl; + std::cout << "\t keyBits: " << _keyBits << " , loopCount: " << _loopCount << std::endl; + srand(bcos::utcSteadyTime()); + /// generate keyPair + KeyPair::UniquePtr keyPair; + auto startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + keyPair = paillier->generateKeyPair(_keyBits); + } + auto pk = (PaillierPublicKey*)(keyPair->pk()); + auto codec = std::make_shared(pk->n); + auto generateKeyPairTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* GenerateKeyPair TPS: " << generateKeyPairTPS.first + << "\ttimecost per count: " << generateKeyPairTPS.second << " us" << std::endl; + + /// encrypt + startT = utcSteadyTimeUs(); + bcos::bytes cipherData; + int64_t value = 0; + for (size_t i = 0; i < _loopCount; i++) + { + value = i % 2 ? rand() : (-rand()); + BigNum v(value); + cipherData = paillier->encrypt_with_crt(v.bn().get(), (void*)keyPair.get()); + } + auto encryptTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Encrypt TPS: " << encryptTPS.first + << "\t\ttimecost per count: " << encryptTPS.second << " us" << std::endl; + + /// decrypt + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + auto result = paillier->decrypt(bcos::ref(cipherData), (void*)keyPair.get()); + } + auto decryptTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Decrypt TPS: " << decryptTPS.first + << "\t\ttimecost per count: " << decryptTPS.second << " us" << std::endl; + + /// paillier add + startT = utcSteadyTimeUs(); + bcos::bytes addCipher; + for (size_t i = 0; i < _loopCount; i++) + { + addCipher = paillier->add(bcos::ref(cipherData), bcos::ref(cipherData), keyPair->pk()); + } + auto addTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Add TPS: " << addTPS.first << "\t\ttimecost per count: " << addTPS.second + << " us" << std::endl; + + /// paillier sub + startT = utcSteadyTimeUs(); + bcos::bytes subCipher; + for (size_t i = 0; i < _loopCount; i++) + { + subCipher = paillier->sub(bcos::ref(cipherData), bcos::ref(cipherData), keyPair->pk()); + } + auto subTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Sub TPS: " << subTPS.first << "\t\ttimecost per count: " << subTPS.second + << " us" << std::endl; + /// paillier scalaMul + startT = utcSteadyTimeUs(); + bcos::bytes scalaMulCipher; + for (size_t i = 0; i < _loopCount; i++) + { + BigNum v(value); + scalaMulCipher = paillier->scalaMul(v.bn().get(), bcos::ref(cipherData), keyPair->pk()); + } + auto scalaMulTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* ScalaMul TPS: " << scalaMulTPS.first + << "\t\ttimecost per count: " << scalaMulTPS.second << " us" << std::endl; + std::cout << "----------- paillier perf end -----------" << std::endl; + std::cout << std::endl; +} + +void floatingPointPaillierPerf(int const _keyBits, size_t _loopCount, bool _needAlign) +{ + std::cout << "----------- floating-point-paillier perf start, need align: " << _needAlign + << "-----------" << std::endl; + std::cout << "\t keyBits: " << _keyBits << " , loopCount: " << _loopCount << std::endl; + auto paillierImpl = std::make_shared(); + auto fpPaillier = std::make_shared(paillierImpl); + /// prepare + // generate keyPair + auto keyPair = paillierImpl->generateKeyPair(_keyBits); + auto pk = keyPair->pk(); + std::string s1("1.23432"); + std::string s2("234234.234234324"); + std::string s3("23423.133243"); + + float50 m1(s1); + float50 m2(s2); + float50 v(s3); + /// Note: the align operation has the overhead of scalaMul + if (!_needAlign) + { + m1 = float50("10.0"); + m2 = float50("20.0"); + v = float50("1"); + } + auto codec = std::make_shared(); + auto c1Fp = codec->toFloatingPoint(s1); + auto c2Fp = codec->toFloatingPoint(s2); + auto vFp = codec->toFloatingPoint(s3); + // precision to 10 + double epsilon = 0.00000000001; + /// encrypt + bcos::bytes c1(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer c1Buffer{c1.data(), c1.size()}; + bcos::bytes c2(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer c2Buffer{c2.data(), c2.size()}; + auto startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + fpPaillier->encrypt(&c1Buffer, c1Fp, pk); + } + auto encryptTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Encrypt TPS: " << encryptTPS.first + << "\t\ttimecost per count: " << encryptTPS.second << " us" << std::endl; + fpPaillier->encrypt(&c2Buffer, c2Fp, pk); + /// decrypt + startT = utcSteadyTimeUs(); + FloatingPointNumber result; + for (size_t i = 0; i < _loopCount; i++) + { + result = fpPaillier->decrypt(ref(c1), (void*)keyPair.get()); + } + auto decryptTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Decrypt TPS: " << decryptTPS.first + << "\t\ttimecost per count: " << decryptTPS.second << " us" << std::endl; + // check the result + assert(fabs(codec->toFloat50(result) - m1) <= epsilon); + /// add + bcos::bytes addResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer addResultBuffer{addResult.data(), addResult.size()}; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + fpPaillier->add(&addResultBuffer, ref(c1), ref(c2), pk); + } + auto addTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Add TPS: " << addTPS.first << "\t\ttimecost per count: " << addTPS.second + << " us" << std::endl; + + /// sub + bcos::bytes subResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer subResultBuffer{subResult.data(), subResult.size()}; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + fpPaillier->sub(&subResultBuffer, ref(c1), ref(c2), pk); + } + auto subTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* Sub TPS: " << subTPS.first << "\t\ttimecost per count: " << subTPS.second + << " us" << std::endl; + /// scalaMul + bcos::bytes scalaMulResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer scalaMulResultBuffer{scalaMulResult.data(), scalaMulResult.size()}; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < _loopCount; i++) + { + fpPaillier->scalaMul(&scalaMulResultBuffer, vFp, ref(c1), pk); + } + auto scalaMulTPS = getTPS(utcSteadyTimeUs(), startT, _loopCount); + std::cout << "* ScalaMul TPS: " << scalaMulTPS.first + << "\t\ttimecost per count: " << scalaMulTPS.second << " us" << std::endl; + std::cout << "----------- floating-point-paillier perf finished -----------" << std::endl; +} + +void fahePerf(int lambda, int alpha, int mmax, size_t loopCount) +{ + std::cout << "----------- fahe perf start -----------" << std::endl; + std::cout << "* lambda: " << lambda << std::endl; + std::cout << "* alpha: " << alpha << std::endl; + std::cout << "* mmax: " << mmax << std::endl; + std::cout << "* loopCount: " << loopCount << std::endl; + auto faheImpl = std::make_shared(); + FaheKey::UniquePtr key; + auto startT = utcSteadyTimeUs(); + for (size_t i = 0; i < loopCount; i++) + { + key = faheImpl->generateKey(lambda, mmax, alpha); + } + auto genKeyTPS = getTPS(utcSteadyTimeUs(), startT, loopCount); + std::cout << "* generateKey TPS: " << genKeyTPS.first + << "\t\ttimecost per count: " << genKeyTPS.second << " us" << std::endl; + // encrypt + BigNum m1(123213231 + bcos::utcSteadyTime()); + BigNum c1; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < loopCount; i++) + { + c1 = faheImpl->encrypt(m1.bn().get(), key.get()); + } + auto encryptTPS = getTPS(utcSteadyTimeUs(), startT, loopCount); + std::cout << "* encrypt TPS: " << encryptTPS.first + << "\t\ttimecost per count: " << encryptTPS.second << " us" << std::endl; + BigNum m2(-(1232231 + bcos::utcSteadyTime())); + auto c2 = faheImpl->encrypt(m2.bn().get(), key.get()); + + // add + BigNum addResult; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < loopCount; i++) + { + addResult = faheImpl->add(c1.bn().get(), c2.bn().get()); + } + auto addTPS = getTPS(utcSteadyTimeUs(), startT, loopCount); + std::cout << "* fahe add TPS: " << addTPS.first << "\t\ttimecost per count: " << addTPS.second + << " us" << std::endl; + // decrypt + BigNum decryptedResult; + startT = utcSteadyTimeUs(); + for (size_t i = 0; i < loopCount; i++) + { + decryptedResult = faheImpl->decrypt(addResult.bn().get(), key.get()); + } + auto decryptTPS = getTPS(utcSteadyTimeUs(), startT, loopCount); + std::cout << "* fahe decrypt TPS: " << decryptTPS.first + << "\t\ttimecost per count: " << decryptTPS.second << " us" << std::endl; + // check the result + auto expectedAddResult = m1.add(m2.bn().get()); + assert(expectedAddResult.cmp(decryptedResult.bn().get()) == 0); + std::cout << "----------- fahe perf end -----------" << std::endl; + + /// check the correctness of fahe + std::cout << "----------- fahe perf correctness check -----------" << std::endl; + uint64_t count = 1000000; + srand(utcSteadyTime()); + for (size_t i = 0; i < count; i++) + { + BigNum randM1(rand() + utcSteadyTime()); + BigNum randM2(rand() + utcSteadyTimeUs()); + auto randC1 = faheImpl->encrypt(randM1.bn().get(), key.get()); + auto randC2 = faheImpl->encrypt(randM2.bn().get(), key.get()); + // add + auto addCipher = faheImpl->add(randC1.bn().get(), randC2.bn().get()); + // decrypt + auto addResult = faheImpl->decrypt(addCipher.bn().get(), key.get()); + auto expectedAddResult = randM1.add(randM2.bn().get()); + assert(expectedAddResult.cmp(addResult.bn().get()) == 0); + } + std::cout << "----------- fahe perf correctness check finished -----------" << std::endl; +} + +int main(int argc, char* argv[]) +{ + if (argc < 2) + { + std::cout << "Usage: " << argv[0] << "\n loopCount [required]" + << "\n keyBitLength[optional]: default is 2048" << std::endl; + return -1; + } + int keyBits = 2048; + int loopCount = atol(argv[1]); + if (argc >= 3) + { + keyBits = atol(argv[2]); + } + paillierPerf(keyBits, loopCount); + floatingPointPaillierPerf(keyBits, loopCount, false); + floatingPointPaillierPerf(keyBits, loopCount, true); + int lambda = 128; + int alpha = 33; + int mmax = 64; + fahePerf(lambda, alpha, mmax, loopCount); + return 0; +} \ No newline at end of file diff --git a/cpp/ppc-homo/fahe/CMakeLists.txt b/cpp/ppc-homo/fahe/CMakeLists.txt new file mode 100644 index 00000000..af920edf --- /dev/null +++ b/cpp/ppc-homo/fahe/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${FAHE_TARGET} ${SRCS}) + +find_package(OpenSSL REQUIRED) +target_link_libraries(${FAHE_TARGET} PUBLIC ${CRYPTO_CODEC_TARGET} ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-homo/fahe/Fahe.cpp b/cpp/ppc-homo/fahe/Fahe.cpp new file mode 100644 index 00000000..3ff270ff --- /dev/null +++ b/cpp/ppc-homo/fahe/Fahe.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Fahe.cpp + * @author: yujiechen + * @date 2023-08-30 + */ +// clang-format off +#include +#include +// clang-format on +#include "Fahe.h" +#include "../codec/SignedNumberCodec.h" +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace std; + +FaheKey::UniquePtr Fahe::generateKey(int16_t _lambda, int16_t _mmax, int16_t _alpha) +{ + auto key = std::make_unique(); + key->lambda = _lambda; + key->alpha = _alpha; + key->mmax = _mmax; + srand(bcos::utcTime()); + + key->pos = rand() % _lambda; + double tmp = _lambda + _alpha + _mmax; + auto n = tmp + _alpha; + // p + key->p.generatePrime(n); + // x = 2^r/p + BigNum base(2); + // r = p/lgp *（(n-tmp)^2） + // divider = (log10(_lambda + _alpha + _mmax)); + int16_t r = (int16_t)((double)tmp / (log10((double)tmp)) * _alpha * _alpha); + BigNum exponent(r); + auto ctx = createBNContext(); + key->x = base.exp(exponent.bn().get(), ctx.get()); + key->x.div(key->x.bn().get(), NULL, key->p.bn().get(), ctx.get()); + return key; +} + +BigNum Fahe::encrypt(BIGNUM const* _m, FaheKey const* _key) +{ + // convert _m to signed + SignedNumberCodec codec(_key->mmax); + auto m = codec.encode(_m); + + auto noise1 = generateRand(_key->pos); + auto noise2 = generateRand(_key->lambda - _key->pos); + // M = (noise2 << (pos + mmax + alpha) + (m << (pos + alpha)) + noise1) + BigNum M1(noise2); + auto shiftStep = _key->pos + _key->mmax + _key->alpha; + if (BN_lshift(M1.bn().get(), M1.bn().get(), shiftStep) != 1) + { + BOOST_THROW_EXCEPTION( + FaheException() << bcos::errinfo_comment( + "encrypt error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + BigNum M2; + if (BN_lshift(M2.bn().get(), m.bn().get(), (_key->pos + _key->alpha)) != 1) + { + BOOST_THROW_EXCEPTION( + FaheException() << bcos::errinfo_comment( + "encrypt error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + auto M = M1.add(M2.bn().get()).add(noise1.bn().get()); + // select q from [0, x) + BigNum q; + randRange(q.bn().get(), _key->x.bn().get()); + // n = p * q + BigNum n; + auto ctx = createBNContext(); + q.mul(n.bn().get(), _key->p.bn().get(), ctx.get()); + // c = n + M + return n.add(M.bn().get()); +} + +BigNum Fahe::decrypt(BIGNUM const* _cipher, FaheKey const* _key) +{ + BigNum result; + /// result = (c mod p) >> (pos + alpha) + // cipher mod p + auto ctx = createBNContext(); + if (BN_div(NULL, result.bn().get(), _cipher, _key->p.bn().get(), ctx.get()) != 1) + { + BOOST_THROW_EXCEPTION( + FaheException() << bcos::errinfo_comment( + "decrypt error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + // result = (cipher mod p) >> (pos + alpha) + if (BN_rshift(result.bn().get(), result.bn().get(), (_key->pos + _key->alpha)) != 1) + { + BOOST_THROW_EXCEPTION( + FaheException() << bcos::errinfo_comment( + "decrypt error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + BN_mask_bits(result.bn().get(), _key->mmax); + SignedNumberCodec codec(_key->mmax); + return codec.decode(result); +} + +BigNum Fahe::add(BIGNUM const* _cipher1, BIGNUM const* _cipher2) +{ + BigNum result; + if (BN_add(result.bn().get(), _cipher1, _cipher2) != 1) + { + BOOST_THROW_EXCEPTION( + FaheException() << bcos::errinfo_comment( + "add error: " + std::string(ERR_error_string(ERR_get_error(), NULL)))); + } + return result; +} diff --git a/cpp/ppc-homo/fahe/Fahe.h b/cpp/ppc-homo/fahe/Fahe.h new file mode 100644 index 00000000..a43372da --- /dev/null +++ b/cpp/ppc-homo/fahe/Fahe.h @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Fahe.h + * @author: yujiechen + * @date 2023-08-30 + */ +#pragma once +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include +#include +#include +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(FaheException); +struct FaheKey +{ + int16_t lambda; + int16_t alpha; + int16_t pos; + int16_t mmax; + ppc::crypto::BigNum p; + ppc::crypto::BigNum x; + + using UniquePtr = std::unique_ptr; + FaheKey() = default; +}; +// FAHE2 implementation: +// Fast Additive Partially Homomorphic Encryption from the Approximate Common Divisor problem +class Fahe +{ +public: + using Ptr = std::shared_ptr; + Fahe() = default; + virtual ~Fahe() = default; + + virtual FaheKey::UniquePtr generateKey(int16_t _lambda, int16_t _mmax, int16_t _alpha); + virtual ppc::crypto::BigNum encrypt(BIGNUM const* _plain, FaheKey const* _key); + virtual ppc::crypto::BigNum decrypt(BIGNUM const* _cipher, FaheKey const* _key); + virtual ppc::crypto::BigNum add(BIGNUM const* _cipher1, BIGNUM const* _cipher2); +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/ihc/CMakeLists.txt b/cpp/ppc-homo/ihc/CMakeLists.txt new file mode 100644 index 00000000..d0dc57d5 --- /dev/null +++ b/cpp/ppc-homo/ihc/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library( ${IHC_TARGET} ${SRCS}) + +find_package(OpenSSL REQUIRED) +# target_link_libraries(${IHC_TARGET} PUBLIC ${CRYPTO_CODEC_TARGET} ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) +target_link_libraries(${IHC_TARGET} PUBLIC ${CRYPTO_CODEC_TARGET} ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-homo/ihc/FloatingPointIhc.cpp b/cpp/ppc-homo/ihc/FloatingPointIhc.cpp new file mode 100644 index 00000000..5d7267f8 --- /dev/null +++ b/cpp/ppc-homo/ihc/FloatingPointIhc.cpp @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License") {} + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointIhc.h + * @author: asherli + * @date 2023-12-29 + */ +#include "FloatingPointIhc.h" + +using namespace ppc::homo; +using namespace ppc; + +void FloatingPointIhc::encrypt( + OutputBuffer* _result, bcos::bytesConstRef const& sk, FloatingPointNumber const& _value) const +{ + auto cipher = m_ihc->encrypt(sk, _value.value.bn().get()); + FloatingPointCipher cipherObject(std::move(cipher), _value.exponent); + cipherObject.encode(_result); +} + +FloatingPointNumber FloatingPointIhc::decrypt( + bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const +{ + FloatingPointCipher cipherObject(_cipher); + auto resultV = m_ihc->decrypt(_sk, bcos::ref(cipherObject.cipher())); + return FloatingPointNumber(std::move(resultV), cipherObject.exponent()); +} + +void FloatingPointIhc::add( + OutputBuffer* _result, bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const +{ + FloatingPointCipher cipher1(_c1); + FloatingPointCipher cipher2(_c2); + align(cipher1, cipher2); + auto cipher = m_ihc->add(bcos::ref(cipher1.cipher()), bcos::ref(cipher2.cipher())); + FloatingPointCipher result(std::move(cipher), cipher1.exponent()); + result.encode(_result); +} + +void FloatingPointIhc::sub( + OutputBuffer* _result, bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const +{ + FloatingPointCipher cipher1(_c1); + FloatingPointCipher cipher2(_c2); + align(cipher1, cipher2); + auto cipher = m_ihc->sub(bcos::ref(cipher1.cipher()), bcos::ref(cipher2.cipher())); + FloatingPointCipher result(std::move(cipher), cipher1.exponent()); + result.encode(_result); +} + +void FloatingPointIhc::scalaMul( + OutputBuffer* _result, FloatingPointNumber const& _v, bcos::bytesConstRef const& _c) const +{ + FloatingPointCipher cipher(_c); + auto resultBytes = m_ihc->scalaMul(_v.value.bn().get(), bcos::ref(cipher.cipher())); + auto exponent = _v.exponent + cipher.exponent(); + FloatingPointCipher result(std::move(resultBytes), exponent); + result.encode(_result); +} \ No newline at end of file diff --git a/cpp/ppc-homo/ihc/FloatingPointIhc.h b/cpp/ppc-homo/ihc/FloatingPointIhc.h new file mode 100644 index 00000000..adcaee7d --- /dev/null +++ b/cpp/ppc-homo/ihc/FloatingPointIhc.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointIhc.h + * @author: asherli + * @date 2023-12-29 + */ +#pragma once +#include "../codec/Common.h" +#include "IhcImpl.h" +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include +namespace ppc::homo +{ +class FloatingPointIhc : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + FloatingPointIhc(Ihc::Ptr _ihc) : m_ihc(std::move(_ihc)) {} + virtual ~FloatingPointIhc() = default; + + virtual void encrypt(OutputBuffer* _result, bcos::bytesConstRef const& sk, + ppc::FloatingPointNumber const& _value) const; + + virtual bcos::bytes encrypt( + bcos::bytesConstRef const& sk, ppc::FloatingPointNumber const& _value) const + { + bcos::bytes result(cipherBytes()); + OutputBuffer resultBuffer{result.data(), result.size()}; + encrypt(&resultBuffer, sk, _value); + result.resize(resultBuffer.len); + return result; + } + + virtual ppc::FloatingPointNumber decrypt( + bcos::bytesConstRef const& _sk, bcos::bytesConstRef const& _cipher) const; + + virtual void add(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const; + + virtual bcos::bytes add(bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const + { + bcos::bytes result(cipherBytes()); + OutputBuffer resultBuffer{result.data(), result.size()}; + add(&resultBuffer, _c1, _c2); + result.resize(resultBuffer.len); + return result; + } + + virtual void sub(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const; + virtual bcos::bytes sub(bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) + { + bcos::bytes result(cipherBytes()); + OutputBuffer resultBuffer{result.data(), result.size()}; + sub(&resultBuffer, _c1, _c2); + result.resize(resultBuffer.len); + return result; + } + + virtual void scalaMul(OutputBuffer* _result, ppc::FloatingPointNumber const& _v, + bcos::bytesConstRef const& _c) const; + + virtual bcos::bytes scalaMul(ppc::FloatingPointNumber const& _v, bcos::bytesConstRef const& _c) + { + bcos::bytes result(cipherBytes()); + OutputBuffer resultBuffer{result.data(), result.size()}; + scalaMul(&resultBuffer, _v, _c); + result.resize(resultBuffer.len); + return result; + } + + virtual uint64_t cipherBytes() const + { + return m_ihc->cipherBytes() + sizeof(uint16_t) + sizeof(int16_t); + } + +private: + inline void align(FloatingPointCipher& _c1, FloatingPointCipher& _c2) const + { + auto self = weak_from_this(); + precisionAlign( + _c1, _c2, [self](BIGNUM const* v, bcos::bytesConstRef const& _cipher) -> bcos::bytes { + auto impl = self.lock(); + if (!impl) + { + return bcos::bytes(); + } + return impl->m_ihc->scalaMul(v, _cipher); + }); + } + +private: + Ihc::Ptr m_ihc; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/ihc/IhcImpl.cpp b/cpp/ppc-homo/ihc/IhcImpl.cpp new file mode 100644 index 00000000..b63cc6fd --- /dev/null +++ b/cpp/ppc-homo/ihc/IhcImpl.cpp @@ -0,0 +1,179 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IhcImpl.cpp + * @author: asherli + * @date 2023-11-23 + */ +#include "IhcImpl.h" +#include "ppc-tools/src/codec/CodecUtility.h" + +using namespace ppc::homo; +using namespace ppc::crypto; + +// generate key according to given mode +BigNum IhcImpl::generateKeyImpl() const +{ + BigNum key; + do + { + randRange(key.bn().get(), m_codec->n().bn().get()); + } while (BN_is_zero(key.bn().get())); + return key; +} + +void IhcImpl::generateKey(OutputBuffer* _result) const +{ + if (_result->len < keyBytes()) + { + BOOST_THROW_EXCEPTION( + IhcException() << bcos::errinfo_comment( + "IHC error for un-enough key buffer, current len: " + std::to_string(_result->len) + + ", min key len: " + std::to_string(keyBytes()))); + } + auto key = generateKeyImpl(); + key.toOutputBuffer(_result, false); +} + +bcos::bytes IhcImpl::generateKey() const +{ + bcos::bytes result(keyBytes()); + generateKeyImpl().toBytes(result, false); + return result; +} +// encrypt the given value to cipher using given key +void IhcImpl::encrypt( + OutputBuffer* _cipher, bcos::bytesConstRef const& _key, BIGNUM const* _value) const +{ + auto v = m_codec->encode(_value); + + BigNum lastRoundV; + randRange(lastRoundV.bn().get(), m_codec->n().bn().get()); + BigNum key(_key, false); + auto ctx = createBNContext(); + for (int i = 0; i < m_iterRound; ++i) + { + BigNum tmp; + v.modMul(tmp.bn().get(), key.bn().get(), m_codec->n().bn().get(), ctx.get()); + tmp.modSub(tmp.bn().get(), lastRoundV.bn().get(), m_codec->n().bn().get(), ctx.get()); + lastRoundV = v; + v = tmp; + } + // c1: v + auto pEnd = _cipher->data + _cipher->len; + auto pBuffer = encodeBigNum(_cipher->data, pEnd, v); + // c2: lastRoundV + pBuffer = encodeBigNum(pBuffer, pEnd, lastRoundV); + _cipher->len = (pBuffer - _cipher->data); +} + +// decrypt the given cipher to value using given key +ppc::crypto::BigNum IhcImpl::decrypt( + bcos::bytesConstRef const& _key, bcos::bytesConstRef const& _cipher) const +{ + BigNum lastRoundV; + auto offset = decodeBigNum(lastRoundV, _cipher.data(), _cipher.size(), 0); + BigNum v; + decodeBigNum(v, _cipher.data(), _cipher.size(), offset); + BigNum key(_key, false); + auto ctx = createBNContext(); + for (int i = 0; i < m_iterRound - 1; ++i) + { + BigNum tmp; + v.modMul(tmp.bn().get(), key.bn().get(), m_codec->n().bn().get(), ctx.get()); + tmp.modSub(tmp.bn().get(), lastRoundV.bn().get(), m_codec->n().bn().get(), ctx.get()); + lastRoundV = v; + v = tmp; + } + return m_codec->decode(v); +} + +// ihc add +void IhcImpl::arithmeticImpl(OutputBuffer* _addResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2, ArithmeticType _type) const +{ + // c1 + BigNum leftV; + auto offset = decodeBigNum(leftV, _c1.data(), _c1.size(), 0); + BigNum leftLastRoundV; + decodeBigNum(leftLastRoundV, _c1.data(), _c1.size(), offset); + // c2 + BigNum rightV; + offset = decodeBigNum(rightV, _c2.data(), _c2.size(), 0); + BigNum rightLastRoundV; + decodeBigNum(rightLastRoundV, _c2.data(), _c2.size(), offset); + // c1 + c2 + BigNum resultV; + BigNum resultLastRoundV; + auto ctx = createBNContext(); + switch (_type) + { + case IhcImpl::ArithmeticType::ADD: + { + resultV = leftV.modAdd(rightV.bn().get(), m_codec->n().bn().get()); + resultLastRoundV = + leftLastRoundV.modAdd(rightLastRoundV.bn().get(), m_codec->n().bn().get()); + break; + } + case IhcImpl::ArithmeticType::SUB: + { + leftV.modSub(resultV.bn().get(), rightV.bn().get(), m_codec->n().bn().get(), ctx.get()); + leftLastRoundV.modSub(resultLastRoundV.bn().get(), rightLastRoundV.bn().get(), + m_codec->n().bn().get(), ctx.get()); + break; + } + default: + BOOST_THROW_EXCEPTION(IhcException() << bcos::errinfo_comment( + "Unsupported ArithmeticType: " + std::to_string((int)_type))); + } + auto pEnd = _addResult->data + _addResult->len; + auto pBuffer = encodeBigNum(_addResult->data, pEnd, resultV); + pBuffer = encodeBigNum(pBuffer, pEnd, resultLastRoundV); + _addResult->len = (pBuffer - _addResult->data); +} + +void IhcImpl::add( + OutputBuffer* _addResult, bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const +{ + arithmeticImpl(_addResult, _c1, _c2, IhcImpl::ArithmeticType::ADD); +} + +void IhcImpl::sub( + OutputBuffer* _subResult, bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const +{ + arithmeticImpl(_subResult, _c1, _c2, IhcImpl::ArithmeticType::SUB); +} + +// ihc scalaMul +void IhcImpl::scalaMul( + OutputBuffer* _mulResult, BIGNUM const* _value, bcos::bytesConstRef const& _cipher) const +{ + // c1 + BigNum leftV; + auto offset = decodeBigNum(leftV, _cipher.data(), _cipher.size(), 0); + BigNum leftLastRoundV; + decodeBigNum(leftLastRoundV, _cipher.data(), _cipher.size(), offset); + BigNum resultV; + BigNum resultLastRoundV; + auto ctx = createBNContext(); + + leftV.modMul(resultV.bn().get(), _value, m_codec->n().bn().get(), ctx.get()); + leftLastRoundV.modMul(resultLastRoundV.bn().get(), _value, m_codec->n().bn().get(), ctx.get()); + + auto pEnd = _mulResult->data + _mulResult->len; + auto pBuffer = encodeBigNum(_mulResult->data, pEnd, resultV); + pBuffer = encodeBigNum(pBuffer, pEnd, resultLastRoundV); + _mulResult->len = (pBuffer - _mulResult->data); +} \ No newline at end of file diff --git a/cpp/ppc-homo/ihc/IhcImpl.h b/cpp/ppc-homo/ihc/IhcImpl.h new file mode 100644 index 00000000..242a80a6 --- /dev/null +++ b/cpp/ppc-homo/ihc/IhcImpl.h @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IhcImpl.h + * @author: asherli + * @date 2023-11-23 + */ + +#pragma once +#include "ppc-framework/Common.h" +#include "ppc-framework/crypto/Ihc.h" +#include "ppc-homo/codec/SignedNumberCodec.h" +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(IhcException); +// Ihc implementation: +class IhcImpl : public Ihc +{ +public: + using Ptr = std::shared_ptr; + IhcImpl(int _mode, int _iterRound) : m_mode(_mode), m_iterRound(_iterRound) + { + switch (_mode) + { + case (int)Ihc::IhcMode::IHC_128: + m_keyBits = 128; + break; + case (int)Ihc::IhcMode::IHC_256: + m_keyBits = 256; + break; + default: + BOOST_THROW_EXCEPTION(IhcException() << bcos::errinfo_comment( + "Unsupported mode, only support IHC_128/IHC_256 now")); + } + m_iterRound = 16; + m_codec = std::make_shared(m_keyBits); + } + ~IhcImpl() override {} + + // generate key according to given mode + bcos::bytes generateKey() const override; + void generateKey(OutputBuffer* _result) const override; + + // encrypt the given value to cipher using given key + void encrypt(OutputBuffer* _cipher, bcos::bytesConstRef const& _key, + BIGNUM const* _value) const override; + + bcos::bytes encrypt(bcos::bytesConstRef const& _key, BIGNUM const* _value) const override + { + bcos::bytes result(cipherBytes()); + OutputBuffer resultCipher{result.data(), result.size()}; + encrypt(&resultCipher, _key, _value); + result.resize(resultCipher.len); + return result; + } + + // decrypt the given cipher to value using given key + ppc::crypto::BigNum decrypt( + bcos::bytesConstRef const& _key, bcos::bytesConstRef const& _cipher) const override; + + // ihc add + void add(OutputBuffer* _addResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const override; + bcos::bytes add(bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const override + { + bcos::bytes result(cipherBytes()); + OutputBuffer buffer{result.data(), result.size()}; + add(&buffer, _c1, _c2); + result.resize(buffer.len); + return result; + } + // ihc sub + void sub(OutputBuffer* _subResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2) const override; + + bcos::bytes sub(bcos::bytesConstRef const& _c1, bcos::bytesConstRef const& _c2) const override + { + bcos::bytes result(cipherBytes()); + OutputBuffer buffer{result.data(), result.size()}; + sub(&buffer, _c1, _c2); + result.resize(buffer.len); + return result; + } + // ihc scalaMul + void scalaMul(OutputBuffer* _mulResult, BIGNUM const* _value, + bcos::bytesConstRef const& _cipher) const override; + + bcos::bytes scalaMul(BIGNUM const* _value, bcos::bytesConstRef const& _cipher) const override + { + bcos::bytes result(cipherBytes()); + OutputBuffer buffer{result.data(), result.size()}; + scalaMul(&buffer, _value, _cipher); + result.resize(buffer.len); + return result; + } + + unsigned int keyBits() const override { return m_keyBits; } + unsigned int keyBytes() const override { return (m_keyBits + 7) / 8; } + uint64_t cipherBytes() const override { return 6 + (m_keyBits * 2) / 8; } + + int mode() const { return m_mode; } + +private: + enum class ArithmeticType + { + ADD, + SUB, + }; + void arithmeticImpl(OutputBuffer* _addResult, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2, ArithmeticType _type) const; + + ppc::crypto::BigNum generateKeyImpl() const; + +private: + int m_mode; + int m_iterRound; + unsigned int m_keyBits; + SignedNumberCodec::Ptr m_codec; +}; +} // namespace ppc::homo diff --git a/cpp/ppc-homo/paillier/CMakeLists.txt b/cpp/ppc-homo/paillier/CMakeLists.txt new file mode 100644 index 00000000..d6856fcb --- /dev/null +++ b/cpp/ppc-homo/paillier/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${PAILLIER_TARGET} ${SRCS}) + +find_package(OpenSSL REQUIRED) +target_link_libraries(${PAILLIER_TARGET} PUBLIC ${CRYPTO_CODEC_TARGET} ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/Common.h b/cpp/ppc-homo/paillier/Common.h new file mode 100644 index 00000000..ba332866 --- /dev/null +++ b/cpp/ppc-homo/paillier/Common.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2023-8-4 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +#include + +#define HOMO_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("HOMOCRYPTO") + +namespace ppc::homo +{ +DERIVE_PPC_EXCEPTION(InvalidHomoKeyPair); +DERIVE_PPC_EXCEPTION(InvalidHomoPublicKey); +DERIVE_PPC_EXCEPTION(OpenSSLPaillierException); +DERIVE_PPC_EXCEPTION(FloatingPointPaillierException); +} // namespace ppc::homo diff --git a/cpp/ppc-homo/paillier/FloatingPointPaillier.cpp b/cpp/ppc-homo/paillier/FloatingPointPaillier.cpp new file mode 100644 index 00000000..8c8fba9c --- /dev/null +++ b/cpp/ppc-homo/paillier/FloatingPointPaillier.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointPaillier.cpp + * @author: yujiechen + * @date 2023-08-17 + */ + +#include "FloatingPointPaillier.h" +#include "../codec/Common.h" + +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace ppc; + +void FloatingPointPaillier::encryptFast( + OutputBuffer* _result, FloatingPointNumber const& _value, void* _keyPair) const +{ + auto cipher = m_paillierImpl->encrypt_with_crt(_value.value.bn().get(), _keyPair); + FloatingPointCipher cipherObject(std::move(cipher), _value.exponent); + cipherObject.encode(_result); +} + +void FloatingPointPaillier::encrypt( + OutputBuffer* _result, FloatingPointNumber const& _value, void* _publicKey) const +{ + auto cipher = m_paillierImpl->encrypt(_value.value.bn().get(), _publicKey); + FloatingPointCipher cipherObject(std::move(cipher), _value.exponent); + cipherObject.encode(_result); +} + + +FloatingPointNumber FloatingPointPaillier::decrypt( + bcos::bytesConstRef const& _cipherData, void* _keyPair) const +{ + FloatingPointCipher cipherObject(_cipherData); + // decrypt + auto decodedV = m_paillierImpl->decrypt(bcos::ref(cipherObject.cipher()), _keyPair); + return FloatingPointNumber(std::move(decodedV), cipherObject.exponent()); +} + +void FloatingPointPaillier::align( + FloatingPointCipher& _c1, FloatingPointCipher& _c2, void* _publicKey) const +{ + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(FloatingPointPaillierException() << bcos::errinfo_comment( + "precisionAlign for two-cipher error for invalid public key")); + } + auto self = weak_from_this(); + precisionAlign(_c1, _c2, + [self, _publicKey](BIGNUM const* v, bcos::bytesConstRef const& _cipher) -> bcos::bytes { + auto fpPaillier = self.lock(); + if (!fpPaillier) + { + return bcos::bytes(); + } + return fpPaillier->m_paillierImpl->scalaMul(v, _cipher, _publicKey); + }); +} + +void FloatingPointPaillier::add(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2, void* _publicKey) const +{ + FloatingPointCipher c1Obj(_c1); + FloatingPointCipher c2Obj(_c2); + // align the precision + align(c1Obj, c2Obj, _publicKey); + auto addResult = + m_paillierImpl->add(bcos::ref(c1Obj.cipher()), bcos::ref(c2Obj.cipher()), _publicKey); + FloatingPointCipher cipherAddResult(std::move(addResult), c1Obj.exponent()); + cipherAddResult.encode(_result); +} + +void FloatingPointPaillier::sub(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& _c2, void* _publicKey) const +{ + FloatingPointCipher c1Obj(_c1); + FloatingPointCipher c2Obj(_c2); + // align the precision + align(c1Obj, c2Obj, _publicKey); + auto subResult = + m_paillierImpl->sub(bcos::ref(c1Obj.cipher()), bcos::ref(c2Obj.cipher()), _publicKey); + FloatingPointCipher cipherSubResult(std::move(subResult), c1Obj.exponent()); + cipherSubResult.encode(_result); +} + +void FloatingPointPaillier::scalaMul(OutputBuffer* _result, FloatingPointNumber const& _v, + bcos::bytesConstRef const& _c, void* _publicKey) const +{ + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(FloatingPointPaillierException() + << bcos::errinfo_comment("scalaMul error for invalid public key")); + } + FloatingPointCipher cipherObject(_c); + auto mulResult = + m_paillierImpl->scalaMul(_v.value.bn().get(), bcos::ref(cipherObject.cipher()), _publicKey); + FloatingPointCipher cipherMulResult( + std::move(mulResult), cipherObject.exponent() + _v.exponent); + cipherMulResult.encode(_result); +} \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/FloatingPointPaillier.h b/cpp/ppc-homo/paillier/FloatingPointPaillier.h new file mode 100644 index 00000000..a2e164e2 --- /dev/null +++ b/cpp/ppc-homo/paillier/FloatingPointPaillier.h @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FloatingPointPaillier.h + * @author: yujiechen + * @date 2023-08-17 + */ +#pragma once +#include "../codec/FloatingPointCipher.h" +#include "../codec/SignedNumberCodec.h" +#include "OpenSSLPaillier.h" +#include "ppc-framework/crypto/Paillier.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include + +namespace ppc::homo +{ +class FloatingPointPaillier : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + FloatingPointPaillier(Paillier::Ptr const& _paillierImpl) : m_paillierImpl(_paillierImpl) {} + virtual ~FloatingPointPaillier() = default; + + virtual void encryptFast( + OutputBuffer* _result, ppc::FloatingPointNumber const& _value, void* _keyPair) const; + virtual void encrypt( + OutputBuffer* _result, ppc::FloatingPointNumber const& _value, void* _publicKey) const; + virtual ppc::FloatingPointNumber decrypt( + bcos::bytesConstRef const& _cipherData, void* _keyPair) const; + + virtual void add(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& c2, void* _publicKey) const; + virtual void sub(OutputBuffer* _result, bcos::bytesConstRef const& _c1, + bcos::bytesConstRef const& c2, void* _publicKey) const; + + // Note: the FloatingPointNumber maybe modified when aligning + virtual void scalaMul(OutputBuffer* _result, ppc::FloatingPointNumber const& _v, + bcos::bytesConstRef const& c, void* _publicKey) const; + + Paillier::Ptr const paillierImpl() const { return m_paillierImpl; } + static unsigned int maxCipherBytesLen(unsigned int _keyBits) + { + return sizeof(uint16_t) + sizeof(int16_t) + OpenSSLPaillier::maxCipherBytesLen(_keyBits); + } + +private: + // align the exponent between cipher c1 and c2 + void align(FloatingPointCipher& c1, FloatingPointCipher& c2, void* _publicKey) const; + +private: + Paillier::Ptr m_paillierImpl; + const ppc::crypto::BigNum C_BASE = ppc::crypto::BigNum(10); +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillier.cpp b/cpp/ppc-homo/paillier/OpenSSLPaillier.cpp new file mode 100644 index 00000000..302b2f27 --- /dev/null +++ b/cpp/ppc-homo/paillier/OpenSSLPaillier.cpp @@ -0,0 +1,315 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLPaillier.cpp + * @author: yujiechen + * @date 2023-08-04 + */ +#include "OpenSSLPaillier.h" +#include "Common.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include + +using namespace ppc::homo; +using namespace ppc::crypto; +using namespace bcos; + +// generate the keypair +KeyPair::UniquePtr OpenSSLPaillier::generateKeyPair(unsigned const _keyBits) const +{ + BigNum gcdResult; + BigNum two; + BigNum pMinus; + BigNum qMinus; + two.setWord(2); + auto ctx = createBNContext(); + auto sk = std::make_shared(); + + do + { + // generate prime p and q, ensure p != q, and p = 3 mod 4, q = 3 mod 4 + do + { + sk->p.generatePrime(_keyBits / 2); + } while (!sk->p.isBitSet(1)); + do + { + sk->q.generatePrime(_keyBits / 2); + } while (!sk->q.isBitSet(1) || (sk->q.cmp(sk->p.bn().get()) == 0)); + // calculate p-1 and q-1 + pMinus = sk->p.sub(BN_value_one()); + qMinus = sk->q.sub(BN_value_one()); + + // calculate gcd(p-1, q-1), break when gcd(p-1, q-1) == 2 + pMinus.gcd(gcdResult.bn().get(), qMinus.bn().get(), ctx.get()); + } while (gcdResult.cmp(two.bn().get())); + + auto pk = std::make_shared(); + pk->keyBits = _keyBits; + sk->keyBits = _keyBits; + + // calculate n = p * q + sk->p.mul(pk->n.bn().get(), sk->q.bn().get(), ctx.get()); + + // generate random x + BigNum x; + randRange(x.bn().get(), pk->n.bn().get()); + // calculate h = -x^2 + x.mul(pk->h.bn().get(), x.bn().get(), ctx.get()); + BN_set_negative(pk->h.bn().get(), 1); + //// precompute + pk->precompute(); + + /// calculate secret key + // lambda = (p - 1) * (q-1)/2 + pMinus.mul(sk->lambda.bn().get(), qMinus.bn().get(), ctx.get()); + sk->lambda.divConst(2); + // precompute + sk->precompute(); + // return the paillier keypair + return std::make_unique(std::move(sk), std::move(pk)); +} + + +/// optimize (_base^_exp)mod(n^2) using CRT +/// refer to: https://zhuanlan.zhihu.com/p/420503254 +BigNum OpenSSLPaillier::powerModSqrtCrt( + BigNum const& _base, BigNum const& _exp, OpenSSLPaillierKeyPair* _keyPair) const +{ + auto sk = (PaillierPrivateKey*)(_keyPair->sk()); + auto pk = (PaillierPublicKey*)(_keyPair->pk()); + + auto ctx = createBNContext(); + // expp = _exp mod (p^2 - p) + BigNum expp; + _exp.div(NULL, expp.bn().get(), sk->pOrderSqrt.bn().get(), ctx.get()); + // baseP = _base mod p^2 + BigNum baseP; + _base.div(NULL, baseP.bn().get(), sk->pSqrt.bn().get(), ctx.get()); + // xp = baseP ^ expp mod p^2 + auto xp = baseP.modExp(expp.bn().get(), sk->pSqrt.bn().get(), ctx.get()); + + // expq = _exp mod (q^2 - q) + BigNum expq; + _exp.div(NULL, expq.bn().get(), sk->qOrderSqrt.bn().get(), ctx.get()); + // baseQ = _base mod q^2 + BigNum baseQ; + _base.div(NULL, baseQ.bn().get(), sk->qSqrt.bn().get(), ctx.get()); + // xq = baseQ ^ expq mod q^2 + auto xq = baseQ.modExp(expq.bn().get(), sk->qSqrt.bn().get(), ctx.get()); + + // crt to calculate (_base^_exp)mod(n^2) + // h = ((xp - xq)*(qSqrtInv)) mod p^2 + // _base^exp = xq + h * q^2 + auto result = xp.sub(xq.bn().get()); + BigNum tmp; + result.modMul(tmp.bn().get(), sk->qSqrtInverse.bn().get(), sk->pSqrt.bn().get(), ctx.get()); + tmp.mul(tmp.bn().get(), sk->qSqrt.bn().get(), ctx.get()); + + return tmp.modAdd(xq.bn().get(), pk->nSqrt.bn().get()); +} + +void OpenSSLPaillier::encryptImpl(OutputBuffer* _resultBuffer, OutputBuffer* _rBuffer, + BIGNUM const* _m, OpenSSLPaillierKeyPair* _keyPair, void* _pk) const +{ + auto pk = (PaillierPublicKey*)_pk; + // encode _m to support negative case + SignedNumberCodec codec(pk->n); + auto m = codec.encode(_m); + // calculate: g^m + // Note: since g=(n+1) when generate keypair, g^m can be speed up to [(mn + 1) mod n^2] + BigNum tmp; + auto ctx = createBNContext(); + m.mul(tmp.bn().get(), pk->n.bn().get(), ctx.get()); + auto g_power_m = tmp.modAdd(BN_value_one(), pk->nSqrt.bn().get()); + + // generate random r with ceil(|n|/2) bits + // rBits is ceil(keyBits) + auto rBits = pk->keyBits % 2 ? ((pk->keyBits / 2) + 1) : (pk->keyBits / 2); + auto r = generateRand(rBits); + // convert r to buffer + if (_rBuffer) + { + r.toOutputBuffer(_rBuffer, false); + } + // calculate r^n mod n^2, namely (h_s^r mod n^2) + BigNum r_power_n; + // with crt optimization + if (_keyPair && _keyPair->sk()) + { + r_power_n = powerModSqrtCrt(pk->h_s, r, _keyPair); + } + else + { + // without crt optmization + r_power_n = pk->h_s.modExp(r.bn().get(), pk->nSqrt.bn().get(), ctx.get()); + } + // (g^m) * (r^n) mod n^2 + BigNum result; + g_power_m.modMul(result.bn().get(), r_power_n.bn().get(), pk->nSqrt.bn().get(), ctx.get()); + result.toOutputBuffer(_resultBuffer, true); +} + +// encrypt the plain data +void OpenSSLPaillier::encrypt_with_crt( + OutputBuffer* _cipherBytes, OutputBuffer* _rBuffer, BIGNUM const* _v, void* _keyPair) const +{ + if (!_cipherBytes) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "encrypt_with_crt error for Invalid OutputBuffer")); + } + if (!_v) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() + << bcos::errinfo_comment("encrypt_with_crt error for invalid _v")); + } + if (!_keyPair) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "encrypt_with_crt error for invalid _keyPair")); + } + auto keyPair = (OpenSSLPaillierKeyPair*)_keyPair; + if (!keyPair->pk()) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "encrypt_with_crt error for invalid publicKey")); + } + encryptImpl(_cipherBytes, _rBuffer, _v, keyPair, keyPair->pk()); +} + +// decrypt the cipher data +BigNum OpenSSLPaillier::decrypt(bcos::bytesConstRef const& _cipherData, void* _keyPair) const +{ + if (!_keyPair) + { + BOOST_THROW_EXCEPTION( + InvalidHomoKeyPair() << bcos::errinfo_comment("decrypt error for empty keypair")); + } + BigNum cipher(_cipherData); + auto keyPair = (OpenSSLPaillierKeyPair*)_keyPair; + auto sk = (PaillierPrivateKey*)(keyPair->sk()); + if (!sk) + { + BOOST_THROW_EXCEPTION( + InvalidHomoKeyPair() << bcos::errinfo_comment("decrypt error for empty sk")); + } + auto pk = (PaillierPublicKey*)(keyPair->pk()); + if (!pk) + { + BOOST_THROW_EXCEPTION( + InvalidHomoKeyPair() << bcos::errinfo_comment("decrypt error for empty pk")); + } + auto ctx = createBNContext(); + // calculate x = (cipher^lambda mod n^2) + auto x = powerModSqrtCrt(cipher, sk->lambda, keyPair); + // calculate (x-1)/n + BigNum result; + result = x.sub(BN_value_one()); + result.div(result.bn().get(), NULL, pk->n.bn().get(), ctx.get()); + + // result * lambdaInverse mod n + // TODO: check the overhead here + auto lambdaInverse = sk->lambda.Invert(pk->n); + result.modMul(result.bn().get(), lambdaInverse.bn().get(), pk->n.bn().get(), ctx.get()); + // decode result to support negative case + SignedNumberCodec codec(pk->n); + return codec.decode(result); +} + +void OpenSSLPaillier::addImpl(OutputBuffer* _resultBytes, ppc::crypto::BigNum const& _cipher1, + ppc::crypto::BigNum const& _cipher2, void* _publicKey) const +{ + // Note: the keypair contain only the public key + auto pk = (PaillierPublicKey*)(_publicKey); + // c1 * c2 mod n*n + auto ctx = createBNContext(); + BigNum result; + _cipher1.modMul(result.bn().get(), _cipher2.bn().get(), pk->nSqrt.bn().get(), ctx.get()); + bcos::bytes cipherResult; + result.toOutputBuffer(_resultBytes, true); +} + +void OpenSSLPaillier::add(OutputBuffer* _result, bcos::bytesConstRef const& _cipherData1, + bcos::bytesConstRef const& _cipherData2, void* _publicKey) const +{ + if (!_result) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier add error for Invalid OutputBuffer")); + } + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier add error for invalid public key")); + } + BigNum cipher1(_cipherData1); + BigNum cipher2(_cipherData2); + addImpl(_result, cipher1, cipher2, _publicKey); +} + +void OpenSSLPaillier::sub(OutputBuffer* _resultBytes, bcos::bytesConstRef const& _cipherData1, + bcos::bytesConstRef const& _cipherData2, void* _publicKey) const +{ + if (!_resultBytes) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier sub error for Invalid OutputBuffer")); + } + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier sub error for invalid public key")); + } + // Note: the keypair contain only the public key + auto pk = (PaillierPublicKey*)(_publicKey); + BigNum cipher1(_cipherData1); + BigNum cipher2(_cipherData2); + // ((c1 * (c2^-1)) mod n*n + auto cipher2Invert = cipher2.Invert(pk->nSqrt); + return addImpl(_resultBytes, cipher1, cipher2Invert, _publicKey); +} + +void OpenSSLPaillier::scalaMul(OutputBuffer* _resultBytes, BIGNUM const* _value, + bcos::bytesConstRef const& _cipherData, void* _publicKey) const +{ + if (!_resultBytes) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier scalaMul error for invalid OutputBuffer")); + } + if (!_value) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier scalaMul error for invalid input v")); + } + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() << bcos::errinfo_comment( + "paillier scalaMul error for invalid public key")); + } + // Note: the keypair contain only the public key + auto pk = (PaillierPublicKey*)(_publicKey); + // encode to support negative v + SignedNumberCodec codec(pk->n); + auto v = codec.encode(_value); + + BigNum cipher(_cipherData); + + // cipher^v mod (n^2) + auto ctx = createBNContext(); + auto result = cipher.modExp(v.bn().get(), pk->nSqrt.bn().get(), ctx.get()); + result.toOutputBuffer(_resultBytes, true); +} \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillier.h b/cpp/ppc-homo/paillier/OpenSSLPaillier.h new file mode 100644 index 00000000..ef8810e1 --- /dev/null +++ b/cpp/ppc-homo/paillier/OpenSSLPaillier.h @@ -0,0 +1,170 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLPaillier.h + * @author: yujiechen + * @date 2023-08-04 + */ +#pragma once +#include "../codec/SignedNumberCodec.h" +#include "Common.h" +#include "OpenSSLPaillierKeyPair.h" +#include + +namespace ppc::homo +{ +class OpenSSLPaillier : public Paillier +{ +public: + using Ptr = std::shared_ptr; + OpenSSLPaillier() = default; + virtual ~OpenSSLPaillier() = default; + + // generate the keypair + ppc::crypto::KeyPair::UniquePtr generateKeyPair(unsigned const _keyLength) const override; + + // encrypt the plain data + void encrypt_with_crt(OutputBuffer* _resultBuffer, OutputBuffer* _rBuffer, BIGNUM const* v, + void* _keyPair) const override; + + bcos::bytes encrypt_with_crt(BIGNUM const* _value, void* _keyPair) const override + { + auto keyPair = (OpenSSLPaillierKeyPair*)_keyPair; + if (!keyPair || !keyPair->sk() || !keyPair->pk()) + { + BOOST_THROW_EXCEPTION(InvalidHomoKeyPair()); + } + auto pk = (PaillierPublicKey*)keyPair->pk(); + bcos::bytes result(maxCipherBytesLen(pk->keyBits), 0); + OutputBuffer buffer{result.data(), result.size()}; + encrypt_with_crt(&buffer, nullptr, _value, keyPair); + result.resize(buffer.len); + return result; + } + + void encrypt(OutputBuffer* _result, OutputBuffer* _rBuffer, BIGNUM const* _value, + void* _pk) const override + { + encryptImpl(_result, _rBuffer, _value, nullptr, _pk); + } + bcos::bytes encrypt(BIGNUM const* _value, void* _pk) const override + { + if (!_value) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() + << bcos::errinfo_comment("encrypt error for invalid input v")); + } + if (!_pk) + { + BOOST_THROW_EXCEPTION(OpenSSLPaillierException() + << bcos::errinfo_comment("encrypt error for invalid public key")); + } + auto pk = (PaillierPublicKey*)_pk; + bcos::bytes result(maxCipherBytesLen(pk->keyBits), 0); + OutputBuffer buffer{result.data(), result.size()}; + encrypt(&buffer, nullptr, _value, _pk); + result.resize(buffer.len); + return result; + } + + // decrypt the cipher data + ppc::crypto::BigNum decrypt( + bcos::bytesConstRef const& _cipherData, void* _keyPair) const override; + + // _result = _cipher1 + _cipher2 + void add(OutputBuffer* _result, bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const override; + + bcos::bytes add(bcos::bytesConstRef const& _cipher1, bcos::bytesConstRef const& _cipher2, + void* _publicKey) const override + { + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(InvalidHomoPublicKey()); + } + auto pk = (PaillierPublicKey*)_publicKey; + bcos::bytes result(maxCipherBytesLen(pk->keyBits), 0); + OutputBuffer buffer{result.data(), result.size()}; + add(&buffer, _cipher1, _cipher2, _publicKey); + result.resize(buffer.len); + return result; + } + + // _result = _cipher1 - _cipher2 + void sub(OutputBuffer* _result, bcos::bytesConstRef const& _cipher1, + bcos::bytesConstRef const& _cipher2, void* _publicKey) const override; + + bcos::bytes sub(bcos::bytesConstRef const& _cipher1, bcos::bytesConstRef const& _cipher2, + void* _publicKey) const override + { + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(InvalidHomoPublicKey()); + } + auto pk = (PaillierPublicKey*)_publicKey; + bcos::bytes result(maxCipherBytesLen(pk->keyBits), 0); + + OutputBuffer buffer{result.data(), result.size()}; + sub(&buffer, _cipher1, _cipher2, _publicKey); + result.resize(buffer.len); + return result; + } + + // _result = _v * _cipher + void scalaMul(OutputBuffer* _result, BIGNUM const* _v, bcos::bytesConstRef const& _cipher, + void* _publicKey) const override; + + bcos::bytes scalaMul( + BIGNUM const* v, bcos::bytesConstRef const& _cipher, void* _publicKey) const override + { + if (!_publicKey) + { + BOOST_THROW_EXCEPTION(InvalidHomoPublicKey()); + } + auto pk = (PaillierPublicKey*)_publicKey; + bcos::bytes result(maxCipherBytesLen(pk->keyBits), 0); + + OutputBuffer buffer{result.data(), result.size()}; + scalaMul(&buffer, v, _cipher, _publicKey); + result.resize(buffer.len); + + return result; + } + + // calculate the cipherBytesLen + static unsigned int maxCipherBytesLen(unsigned int _keyBits) + { + // 1(the signed/unsigned flag) + _keyBits * 2/ 8 + return ((_keyBits) >> 2) + 1; + } + + static unsigned int rBytesLen(unsigned int _keyBits) + { + auto rBits = _keyBits % 2 ? ((_keyBits / 2) + 1) : (_keyBits / 2); + return (rBits + 7) >> 3; + } + +protected: + virtual void encryptImpl(OutputBuffer* _resultBuffer, OutputBuffer* _rBuffer, BIGNUM const* m, + OpenSSLPaillierKeyPair* _keyPair, void* _pk) const; + + void addImpl(OutputBuffer* _result, ppc::crypto::BigNum const& _cipher1, + ppc::crypto::BigNum const& _cipher2, void* _publicKey) const; + +private: + ppc::crypto::BigNum powerModSqrtCrt(ppc::crypto::BigNum const& _base, + ppc::crypto::BigNum const& _exp, OpenSSLPaillierKeyPair* _keyPair) const; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp b/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp new file mode 100644 index 00000000..40b04d23 --- /dev/null +++ b/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLPaillierKeyPair.cpp + * @author: yujiechen + * @date 2023-08-07 + */ +#include "OpenSSLPaillierKeyPair.h" + +using namespace ppc::homo; +using namespace ppc; +using namespace ppc::crypto; + +// serialize the sk +int PaillierPrivateKey::serialize(bcos::byte* _encodedData, unsigned int _dataLen) const +{ + auto pEnd = _encodedData + _dataLen; + // keyBits + auto pBuffer = encodeInteger(_encodedData, pEnd, keyBits); + // lambda + pBuffer = encodeBigNum(pBuffer, pEnd, lambda); + // p + pBuffer = encodeBigNum(pBuffer, pEnd, p); + // q + pBuffer = encodeBigNum(pBuffer, pEnd, q); + return (pBuffer - _encodedData); +} + +// serialize the pk +int PaillierPublicKey::serialize(bcos::byte* _encodedData, unsigned int _dataLen) const +{ + auto pEnd = _encodedData + _dataLen; + // encode the keyBits + auto pBuffer = encodeInteger(_encodedData, pEnd, keyBits); + // n + pBuffer = encodeBigNum(pBuffer, pEnd, n); + // h + pBuffer = encodeBigNum(pBuffer, pEnd, h); + return (pBuffer - _encodedData); +} + +// deserialize the sk +void PaillierPrivateKey::deserialize(bcos::byte const* _sk, unsigned int _len) +{ + // keyBits + auto offset = decodeInteger(keyBits, _sk, _len, 0); + // lambda + offset = decodeBigNum(lambda, _sk, _len, offset); + // p + offset = decodeBigNum(p, _sk, _len, offset); + // q + offset = decodeBigNum(q, _sk, _len, offset); +} + +// deserialize the pk +void PaillierPublicKey::deserialize(bcos::byte const* _pk, unsigned int _len) +{ + // keyBits + auto offset = decodeInteger(keyBits, _pk, _len, 0); + // n + offset = decodeBigNum(n, _pk, _len, offset); + // h + offset = decodeBigNum(h, _pk, _len, offset); +} \ No newline at end of file diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h b/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h new file mode 100644 index 00000000..75d13cf7 --- /dev/null +++ b/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h @@ -0,0 +1,190 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OpenSSLPaillierKeyPair.h + * @author: yujiechen + * @date 2023-08-07 + */ +#pragma once + +#include "ppc-framework/crypto/KeyPair.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-tools/src/codec/CodecUtility.h" + +namespace ppc::homo +{ +///////////// Paillier-DJN ///////////// +/// the paillier public key +class PaillierPublicKey +{ +public: + using Ptr = std::shared_ptr; + using UniquePtr = std::unique_ptr; + PaillierPublicKey() = default; + PaillierPublicKey(bcos::byte const* _pk, unsigned int _len) + { + deserialize(_pk, _len); + precompute(); + } + virtual ~PaillierPublicKey() = default; + + /////// the stored-elements + unsigned int keyBits; // the length of the public key + ppc::crypto::BigNum n; // the public key + ppc::crypto::BigNum h; // -x^2 + + ////// the precomputed-fields + ppc::crypto::BigNum nSqrt; // cached for avoid recomputing, precalculation to improve + // performance, size is keyBits*2 + ppc::crypto::BigNum h_s; // h^n mod n^2, precalculation to improve performance, size is + // keyBits*2 + + // serialize the pk into _encodedData, return the serialized size + virtual int serialize(bcos::byte* _encodedData, unsigned int _dataLen) const; + // deserialize the pk + virtual void deserialize(bcos::byte const* _pk, unsigned int _len); + + void precompute() + { + // calculate nSqrt = n^2 + auto ctx = ppc::crypto::createBNContext(); + n.mul(nSqrt.bn().get(), n.bn().get(), ctx.get()); + // calculate h_s = (h^n) mod (n^2) + h_s = h.modExp(n.bn().get(), nSqrt.bn().get(), ctx.get()); + } + + static unsigned int maxBytes(unsigned int _keyBits) + { + // sizeof(keyBits) + keyBits/8(n Bytes) + 2 * keyBits/8(h Bytes) + sizeof(uint16_t) * + // BigNumFieldNum + BigNumFieldNum + // = sizeof(keyBits) + (3*KeyBits + 7)/8 + sizeof(uint16_t) * 2 + 2; + return sizeof(_keyBits) + (3 * _keyBits + 7) / 8 + 6; + } +}; + +/// the paillier private key +class PaillierPrivateKey +{ +public: + using Ptr = std::shared_ptr; + using UniquePtr = std::unique_ptr; + PaillierPrivateKey() = default; + PaillierPrivateKey(bcos::byte const* _pk, unsigned int _len) + { + deserialize(_pk, _len); + precompute(); + } + virtual ~PaillierPrivateKey() = default; + + ////// the stored-fields + unsigned int keyBits; // the length of the public key + ppc::crypto::BigNum lambda; // the private key, keyBits size + ppc::crypto::BigNum p; // p, keyBits/2 + ppc::crypto::BigNum q; // q, keyBits/2 + + ////// the precompute fields + ppc::crypto::BigNum pSqrt; // p^2, precalculation to improve performance, keyBits size + ppc::crypto::BigNum qSqrt; // q^2, precalculation to improve performance, keyBits size + ppc::crypto::BigNum qSqrtInverse; // (q^2)^(-1) mod(p^2), precalculation to improve + // performance, keyBits size + ppc::crypto::BigNum pOrderSqrt; // (p*(p-1)), keyBits size + ppc::crypto::BigNum qOrderSqrt; // (q*(q - 1)), keyBits size + + + // serialize the sk into _encodedData, return the serialized size + virtual int serialize(bcos::byte* _encodedData, unsigned int _dataLen) const; + // deserialize the sk + virtual void deserialize(bcos::byte const* _pk, unsigned int _len); + + virtual void precompute() + { + auto ctx = ppc::crypto::createBNContext(); + /// calculate pSqrt and qSqrt + p.mul(pSqrt.bn().get(), p.bn().get(), ctx.get()); + q.mul(qSqrt.bn().get(), q.bn().get(), ctx.get()); + // (q^2)^(-1) mod(p^2) + qSqrtInverse = qSqrt.Invert(pSqrt); + // calculate pOrderSqrt(p*(p-1)) and qOrderSqrt(q*(q - 1)) + pOrderSqrt = pSqrt.sub(p.bn().get()); + qOrderSqrt = qSqrt.sub(q.bn().get()); + } + + static unsigned int maxBytes(unsigned int _keyBits) + { + // sizeof(keyBits) + (lambda_length + p_length + q_length) + (lambda_signed_flag + + // p_signed_flag + q_signed_flag) + (lambda_buffer_len + p_buffer_len + q_buffer_len) + // = sizeof(keyBits) + (2*_keyBits + 7)/8 + BigNumFieldNum + BigNumFieldNum * + // = sizeof(keyBits) + (2*_keyBits + 7)/8 + 3 + 3 * sizeof(int16_t) + // = sizeof(uint16_t) + (2 * _keyBits + 7)/8 + 9 + return sizeof(_keyBits) + (2 * _keyBits + 7) / 8 + 9; + } +}; + +class OpenSSLPaillierKeyPair : public ppc::crypto::KeyPair +{ +public: + using Ptr = std::shared_ptr; + using UniquePtr = std::unique_ptr; + OpenSSLPaillierKeyPair(PaillierPrivateKey::Ptr&& _sk, PaillierPublicKey::Ptr&& _pk) + : m_sk(std::move(_sk)), m_pk(std::move(_pk)) + {} + + OpenSSLPaillierKeyPair( + bcos::byte const* _sk, unsigned int _skLen, bcos::byte const* _pk, unsigned int _pkLen) + : m_sk(std::make_shared(_sk, _skLen)), + m_pk(std::make_shared(_pk, _pkLen)) + {} + + OpenSSLPaillierKeyPair(bcos::bytes const& _sk, bcos::bytes const& _pk) + : OpenSSLPaillierKeyPair(_sk.data(), _sk.size(), _pk.data(), _pk.size()) + {} + + void* sk() const override { return (void*)m_sk.get(); } + void* pk() const override { return (void*)m_pk.get(); } + + // serialize the sk + bcos::bytes serializeSK() const override + { + bcos::bytes skData; + skData.resize(PaillierPrivateKey::maxBytes(m_sk->keyBits)); + if (m_sk) + { + auto dataSize = m_sk->serialize(skData.data(), skData.size()); + skData.resize(dataSize); + } + return skData; + } + + // serialize the pk + bcos::bytes serializePK() const override + { + bcos::bytes pkData; + pkData.resize(PaillierPublicKey::maxBytes(m_pk->keyBits)); + if (m_pk) + { + auto dataSize = m_pk->serialize(pkData.data(), pkData.size()); + pkData.resize(dataSize); + } + return pkData; + } + +private: + // the private key + PaillierPrivateKey::Ptr m_sk; + + // the public key + PaillierPublicKey::Ptr m_pk; +}; +} // namespace ppc::homo \ No newline at end of file diff --git a/cpp/ppc-homo/tests/CMakeLists.txt b/cpp/ppc-homo/tests/CMakeLists.txt new file mode 100644 index 00000000..de4051c9 --- /dev/null +++ b/cpp/ppc-homo/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-homo) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${FAHE_TARGET} ${PAILLIER_TARGET} ${IHC_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-ppc-homo WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestFahe.cpp b/cpp/ppc-homo/tests/TestFahe.cpp new file mode 100644 index 00000000..2f31b731 --- /dev/null +++ b/cpp/ppc-homo/tests/TestFahe.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestFahe.cpp + * @author: yujiechen + * @date 2023-08-30 + */ +#include "ppc-homo/fahe/Fahe.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(faheTest, TestPromptFixture) +void testFaheImpl(BigNum const& m1, BigNum const& m2, int lambda, int mmax, int alpha) +{ + auto faheImpl = std::make_shared(); + auto key = faheImpl->generateKey(lambda, mmax, alpha); + + // test encrypt/decrypt + auto c1 = faheImpl->encrypt(m1.bn().get(), key.get()); + bcos::bytes cipherBytes; + c1.toBytes(cipherBytes); + std::cout << "### cipherLen: " << cipherBytes.size() << std::endl; + auto decryptedM1 = faheImpl->decrypt(c1.bn().get(), key.get()); + std::cout << "### m1: "; + BN_print_fp(stdout, m1.bn().get()); + std::cout << std::endl; + std::cout << "### decryptedM1: "; + BN_print_fp(stdout, decryptedM1.bn().get()); + std::cout << std::endl; + + BOOST_CHECK(decryptedM1.cmp(m1.bn().get()) == 0); + + auto c2 = faheImpl->encrypt(m2.bn().get(), key.get()); + auto decryptedM2 = faheImpl->decrypt(c2.bn().get(), key.get()); + + std::cout << "### m2: "; + BN_print_fp(stdout, m2.bn().get()); + std::cout << std::endl; + std::cout << "### decryptedM2: "; + BN_print_fp(stdout, decryptedM2.bn().get()); + std::cout << std::endl; + + BOOST_CHECK(decryptedM2.cmp(m2.bn().get()) == 0); + + // add + auto addResult = faheImpl->add(c1.bn().get(), c2.bn().get()); + auto decryptAddResult = faheImpl->decrypt(addResult.bn().get(), key.get()); + auto expectedAddResult = m1.add(m2.bn().get()); + + std::cout << "### decryptAddResult: "; + BN_print_fp(stdout, decryptAddResult.bn().get()); + std::cout << std::endl; + std::cout << "### expectedAddResult: "; + BN_print_fp(stdout, expectedAddResult.bn().get()); + std::cout << std::endl; + BOOST_CHECK(decryptAddResult.cmp(expectedAddResult.bn().get()) == 0); +} + +BOOST_AUTO_TEST_CASE(testFahe) +{ + int lambda = 128; + int alpha = 32; + int mmax = 64; + + // positive case + int64_t m1 = 123213231; + int64_t m2 = 234343430; + std::cout << "==== testFahe positive =====" << std::endl; + testFaheImpl(BigNum(m1), BigNum(m2), lambda, mmax, alpha); + std::cout << "==== testFahe positive finished=====" << std::endl; + + // zero case + std::cout << "==== testFahe zero case =====" << std::endl; + m2 = 0; + testFaheImpl(BigNum(m1), BigNum(m2), lambda, mmax, alpha); + m1 = 0; + testFaheImpl(BigNum(m1), BigNum(m2), lambda, mmax, alpha); + std::cout << "==== testFahe zero case finished =====" << std::endl; + + // negative case + std::cout << "==== testFahe negative case =====" << std::endl; + m1 = 123213231 + bcos::utcSteadyTime(); + srand(bcos::utcSteadyTime()); + m2 = -(123213231 + bcos::utcSteadyTime() + rand()); + testFaheImpl(BigNum(m1), BigNum(m2), lambda, mmax, alpha); + std::cout << "==== testFahe negative case finished =====" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestFloatingPointCodec.cpp b/cpp/ppc-homo/tests/TestFloatingPointCodec.cpp new file mode 100644 index 00000000..857bb686 --- /dev/null +++ b/cpp/ppc-homo/tests/TestFloatingPointCodec.cpp @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestFloatingPointCodec.cpp + * @author: yujiechen + * @date 2023-08-29 + */ +#include "ppc-homo/codec/FloatingPointCodec.h" +#include +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(floatingPointCodecTest, TestPromptFixture) +void testDoubleCase(FloatingPointNumber const& _number, std::string const& _expectedVStr) +{ + float50 _expectedV(_expectedVStr); + auto codec = std::make_shared(); + auto float50V = codec->toFloat50(_number); + std::cout << std::fixed << std::setprecision(8); + std::cout << "float50V: " << float50V << ", expected:" << _expectedV << std::endl; + std::cout << "epsilon: " << std::numeric_limits::epsilon() << std::endl; + BOOST_CHECK(fabs(float50V - _expectedV) < std::numeric_limits::epsilon()); + + // convert float50V to FloatingPointNumber + auto convertedNumber = codec->toFloatingPoint(_expectedVStr); + std::cout << "exp: " << convertedNumber.exponent << std::endl; + std::cout << "value: " << convertedNumber.value.getWord() << std::endl; + + // encode the convertedNumber again + auto convertedV = codec->toFloat50(convertedNumber); + BOOST_CHECK(fabs(convertedV - _expectedV) < std::numeric_limits::epsilon()); +} + +void testIntCase(FloatingPointNumber const& _number, s1024 const& _expectedV) +{ + // encode + auto codec = std::make_shared(); + auto v = codec->toInt(_number); + std::cout << "v: " << v << ", _expectedV: " << _expectedV << std::endl; + BOOST_CHECK(v == _expectedV); + + // decode + auto convertedNumber = codec->toFloatingPoint(_expectedV); + + // encode again and check + auto convertedV = codec->toInt(convertedNumber); + BOOST_CHECK(_expectedV == convertedV); +} + +void testBigNumCovert(s1024 _expectedV) +{ + BigNum bn(_expectedV); + BigNum expectedBn(_expectedV); + BOOST_CHECK(bn.cmp(expectedBn.bn().get()) == 0); + + // test to S1024 + auto expectedS1024 = bn.toS1024(); + BOOST_CHECK(expectedS1024 == _expectedV); +} + +BOOST_AUTO_TEST_CASE(testBigNumConvert) +{ + s1024 value("123243432444440000234234323423424"); + testBigNumCovert(value); + + for (int i = 0; i < 5; i++) + { + int64_t expectedV = 123123342343 + utcSteadyTime(); + s1024 value(expectedV); + testBigNumCovert(value); + + expectedV = -(123123342343 + utcSteadyTime()); + value = s1024(expectedV); + testBigNumCovert(value); + } +} + +BOOST_AUTO_TEST_CASE(testDouble) +{ + std::cout << "==== test decimal case ===== " << std::endl; + std::string expectedV("2.23434"); + FloatingPointNumber fpNumber; + fpNumber.exponent = -5; + s1024 significant = s1024(223434); + BigNum significantBn(significant); + fpNumber.value = significantBn; + testDoubleCase(fpNumber, expectedV); + + expectedV = "2.2343432423423431"; + fpNumber.exponent = -16; + significant = s1024(22343432423423431); + BigNum significantBn2(significant); + fpNumber.value = significantBn2; + testDoubleCase(fpNumber, expectedV); + + + expectedV = "2234.3432423423431"; + fpNumber.exponent = -13; + significant = s1024(22343432423423431); + BigNum significantBn3(significant); + fpNumber.value = significantBn3; + testDoubleCase(fpNumber, expectedV); + + + expectedV = "2234343242342.3431"; + fpNumber.exponent = -4; + significant = s1024(22343432423423431); + BigNum significantBn4(significant); + fpNumber.value = significantBn4; + testDoubleCase(fpNumber, expectedV); + + expectedV = "2234343242342343.1"; + fpNumber.exponent = -1; + significant = s1024(22343432423423431); + BigNum significantBn5(significant); + fpNumber.value = significantBn5; + testDoubleCase(fpNumber, expectedV); + + expectedV = "22343432423423431"; + fpNumber.exponent = 0; + significant = s1024(22343432423423431); + BigNum significantBn6(significant); + fpNumber.value = significantBn6; + testDoubleCase(fpNumber, expectedV); + std::cout << "==== test decimal case finished ===== " << std::endl; + + // the positive case + std::cout << "==== test positive case ===== " << std::endl; + expectedV = "22343432423100"; + fpNumber.exponent = 2; + BigNum significantBn7(223434324231); + fpNumber.value = significantBn7; + testDoubleCase(fpNumber, expectedV); + std::cout << "==== test positive case finish ===== " << std::endl; + + std::cout << "==== test zero case ===== " << std::endl; + // zero + expectedV = "0"; + fpNumber.exponent = 10; + BigNum zero(0); + fpNumber.value = zero; + testDoubleCase(fpNumber, expectedV); + std::cout << "==== test zero case finish===== " << std::endl; + + std::cout << "==== test negative case =====" << std::endl; + expectedV = "-22343432423423431"; + fpNumber.exponent = 0; + significant = s1024(-22343432423423431); + BigNum significantBn8(significant); + fpNumber.value = significantBn8; + testDoubleCase(fpNumber, expectedV); + + expectedV = "-22343432423423.431"; + fpNumber.exponent = -3; + significant = s1024(-22343432423423431); + BigNum significantBn9(significant); + fpNumber.value = significantBn9; + testDoubleCase(fpNumber, expectedV); + + expectedV = "-22.343432423423431"; + fpNumber.exponent = -15; + significant = s1024(-22343432423423431); + BigNum significantBn10(significant); + fpNumber.value = significantBn10; + testDoubleCase(fpNumber, expectedV); + std::cout << "==== test negative case finish===== " << std::endl; +} +BOOST_AUTO_TEST_CASE(testInt) +{ + std::cout << "==== test int case =====" << std::endl; + s1024 expectedV = s1024("223434323222342343242342343234300000"); + FloatingPointNumber fpNumber; + fpNumber.exponent = 5; + s1024 significant = s1024("2234343232223423432423423432343"); + BigNum tmp(significant); + fpNumber.value = tmp; + testIntCase(fpNumber, expectedV); + + expectedV = 200; + fpNumber.exponent = 2; + BigNum tmp1(2); + fpNumber.value = tmp1; + testIntCase(fpNumber, expectedV); + + expectedV = 2000000000000; + fpNumber.exponent = -2; + BigNum tmp2(200000000000000); + fpNumber.value = tmp2; + testIntCase(fpNumber, expectedV); + + expectedV = 2000000000001; + fpNumber.exponent = -2; + BigNum tmp3(200000000000100); + fpNumber.value = tmp3; + testIntCase(fpNumber, expectedV); + + expectedV = 200000000000; + fpNumber.exponent = -3; + BigNum tmp4(200000000000100); + fpNumber.value = tmp4; + testIntCase(fpNumber, expectedV); + + // Note: not support floor + expectedV = 200000000000; + fpNumber.exponent = -3; + BigNum tmp5(200000000000900); + fpNumber.value = tmp5; + testIntCase(fpNumber, expectedV); + std::cout << "==== test int case finish=====" << std::endl; + + std::cout << "==== test zero case =====" << std::endl; + expectedV = 0; + BigNum tmp6(0); + fpNumber.value = tmp6; + testIntCase(fpNumber, expectedV); + std::cout << "==== test zero case finish=====" << std::endl; + + std::cout << "==== test negative case =====" << std::endl; + expectedV = -200; + fpNumber.exponent = 2; + BigNum tmp7(-2); + fpNumber.value = tmp7; + testIntCase(fpNumber, expectedV); + + expectedV = -2000000000000; + fpNumber.exponent = -2; + BigNum tmp8(-200000000000000); + fpNumber.value = tmp8; + testIntCase(fpNumber, expectedV); + + expectedV = -200000000000; + fpNumber.exponent = -3; + BigNum tmp9(-200000000000900); + fpNumber.value = tmp9; + testIntCase(fpNumber, expectedV); + std::cout << "==== test negative case finish=====" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp b/cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp new file mode 100644 index 00000000..f04c8a88 --- /dev/null +++ b/cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestFloatingPointPaillier.cpp + * @author: yujiechen + * @date 2023-08-29 + */ +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-homo/paillier/FloatingPointPaillier.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(floatingPointPaillierTest, TestPromptFixture) + +void testPaillierHomoProperty(int _keyBits, std::string const& s1, std::string const& s2, + std::string const& s3, double epsilon) +{ + float50 m1(s1); + float50 m2(s2); + float50 v(s3); + auto codec = std::make_shared(); + auto m1Fp = codec->toFloatingPoint(s1); + auto m2Fp = codec->toFloatingPoint(s2); + auto vFp = codec->toFloatingPoint(s3); + std::cout << "* exp for m1: " << m1Fp.exponent << ", value: " << m1Fp.value.getWord() + << std::endl; + std::cout << "* exp for m2: " << m2Fp.exponent << ", value: " << m2Fp.value.getWord() + << std::endl; + + auto paillierImpl = std::make_shared(); + auto fpPaillier = std::make_shared(paillierImpl); + // generate keypair + auto keyPair = paillierImpl->generateKeyPair(_keyBits); + std::cout << "===== encrypt/decrypt check ======" << std::endl; + // encrypt m1 + bcos::bytes c1(FloatingPointPaillier::maxCipherBytesLen(_keyBits), 0); + OutputBuffer c1Buffer{c1.data(), c1.size()}; + fpPaillier->encryptFast(&c1Buffer, m1Fp, (void*)keyPair.get()); + // decrypt c1 + auto decrypted_m1 = fpPaillier->decrypt(ref(c1), (void*)keyPair.get()); + // check m1 + BOOST_CHECK(fabs(codec->toFloat50(decrypted_m1) - m1) <= epsilon); + + // encrypt m2 + bcos::bytes c2(FloatingPointPaillier::maxCipherBytesLen(_keyBits), 0); + OutputBuffer c2Buffer{c2.data(), c2.size()}; + fpPaillier->encryptFast(&c2Buffer, m2Fp, (void*)keyPair.get()); + // decrypt c2 + auto decrypted_m2 = fpPaillier->decrypt(ref(c2), (void*)keyPair.get()); + // check m2 + std::cout << std::fixed << std::setprecision(11); + std::cout << "#### decrypted_m2: " << codec->toFloat50(decrypted_m2) << std::endl; + std::cout << "#### m2: " << m2 << std::endl; + BOOST_CHECK(fabs(codec->toFloat50(decrypted_m2) - m2) <= epsilon); + std::cout << "===== encrypt/decrypt check done======" << std::endl; + + std::cout << "===== paillier add check ======" << std::endl; + bcos::bytes addResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits), 0); + OutputBuffer addResultBuffer{addResult.data(), addResult.size()}; + auto pk = keyPair->pk(); + fpPaillier->add(&addResultBuffer, ref(c1), ref(c2), pk); + // decrypt and check + auto decryptAddResult = fpPaillier->decrypt(ref(addResult), (void*)keyPair.get()); + auto expectedAddResult = m1 + m2; + + std::cout << std::fixed << std::setprecision(15); + std::cout << "#### expectedAddResult: " << expectedAddResult << std::endl; + std::cout << "#### decryptAddResult: " << codec->toFloat50(decryptAddResult) << std::endl; + std::cout << "### fabs: " << fabs(codec->toFloat50(decryptAddResult) - expectedAddResult) + << std::endl; + BOOST_CHECK(fabs(codec->toFloat50(decryptAddResult) - expectedAddResult) <= epsilon); + std::cout << "===== paillier add check done======" << std::endl; + + std::cout << "===== paillier sub check ======" << std::endl; + bcos::bytes subResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits), 0); + OutputBuffer subResultBuffer{subResult.data(), subResult.size()}; + fpPaillier->sub(&subResultBuffer, ref(c1), ref(c2), pk); + // decrypt and check + auto decryptSubResult = fpPaillier->decrypt(ref(subResult), (void*)keyPair.get()); + auto expectedSubResult = m1 - m2; + + std::cout << std::fixed << std::setprecision(15); + std::cout << "#### expectedSubResult: " << expectedSubResult << std::endl; + std::cout << "#### decryptSubResult: " << codec->toFloat50(decryptSubResult) << std::endl; + std::cout << "### fabs: " << fabs(codec->toFloat50(decryptSubResult) - expectedSubResult) + << std::endl; + BOOST_CHECK(fabs(codec->toFloat50(decryptSubResult) - expectedSubResult) <= epsilon); + std::cout << "===== paillier sub check done======" << std::endl; + + + std::cout << "===== paillier scalaMul check ======" << std::endl; + bcos::bytes mulResult(FloatingPointPaillier::maxCipherBytesLen(_keyBits), 0); + OutputBuffer mulResultBuffer{mulResult.data(), mulResult.size()}; + fpPaillier->scalaMul(&mulResultBuffer, vFp, ref(c1), pk); + // decrypt and check + auto decryptMulResult = fpPaillier->decrypt(ref(mulResult), (void*)keyPair.get()); + auto expectedMulResult = v * m1; + + std::cout << std::fixed << std::setprecision(15); + std::cout << "#### expectedMulResult: " << expectedMulResult << std::endl; + std::cout << "#### decryptMulResult: " << codec->toFloat50(decryptMulResult) << std::endl; + std::cout << "### fabs: " << fabs(codec->toFloat50(decryptMulResult) - expectedMulResult) + << std::endl; + BOOST_CHECK(fabs(codec->toFloat50(decryptMulResult) - expectedMulResult) <= epsilon); + std::cout << "===== paillier scalaMul check done======" << std::endl; + + // test the overflow case + std::cout << "====== test the overflow case ======" << std::endl; + FloatingPointNumber fpN; + fpN.value = ((PaillierPublicKey*)pk)->n; + fpN.exponent = 0; + bcos::bytes resultBytes(FloatingPointPaillier::maxCipherBytesLen(_keyBits)); + OutputBuffer result{resultBytes.data(), resultBytes.size()}; + BOOST_CHECK_THROW(fpPaillier->encrypt(&result, fpN, (void*)pk), SignedNumberCodecException); + std::cout << "====== test the overflow case done======" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testFloatingPointPaillier) +{ + int keyBits = 2048; + std::string m1("2.323"); + std::string m2("4.5"); + std::string v("123.23"); + // set precison to 10 + double epsilon = 0.0000000001; + std::cout << "===== testFloatingPointPaillier: positive case =========" << std::endl; + testPaillierHomoProperty(keyBits, m1, m2, v, epsilon); + std::cout << "===== testFloatingPointPaillier: positive case done =========" << std::endl; + + // the negative case + std::cout << "===== testFloatingPointPaillier: negative case =========" << std::endl; + m1 = "-23.2344234"; + m2 = "4.234"; + v = "-2343.23434324324"; + testPaillierHomoProperty(keyBits, m1, m2, v, epsilon); + std::cout << "===== testFloatingPointPaillier: negative case done=========" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestIhc.cpp b/cpp/ppc-homo/tests/TestIhc.cpp new file mode 100644 index 00000000..c5c0109d --- /dev/null +++ b/cpp/ppc-homo/tests/TestIhc.cpp @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestIhc.cpp + * @author: asherli + * @date 2023-11-27 + */ +#include "ppc-homo/ihc/IhcImpl.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(ihcTest, TestPromptFixture) + +bcos::bytes testIhcEncryptionDecryption( + Ihc::Ptr const& _ihc, bcos::bytesConstRef const& key, BigNum const& m) +{ + std::cout << "#### ihc encrypt" << std::endl; + // encrypt + auto cipher = _ihc->encrypt(key, m.bn().get()); + std::cout << "#### ihc decrypt" << std::endl; + // decrypt + auto decodedM = _ihc->decrypt(key, bcos::ref(cipher)); + std::cout << "#### ihc decrypt success" << std::endl; + BOOST_CHECK(decodedM.cmp(m.bn().get()) == 0); + return cipher; +} +void testIhcImpl(BigNum const& m1, BigNum const& m2, BigNum const& m3, Ihc::IhcMode _mode, + unsigned int _iterRound) +{ + auto ihc = std::make_shared((int)_mode, _iterRound); + auto key = ihc->generateKey(); + // encryption + auto c1 = testIhcEncryptionDecryption(ihc, bcos::ref(key), m1); + auto c2 = testIhcEncryptionDecryption(ihc, bcos::ref(key), m2); + auto c3 = testIhcEncryptionDecryption(ihc, bcos::ref(key), m3); + // add test + std::cout << "#### ihc add test" << std::endl; + auto addResult = ihc->add(bcos::ref(c1), bcos::ref(c2)); + std::cout << "##### addResult: " << bcos::toHex(addResult) << ", size: " << addResult.size() + << std::endl; + auto addM = ihc->decrypt(bcos::ref(key), bcos::ref(addResult)); + auto expectedAddM = m1.add(m2.bn().get()); + BOOST_CHECK(addM.cmp(expectedAddM.bn().get()) == 0); + std::cout << "#### ihc add test success!" << std::endl; + // sub test + std::cout << "#### ihc sub test" << std::endl; + auto subResult = ihc->sub(bcos::ref(c1), bcos::ref(c2)); + auto subM = ihc->decrypt(bcos::ref(key), bcos::ref(subResult)); + auto expectedSubM = m1.sub(m2.bn().get()); + std::cout << "##### subM: " << std::endl; + BN_print_fp(stdout, subM.bn().get()); + std::cout << "#### expectedSubM: " << std::endl; + BN_print_fp(stdout, expectedSubM.bn().get()); + BOOST_CHECK(subM.cmp(expectedSubM.bn().get()) == 0); + std::cout << "#### ihc sub test success" << std::endl; + + // scalaMul test + std::cout << "#### ihc sub scalaMul" << std::endl; + auto mulCipher = ihc->scalaMul(m3.bn().get(), bcos::ref(c2)); + auto mulM = ihc->decrypt(bcos::ref(key), bcos::ref(mulCipher)); + auto ctx = createBNContext(); + BigNum expectedMulResult; + m3.mul(expectedMulResult.bn().get(), m2.bn().get(), ctx.get()); + BOOST_CHECK(mulM.cmp(expectedMulResult.bn().get()) == 0); + std::cout << "#### ihc sub scalaMul success" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testIhc) +{ + int iterRound = 16; + + // positive case + BigNum m1(123213231); + BigNum m2(234343430); + BigNum m3(123456); + std::cout << "==== testIhc positive =====" << std::endl; + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, 16); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, 16); + std::cout << "==== testIhc positive finished=====" << std::endl; + + // zero case + std::cout << "==== testIhc zero case =====" << std::endl; + m2 = BigNum((int64_t)0); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, 16); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, 16); + m1 = BigNum((int64_t)0); + m3 = BigNum((int64_t)1); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, 16); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, 16); + std::cout << "==== testIhc zero case finished =====" << std::endl; + + // negative case + std::cout << "==== testIhc negative case =====" << std::endl; + m1 = BigNum(123213231 + bcos::utcSteadyTime()); + srand(bcos::utcSteadyTime()); + m2 = BigNum(-(123213231 + bcos::utcSteadyTime() + rand())); + m3 = BigNum(122); + testIhcImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, 16); + std::cout << "==== testIhc negative case finished =====" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestIhcFloating.cpp b/cpp/ppc-homo/tests/TestIhcFloating.cpp new file mode 100644 index 00000000..216cf7b5 --- /dev/null +++ b/cpp/ppc-homo/tests/TestIhcFloating.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestIhcFloating.cpp + * @author: asherli + * @date 2023-11-27 + */ +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-homo/ihc/FloatingPointIhc.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(ihcFloatingTest, TestPromptFixture) + +bcos::bytes testEncryptDecrypt(FloatingPointIhc::Ptr const& fpIhc, + FloatingPointCodec::Ptr const& codec, bcos::bytes const& _key, std::string const& s) +{ + float50 m(s); + auto ffpNumber = codec->toFloatingPoint(s); + auto cipher = fpIhc->encrypt(bcos::ref(_key), ffpNumber); + // decrypt + auto decryptffp = fpIhc->decrypt(bcos::ref(_key), bcos::ref(cipher)); + auto decryptValue = codec->toFloat50(decryptffp); + // check + std::cout << "#### m: " << m.str(32) << std::endl; + std::cout << "#### decryptValue: " << decryptValue.str(32) << std::endl; + BOOST_CHECK(m == decryptValue); + return cipher; +} + +void testIhcFloatingImpl( + std::string s1, std::string s2, std::string s3, Ihc::IhcMode _mode, int iterRound) +{ + auto ihc = std::make_shared((int)_mode, iterRound); + auto key = ihc->generateKey(); + + float50 m1(s1); + float50 m2(s2); + float50 m3(s3); + auto fpIhc = std::make_shared(ihc); + auto codec = std::make_shared(); + // c1 + auto c1 = testEncryptDecrypt(fpIhc, codec, key, s1); + // c2 + auto c2 = testEncryptDecrypt(fpIhc, codec, key, s2); + // c3 + auto c3 = testEncryptDecrypt(fpIhc, codec, key, s3); + // add + std::cout << "### testIhcFloatingImpl: add case" << std::endl; + auto addResult = fpIhc->add(bcos::ref(c1), bcos::ref(c2)); + auto addFfpResult = fpIhc->decrypt(bcos::ref(key), bcos::ref(addResult)); + auto expectedAddResult = m1 + m2; + BOOST_CHECK(codec->toFloat50(addFfpResult) == expectedAddResult); + std::cout << "### testIhcFloatingImpl: add case end" << std::endl; + // sub case + std::cout << "### testIhcFloatingImpl: sub case" << std::endl; + auto subResult = fpIhc->sub(bcos::ref(c1), bcos::ref(c2)); + auto subFfpResult = fpIhc->decrypt(bcos::ref(key), bcos::ref(subResult)); + auto expectedSubResult = m1 - m2; + BOOST_CHECK(codec->toFloat50(subFfpResult) == expectedSubResult); + std::cout << "### testIhcFloatingImpl: sub case end" << std::endl; + // scalaMul case + std::cout << "### testIhcFloatingImpl: scalaMul case" << std::endl; + auto v = codec->toFloatingPoint(s3); + auto mulResult = fpIhc->scalaMul(v, bcos::ref(c1)); + auto mulFfpResult = fpIhc->decrypt(bcos::ref(key), bcos::ref(mulResult)); + auto expectedMulResult = m1 * m3; + std::cout << "#### expectedMulResult: " << expectedMulResult << std::endl; + std::cout << "#### mulFfpResult: " << codec->toFloat50(mulFfpResult) << std::endl; + BOOST_CHECK(codec->toFloat50(mulFfpResult) == expectedMulResult); + std::cout << "### testIhcFloatingImpl: mul case end" << std::endl; +} + + +BOOST_AUTO_TEST_CASE(testIhcFloating) +{ + int iterRound = 16; + + // positive case + std::cout << "==== testIhcFloatingImpl positive =====" << std::endl; + std::string m1("234324.345435345345"); + std::string m2("534252.45670234"); + std::string m3("234.456452"); + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, iterRound); + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, iterRound); + std::cout << "==== testIhcFloatingImpl positive finished=====" << std::endl; + + // zero case + std::cout << "==== testIhcFloatingImpl zero case =====" << std::endl; + m1 = "0.0"; + m2 = "0.1"; + m3 = "0.000"; + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, iterRound); + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, iterRound); + std::cout << "==== testIhc zero case finished =====" << std::endl; + + // negative case + std::cout << "==== testIhcFloatingImpl negative case =====" << std::endl; + m1 = "-234234.00234324"; + m2 = "-234.543645"; + m3 = "-23432435234"; + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_128, iterRound); + testIhcFloatingImpl(m1, m2, m3, Ihc::IhcMode::IHC_256, iterRound); + std::cout << "==== testIhcFloatingImpl negative case finished =====" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/TestPaillier.cpp b/cpp/ppc-homo/tests/TestPaillier.cpp new file mode 100644 index 00000000..a7b04bbb --- /dev/null +++ b/cpp/ppc-homo/tests/TestPaillier.cpp @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestPaillier.cpp + * @author: yujiechen + * @date 2023-08-08 + */ +#include "ppc-homo/codec/SignedNumberCodec.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include +#include +#include + +using namespace ppc::crypto; +using namespace ppc::homo; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(paillierTest, TestPromptFixture) + +void testPaillierEncryptDecrypt(bcos::bytes& _cipherData, OpenSSLPaillier::Ptr const& _impl, + SignedNumberCodec::Ptr const& _codec, KeyPair::UniquePtr const& _keyPair, int64_t const _m, + uint64_t const _perfCount) +{ + (void)_codec; + ////// test encrypt and decrypt, return the cipher data + // encrypt + BigNum m(_m); + + auto startT = bcos::utcSteadyTimeUs(); + for (uint64_t i = 0; i < _perfCount; i++) + { + _cipherData = _impl->encrypt_with_crt(m.bn().get(), (void*)_keyPair.get()); + } + std::cout << "#### paillier encrypt perf:" << ((bcos::utcSteadyTimeUs() - startT) / _perfCount) + << " us" << std::endl; + + /// case not enough resultBuffer; + bcos::bytes resultBytes(2); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + BOOST_CHECK_THROW( + _impl->encrypt_with_crt(&resultBuffer, nullptr, m.bn().get(), (void*)_keyPair.get()), + std::exception); + + // decrypt + startT = utcSteadyTimeUs(); + BigNum decryptedV; + for (uint64_t i = 0; i < _perfCount; i++) + { + decryptedV = _impl->decrypt(bcos::ref(_cipherData), (void*)_keyPair.get()); + } + std::cout << "#### paillier decrypt perf:" << ((bcos::utcSteadyTimeUs() - startT) / _perfCount) + << " us" << std::endl; + std::cout << std::endl; + + BN_print_fp(stdout, decryptedV.bn().get()); + BigNum mBN(_m); + BOOST_CHECK(mBN.cmp(decryptedV.bn().get()) == 0); +} + +void testPaillierHomoProperty(OpenSSLPaillier::Ptr const& _impl, + SignedNumberCodec::Ptr const& _codec, KeyPair::UniquePtr const& _keyPair, int64_t _value, + bcos::bytes& _c1, bcos::bytes& _c2, int64_t const _m1, int64_t _m2, uint64_t _perfCount) +{ + std::cout << "### c1 size:" << _c1.size() << std::endl; + std::cout << "### c2 size:" << _c2.size() << std::endl; + BigNum m1(_m1); + BigNum m2(_m2); + // ==== test add ===== + // _c1 + _c2 + bcos::bytes sumCipher; + auto startT = bcos::utcSteadyTimeUs(); + for (uint64_t i = 0; i < _perfCount; i++) + { + sumCipher = _impl->add(bcos::ref(_c1), bcos::ref(_c2), _keyPair->pk()); + } + std::cout << "#### paillier add perf:" << ((bcos::utcSteadyTimeUs() - startT) / _perfCount) + << " us" << std::endl; + + /// case not enough resultBuffer; + bcos::bytes resultBytes(10); + OutputBuffer resultBuffer{resultBytes.data(), resultBytes.size()}; + BOOST_CHECK_THROW( + _impl->add(&resultBuffer, bcos::ref(_c1), bcos::ref(_c2), _keyPair->pk()), std::exception); + + // decrypt and check the result + auto result = _impl->decrypt(bcos::ref(sumCipher), (void*)_keyPair.get()); + auto expectedBN = m1.add(m2.bn().get()); + BOOST_CHECK(expectedBN.cmp(result.bn().get()) == 0); + + // ==== test sub ===== + // _c1 - _c2 + startT = bcos::utcSteadyTimeUs(); + bcos::bytes subCipher; + for (uint64_t i = 0; i < _perfCount; i++) + { + subCipher = _impl->sub(bcos::ref(_c1), bcos::ref(_c2), _keyPair->pk()); + } + std::cout << "#### paillier sub perf:" << ((bcos::utcSteadyTimeUs() - startT) / _perfCount) + << " us" << std::endl; + /// case not enough resultBuffer; + BOOST_CHECK_THROW( + _impl->sub(&resultBuffer, bcos::ref(_c1), bcos::ref(_c2), _keyPair->pk()), std::exception); + + result = _impl->decrypt(bcos::ref(subCipher), (void*)_keyPair.get()); + auto expectedSubBN = m1.sub(m2.bn().get()); + BOOST_CHECK(expectedSubBN.cmp(result.bn().get()) == 0); + + // ==== test mul ===== + // _value * _c1 + bcos::bytes mulCipher; + startT = bcos::utcSteadyTimeUs(); + ppc::crypto::BigNum v(_value); + for (uint64_t i = 0; i < _perfCount; i++) + { + mulCipher = _impl->scalaMul(v.bn().get(), bcos::ref(_c1), _keyPair->pk()); + } + std::cout << "#### paillier scalMul perf:" << ((bcos::utcSteadyTimeUs() - startT) / _perfCount) + << " us" << std::endl; + /// case not enough resultBuffer; + BOOST_CHECK_THROW(_impl->scalaMul(&resultBuffer, v.bn().get(), bcos::ref(_c1), _keyPair->pk()), + std::exception); + + // decrypt and check the result + auto resultBn = _codec->decode(_impl->decrypt(bcos::ref(mulCipher), (void*)_keyPair.get())); + BigNum valueBn(_value); + BigNum valueM1(_m1); + BigNum expectedBn; + auto ctx = createBNContext(); + valueBn.mul(expectedBn.bn().get(), valueM1.bn().get(), ctx.get()); + std::cout << "## scalaMul resultBn: "; + BN_print_fp(stdout, resultBn.bn().get()); + std::cout << std::endl; + std::cout << "## scalaMul expectedResult: "; + BN_print_fp(stdout, expectedBn.bn().get()); + std::cout << std::endl; + BOOST_CHECK(expectedBn.cmp(resultBn.bn().get()) == 0); +} + +void checkKeyPair(KeyPair::UniquePtr&& _keyPair1, OpenSSLPaillierKeyPair::UniquePtr&& _keyPair2) +{ + auto pk1 = (PaillierPublicKey*)_keyPair1->pk(); + auto pk2 = (PaillierPublicKey*)_keyPair2->pk(); + BOOST_CHECK(pk1->keyBits == pk2->keyBits); + BOOST_CHECK(pk1->n.cmp(pk2->n.bn().get()) == 0); + BOOST_CHECK(pk1->nSqrt.cmp(pk2->nSqrt.bn().get()) == 0); + BOOST_CHECK(pk1->h.cmp(pk2->h.bn().get()) == 0); + + BOOST_CHECK(pk1->h_s.cmp(pk2->h_s.bn().get()) == 0); + + auto sk1 = (PaillierPrivateKey*)_keyPair1->sk(); + auto sk2 = (PaillierPrivateKey*)_keyPair2->sk(); + BOOST_CHECK(sk1->keyBits == sk2->keyBits); + BOOST_CHECK(sk1->lambda.cmp(sk2->lambda.bn().get()) == 0); + BOOST_CHECK(sk1->p.cmp(sk2->p.bn().get()) == 0); + BOOST_CHECK(sk1->q.cmp(sk2->q.bn().get()) == 0); + BOOST_CHECK(sk1->pSqrt.cmp(sk2->pSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qSqrt.cmp(sk2->qSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qSqrtInverse.cmp(sk2->qSqrtInverse.bn().get()) == 0); + BOOST_CHECK(sk1->pOrderSqrt.cmp(sk2->pOrderSqrt.bn().get()) == 0); + BOOST_CHECK(sk1->qOrderSqrt.cmp(sk2->qOrderSqrt.bn().get()) == 0); +} + +void testOverflow(SignedNumberCodec::Ptr codec, OpenSSLPaillier::Ptr paillierImpl, + ppc::crypto::KeyPair::UniquePtr&& keyPair) +{ + // overflow case + auto v = codec->maxPositive(); + BigNum tmp(1); + tmp.add(tmp.bn().get(), v.bn().get()); + BOOST_CHECK_THROW(paillierImpl->encrypt_with_crt(tmp.bn().get(), (void*)keyPair.get()), + SignedNumberCodecException); + + // add overflow + BigNum two(2); + auto m1 = tmp.sub(two.bn().get()); + auto c1 = paillierImpl->encrypt_with_crt(m1.bn().get(), (void*)keyPair.get()); + BigNum m2(1000); + auto c2 = paillierImpl->encrypt_with_crt(m2.bn().get(), (void*)keyPair.get()); + auto pk = (PaillierPublicKey*)(keyPair->pk()); + auto cipher = paillierImpl->add(bcos::ref(c1), bcos::ref(c2), (void*)pk); + // decrypt failed for overflow + BOOST_CHECK_THROW( + paillierImpl->decrypt(bcos::ref(cipher), (void*)keyPair.get()), SignedNumberCodecException); +} + +BOOST_AUTO_TEST_CASE(testPaillier) +{ + auto paillierImpl = std::make_shared(); + // generate keyPair with keyLength bits + unsigned keyLength = 3072; + auto startT = bcos::utcSteadyTime(); + auto keyPair = paillierImpl->generateKeyPair(keyLength); + std::cout << "### generate keypair timecost:" << (bcos::utcSteadyTime() - startT) << " ms" + << std::endl; + auto pk = (PaillierPublicKey*)(keyPair->pk()); + + auto codec = std::make_shared(pk->n); + + /// === normal test case + // check encrypt/decrypt + int64_t perfCount = 2; + bcos::bytes cipherData1; + srand(bcos::utcSteadyTime()); + int64_t m1 = 12323423434 + rand(); + testPaillierEncryptDecrypt(cipherData1, paillierImpl, codec, keyPair, m1, perfCount); + + bcos::bytes cipherData2; + int64_t m2 = 2342342344 + rand(); + testPaillierEncryptDecrypt(cipherData2, paillierImpl, codec, keyPair, m2, perfCount); + + // check homo-property + int value = rand(); + testPaillierHomoProperty( + paillierImpl, codec, keyPair, value, cipherData1, cipherData2, m1, m2, perfCount); + + ////// negative case + m1 = -1234324 - rand(); + testPaillierEncryptDecrypt(cipherData1, paillierImpl, codec, keyPair, m1, 1); + m2 = 23434 + rand(); + testPaillierEncryptDecrypt(cipherData2, paillierImpl, codec, keyPair, m2, 1); + testPaillierHomoProperty( + paillierImpl, codec, keyPair, value, cipherData1, cipherData2, m1, m2, perfCount); + + value = -34534 - rand(); + testPaillierHomoProperty( + paillierImpl, codec, keyPair, value, cipherData1, cipherData2, m1, m2, perfCount); + + testOverflow(codec, paillierImpl, std::move(keyPair)); +} + +// test paillier keyPair +BOOST_AUTO_TEST_CASE(testPaillierKeyPair) +{ + int keyLength = 2048; + auto paillierImpl = std::make_shared(); + auto keyPair = paillierImpl->generateKeyPair(keyLength); + + auto skData = keyPair->serializeSK(); + auto pkData = keyPair->serializePK(); + + // create new keyPair according to skData and pkData + auto keyPair2 = std::make_unique( + skData.data(), skData.size(), pkData.data(), pkData.size()); + // check serialize again + auto skData2 = keyPair2->serializeSK(); + auto pkData2 = keyPair2->serializePK(); + BOOST_CHECK(skData == skData2); + BOOST_CHECK(pkData == pkData2); + checkKeyPair(std::move(keyPair), std::move(keyPair2)); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-homo/tests/main.cpp b/cpp/ppc-homo/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-homo/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-http/CMakeLists.txt b/cpp/ppc-http/CMakeLists.txt new file mode 100644 index 00000000..6039e2ed --- /dev/null +++ b/cpp/ppc-http/CMakeLists.txt @@ -0,0 +1,6 @@ +project(ppc-http VERSION ${VERSION}) +add_subdirectory(src) + +if (DEMO) + add_subdirectory(demo) +endif() \ No newline at end of file diff --git a/cpp/ppc-http/demo/CMakeLists.txt b/cpp/ppc-http/demo/CMakeLists.txt new file mode 100644 index 00000000..56b3c4bd --- /dev/null +++ b/cpp/ppc-http/demo/CMakeLists.txt @@ -0,0 +1,16 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(BINARY_NAME http-demo) +set(BINARY_CLIENT_NAME http-client-demo) + +add_executable(${BINARY_NAME} http_demo.cpp) +target_include_directories(${BINARY_NAME} PRIVATE .) +target_link_libraries(${BINARY_NAME} ${HTTP_TARGET} ${HELPER_TARGET} ${PROTOCOL_TARGET}) + +add_executable(${BINARY_CLIENT_NAME} http_client_demo.cpp) +target_include_directories(${BINARY_CLIENT_NAME} PRIVATE .) +target_link_libraries(${BINARY_CLIENT_NAME} ${HTTP_TARGET} ${HELPER_TARGET} ${PROTOCOL_TARGET} ${TOOLS-TARGET}) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-http/demo/config.ini b/cpp/ppc-http/demo/config.ini new file mode 100644 index 00000000..5d9a0ffb --- /dev/null +++ b/cpp/ppc-http/demo/config.ini @@ -0,0 +1,18 @@ +[agency] + id = agency0 + +[gateway] + listen_ip=0.0.0.0 + listen_port=8813 + ;thread_count = 4 + ; ssl or sm ssl + sm_ssl=true + ; the max allowed message size in MBytes, default is 100MBytes + max_allow_msg_size = 100 + ;ssl connection switch, if disable the ssl connection, default: false + disable_ssl = true + ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes + holding_msg_minutes = 10 + ;the protocol of gateway : websocket(0)/http(1) + protocol = 1 + url = /api/v1/interconn/invoke \ No newline at end of file diff --git a/cpp/ppc-http/demo/http_client_demo.cpp b/cpp/ppc-http/demo/http_client_demo.cpp new file mode 100644 index 00000000..106cb10a --- /dev/null +++ b/cpp/ppc-http/demo/http_client_demo.cpp @@ -0,0 +1,140 @@ +#include "../src/HttpClient.h" +#include "ppc-tools/src/common/TransTools.h" +#include +#include +#include +#include +#include +#include +#include + +using tcp = boost::asio::ip::tcp; +namespace http = boost::beast::http; +namespace beast = boost::beast; +namespace net = boost::asio; +using namespace ppc::http; +using namespace ppc::tools; + +#define PORT 1234 +#define PATH "/test" + +// Function to handle incoming HTTP requests +void handle_request(http::request& req, http::response& res) +{ + if (req.method() == http::verb::post && req.target() == PATH) + { + res.result(http::status::ok); + res.set(http::field::content_type, "text/plain"); + + // Access the binary data received in the request body + std::string& bodyData = req.body(); + + std::cout << "Server receives body: " << bodyData.data() << std::endl; + + // Save the binary data to a local file + std::ofstream outfile("http_out", std::ios::binary); + if (outfile.is_open()) + { + outfile.write(reinterpret_cast(bodyData.data()), bodyData.size()); + outfile.close(); + res.body() = "Binary data received and saved successfully."; + } + else + { + res.result(http::status::internal_server_error); + res.body() = "Failed to save binary data."; + } + } + else + { + res.result(http::status::not_found); + res.set(http::field::content_type, "text/plain"); + res.body() = "Not Found"; + } +} + +void newServer() +{ + try + { + // Create a Boost ASIO io_context + net::io_context ioc; + + // Create a TCP acceptor to listen on port + tcp::acceptor acceptor(ioc, {tcp::v4(), PORT}); + + while (true) + { + // Wait for an incoming connection + tcp::socket socket(ioc); + acceptor.accept(socket); + + // Read the request + beast::flat_buffer buffer; + http::request request; + http::read(socket, buffer, request); + + // Prepare the response + http::response response; + handle_request(request, response); + + // Send the response + http::write(socket, response); + socket.shutdown(tcp::socket::shutdown_send); + } + } + catch (const std::exception& e) + { + std::cerr << "Server Error: " << e.what() << std::endl; + } +} + +void testClient() +{ + try + { + net::io_context io_context; + std::thread io_thread([&io_context]() { io_context.run(); }); + + HttpClient client(io_context, "127.0.0.1", PORT); + + std::map headers = {{"Content-Type", "application/octet-stream"}}; + + bcos::bytes body = {'a', 'b', 'c', 'd'}; + auto response = client.post(PATH, headers, body); + std::cout << "Client receives body: " << std::string(response.begin(), response.end()) + << std::endl; + + body = {'1', '2', '3', '4'}; + response.clear(); + response = client.post(PATH, headers, body); + std::cout << "Client receives body: " << std::string(response.begin(), response.end()) + << std::endl; + + uint32_t a = 37; + auto body1 = std::make_shared(); + encodeUnsignedNum(body1, a); + response.clear(); + response = client.post(PATH, headers, *body1); + std::cout << "Client receives body: " << std::string(response.begin(), response.end()) + << std::endl; + + io_thread.join(); + } + catch (const std::exception& e) + { + std::cerr << "Client Error: " << e.what() << std::endl; + } +} + +int main(int argc, char const* argv[]) +{ + std::thread myThread(newServer); + std::cout << "Server started!" << std::endl; + + testClient(); + + // Wait for the new thread to finish (optional) + myThread.join(); + return 0; +} diff --git a/cpp/ppc-http/demo/http_demo.cpp b/cpp/ppc-http/demo/http_demo.cpp new file mode 100644 index 00000000..463f078c --- /dev/null +++ b/cpp/ppc-http/demo/http_demo.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file http_demo.cpp + * @author: zachma + * @date 2023-8-23 + */ + +#include "libhelper/CommandHelper.h" +#include "libhelper/ExitHandler.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-http/src/HttpFactory.h" +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::http; +using namespace ppc::tools; + +void registerUrlHandler(const std::string& _url, HttpInterface::Ptr _http) +{ + _http->registerUrlHandler( + _url, [](bcos::boostssl::http::HttpRequest const& _httpReq, RespUrlFunc _handler) { + auto reqs = _httpReq.body(); + std::cout << " [Url] ===>>>> " << LOG_KV("url_request", reqs) << std::endl; + _handler(nullptr, std::move(bcos::bytes(reqs.begin(), reqs.end()))); + }); +} + +int main(int argc, char const* argv[]) +{ + ExitHandler exitHandler; + signal(SIGTERM, &ExitHandler::exitHandler); + signal(SIGABRT, &ExitHandler::exitHandler); + signal(SIGINT, &ExitHandler::exitHandler); + + try + { + auto param = initCommandLine(argc, argv); + auto ppcConfig = std::make_shared(); + // not specify the certPath in air-mode + ppcConfig->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, param.configFilePath); + auto httpFactory = std::make_shared("selfParty"); + auto url = ppcConfig->gatewayConfig().networkConfig.url; + auto http = httpFactory->buildHttp(ppcConfig); + registerUrlHandler(url, http); + // start the http + http->start(); + while (!exitHandler.shouldExit()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "http-demo program exit normally." << std::endl; + } + catch (std::exception const& e) + { + ppc::printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "start http-demo failed, error:" << boost::diagnostic_information(e) + << std::endl; + return -1; + } + return 0; +} diff --git a/cpp/ppc-http/src/CMakeLists.txt b/cpp/ppc-http/src/CMakeLists.txt new file mode 100644 index 00000000..672dccd9 --- /dev/null +++ b/cpp/ppc-http/src/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${HTTP_TARGET} ${SRCS}) + +target_link_libraries(${HTTP_TARGET} PUBLIC ${TOOLS_TARGET} ${PROTOCOL_TARGET} ${BCOS_BOOSTSSL_TARGET}) \ No newline at end of file diff --git a/cpp/ppc-http/src/Http.cpp b/cpp/ppc-http/src/Http.cpp new file mode 100644 index 00000000..5f871d29 --- /dev/null +++ b/cpp/ppc-http/src/Http.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Http.cpp + * @author: zachma + * @date 2023-8-23 + */ + +#include "Http.h" +#include "ppc-framework/http/HttpTypeDef.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-tools/src/config/ParamChecker.h" +#include + +using namespace bcos; +using namespace ppc::http; +using namespace ppc::tools; +using namespace ppc::protocol; + +Http::Http(std::shared_ptr _wsService, std::string const& _selfPartyID, + std::string const& _token) + : m_wsService(std::move(_wsService)), m_token(_token) +{ + // register handler for http + auto httpServer = m_wsService->httpServer(); + if (httpServer) + { + httpServer->setHttpReqHandler([this](bcos::boostssl::http::HttpRequest&& _request, + std::function _responseHandler) { + this->onHttpRequest(std::move(_request), _responseHandler); + }); + } + HTTP_LOG(INFO) << LOG_DESC("init http success") << LOG_KV("selfParty", _selfPartyID); +} + +void Http::onHttpRequest( + bcos::boostssl::http::HttpRequest&& _request, std::function _responseHandler) +{ + std::string uri; + std::map params; + std::string url = std::string(_request.target().data(), _request.target().size()); + parseURL(url, uri, params); + auto it = m_urlToHandler.find(uri); + if (it == m_urlToHandler.end()) + { + HTTP_LOG(DEBUG) << LOG_DESC("onHttpRequest: uri not found") << LOG_KV("uri", uri); + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)HttpError::UriNotFound, "The uri does not exist/is not available.")); + } + auto const& methodHandler = it->second; + HTTP_LOG(DEBUG) << LOG_DESC("onHttpRequest: handle uri") << LOG_KV("uri", uri); + methodHandler(std::move(_request), [_responseHandler](Error::Ptr _error, + bcos::bytes&& _result) { _responseHandler(_result); }); +} \ No newline at end of file diff --git a/cpp/ppc-http/src/Http.h b/cpp/ppc-http/src/Http.h new file mode 100644 index 00000000..750f1973 --- /dev/null +++ b/cpp/ppc-http/src/Http.h @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Http.h + * @author: zachma + * @date 2023-8-23 + */ +#pragma once +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/http/HttpInterface.h" +#include +#include +#include +#include +#include + +namespace ppc::http +{ +class Http : public HttpInterface +{ +public: + using Ptr = std::shared_ptr; + Http(std::shared_ptr _wsService, std::string const& _selfPartyID, + std::string const& _token); + ~Http() override { stop(); } + + void start() override + { + if (m_wsService) + { + m_wsService->start(); + } + HTTP_LOG(INFO) << LOG_DESC("start HTTP"); + } + + void stop() override + { + if (m_wsService) + { + m_wsService->stop(); + } + HTTP_LOG(INFO) << LOG_DESC("stop HTTP"); + } + + void registerUrlHandler(std::string const& _urlName, + std::function _handler) + override + { + bcos::UpgradableGuard l(x_urlToHandler); + if (m_urlToHandler.count(_urlName)) + { + HTTP_LOG(INFO) << LOG_DESC("registerUrlHandler return for url handler already exists") + << LOG_KV("url", _urlName); + return; + } + bcos::UpgradeGuard ul(l); + m_urlToHandler[_urlName] = _handler; + HTTP_LOG(INFO) << LOG_DESC("registerUrlHandler success") << LOG_KV("url", _urlName); + } + +protected: + virtual void onHttpRequest(bcos::boostssl::http::HttpRequest&& _request, + std::function _responseHandler); + + void parseURL( + const std::string& _url, std::string& uri, std::map& params_map) + { + std::string params; + std::string url; + auto paramsStart = _url.find("?"); + if (paramsStart != std::string::npos) + { + params = _url.substr(paramsStart + 1); + uri = _url.substr(0, paramsStart - 1); + } + else + { + uri = _url; + } + + auto pos = 0U; + while (pos < params.size()) + { + auto eqPos = params.find("=", pos); + auto ampPos = params.find("&", pos); + + std::string paramName = params.substr(pos, eqPos - pos); + std::string paramValue; + if (ampPos != std::string::npos) + { + paramValue = params.substr(eqPos + 1, ampPos - eqPos - 1); + } + else + { + paramValue = params.substr(eqPos + 1); + } + + params_map[paramName] = paramValue; + if (ampPos == std::string::npos) + { + break; + } + + pos = ampPos + 1; + } + } + +private: + std::shared_ptr m_wsService; + + // the url_name to function + std::map> + m_urlToHandler; + + bcos::SharedMutex x_urlToHandler; + + std::string m_token; +}; +} // namespace ppc::http diff --git a/cpp/ppc-http/src/HttpClient.cpp b/cpp/ppc-http/src/HttpClient.cpp new file mode 100644 index 00000000..bdd52b99 --- /dev/null +++ b/cpp/ppc-http/src/HttpClient.cpp @@ -0,0 +1,122 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpClient.cpp + * @author: shawnhe + * @date 2023-07-25 + */ + +#include "HttpClient.h" + +using tcp = boost::asio::ip::tcp; +namespace http = boost::beast::http; +namespace ssl = boost::asio::ssl; +using namespace boost::system; + +bcos::bytes ppc::http::HttpClient::post(const std::string& path, + const std::map& headers, const bcos::bytes& body) +{ + return query(boost::beast::http::verb::post, path, headers, body); +} + +bcos::bytes ppc::http::HttpClient::get(const std::string& path, + const std::map& headers, const bcos::bytes& body) +{ + return query(boost::beast::http::verb::get, path, headers, body); +} + +bcos::bytes ppc::http::HttpClient::query(boost::beast::http::verb method, const std::string& path, + const std::map& headers, const bcos::bytes& body) +{ + boost::system::error_code ec; + // Declare a container to hold the response + boost::beast::http::response response; + + // This buffer is used for reading and must be persisted + boost::beast::flat_buffer buffer; + + // Make the connection on the IP address we get from a lookup + if (m_enableSsl) + { + boost::asio::ssl::stream socket_stream(m_ioContext, m_sslContext); + boost::asio::connect(socket_stream.next_layer(), m_endpoints); + socket_stream.handshake(ssl::stream_base::client); + + // Set up an HTTP GET request message + boost::beast::http::request request{ + boost::beast::http::verb::post, path, m_version}; + request.set(boost::beast::http::field::host, m_serverHost); + request.set(boost::beast::http::field::user_agent, BOOST_BEAST_VERSION_STRING); + request.set(boost::beast::http::field::content_length, std::to_string(body.size())); + + for (const auto& header : headers) + { + request.set(header.first, header.second); + } + + request.body() = std::string(reinterpret_cast(body.data()), body.size()); + + // Send the HTTP request to the remote host + boost::beast::http::write(socket_stream, request); + + + // Receive the HTTP response + boost::beast::http::read(socket_stream, buffer, response); + + // Gracefully close the socket + socket_stream.shutdown(ec); + } + else + { + boost::asio::ip::tcp::socket newSocket(m_ioContext); + boost::asio::connect(newSocket, m_endpoints.begin(), m_endpoints.end()); + + // Set up an HTTP GET request message + boost::beast::http::request request{ + boost::beast::http::verb::post, path, m_version}; + request.set(boost::beast::http::field::host, m_serverHost); + request.set(boost::beast::http::field::user_agent, BOOST_BEAST_VERSION_STRING); + request.set(boost::beast::http::field::content_length, std::to_string(body.size())); + + for (const auto& header : headers) + { + request.set(header.first, header.second); + } + + request.body() = std::string(reinterpret_cast(body.data()), body.size()); + + // Send the HTTP request to the remote host + boost::beast::http::write(newSocket, request); + + // Receive the HTTP response + boost::beast::http::read(newSocket, buffer, response); + + // Gracefully close the socket + newSocket.shutdown(tcp::socket::shutdown_both, ec); + } + + // not_connected happens sometimes + // so don't bother reporting it. + if (ec && ec != boost::system::errc::not_connected) + throw boost::system::system_error{ec}; + + // Copy the response body to a vector + std::vector response_data; + response_data.reserve(boost::asio::buffer_size(response.body().data())); + response_data.assign(boost::asio::buffers_begin(response.body().data()), + boost::asio::buffers_end(response.body().data())); + + return response_data; +} diff --git a/cpp/ppc-http/src/HttpClient.h b/cpp/ppc-http/src/HttpClient.h new file mode 100644 index 00000000..ee435bc2 --- /dev/null +++ b/cpp/ppc-http/src/HttpClient.h @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpClient.h + * @author: shawnhe + * @date 2023-07-25 + */ + +#pragma once +#include "bcos-utilities/Common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ppc::http +{ +class HttpClient +{ +public: + using Ptr = std::shared_ptr; + HttpClient(boost::asio::io_context& ioContext, const std::string& hostOrIp, unsigned short port, + bool enableSsl = false, unsigned short version = 11) + : m_ioContext(ioContext), + m_resolver(ioContext), + m_serverHost(hostOrIp), + m_port(port), + m_enableSsl(enableSsl), + m_version(version) + { + if (enableSsl) + { + m_sslContext.set_verify_mode(boost::asio::ssl::verify_peer); + } + m_endpoints = m_resolver.resolve(m_serverHost, std::to_string(m_port)); + } + + bcos::bytes post(const std::string& path, const std::map& headers, + const bcos::bytes& body); + + bcos::bytes get(const std::string& path, const std::map& headers, + const bcos::bytes& body); + +private: + bcos::bytes query(boost::beast::http::verb method, const std::string& path, + const std::map& headers, const bcos::bytes& body); + +private: + boost::asio::io_context& m_ioContext; + boost::asio::ip::tcp::resolver m_resolver; + boost::asio::ssl::context m_sslContext = + boost::asio::ssl::context(boost::asio::ssl::context::sslv23_client); + + std::string m_serverHost; + unsigned short m_port; + bool m_enableSsl; + unsigned short m_version; + + boost::asio::ip::tcp::resolver::results_type m_endpoints; +}; +} // namespace ppc::http diff --git a/cpp/ppc-http/src/HttpFactory.cpp b/cpp/ppc-http/src/HttpFactory.cpp new file mode 100644 index 00000000..cca22f21 --- /dev/null +++ b/cpp/ppc-http/src/HttpFactory.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpFactory.cpp + * @author: zachma + * @date 2023-8-23 + */ + +#include "HttpFactory.h" +#include +#include + +using namespace bcos; +using namespace ppc::http; +using namespace ppc::tools; + +Http::Ptr HttpFactory::buildHttp(ppc::tools::PPCConfig::ConstPtr _config) +{ + auto wsConfig = initConfig(_config); + // create the wsConfig + auto wsService = std::make_shared(); + wsService->setTimerFactory(std::make_shared()); + auto initializer = std::make_shared(); + + initializer->setConfig(wsConfig); + initializer->initWsService(wsService); + + return std::make_shared(wsService, m_selfPartyID, _config->gatewayConfig().networkConfig.token); +} + + +std::shared_ptr HttpFactory::initConfig( + ppc::tools::PPCConfig::ConstPtr _config) +{ + // init the wsConfig + auto wsConfig = std::make_shared(); + wsConfig->setModel(boostssl::ws::WsModel::Server); + + wsConfig->setListenIP(_config->gatewayConfig().networkConfig.listenIp); + wsConfig->setListenPort(_config->gatewayConfig().networkConfig.listenPort); + wsConfig->setThreadPoolSize(_config->gatewayConfig().networkConfig.threadPoolSize); + wsConfig->setDisableSsl(_config->gatewayConfig().networkConfig.disableSsl); + if (_config->gatewayConfig().networkConfig.disableSsl) + { + return wsConfig; + } + // non-sm-ssl + auto contextConfig = std::make_shared(); + if (!_config->gatewayConfig().networkConfig.enableSM) + { + boostssl::context::ContextConfig::CertConfig certConfig; + certConfig.caCert = _config->gatewayConfig().networkConfig.caCertPath; + certConfig.nodeCert = _config->gatewayConfig().networkConfig.sslCertPath; + certConfig.nodeKey = _config->gatewayConfig().networkConfig.sslKeyPath; + contextConfig->setCertConfig(certConfig); + contextConfig->setSslType("ssl"); + HTTP_LOG(INFO) << LOG_DESC("initConfig: http work in non-sm-ssl model") + << LOG_KV("caCert", certConfig.caCert) + << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("nodeKey", certConfig.nodeKey); + wsConfig->setContextConfig(contextConfig); + return wsConfig; + } + // sm-ssl + boostssl::context::ContextConfig::SMCertConfig certConfig; + certConfig.caCert = _config->gatewayConfig().networkConfig.smCaCertPath; + certConfig.nodeCert = _config->gatewayConfig().networkConfig.smSslCertPath; + certConfig.nodeKey = _config->gatewayConfig().networkConfig.smSslKeyPath; + certConfig.enNodeCert = _config->gatewayConfig().networkConfig.smEnSslCertPath; + certConfig.enNodeKey = _config->gatewayConfig().networkConfig.smEnSslKeyPath; + contextConfig->setSmCertConfig(certConfig); + contextConfig->setSslType("sm_ssl"); + + BCOS_LOG(INFO) << LOG_DESC("initConfig") << LOG_DESC("http work in sm ssl model") + << LOG_KV("caCert", certConfig.caCert) << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("enNodeCert", certConfig.enNodeCert); + + wsConfig->setContextConfig(contextConfig); + return wsConfig; +} diff --git a/cpp/ppc-http/src/HttpFactory.h b/cpp/ppc-http/src/HttpFactory.h new file mode 100644 index 00000000..d3d063f5 --- /dev/null +++ b/cpp/ppc-http/src/HttpFactory.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HttpFactory.h + * @author: zachma + * @date 2023-8-23 + */ +#pragma once +#include "Http.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include + +namespace bcos::boostssl::ws +{ +class WsConfig; +} + +namespace ppc::http +{ +class HttpFactory +{ +public: + using Ptr = std::shared_ptr; + HttpFactory(std::string const& _selfPartyID) : m_selfPartyID(_selfPartyID) {} + virtual ~HttpFactory() = default; + + Http::Ptr buildHttp(ppc::tools::PPCConfig::ConstPtr _config); + +private: + std::shared_ptr initConfig( + ppc::tools::PPCConfig::ConstPtr _config); + +private: + std::string m_selfPartyID; +}; +} // namespace ppc::http \ No newline at end of file diff --git a/cpp/ppc-io/CMakeLists.txt b/cpp/ppc-io/CMakeLists.txt new file mode 100644 index 00000000..94d81491 --- /dev/null +++ b/cpp/ppc-io/CMakeLists.txt @@ -0,0 +1,12 @@ +project(ppc-io VERSION ${VERSION}) + +add_subdirectory(src) +if (DEMO) + add_subdirectory(demo) +endif() + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/cpp/ppc-io/demo/CMakeLists.txt b/cpp/ppc-io/demo/CMakeLists.txt new file mode 100644 index 00000000..6daccc8c --- /dev/null +++ b/cpp/ppc-io/demo/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(BINARY_NAME io-demo) + +add_executable(${BINARY_NAME} ${SOURCES}) +target_include_directories(${BINARY_NAME} PRIVATE .) + +target_link_libraries(${BINARY_NAME} ${IO_TARGET} TBB::tbb) +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-io/demo/io_demo.cpp b/cpp/ppc-io/demo/io_demo.cpp new file mode 100644 index 00000000..4a1b1e42 --- /dev/null +++ b/cpp/ppc-io/demo/io_demo.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file io_demo.cpp + * @desc: demo for io + * @author: yujiechen + * @date 2022-10-19 + */ +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include + +using namespace bcos; +using namespace ppc::io; + +int main(int argc, char* argv[]) +{ + if (argc < 2) + { + std::cout << "Usage: " << argv[0] + << ": \n - [fileToRead(required)]\n - mmapGranularity: default is 500MB \n - " + "readPerBatch(default read all)\n - writeLines: default is 1000w"; + return -1; + } + std::string fileToRead = argv[1]; + uint64_t mmapGranularity = 500 * 1024 * 1024; + if (argc == 3) + { + mmapGranularity = atol(argv[2]); + } + int16_t readPerBatch = -1; + if (argc == 4) + { + readPerBatch = atoi(argv[3]); + } + uint64_t dataToWrite = 10000000; + if (argc == 5) + { + dataToWrite = atoi(argv[4]); + } + std::cout << "fileToRead:\t" << fileToRead << std::endl; + std::cout << "mmapGranularity:\t" << mmapGranularity << std::endl; + if (readPerBatch == -1) + { + std::cout << "readPerBatch:\t readAll" << std::endl; + } + else + { + std::cout << "readPerBatch:\t" << readPerBatch << std::endl; + } + + std::cout << "----- parseFileBegin----------"; + auto factory = std::make_shared(); + auto lineReader = factory->createLineReader(fileToRead, mmapGranularity); + auto startT = utcSteadyTime(); + uint64_t lineSize = 0; + while (true) + { + auto dataBatch = lineReader->next(readPerBatch); + if (!dataBatch) + { + break; + } + lineSize += dataBatch->size(); + } + std::cout << "timecost:\t" << (utcSteadyTime() - startT) << "ms " << std::endl; + std::cout << "fileSize:\t" << lineReader->capacity() << std::endl; + std::cout << "lines:\t\t" << lineSize << std::endl; + std::cout << "----- parseFileFinished----------"; + std::cout << "------ test writer --------------)"; + uint64_t batchSize = 10; + uint64_t dataGranularity = dataToWrite / batchSize; + std::vector dataVec; + for (uint64_t j = 0; j < dataGranularity; j++) + { + std::string data = "234k9lwekrwejrlewkrjwe"; + dataVec.emplace_back(bcos::bytes(data.begin(), data.end())); + } + for (uint64_t i = 0; i < 10; i++) + { + auto writer = std::make_shared("output.txt", false); + auto dataBatch = std::make_shared(); + auto tmpData = dataVec; + dataBatch->setData(std::move(tmpData)); + writer->writeLine(dataBatch, DataSchema::Bytes, "\n"); + writer->flush(); + writer->close(); + } +} diff --git a/cpp/ppc-io/src/BaseFileLineReader.cpp b/cpp/ppc-io/src/BaseFileLineReader.cpp new file mode 100644 index 00000000..a16a5875 --- /dev/null +++ b/cpp/ppc-io/src/BaseFileLineReader.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BaseFileLineReader.cpp + * @author: yujiechen + * @date 2022-10-19 + */ + +#include "BaseFileLineReader.h" +#include "Common.h" +#include +#include +#include +#include +#include +#include +using namespace ppc::io; +using namespace bcos; +// get the next _size line data +DataBatch::Ptr BaseFileLineReader::next(int64_t _size, DataSchema schema) +{ + // the file has already been read-finished + if (readFinished()) + { + return nullptr; + } + switch (schema) + { + case DataSchema::String: + { + if (-1 == _size) + { + return readAllData(DataSchema::String); + } + else if (_size <= 0) + { + BOOST_THROW_EXCEPTION(InvalidParam() << bcos::errinfo_comment( + "The read-size parameter must be -1 or positive")); + } + else + { + return readLineData(_size, DataSchema::String); + } + } + case DataSchema::Bytes: + { + if (-1 == _size) + { + return readAllData(DataSchema::Bytes); + } + else if (_size <= 0) + { + BOOST_THROW_EXCEPTION(InvalidParam() << bcos::errinfo_comment( + "The read-size parameter must be -1 or positive")); + } + else + { + return readLineData(_size, DataSchema::Bytes); + } + } + default: + { + BOOST_THROW_EXCEPTION( + UnsupportedDataSchema() << errinfo_comment("unsupported data schema")); + } + } +} + +// read all bytes from the file, without parse +bcos::bytes BaseFileLineReader::readBytes() +{ + bcos::bytes result; + while (allocateCurrentBlock()) + { + m_currentBlockParser->readAll(result); + m_offset = result.size(); + } + IO_LOG(INFO) << LOG_DESC("readBytes") << LOG_KV("result", result.size()) + << LOG_KV("length", m_length); + return result; +} \ No newline at end of file diff --git a/cpp/ppc-io/src/BaseFileLineReader.h b/cpp/ppc-io/src/BaseFileLineReader.h new file mode 100644 index 00000000..001d6812 --- /dev/null +++ b/cpp/ppc-io/src/BaseFileLineReader.h @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BaseFileLineReader.h + * @author: yujiechen + * @date 2022-10-19 + */ +#pragma once +#include "parser/MmapParser.h" +#include "ppc-framework/io/LineReader.h" +#include + +namespace ppc::io +{ +class BaseFileLineReader : public LineReader +{ +public: + BaseFileLineReader(char _lineSpliter) : m_lineSplitter(_lineSpliter) {} + ~BaseFileLineReader() override {} + + /** + * @brief get the next _offset line data + * + * @param _size: the data-size to be read + * read all rows at once when the _size is -1 + * @return DataBatch::Ptr: the container that contains the readed-data + */ + DataBatch::Ptr next(int64_t _size = -1, DataSchema schema = DataSchema::String) override; + + // the capacity of the file or memory-bytes + uint64_t capacity() const override { return m_length; } + + uint64_t columnSize() const override { return 1; } + bcos::bytes readBytes() override; + +protected: + virtual bool allocateCurrentBlock() = 0; + virtual bool readFinished() const = 0; + + template + int64_t readLineFromBlock(DataBatch::Ptr _dataBatch, int64_t _size) + { + int64_t readedLineNum = 0; + int64_t readedBytes = 0; + for (; readedLineNum < _size; readedLineNum++) + { + T dataItem; + auto readLineResult = + m_currentBlockParser->readLine(readedBytes, dataItem, m_lineSplitter); + // update the offset + m_offset += readedBytes; + if (m_appendToLine) + { + m_appendToLine = false; + _dataBatch->appendToLine(std::move(dataItem)); + } + else + { + _dataBatch->append(std::move(dataItem)); + } + if (ReadLineResult::ReadFinishWithoutSpliter == readLineResult) + { + m_appendToLine = true; + } + // reset the current mmap-file since it has been read-finished + if (ReadLineResult::ReadFinish == readLineResult || + ReadLineResult::ReadFinishWithoutSpliter == readLineResult) + { + m_currentBlockParser.reset(); + break; + } + } + IO_LOG(INFO) << LOG_DESC("readLineFromBlock") << LOG_KV("readedLineNum", readedLineNum) + << LOG_KV("expectedSize", _size) << LOG_KV("offset", m_offset); + return readedLineNum; + } + + template + DataBatch::Ptr readLineData(int64_t _size, DataSchema _schema) + { + auto dataBatch = std::make_shared(); + dataBatch->setDataSchema(_schema); + int64_t expectedSize = _size; + while (expectedSize > 0 && (allocateCurrentBlock())) + { + auto readLineNum = readLineFromBlock(dataBatch, expectedSize); + expectedSize = _size - readLineNum; + } + IO_LOG(INFO) << LOG_DESC("readLineData") << LOG_KV("size", _size) + << LOG_KV("readedSize", dataBatch->size()); + return dataBatch; + } + + template + DataBatch::Ptr readAllData(DataSchema _dataSchema) + { + auto dataBatch = std::make_shared(); + dataBatch->setDataSchema(_dataSchema); + while (allocateCurrentBlock()) + { + readLineFromBlock(dataBatch, m_readGranularity); + } + IO_LOG(INFO) << LOG_DESC("readAllData") << LOG_KV("size", dataBatch->size()); + return dataBatch; + } + +protected: + char m_lineSplitter; + uint64_t m_length; + BaseBufferParser::Ptr m_currentBlockParser; // the current block-parser + + mutable uint64_t m_offset = 0; // the current read pointer + bool m_appendToLine = false; + + // Note: the m_readGranularity should be set by the override class + uint64_t m_readGranularity; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/CMakeLists.txt b/cpp/ppc-io/src/CMakeLists.txt new file mode 100644 index 00000000..4be7d7a8 --- /dev/null +++ b/cpp/ppc-io/src/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${IO_TARGET} ${SRCS}) +target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-io/src/Common.h b/cpp/ppc-io/src/Common.h new file mode 100644 index 00000000..f771627a --- /dev/null +++ b/cpp/ppc-io/src/Common.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-20 + */ +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-framework/storage/FileStorage.h" +#include +#include + +#define IO_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("IO") + +namespace ppc::io +{ +DERIVE_PPC_EXCEPTION(ConnectionOptionNotFound); +DERIVE_PPC_EXCEPTION(InvalidMmapGranularity); +DERIVE_PPC_EXCEPTION(MmapFileException); +DERIVE_PPC_EXCEPTION(InvalidParam); +DERIVE_PPC_EXCEPTION(OpenFileLineWriterException); +DERIVE_PPC_EXCEPTION(CloseFileLineWriterException); +DERIVE_PPC_EXCEPTION(UnSupportedDataResource); +DERIVE_PPC_EXCEPTION(LoadDataResourceException); + +DERIVE_PPC_EXCEPTION(HDFSOpenMetaInfoFailed); +DERIVE_PPC_EXCEPTION(HDFSReadDataFailed); +DERIVE_PPC_EXCEPTION(HDFSWriteDataFailed); +DERIVE_PPC_EXCEPTION(HDFSFlushFailed); + +struct HdfsFileWrapper +{ + HdfsFileWrapper(ppc::storage::FileHandler::Ptr _handler, HdfsFileInternalWrapper* _fs) + : handler(std::move(_handler)), fs(_fs) + {} + ppc::storage::FileHandler::Ptr handler; + HdfsFileInternalWrapper* fs; +}; + +struct HDFSFileDeleter +{ + void operator()(HdfsFileWrapper* _wrapper) const + { + if (!_wrapper->fs) + { + return; + } + hdfsCloseFile((hdfsFS)_wrapper->handler->storageHandler(), _wrapper->fs); + } + ppc::storage::FileHandler::Ptr handler; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/DataResourceLoaderImpl.cpp b/cpp/ppc-io/src/DataResourceLoaderImpl.cpp new file mode 100644 index 00000000..c312bed5 --- /dev/null +++ b/cpp/ppc-io/src/DataResourceLoaderImpl.cpp @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataResourceLoaderImpl.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "DataResourceLoaderImpl.h" +#include "Common.h" +#include "FileLineReader.h" +#include "FileLineWriter.h" +#include "HDFSReader.h" +#include "HDFSWriter.h" +#include "SQLResultReader.h" +#include +#include + +using namespace ppc::io; +using namespace ppc::storage; +using namespace ppc::protocol; +using namespace bcos; + +LineReader::Ptr DataResourceLoaderImpl::loadReader(DataResourceDesc::ConstPtr _desc, + DataSchema _schema, bool _parseByColumn, FileStorage::Ptr const& _fileStorage) +{ + if (!_desc) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() + << bcos::errinfo_comment("must define the desc for data-resource")); + } + switch (_desc->type()) + { + // load lineReader from file + case (int)(DataResourceType::FILE): + { + return std::make_shared(_desc->path()); + } + // load reader from the sql + case (int)(DataResourceType::MySQL): + { + return loadSQLResource(_desc, _schema, _parseByColumn); + } + case (int)(DataResourceType::HDFS): + { + return loadHDFSResource(_desc, _fileStorage); + } + default: + { + BOOST_THROW_EXCEPTION( + UnSupportedDataResource() << errinfo_comment("Only support File/MySQL/HDFS now")); + } + } +} + +LineReader::Ptr DataResourceLoaderImpl::loadSQLResource( + DataResourceDesc::ConstPtr _desc, DataSchema, bool _parseByColumn) +{ + IO_LOG(INFO) << LOG_DESC("loadSQLResource") << LOG_KV("command", _desc->accessCommand()); + SQLStorage::Ptr storage; + if (_desc->sqlConnectionOption()) + { + storage = m_sqlStorageFactory->createSQLStorage( + (ppc::protocol::DataResourceType)_desc->type(), _desc->sqlConnectionOption()); + } + else if (m_sqlConnectionOpt) + { + storage = m_sqlStorageFactory->createSQLStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_sqlConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION( + ConnectionOptionNotFound() << errinfo_comment("mysql connection option not found")); + } + + // exec command + auto result = storage->execQuery( + _parseByColumn, _desc->accessCommand().c_str(), FieldDataType::TERMINATE); + return std::make_shared(std::move(result), _parseByColumn); +} + +LineReader::Ptr DataResourceLoaderImpl::loadHDFSResource( + ppc::protocol::DataResourceDesc::ConstPtr _desc, FileStorage::Ptr const& _storage) +{ + IO_LOG(INFO) << LOG_DESC("loadHDFSResource") << LOG_KV("path", _desc->path()); + auto storage = _storage; + if (!storage) + { + if (_desc->fileStorageConnectionOption()) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), + _desc->fileStorageConnectionOption()); + } + else if (m_fileStorageConnectionOpt) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION( + ConnectionOptionNotFound() << errinfo_comment("hdfs connection option not found")); + } + } + // load the HDFSReader, open the file + auto handler = storage->openFile(_desc->path()); + return std::make_shared(handler); +} + +void DataResourceLoaderImpl::deleteResource( + ppc::protocol::DataResourceDesc::ConstPtr _desc, ppc::storage::FileStorage::Ptr const& _storage) +{ + if (!_desc) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() + << bcos::errinfo_comment("must define the desc for data-resource")); + } + switch (_desc->type()) + { + case (int)(DataResourceType::FILE): + case (int)(DataResourceType::HDFS): + { + auto storage = _storage; + if (!storage) + { + if (_desc->fileStorageConnectionOption()) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), + _desc->fileStorageConnectionOption()); + } + else if (m_fileStorageConnectionOpt) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION(ConnectionOptionNotFound() + << errinfo_comment("hdfs connection option not found")); + } + } + + if (storage->fileExists(_desc->path())) + { + storage->deleteFile(_desc->path()); + } + break; + } + default: + { + BOOST_THROW_EXCEPTION( + UnSupportedDataResource() << errinfo_comment( + "checkResourceExists: Only support File/HDFS now, passed in resource type: " + + std::to_string(_desc->type()))); + } + } +} + +void DataResourceLoaderImpl::renameResource(ppc::protocol::DataResourceDesc::ConstPtr _desc, + std::string const& _new_path, ppc::storage::FileStorage::Ptr const& _storage) +{ + if (!_desc) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() + << bcos::errinfo_comment("must define the desc for data-resource")); + } + switch (_desc->type()) + { + case (int)(DataResourceType::FILE): + case (int)(DataResourceType::HDFS): + { + auto storage = _storage; + if (!storage) + { + if (_desc->fileStorageConnectionOption()) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), + _desc->fileStorageConnectionOption()); + } + else if (m_fileStorageConnectionOpt) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION(ConnectionOptionNotFound() + << errinfo_comment("hdfs connection option not found")); + } + } + + if (storage->fileExists(_new_path)) + { + storage->deleteFile(_new_path); + } + storage->renameFile(_desc->path(), _new_path); + break; + } + default: + { + BOOST_THROW_EXCEPTION( + UnSupportedDataResource() << errinfo_comment( + "checkResourceExists: Only support File/HDFS now, passed in resource type: " + + std::to_string(_desc->type()))); + } + } +} + +void DataResourceLoaderImpl::checkResourceExists( + ppc::protocol::DataResourceDesc::ConstPtr _desc, ppc::storage::FileStorage::Ptr const& _storage) +{ + if (!_desc) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() + << bcos::errinfo_comment("must define the desc for data-resource")); + } + switch (_desc->type()) + { + case (int)(DataResourceType::FILE): + case (int)(DataResourceType::HDFS): + { + auto storage = _storage; + if (!storage) + { + if (_desc->fileStorageConnectionOption()) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), + _desc->fileStorageConnectionOption()); + } + else if (m_fileStorageConnectionOpt) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION(ConnectionOptionNotFound() + << errinfo_comment("hdfs connection option not found")); + } + } + if (storage->fileExists(_desc->path())) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() << bcos::errinfo_comment( + "The hdfs file: " + _desc->path() + " already exists!")); + } + break; + } + default: + { + BOOST_THROW_EXCEPTION( + UnSupportedDataResource() << errinfo_comment( + "checkResourceExists: Only support File/HDFS now, passed in resource type: " + + std::to_string(_desc->type()))); + } + } +} + + +LineWriter::Ptr DataResourceLoaderImpl::loadWriter( + DataResourceDesc::ConstPtr _desc, bool _truncate, FileStorage::Ptr const& _storage) +{ + if (!_desc) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() + << bcos::errinfo_comment("must define the desc for data-resource")); + } + switch (_desc->type()) + { + case (int)(DataResourceType::FILE): + { + return std::make_shared(_desc->path(), _truncate); + } + case (int)(DataResourceType::HDFS): + { + auto storage = _storage; + if (!storage) + { + if (_desc->fileStorageConnectionOption()) + { + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), + _desc->fileStorageConnectionOption()); + } + else if (m_fileStorageConnectionOpt) + { + // create the hdfsStorage + storage = m_fileStorageFactory->createFileStorage( + (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt); + } + else + { + BOOST_THROW_EXCEPTION(ConnectionOptionNotFound() + << errinfo_comment("hdfs connection option not found")); + } + } + auto handler = storage->openFile(_desc->path(), true); + return std::make_shared(handler, _truncate); + } + default: + { + BOOST_THROW_EXCEPTION( + UnSupportedDataResource() << errinfo_comment("Only support File/HDFS now")); + } + } +} \ No newline at end of file diff --git a/cpp/ppc-io/src/DataResourceLoaderImpl.h b/cpp/ppc-io/src/DataResourceLoaderImpl.h new file mode 100644 index 00000000..a1c850fa --- /dev/null +++ b/cpp/ppc-io/src/DataResourceLoaderImpl.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataResourceLoaderImpl.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/storage/FileStorage.h" +#include "ppc-framework/storage/RemoteStorage.h" +#include "ppc-framework/storage/SQLStorage.h" +#include + +namespace ppc::io +{ +class DataResourceLoaderImpl : public DataResourceLoader +{ +public: + using Ptr = std::shared_ptr; + DataResourceLoaderImpl(ppc::protocol::SQLConnectionOption::Ptr const& _sqlConnectionOpt, + ppc::protocol::FileStorageConnectionOption::Ptr const& _fileStorageConnectionOpt, + ppc::protocol::RemoteStorageConnectionOption::Ptr const& _remoteStorageConnectionOpt, + ppc::storage::SQLStorageFactory::Ptr const& _sqlStorageFactory, + ppc::storage::FileStorageFactory::Ptr const& _fileStorageFactory, + ppc::storage::RemoteStorageFactory::Ptr const& _remoteStorageFactory) + : m_sqlConnectionOpt(_sqlConnectionOpt), + m_fileStorageConnectionOpt(_fileStorageConnectionOpt), + m_remoteStorageConnectionOpt(_remoteStorageConnectionOpt), + m_sqlStorageFactory(_sqlStorageFactory), + m_fileStorageFactory(_fileStorageFactory), + m_remoteStorageFactory(_remoteStorageFactory) + {} + + DataResourceLoaderImpl() = default; + ~DataResourceLoaderImpl() override = default; + + LineReader::Ptr loadReader(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::io::DataSchema _schema = ppc::io::DataSchema::String, bool _parseByColumn = true, + ppc::storage::FileStorage::Ptr const& _storage = nullptr) override; + + LineWriter::Ptr loadWriter(ppc::protocol::DataResourceDesc::ConstPtr _desc, + bool _truncate = false, ppc::storage::FileStorage::Ptr const& _storage = nullptr) override; + + void checkResourceExists(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::storage::FileStorage::Ptr const& _storage = nullptr) override; + + void deleteResource(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::storage::FileStorage::Ptr const& _storage = nullptr); + + void renameResource(ppc::protocol::DataResourceDesc::ConstPtr _desc, + std::string const& _new_path, ppc::storage::FileStorage::Ptr const& _storage = nullptr); + +protected: + virtual LineReader::Ptr loadSQLResource(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::io::DataSchema _schema, bool _parseByColumn); + + virtual LineReader::Ptr loadHDFSResource(ppc::protocol::DataResourceDesc::ConstPtr _desc, + ppc::storage::FileStorage::Ptr const& _fileStorage); + +private: + ppc::protocol::SQLConnectionOption::Ptr m_sqlConnectionOpt; + ppc::protocol::FileStorageConnectionOption::Ptr m_fileStorageConnectionOpt; + ppc::protocol::RemoteStorageConnectionOption::Ptr m_remoteStorageConnectionOpt; + + ppc::storage::SQLStorageFactory::Ptr m_sqlStorageFactory; + ppc::storage::FileStorageFactory::Ptr m_fileStorageFactory; + ppc::storage::RemoteStorageFactory::Ptr m_remoteStorageFactory; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/FileLineReader.cpp b/cpp/ppc-io/src/FileLineReader.cpp new file mode 100644 index 00000000..7a71585d --- /dev/null +++ b/cpp/ppc-io/src/FileLineReader.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileLineReader.cpp + * @author: yujiechen + * @date 2022-10-19 + */ + +#include "FileLineReader.h" +#include "Common.h" +#include +#include +#include +#include +#include +#include +using namespace ppc::io; +using namespace bcos; + +FileLineReader::FileLineReader( + std::string const& _filePath, int64_t _mmapGranularity, char _lineSpliter) + : BaseFileLineReader(_lineSpliter), + m_filePath(_filePath), + c_pageSize(getpagesize()), + m_mmapGranularity(_mmapGranularity) +{ + if (-1 == _mmapGranularity) + { + m_mmapGranularity = 500 * 1024 * c_pageSize; + } + // check the mmap-granularity + if (m_mmapGranularity % c_pageSize != 0) + { + BOOST_THROW_EXCEPTION( + InvalidMmapGranularity() << errinfo_comment( + "the mmapGranularity must a multiple of page-size: " + std::to_string(c_pageSize))); + } + // check whether the file exists + if (!boost::filesystem::exists(boost::filesystem::path(m_filePath))) + { + BOOST_THROW_EXCEPTION(OpenFileFailed() << errinfo_comment(_filePath + " not exist!")); + } + // the filePath should not be directory + if (boost::filesystem::is_directory(boost::filesystem::path(m_filePath))) + { + BOOST_THROW_EXCEPTION( + OpenFileFailed() << errinfo_comment( + "The " + _filePath + " is a directory. Only support load content from file!")); + } + m_readGranularity = m_mmapGranularity / 8; + // open the file + m_fd = open(m_filePath.c_str(), O_RDONLY, 0644); + if (-1 == m_fd) + { + BOOST_THROW_EXCEPTION( + OpenFileFailed() << errinfo_comment("open file " + _filePath + " exception")); + } + m_length = lseek(m_fd, 0, SEEK_END); + IO_LOG(INFO) << LOG_DESC("FileLineReader: open file success") << LOG_KV("file", _filePath) + << LOG_KV("length", m_length) << LOG_KV("pageSize", c_pageSize); +} + +FileLineReader::~FileLineReader() +{ + if (-1 == m_fd) + { + return; + } + close(m_fd); +} + +bool FileLineReader::allocateCurrentBlock() +{ + if (readFinished()) + { + return false; + } + // holds mmap that has not been read-finished + if (m_currentBlockParser && m_currentBlockParser->pointer() < m_currentBlockParser->length()) + { + return true; + } + // without mmap or the mmap has already been read-finished, re-allocate a new mmapFile + auto mmapSize = std::min(m_mmapGranularity, (m_length - m_offset)); + m_currentBlockParser = std::make_unique(m_fd, m_offset, mmapSize); + IO_LOG(TRACE) << LOG_DESC("allocate new mmap") << LOG_KV("size", mmapSize) + << LOG_KV("offset", m_offset); + return true; +} \ No newline at end of file diff --git a/cpp/ppc-io/src/FileLineReader.h b/cpp/ppc-io/src/FileLineReader.h new file mode 100644 index 00000000..612a6654 --- /dev/null +++ b/cpp/ppc-io/src/FileLineReader.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileLineReader.h + * @author: yujiechen + * @date 2022-10-19 + */ +#pragma once +#include "BaseFileLineReader.h" +#include "parser/MmapParser.h" +#include "ppc-framework/io/LineReader.h" +#include + +namespace ppc::io +{ +class FileLineReader : public BaseFileLineReader +{ +public: + using Ptr = std::shared_ptr; + // Note: the same file can't be read/write in the same time + FileLineReader( + std::string const& _filePath, int64_t _mmapGranularity = -1, char _lineSpliter = '\n'); + ~FileLineReader() override; + + ppc::protocol::DataResourceType type() const override + { + return ppc::protocol::DataResourceType::FILE; + } + +protected: + bool allocateCurrentBlock() override; + bool readFinished() const override { return (m_offset == m_length); } + +protected: + std::string m_filePath; + int c_pageSize; + // Note: the mmapGranularity must a multiple of page-size + uint64_t m_mmapGranularity; // the mmap-granularity, default is (500 * 1024 * pageSize) + + int m_fd = -1; +}; + +class FileLineReaderFactory : public LineReaderFactory +{ +public: + using Ptr = std::shared_ptr; + FileLineReaderFactory() = default; + ~FileLineReaderFactory() override = default; + + LineReader::Ptr createLineReader(std::string const& _filePath, + uint64_t _mmapGranularity = 500 * 1024 * 1024, char _lineSpliter = '\n') override + { + return std::make_shared(_filePath, _mmapGranularity, _lineSpliter); + } +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/FileLineWriter.cpp b/cpp/ppc-io/src/FileLineWriter.cpp new file mode 100644 index 00000000..c8f74832 --- /dev/null +++ b/cpp/ppc-io/src/FileLineWriter.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileLineWriter.cpp + * @author: yujiechen + * @date 2022-10-21 + */ +#include "FileLineWriter.h" +#include +#include + +using namespace ppc::io; +using namespace bcos; +FileLineWriter::FileLineWriter(std::string const& _filePath, bool _trunc) : m_filePath(_filePath) +{ + boost::filesystem::path filePath(m_filePath); + auto parentPath = filePath.parent_path(); + // the parentPath is a file + if (!parentPath.empty() && boost::filesystem::exists(filePath.parent_path()) && + !boost::filesystem::is_directory(parentPath)) + { + BOOST_THROW_EXCEPTION( + OpenFileLineWriterException() << bcos::errinfo_comment( + "open " + m_filePath + " error for the non-dir-parent-path exists!")); + } + try + { + // the parentPath not exists, create the parentPath + if (!parentPath.empty() && !boost::filesystem::is_directory(parentPath)) + { + boost::filesystem::create_directories(parentPath); + } + m_outStream.open(m_filePath, std::ios::binary | (_trunc ? std::ios::trunc : std::ios::app)); + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + OpenFileLineWriterException() << bcos::errinfo_comment( + "open " + m_filePath + " error: " + boost::diagnostic_information(e))); + } +} + +// write the data into the writer(Note: only support row-data) +bool FileLineWriter::writeLine( + DataBatch::ConstPtr _data, DataSchema _schema, std::string _lineSplitter) +{ + switch (_schema) + { + case DataSchema::String: + { + writeData(_data, _lineSplitter); + return true; + } + case DataSchema::Bytes: + { + writeData(_data, _lineSplitter); + return true; + } + default: + { + BOOST_THROW_EXCEPTION(UnsupportedDataSchema() << bcos::errinfo_comment( + "Only support String and Bytes schema now!")); + } + } +} + +void FileLineWriter::writeBytes(bytesConstRef _data) +{ + m_outStream.write(reinterpret_cast(_data.data()), _data.size()); +} + + +// flush the data into the storage-backend +void FileLineWriter::flush() +{ + m_outStream.flush(); +} +// close the storage-backend resource(Note: flush before close) +void FileLineWriter::close() +{ + m_outStream.close(); +} \ No newline at end of file diff --git a/cpp/ppc-io/src/FileLineWriter.h b/cpp/ppc-io/src/FileLineWriter.h new file mode 100644 index 00000000..19b762fa --- /dev/null +++ b/cpp/ppc-io/src/FileLineWriter.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileLineWriter.h + * @author: yujiechen + * @date 2022-10-21 + */ +#pragma once +#include "Common.h" +#include "ppc-framework/io/LineWriter.h" +#include + +namespace ppc::io +{ +class FileLineWriter : public LineWriter +{ +public: + using Ptr = std::shared_ptr; + FileLineWriter(std::string const& _filePath, bool _trunc = false); + ~FileLineWriter() override + { + try + { + flush(); + close(); + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION( + CloseFileLineWriterException() << bcos::errinfo_comment( + "close " + m_filePath + " error: " + boost::diagnostic_information(e))); + } + } + + ppc::protocol::DataResourceType type() const override + { + return ppc::protocol::DataResourceType::FILE; + } + + // write the data into the writer(Note: only support row-data) + bool writeLine( + DataBatch::ConstPtr _data, DataSchema _schema, std::string _lineSplitter) override; + + void writeBytes(bcos::bytesConstRef _data) override; + + // flush the data into the storage-backend + void flush() override; + // close the storage-backend resource(Note: flush before close) + void close() override; + +private: + template + void writeData(DataBatch::ConstPtr _dataBatch, std::string _lineSplitter) + { + IO_LOG(TRACE) << LOG_DESC("writeData, size:") << _dataBatch->size(); + for (uint64_t i = 0; i < _dataBatch->size(); i++) + { + auto const& element = _dataBatch->get(i); + m_outStream.write(reinterpret_cast(element.data()), element.size()); + if (_lineSplitter.empty()) + { + continue; + } + m_outStream.write(_lineSplitter.c_str(), _lineSplitter.size()); + } + } + +protected: + std::ofstream m_outStream; + std::string m_filePath; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/HDFSReader.cpp b/cpp/ppc-io/src/HDFSReader.cpp new file mode 100644 index 00000000..a6432b4f --- /dev/null +++ b/cpp/ppc-io/src/HDFSReader.cpp @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSReader.cpp + * @author: yujiechen + * @date 2022-11-30 + */ +#include "HDFSReader.h" + +using namespace bcos; +using namespace ppc::io; +using namespace ppc::storage; + +HDFSReader::HDFSReader(FileHandler::Ptr _handler, uint64_t _bufferSize, char _lineSpliter) + : BaseFileLineReader(_lineSpliter), m_handler(_handler), m_bufferSize(_bufferSize) +{ + m_length = ((hdfsFileInfo*)_handler->fileInfoHandler())->mSize; + m_hostsInfo = + hdfsGetHosts((hdfsFS)_handler->storageHandler(), _handler->path().c_str(), 0, m_length); + if (!m_hostsInfo) + { + BOOST_THROW_EXCEPTION( + HDFSOpenMetaInfoFailed() + << errinfo_comment("hdfsGetHosts failed, error: " + std::string(hdfsGetLastError()))); + } + // set the m_readGranularity + m_readGranularity = m_bufferSize / 8; + IO_LOG(DEBUG) << LOG_DESC("create HDFSReader success") << LOG_KV("path", _handler->path()) + << LOG_KV("fileSize", m_length) << LOG_KV("bufferSize", m_bufferSize) + << LOG_KV("readGranularity", m_readGranularity); +} + +bool HDFSReader::allocateCurrentBlock() +{ + if (readFinished()) + { + return false; + } + // the loaded block-parser has not been read-finished + if (m_currentBlockParser && m_currentBlockParser->pointer() < m_currentBlockParser->length()) + { + return true; + } + + // read data from the current fd + if (m_currentFd && loadDataAndCreateBufferParser()) + { + return true; + } + // the current fd is null or read-empty-data from the current fd + if (!tryToOpenNewFile()) + { + return false; + } + // open new file success, try to load-data and create buffer parser + return loadDataAndCreateBufferParser(); +} + +bool HDFSReader::tryToOpenNewFile() +{ + m_currentBlockOffset++; + if (!m_hostsInfo) + { + return false; + } + if (!m_hostsInfo[m_currentBlock]) + { + return false; + } + char* hostName = m_hostsInfo[m_currentBlock][m_currentBlockOffset]; + // the block has been read-finished + if (!hostName) + { + m_currentBlock++; + // reset the offset to 0 + m_currentBlockOffset = 0; + } + // all blocks has been read-finished + if (!m_hostsInfo[m_currentBlock]) + { + IO_LOG(INFO) << LOG_DESC("tryToOpenNewFile finish for null block") + << LOG_KV("block", m_currentBlock); + + return false; + } + // open the file + hostName = m_hostsInfo[m_currentBlock][m_currentBlockOffset]; + if (!hostName) + { + IO_LOG(INFO) << LOG_DESC("tryToOpenNewFile finish for null block items") + << LOG_KV("block", m_currentBlock) + << LOG_KV("blockOffset", m_currentBlockOffset); + return false; + } + // release the old fd + hdfsFS fs = (hdfsFS)m_handler->storageHandler(); + // try to open the new file + m_currentFd.reset(new HdfsFileWrapper( + m_handler, hdfsOpenFile2(fs, hostName, m_handler->path().c_str(), O_RDONLY, + ((hdfsFileInfo*)(m_handler->fileInfoHandler()))->mBlockSize, 0, 0))); + if (!m_currentFd) + { + BOOST_THROW_EXCEPTION(OpenFileFailed() << errinfo_comment( + "HDFSOpenFileFailed failed, host:" + std::string(hostName) + + ", error: " + std::string(hdfsGetLastError()))); + } + return true; +} \ No newline at end of file diff --git a/cpp/ppc-io/src/HDFSReader.h b/cpp/ppc-io/src/HDFSReader.h new file mode 100644 index 00000000..c8169d97 --- /dev/null +++ b/cpp/ppc-io/src/HDFSReader.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSReader.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "BaseFileLineReader.h" +#include "parser/BufferParser.h" +#include "ppc-framework/io/LineReader.h" +#include "ppc-framework/storage/FileStorage.h" +#include + +namespace ppc::io +{ +class HDFSReader : public BaseFileLineReader +{ +public: + using Ptr = std::shared_ptr; + HDFSReader(ppc::storage::FileHandler::Ptr _handler, uint64_t _bufferSize = 500 * 1024 * 1024, + char _lineSpliter = '\n'); + ~HDFSReader() override = default; + + // the capacity of the file or memory-bytes + uint64_t capacity() const override { return m_length; } + ppc::protocol::DataResourceType type() const override + { + return ppc::protocol::DataResourceType::HDFS; + } + +protected: + bool allocateCurrentBlock() override; + bool readFinished() const override { return (m_offset == m_length); } + virtual bool tryToOpenNewFile(); + + inline bool loadDataAndCreateBufferParser() + { + auto buffer = std::make_shared(); + auto dataLen = std::min(m_bufferSize, (m_length - m_offset)); + buffer->resize(dataLen); + + hdfsFS fs = (hdfsFS)m_handler->storageHandler(); + // read data from the hdfs + uint64_t readedCount = 0; + int64_t readedBytes = 0; + // Note: when read-finished, hdfsRead return 0 + // the hdfsRead api only return 65536Bytes data at one time + do + { + readedBytes = hdfsRead(fs, m_currentFd->fs, buffer->data() + readedCount, dataLen); + readedCount += readedBytes; + } while (readedBytes != 0 && readedBytes != -1 && readedCount < dataLen); + // read error + if (readedBytes == -1) + { + BOOST_THROW_EXCEPTION( + HDFSReadDataFailed() << bcos::errinfo_comment( + "loadDataAndCreateBufferParser: read-hdfs-data failed, error: " + + std::string(hdfsGetLastError()))); + } + // the file has been read finished + if (readedCount > 0) + { + IO_LOG(TRACE) << LOG_DESC("loadDataAndCreateBufferParser") + << LOG_KV("readedCount", readedCount); + m_readedBytes += readedCount; + buffer->resize(readedCount); + // seek to the pointer + hdfsSeek(fs, m_currentFd->fs, m_readedBytes); + // allocate the currentBlockParser + m_currentBlockParser = std::make_shared(std::move(*buffer)); + return true; + } + return false; + } + +private: + // the file handler + ppc::storage::FileHandler::Ptr m_handler; + // buffer size loaded per time + uint64_t m_bufferSize; + std::unique_ptr m_currentFd; + char*** m_hostsInfo; + uint64_t m_readedBytes = 0; + int m_currentBlock = 0; + int m_currentBlockOffset = -1; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/HDFSWriter.cpp b/cpp/ppc-io/src/HDFSWriter.cpp new file mode 100644 index 00000000..47dd6090 --- /dev/null +++ b/cpp/ppc-io/src/HDFSWriter.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSWriter.h + * @author: yujiechen + * @date 2022-12-1 + */ +#include "HDFSWriter.h" + +using namespace ppc::io; +using namespace ppc::storage; +using namespace bcos; + +HDFSWriter::HDFSWriter(FileHandler::Ptr _handler, bool _trunc, int64_t _cacheSize) + : m_handler(std::move(_handler)), m_cacheSize(_cacheSize) +{ + IO_LOG(INFO) << LOG_DESC("HDFSWriter") << LOG_KV("path", m_handler->path()); + int flag = O_WRONLY; + if (!_trunc) + { + flag |= O_APPEND; + } + auto rawFd = + hdfsOpenFile((hdfsFS)m_handler->storageHandler(), m_handler->path().c_str(), flag, 0, 0, 0); + if (!rawFd) + { + BOOST_THROW_EXCEPTION(OpenFileFailed() << errinfo_comment( + "HDFSWriter: open file " + m_handler->path() + " failed!")); + } + + m_fd.reset(new HdfsFileWrapper(m_handler, rawFd)); +} +// write the data into the writer(Note: only support row-data) +bool HDFSWriter::writeLine(DataBatch::ConstPtr _data, DataSchema _schema, std::string _lineSplitter) +{ + switch (_schema) + { + case DataSchema::String: + { + writeData(_data, _lineSplitter); + return true; + } + case DataSchema::Bytes: + { + writeData(_data, _lineSplitter); + return true; + } + default: + { + BOOST_THROW_EXCEPTION(UnsupportedDataSchema() << bcos::errinfo_comment( + "Only support String and Bytes schema now!")); + } + } + return false; +} + +void HDFSWriter::writeBytes(bcos::bytesConstRef _data) +{ + auto ret = hdfsWrite((hdfsFS)m_handler->storageHandler(), m_fd->fs, _data.data(), _data.size()); + if (ret == -1) + { + BOOST_THROW_EXCEPTION(HDFSWriteDataFailed() << bcos::errinfo_comment( + "writeBytes failed, error: " + std::string(hdfsGetLastError()))); + } +} + +// flush the data into the storage-backend +// Note: we should not calls flush multiple times +void HDFSWriter::flush() +{ + if (hdfsFlush((hdfsFS)m_handler->storageHandler(), m_fd->fs) == -1) + { + BOOST_THROW_EXCEPTION(HDFSFlushFailed() << bcos::errinfo_comment( + "flush failed, error: " + std::string(hdfsGetLastError()))); + } +} +// close the storage-backend resource(Note: flush before close) +void HDFSWriter::close() +{ + m_fd.reset(); +} \ No newline at end of file diff --git a/cpp/ppc-io/src/HDFSWriter.h b/cpp/ppc-io/src/HDFSWriter.h new file mode 100644 index 00000000..3c97b9da --- /dev/null +++ b/cpp/ppc-io/src/HDFSWriter.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSWriter.h + * @author: yujiechen + * @date 2022-12-1 + */ +#pragma once +#include "Common.h" +#include "ppc-framework/io/LineWriter.h" +#include "ppc-framework/storage/FileStorage.h" + +namespace ppc::io +{ +class HDFSWriter : public LineWriter +{ +public: + using Ptr = std::shared_ptr; + HDFSWriter(ppc::storage::FileHandler::Ptr _handler, bool _trunc, + int64_t _cacheSize = 100 * 1024 * 1024); + ~HDFSWriter() override { close(); } + + ppc::protocol::DataResourceType type() const override + { + return ppc::protocol::DataResourceType::HDFS; + } + + // write the data into the writer(Note: only support row-data) + bool writeLine( + DataBatch::ConstPtr _data, DataSchema _schema, std::string _lineSplitter = "\n") override; + + void writeBytes(bcos::bytesConstRef _data) override; + + // flush the data into the storage-backend + void flush() override; + // close the storage-backend resource(Note: flush before close) + void close() override; + +protected: + void flushCache(bool _enforce) + { + if (m_cache.size() == 0) + { + return; + } + if (!_enforce && m_cache.size() < m_cacheSize) + { + return; + } + // flush cache into the hdfs, clear the cache + auto ret = hdfsWrite( + (hdfsFS)m_handler->storageHandler(), m_fd->fs, m_cache.data(), m_cache.size()); + if (ret == -1) + { + BOOST_THROW_EXCEPTION( + HDFSWriteDataFailed() << bcos::errinfo_comment( + "write cache-data failed, error: " + std::string(hdfsGetLastError()))); + } + m_cache.clear(); + } + + template + void writeData(DataBatch::ConstPtr _data, std::string _lineSplitter) + { + for (uint64_t i = 0; i < _data->size(); i++) + { + if (m_cache.size() >= m_cacheSize) + { + flushCache(false); + } + auto& lineData = _data->get(i); + m_cache.insert(m_cache.end(), reinterpret_cast(lineData.data()), + reinterpret_cast(lineData.data() + lineData.size())); + if (_lineSplitter.empty()) + { + continue; + } + m_cache.insert(m_cache.end(), reinterpret_cast(_lineSplitter.data()), + reinterpret_cast(_lineSplitter.data() + _lineSplitter.size())); + } + flushCache(true); + } + +private: + ppc::storage::FileHandler::Ptr m_handler; + std::unique_ptr m_fd; + + int64_t m_cacheSize; + bcos::bytes m_cache; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/SQLResultReader.h b/cpp/ppc-io/src/SQLResultReader.h new file mode 100644 index 00000000..d2d9439d --- /dev/null +++ b/cpp/ppc-io/src/SQLResultReader.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SQLResultReader.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include "ppc-framework/io/LineReader.h" +#include "ppc-framework/storage/SQLStorage.h" +#include + +namespace ppc::io +{ +class SQLResultReader : public LineReader +{ +public: + using Ptr = std::shared_ptr; + SQLResultReader(ppc::storage::QueryResult::Ptr _result, bool _parseByColumn) + : m_result(std::move(_result)), m_parseByColumn(_parseByColumn) + {} + ~SQLResultReader() override {} + // get the next _offset line data + // Note: the _schema is not used by mysql-result-reader + DataBatch::Ptr next(int64_t _offset = 0, DataSchema _schema = DataSchema::Bytes) override + { + boost::ignore_unused(_schema); + if ((int64_t)m_result->data().size() <= _offset) + { + return nullptr; + } + return m_result->mutableData().at(_offset); + } + + // the capacity the query-result + uint64_t capacity() const override { return m_result->data().size(); } + uint64_t columnSize() const override + { + if (m_parseByColumn) + { + return m_result->data().size(); + } + return 1; + } + + ppc::protocol::DataResourceType type() const override + { + return ppc::protocol::DataResourceType::MySQL; + } + + bcos::bytes readBytes() override + { + throw std::runtime_error("SQLResultReader: unimplemented readBytes!"); + } + +private: + ppc::storage::QueryResult::Ptr m_result; + bool m_parseByColumn; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/parser/BaseBufferParser.h b/cpp/ppc-io/src/parser/BaseBufferParser.h new file mode 100644 index 00000000..de2093c0 --- /dev/null +++ b/cpp/ppc-io/src/parser/BaseBufferParser.h @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BaseBufferParser.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include +namespace ppc::io +{ +enum class ReadLineResult +{ + ReadFinish, + ReadFinishWithoutSpliter, + NotReadFinish, +}; +class BaseBufferParser +{ +public: + using Ptr = std::shared_ptr; + BaseBufferParser() = default; + BaseBufferParser(char* _address, uint64_t _length) : m_address(_address), m_length(_length) {} + virtual ~BaseBufferParser() = default; + + virtual void* address() const { return (void*)m_address; } + virtual uint64_t length() const { return m_length; } + + template + ReadLineResult readLine(int64_t& readBytes, T& _lineData, char _delim = '\n') + { + if (m_pointer >= m_length) + { + return ReadLineResult::ReadFinish; + } + auto orgPointer = m_pointer; + auto startAddr = m_address + m_pointer; + uint64_t copyLen = 0; + bool findDelim = false; + for (; m_pointer < m_length; m_pointer++) + { + while (m_pointer < m_length && (((char)*(m_address + m_pointer) == _delim) || + ((char)*(m_address + m_pointer) == c_ignoreDelim))) + { + // skip the delim when read-line next-round + m_pointer++; + findDelim = true; + } + if (findDelim) + { + break; + } + copyLen++; + } + readBytes = (uint64_t)(m_pointer - orgPointer); + _lineData.resize(copyLen); + memcpy(_lineData.data(), (char*)startAddr, copyLen); + // not read-finished + if (m_pointer < m_length) + { + return ReadLineResult::NotReadFinish; + } + if (findDelim) + { + return ReadLineResult::ReadFinish; + } + return ReadLineResult::ReadFinishWithoutSpliter; + } + + virtual void readAll(bcos::bytes& _data) + { + _data.insert(_data.end(), (char*)m_address, (char*)m_address + m_length); + m_pointer = m_length; + } + virtual uint64_t pointer() const { return m_pointer; } + +protected: + char* m_address = nullptr; + uint64_t m_length; + uint64_t m_pointer = 0; + + const char c_ignoreDelim = '\r'; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/parser/BufferParser.h b/cpp/ppc-io/src/parser/BufferParser.h new file mode 100644 index 00000000..ed4791fc --- /dev/null +++ b/cpp/ppc-io/src/parser/BufferParser.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BufferParser.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "BaseBufferParser.h" + +namespace ppc::io +{ +class BufferParser : public BaseBufferParser +{ +public: + using Ptr = std::shared_ptr; + BufferParser(std::vector&& _data) : BaseBufferParser(), m_data(std::move(_data)) + { + m_length = m_data.size(); + m_address = (char*)m_data.data(); + } + ~BufferParser() override = default; + +protected: + std::vector m_data; +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/src/parser/MmapParser.h b/cpp/ppc-io/src/parser/MmapParser.h new file mode 100644 index 00000000..90e8ed2b --- /dev/null +++ b/cpp/ppc-io/src/parser/MmapParser.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MmapParser.h + * @author: yujiechen + * @date 2022-10-20 + */ +#pragma once +#include "../Common.h" +#include "BaseBufferParser.h" +#include +#include + +namespace ppc::io +{ +class MmapParser : public BaseBufferParser +{ +public: + using Ptr = std::unique_ptr; + MmapParser(int _fd, uint64_t _offset, uint64_t _length, bool _readOnly = true) + : BaseBufferParser() + { + m_length = _length; + int prot = PROT_READ; + if (!_readOnly) + { + prot = PROT_READ | PROT_WRITE; + } + // Note: the mmap offset param must be a multiple of page-size(4096), otherwise, Invalid + // Argument error will be raised + m_address = (char*)mmap(nullptr, _length, prot, MAP_SHARED, _fd, _offset); + if (MAP_FAILED == m_address) + { + BOOST_THROW_EXCEPTION(MmapFileException() << bcos::errinfo_comment( + "mmap file exception, code: " + std::to_string(errno) + + ", error: " + strerror(errno))); + } + IO_LOG(INFO) << LOG_DESC("MmapFile success") << LOG_KV("offset", _offset) + << LOG_KV("length", _length) << LOG_KV("readOnly", _readOnly) + << ", address:" << std::hex << (uint64_t)m_address; + } + + ~MmapParser() override + { + if (!m_address) + { + return; + } + munmap(m_address, m_length); + } +}; +} // namespace ppc::io \ No newline at end of file diff --git a/cpp/ppc-io/tests/CMakeLists.txt b/cpp/ppc-io/tests/CMakeLists.txt new file mode 100644 index 00000000..993e8bc0 --- /dev/null +++ b/cpp/ppc-io/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-io) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${IO_TARGET} ${BOOST_UNIT_TEST} TBB::tbb) +add_test(NAME test-io WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) diff --git a/cpp/ppc-io/tests/TestLineReader.cpp b/cpp/ppc-io/tests/TestLineReader.cpp new file mode 100644 index 00000000..47c0a074 --- /dev/null +++ b/cpp/ppc-io/tests/TestLineReader.cpp @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestLineReader.cpp + * @author: yujiechen + * @date 2022-10-21 + */ +#include "ppc-io/src/Common.h" +#include "ppc-io/src/FileLineReader.h" +#include +#include + +using namespace ppc::io; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(lineReaderTest, TestPromptFixture) + +void testFileLineReaderFunc(FileLineReaderFactory::Ptr _factory, std::string const& _filePath, + char _lineSpliter, uint64_t _mmapGranularity, uint64_t _fileTotalLine, + std::vector const& _expectedData, uint64_t _batchSize) +{ + if (_mmapGranularity % getpagesize() != 0) + { + BOOST_CHECK_THROW(_factory->createLineReader(_filePath, _mmapGranularity, _lineSpliter), + InvalidMmapGranularity); + return; + } + auto lineReader = _factory->createLineReader(_filePath, _mmapGranularity, _lineSpliter); + // case1: read all data from the file + auto dataBatch = lineReader->next(-1); + BOOST_CHECK(dataBatch->size() == _fileTotalLine); + std::cout << "#### dataBatch size: " << dataBatch->size() << std::endl; + + // check the data + uint64_t offset = 0; + for (auto const& item : _expectedData) + { + BOOST_CHECK(item == dataBatch->get(offset)); + offset++; + } + // try to read again while the file has been read-finished + BOOST_CHECK(nullptr == lineReader->next(1)); + + // case2: read _batchSize everyTime + lineReader = _factory->createLineReader(_filePath, _mmapGranularity, _lineSpliter); + uint64_t readedLine = 0; + std::vector dataBatches; + while (true) + { + dataBatch = lineReader->next(_batchSize); + if (!dataBatch) + { + break; + } + readedLine += dataBatch->size(); + dataBatches.emplace_back(dataBatch); + } + BOOST_CHECK(readedLine == _expectedData.size()); + // check the dataBatch + offset = 0; + for (auto const& batch : dataBatches) + { + for (uint64_t i = 0; i < batch->size(); i++) + { + BOOST_CHECK(batch->get(i) == _expectedData[offset]); + offset++; + } + } +} +BOOST_AUTO_TEST_CASE(testDataBatch) +{ + auto dataBatch = std::make_shared(); + uint64_t dataSize = 1002; + std::vector stringData; + std::vector intData; + uint64_t expectedStringCapacity = 0; + for (uint64_t i = 0; i < dataSize; i++) + { + auto str = std::to_string(i); + expectedStringCapacity += str.size(); + stringData.emplace_back(str); + intData.emplace_back(i); + } + // setData and check capacity + auto tmpData = stringData; + dataBatch->setData(std::move(tmpData)); + std::cout << "#### capacityBytes" << dataBatch->capacityBytes() << std::endl; + std::cout << "### expectedStringCapacity: " << expectedStringCapacity << std::endl; + BOOST_CHECK(dataBatch->capacityBytes() == expectedStringCapacity); + // append + for (uint64_t i = 0; i < stringData.size(); i++) + { + BOOST_CHECK(dataBatch->get(i) == stringData.at(i)); + BOOST_CHECK(dataBatch->elementSize(i) == stringData.at(i).size()); + auto tmp = stringData.at(i); + dataBatch->append(std::move(tmp)); + } + BOOST_CHECK(dataBatch->capacityBytes() == 2 * expectedStringCapacity); + + // appendToLine + auto orgSize = dataBatch->elementSize(dataBatch->size() - 1); + auto data = stringData.at(0); + dataBatch->appendToLine(std::move(data)); + BOOST_CHECK(dataBatch->elementSize(dataBatch->size() - 1) == + (stringData.at(0).size() + orgSize)); + // test set + std::string resetData = "reset"; + auto tmp = resetData; + dataBatch->set(0, std::move(tmp)); + BOOST_CHECK(resetData == dataBatch->get(0)); + + auto intDataBatch = std::make_shared(); + auto tmpIntData = intData; + intDataBatch->setData(std::move(tmpIntData)); + BOOST_CHECK(intDataBatch->capacityBytes() == dataSize * sizeof(uint64_t)); + for (uint64_t i = 0; i < intData.size(); i++) + { + BOOST_CHECK(intDataBatch->get(i) == intData.at(i)); + BOOST_CHECK(intDataBatch->elementSize(i) == sizeof(intData.at(i))); + auto tmp = intData.at(i); + intDataBatch->append(std::move(tmp)); + } + BOOST_CHECK(intDataBatch->capacityBytes() == 2 * dataSize * sizeof(uint64_t)); + + // the data-schema not set + BOOST_CHECK_THROW(intDataBatch->getBytes(0), DataSchemaNotSetted); + intDataBatch->setDataSchema(DataSchema::Uint); + + auto expectedData = boost::lexical_cast(intData.at(0)); + BOOST_CHECK(intDataBatch->getBytes(0) == bcos::bytes(expectedData.begin(), expectedData.end())); +} +BOOST_AUTO_TEST_CASE(testFileLineReader) +{ + std::string filePath = "../../../ppc-io/tests/data/testData"; + uint64_t fileTotalLine = 10000; + std::vector fileData; + for (uint64_t i = 1; i <= fileTotalLine; i++) + { + fileData.emplace_back(std::to_string(i)); + } + auto factory = std::make_shared(); + // case1: invalid mmap-granularity + testFileLineReaderFunc(factory, filePath, '\n', 1024, fileTotalLine, fileData, 100); + + // case2: valid mmap-granularity + auto pageSize = getpagesize(); + auto startT = utcSteadyTime(); + testFileLineReaderFunc(factory, filePath, '\n', pageSize, fileTotalLine, fileData, 100); + std::cout << "### timecost with 1-page mmap-granularity:" << (utcSteadyTime() - startT) << " ms" + << ", pageSize:" << pageSize << std::endl; + + // case3: large mmap-granularity + startT = bcos::utcSteadyTime(); + testFileLineReaderFunc( + factory, filePath, '\n', 500 * 1024 * pageSize, fileTotalLine, fileData, 100); + std::cout << "### timecost with 500K-page mmap-granularity:" << (utcSteadyTime() - startT) + << " ms" + << ", pageSize:" << pageSize << std::endl; + + // case4: the windows file + auto windowsFilePath = "../../../ppc-io/tests/data/windows_file.txt"; + auto lineReader = factory->createLineReader(windowsFilePath, pageSize, '\n'); + auto windowsDataBatch = lineReader->next(-1, DataSchema::Bytes); + + auto linuxFilePath = "../../../ppc-io/tests/data/linux_file.txt"; + lineReader = factory->createLineReader(linuxFilePath, pageSize, '\n'); + auto linuxDataBatch = lineReader->next(-1, DataSchema::Bytes); + BOOST_CHECK(windowsDataBatch->size() == linuxDataBatch->size()); + for (uint64_t i = 0; i < windowsDataBatch->size(); i++) + { + BOOST_CHECK(windowsDataBatch->get(i) == linuxDataBatch->get(i)); + } +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-io/tests/TestLineWriter.cpp b/cpp/ppc-io/tests/TestLineWriter.cpp new file mode 100644 index 00000000..e33fda2e --- /dev/null +++ b/cpp/ppc-io/tests/TestLineWriter.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestLineReader.cpp + * @author: yujiechen + * @date 2022-10-21 + */ + +#include "ppc-io/src/Common.h" +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include +#include + +using namespace ppc::io; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(lineWriterTest, TestPromptFixture) +BOOST_AUTO_TEST_CASE(testFileLineWriter) +{ + auto dataBatch = std::make_shared(); + for (uint64_t i = 0; i < 10000; i++) + { + dataBatch->append(std::to_string(i)); + } + std::string fileName = "testWriter"; + auto writer = std::make_shared(fileName, true); + writer->writeLine(dataBatch, DataSchema::String, "\n"); + writer->close(); + // check the content use FileLineReader + auto reader = std::make_shared(fileName); + auto readedData = reader->next(-1); + BOOST_CHECK(readedData->size() == dataBatch->size()); + for (uint64_t i = 0; i < readedData->size(); i++) + { + BOOST_CHECK(readedData->get(i) == dataBatch->get(i)); + } +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-io/tests/data/linux_file.txt b/cpp/ppc-io/tests/data/linux_file.txt new file mode 100644 index 00000000..f00c965d --- /dev/null +++ b/cpp/ppc-io/tests/data/linux_file.txt @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 diff --git a/cpp/ppc-io/tests/data/testData b/cpp/ppc-io/tests/data/testData new file mode 100644 index 00000000..5bcdcf0d --- /dev/null +++ b/cpp/ppc-io/tests/data/testData @@ -0,0 +1,10002 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 + + +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 +1001 +1002 +1003 +1004 +1005 +1006 +1007 +1008 +1009 +1010 +1011 +1012 +1013 +1014 +1015 +1016 +1017 +1018 +1019 +1020 +1021 +1022 +1023 +1024 +1025 +1026 +1027 +1028 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1037 +1038 +1039 +1040 +1041 +1042 +1043 +1044 +1045 +1046 +1047 +1048 +1049 +1050 +1051 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1059 +1060 +1061 +1062 +1063 +1064 +1065 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1075 +1076 +1077 +1078 +1079 +1080 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1091 +1092 +1093 +1094 +1095 +1096 +1097 +1098 +1099 +1100 +1101 +1102 +1103 +1104 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1116 +1117 +1118 +1119 +1120 +1121 +1122 +1123 +1124 +1125 +1126 +1127 +1128 +1129 +1130 +1131 +1132 +1133 +1134 +1135 +1136 +1137 +1138 +1139 +1140 +1141 +1142 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1153 +1154 +1155 +1156 +1157 +1158 +1159 +1160 +1161 +1162 +1163 +1164 +1165 +1166 +1167 +1168 +1169 +1170 +1171 +1172 +1173 +1174 +1175 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1187 +1188 +1189 +1190 +1191 +1192 +1193 +1194 +1195 +1196 +1197 +1198 +1199 +1200 +1201 +1202 +1203 +1204 +1205 +1206 +1207 +1208 +1209 +1210 +1211 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1221 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1229 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1240 +1241 +1242 +1243 +1244 +1245 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1254 +1255 +1256 +1257 +1258 +1259 +1260 +1261 +1262 +1263 +1264 +1265 +1266 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1275 +1276 +1277 +1278 +1279 +1280 +1281 +1282 +1283 +1284 +1285 +1286 +1287 +1288 +1289 +1290 +1291 +1292 +1293 +1294 +1295 +1296 +1297 +1298 +1299 +1300 +1301 +1302 +1303 +1304 +1305 +1306 +1307 +1308 +1309 +1310 +1311 +1312 +1313 +1314 +1315 +1316 +1317 +1318 +1319 +1320 +1321 +1322 +1323 +1324 +1325 +1326 +1327 +1328 +1329 +1330 +1331 +1332 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 +1341 +1342 +1343 +1344 +1345 +1346 +1347 +1348 +1349 +1350 +1351 +1352 +1353 +1354 +1355 +1356 +1357 +1358 +1359 +1360 +1361 +1362 +1363 +1364 +1365 +1366 +1367 +1368 +1369 +1370 +1371 +1372 +1373 +1374 +1375 +1376 +1377 +1378 +1379 +1380 +1381 +1382 +1383 +1384 +1385 +1386 +1387 +1388 +1389 +1390 +1391 +1392 +1393 +1394 +1395 +1396 +1397 +1398 +1399 +1400 +1401 +1402 +1403 +1404 +1405 +1406 +1407 +1408 +1409 +1410 +1411 +1412 +1413 +1414 +1415 +1416 +1417 +1418 +1419 +1420 +1421 +1422 +1423 +1424 +1425 +1426 +1427 +1428 +1429 +1430 +1431 +1432 +1433 +1434 +1435 +1436 +1437 +1438 +1439 +1440 +1441 +1442 +1443 +1444 +1445 +1446 +1447 +1448 +1449 +1450 +1451 +1452 +1453 +1454 +1455 +1456 +1457 +1458 +1459 +1460 +1461 +1462 +1463 +1464 +1465 +1466 +1467 +1468 +1469 +1470 +1471 +1472 +1473 +1474 +1475 +1476 +1477 +1478 +1479 +1480 +1481 +1482 +1483 +1484 +1485 +1486 +1487 +1488 +1489 +1490 +1491 +1492 +1493 +1494 +1495 +1496 +1497 +1498 +1499 +1500 +1501 +1502 +1503 +1504 +1505 +1506 +1507 +1508 +1509 +1510 +1511 +1512 +1513 +1514 +1515 +1516 +1517 +1518 +1519 +1520 +1521 +1522 +1523 +1524 +1525 +1526 +1527 +1528 +1529 +1530 +1531 +1532 +1533 +1534 +1535 +1536 +1537 +1538 +1539 +1540 +1541 +1542 +1543 +1544 +1545 +1546 +1547 +1548 +1549 +1550 +1551 +1552 +1553 +1554 +1555 +1556 +1557 +1558 +1559 +1560 +1561 +1562 +1563 +1564 +1565 +1566 +1567 +1568 +1569 +1570 +1571 +1572 +1573 +1574 +1575 +1576 +1577 +1578 +1579 +1580 +1581 +1582 +1583 +1584 +1585 +1586 +1587 +1588 +1589 +1590 +1591 +1592 +1593 +1594 +1595 +1596 +1597 +1598 +1599 +1600 +1601 +1602 +1603 +1604 +1605 +1606 +1607 +1608 +1609 +1610 +1611 +1612 +1613 +1614 +1615 +1616 +1617 +1618 +1619 +1620 +1621 +1622 +1623 +1624 +1625 +1626 +1627 +1628 +1629 +1630 +1631 +1632 +1633 +1634 +1635 +1636 +1637 +1638 +1639 +1640 +1641 +1642 +1643 +1644 +1645 +1646 +1647 +1648 +1649 +1650 +1651 +1652 +1653 +1654 +1655 +1656 +1657 +1658 +1659 +1660 +1661 +1662 +1663 +1664 +1665 +1666 +1667 +1668 +1669 +1670 +1671 +1672 +1673 +1674 +1675 +1676 +1677 +1678 +1679 +1680 +1681 +1682 +1683 +1684 +1685 +1686 +1687 +1688 +1689 +1690 +1691 +1692 +1693 +1694 +1695 +1696 +1697 +1698 +1699 +1700 +1701 +1702 +1703 +1704 +1705 +1706 +1707 +1708 +1709 +1710 +1711 +1712 +1713 +1714 +1715 +1716 +1717 +1718 +1719 +1720 +1721 +1722 +1723 +1724 +1725 +1726 +1727 +1728 +1729 +1730 +1731 +1732 +1733 +1734 +1735 +1736 +1737 +1738 +1739 +1740 +1741 +1742 +1743 +1744 +1745 +1746 +1747 +1748 +1749 +1750 +1751 +1752 +1753 +1754 +1755 +1756 +1757 +1758 +1759 +1760 +1761 +1762 +1763 +1764 +1765 +1766 +1767 +1768 +1769 +1770 +1771 +1772 +1773 +1774 +1775 +1776 +1777 +1778 +1779 +1780 +1781 +1782 +1783 +1784 +1785 +1786 +1787 +1788 +1789 +1790 +1791 +1792 +1793 +1794 +1795 +1796 +1797 +1798 +1799 +1800 +1801 +1802 +1803 +1804 +1805 +1806 +1807 +1808 +1809 +1810 +1811 +1812 +1813 +1814 +1815 +1816 +1817 +1818 +1819 +1820 +1821 +1822 +1823 +1824 +1825 +1826 +1827 +1828 +1829 +1830 +1831 +1832 +1833 +1834 +1835 +1836 +1837 +1838 +1839 +1840 +1841 +1842 +1843 +1844 +1845 +1846 +1847 +1848 +1849 +1850 +1851 +1852 +1853 +1854 +1855 +1856 +1857 +1858 +1859 +1860 +1861 +1862 +1863 +1864 +1865 +1866 +1867 +1868 +1869 +1870 +1871 +1872 +1873 +1874 +1875 +1876 +1877 +1878 +1879 +1880 +1881 +1882 +1883 +1884 +1885 +1886 +1887 +1888 +1889 +1890 +1891 +1892 +1893 +1894 +1895 +1896 +1897 +1898 +1899 +1900 +1901 +1902 +1903 +1904 +1905 +1906 +1907 +1908 +1909 +1910 +1911 +1912 +1913 +1914 +1915 +1916 +1917 +1918 +1919 +1920 +1921 +1922 +1923 +1924 +1925 +1926 +1927 +1928 +1929 +1930 +1931 +1932 +1933 +1934 +1935 +1936 +1937 +1938 +1939 +1940 +1941 +1942 +1943 +1944 +1945 +1946 +1947 +1948 +1949 +1950 +1951 +1952 +1953 +1954 +1955 +1956 +1957 +1958 +1959 +1960 +1961 +1962 +1963 +1964 +1965 +1966 +1967 +1968 +1969 +1970 +1971 +1972 +1973 +1974 +1975 +1976 +1977 +1978 +1979 +1980 +1981 +1982 +1983 +1984 +1985 +1986 +1987 +1988 +1989 +1990 +1991 +1992 +1993 +1994 +1995 +1996 +1997 +1998 +1999 +2000 +2001 +2002 +2003 +2004 +2005 +2006 +2007 +2008 +2009 +2010 +2011 +2012 +2013 +2014 +2015 +2016 +2017 +2018 +2019 +2020 +2021 +2022 +2023 +2024 +2025 +2026 +2027 +2028 +2029 +2030 +2031 +2032 +2033 +2034 +2035 +2036 +2037 +2038 +2039 +2040 +2041 +2042 +2043 +2044 +2045 +2046 +2047 +2048 +2049 +2050 +2051 +2052 +2053 +2054 +2055 +2056 +2057 +2058 +2059 +2060 +2061 +2062 +2063 +2064 +2065 +2066 +2067 +2068 +2069 +2070 +2071 +2072 +2073 +2074 +2075 +2076 +2077 +2078 +2079 +2080 +2081 +2082 +2083 +2084 +2085 +2086 +2087 +2088 +2089 +2090 +2091 +2092 +2093 +2094 +2095 +2096 +2097 +2098 +2099 +2100 +2101 +2102 +2103 +2104 +2105 +2106 +2107 +2108 +2109 +2110 +2111 +2112 +2113 +2114 +2115 +2116 +2117 +2118 +2119 +2120 +2121 +2122 +2123 +2124 +2125 +2126 +2127 +2128 +2129 +2130 +2131 +2132 +2133 +2134 +2135 +2136 +2137 +2138 +2139 +2140 +2141 +2142 +2143 +2144 +2145 +2146 +2147 +2148 +2149 +2150 +2151 +2152 +2153 +2154 +2155 +2156 +2157 +2158 +2159 +2160 +2161 +2162 +2163 +2164 +2165 +2166 +2167 +2168 +2169 +2170 +2171 +2172 +2173 +2174 +2175 +2176 +2177 +2178 +2179 +2180 +2181 +2182 +2183 +2184 +2185 +2186 +2187 +2188 +2189 +2190 +2191 +2192 +2193 +2194 +2195 +2196 +2197 +2198 +2199 +2200 +2201 +2202 +2203 +2204 +2205 +2206 +2207 +2208 +2209 +2210 +2211 +2212 +2213 +2214 +2215 +2216 +2217 +2218 +2219 +2220 +2221 +2222 +2223 +2224 +2225 +2226 +2227 +2228 +2229 +2230 +2231 +2232 +2233 +2234 +2235 +2236 +2237 +2238 +2239 +2240 +2241 +2242 +2243 +2244 +2245 +2246 +2247 +2248 +2249 +2250 +2251 +2252 +2253 +2254 +2255 +2256 +2257 +2258 +2259 +2260 +2261 +2262 +2263 +2264 +2265 +2266 +2267 +2268 +2269 +2270 +2271 +2272 +2273 +2274 +2275 +2276 +2277 +2278 +2279 +2280 +2281 +2282 +2283 +2284 +2285 +2286 +2287 +2288 +2289 +2290 +2291 +2292 +2293 +2294 +2295 +2296 +2297 +2298 +2299 +2300 +2301 +2302 +2303 +2304 +2305 +2306 +2307 +2308 +2309 +2310 +2311 +2312 +2313 +2314 +2315 +2316 +2317 +2318 +2319 +2320 +2321 +2322 +2323 +2324 +2325 +2326 +2327 +2328 +2329 +2330 +2331 +2332 +2333 +2334 +2335 +2336 +2337 +2338 +2339 +2340 +2341 +2342 +2343 +2344 +2345 +2346 +2347 +2348 +2349 +2350 +2351 +2352 +2353 +2354 +2355 +2356 +2357 +2358 +2359 +2360 +2361 +2362 +2363 +2364 +2365 +2366 +2367 +2368 +2369 +2370 +2371 +2372 +2373 +2374 +2375 +2376 +2377 +2378 +2379 +2380 +2381 +2382 +2383 +2384 +2385 +2386 +2387 +2388 +2389 +2390 +2391 +2392 +2393 +2394 +2395 +2396 +2397 +2398 +2399 +2400 +2401 +2402 +2403 +2404 +2405 +2406 +2407 +2408 +2409 +2410 +2411 +2412 +2413 +2414 +2415 +2416 +2417 +2418 +2419 +2420 +2421 +2422 +2423 +2424 +2425 +2426 +2427 +2428 +2429 +2430 +2431 +2432 +2433 +2434 +2435 +2436 +2437 +2438 +2439 +2440 +2441 +2442 +2443 +2444 +2445 +2446 +2447 +2448 +2449 +2450 +2451 +2452 +2453 +2454 +2455 +2456 +2457 +2458 +2459 +2460 +2461 +2462 +2463 +2464 +2465 +2466 +2467 +2468 +2469 +2470 +2471 +2472 +2473 +2474 +2475 +2476 +2477 +2478 +2479 +2480 +2481 +2482 +2483 +2484 +2485 +2486 +2487 +2488 +2489 +2490 +2491 +2492 +2493 +2494 +2495 +2496 +2497 +2498 +2499 +2500 +2501 +2502 +2503 +2504 +2505 +2506 +2507 +2508 +2509 +2510 +2511 +2512 +2513 +2514 +2515 +2516 +2517 +2518 +2519 +2520 +2521 +2522 +2523 +2524 +2525 +2526 +2527 +2528 +2529 +2530 +2531 +2532 +2533 +2534 +2535 +2536 +2537 +2538 +2539 +2540 +2541 +2542 +2543 +2544 +2545 +2546 +2547 +2548 +2549 +2550 +2551 +2552 +2553 +2554 +2555 +2556 +2557 +2558 +2559 +2560 +2561 +2562 +2563 +2564 +2565 +2566 +2567 +2568 +2569 +2570 +2571 +2572 +2573 +2574 +2575 +2576 +2577 +2578 +2579 +2580 +2581 +2582 +2583 +2584 +2585 +2586 +2587 +2588 +2589 +2590 +2591 +2592 +2593 +2594 +2595 +2596 +2597 +2598 +2599 +2600 +2601 +2602 +2603 +2604 +2605 +2606 +2607 +2608 +2609 +2610 +2611 +2612 +2613 +2614 +2615 +2616 +2617 +2618 +2619 +2620 +2621 +2622 +2623 +2624 +2625 +2626 +2627 +2628 +2629 +2630 +2631 +2632 +2633 +2634 +2635 +2636 +2637 +2638 +2639 +2640 +2641 +2642 +2643 +2644 +2645 +2646 +2647 +2648 +2649 +2650 +2651 +2652 +2653 +2654 +2655 +2656 +2657 +2658 +2659 +2660 +2661 +2662 +2663 +2664 +2665 +2666 +2667 +2668 +2669 +2670 +2671 +2672 +2673 +2674 +2675 +2676 +2677 +2678 +2679 +2680 +2681 +2682 +2683 +2684 +2685 +2686 +2687 +2688 +2689 +2690 +2691 +2692 +2693 +2694 +2695 +2696 +2697 +2698 +2699 +2700 +2701 +2702 +2703 +2704 +2705 +2706 +2707 +2708 +2709 +2710 +2711 +2712 +2713 +2714 +2715 +2716 +2717 +2718 +2719 +2720 +2721 +2722 +2723 +2724 +2725 +2726 +2727 +2728 +2729 +2730 +2731 +2732 +2733 +2734 +2735 +2736 +2737 +2738 +2739 +2740 +2741 +2742 +2743 +2744 +2745 +2746 +2747 +2748 +2749 +2750 +2751 +2752 +2753 +2754 +2755 +2756 +2757 +2758 +2759 +2760 +2761 +2762 +2763 +2764 +2765 +2766 +2767 +2768 +2769 +2770 +2771 +2772 +2773 +2774 +2775 +2776 +2777 +2778 +2779 +2780 +2781 +2782 +2783 +2784 +2785 +2786 +2787 +2788 +2789 +2790 +2791 +2792 +2793 +2794 +2795 +2796 +2797 +2798 +2799 +2800 +2801 +2802 +2803 +2804 +2805 +2806 +2807 +2808 +2809 +2810 +2811 +2812 +2813 +2814 +2815 +2816 +2817 +2818 +2819 +2820 +2821 +2822 +2823 +2824 +2825 +2826 +2827 +2828 +2829 +2830 +2831 +2832 +2833 +2834 +2835 +2836 +2837 +2838 +2839 +2840 +2841 +2842 +2843 +2844 +2845 +2846 +2847 +2848 +2849 +2850 +2851 +2852 +2853 +2854 +2855 +2856 +2857 +2858 +2859 +2860 +2861 +2862 +2863 +2864 +2865 +2866 +2867 +2868 +2869 +2870 +2871 +2872 +2873 +2874 +2875 +2876 +2877 +2878 +2879 +2880 +2881 +2882 +2883 +2884 +2885 +2886 +2887 +2888 +2889 +2890 +2891 +2892 +2893 +2894 +2895 +2896 +2897 +2898 +2899 +2900 +2901 +2902 +2903 +2904 +2905 +2906 +2907 +2908 +2909 +2910 +2911 +2912 +2913 +2914 +2915 +2916 +2917 +2918 +2919 +2920 +2921 +2922 +2923 +2924 +2925 +2926 +2927 +2928 +2929 +2930 +2931 +2932 +2933 +2934 +2935 +2936 +2937 +2938 +2939 +2940 +2941 +2942 +2943 +2944 +2945 +2946 +2947 +2948 +2949 +2950 +2951 +2952 +2953 +2954 +2955 +2956 +2957 +2958 +2959 +2960 +2961 +2962 +2963 +2964 +2965 +2966 +2967 +2968 +2969 +2970 +2971 +2972 +2973 +2974 +2975 +2976 +2977 +2978 +2979 +2980 +2981 +2982 +2983 +2984 +2985 +2986 +2987 +2988 +2989 +2990 +2991 +2992 +2993 +2994 +2995 +2996 +2997 +2998 +2999 +3000 +3001 +3002 +3003 +3004 +3005 +3006 +3007 +3008 +3009 +3010 +3011 +3012 +3013 +3014 +3015 +3016 +3017 +3018 +3019 +3020 +3021 +3022 +3023 +3024 +3025 +3026 +3027 +3028 +3029 +3030 +3031 +3032 +3033 +3034 +3035 +3036 +3037 +3038 +3039 +3040 +3041 +3042 +3043 +3044 +3045 +3046 +3047 +3048 +3049 +3050 +3051 +3052 +3053 +3054 +3055 +3056 +3057 +3058 +3059 +3060 +3061 +3062 +3063 +3064 +3065 +3066 +3067 +3068 +3069 +3070 +3071 +3072 +3073 +3074 +3075 +3076 +3077 +3078 +3079 +3080 +3081 +3082 +3083 +3084 +3085 +3086 +3087 +3088 +3089 +3090 +3091 +3092 +3093 +3094 +3095 +3096 +3097 +3098 +3099 +3100 +3101 +3102 +3103 +3104 +3105 +3106 +3107 +3108 +3109 +3110 +3111 +3112 +3113 +3114 +3115 +3116 +3117 +3118 +3119 +3120 +3121 +3122 +3123 +3124 +3125 +3126 +3127 +3128 +3129 +3130 +3131 +3132 +3133 +3134 +3135 +3136 +3137 +3138 +3139 +3140 +3141 +3142 +3143 +3144 +3145 +3146 +3147 +3148 +3149 +3150 +3151 +3152 +3153 +3154 +3155 +3156 +3157 +3158 +3159 +3160 +3161 +3162 +3163 +3164 +3165 +3166 +3167 +3168 +3169 +3170 +3171 +3172 +3173 +3174 +3175 +3176 +3177 +3178 +3179 +3180 +3181 +3182 +3183 +3184 +3185 +3186 +3187 +3188 +3189 +3190 +3191 +3192 +3193 +3194 +3195 +3196 +3197 +3198 +3199 +3200 +3201 +3202 +3203 +3204 +3205 +3206 +3207 +3208 +3209 +3210 +3211 +3212 +3213 +3214 +3215 +3216 +3217 +3218 +3219 +3220 +3221 +3222 +3223 +3224 +3225 +3226 +3227 +3228 +3229 +3230 +3231 +3232 +3233 +3234 +3235 +3236 +3237 +3238 +3239 +3240 +3241 +3242 +3243 +3244 +3245 +3246 +3247 +3248 +3249 +3250 +3251 +3252 +3253 +3254 +3255 +3256 +3257 +3258 +3259 +3260 +3261 +3262 +3263 +3264 +3265 +3266 +3267 +3268 +3269 +3270 +3271 +3272 +3273 +3274 +3275 +3276 +3277 +3278 +3279 +3280 +3281 +3282 +3283 +3284 +3285 +3286 +3287 +3288 +3289 +3290 +3291 +3292 +3293 +3294 +3295 +3296 +3297 +3298 +3299 +3300 +3301 +3302 +3303 +3304 +3305 +3306 +3307 +3308 +3309 +3310 +3311 +3312 +3313 +3314 +3315 +3316 +3317 +3318 +3319 +3320 +3321 +3322 +3323 +3324 +3325 +3326 +3327 +3328 +3329 +3330 +3331 +3332 +3333 +3334 +3335 +3336 +3337 +3338 +3339 +3340 +3341 +3342 +3343 +3344 +3345 +3346 +3347 +3348 +3349 +3350 +3351 +3352 +3353 +3354 +3355 +3356 +3357 +3358 +3359 +3360 +3361 +3362 +3363 +3364 +3365 +3366 +3367 +3368 +3369 +3370 +3371 +3372 +3373 +3374 +3375 +3376 +3377 +3378 +3379 +3380 +3381 +3382 +3383 +3384 +3385 +3386 +3387 +3388 +3389 +3390 +3391 +3392 +3393 +3394 +3395 +3396 +3397 +3398 +3399 +3400 +3401 +3402 +3403 +3404 +3405 +3406 +3407 +3408 +3409 +3410 +3411 +3412 +3413 +3414 +3415 +3416 +3417 +3418 +3419 +3420 +3421 +3422 +3423 +3424 +3425 +3426 +3427 +3428 +3429 +3430 +3431 +3432 +3433 +3434 +3435 +3436 +3437 +3438 +3439 +3440 +3441 +3442 +3443 +3444 +3445 +3446 +3447 +3448 +3449 +3450 +3451 +3452 +3453 +3454 +3455 +3456 +3457 +3458 +3459 +3460 +3461 +3462 +3463 +3464 +3465 +3466 +3467 +3468 +3469 +3470 +3471 +3472 +3473 +3474 +3475 +3476 +3477 +3478 +3479 +3480 +3481 +3482 +3483 +3484 +3485 +3486 +3487 +3488 +3489 +3490 +3491 +3492 +3493 +3494 +3495 +3496 +3497 +3498 +3499 +3500 +3501 +3502 +3503 +3504 +3505 +3506 +3507 +3508 +3509 +3510 +3511 +3512 +3513 +3514 +3515 +3516 +3517 +3518 +3519 +3520 +3521 +3522 +3523 +3524 +3525 +3526 +3527 +3528 +3529 +3530 +3531 +3532 +3533 +3534 +3535 +3536 +3537 +3538 +3539 +3540 +3541 +3542 +3543 +3544 +3545 +3546 +3547 +3548 +3549 +3550 +3551 +3552 +3553 +3554 +3555 +3556 +3557 +3558 +3559 +3560 +3561 +3562 +3563 +3564 +3565 +3566 +3567 +3568 +3569 +3570 +3571 +3572 +3573 +3574 +3575 +3576 +3577 +3578 +3579 +3580 +3581 +3582 +3583 +3584 +3585 +3586 +3587 +3588 +3589 +3590 +3591 +3592 +3593 +3594 +3595 +3596 +3597 +3598 +3599 +3600 +3601 +3602 +3603 +3604 +3605 +3606 +3607 +3608 +3609 +3610 +3611 +3612 +3613 +3614 +3615 +3616 +3617 +3618 +3619 +3620 +3621 +3622 +3623 +3624 +3625 +3626 +3627 +3628 +3629 +3630 +3631 +3632 +3633 +3634 +3635 +3636 +3637 +3638 +3639 +3640 +3641 +3642 +3643 +3644 +3645 +3646 +3647 +3648 +3649 +3650 +3651 +3652 +3653 +3654 +3655 +3656 +3657 +3658 +3659 +3660 +3661 +3662 +3663 +3664 +3665 +3666 +3667 +3668 +3669 +3670 +3671 +3672 +3673 +3674 +3675 +3676 +3677 +3678 +3679 +3680 +3681 +3682 +3683 +3684 +3685 +3686 +3687 +3688 +3689 +3690 +3691 +3692 +3693 +3694 +3695 +3696 +3697 +3698 +3699 +3700 +3701 +3702 +3703 +3704 +3705 +3706 +3707 +3708 +3709 +3710 +3711 +3712 +3713 +3714 +3715 +3716 +3717 +3718 +3719 +3720 +3721 +3722 +3723 +3724 +3725 +3726 +3727 +3728 +3729 +3730 +3731 +3732 +3733 +3734 +3735 +3736 +3737 +3738 +3739 +3740 +3741 +3742 +3743 +3744 +3745 +3746 +3747 +3748 +3749 +3750 +3751 +3752 +3753 +3754 +3755 +3756 +3757 +3758 +3759 +3760 +3761 +3762 +3763 +3764 +3765 +3766 +3767 +3768 +3769 +3770 +3771 +3772 +3773 +3774 +3775 +3776 +3777 +3778 +3779 +3780 +3781 +3782 +3783 +3784 +3785 +3786 +3787 +3788 +3789 +3790 +3791 +3792 +3793 +3794 +3795 +3796 +3797 +3798 +3799 +3800 +3801 +3802 +3803 +3804 +3805 +3806 +3807 +3808 +3809 +3810 +3811 +3812 +3813 +3814 +3815 +3816 +3817 +3818 +3819 +3820 +3821 +3822 +3823 +3824 +3825 +3826 +3827 +3828 +3829 +3830 +3831 +3832 +3833 +3834 +3835 +3836 +3837 +3838 +3839 +3840 +3841 +3842 +3843 +3844 +3845 +3846 +3847 +3848 +3849 +3850 +3851 +3852 +3853 +3854 +3855 +3856 +3857 +3858 +3859 +3860 +3861 +3862 +3863 +3864 +3865 +3866 +3867 +3868 +3869 +3870 +3871 +3872 +3873 +3874 +3875 +3876 +3877 +3878 +3879 +3880 +3881 +3882 +3883 +3884 +3885 +3886 +3887 +3888 +3889 +3890 +3891 +3892 +3893 +3894 +3895 +3896 +3897 +3898 +3899 +3900 +3901 +3902 +3903 +3904 +3905 +3906 +3907 +3908 +3909 +3910 +3911 +3912 +3913 +3914 +3915 +3916 +3917 +3918 +3919 +3920 +3921 +3922 +3923 +3924 +3925 +3926 +3927 +3928 +3929 +3930 +3931 +3932 +3933 +3934 +3935 +3936 +3937 +3938 +3939 +3940 +3941 +3942 +3943 +3944 +3945 +3946 +3947 +3948 +3949 +3950 +3951 +3952 +3953 +3954 +3955 +3956 +3957 +3958 +3959 +3960 +3961 +3962 +3963 +3964 +3965 +3966 +3967 +3968 +3969 +3970 +3971 +3972 +3973 +3974 +3975 +3976 +3977 +3978 +3979 +3980 +3981 +3982 +3983 +3984 +3985 +3986 +3987 +3988 +3989 +3990 +3991 +3992 +3993 +3994 +3995 +3996 +3997 +3998 +3999 +4000 +4001 +4002 +4003 +4004 +4005 +4006 +4007 +4008 +4009 +4010 +4011 +4012 +4013 +4014 +4015 +4016 +4017 +4018 +4019 +4020 +4021 +4022 +4023 +4024 +4025 +4026 +4027 +4028 +4029 +4030 +4031 +4032 +4033 +4034 +4035 +4036 +4037 +4038 +4039 +4040 +4041 +4042 +4043 +4044 +4045 +4046 +4047 +4048 +4049 +4050 +4051 +4052 +4053 +4054 +4055 +4056 +4057 +4058 +4059 +4060 +4061 +4062 +4063 +4064 +4065 +4066 +4067 +4068 +4069 +4070 +4071 +4072 +4073 +4074 +4075 +4076 +4077 +4078 +4079 +4080 +4081 +4082 +4083 +4084 +4085 +4086 +4087 +4088 +4089 +4090 +4091 +4092 +4093 +4094 +4095 +4096 +4097 +4098 +4099 +4100 +4101 +4102 +4103 +4104 +4105 +4106 +4107 +4108 +4109 +4110 +4111 +4112 +4113 +4114 +4115 +4116 +4117 +4118 +4119 +4120 +4121 +4122 +4123 +4124 +4125 +4126 +4127 +4128 +4129 +4130 +4131 +4132 +4133 +4134 +4135 +4136 +4137 +4138 +4139 +4140 +4141 +4142 +4143 +4144 +4145 +4146 +4147 +4148 +4149 +4150 +4151 +4152 +4153 +4154 +4155 +4156 +4157 +4158 +4159 +4160 +4161 +4162 +4163 +4164 +4165 +4166 +4167 +4168 +4169 +4170 +4171 +4172 +4173 +4174 +4175 +4176 +4177 +4178 +4179 +4180 +4181 +4182 +4183 +4184 +4185 +4186 +4187 +4188 +4189 +4190 +4191 +4192 +4193 +4194 +4195 +4196 +4197 +4198 +4199 +4200 +4201 +4202 +4203 +4204 +4205 +4206 +4207 +4208 +4209 +4210 +4211 +4212 +4213 +4214 +4215 +4216 +4217 +4218 +4219 +4220 +4221 +4222 +4223 +4224 +4225 +4226 +4227 +4228 +4229 +4230 +4231 +4232 +4233 +4234 +4235 +4236 +4237 +4238 +4239 +4240 +4241 +4242 +4243 +4244 +4245 +4246 +4247 +4248 +4249 +4250 +4251 +4252 +4253 +4254 +4255 +4256 +4257 +4258 +4259 +4260 +4261 +4262 +4263 +4264 +4265 +4266 +4267 +4268 +4269 +4270 +4271 +4272 +4273 +4274 +4275 +4276 +4277 +4278 +4279 +4280 +4281 +4282 +4283 +4284 +4285 +4286 +4287 +4288 +4289 +4290 +4291 +4292 +4293 +4294 +4295 +4296 +4297 +4298 +4299 +4300 +4301 +4302 +4303 +4304 +4305 +4306 +4307 +4308 +4309 +4310 +4311 +4312 +4313 +4314 +4315 +4316 +4317 +4318 +4319 +4320 +4321 +4322 +4323 +4324 +4325 +4326 +4327 +4328 +4329 +4330 +4331 +4332 +4333 +4334 +4335 +4336 +4337 +4338 +4339 +4340 +4341 +4342 +4343 +4344 +4345 +4346 +4347 +4348 +4349 +4350 +4351 +4352 +4353 +4354 +4355 +4356 +4357 +4358 +4359 +4360 +4361 +4362 +4363 +4364 +4365 +4366 +4367 +4368 +4369 +4370 +4371 +4372 +4373 +4374 +4375 +4376 +4377 +4378 +4379 +4380 +4381 +4382 +4383 +4384 +4385 +4386 +4387 +4388 +4389 +4390 +4391 +4392 +4393 +4394 +4395 +4396 +4397 +4398 +4399 +4400 +4401 +4402 +4403 +4404 +4405 +4406 +4407 +4408 +4409 +4410 +4411 +4412 +4413 +4414 +4415 +4416 +4417 +4418 +4419 +4420 +4421 +4422 +4423 +4424 +4425 +4426 +4427 +4428 +4429 +4430 +4431 +4432 +4433 +4434 +4435 +4436 +4437 +4438 +4439 +4440 +4441 +4442 +4443 +4444 +4445 +4446 +4447 +4448 +4449 +4450 +4451 +4452 +4453 +4454 +4455 +4456 +4457 +4458 +4459 +4460 +4461 +4462 +4463 +4464 +4465 +4466 +4467 +4468 +4469 +4470 +4471 +4472 +4473 +4474 +4475 +4476 +4477 +4478 +4479 +4480 +4481 +4482 +4483 +4484 +4485 +4486 +4487 +4488 +4489 +4490 +4491 +4492 +4493 +4494 +4495 +4496 +4497 +4498 +4499 +4500 +4501 +4502 +4503 +4504 +4505 +4506 +4507 +4508 +4509 +4510 +4511 +4512 +4513 +4514 +4515 +4516 +4517 +4518 +4519 +4520 +4521 +4522 +4523 +4524 +4525 +4526 +4527 +4528 +4529 +4530 +4531 +4532 +4533 +4534 +4535 +4536 +4537 +4538 +4539 +4540 +4541 +4542 +4543 +4544 +4545 +4546 +4547 +4548 +4549 +4550 +4551 +4552 +4553 +4554 +4555 +4556 +4557 +4558 +4559 +4560 +4561 +4562 +4563 +4564 +4565 +4566 +4567 +4568 +4569 +4570 +4571 +4572 +4573 +4574 +4575 +4576 +4577 +4578 +4579 +4580 +4581 +4582 +4583 +4584 +4585 +4586 +4587 +4588 +4589 +4590 +4591 +4592 +4593 +4594 +4595 +4596 +4597 +4598 +4599 +4600 +4601 +4602 +4603 +4604 +4605 +4606 +4607 +4608 +4609 +4610 +4611 +4612 +4613 +4614 +4615 +4616 +4617 +4618 +4619 +4620 +4621 +4622 +4623 +4624 +4625 +4626 +4627 +4628 +4629 +4630 +4631 +4632 +4633 +4634 +4635 +4636 +4637 +4638 +4639 +4640 +4641 +4642 +4643 +4644 +4645 +4646 +4647 +4648 +4649 +4650 +4651 +4652 +4653 +4654 +4655 +4656 +4657 +4658 +4659 +4660 +4661 +4662 +4663 +4664 +4665 +4666 +4667 +4668 +4669 +4670 +4671 +4672 +4673 +4674 +4675 +4676 +4677 +4678 +4679 +4680 +4681 +4682 +4683 +4684 +4685 +4686 +4687 +4688 +4689 +4690 +4691 +4692 +4693 +4694 +4695 +4696 +4697 +4698 +4699 +4700 +4701 +4702 +4703 +4704 +4705 +4706 +4707 +4708 +4709 +4710 +4711 +4712 +4713 +4714 +4715 +4716 +4717 +4718 +4719 +4720 +4721 +4722 +4723 +4724 +4725 +4726 +4727 +4728 +4729 +4730 +4731 +4732 +4733 +4734 +4735 +4736 +4737 +4738 +4739 +4740 +4741 +4742 +4743 +4744 +4745 +4746 +4747 +4748 +4749 +4750 +4751 +4752 +4753 +4754 +4755 +4756 +4757 +4758 +4759 +4760 +4761 +4762 +4763 +4764 +4765 +4766 +4767 +4768 +4769 +4770 +4771 +4772 +4773 +4774 +4775 +4776 +4777 +4778 +4779 +4780 +4781 +4782 +4783 +4784 +4785 +4786 +4787 +4788 +4789 +4790 +4791 +4792 +4793 +4794 +4795 +4796 +4797 +4798 +4799 +4800 +4801 +4802 +4803 +4804 +4805 +4806 +4807 +4808 +4809 +4810 +4811 +4812 +4813 +4814 +4815 +4816 +4817 +4818 +4819 +4820 +4821 +4822 +4823 +4824 +4825 +4826 +4827 +4828 +4829 +4830 +4831 +4832 +4833 +4834 +4835 +4836 +4837 +4838 +4839 +4840 +4841 +4842 +4843 +4844 +4845 +4846 +4847 +4848 +4849 +4850 +4851 +4852 +4853 +4854 +4855 +4856 +4857 +4858 +4859 +4860 +4861 +4862 +4863 +4864 +4865 +4866 +4867 +4868 +4869 +4870 +4871 +4872 +4873 +4874 +4875 +4876 +4877 +4878 +4879 +4880 +4881 +4882 +4883 +4884 +4885 +4886 +4887 +4888 +4889 +4890 +4891 +4892 +4893 +4894 +4895 +4896 +4897 +4898 +4899 +4900 +4901 +4902 +4903 +4904 +4905 +4906 +4907 +4908 +4909 +4910 +4911 +4912 +4913 +4914 +4915 +4916 +4917 +4918 +4919 +4920 +4921 +4922 +4923 +4924 +4925 +4926 +4927 +4928 +4929 +4930 +4931 +4932 +4933 +4934 +4935 +4936 +4937 +4938 +4939 +4940 +4941 +4942 +4943 +4944 +4945 +4946 +4947 +4948 +4949 +4950 +4951 +4952 +4953 +4954 +4955 +4956 +4957 +4958 +4959 +4960 +4961 +4962 +4963 +4964 +4965 +4966 +4967 +4968 +4969 +4970 +4971 +4972 +4973 +4974 +4975 +4976 +4977 +4978 +4979 +4980 +4981 +4982 +4983 +4984 +4985 +4986 +4987 +4988 +4989 +4990 +4991 +4992 +4993 +4994 +4995 +4996 +4997 +4998 +4999 +5000 +5001 +5002 +5003 +5004 +5005 +5006 +5007 +5008 +5009 +5010 +5011 +5012 +5013 +5014 +5015 +5016 +5017 +5018 +5019 +5020 +5021 +5022 +5023 +5024 +5025 +5026 +5027 +5028 +5029 +5030 +5031 +5032 +5033 +5034 +5035 +5036 +5037 +5038 +5039 +5040 +5041 +5042 +5043 +5044 +5045 +5046 +5047 +5048 +5049 +5050 +5051 +5052 +5053 +5054 +5055 +5056 +5057 +5058 +5059 +5060 +5061 +5062 +5063 +5064 +5065 +5066 +5067 +5068 +5069 +5070 +5071 +5072 +5073 +5074 +5075 +5076 +5077 +5078 +5079 +5080 +5081 +5082 +5083 +5084 +5085 +5086 +5087 +5088 +5089 +5090 +5091 +5092 +5093 +5094 +5095 +5096 +5097 +5098 +5099 +5100 +5101 +5102 +5103 +5104 +5105 +5106 +5107 +5108 +5109 +5110 +5111 +5112 +5113 +5114 +5115 +5116 +5117 +5118 +5119 +5120 +5121 +5122 +5123 +5124 +5125 +5126 +5127 +5128 +5129 +5130 +5131 +5132 +5133 +5134 +5135 +5136 +5137 +5138 +5139 +5140 +5141 +5142 +5143 +5144 +5145 +5146 +5147 +5148 +5149 +5150 +5151 +5152 +5153 +5154 +5155 +5156 +5157 +5158 +5159 +5160 +5161 +5162 +5163 +5164 +5165 +5166 +5167 +5168 +5169 +5170 +5171 +5172 +5173 +5174 +5175 +5176 +5177 +5178 +5179 +5180 +5181 +5182 +5183 +5184 +5185 +5186 +5187 +5188 +5189 +5190 +5191 +5192 +5193 +5194 +5195 +5196 +5197 +5198 +5199 +5200 +5201 +5202 +5203 +5204 +5205 +5206 +5207 +5208 +5209 +5210 +5211 +5212 +5213 +5214 +5215 +5216 +5217 +5218 +5219 +5220 +5221 +5222 +5223 +5224 +5225 +5226 +5227 +5228 +5229 +5230 +5231 +5232 +5233 +5234 +5235 +5236 +5237 +5238 +5239 +5240 +5241 +5242 +5243 +5244 +5245 +5246 +5247 +5248 +5249 +5250 +5251 +5252 +5253 +5254 +5255 +5256 +5257 +5258 +5259 +5260 +5261 +5262 +5263 +5264 +5265 +5266 +5267 +5268 +5269 +5270 +5271 +5272 +5273 +5274 +5275 +5276 +5277 +5278 +5279 +5280 +5281 +5282 +5283 +5284 +5285 +5286 +5287 +5288 +5289 +5290 +5291 +5292 +5293 +5294 +5295 +5296 +5297 +5298 +5299 +5300 +5301 +5302 +5303 +5304 +5305 +5306 +5307 +5308 +5309 +5310 +5311 +5312 +5313 +5314 +5315 +5316 +5317 +5318 +5319 +5320 +5321 +5322 +5323 +5324 +5325 +5326 +5327 +5328 +5329 +5330 +5331 +5332 +5333 +5334 +5335 +5336 +5337 +5338 +5339 +5340 +5341 +5342 +5343 +5344 +5345 +5346 +5347 +5348 +5349 +5350 +5351 +5352 +5353 +5354 +5355 +5356 +5357 +5358 +5359 +5360 +5361 +5362 +5363 +5364 +5365 +5366 +5367 +5368 +5369 +5370 +5371 +5372 +5373 +5374 +5375 +5376 +5377 +5378 +5379 +5380 +5381 +5382 +5383 +5384 +5385 +5386 +5387 +5388 +5389 +5390 +5391 +5392 +5393 +5394 +5395 +5396 +5397 +5398 +5399 +5400 +5401 +5402 +5403 +5404 +5405 +5406 +5407 +5408 +5409 +5410 +5411 +5412 +5413 +5414 +5415 +5416 +5417 +5418 +5419 +5420 +5421 +5422 +5423 +5424 +5425 +5426 +5427 +5428 +5429 +5430 +5431 +5432 +5433 +5434 +5435 +5436 +5437 +5438 +5439 +5440 +5441 +5442 +5443 +5444 +5445 +5446 +5447 +5448 +5449 +5450 +5451 +5452 +5453 +5454 +5455 +5456 +5457 +5458 +5459 +5460 +5461 +5462 +5463 +5464 +5465 +5466 +5467 +5468 +5469 +5470 +5471 +5472 +5473 +5474 +5475 +5476 +5477 +5478 +5479 +5480 +5481 +5482 +5483 +5484 +5485 +5486 +5487 +5488 +5489 +5490 +5491 +5492 +5493 +5494 +5495 +5496 +5497 +5498 +5499 +5500 +5501 +5502 +5503 +5504 +5505 +5506 +5507 +5508 +5509 +5510 +5511 +5512 +5513 +5514 +5515 +5516 +5517 +5518 +5519 +5520 +5521 +5522 +5523 +5524 +5525 +5526 +5527 +5528 +5529 +5530 +5531 +5532 +5533 +5534 +5535 +5536 +5537 +5538 +5539 +5540 +5541 +5542 +5543 +5544 +5545 +5546 +5547 +5548 +5549 +5550 +5551 +5552 +5553 +5554 +5555 +5556 +5557 +5558 +5559 +5560 +5561 +5562 +5563 +5564 +5565 +5566 +5567 +5568 +5569 +5570 +5571 +5572 +5573 +5574 +5575 +5576 +5577 +5578 +5579 +5580 +5581 +5582 +5583 +5584 +5585 +5586 +5587 +5588 +5589 +5590 +5591 +5592 +5593 +5594 +5595 +5596 +5597 +5598 +5599 +5600 +5601 +5602 +5603 +5604 +5605 +5606 +5607 +5608 +5609 +5610 +5611 +5612 +5613 +5614 +5615 +5616 +5617 +5618 +5619 +5620 +5621 +5622 +5623 +5624 +5625 +5626 +5627 +5628 +5629 +5630 +5631 +5632 +5633 +5634 +5635 +5636 +5637 +5638 +5639 +5640 +5641 +5642 +5643 +5644 +5645 +5646 +5647 +5648 +5649 +5650 +5651 +5652 +5653 +5654 +5655 +5656 +5657 +5658 +5659 +5660 +5661 +5662 +5663 +5664 +5665 +5666 +5667 +5668 +5669 +5670 +5671 +5672 +5673 +5674 +5675 +5676 +5677 +5678 +5679 +5680 +5681 +5682 +5683 +5684 +5685 +5686 +5687 +5688 +5689 +5690 +5691 +5692 +5693 +5694 +5695 +5696 +5697 +5698 +5699 +5700 +5701 +5702 +5703 +5704 +5705 +5706 +5707 +5708 +5709 +5710 +5711 +5712 +5713 +5714 +5715 +5716 +5717 +5718 +5719 +5720 +5721 +5722 +5723 +5724 +5725 +5726 +5727 +5728 +5729 +5730 +5731 +5732 +5733 +5734 +5735 +5736 +5737 +5738 +5739 +5740 +5741 +5742 +5743 +5744 +5745 +5746 +5747 +5748 +5749 +5750 +5751 +5752 +5753 +5754 +5755 +5756 +5757 +5758 +5759 +5760 +5761 +5762 +5763 +5764 +5765 +5766 +5767 +5768 +5769 +5770 +5771 +5772 +5773 +5774 +5775 +5776 +5777 +5778 +5779 +5780 +5781 +5782 +5783 +5784 +5785 +5786 +5787 +5788 +5789 +5790 +5791 +5792 +5793 +5794 +5795 +5796 +5797 +5798 +5799 +5800 +5801 +5802 +5803 +5804 +5805 +5806 +5807 +5808 +5809 +5810 +5811 +5812 +5813 +5814 +5815 +5816 +5817 +5818 +5819 +5820 +5821 +5822 +5823 +5824 +5825 +5826 +5827 +5828 +5829 +5830 +5831 +5832 +5833 +5834 +5835 +5836 +5837 +5838 +5839 +5840 +5841 +5842 +5843 +5844 +5845 +5846 +5847 +5848 +5849 +5850 +5851 +5852 +5853 +5854 +5855 +5856 +5857 +5858 +5859 +5860 +5861 +5862 +5863 +5864 +5865 +5866 +5867 +5868 +5869 +5870 +5871 +5872 +5873 +5874 +5875 +5876 +5877 +5878 +5879 +5880 +5881 +5882 +5883 +5884 +5885 +5886 +5887 +5888 +5889 +5890 +5891 +5892 +5893 +5894 +5895 +5896 +5897 +5898 +5899 +5900 +5901 +5902 +5903 +5904 +5905 +5906 +5907 +5908 +5909 +5910 +5911 +5912 +5913 +5914 +5915 +5916 +5917 +5918 +5919 +5920 +5921 +5922 +5923 +5924 +5925 +5926 +5927 +5928 +5929 +5930 +5931 +5932 +5933 +5934 +5935 +5936 +5937 +5938 +5939 +5940 +5941 +5942 +5943 +5944 +5945 +5946 +5947 +5948 +5949 +5950 +5951 +5952 +5953 +5954 +5955 +5956 +5957 +5958 +5959 +5960 +5961 +5962 +5963 +5964 +5965 +5966 +5967 +5968 +5969 +5970 +5971 +5972 +5973 +5974 +5975 +5976 +5977 +5978 +5979 +5980 +5981 +5982 +5983 +5984 +5985 +5986 +5987 +5988 +5989 +5990 +5991 +5992 +5993 +5994 +5995 +5996 +5997 +5998 +5999 +6000 +6001 +6002 +6003 +6004 +6005 +6006 +6007 +6008 +6009 +6010 +6011 +6012 +6013 +6014 +6015 +6016 +6017 +6018 +6019 +6020 +6021 +6022 +6023 +6024 +6025 +6026 +6027 +6028 +6029 +6030 +6031 +6032 +6033 +6034 +6035 +6036 +6037 +6038 +6039 +6040 +6041 +6042 +6043 +6044 +6045 +6046 +6047 +6048 +6049 +6050 +6051 +6052 +6053 +6054 +6055 +6056 +6057 +6058 +6059 +6060 +6061 +6062 +6063 +6064 +6065 +6066 +6067 +6068 +6069 +6070 +6071 +6072 +6073 +6074 +6075 +6076 +6077 +6078 +6079 +6080 +6081 +6082 +6083 +6084 +6085 +6086 +6087 +6088 +6089 +6090 +6091 +6092 +6093 +6094 +6095 +6096 +6097 +6098 +6099 +6100 +6101 +6102 +6103 +6104 +6105 +6106 +6107 +6108 +6109 +6110 +6111 +6112 +6113 +6114 +6115 +6116 +6117 +6118 +6119 +6120 +6121 +6122 +6123 +6124 +6125 +6126 +6127 +6128 +6129 +6130 +6131 +6132 +6133 +6134 +6135 +6136 +6137 +6138 +6139 +6140 +6141 +6142 +6143 +6144 +6145 +6146 +6147 +6148 +6149 +6150 +6151 +6152 +6153 +6154 +6155 +6156 +6157 +6158 +6159 +6160 +6161 +6162 +6163 +6164 +6165 +6166 +6167 +6168 +6169 +6170 +6171 +6172 +6173 +6174 +6175 +6176 +6177 +6178 +6179 +6180 +6181 +6182 +6183 +6184 +6185 +6186 +6187 +6188 +6189 +6190 +6191 +6192 +6193 +6194 +6195 +6196 +6197 +6198 +6199 +6200 +6201 +6202 +6203 +6204 +6205 +6206 +6207 +6208 +6209 +6210 +6211 +6212 +6213 +6214 +6215 +6216 +6217 +6218 +6219 +6220 +6221 +6222 +6223 +6224 +6225 +6226 +6227 +6228 +6229 +6230 +6231 +6232 +6233 +6234 +6235 +6236 +6237 +6238 +6239 +6240 +6241 +6242 +6243 +6244 +6245 +6246 +6247 +6248 +6249 +6250 +6251 +6252 +6253 +6254 +6255 +6256 +6257 +6258 +6259 +6260 +6261 +6262 +6263 +6264 +6265 +6266 +6267 +6268 +6269 +6270 +6271 +6272 +6273 +6274 +6275 +6276 +6277 +6278 +6279 +6280 +6281 +6282 +6283 +6284 +6285 +6286 +6287 +6288 +6289 +6290 +6291 +6292 +6293 +6294 +6295 +6296 +6297 +6298 +6299 +6300 +6301 +6302 +6303 +6304 +6305 +6306 +6307 +6308 +6309 +6310 +6311 +6312 +6313 +6314 +6315 +6316 +6317 +6318 +6319 +6320 +6321 +6322 +6323 +6324 +6325 +6326 +6327 +6328 +6329 +6330 +6331 +6332 +6333 +6334 +6335 +6336 +6337 +6338 +6339 +6340 +6341 +6342 +6343 +6344 +6345 +6346 +6347 +6348 +6349 +6350 +6351 +6352 +6353 +6354 +6355 +6356 +6357 +6358 +6359 +6360 +6361 +6362 +6363 +6364 +6365 +6366 +6367 +6368 +6369 +6370 +6371 +6372 +6373 +6374 +6375 +6376 +6377 +6378 +6379 +6380 +6381 +6382 +6383 +6384 +6385 +6386 +6387 +6388 +6389 +6390 +6391 +6392 +6393 +6394 +6395 +6396 +6397 +6398 +6399 +6400 +6401 +6402 +6403 +6404 +6405 +6406 +6407 +6408 +6409 +6410 +6411 +6412 +6413 +6414 +6415 +6416 +6417 +6418 +6419 +6420 +6421 +6422 +6423 +6424 +6425 +6426 +6427 +6428 +6429 +6430 +6431 +6432 +6433 +6434 +6435 +6436 +6437 +6438 +6439 +6440 +6441 +6442 +6443 +6444 +6445 +6446 +6447 +6448 +6449 +6450 +6451 +6452 +6453 +6454 +6455 +6456 +6457 +6458 +6459 +6460 +6461 +6462 +6463 +6464 +6465 +6466 +6467 +6468 +6469 +6470 +6471 +6472 +6473 +6474 +6475 +6476 +6477 +6478 +6479 +6480 +6481 +6482 +6483 +6484 +6485 +6486 +6487 +6488 +6489 +6490 +6491 +6492 +6493 +6494 +6495 +6496 +6497 +6498 +6499 +6500 +6501 +6502 +6503 +6504 +6505 +6506 +6507 +6508 +6509 +6510 +6511 +6512 +6513 +6514 +6515 +6516 +6517 +6518 +6519 +6520 +6521 +6522 +6523 +6524 +6525 +6526 +6527 +6528 +6529 +6530 +6531 +6532 +6533 +6534 +6535 +6536 +6537 +6538 +6539 +6540 +6541 +6542 +6543 +6544 +6545 +6546 +6547 +6548 +6549 +6550 +6551 +6552 +6553 +6554 +6555 +6556 +6557 +6558 +6559 +6560 +6561 +6562 +6563 +6564 +6565 +6566 +6567 +6568 +6569 +6570 +6571 +6572 +6573 +6574 +6575 +6576 +6577 +6578 +6579 +6580 +6581 +6582 +6583 +6584 +6585 +6586 +6587 +6588 +6589 +6590 +6591 +6592 +6593 +6594 +6595 +6596 +6597 +6598 +6599 +6600 +6601 +6602 +6603 +6604 +6605 +6606 +6607 +6608 +6609 +6610 +6611 +6612 +6613 +6614 +6615 +6616 +6617 +6618 +6619 +6620 +6621 +6622 +6623 +6624 +6625 +6626 +6627 +6628 +6629 +6630 +6631 +6632 +6633 +6634 +6635 +6636 +6637 +6638 +6639 +6640 +6641 +6642 +6643 +6644 +6645 +6646 +6647 +6648 +6649 +6650 +6651 +6652 +6653 +6654 +6655 +6656 +6657 +6658 +6659 +6660 +6661 +6662 +6663 +6664 +6665 +6666 +6667 +6668 +6669 +6670 +6671 +6672 +6673 +6674 +6675 +6676 +6677 +6678 +6679 +6680 +6681 +6682 +6683 +6684 +6685 +6686 +6687 +6688 +6689 +6690 +6691 +6692 +6693 +6694 +6695 +6696 +6697 +6698 +6699 +6700 +6701 +6702 +6703 +6704 +6705 +6706 +6707 +6708 +6709 +6710 +6711 +6712 +6713 +6714 +6715 +6716 +6717 +6718 +6719 +6720 +6721 +6722 +6723 +6724 +6725 +6726 +6727 +6728 +6729 +6730 +6731 +6732 +6733 +6734 +6735 +6736 +6737 +6738 +6739 +6740 +6741 +6742 +6743 +6744 +6745 +6746 +6747 +6748 +6749 +6750 +6751 +6752 +6753 +6754 +6755 +6756 +6757 +6758 +6759 +6760 +6761 +6762 +6763 +6764 +6765 +6766 +6767 +6768 +6769 +6770 +6771 +6772 +6773 +6774 +6775 +6776 +6777 +6778 +6779 +6780 +6781 +6782 +6783 +6784 +6785 +6786 +6787 +6788 +6789 +6790 +6791 +6792 +6793 +6794 +6795 +6796 +6797 +6798 +6799 +6800 +6801 +6802 +6803 +6804 +6805 +6806 +6807 +6808 +6809 +6810 +6811 +6812 +6813 +6814 +6815 +6816 +6817 +6818 +6819 +6820 +6821 +6822 +6823 +6824 +6825 +6826 +6827 +6828 +6829 +6830 +6831 +6832 +6833 +6834 +6835 +6836 +6837 +6838 +6839 +6840 +6841 +6842 +6843 +6844 +6845 +6846 +6847 +6848 +6849 +6850 +6851 +6852 +6853 +6854 +6855 +6856 +6857 +6858 +6859 +6860 +6861 +6862 +6863 +6864 +6865 +6866 +6867 +6868 +6869 +6870 +6871 +6872 +6873 +6874 +6875 +6876 +6877 +6878 +6879 +6880 +6881 +6882 +6883 +6884 +6885 +6886 +6887 +6888 +6889 +6890 +6891 +6892 +6893 +6894 +6895 +6896 +6897 +6898 +6899 +6900 +6901 +6902 +6903 +6904 +6905 +6906 +6907 +6908 +6909 +6910 +6911 +6912 +6913 +6914 +6915 +6916 +6917 +6918 +6919 +6920 +6921 +6922 +6923 +6924 +6925 +6926 +6927 +6928 +6929 +6930 +6931 +6932 +6933 +6934 +6935 +6936 +6937 +6938 +6939 +6940 +6941 +6942 +6943 +6944 +6945 +6946 +6947 +6948 +6949 +6950 +6951 +6952 +6953 +6954 +6955 +6956 +6957 +6958 +6959 +6960 +6961 +6962 +6963 +6964 +6965 +6966 +6967 +6968 +6969 +6970 +6971 +6972 +6973 +6974 +6975 +6976 +6977 +6978 +6979 +6980 +6981 +6982 +6983 +6984 +6985 +6986 +6987 +6988 +6989 +6990 +6991 +6992 +6993 +6994 +6995 +6996 +6997 +6998 +6999 +7000 +7001 +7002 +7003 +7004 +7005 +7006 +7007 +7008 +7009 +7010 +7011 +7012 +7013 +7014 +7015 +7016 +7017 +7018 +7019 +7020 +7021 +7022 +7023 +7024 +7025 +7026 +7027 +7028 +7029 +7030 +7031 +7032 +7033 +7034 +7035 +7036 +7037 +7038 +7039 +7040 +7041 +7042 +7043 +7044 +7045 +7046 +7047 +7048 +7049 +7050 +7051 +7052 +7053 +7054 +7055 +7056 +7057 +7058 +7059 +7060 +7061 +7062 +7063 +7064 +7065 +7066 +7067 +7068 +7069 +7070 +7071 +7072 +7073 +7074 +7075 +7076 +7077 +7078 +7079 +7080 +7081 +7082 +7083 +7084 +7085 +7086 +7087 +7088 +7089 +7090 +7091 +7092 +7093 +7094 +7095 +7096 +7097 +7098 +7099 +7100 +7101 +7102 +7103 +7104 +7105 +7106 +7107 +7108 +7109 +7110 +7111 +7112 +7113 +7114 +7115 +7116 +7117 +7118 +7119 +7120 +7121 +7122 +7123 +7124 +7125 +7126 +7127 +7128 +7129 +7130 +7131 +7132 +7133 +7134 +7135 +7136 +7137 +7138 +7139 +7140 +7141 +7142 +7143 +7144 +7145 +7146 +7147 +7148 +7149 +7150 +7151 +7152 +7153 +7154 +7155 +7156 +7157 +7158 +7159 +7160 +7161 +7162 +7163 +7164 +7165 +7166 +7167 +7168 +7169 +7170 +7171 +7172 +7173 +7174 +7175 +7176 +7177 +7178 +7179 +7180 +7181 +7182 +7183 +7184 +7185 +7186 +7187 +7188 +7189 +7190 +7191 +7192 +7193 +7194 +7195 +7196 +7197 +7198 +7199 +7200 +7201 +7202 +7203 +7204 +7205 +7206 +7207 +7208 +7209 +7210 +7211 +7212 +7213 +7214 +7215 +7216 +7217 +7218 +7219 +7220 +7221 +7222 +7223 +7224 +7225 +7226 +7227 +7228 +7229 +7230 +7231 +7232 +7233 +7234 +7235 +7236 +7237 +7238 +7239 +7240 +7241 +7242 +7243 +7244 +7245 +7246 +7247 +7248 +7249 +7250 +7251 +7252 +7253 +7254 +7255 +7256 +7257 +7258 +7259 +7260 +7261 +7262 +7263 +7264 +7265 +7266 +7267 +7268 +7269 +7270 +7271 +7272 +7273 +7274 +7275 +7276 +7277 +7278 +7279 +7280 +7281 +7282 +7283 +7284 +7285 +7286 +7287 +7288 +7289 +7290 +7291 +7292 +7293 +7294 +7295 +7296 +7297 +7298 +7299 +7300 +7301 +7302 +7303 +7304 +7305 +7306 +7307 +7308 +7309 +7310 +7311 +7312 +7313 +7314 +7315 +7316 +7317 +7318 +7319 +7320 +7321 +7322 +7323 +7324 +7325 +7326 +7327 +7328 +7329 +7330 +7331 +7332 +7333 +7334 +7335 +7336 +7337 +7338 +7339 +7340 +7341 +7342 +7343 +7344 +7345 +7346 +7347 +7348 +7349 +7350 +7351 +7352 +7353 +7354 +7355 +7356 +7357 +7358 +7359 +7360 +7361 +7362 +7363 +7364 +7365 +7366 +7367 +7368 +7369 +7370 +7371 +7372 +7373 +7374 +7375 +7376 +7377 +7378 +7379 +7380 +7381 +7382 +7383 +7384 +7385 +7386 +7387 +7388 +7389 +7390 +7391 +7392 +7393 +7394 +7395 +7396 +7397 +7398 +7399 +7400 +7401 +7402 +7403 +7404 +7405 +7406 +7407 +7408 +7409 +7410 +7411 +7412 +7413 +7414 +7415 +7416 +7417 +7418 +7419 +7420 +7421 +7422 +7423 +7424 +7425 +7426 +7427 +7428 +7429 +7430 +7431 +7432 +7433 +7434 +7435 +7436 +7437 +7438 +7439 +7440 +7441 +7442 +7443 +7444 +7445 +7446 +7447 +7448 +7449 +7450 +7451 +7452 +7453 +7454 +7455 +7456 +7457 +7458 +7459 +7460 +7461 +7462 +7463 +7464 +7465 +7466 +7467 +7468 +7469 +7470 +7471 +7472 +7473 +7474 +7475 +7476 +7477 +7478 +7479 +7480 +7481 +7482 +7483 +7484 +7485 +7486 +7487 +7488 +7489 +7490 +7491 +7492 +7493 +7494 +7495 +7496 +7497 +7498 +7499 +7500 +7501 +7502 +7503 +7504 +7505 +7506 +7507 +7508 +7509 +7510 +7511 +7512 +7513 +7514 +7515 +7516 +7517 +7518 +7519 +7520 +7521 +7522 +7523 +7524 +7525 +7526 +7527 +7528 +7529 +7530 +7531 +7532 +7533 +7534 +7535 +7536 +7537 +7538 +7539 +7540 +7541 +7542 +7543 +7544 +7545 +7546 +7547 +7548 +7549 +7550 +7551 +7552 +7553 +7554 +7555 +7556 +7557 +7558 +7559 +7560 +7561 +7562 +7563 +7564 +7565 +7566 +7567 +7568 +7569 +7570 +7571 +7572 +7573 +7574 +7575 +7576 +7577 +7578 +7579 +7580 +7581 +7582 +7583 +7584 +7585 +7586 +7587 +7588 +7589 +7590 +7591 +7592 +7593 +7594 +7595 +7596 +7597 +7598 +7599 +7600 +7601 +7602 +7603 +7604 +7605 +7606 +7607 +7608 +7609 +7610 +7611 +7612 +7613 +7614 +7615 +7616 +7617 +7618 +7619 +7620 +7621 +7622 +7623 +7624 +7625 +7626 +7627 +7628 +7629 +7630 +7631 +7632 +7633 +7634 +7635 +7636 +7637 +7638 +7639 +7640 +7641 +7642 +7643 +7644 +7645 +7646 +7647 +7648 +7649 +7650 +7651 +7652 +7653 +7654 +7655 +7656 +7657 +7658 +7659 +7660 +7661 +7662 +7663 +7664 +7665 +7666 +7667 +7668 +7669 +7670 +7671 +7672 +7673 +7674 +7675 +7676 +7677 +7678 +7679 +7680 +7681 +7682 +7683 +7684 +7685 +7686 +7687 +7688 +7689 +7690 +7691 +7692 +7693 +7694 +7695 +7696 +7697 +7698 +7699 +7700 +7701 +7702 +7703 +7704 +7705 +7706 +7707 +7708 +7709 +7710 +7711 +7712 +7713 +7714 +7715 +7716 +7717 +7718 +7719 +7720 +7721 +7722 +7723 +7724 +7725 +7726 +7727 +7728 +7729 +7730 +7731 +7732 +7733 +7734 +7735 +7736 +7737 +7738 +7739 +7740 +7741 +7742 +7743 +7744 +7745 +7746 +7747 +7748 +7749 +7750 +7751 +7752 +7753 +7754 +7755 +7756 +7757 +7758 +7759 +7760 +7761 +7762 +7763 +7764 +7765 +7766 +7767 +7768 +7769 +7770 +7771 +7772 +7773 +7774 +7775 +7776 +7777 +7778 +7779 +7780 +7781 +7782 +7783 +7784 +7785 +7786 +7787 +7788 +7789 +7790 +7791 +7792 +7793 +7794 +7795 +7796 +7797 +7798 +7799 +7800 +7801 +7802 +7803 +7804 +7805 +7806 +7807 +7808 +7809 +7810 +7811 +7812 +7813 +7814 +7815 +7816 +7817 +7818 +7819 +7820 +7821 +7822 +7823 +7824 +7825 +7826 +7827 +7828 +7829 +7830 +7831 +7832 +7833 +7834 +7835 +7836 +7837 +7838 +7839 +7840 +7841 +7842 +7843 +7844 +7845 +7846 +7847 +7848 +7849 +7850 +7851 +7852 +7853 +7854 +7855 +7856 +7857 +7858 +7859 +7860 +7861 +7862 +7863 +7864 +7865 +7866 +7867 +7868 +7869 +7870 +7871 +7872 +7873 +7874 +7875 +7876 +7877 +7878 +7879 +7880 +7881 +7882 +7883 +7884 +7885 +7886 +7887 +7888 +7889 +7890 +7891 +7892 +7893 +7894 +7895 +7896 +7897 +7898 +7899 +7900 +7901 +7902 +7903 +7904 +7905 +7906 +7907 +7908 +7909 +7910 +7911 +7912 +7913 +7914 +7915 +7916 +7917 +7918 +7919 +7920 +7921 +7922 +7923 +7924 +7925 +7926 +7927 +7928 +7929 +7930 +7931 +7932 +7933 +7934 +7935 +7936 +7937 +7938 +7939 +7940 +7941 +7942 +7943 +7944 +7945 +7946 +7947 +7948 +7949 +7950 +7951 +7952 +7953 +7954 +7955 +7956 +7957 +7958 +7959 +7960 +7961 +7962 +7963 +7964 +7965 +7966 +7967 +7968 +7969 +7970 +7971 +7972 +7973 +7974 +7975 +7976 +7977 +7978 +7979 +7980 +7981 +7982 +7983 +7984 +7985 +7986 +7987 +7988 +7989 +7990 +7991 +7992 +7993 +7994 +7995 +7996 +7997 +7998 +7999 +8000 +8001 +8002 +8003 +8004 +8005 +8006 +8007 +8008 +8009 +8010 +8011 +8012 +8013 +8014 +8015 +8016 +8017 +8018 +8019 +8020 +8021 +8022 +8023 +8024 +8025 +8026 +8027 +8028 +8029 +8030 +8031 +8032 +8033 +8034 +8035 +8036 +8037 +8038 +8039 +8040 +8041 +8042 +8043 +8044 +8045 +8046 +8047 +8048 +8049 +8050 +8051 +8052 +8053 +8054 +8055 +8056 +8057 +8058 +8059 +8060 +8061 +8062 +8063 +8064 +8065 +8066 +8067 +8068 +8069 +8070 +8071 +8072 +8073 +8074 +8075 +8076 +8077 +8078 +8079 +8080 +8081 +8082 +8083 +8084 +8085 +8086 +8087 +8088 +8089 +8090 +8091 +8092 +8093 +8094 +8095 +8096 +8097 +8098 +8099 +8100 +8101 +8102 +8103 +8104 +8105 +8106 +8107 +8108 +8109 +8110 +8111 +8112 +8113 +8114 +8115 +8116 +8117 +8118 +8119 +8120 +8121 +8122 +8123 +8124 +8125 +8126 +8127 +8128 +8129 +8130 +8131 +8132 +8133 +8134 +8135 +8136 +8137 +8138 +8139 +8140 +8141 +8142 +8143 +8144 +8145 +8146 +8147 +8148 +8149 +8150 +8151 +8152 +8153 +8154 +8155 +8156 +8157 +8158 +8159 +8160 +8161 +8162 +8163 +8164 +8165 +8166 +8167 +8168 +8169 +8170 +8171 +8172 +8173 +8174 +8175 +8176 +8177 +8178 +8179 +8180 +8181 +8182 +8183 +8184 +8185 +8186 +8187 +8188 +8189 +8190 +8191 +8192 +8193 +8194 +8195 +8196 +8197 +8198 +8199 +8200 +8201 +8202 +8203 +8204 +8205 +8206 +8207 +8208 +8209 +8210 +8211 +8212 +8213 +8214 +8215 +8216 +8217 +8218 +8219 +8220 +8221 +8222 +8223 +8224 +8225 +8226 +8227 +8228 +8229 +8230 +8231 +8232 +8233 +8234 +8235 +8236 +8237 +8238 +8239 +8240 +8241 +8242 +8243 +8244 +8245 +8246 +8247 +8248 +8249 +8250 +8251 +8252 +8253 +8254 +8255 +8256 +8257 +8258 +8259 +8260 +8261 +8262 +8263 +8264 +8265 +8266 +8267 +8268 +8269 +8270 +8271 +8272 +8273 +8274 +8275 +8276 +8277 +8278 +8279 +8280 +8281 +8282 +8283 +8284 +8285 +8286 +8287 +8288 +8289 +8290 +8291 +8292 +8293 +8294 +8295 +8296 +8297 +8298 +8299 +8300 +8301 +8302 +8303 +8304 +8305 +8306 +8307 +8308 +8309 +8310 +8311 +8312 +8313 +8314 +8315 +8316 +8317 +8318 +8319 +8320 +8321 +8322 +8323 +8324 +8325 +8326 +8327 +8328 +8329 +8330 +8331 +8332 +8333 +8334 +8335 +8336 +8337 +8338 +8339 +8340 +8341 +8342 +8343 +8344 +8345 +8346 +8347 +8348 +8349 +8350 +8351 +8352 +8353 +8354 +8355 +8356 +8357 +8358 +8359 +8360 +8361 +8362 +8363 +8364 +8365 +8366 +8367 +8368 +8369 +8370 +8371 +8372 +8373 +8374 +8375 +8376 +8377 +8378 +8379 +8380 +8381 +8382 +8383 +8384 +8385 +8386 +8387 +8388 +8389 +8390 +8391 +8392 +8393 +8394 +8395 +8396 +8397 +8398 +8399 +8400 +8401 +8402 +8403 +8404 +8405 +8406 +8407 +8408 +8409 +8410 +8411 +8412 +8413 +8414 +8415 +8416 +8417 +8418 +8419 +8420 +8421 +8422 +8423 +8424 +8425 +8426 +8427 +8428 +8429 +8430 +8431 +8432 +8433 +8434 +8435 +8436 +8437 +8438 +8439 +8440 +8441 +8442 +8443 +8444 +8445 +8446 +8447 +8448 +8449 +8450 +8451 +8452 +8453 +8454 +8455 +8456 +8457 +8458 +8459 +8460 +8461 +8462 +8463 +8464 +8465 +8466 +8467 +8468 +8469 +8470 +8471 +8472 +8473 +8474 +8475 +8476 +8477 +8478 +8479 +8480 +8481 +8482 +8483 +8484 +8485 +8486 +8487 +8488 +8489 +8490 +8491 +8492 +8493 +8494 +8495 +8496 +8497 +8498 +8499 +8500 +8501 +8502 +8503 +8504 +8505 +8506 +8507 +8508 +8509 +8510 +8511 +8512 +8513 +8514 +8515 +8516 +8517 +8518 +8519 +8520 +8521 +8522 +8523 +8524 +8525 +8526 +8527 +8528 +8529 +8530 +8531 +8532 +8533 +8534 +8535 +8536 +8537 +8538 +8539 +8540 +8541 +8542 +8543 +8544 +8545 +8546 +8547 +8548 +8549 +8550 +8551 +8552 +8553 +8554 +8555 +8556 +8557 +8558 +8559 +8560 +8561 +8562 +8563 +8564 +8565 +8566 +8567 +8568 +8569 +8570 +8571 +8572 +8573 +8574 +8575 +8576 +8577 +8578 +8579 +8580 +8581 +8582 +8583 +8584 +8585 +8586 +8587 +8588 +8589 +8590 +8591 +8592 +8593 +8594 +8595 +8596 +8597 +8598 +8599 +8600 +8601 +8602 +8603 +8604 +8605 +8606 +8607 +8608 +8609 +8610 +8611 +8612 +8613 +8614 +8615 +8616 +8617 +8618 +8619 +8620 +8621 +8622 +8623 +8624 +8625 +8626 +8627 +8628 +8629 +8630 +8631 +8632 +8633 +8634 +8635 +8636 +8637 +8638 +8639 +8640 +8641 +8642 +8643 +8644 +8645 +8646 +8647 +8648 +8649 +8650 +8651 +8652 +8653 +8654 +8655 +8656 +8657 +8658 +8659 +8660 +8661 +8662 +8663 +8664 +8665 +8666 +8667 +8668 +8669 +8670 +8671 +8672 +8673 +8674 +8675 +8676 +8677 +8678 +8679 +8680 +8681 +8682 +8683 +8684 +8685 +8686 +8687 +8688 +8689 +8690 +8691 +8692 +8693 +8694 +8695 +8696 +8697 +8698 +8699 +8700 +8701 +8702 +8703 +8704 +8705 +8706 +8707 +8708 +8709 +8710 +8711 +8712 +8713 +8714 +8715 +8716 +8717 +8718 +8719 +8720 +8721 +8722 +8723 +8724 +8725 +8726 +8727 +8728 +8729 +8730 +8731 +8732 +8733 +8734 +8735 +8736 +8737 +8738 +8739 +8740 +8741 +8742 +8743 +8744 +8745 +8746 +8747 +8748 +8749 +8750 +8751 +8752 +8753 +8754 +8755 +8756 +8757 +8758 +8759 +8760 +8761 +8762 +8763 +8764 +8765 +8766 +8767 +8768 +8769 +8770 +8771 +8772 +8773 +8774 +8775 +8776 +8777 +8778 +8779 +8780 +8781 +8782 +8783 +8784 +8785 +8786 +8787 +8788 +8789 +8790 +8791 +8792 +8793 +8794 +8795 +8796 +8797 +8798 +8799 +8800 +8801 +8802 +8803 +8804 +8805 +8806 +8807 +8808 +8809 +8810 +8811 +8812 +8813 +8814 +8815 +8816 +8817 +8818 +8819 +8820 +8821 +8822 +8823 +8824 +8825 +8826 +8827 +8828 +8829 +8830 +8831 +8832 +8833 +8834 +8835 +8836 +8837 +8838 +8839 +8840 +8841 +8842 +8843 +8844 +8845 +8846 +8847 +8848 +8849 +8850 +8851 +8852 +8853 +8854 +8855 +8856 +8857 +8858 +8859 +8860 +8861 +8862 +8863 +8864 +8865 +8866 +8867 +8868 +8869 +8870 +8871 +8872 +8873 +8874 +8875 +8876 +8877 +8878 +8879 +8880 +8881 +8882 +8883 +8884 +8885 +8886 +8887 +8888 +8889 +8890 +8891 +8892 +8893 +8894 +8895 +8896 +8897 +8898 +8899 +8900 +8901 +8902 +8903 +8904 +8905 +8906 +8907 +8908 +8909 +8910 +8911 +8912 +8913 +8914 +8915 +8916 +8917 +8918 +8919 +8920 +8921 +8922 +8923 +8924 +8925 +8926 +8927 +8928 +8929 +8930 +8931 +8932 +8933 +8934 +8935 +8936 +8937 +8938 +8939 +8940 +8941 +8942 +8943 +8944 +8945 +8946 +8947 +8948 +8949 +8950 +8951 +8952 +8953 +8954 +8955 +8956 +8957 +8958 +8959 +8960 +8961 +8962 +8963 +8964 +8965 +8966 +8967 +8968 +8969 +8970 +8971 +8972 +8973 +8974 +8975 +8976 +8977 +8978 +8979 +8980 +8981 +8982 +8983 +8984 +8985 +8986 +8987 +8988 +8989 +8990 +8991 +8992 +8993 +8994 +8995 +8996 +8997 +8998 +8999 +9000 +9001 +9002 +9003 +9004 +9005 +9006 +9007 +9008 +9009 +9010 +9011 +9012 +9013 +9014 +9015 +9016 +9017 +9018 +9019 +9020 +9021 +9022 +9023 +9024 +9025 +9026 +9027 +9028 +9029 +9030 +9031 +9032 +9033 +9034 +9035 +9036 +9037 +9038 +9039 +9040 +9041 +9042 +9043 +9044 +9045 +9046 +9047 +9048 +9049 +9050 +9051 +9052 +9053 +9054 +9055 +9056 +9057 +9058 +9059 +9060 +9061 +9062 +9063 +9064 +9065 +9066 +9067 +9068 +9069 +9070 +9071 +9072 +9073 +9074 +9075 +9076 +9077 +9078 +9079 +9080 +9081 +9082 +9083 +9084 +9085 +9086 +9087 +9088 +9089 +9090 +9091 +9092 +9093 +9094 +9095 +9096 +9097 +9098 +9099 +9100 +9101 +9102 +9103 +9104 +9105 +9106 +9107 +9108 +9109 +9110 +9111 +9112 +9113 +9114 +9115 +9116 +9117 +9118 +9119 +9120 +9121 +9122 +9123 +9124 +9125 +9126 +9127 +9128 +9129 +9130 +9131 +9132 +9133 +9134 +9135 +9136 +9137 +9138 +9139 +9140 +9141 +9142 +9143 +9144 +9145 +9146 +9147 +9148 +9149 +9150 +9151 +9152 +9153 +9154 +9155 +9156 +9157 +9158 +9159 +9160 +9161 +9162 +9163 +9164 +9165 +9166 +9167 +9168 +9169 +9170 +9171 +9172 +9173 +9174 +9175 +9176 +9177 +9178 +9179 +9180 +9181 +9182 +9183 +9184 +9185 +9186 +9187 +9188 +9189 +9190 +9191 +9192 +9193 +9194 +9195 +9196 +9197 +9198 +9199 +9200 +9201 +9202 +9203 +9204 +9205 +9206 +9207 +9208 +9209 +9210 +9211 +9212 +9213 +9214 +9215 +9216 +9217 +9218 +9219 +9220 +9221 +9222 +9223 +9224 +9225 +9226 +9227 +9228 +9229 +9230 +9231 +9232 +9233 +9234 +9235 +9236 +9237 +9238 +9239 +9240 +9241 +9242 +9243 +9244 +9245 +9246 +9247 +9248 +9249 +9250 +9251 +9252 +9253 +9254 +9255 +9256 +9257 +9258 +9259 +9260 +9261 +9262 +9263 +9264 +9265 +9266 +9267 +9268 +9269 +9270 +9271 +9272 +9273 +9274 +9275 +9276 +9277 +9278 +9279 +9280 +9281 +9282 +9283 +9284 +9285 +9286 +9287 +9288 +9289 +9290 +9291 +9292 +9293 +9294 +9295 +9296 +9297 +9298 +9299 +9300 +9301 +9302 +9303 +9304 +9305 +9306 +9307 +9308 +9309 +9310 +9311 +9312 +9313 +9314 +9315 +9316 +9317 +9318 +9319 +9320 +9321 +9322 +9323 +9324 +9325 +9326 +9327 +9328 +9329 +9330 +9331 +9332 +9333 +9334 +9335 +9336 +9337 +9338 +9339 +9340 +9341 +9342 +9343 +9344 +9345 +9346 +9347 +9348 +9349 +9350 +9351 +9352 +9353 +9354 +9355 +9356 +9357 +9358 +9359 +9360 +9361 +9362 +9363 +9364 +9365 +9366 +9367 +9368 +9369 +9370 +9371 +9372 +9373 +9374 +9375 +9376 +9377 +9378 +9379 +9380 +9381 +9382 +9383 +9384 +9385 +9386 +9387 +9388 +9389 +9390 +9391 +9392 +9393 +9394 +9395 +9396 +9397 +9398 +9399 +9400 +9401 +9402 +9403 +9404 +9405 +9406 +9407 +9408 +9409 +9410 +9411 +9412 +9413 +9414 +9415 +9416 +9417 +9418 +9419 +9420 +9421 +9422 +9423 +9424 +9425 +9426 +9427 +9428 +9429 +9430 +9431 +9432 +9433 +9434 +9435 +9436 +9437 +9438 +9439 +9440 +9441 +9442 +9443 +9444 +9445 +9446 +9447 +9448 +9449 +9450 +9451 +9452 +9453 +9454 +9455 +9456 +9457 +9458 +9459 +9460 +9461 +9462 +9463 +9464 +9465 +9466 +9467 +9468 +9469 +9470 +9471 +9472 +9473 +9474 +9475 +9476 +9477 +9478 +9479 +9480 +9481 +9482 +9483 +9484 +9485 +9486 +9487 +9488 +9489 +9490 +9491 +9492 +9493 +9494 +9495 +9496 +9497 +9498 +9499 +9500 +9501 +9502 +9503 +9504 +9505 +9506 +9507 +9508 +9509 +9510 +9511 +9512 +9513 +9514 +9515 +9516 +9517 +9518 +9519 +9520 +9521 +9522 +9523 +9524 +9525 +9526 +9527 +9528 +9529 +9530 +9531 +9532 +9533 +9534 +9535 +9536 +9537 +9538 +9539 +9540 +9541 +9542 +9543 +9544 +9545 +9546 +9547 +9548 +9549 +9550 +9551 +9552 +9553 +9554 +9555 +9556 +9557 +9558 +9559 +9560 +9561 +9562 +9563 +9564 +9565 +9566 +9567 +9568 +9569 +9570 +9571 +9572 +9573 +9574 +9575 +9576 +9577 +9578 +9579 +9580 +9581 +9582 +9583 +9584 +9585 +9586 +9587 +9588 +9589 +9590 +9591 +9592 +9593 +9594 +9595 +9596 +9597 +9598 +9599 +9600 +9601 +9602 +9603 +9604 +9605 +9606 +9607 +9608 +9609 +9610 +9611 +9612 +9613 +9614 +9615 +9616 +9617 +9618 +9619 +9620 +9621 +9622 +9623 +9624 +9625 +9626 +9627 +9628 +9629 +9630 +9631 +9632 +9633 +9634 +9635 +9636 +9637 +9638 +9639 +9640 +9641 +9642 +9643 +9644 +9645 +9646 +9647 +9648 +9649 +9650 +9651 +9652 +9653 +9654 +9655 +9656 +9657 +9658 +9659 +9660 +9661 +9662 +9663 +9664 +9665 +9666 +9667 +9668 +9669 +9670 +9671 +9672 +9673 +9674 +9675 +9676 +9677 +9678 +9679 +9680 +9681 +9682 +9683 +9684 +9685 +9686 +9687 +9688 +9689 +9690 +9691 +9692 +9693 +9694 +9695 +9696 +9697 +9698 +9699 +9700 +9701 +9702 +9703 +9704 +9705 +9706 +9707 +9708 +9709 +9710 +9711 +9712 +9713 +9714 +9715 +9716 +9717 +9718 +9719 +9720 +9721 +9722 +9723 +9724 +9725 +9726 +9727 +9728 +9729 +9730 +9731 +9732 +9733 +9734 +9735 +9736 +9737 +9738 +9739 +9740 +9741 +9742 +9743 +9744 +9745 +9746 +9747 +9748 +9749 +9750 +9751 +9752 +9753 +9754 +9755 +9756 +9757 +9758 +9759 +9760 +9761 +9762 +9763 +9764 +9765 +9766 +9767 +9768 +9769 +9770 +9771 +9772 +9773 +9774 +9775 +9776 +9777 +9778 +9779 +9780 +9781 +9782 +9783 +9784 +9785 +9786 +9787 +9788 +9789 +9790 +9791 +9792 +9793 +9794 +9795 +9796 +9797 +9798 +9799 +9800 +9801 +9802 +9803 +9804 +9805 +9806 +9807 +9808 +9809 +9810 +9811 +9812 +9813 +9814 +9815 +9816 +9817 +9818 +9819 +9820 +9821 +9822 +9823 +9824 +9825 +9826 +9827 +9828 +9829 +9830 +9831 +9832 +9833 +9834 +9835 +9836 +9837 +9838 +9839 +9840 +9841 +9842 +9843 +9844 +9845 +9846 +9847 +9848 +9849 +9850 +9851 +9852 +9853 +9854 +9855 +9856 +9857 +9858 +9859 +9860 +9861 +9862 +9863 +9864 +9865 +9866 +9867 +9868 +9869 +9870 +9871 +9872 +9873 +9874 +9875 +9876 +9877 +9878 +9879 +9880 +9881 +9882 +9883 +9884 +9885 +9886 +9887 +9888 +9889 +9890 +9891 +9892 +9893 +9894 +9895 +9896 +9897 +9898 +9899 +9900 +9901 +9902 +9903 +9904 +9905 +9906 +9907 +9908 +9909 +9910 +9911 +9912 +9913 +9914 +9915 +9916 +9917 +9918 +9919 +9920 +9921 +9922 +9923 +9924 +9925 +9926 +9927 +9928 +9929 +9930 +9931 +9932 +9933 +9934 +9935 +9936 +9937 +9938 +9939 +9940 +9941 +9942 +9943 +9944 +9945 +9946 +9947 +9948 +9949 +9950 +9951 +9952 +9953 +9954 +9955 +9956 +9957 +9958 +9959 +9960 +9961 +9962 +9963 +9964 +9965 +9966 +9967 +9968 +9969 +9970 +9971 +9972 +9973 +9974 +9975 +9976 +9977 +9978 +9979 +9980 +9981 +9982 +9983 +9984 +9985 +9986 +9987 +9988 +9989 +9990 +9991 +9992 +9993 +9994 +9995 +9996 +9997 +9998 +9999 +10000 diff --git a/cpp/ppc-io/tests/data/windows_file.txt b/cpp/ppc-io/tests/data/windows_file.txt new file mode 100644 index 00000000..bed6107d --- /dev/null +++ b/cpp/ppc-io/tests/data/windows_file.txt @@ -0,0 +1,10 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 \ No newline at end of file diff --git a/cpp/ppc-io/tests/main.cpp b/cpp/ppc-io/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-io/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-main/CMakeLists.txt b/cpp/ppc-main/CMakeLists.txt new file mode 100644 index 00000000..f73c486d --- /dev/null +++ b/cpp/ppc-main/CMakeLists.txt @@ -0,0 +1,34 @@ +project(ppc-main VERSION ${VERSION}) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +add_subdirectory(air-node) + +if (BUILD_CEM) + add_subdirectory(cem-node) +endif () +add_subdirectory(mpc-node) + +macro(compile_service SERVICE_SOURCE_PATH BINARY_NAME PKG_NAME) + add_subdirectory(${SERVICE_SOURCE_PATH}) + add_custom_command(OUTPUT ${PKG_NAME}.tgz + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/ppc-main/${SERVICE_SOURCE_PATH}/${BINARY_NAME} ${CMAKE_BINARY_DIR}/${PKG_NAME}/${PKG_NAME} + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_BINARY_DIR} tar czfv ${PKG_NAME}.tgz ${PKG_NAME}/${PKG_NAME} + COMMAND ${CMAKE_COMMAND} -E copy ${PKG_NAME}.tgz tars/${PKG_NAME}.tgz + COMMENT "Compressing ${BINARY_NAME} into ${PKG_NAME}.tgz ..." + COMMAND ${CMAKE_COMMAND} -E rm ${PKG_NAME}.tgz + ) + add_custom_target(${PKG_NAME}-tar DEPENDS ${PKG_NAME}.tgz ${BINARY_NAME}) + list(APPEND SERVICE_TAR_LIST ${PKG_NAME}-tar) +endmacro() + +# gateway +# Note: the tar-pkg is only useful when using tars-admin +compile_service("gateway" ${GATEWAY_BINARY_NAME} ${GATEWAY_PKG_NAME}) +# ppc-node +compile_service("pro-node" ${PRO_BINARY_NAME} ${PPC_NODE_PKG_NAME}) + +# for make tar +add_custom_target(tar DEPENDS ${SERVICE_TAR_LIST}) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) diff --git a/cpp/ppc-main/air-node/AirNodeInitializer.cpp b/cpp/ppc-main/air-node/AirNodeInitializer.cpp new file mode 100644 index 00000000..7bad21b7 --- /dev/null +++ b/cpp/ppc-main/air-node/AirNodeInitializer.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AirNodeInitializer.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "AirNodeInitializer.h" +#include "ppc-gateway/Gateway.h" +#include "ppc-gateway/GatewayConfigContext.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-rpc/src/RpcMemory.h" +#include "ppc-storage/src/redis/RedisStorage.h" + +using namespace ppc::protocol; +using namespace ppc::node; +using namespace ppc::gateway; +using namespace ppc::rpc; +using namespace ppc::storage; +using namespace ppc::initializer; +using namespace bcos; + +void AirNodeInitializer::init(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + INIT_LOG(INFO) << LOG_DESC("initLog success"); + + // init the node + m_nodeInitializer = std::make_shared(_configPath); + m_nodeInitializer->init(ppc::protocol::NodeArch::AIR); + auto front = m_nodeInitializer->frontInitializer()->front(); + front->setSelfEndPoint("localhost"); + + // init the gateway + initGateway(_configPath); + // set the gateway into front + front->setGatewayInterface(m_gateway); + + INIT_LOG(INFO) << LOG_DESC("init the rpc"); + // load the rpc config + // not specify the certPath in air-mode + m_nodeInitializer->config()->loadRpcConfig(nullptr, pt); + // init RpcStatusInterface + RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(m_gateway); + + m_nodeInitializer->frontInitializer()->setRpcStatus(rpcStatusInterface); + auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID()); + m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config()); + m_rpc->setRpcStorage(rpcStatusInterface); + m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); + m_nodeInitializer->registerRpcHandler(m_rpc); + + // Note: only can fetchAgencyList after the gatewayInterface has been setted into front + m_nodeInitializer->fetchAgencyList(); + INIT_LOG(INFO) << LOG_DESC("init the rpc success"); +} + +void AirNodeInitializer::initGateway(std::string const& _configPath) +{ + INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath; + // not specify the certPath in air-mode + auto config = m_nodeInitializer->config(); + config->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, _configPath); + + auto threadPool = std::make_shared( + "gateway", config->gatewayConfig().networkConfig.threadPoolSize); + + // Note: no need use redis as cache in-air-mode + GatewayFactory gatewayFactory; + auto gateway = gatewayFactory.buildGateway(ppc::protocol::NodeArch::AIR, config, nullptr, + m_nodeInitializer->protocolInitializer()->ppcMsgFactory(), threadPool); + auto frontInitializer = m_nodeInitializer->frontInitializer(); + gateway->registerFront(frontInitializer->front()->selfEndPoint(), frontInitializer->front()); + m_gateway = gateway; +} + +void AirNodeInitializer::start() +{ + // start the node + if (m_nodeInitializer) + { + m_nodeInitializer->start(); + } + if (m_gateway) + { + m_gateway->start(); + } + if (m_rpc) + { + m_rpc->start(); + } +} + +void AirNodeInitializer::stop() +{ + if (m_rpc) + { + m_rpc->stop(); + } + if (m_gateway) + { + m_gateway->stop(); + } + if (m_nodeInitializer) + { + m_nodeInitializer->stop(); + } +} \ No newline at end of file diff --git a/cpp/ppc-main/air-node/AirNodeInitializer.h b/cpp/ppc-main/air-node/AirNodeInitializer.h new file mode 100644 index 00000000..260b39cb --- /dev/null +++ b/cpp/ppc-main/air-node/AirNodeInitializer.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AirNodeInitializer.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "libinitializer/Common.h" +#include "libinitializer/Initializer.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include +#include +namespace ppc::rpc +{ +class Rpc; +} +namespace ppc::node +{ +class AirNodeInitializer +{ +public: + using Ptr = std::shared_ptr; + AirNodeInitializer() {} + virtual ~AirNodeInitializer() { stop(); } + + virtual void init(std::string const& _configPath); + virtual void start(); + virtual void stop(); + +protected: + virtual void initGateway(std::string const& _configPath); + +private: + bcos::BoostLogInitializer::Ptr m_logInitializer; + ppc::initializer::Initializer::Ptr m_nodeInitializer; + ppc::gateway::GatewayInterface::Ptr m_gateway; + std::shared_ptr m_rpc; +}; +} // namespace ppc::node \ No newline at end of file diff --git a/cpp/ppc-main/air-node/CMakeLists.txt b/cpp/ppc-main/air-node/CMakeLists.txt new file mode 100644 index 00000000..53e7bfb9 --- /dev/null +++ b/cpp/ppc-main/air-node/CMakeLists.txt @@ -0,0 +1,4 @@ +aux_source_directory(. SRC_LIST) + +add_executable(${AIR_BINARY_NAME} ${SRC_LIST}) +target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc) \ No newline at end of file diff --git a/cpp/ppc-main/air-node/main.cpp b/cpp/ppc-main/air-node/main.cpp new file mode 100644 index 00000000..884f5298 --- /dev/null +++ b/cpp/ppc-main/air-node/main.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "AirNodeInitializer.h" +#include +#include +#include +#include +#include + +using namespace ppc; +int main(int argc, const char* argv[]) +{ + /// set LC_ALL + setDefaultOrCLocale(); + std::set_terminate([]() { + std::cerr << "terminate handler called, print stacks" << std::endl; + void* trace_elems[20]; + int trace_elem_count(backtrace(trace_elems, 20)); + char** stack_syms(backtrace_symbols(trace_elems, trace_elem_count)); + for (int i = 0; i < trace_elem_count; ++i) + { + std::cout << stack_syms[i] << "\n"; + } + free(stack_syms); + std::cerr << "terminate handler called, print stack end" << std::endl; + abort(); + }); + // get datetime and output welcome info + ExitHandler exitHandler; + signal(SIGTERM, &ExitHandler::exitHandler); + signal(SIGABRT, &ExitHandler::exitHandler); + signal(SIGINT, &ExitHandler::exitHandler); + + // Note: the initializer must exist in the life time of the whole program + auto initializer = std::make_shared(); + try + { + auto param = initCommandLine(argc, argv); + initializer->init(param.configFilePath); + initializer->start(); + } + catch (std::exception const& e) + { + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "start ppc-psi failed, error:" << boost::diagnostic_information(e) + << std::endl; + return -1; + } + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "The ppc-psi is running..." << std::endl; + while (!exitHandler.shouldExit()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + initializer.reset(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "ppc-psi program exit normally." << std::endl; +} diff --git a/cpp/ppc-main/cem-node/CEMInitializer.cpp b/cpp/ppc-main/cem-node/CEMInitializer.cpp new file mode 100644 index 00000000..7344386c --- /dev/null +++ b/cpp/ppc-main/cem-node/CEMInitializer.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CEMInitializer.cpp + * @author: caryliao + * @date 2022-11-19 + */ +#include "CEMInitializer.h" +#include "ppc-cem/src/CEMService.h" + +using namespace ppc::rpc; +using namespace bcos; +using namespace ppc::cem; +using namespace ppc::tools; + +void CEMInitializer::init(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + INIT_LOG(INFO) << LOG_DESC("initLog success"); + + // init the rpc + INIT_LOG(INFO) << LOG_DESC("init the rpc"); + // load the rpc config + auto ppcConfig = std::make_shared(); + // not specify the certPath in air-mode + ppcConfig->loadRpcConfig(nullptr, pt); + ppcConfig->loadCEMConfig(pt); + // bool useMysql = pt.get("cem.use_mysql", false); + auto storageConfig = ppcConfig->storageConfig(); + auto cemConfig = ppcConfig->cemConfig(); + auto rpcFactory = std::make_shared(ppcConfig->agencyID()); + m_rpc = rpcFactory->buildRpc(ppcConfig); + auto cemService = std::make_shared(); + cemService->setCEMConfig(cemConfig); + cemService->setStorageConfig(storageConfig); + m_rpc->registerHandler("match", std::bind(&CEMService::makeCiphertextEqualityMatchRpc, + cemService, std::placeholders::_1, std::placeholders::_2)); + m_rpc->registerHandler("encrypt", std::bind(&CEMService::encryptDatasetRpc, cemService, + std::placeholders::_1, std::placeholders::_2)); + + INIT_LOG(INFO) << LOG_DESC("init the rpc success"); +} + +void CEMInitializer::start() +{ + // start the ppc cem + if (m_rpc) + { + m_rpc->start(); + } +} + +void CEMInitializer::stop() +{ + if (m_rpc) + { + m_rpc->stop(); + } +} diff --git a/cpp/ppc-main/cem-node/CEMInitializer.h b/cpp/ppc-main/cem-node/CEMInitializer.h new file mode 100644 index 00000000..149e9872 --- /dev/null +++ b/cpp/ppc-main/cem-node/CEMInitializer.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CEMInitializer.h + * @author: caryliao + * @date 2022-11-19 + */ +#pragma once +#include "libinitializer/Common.h" +#include "ppc-rpc/src/RpcFactory.h" +#include +#include +namespace ppc::rpc +{ +class Rpc; +} +namespace ppc::cem +{ +class CEMInitializer +{ +public: + using Ptr = std::shared_ptr; + CEMInitializer() {} + virtual ~CEMInitializer() { stop(); } + + virtual void init(std::string const& _configPath); + virtual void start(); + virtual void stop(); + +private: + bcos::BoostLogInitializer::Ptr m_logInitializer; + std::shared_ptr m_rpc; +}; +} // namespace ppc::cem \ No newline at end of file diff --git a/cpp/ppc-main/cem-node/CMakeLists.txt b/cpp/ppc-main/cem-node/CMakeLists.txt new file mode 100644 index 00000000..0b8957d9 --- /dev/null +++ b/cpp/ppc-main/cem-node/CMakeLists.txt @@ -0,0 +1,4 @@ +aux_source_directory(. SRC_LIST) + +add_executable(${CEM_BINARY_NAME} ${SRC_LIST}) +target_link_libraries(${CEM_BINARY_NAME} PUBLIC ${CEM_TARGET} ${RPC_TARGET} ${HELPER_TARGET}) diff --git a/cpp/ppc-main/cem-node/main.cpp b/cpp/ppc-main/cem-node/main.cpp new file mode 100644 index 00000000..cfd4a724 --- /dev/null +++ b/cpp/ppc-main/cem-node/main.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: caryliao + * @date 2022-11-19 + */ +#include "CEMInitializer.h" +#include +#include +#include +#include +#include + +using namespace ppc; + +int main(int argc, const char* argv[]) +{ + /// set LC_ALL + setDefaultOrCLocale(); + std::set_terminate([]() { + std::cerr << "terminate handler called, print stacks" << std::endl; + void* trace_elems[20]; + int trace_elem_count(backtrace(trace_elems, 20)); + char** stack_syms(backtrace_symbols(trace_elems, trace_elem_count)); + for (int i = 0; i < trace_elem_count; ++i) + { + std::cout << stack_syms[i] << "\n"; + } + free(stack_syms); + std::cerr << "terminate handler called, print stack end" << std::endl; + abort(); + }); + // get datetime and output welcome info + ExitHandler exitHandler; + signal(SIGTERM, &ExitHandler::exitHandler); + signal(SIGABRT, &ExitHandler::exitHandler); + signal(SIGINT, &ExitHandler::exitHandler); + + // Note: the initializer must exist in the life time of the whole program + auto initializer = std::make_shared(); + try + { + auto param = initCommandLine(argc, argv); + initializer->init(param.configFilePath); + initializer->start(); + } + catch (std::exception const& e) + { + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "start ppc-cem failed, error:" << boost::diagnostic_information(e) + << std::endl; + return -1; + } + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "The ppc-cem is running..." << std::endl; + while (!exitHandler.shouldExit()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + initializer.reset(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "ppc-cem program exit normally." << std::endl; +} diff --git a/cpp/ppc-main/gateway/CMakeLists.txt b/cpp/ppc-main/gateway/CMakeLists.txt new file mode 100644 index 00000000..8406c395 --- /dev/null +++ b/cpp/ppc-main/gateway/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.14) + +project(ppctars-Gateway) + +include_directories(${CMAKE_SOURCE_DIR}) + + +aux_source_directory(./ SRC_LIST) +aux_source_directory(../../ppc-tars-service/GatewayService SRC_LIST) +add_executable(${GATEWAY_BINARY_NAME} ${SRC_LIST}) + +target_link_libraries(${GATEWAY_BINARY_NAME} ${GATEWAY_TARGET} ${STORAGE_TARGET} ${PROTOCOL_TARGET} ${HELPER_TARGET}) \ No newline at end of file diff --git a/cpp/ppc-main/gateway/GatewayServiceApp.cpp b/cpp/ppc-main/gateway/GatewayServiceApp.cpp new file mode 100644 index 00000000..70dbf15f --- /dev/null +++ b/cpp/ppc-main/gateway/GatewayServiceApp.cpp @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServiceApp.cpp + * @author: shawnhe + * @date 2022-10-21 + */ +#include "GatewayServiceApp.h" +#include "ppc-storage/src/CacheStorageFactoryImpl.h" +#include "ppc-tars-protocol/Common.h" +#include "ppc-tars-service/GatewayService/GatewayInitializer.h" +#include "ppc-tars-service/GatewayService/GatewayServiceServer.h" +#include "ppc-tools/src/config/NetworkConfig.h" + +using namespace ppctars; +using namespace ppc; +using namespace ppc::tools; +using namespace ppc::gateway; +using namespace ppc::storage; +using namespace ppc::protocol; + +using namespace bcos; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; + +void GatewayServiceApp::initialize() +{ + try + { + m_configPath = tars::ServerConfig::BasePath + "/config.ini"; + addConfig("config.ini"); + + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService") << LOG_KV("config", m_configPath); + initService(m_configPath); + + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService success") << LOG_KV("config", m_configPath); + + GatewayServiceParam param; + param.gateway = m_gatewayInitializer->gateway(); + param.ppcMsgFactory = m_gatewayInitializer->messageFactory(); + addServantWithParams( + getProxyDesc(GATEWAY_SERVANT_NAME), param); + } + catch (std::exception const& e) + { + std::cout << "init GatewayServiceApp failed, error: " << boost::diagnostic_information(e) + << std::endl; + throw e; + } +} + +void GatewayServiceApp::initService(std::string const& _configPath) +{ + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + auto ppcConfig = std::make_shared(); + // init the log + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + + // load the gatewayConfig + auto config = std::make_shared(); + config->loadGatewayConfig( + ppc::protocol::NodeArch::PRO, tars::ServerConfig::BasePath.c_str(), pt); + // add the config + auto const& networkConfig = config->gatewayConfig().networkConfig; + + if (!networkConfig.disableSsl) + { + GATEWAYAPP_LOG(INFO) << LOG_DESC("addConfig") << LOG_KV("enableSM", networkConfig.enableSM); + if (!networkConfig.enableSM) + { + addConfig(std::string(NetworkConfig::CA_CERT_NAME)); + addConfig(std::string(NetworkConfig::SSL_CERT_NAME)); + addConfig(std::string(NetworkConfig::SSL_KEY_NAME)); + } + else + { + addConfig(std::string(NetworkConfig::SM_CA_CERT_NAME)); + addConfig(std::string(NetworkConfig::SM_SSL_CERT_NAME)); + addConfig(std::string(NetworkConfig::SM_SSL_KEY_NAME)); + addConfig(std::string(NetworkConfig::SM_SSL_EN_KEY_NAME)); + addConfig(std::string(NetworkConfig::SM_SSL_EN_CERT_NAME)); + } + GATEWAYAPP_LOG(INFO) << LOG_DESC("addConfig success") + << LOG_KV("enableSM", networkConfig.enableSM); + } + + // redis cache + storage::CacheStorage::Ptr cache; + if (!config->gatewayConfig().disableCache) + { + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: buildRedisStorage") + << config->gatewayConfig().cacheStorageConfig.desc(); + auto cacheStorageFactory = std::make_shared(); + cache = cacheStorageFactory->createCacheStorage(config->gatewayConfig().cacheStorageConfig); + try + { + cache->exists("check_cache"); + } + catch (std::exception& e) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment( + "init cache error:" + std::string(boost::diagnostic_information(e)))); + } + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: buildRedisStorage success"); + } + + // message factory + auto messageFactory = std::make_shared(); + // global thread pool + auto threadPoolSize = config->gatewayConfig().networkConfig.threadPoolSize; + auto threadPool = std::make_shared(GATEWAY_THREAD_POOL_MODULE, threadPoolSize); + + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: build and start gateway"); + m_gatewayInitializer = std::make_shared( + NodeArch::PRO, config, cache, messageFactory, threadPool); + m_gatewayInitializer->start(); + GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: build and start gateway success"); +} \ No newline at end of file diff --git a/cpp/ppc-main/gateway/GatewayServiceApp.h b/cpp/ppc-main/gateway/GatewayServiceApp.h new file mode 100644 index 00000000..7c462f2e --- /dev/null +++ b/cpp/ppc-main/gateway/GatewayServiceApp.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServiceApp.h + * @author: shawnhe + * @date 2022-10-21 + */ +#pragma once +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-tars-protocol/Common.h" +#include "ppc-tars-service/GatewayService/GatewayInitializer.h" +#include +#include + +#define GATEWAYAPP_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GatewayServiceApp]" + +namespace ppctars +{ +class GatewayServiceApp : public tars::Application +{ +public: + GatewayServiceApp() {} + ~GatewayServiceApp() override{}; + + void destroyApp() override + { + if (m_gatewayInitializer) + { + m_gatewayInitializer->stop(); + } + } + void initialize() override; + +protected: + virtual void initService(std::string const& _configPath); + +private: + std::string m_configPath; + bcos::BoostLogInitializer::Ptr m_logInitializer; + ppctars::GatewayInitializer::Ptr m_gatewayInitializer; +}; +} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-main/gateway/main.cpp b/cpp/ppc-main/gateway/main.cpp new file mode 100644 index 00000000..0212cf82 --- /dev/null +++ b/cpp/ppc-main/gateway/main.cpp @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: yujiechen + * @date 2022-11-25 + */ +#include "GatewayServiceApp.h" +#include "libhelper/CommandHelper.h" + +using namespace ppctars; + +int main(int argc, char* argv[]) +{ + try + { + ppc::initAppCommandLine(argc, argv); + GatewayServiceApp app; + app.main(argc, argv); + app.waitForShutdown(); + + return 0; + } + catch (std::exception& e) + { + cerr << "ppc-gateway-service std::exception:" << boost::diagnostic_information(e) + << std::endl; + } + catch (...) + { + cerr << "ppc-gateway-service unknown exception." << std::endl; + } + return -1; +} \ No newline at end of file diff --git a/cpp/ppc-main/mpc-node/CMakeLists.txt b/cpp/ppc-main/mpc-node/CMakeLists.txt new file mode 100644 index 00000000..9b752de2 --- /dev/null +++ b/cpp/ppc-main/mpc-node/CMakeLists.txt @@ -0,0 +1,4 @@ +aux_source_directory(. SRC_LIST) + +add_executable(${MPC_BINARY_NAME} ${SRC_LIST}) +target_link_libraries(${MPC_BINARY_NAME} PUBLIC ${MPC_TARGET} ${RPC_TARGET} ${HELPER_TARGET} TBB::tbb) diff --git a/cpp/ppc-main/mpc-node/MPCInitializer.cpp b/cpp/ppc-main/mpc-node/MPCInitializer.cpp new file mode 100644 index 00000000..350e24fb --- /dev/null +++ b/cpp/ppc-main/mpc-node/MPCInitializer.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MPCInitializer.cpp + * @author: caryliao + * @date 2023-03-24 + */ +#include "MPCInitializer.h" +#include "ppc-mpc/src/MPCService.h" + +using namespace ppc::rpc; +using namespace bcos; +using namespace ppc::mpc; +using namespace ppc::tools; + +void MPCInitializer::init(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + INIT_LOG(INFO) << LOG_DESC("initLog success"); + + // init the rpc + INIT_LOG(INFO) << LOG_DESC("init the rpc"); + // load the rpc config + auto ppcConfig = std::make_shared(); + // not specify the certPath in air-mode + ppcConfig->loadRpcConfig(nullptr, pt); + ppcConfig->loadMPCConfig(pt); + // bool useMysql = pt.get("mpc.use_mysql", false); + auto storageConfig = ppcConfig->storageConfig(); + auto mpcConfig = ppcConfig->mpcConfig(); + auto rpcFactory = std::make_shared(ppcConfig->agencyID()); + m_rpc = rpcFactory->buildRpc(ppcConfig); + auto mpcService = std::make_shared(); + mpcService->setMPCConfig(mpcConfig); + mpcService->setStorageConfig(storageConfig); + m_rpc->registerHandler("run", std::bind(&MPCService::runMpcRpc, mpcService, + std::placeholders::_1, std::placeholders::_2)); + m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService, + std::placeholders::_1, std::placeholders::_2)); + + + INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success"); +} + +void MPCInitializer::start() +{ + // start the ppc mpc + if (m_rpc) + { + m_rpc->start(); + } +} + +void MPCInitializer::stop() +{ + if (m_rpc) + { + m_rpc->stop(); + } +} diff --git a/cpp/ppc-main/mpc-node/MPCInitializer.h b/cpp/ppc-main/mpc-node/MPCInitializer.h new file mode 100644 index 00000000..1d92e675 --- /dev/null +++ b/cpp/ppc-main/mpc-node/MPCInitializer.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MPCInitializer.h + * @author: caryliao + * @date 2023-03-24 + */ +#pragma once +#include "libinitializer/Common.h" +#include "ppc-rpc/src/RpcFactory.h" +#include +#include +namespace ppc::rpc +{ +class Rpc; +} +namespace ppc::mpc +{ +class MPCInitializer +{ +public: + using Ptr = std::shared_ptr; + MPCInitializer() {} + virtual ~MPCInitializer() { stop(); } + + virtual void init(std::string const& _configPath); + virtual void start(); + virtual void stop(); + +private: + bcos::BoostLogInitializer::Ptr m_logInitializer; + std::shared_ptr m_rpc; +}; +} // namespace ppc::mpc \ No newline at end of file diff --git a/cpp/ppc-main/mpc-node/main.cpp b/cpp/ppc-main/mpc-node/main.cpp new file mode 100644 index 00000000..dae16bf5 --- /dev/null +++ b/cpp/ppc-main/mpc-node/main.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: caryliao + * @date 2023-03-24 + */ +#include "MPCInitializer.h" +#include +#include +#include +#include +#include + +using namespace ppc; + +int main(int argc, const char* argv[]) +{ + /// set LC_ALL + setDefaultOrCLocale(); + std::set_terminate([]() { + std::cerr << "terminate handler called, print stacks" << std::endl; + void* trace_elems[20]; + int trace_elem_count(backtrace(trace_elems, 20)); + char** stack_syms(backtrace_symbols(trace_elems, trace_elem_count)); + for (int i = 0; i < trace_elem_count; ++i) + { + std::cout << stack_syms[i] << "\n"; + } + free(stack_syms); + std::cerr << "terminate handler called, print stack end" << std::endl; + abort(); + }); + // get datetime and output welcome info + ExitHandler exitHandler; + signal(SIGTERM, &ExitHandler::exitHandler); + signal(SIGABRT, &ExitHandler::exitHandler); + signal(SIGINT, &ExitHandler::exitHandler); + + // Note: the initializer must exist in the life time of the whole program + auto initializer = std::make_shared(); + try + { + auto param = initCommandLine(argc, argv); + initializer->init(param.configFilePath); + initializer->start(); + } + catch (std::exception const& e) + { + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "start ppc-mpc failed, error:" << boost::diagnostic_information(e) + << std::endl; + return -1; + } + printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "The ppc-mpc is running..." << std::endl; + while (!exitHandler.shouldExit()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + initializer.reset(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "ppc-mpc program exit normally." << std::endl; +} diff --git a/cpp/ppc-main/pro-node/CMakeLists.txt b/cpp/ppc-main/pro-node/CMakeLists.txt new file mode 100644 index 00000000..ad796b5a --- /dev/null +++ b/cpp/ppc-main/pro-node/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.14) + +project(ppctars-node) + +include_directories(${CMAKE_SOURCE_DIR}) + +aux_source_directory(./ SRC_LIST) +aux_source_directory(../../ppc-tars-service/FrontService SRC_LIST) +add_executable(${PRO_BINARY_NAME} ${SRC_LIST}) + +target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc) diff --git a/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp b/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp new file mode 100644 index 00000000..4199d163 --- /dev/null +++ b/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProNodeServiceApp.cpp + * @author: yujiechen + * @date 2022-11-25 + */ +#include "ProNodeServiceApp.h" +#include "GatewayService.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-rpc/src/RpcMemory.h" +#include "ppc-tars-protocol/Common.h" +#include "ppc-tars-protocol/client/GatewayServiceClient.h" +#include "ppc-tars-service/FrontService/FrontServiceServer.h" + +using namespace ppctars; +using namespace ppc::initializer; +using namespace ppc::rpc; +using namespace ppc::tools; +using namespace ppc::protocol; + +void ProNodeServiceApp::destroyApp() +{ + NodeService_LOG(INFO) << LOG_DESC("destroyApp"); + if (m_rpc) + { + m_rpc->stop(); + } + if (m_nodeInitializer) + { + m_nodeInitializer->stop(); + } + NodeService_LOG(INFO) << LOG_DESC("destroyApp success"); +} + +void ProNodeServiceApp::initialize() +{ + try + { + m_configPath = tars::ServerConfig::BasePath + "/config.ini"; + addConfig("config.ini"); + + NodeService_LOG(INFO) << LOG_DESC("initService") << LOG_KV("config", m_configPath); + initService(m_configPath); + NodeService_LOG(INFO) << LOG_DESC("initService success") << LOG_KV("config", m_configPath); + } + catch (std::exception const& e) + { + // since the tars will not print the detailed information when exceptioned, we print here + std::cout << "init NodeServiceApp failed, error: " << boost::diagnostic_information(e) + << std::endl; + throw e; + } +} + +void ProNodeServiceApp::initService(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + auto ppcConfig = std::make_shared(); + ppcConfig->loadSelfTarsEndpoint(pt); + // init the log + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + + // init the nodeInitializer + addConfig(std::string(NODE_PEM_NAME)); + NodeService_LOG(INFO) << LOG_DESC("initService: init the node"); + m_nodeInitializer = std::make_shared(_configPath); + auto privateKeyPath = tars::ServerConfig::BasePath + "/" + std::string(NODE_PEM_NAME); + NodeService_LOG(INFO) << LOG_DESC("generate the node private key path: ") << privateKeyPath; + m_nodeInitializer->config()->setPrivateKeyPath(privateKeyPath); + + m_nodeInitializer->init(ppc::protocol::NodeArch::PRO); + NodeService_LOG(INFO) << LOG_DESC("initService: init the node success"); + + auto config = m_nodeInitializer->config(); + + // init the gateway + auto gatewayServiceName = config->gatewayServiceName(); + auto endPoints = config->getServiceEndPointsByName(gatewayServiceName); + NodeService_LOG(INFO) << LOG_DESC("initGateway") << LOG_KV("serviceName", gatewayServiceName) + << LOG_KV("endPointSize", endPoints.size()); + auto gatewayPrx = + createServantProxy(true, gatewayServiceName, toTarsEndPoints(endPoints)); + auto gateway = std::make_shared( + gatewayServiceName, gatewayPrx, config->holdingMessageMinutes()); + NodeService_LOG(INFO) << LOG_DESC("initGateway success"); + + // addservant for the front-service + auto frontInitializer = m_nodeInitializer->frontInitializer(); + FrontServiceParam frontParam{frontInitializer}; + addServantWithParams( + getProxyDesc(ppc::protocol::FRONT_SERVANT_NAME), frontParam); + // get the endpoint for front-service-object + auto ret = + getEndPointDescByAdapter(this, ppc::protocol::FRONT_SERVANT_NAME, ppcConfig->endpoint()); + if (!ret.first) + { + throw std::runtime_error("get load endpoint for front-service-object information failed"); + } + + std::string selfEndPoint = ret.second; + NodeService_LOG(INFO) << LOG_DESC("get local-endpoint for the front-service-object success") + << LOG_KV("endPoint", selfEndPoint); + frontInitializer->front()->setSelfEndPoint(selfEndPoint); + // set the gateway into front + m_nodeInitializer->frontInitializer()->front()->setGatewayInterface(gateway); + + // init the rpc + NodeService_LOG(INFO) << LOG_DESC("init the rpc"); + // load the rpc config + // not specify the certPath in air-mode + config->loadRpcConfig(nullptr, pt); + // init RpcStatusInterface + RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(gateway); + m_nodeInitializer->frontInitializer()->setRpcStatus(rpcStatusInterface); + auto rpcFactory = std::make_shared(config->agencyID()); + m_rpc = rpcFactory->buildRpc(config); + m_rpc->setRpcStorage(rpcStatusInterface); + m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); + + m_nodeInitializer->registerRpcHandler(m_rpc); + NodeService_LOG(INFO) << LOG_DESC("init the rpc success"); + + // start the node + NodeService_LOG(INFO) << LOG_DESC("start the node"); + m_nodeInitializer->start(); + NodeService_LOG(INFO) << LOG_DESC("start the node success"); + + NodeService_LOG(INFO) << LOG_DESC("start the rpc"); + m_rpc->start(); + NodeService_LOG(INFO) << LOG_DESC("start the rpc success"); +} \ No newline at end of file diff --git a/cpp/ppc-main/pro-node/ProNodeServiceApp.h b/cpp/ppc-main/pro-node/ProNodeServiceApp.h new file mode 100644 index 00000000..13883c62 --- /dev/null +++ b/cpp/ppc-main/pro-node/ProNodeServiceApp.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProNodeServiceApp.h + * @author: yujiechen + * @date 2022-11-25 + */ +#pragma once +#include "libinitializer/Initializer.h" +#include +#include + +#define NodeService_LOG(LEVEL) BCOS_LOG(LEVEL) << "[NodeServiceApp]" + +namespace ppc::rpc +{ +class Rpc; +} +namespace ppctars +{ +class ProNodeServiceApp : public tars::Application +{ +public: + ProNodeServiceApp() {} + ~ProNodeServiceApp() override{}; + + void destroyApp() override; + void initialize() override; + +protected: + virtual void initService(std::string const& _configPath); + +private: + std::string m_configPath; + bcos::BoostLogInitializer::Ptr m_logInitializer; + ppc::initializer::Initializer::Ptr m_nodeInitializer; + // TODO: rpc support pro-mode + std::shared_ptr m_rpc; +}; +} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-main/pro-node/main.cpp b/cpp/ppc-main/pro-node/main.cpp new file mode 100644 index 00000000..881c4304 --- /dev/null +++ b/cpp/ppc-main/pro-node/main.cpp @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: yujiechen + * @date 2022-11-25 + */ +#include "ProNodeServiceApp.h" +#include "libhelper/CommandHelper.h" + +using namespace ppctars; + +int main(int argc, char* argv[]) +{ + try + { + ppc::initAppCommandLine(argc, argv); + ProNodeServiceApp app; + app.main(argc, argv); + app.waitForShutdown(); + return 0; + } + catch (std::exception& e) + { + cerr << "ppc-pro-node std::exception:" << boost::diagnostic_information(e) << std::endl; + } + catch (...) + { + cerr << "ppc-pro-node unknown exception." << std::endl; + } + return -1; +} diff --git a/cpp/ppc-mpc/CMakeLists.txt b/cpp/ppc-mpc/CMakeLists.txt new file mode 100644 index 00000000..cdb60d56 --- /dev/null +++ b/cpp/ppc-mpc/CMakeLists.txt @@ -0,0 +1,9 @@ +project(ppc-mpc VERSION ${VERSION}) +add_subdirectory(src) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() + diff --git a/cpp/ppc-mpc/src/CMakeLists.txt b/cpp/ppc-mpc/src/CMakeLists.txt new file mode 100644 index 00000000..1f218fe1 --- /dev/null +++ b/cpp/ppc-mpc/src/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +add_library(${MPC_TARGET} ${SOURCES}) +target_link_libraries(${MPC_TARGET} PUBLIC ${IO_TARGET}) diff --git a/cpp/ppc-mpc/src/Common.h b/cpp/ppc-mpc/src/Common.h new file mode 100644 index 00000000..38a7d492 --- /dev/null +++ b/cpp/ppc-mpc/src/Common.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: caryliao + * @date 2023-03-24 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +#define MPC_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("MPC") + +namespace ppc::mpc +{ + +DERIVE_PPC_EXCEPTION(MpcCompilerNotExistException); +DERIVE_PPC_EXCEPTION(RunMpcFailException); +DERIVE_PPC_EXCEPTION(OpenPipeFailException); + +struct JobInfo +{ + std::string jobId; + bool mpcNodeUseGateway; + std::string receiverNodeIp; + int mpcNodeDirectPort; + int participantCount; + int selfIndex; + bool isMalicious; + int bitLength; + std::string inputFileName; + std::string outputFileName; + std::string gatewayEngineEndpoint; +}; + +const int MPC_SUCCESS = 0; +const int MPC_FAILED = -1; +const std::string PATH_SEPARATOR = "/"; +const std::string MPC_RELATIVE_PATH = "/Programs/Source/"; +const std::string MPC_ALGORITHM_FILE_SUFFIX = ".mpc"; +const std::string MPC_ALGORITHM_COMPILER = "compile.py"; +const std::string MPC_PREPARE_FILE = "mpc_prepare.csv"; + +enum MpcBinaryType +{ + Hemi, + Replicated, + Shamir, + Mascot +}; +const std::string MPC_BINARY_NAME[] = { + "hemi-party.x", "replicated-ring-party.x", "shamir-party.x", "mascot-party.x"}; + +} // namespace ppc::mpc diff --git a/cpp/ppc-mpc/src/MPCService.cpp b/cpp/ppc-mpc/src/MPCService.cpp new file mode 100644 index 00000000..6f071e5a --- /dev/null +++ b/cpp/ppc-mpc/src/MPCService.cpp @@ -0,0 +1,426 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MPCService.cpp + * @author: caryliao + * @date 2023-03-28 + */ + +#include "MPCService.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include "ppc-storage/src/FileStorageFactoryImpl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ppc; +using namespace ppc::mpc; +using namespace bcos; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::tools; +using namespace ppc::storage; + + +void MPCService::doRun(Json::Value const& request, Json::Value& response) +{ + auto startT = utcSteadyTime(); + try + { // 0 get jobInfo and make command + auto jobInfo = paramsToJobInfo(request); + std::string mpcCmd; + makeCommand(mpcCmd, jobInfo); + + std::string hdfsPathPrefix = + m_mpcConfig.datasetHDFSPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR; + std::string localPathPrefix = + m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR; + + // 1 download algorithm file + std::string algorithmFileHdfsPath = + hdfsPathPrefix + jobInfo.jobId + MPC_ALGORITHM_FILE_SUFFIX; + std::string mpcRootPath = m_mpcConfig.mpcRootPathNoGateway; + if (jobInfo.mpcNodeUseGateway) + { + mpcRootPath = m_mpcConfig.mpcRootPath; + } + std::string algorithmFileLocalPath = + mpcRootPath + MPC_RELATIVE_PATH + jobInfo.jobId + MPC_ALGORITHM_FILE_SUFFIX; + if (!boost::filesystem::exists(algorithmFileLocalPath)) + { + auto lineReader1 = + initialize_lineReader(jobInfo, algorithmFileHdfsPath, DataResourceType::HDFS); + auto lineWriter1 = + initialize_lineWriter(jobInfo, algorithmFileLocalPath, DataResourceType::FILE); + readAndSaveFile(lineReader1, lineWriter1); + } + + // 2 download dataset file + std::string datasetFileHdfsPath = hdfsPathPrefix + jobInfo.inputFileName; + std::string datasetFileLocalPath = localPathPrefix + jobInfo.inputFileName; + if (!boost::filesystem::exists(datasetFileLocalPath)) + { + auto lineReader2 = + initialize_lineReader(jobInfo, datasetFileHdfsPath, DataResourceType::HDFS); + auto lineWriter2 = + initialize_lineWriter(jobInfo, datasetFileLocalPath, DataResourceType::FILE); + readAndSaveFile(lineReader2, lineWriter2); + } + + // 3 run mpc job + int outExitStatus = MPC_SUCCESS; + std::string outResult; + execCommand(mpcCmd, outExitStatus, outResult); + std::string message = "run mpc job successfully"; + + // 4 upload result file + std::string resultFileHdfsPath = hdfsPathPrefix + jobInfo.outputFileName; + std::string resultFileLocalPath = localPathPrefix + jobInfo.outputFileName; + writeStringToFile(outResult, resultFileLocalPath); + + auto lineReader3 = + initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE); + auto lineWriter3 = + initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS); + readAndSaveFile(lineReader3, lineWriter3); + + if (outExitStatus != MPC_SUCCESS) + { + message = "run mpc job failed"; + MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC(message); + BOOST_THROW_EXCEPTION(RunMpcFailException() << errinfo_comment(message)); + } + else + { + MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC(message); + response["code"] = MPC_SUCCESS; + response["message"] = "success"; + } + if (boost::filesystem::exists(localPathPrefix)) + { + boost::filesystem::remove_all(localPathPrefix); + } + } + catch (const std::exception& e) + { + const std::string diagnostic_information = std::string(boost::diagnostic_information(e)); + MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC("run mpc job failed:") + << LOG_DESC(diagnostic_information); + response["code"] = MPC_FAILED; + response["message"] = diagnostic_information; + } + MPC_LOG(INFO) << LOG_DESC("run mpc") << LOG_KV("timecost(ms)", utcSteadyTime() - startT); +} + +void MPCService::runMpcRpc(Json::Value const& request, RespFunc func) +{ + Json::Value response; + doRun(request, response); + func(nullptr, std::move(response)); +} + +void MPCService::doKill(Json::Value const& request, Json::Value& response) +{ + auto startT = utcSteadyTime(); + try + { + auto jobId = request["jobId"].asString(); + MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("jobId", jobId); + + std::string killCmd = "ps -ef |grep mpc | grep " + jobId + + " | grep -v grep | awk '{print $2}' | xargs kill -9"; + MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("killCmd", killCmd); + + int outExitStatus = 0; + std::string outResult; + execCommand(killCmd, outExitStatus, outResult); + std::string message = ""; + if (outExitStatus == 0) + { + message = "Kill mpc job successfully"; + MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_DESC(message); + response["code"] = MPC_SUCCESS; + response["message"] = "success"; + } + else + { + message = "Kill mpc job failed"; + MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_DESC(message); + response["code"] = MPC_FAILED; + response["message"] = message; + } + } + catch (const std::exception& e) + { + const std::string diagnostic_information = std::string(boost::diagnostic_information(e)); + MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_DESC("kill mpc job failed:") + << LOG_DESC(diagnostic_information); + response["code"] = MPC_FAILED; + response["message"] = diagnostic_information; + } + MPC_LOG(INFO) << LOG_DESC("kill mpc job") << LOG_KV("timecost(ms)", utcSteadyTime() - startT); +} + +void MPCService::killMpcRpc(Json::Value const& request, RespFunc func) +{ + Json::Value response; + doKill(request, response); + func(nullptr, std::move(response)); +} + +void MPCService::writeStringToFile(const std::string& content, const std::string& filePath) +{ + std::ostringstream buffer; + buffer << content; + + std::ofstream file(filePath, std::ios::out | std::ios::trunc); + file << buffer.str(); +} + +void MPCService::readAndSaveFile(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter) +{ + uint64_t lineSize = 0; + int64_t readPerBatchLines = m_mpcConfig.readPerBatchLines; + while (true) + { + // batch read dataset line + auto dataBatch = lineReader->next(readPerBatchLines); + if (!dataBatch) + { + break; + } + lineSize += dataBatch->size(); + lineWriter->writeLine(dataBatch, DataSchema::String, "\n"); + } + lineWriter->close(); + MPC_LOG(INFO) << LOG_DESC("save file ok") << LOG_KV("file lines", lineSize); +} + +LineReader::Ptr MPCService::initialize_lineReader( + const JobInfo& jobInfo, const std::string& readerFilePath, DataResourceType type) +{ + auto factory = std::make_shared(); + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, factory, nullptr); + const auto dataResource = std::make_shared(); + dataResource->setResourceID(jobInfo.jobId); + const auto dataResourceDesc = std::make_shared(); + dataResourceDesc->setType((int)type); + if ((int)type == (int)DataResourceType::HDFS) + { + dataResourceDesc->setFileStorageConnectOption(m_storageConfig.fileStorageConnectionOpt); + } + dataResourceDesc->setPath(readerFilePath); + dataResource->setDesc(dataResourceDesc); + return dataResourceLoader->loadReader(dataResource->desc()); +} + +LineWriter::Ptr MPCService::initialize_lineWriter( + const JobInfo& jobInfo, const std::string& writerFilePath, DataResourceType type) +{ + auto factory = std::make_shared(); + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, factory, nullptr); + const auto dataResource = std::make_shared(); + dataResource->setResourceID(jobInfo.jobId); + const auto dataResourceDesc = std::make_shared(); + dataResourceDesc->setType((int)type); + + if ((int)type == (int)DataResourceType::HDFS) + { + dataResourceDesc->setFileStorageConnectOption(m_storageConfig.fileStorageConnectionOpt); + } + dataResourceDesc->setPath(writerFilePath); + dataResource->setOutputDesc(dataResourceDesc); + return dataResourceLoader->loadWriter(dataResource->outputDesc()); +} + + +void MPCService::setMPCConfig(MPCConfig const& mpcConfig) +{ + m_mpcConfig = mpcConfig; +} + +void MPCService::setStorageConfig(StorageConfig const& storageConfig) +{ + m_storageConfig = storageConfig; +} + +JobInfo MPCService::paramsToJobInfo(const Json::Value& params) +{ + try + { + JobInfo jobInfo; + jobInfo.jobId = params["jobId"].asString(); + jobInfo.mpcNodeUseGateway = params["mpcNodeUseGateway"].asBool(); + jobInfo.receiverNodeIp = params["receiverNodeIp"].asString(); + jobInfo.mpcNodeDirectPort = params["mpcNodeDirectPort"].asInt(); + jobInfo.participantCount = params["participantCount"].asInt(); + jobInfo.selfIndex = params["selfIndex"].asInt(); + jobInfo.isMalicious = params["isMalicious"].asBool(); + jobInfo.bitLength = params["bitLength"].asInt(); + jobInfo.inputFileName = params["inputFileName"].asString(); + jobInfo.outputFileName = params["outputFileName"].asString(); + jobInfo.gatewayEngineEndpoint = params["gatewayEngineEndpoint"].asString(); + return jobInfo; + } + catch (const std::exception& e) + { + BOOST_THROW_EXCEPTION( + InvalidParam() << errinfo_comment( + "invalid params:" + std::string(boost::diagnostic_information(e)))); + } +} + +void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo) +{ + std::string mpcRootPath = m_mpcConfig.mpcRootPath; + if (jobInfo.mpcNodeUseGateway) + { + MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] use gateway to connect node"); + } + else + { + mpcRootPath = m_mpcConfig.mpcRootPathNoGateway; + MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] direct connect node"); + } + int r = chdir(mpcRootPath.c_str()); + if (r == 0) + { + MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] change dir ok"); + } + std::string compileFilePath = mpcRootPath + PATH_SEPARATOR + MPC_ALGORITHM_COMPILER; + int participantCount = jobInfo.participantCount; + bool isMalicious = jobInfo.isMalicious; + std::string mpcBinFileName; + std::string compileOption; + getMpcProtocol(participantCount, isMalicious, mpcBinFileName, compileOption); + + if (!boost::filesystem::exists(compileFilePath)) + { + BOOST_THROW_EXCEPTION(MpcCompilerNotExistException() + << errinfo_comment("compile file not exist:" + compileFilePath)); + } + if (jobInfo.mpcNodeUseGateway) + { + cmd = "export LD_LIBRARY_PATH=$PPC_MPC_LIB && python "; + } + else + { + cmd = "export LD_LIBRARY_PATH=$PPC_MPC_NO_GATEWAY_LIB && python "; + } + std::string compileCmd = compileFilePath + " " + compileOption + +" " + + std::to_string(jobInfo.bitLength) + " " + jobInfo.jobId; + cmd += compileCmd + " && "; + cmd += mpcRootPath + PATH_SEPARATOR + mpcBinFileName + " "; + cmd += std::to_string(jobInfo.selfIndex) + " "; + cmd += jobInfo.jobId + " "; + if (jobInfo.mpcNodeUseGateway) + { + cmd += "-gateway " + jobInfo.gatewayEngineEndpoint + " "; + cmd += "-ID " + jobInfo.jobId.substr(jobInfo.jobId.length() - 8, 8) + " "; + } + else + { + cmd += "-h " + jobInfo.receiverNodeIp + " "; + cmd += "-pn " + std::to_string(jobInfo.mpcNodeDirectPort) + " "; + } + if (jobInfo.mpcNodeUseGateway && !jobInfo.isMalicious && jobInfo.participantCount >= 3) + { + cmd += "-u "; + } + std::string inputFilePath = + m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR + MPC_PREPARE_FILE; + cmd += "-IF " + inputFilePath + " "; + if (mpcBinFileName != MPC_BINARY_NAME[MpcBinaryType::Replicated]) + { + cmd += "-N " + std::to_string(jobInfo.participantCount) + " "; + } + MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand]") << LOG_KV("mpcCmd", cmd); +} + +void MPCService::getMpcProtocol(const int participantCount, const bool isMalicious, + std::string& mpcBinFileName, std::string& compileOption) +{ + if (participantCount <= 1) + { + BOOST_THROW_EXCEPTION( + InvalidParam() << errinfo_comment("getMpcProtocol: participantCount need at least 2")); + } + if (isMalicious == true) + { + mpcBinFileName = MPC_BINARY_NAME[MpcBinaryType::Mascot]; + compileOption = "-F"; + } + else + { + if (participantCount == 2) + { + mpcBinFileName = MPC_BINARY_NAME[MpcBinaryType::Hemi]; + compileOption = "-F"; + } + else if (participantCount == 3) + { + mpcBinFileName = MPC_BINARY_NAME[MpcBinaryType::Replicated]; + compileOption = "-R"; + } + else + { + mpcBinFileName = MPC_BINARY_NAME[MpcBinaryType::Shamir]; + compileOption = "-F"; + } + } +} + +void MPCService::execCommand(const std::string cmd, int& outExitStatus, std::string& outResult) +{ + try + { + auto pPipe = popen((cmd + " 2>&1").c_str(), "r"); + if (pPipe == nullptr) + { + BOOST_THROW_EXCEPTION(OpenPipeFailException() << errinfo_comment("open pipe fail")); + } + std::array buffer; + while (not std::feof(pPipe)) + { + auto bytes = std::fread(buffer.data(), 1, buffer.size(), pPipe); + outResult.append(buffer.data(), bytes); + } + auto rc = pclose(pPipe); + if (WIFEXITED(rc)) + { + outExitStatus = WEXITSTATUS(rc); + } + MPC_LOG(INFO) << LOG_DESC("[MPCService][execCommand]") + << LOG_KV("outExitStatus", outExitStatus); + } + catch (const std::exception& e) + { + BOOST_THROW_EXCEPTION( + RunMpcFailException() << errinfo_comment( + "invalid params:" + std::string(boost::diagnostic_information(e)))); + } +} diff --git a/cpp/ppc-mpc/src/MPCService.h b/cpp/ppc-mpc/src/MPCService.h new file mode 100644 index 00000000..9827d6bd --- /dev/null +++ b/cpp/ppc-mpc/src/MPCService.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MPCService.h + * @author: caryliao + * @date 2023-03-24 + */ +#pragma once +#include "Common.h" +#include "ppc-io/src/FileLineReader.h" +#include "ppc-io/src/FileLineWriter.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::io; +using namespace ppc::rpc; +using namespace ppc::tools; + +namespace ppc::mpc +{ +class MPCService +{ +public: + using Ptr = std::shared_ptr; + MPCService() = default; + virtual ~MPCService() = default; + + void runMpcRpc(Json::Value const& request, RespFunc func); + void killMpcRpc(Json::Value const& request, RespFunc func); + void setMPCConfig(MPCConfig const& mpcConfig); + void setStorageConfig(StorageConfig const& storageConfig); + + JobInfo paramsToJobInfo(const Json::Value& params); + void makeCommand(std::string& cmd, const JobInfo& jobInfo); + void getMpcProtocol(const int participantCount, const bool isMalicious, + std::string& mpcBinFileName, std::string& compileOption); + void doRun(Json::Value const& request, Json::Value& response); + void doKill(Json::Value const& request, Json::Value& response); + void execCommand(const std::string cmd, int& outExitStatus, std::string& outResult); + + void writeStringToFile(const std::string& content, const std::string& filePath); + void readAndSaveFile(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter); + LineReader::Ptr initialize_lineReader(const JobInfo& jobInfo, const std::string& readerFilePath, + ppc::protocol::DataResourceType type); + LineWriter::Ptr initialize_lineWriter(const JobInfo& jobInfo, const std::string& writerFilePath, + ppc::protocol::DataResourceType type); + +private: + MPCConfig m_mpcConfig; + StorageConfig m_storageConfig; +}; +} // namespace ppc::mpc diff --git a/cpp/ppc-mpc/tests/CMakeLists.txt b/cpp/ppc-mpc/tests/CMakeLists.txt new file mode 100644 index 00000000..ef2135ec --- /dev/null +++ b/cpp/ppc-mpc/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-mpc) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) +target_link_libraries(${TEST_BINARY_NAME} ${MPC_TARGET} ${RPC_TARGET} ${BOOST_UNIT_TEST} TBB::tbb) +add_test(NAME test-mpc WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-mpc/tests/TestMPCService.cpp b/cpp/ppc-mpc/tests/TestMPCService.cpp new file mode 100644 index 00000000..51277660 --- /dev/null +++ b/cpp/ppc-mpc/tests/TestMPCService.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestmpcService->cpp + * @author: caryliao + * @date 2023-03-28 + */ +#include "ppc-mpc/src/Common.h" +#include "ppc-mpc/src/MPCService.h" +#include +#include + +using namespace ppc::mpc; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::io; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(MPCServiceTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(testMPCService) +{ + auto mpcService = std::make_shared(); + std::string configPath{"../../../ppc-mpc/tests/data/config.ini"}; + boost::property_tree::ptree pt; + boost::property_tree::read_ini(configPath, pt); + // load the mpc config + auto ppcConfig = std::make_shared(); + ppcConfig->loadMPCConfig(pt); + auto mpcConfig = ppcConfig->mpcConfig(); + auto storageConfig = ppcConfig->storageConfig(); + mpcService->setMPCConfig(mpcConfig); + mpcService->setStorageConfig(storageConfig); + + // test mpc config + BOOST_CHECK(mpcConfig.datasetHDFSPath == "/user/ppc/"); + BOOST_CHECK(mpcConfig.jobPath == "/data/app/ppc/mpc-job/"); + BOOST_CHECK(mpcConfig.mpcRootPath == "/ppc/scripts/ppc-mpc/"); + BOOST_CHECK(mpcConfig.mpcRootPathNoGateway == "/ppc/scripts/ppc-mpc-no-gateway/"); + BOOST_CHECK(mpcConfig.readPerBatchLines == 100000); + + // test getMpcProtocol + std::string mpcBinFileName, compileOption; + BOOST_CHECK_THROW( + mpcService->getMpcProtocol(1, false, mpcBinFileName, compileOption), InvalidParam); + + mpcService->getMpcProtocol(2, true, mpcBinFileName, compileOption); + BOOST_CHECK_EQUAL(mpcBinFileName, MPC_BINARY_NAME[MpcBinaryType::Mascot]); + BOOST_CHECK_EQUAL(compileOption, "-F"); + + mpcService->getMpcProtocol(2, false, mpcBinFileName, compileOption); + BOOST_CHECK_EQUAL(mpcBinFileName, MPC_BINARY_NAME[MpcBinaryType::Hemi]); + BOOST_CHECK_EQUAL(compileOption, "-F"); + + mpcService->getMpcProtocol(3, false, mpcBinFileName, compileOption); + BOOST_CHECK_EQUAL(mpcBinFileName, MPC_BINARY_NAME[MpcBinaryType::Replicated]); + BOOST_CHECK_EQUAL(compileOption, "-R"); + + mpcService->getMpcProtocol(4, false, mpcBinFileName, compileOption); + BOOST_CHECK_EQUAL(mpcBinFileName, MPC_BINARY_NAME[MpcBinaryType::Shamir]); + BOOST_CHECK_EQUAL(compileOption, "-F"); + + // test makeCommand + std::string cmd; + JobInfo jobInfo; + jobInfo.jobId = "j-123456789"; + jobInfo.mpcNodeUseGateway = true; + jobInfo.receiverNodeIp = ""; + jobInfo.mpcNodeDirectPort = 5899; + jobInfo.participantCount = 2; + jobInfo.selfIndex = 0; + jobInfo.isMalicious = false; + jobInfo.bitLength = 128; + jobInfo.inputFileName = "mpc_prepare.csv"; + jobInfo.outputFileName = "mpc_output.txt"; + jobInfo.gatewayEngineEndpoint = "127.0.0.1:6789"; + + mpcService->makeCommand(cmd, jobInfo); + BOOST_CHECK_EQUAL(cmd, + "/ppc/scripts/ppc-mpc//compile.py -F 128 j-123456789 && /ppc/scripts/ppc-mpc//hemi-party.x " + "0 j-123456789 -gateway 127.0.0.1:6789 -ID 23456789 -IF " + "/data/app/ppc/mpc-job//j-123456789/mpc_prepare.csv -N 2 "); + + + jobInfo.participantCount = 3; + mpcService->makeCommand(cmd, jobInfo); + BOOST_CHECK_EQUAL(cmd, + "/ppc/scripts/ppc-mpc//compile.py -R 128 j-123456789 && " + "/ppc/scripts/ppc-mpc//replicated-ring-party.x " + "0 j-123456789 -gateway 127.0.0.1:6789 -ID 23456789 -u -IF " + "/data/app/ppc/mpc-job//j-123456789/mpc_prepare.csv "); + + jobInfo.participantCount = 4; + mpcService->makeCommand(cmd, jobInfo); + BOOST_CHECK_EQUAL(cmd, + "/ppc/scripts/ppc-mpc//compile.py -F 128 j-123456789 && " + "/ppc/scripts/ppc-mpc//shamir-party.x 0 j-123456789 -gateway 127.0.0.1:6789 -ID 23456789 " + "-u -IF /data/app/ppc/mpc-job//j-123456789/mpc_prepare.csv -N 4 "); + + jobInfo.participantCount = 4; + jobInfo.isMalicious = true; + jobInfo.mpcNodeUseGateway = false; + jobInfo.receiverNodeIp = "192.168.1.2"; + mpcService->makeCommand(cmd, jobInfo); + BOOST_CHECK_EQUAL(cmd, + "/ppc/scripts/ppc-mpc-no-gateway//compile.py -F 128 j-123456789 && " + "/ppc/scripts/ppc-mpc-no-gateway//mascot-party.x 0 j-123456789 -h 192.168.1.2 -pn 5899 " + "-IF /data/app/ppc/mpc-job//j-123456789/mpc_prepare.csv -N 4 "); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-mpc/tests/data/config.ini b/cpp/ppc-mpc/tests/data/config.ini new file mode 100644 index 00000000..a4a9fb9c --- /dev/null +++ b/cpp/ppc-mpc/tests/data/config.ini @@ -0,0 +1,56 @@ +[agency] + ; the agency-id of self-party + id = agency + ; the agency info + +[mpc] + dataset_hdfs_path = /user/ppc/ + job_path = /data/app/ppc/mpc-job/ + mpc_root_path = /ppc/scripts/ppc-mpc/ + mpc_root_path_no_gateway = /ppc/scripts/ppc-mpc-no-gateway/ + read_per_batch_lines = 100000 + +[crypto] + sm_crypto = false + +[rpc] + listen_ip=0.0.0.0 + listen_port=5002 + thread_count=4 + ; ssl or sm ssl + sm_ssl=false + ; ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + +; [storage] +; host = 127.0.0.1 +; ; the mysqlx_port +; port = 3306 +; user = root +; password = 12345678 +; database = ppc15 + +[hdfs_storage] + ; the hdfs configuration + user = ppc + name_node = 127.0.0.1 + name_node_port = 9900 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 2000 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[log] + enable=true + ; print the log to std::cout or not, default print to the log files + enable_console_output = false + log_path=./log + ; info debug trace + level=info + ; MB + max_log_file_size=200 diff --git a/cpp/ppc-mpc/tests/main.cpp b/cpp/ppc-mpc/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-mpc/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-pir/CMakeLists.txt b/cpp/ppc-pir/CMakeLists.txt new file mode 100644 index 00000000..bf3b53b3 --- /dev/null +++ b/cpp/ppc-pir/CMakeLists.txt @@ -0,0 +1,8 @@ +project(ppc-pir VERSION ${VERSION}) +add_subdirectory(src) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() diff --git a/cpp/ppc-pir/src/BaseOT.cpp b/cpp/ppc-pir/src/BaseOT.cpp new file mode 100644 index 00000000..98530d80 --- /dev/null +++ b/cpp/ppc-pir/src/BaseOT.cpp @@ -0,0 +1,288 @@ +/** + * Copyright (C) 2021 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BaseOT.cpp + * @author: asherli + * @date 2023-03-13 + */ + +#include "BaseOT.h" +#include "Common.h" +#include "ppc-crypto-core/src/sym-crypto/OpenSSLAES.h" +#include "ppc-framework/crypto/SymCrypto.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +using namespace ppc::crypto; +using namespace ppc::protocol; +using namespace ppc::pir; +using namespace bcos; + +BaseOT::BaseOT(EccCrypto::Ptr _ecc, Hash::Ptr _hash) + : m_ecc(std::move(_ecc)), m_hash(std::move(_hash)) +{ + m_acculator = std::make_shared(OpenSSLAES::AESType::AES128); +} + +std::string genRandom(const int len) +{ + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + std::string tmp_s; + tmp_s.reserve(len); + + for (int i = 0; i < len; ++i) + { + tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)]; + } + + return tmp_s; +} + + +bool isPrefix(bcos::bytes prefixBytes, bcos::bytes targetBytes) +{ + int len1 = prefixBytes.size(); + int len2 = targetBytes.size(); + + // If prefix is larger than target, then prefix can't be a prefix anymore. + if (len1 > len2) + { + return false; + } + + // Check if the first len1 bytes of "targetBytes" are equals to "prefixBytes". + for (int i = 0; i < len1; i++) + { + if (prefixBytes[i] != targetBytes[i]) + { + return false; + } + } + + return true; +} + + +SenderMessage BaseOT::senderGenerateCipher( + const bcos::bytes _choicesMessage, const uint32_t obfuscatedOrder) + +{ + PIR_LOG(INFO) << LOG_DESC("senderGenerateCipher"); + + bcos::bytes scalarBlidingA = m_ecc->generateRandomScalar(); + bcos::bytes scalarBlidingB = m_ecc->generateRandomScalar(); + bcos::bytes pointX = m_ecc->mulGenerator(scalarBlidingA); + bcos::bytes pointY = m_ecc->mulGenerator(scalarBlidingB); + bcos::bytes scalarC = m_ecc->scalarMul(scalarBlidingA, scalarBlidingB); + // PIR_LOG(INFO) << LOG_DESC("scalarBlidingA") << LOG_KV("scalarBlidingA", + // toHex(scalarBlidingA)); bcos::bytes scalarC_B = m_ecc->scalarSub(scalarC, scalarBlidingB); + // PIR_LOG(INFO) << LOG_DESC("scalarC_B") << LOG_KV("scalarC_B", toHex(scalarC_B)); + + + bcos::bytes pointZ = m_ecc->mulGenerator(scalarC); + // std::vector z0List; + // std::vector sendObfuscatedHash; + uint32_t reviewLength = 0; + if (_choicesMessage.size() < obfuscatedOrder) + { + reviewLength = _choicesMessage.size(); + } + else + { + reviewLength = obfuscatedOrder; + } + // 取vector的前k个字符 + bcos::bytes sendObfuscatedHash( + _choicesMessage.begin(), _choicesMessage.begin() + obfuscatedOrder); + bcos::bytes scalarId = m_ecc->hashToScalar(_choicesMessage); + PIR_LOG(INFO) << LOG_DESC("scalarId") << LOG_KV("scalarId", toHex(scalarId)); + + bcos::bytes scalarCSub = m_ecc->scalarSub(scalarC, scalarId); + bcos::bytes z0 = m_ecc->mulGenerator(scalarCSub); + + + SenderMessage senderMessage; + senderMessage.pointX = pointX; + senderMessage.pointY = pointY; + senderMessage.scalarBlidingA = scalarBlidingA; + senderMessage.scalarBlidingB = scalarBlidingB; + senderMessage.pointZ = z0; + senderMessage.sendObfuscatedHash = sendObfuscatedHash; + return senderMessage; +} + +ReceiverMessage BaseOT::receiverGenerateMessage(const bcos::bytes pointX, const bcos::bytes pointY, + const std::vector> messageKeypair, const bcos::bytes pointZ) +{ + PIR_LOG(INFO) << LOG_DESC("receiverGenerateMessage") + << LOG_KV("messageKeypair size", messageKeypair.size()); + PIR_LOG(INFO) << LOG_DESC("receiverGenerateMessage") << LOG_KV("pointX", toHex(pointX)) + << LOG_KV("pointY", toHex(pointY)) << LOG_KV("pointZ", toHex(pointZ)); + // std::vector encryptMessagePair(messageKeypair.size()); + // std::vector pointWList(messageKeypair.size()); + // std::vector encryptCipher(messageKeypair.size()); + std::vector encryptMessagePair; + std::vector pointWList; + std::vector encryptCipher; + for (uint32_t i = 0; i < messageKeypair.size(); ++i) + { + // PIR_LOG(TRACE) << LOG_DESC("round") << LOG_KV("i", i); + bcos::bytes scalarBlidingR = m_ecc->generateRandomScalar(); + bcos::bytes scalarBlidingS = m_ecc->generateRandomScalar(); + bcos::bytes pointWLeft = m_ecc->ecMultiply(pointX, scalarBlidingS); + bcos::bytes pointWRight = m_ecc->mulGenerator(scalarBlidingR); + bcos::bytes pointW = m_ecc->ecAdd(pointWLeft, pointWRight); + bcos::bytes pointKeyRight = m_ecc->ecMultiply(pointY, scalarBlidingR); + // std::cout << "pointW" << toHex(pointW) << std::endl; + pointWList.push_back(pointW); + bcos::bytes scalarId = m_ecc->hashToScalar(messageKeypair[i].first); + // PIR_LOG(INFO) << LOG_DESC("scalarId") << LOG_KV("scalarId", toHex(scalarId)); + + // std::cout << "iter->first" << std::string(messageKeypair[i].first.begin(), + // messageKeypair[i].first.end()) << std::endl; + + // point是33个字节uint_8 + bcos::bytes pointZRight = m_ecc->mulGenerator(scalarId); + // PIR_LOG(INFO) << LOG_DESC("pointZRight") << LOG_KV("pointZRight.size()", + // pointZRight.size()); + + // TODO: use random aes key 128bit + bcos::bytes message = messageKeypair[i].second; + // std::vector encryptMessageList; + bcos::bytes defaultPrefix(pir::DEFAULT_PREFIX.begin(), pir::DEFAULT_PREFIX.end()); + bcos::bytes randomKey = m_ecc->generateRandomScalar(); + + + // 截取defaultPrefix和randomKey的前16个字节拼接成一个新的bcos::bytes类型 + bcos::bytes newBytes; + bcos::bytes aesKey; + aesKey.insert(aesKey.end(), randomKey.begin(), randomKey.begin() + 16); + newBytes.insert(newBytes.end(), defaultPrefix.begin(), defaultPrefix.begin() + 16); + newBytes.insert(newBytes.end(), randomKey.begin(), randomKey.begin() + 16); + bcos::bytes encryptedData = m_acculator->encrypt( + SymCrypto::OperationMode::CBC, bcos::ref(aesKey), bcos::ref(m_iv), bcos::ref(message)); + encryptCipher.push_back(encryptedData); + // PIR_LOG(INFO) << LOG_DESC("aesKey") << LOG_KV("aesKey", toHex(aesKey)); + + bcos::bytes pointZi = m_ecc->ecAdd(pointZ, pointZRight); + bcos::bytes pointKeyLeft = m_ecc->ecMultiply(pointZi, scalarBlidingS); + bcos::bytes pointKey = m_ecc->ecAdd(pointKeyLeft, pointKeyRight); + // PIR_LOG(INFO) << LOG_DESC("pointKey") << LOG_KV("pointKey", toHex(pointKey)); + // PIR_LOG(INFO) << LOG_DESC("message") << LOG_KV("message", + // std::string(message.begin(),message.end())); + + if (pointKey.size() < newBytes.size()) + { + BOOST_THROW_EXCEPTION( + CipherLengthFailException() << errinfo_comment("encrypt ot newBytes fail")); + } + bcos::bytes encryptMessage; + for (uint32_t k = 0; k < newBytes.size(); ++k) + { + encryptMessage.push_back(newBytes[k] ^ pointKey[k]); + // encryptMessage.push_back(message[k]); + } + encryptMessagePair.push_back(encryptMessage); + } + PIR_LOG(INFO) << LOG_DESC("init result"); + ReceiverMessage result; + result.pointWList = pointWList; + result.encryptMessagePair = encryptMessagePair; + result.encryptCipher = encryptCipher; + PIR_LOG(INFO) << LOG_DESC("end ReceiverMessage"); + return result; +} + +bcos::bytes BaseOT::finishSender(const bcos::bytes scalarBlidingB, + const std::vector pointWList, const std::vector encryptMessagePair, + std::vector encryptCipher) +{ + PIR_LOG(INFO) << LOG_DESC("finishSender"); + // std::vector indexList; + std::vector decryptKeyList; + if (encryptMessagePair.size() != pointWList.size()) + { + PIR_LOG(ERROR) << LOG_DESC("check encryptMessagePair and pointWList fail") + << LOG_KV("encryptMessagePair.size", encryptMessagePair.size()) + << LOG_KV("pointWList.size", pointWList.size()); + BOOST_THROW_EXCEPTION(CipherLengthFailException() + << errinfo_comment("check sendObfuscatedHash and pointWList fail")); + } + + for (uint32_t i = 0; i < pointWList.size(); ++i) + { + bcos::bytes pointW = pointWList[i]; + PIR_LOG(INFO) << LOG_DESC("BaseOT::finishSender") << LOG_KV("pointW", toHex(pointW)); + + bcos::bytes messagePair = encryptMessagePair[i]; + bcos::bytes key = m_ecc->ecMultiply(pointW, scalarBlidingB); + PIR_LOG(INFO) << LOG_DESC("key") << LOG_KV("key", toHex(key)); + bcos::bytes decryptMessage; + for (uint32_t k = 0; k < messagePair.size(); ++k) + { + decryptMessage.push_back(messagePair[k] ^ key[k]); + // decryptMessage.push_back(messagePair[i][k]); + } + PIR_LOG(INFO) << LOG_DESC("decryptMessage") + << LOG_KV("decryptMessage", + std::string(decryptMessage.begin(), decryptMessage.end())); + if (isPrefix(bcos::bytes(DEFAULT_PREFIX.begin(), DEFAULT_PREFIX.end()), decryptMessage)) + { + // 截取bcos::bytes的16-32个字节 + bcos::bytes aesKey(decryptMessage.begin() + 16, decryptMessage.begin() + 32); + PIR_LOG(INFO) << LOG_DESC("aesKey") << LOG_KV("aesKey", toHex(aesKey)); + PIR_LOG(INFO) << LOG_DESC("encryptCipher") + << LOG_KV("encryptCipher[i]", toHex(encryptCipher[i])); + bcos::bytes decryptData(m_acculator->blockSize()); + OutputBuffer decryptedBytes{decryptData.data(), decryptData.size()}; + m_acculator->decrypt(&decryptedBytes, SymCrypto::OperationMode::CBC, bcos::ref(aesKey), + bcos::ref(m_iv), bcos::ref(encryptCipher[i])); + PIR_LOG(INFO) << LOG_DESC("decryptData") + << LOG_KV("decryptData", + std::string(decryptData.begin(), decryptData.end())); + + return decryptData; + } + } + + return bcos::bytes(); +} + + +std::vector> BaseOT::prepareDataset( + bcos::bytes sendObfuscatedHashPrefix, std::string datasetPath) +{ + PIR_LOG(INFO) << LOG_DESC("prepareDataset") + << LOG_KV( + "sendObfuscatedHashPrefix", std::string(sendObfuscatedHashPrefix.begin(), + sendObfuscatedHashPrefix.end())); + PIR_LOG(INFO) << LOG_DESC("prepareDataset") << LOG_KV("datasetPath", datasetPath); + + std::vector> messageKeypair = + pir::readInputsByMmapWithPrefix(datasetPath, sendObfuscatedHashPrefix); + + return messageKeypair; +} diff --git a/cpp/ppc-pir/src/BaseOT.h b/cpp/ppc-pir/src/BaseOT.h new file mode 100644 index 00000000..cf3888d7 --- /dev/null +++ b/cpp/ppc-pir/src/BaseOT.h @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BaseOT.h + * @author: asherli + * @date 2023-03-13 + */ + +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-framework/crypto/EccCrypto.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/crypto/SymCrypto.h" + +#include "ppc-framework/protocol/Protocol.h" +#include +#include +#include + + +namespace ppc::crypto +{ +struct SenderMessage +{ + std::string taskId; + // std::vector idIndexList; + // 私有保存的密钥 + bcos::bytes scalarBlidingA; + bcos::bytes scalarBlidingB; + // 发送的数据 + bcos::bytes pointX; + bcos::bytes pointY; + bcos::bytes pointZ; + // 筛选前缀 + bcos::bytes sendObfuscatedHash; +}; + + +struct ReceiverMessage +{ + std::string path; + std::vector pointWList; + // 加密的AES key, baseOT交互的结果 + std::vector encryptMessagePair; + // 加密的AES cipher + std::vector encryptCipher; +}; + +class BaseOT : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + BaseOT() = delete; + ~BaseOT() = default; + + BaseOT(EccCrypto::Ptr _ecc, Hash::Ptr _hash); + + + // sender use id_list = [id1,id2,...], generate bliding a, b + // calculate X = a * G, Y = b * G, c_id = a * b, z_delta = c_id * Gm z_0 = (c_id - id) * G + // z_0_list = [z_0_id1,z_0_id2,...] + // obfuscatedOrder披露多少个字符 + SenderMessage senderGenerateCipher( + const bcos::bytes _choicesMessage, const uint32_t obfuscatedOrder); + + // receiver generate random ri, si + // calculate wi = si * X + ri * G, zi = z0 + id * G + // key = si * zi + ri * Y, Ei = key xor mi + // mi is AES key for Enc(message) + ReceiverMessage receiverGenerateMessage(const bcos::bytes pointX, const bcos::bytes pointY, + const std::vector> messageKeypair, + const bcos::bytes pointZ); + + // calculate b * G xor Ei = key + // AES decrypt message + bcos::bytes finishSender(const bcos::bytes scalarBlidingB, + const std::vector pointWList, + const std::vector encryptMessagePair, std::vector encryptCipher); + + std::vector> prepareDataset( + bcos::bytes sendObfuscatedHash, std::string datasetPath); + + +private: + EccCrypto::Ptr m_ecc; + Hash::Ptr m_hash; + bcos::bytes m_iv = bcos::bytes(0); + ppc::crypto::SymCrypto::Ptr m_acculator; +}; + +} // namespace ppc::crypto diff --git a/cpp/ppc-pir/src/CMakeLists.txt b/cpp/ppc-pir/src/CMakeLists.txt new file mode 100644 index 00000000..5f73920e --- /dev/null +++ b/cpp/ppc-pir/src/CMakeLists.txt @@ -0,0 +1,31 @@ +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-pir/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +file(GLOB_RECURSE TARS_INPUT "*.tars") + +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () + +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +add_library(${PIR_TARGET} ${SOURCES} ${OUT_TARS_H_LIST}) +target_include_directories(${PIR_TARGET} PUBLIC + $) + +target_link_libraries(${PIR_TARGET} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) diff --git a/cpp/ppc-pir/src/Common.h b/cpp/ppc-pir/src/Common.h new file mode 100644 index 00000000..77051444 --- /dev/null +++ b/cpp/ppc-pir/src/Common.h @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: asherli + * @date 2023-03-13 + */ +#pragma once +#include "ppc-framework/Common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PIR_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PIR") + +namespace ppc::pir +{ +// const std::string DEFAULT_MESSAGE = "message not found"; +const std::string DEFAULT_PREFIX = "PPC_DEFAULT_MESG"; + + + +enum class OTPIRMessageType : uint8_t +{ + HELLO_RECEIVER = 0x01, + RESULTS = 0x02 +}; + +enum class OTPIRRetCode : int +{ + ON_EXCEPTION = -3000, + UNDEFINED_TASK_ROLE = -3001, + INVALID_TASK_PARAM = -3002 +}; + +inline bool isPrefixMatched(const bcos::bytes& _prefix, const char* begin, const char* end) +{ + size_t prefixLen = _prefix.size(); + if (end - begin < prefixLen) return false; + for (size_t i = 0; i < prefixLen; ++i) + { + if (_prefix[i] != *(begin + i)) + return false; + } + return true; +} + +// 从Task param解析 +struct PirTaskMessage { + std::string searchId; + std::string requestAgencyId; + std::string requestAgencyDataset; + uint32_t prefixLength; +}; + +// 通过前缀读取符合要求的行，写到buffer中 +// 我们约定id在每行最前 +inline std::vector> readInputsByMmapWithPrefix( + const std::string& _path, const bcos::bytes& _prefix) +{ + std::vector> _buffer; + int fd = open(_path.c_str(), O_RDONLY); + struct stat fs + { + }; + + if (fd == -1 || fstat(fd, &fs) == -1) + { + BOOST_THROW_EXCEPTION(std::runtime_error("open file error: " + _path)); + } + + char* buf = (char*)mmap(nullptr, fs.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (buf == MAP_FAILED) + { + close(fd); + BOOST_THROW_EXCEPTION(std::runtime_error("mmap file to memory error: " + _path)); + } + + char* buf_end = buf + fs.st_size; + + char *begin = buf, *end = buf, c; + while (true) + { + if (!(*end == '\r' || *end == '\n')) + { + if (++end < buf_end) + continue; + } + else if (1 + end < buf_end) + { + // see if we got "\r\n" or "\n\r" here + c = *(1 + end); + if ((c == '\r' || c == '\n') && c != *end) + ++end; + } + // 判断_prefix和buf中begin, end这一行的前缀是否相同 + if(isPrefixMatched(_prefix, begin, end)) + { + std::string st(begin, end); + std::stringstream ss(st); + std::string idIndex; + std::getline(ss, idIndex, ','); + // 将begin, end这一行的前缀是否相同按逗号分割，取逗号前的内容 + std::pair pairTmp; + pairTmp.first = bcos::bytes(idIndex.begin(), idIndex.end()); + pairTmp.second = bcos::bytes(begin, end); + _buffer.push_back(pairTmp); + // _buffer[bcos::bytes(idIndex.begin(), idIndex.end())] = bcos::bytes(begin, end); + // _buffer->push_back(bcos::bytes(begin, end)); + } + + if ((begin = ++end) >= buf_end) + break; + } + + munmap(buf, fs.st_size); + close(fd); + return _buffer; +} + +inline PirTaskMessage parseJson(std::string_view _param) +{ + PIR_LOG(TRACE) << LOG_BADGE("parseJson") LOG_KV("_param", _param); + Json::Reader reader; + Json::Value result; + if (!reader.parse(_param.begin(), _param.end(), result)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param: invalid json")); + } + PIR_LOG(TRACE) << LOG_BADGE("result type") LOG_KV("result", result.type()); + if (!result.isObject() || result.empty()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param must be object")); + } + PirTaskMessage taskMessage; + + if(!result.isMember("searchId")){ + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param searchId not found")); + } + if(!result.isMember("requestAgencyId")){ + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param requestAgencyId not found")); + } + if(!result.isMember("prefixLength")){ + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param prefixLength not found")); + } + // if(!result.isMember("requestAgencyDataset")){ + // BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, + // "invalid task param:: the param requestAgencyDataset not found")); + // } + taskMessage.searchId = result["searchId"].asString(); + taskMessage.requestAgencyId = result["requestAgencyId"].asString(); + // taskMessage.requestAgencyDataset = result["requestAgencyDataset"].asString(); + taskMessage.prefixLength = result.get("prefixLength", 0).asInt(); + // taskMessage.prefixLength = result['prefixLength'].asInt(); + // if(result.isMember("requestAgencyDataset")){ + // taskMessage.requestAgencyDataset = result["requestAgencyDataset"].asString(); + // } + return taskMessage; +} + + +DERIVE_PPC_EXCEPTION(CipherLengthFailException); +DERIVE_PPC_EXCEPTION(FIleDeleteFailException); +} // namespace ppc::pir \ No newline at end of file diff --git a/cpp/ppc-pir/src/OtPIRConfig.h b/cpp/ppc-pir/src/OtPIRConfig.h new file mode 100644 index 00000000..dd4d6bda --- /dev/null +++ b/cpp/ppc-pir/src/OtPIRConfig.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIConfig.h + * @author: shawnhe + * @date 2022-12-7 + */ + +#pragma once +#include + +#include + +#include "Common.h" +#include "ppc-crypto/src/oprf/EcdhOprf.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/crypto/Oprf.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/PSIConfig.h" + +using namespace ppc::psi; + +namespace ppc::pir +{ +class OtPIRConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + + OtPIRConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front, + ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes, + uint16_t _parallelism = 3, + const front::PPCMessageFactory::Ptr& _msgFactory = + std::make_shared()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::OT_PIR_2PC, _selfPartyID, std::move(_front), + _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes), + m_cryptoBox(std::move(_cryptoBox)), + m_threadPool(std::move(_threadPool)), + m_parallelism(_parallelism) + {} + + virtual ~OtPIRConfig() = default; + +public: + crypto::Hash::Ptr const& hash() const { return m_cryptoBox->hashImpl(); } + crypto::EccCrypto::Ptr const& eccCrypto() const { return m_cryptoBox->eccCrypto(); } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + uint16_t parallelism() const { return m_parallelism; } + +private: + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + bcos::ThreadPool::Ptr m_threadPool; + uint16_t m_parallelism; +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-pir/src/OtPIRFactory.h b/cpp/ppc-pir/src/OtPIRFactory.h new file mode 100644 index 00000000..caf29cd9 --- /dev/null +++ b/cpp/ppc-pir/src/OtPIRFactory.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIFactory.h + * @author: shawnhe + * @date 2022-12-7 + */ +#pragma once +#include "OtPIRImpl.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-tools/src/config/PPCConfig.h" + +using namespace ppc::psi; + +namespace ppc::pir +{ +class OtPIRFactory +{ +public: + using Ptr = std::shared_ptr; + OtPIRFactory() = default; + virtual ~OtPIRFactory() = default; + + virtual OtPIRImpl::Ptr buildOtPIR(std::string const& _selfParty, + ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader, + int _holdingMessageMinutes) + { + auto config = std::make_shared(_selfParty, std::move(_front), + std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), + _holdingMessageMinutes); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-pir/src/OtPIRImpl.cpp b/cpp/ppc-pir/src/OtPIRImpl.cpp new file mode 100644 index 00000000..3ee59301 --- /dev/null +++ b/cpp/ppc-pir/src/OtPIRImpl.cpp @@ -0,0 +1,562 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AYSService.cpp + * @author: asherli + * @date 2023-03-20 + */ + +#include "OtPIRImpl.h" +#include "BaseOT.h" +#include "Common.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-front/Common.h" +#include "OtPIR.h" +#include "ppc-tars-protocol/TarsSerialize.h" +#include + + +using namespace ppc; +using namespace bcos; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::pir; + + +OtPIRImpl::OtPIRImpl(const OtPIRConfig::Ptr& _config, unsigned _idleTimeMs) + : Worker("OT-PIR", _idleTimeMs), + TaskGuarder(_config, PSIAlgorithmType::OT_PIR_2PC, "OT-PIR-Timer"), + m_config(_config), + m_msgQueue(std::make_shared()), + m_ioService(std::make_shared()), + m_parallelism(m_config->parallelism()) +{ + m_senderThreadPool = std::make_shared( + "OT-PIR-Sender", std::thread::hardware_concurrency()); + m_ot = std::make_shared(m_config->eccCrypto(), m_config->hash()); +} + +void OtPIRImpl::start() +{ + if (m_started) + { + PIR_LOG(WARNING) << LOG_DESC("The OT-PIR has already been started"); + return; + } + PIR_LOG(INFO) << LOG_DESC("Start the OT-PIR"); + m_started = true; + + // start a thread to execute task + startWorking(); + + m_thread = std::make_shared([&] { + bcos::pthread_setThreadName("otPir_io_service"); + while (m_started) + { + try + { + m_ioService->run(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (m_started && m_ioService->stopped()) + { + m_ioService->restart(); + } + } + catch (std::exception& e) + { + FRONT_LOG(WARNING) << LOG_DESC("Exception in OT-PIR Thread:") + << boost::diagnostic_information(e); + } + } + PIR_LOG(INFO) << "OT-PIR exit"; + }); + + startPingTimer(); +} + +void OtPIRImpl::stop() +{ + if (!m_started) + { + return; + } + PIR_LOG(INFO) << LOG_DESC("Stop OT-PIR"); + m_started = false; + + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } + + if (m_senderThreadPool) + { + m_senderThreadPool->stop(); + } + if (m_ioService) + { + m_ioService->stop(); + } + // stop the thread + if (m_thread->get_id() != std::this_thread::get_id()) + { + m_thread->join(); + } + else + { + m_thread->detach(); + } + stopPingTimer(); + + PIR_LOG(INFO) << LOG_DESC("OT-PIR stopped"); +} + +// register to the front to get the message related to cm2020-psi +void OtPIRImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) +{ + try + { + m_msgQueue->push(_msg); + + // notify to handle the message + m_signal.notify_all(); + } + catch (std::exception const& e) + { + PIR_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + +void OtPIRImpl::onReceivedErrorNotification(const std::string& _taskID) +{ + // finish the task while the peer is failed + auto taskState = findPendingTask(_taskID); + if (taskState) + { + taskState->onPeerNotifyFinish(); + + wakeupWorker(); + } +} + +void OtPIRImpl::onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +{ + PIR_LOG(WARNING) << LOG_DESC("onSelfError") << LOG_KV("task", _taskID) + << LOG_KV("error", _error->errorMessage()) + << LOG_KV("noticePeer", _noticePeer); + + auto taskState = findPendingTask(_taskID); + if (!taskState) + { + return; + } + + auto result = std::make_shared(taskState->task()->id()); + result->setError(std::move(_error)); + taskState->onTaskFinished(result, _noticePeer); + + wakeupWorker(); +} + + +// ot-pir main processing function +// for ut to make this function public +void OtPIRImpl::executeWorker() +{ + checkFinishedTask(); + auto result = m_msgQueue->tryPop(c_popWaitMs); + if (result.first) + { + handleReceivedMessage(result.second); + return; + } + waitSignal(); +} + + +void OtPIRImpl::checkFinishedTask() +{ + std::set finishedTask; + { + bcos::WriteGuard l(x_pendingTasks); + if (m_pendingTasks.empty()) + { + return; + } + + for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();) + { + auto task = it->second; + if (task->finished()) + { + finishedTask.insert(it->first); + } + it++; + } + } + for (auto& taskID : finishedTask) + { + removeReceiver(taskID); + removeSender(taskID); + removePendingTask(taskID); + // TODO: 为什么这里要再调用一下执行任务 + // asyncRunTask(); + } +} + + +void OtPIRImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message) +{ + PIR_LOG(TRACE) << LOG_DESC("handleReceivedMessage") << printPPCMsg(_message); + + m_config->threadPool()->enqueue([self = weak_from_this(), _message]() { + try + { + auto pir = self.lock(); + if (!pir) + { + return; + } + switch (int(_message->messageType())) + { + case int(CommonMessageType::ErrorNotification): + { + pir->onReceivedErrorNotification(_message->taskID()); + break; + } + case int(CommonMessageType::PingPeer): + { + break; + } + case int(OTPIRMessageType::HELLO_RECEIVER): + { + pir->onHelloReceiver(_message); + break; + } + case int(OTPIRMessageType::RESULTS): + { + pir->onSnederResults(_message); + break; + } + default: + { + PIR_LOG(WARNING) + << LOG_DESC("unsupported messageType ") << unsigned(_message->messageType()); + break; + } + } + } + catch (std::exception const& e) + { + PIR_LOG(WARNING) + << LOG_DESC("handleReceivedMessage exception") + << LOG_KV("type", unsigned(_message->messageType())) << printPPCMsg(_message) + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + +void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) +{ + // 接收方不需要记录taskID + // if (m_taskState->taskDone()) + // { + // return; + // } + PIR_LOG(DEBUG) << LOG_BADGE("onHelloReceiver") << LOG_KV("taskID", _message->taskID()) + << LOG_KV("seq", _message->seq()) << LOG_KV("length", _message->length()); + ppctars::SenderMessageParams senderMessageParams; + ppctars::serialize::decode(*_message->data(), senderMessageParams); + // crypto::SenderMessage senderMessage; + // TODO: how to find my dataset + // m_taskState->setReader(io::LineReader::Ptr _reader, int64_t _readerParam) + + try { + // auto writer = m_taskState->reader(); + auto receiver = findReceiver(_message->taskID()); + auto path = receiver.path; + PIR_LOG(INFO) << LOG_BADGE("onHelloReceiver") << LOG_KV("taskID", _message->taskID()) << LOG_KV("requestAgencyDataset", path) << LOG_KV("sendObfuscatedHash", std::string(senderMessageParams.sendObfuscatedHash.begin(), senderMessageParams.sendObfuscatedHash.end())); + + auto messageKeypair = m_ot->prepareDataset(senderMessageParams.sendObfuscatedHash, path); + auto receiverMessage = m_ot->receiverGenerateMessage(senderMessageParams.pointX, senderMessageParams.pointY, messageKeypair, senderMessageParams.pointZ); + ppctars::ReceiverMessageParams receiverMessageParams; + receiverMessageParams.encryptMessagePair = receiverMessage.encryptMessagePair; + receiverMessageParams.encryptCipher = receiverMessage.encryptCipher; + receiverMessageParams.pointWList = receiverMessage.pointWList; + // PIR_LOG(INFO) << LOG_BADGE("buildPPCMessage"); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR), + uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, + std::make_shared()); + message->setMessageType(uint8_t(OTPIRMessageType::RESULTS)); + ppctars::serialize::encode(receiverMessageParams, *message->data()); + // PIR_LOG(INFO) << LOG_BADGE("asyncSendMessage"); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); + auto endTask = std::make_shared(m_taskState->task()->id()); + m_taskState->onTaskFinished(endTask, true); + } + catch (bcos::Error const& e) { + PIR_LOG(WARNING) << LOG_DESC("onHelloReceiver exception") + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + onSelfError( + m_taskID, std::make_shared(e.errorCode(), e.errorMessage()), true); + } +} + +void OtPIRImpl::onSnederResults(ppc::front::PPCMessageFace::Ptr _message) +{ + PIR_LOG(DEBUG) << LOG_BADGE("onSnederResults") << LOG_KV("taskID", _message->taskID()) + << LOG_KV("seq", _message->seq()); + + ppctars::ReceiverMessageParams receiverMessageParams; + ppctars::serialize::decode(*_message->data(), receiverMessageParams); + crypto::SenderMessage senderMessage = findSender(_message->taskID()); + PIR_LOG(DEBUG) << LOG_BADGE("onSnederResults") << LOG_KV("scalarBlidingB", toHex(senderMessage.scalarBlidingB)) + << LOG_KV("pointWList Size", receiverMessageParams.pointWList.size()) + << LOG_KV("encryptCipher Size", receiverMessageParams.encryptCipher.size()) + << LOG_KV("encryptMessagePair Size", receiverMessageParams.encryptMessagePair.size()); + bcos::bytes result = m_ot->finishSender(senderMessage.scalarBlidingB, receiverMessageParams.pointWList, receiverMessageParams.encryptMessagePair, receiverMessageParams.encryptCipher); + saveResults(result); + auto endTask = std::make_shared(m_taskState->task()->id()); + m_taskState->onTaskFinished(endTask, true); +} + +void OtPIRImpl::asyncRunTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +{ + //TODO + PIR_LOG(INFO) << LOG_DESC("receive a task") << LOG_KV("taskID", _task->id()); + m_taskID = _task->id(); + addTask(_task, [self = weak_from_this(), taskID = _task->id(), _onTaskFinished]( + ppc::protocol::TaskResult::Ptr&& _result) { + auto result = std::move(_result); + _onTaskFinished(std::move(result)); + PIR_LOG(INFO) << LOG_DESC("finish a task") << LOG_KV("taskID", taskID); + auto pir = self.lock(); + if (!pir) + { + return; + } + pir->m_parallelism++; + }); + asyncRunTask(); +} + +void OtPIRImpl::asyncRunTask() +{ + PIR_LOG(INFO) << LOG_DESC("asyncRunTask") << LOG_KV("current semaphore", m_parallelism); + + if (m_parallelism <= 0) + { + return; + } + + std::pair taskPair; + { + bcos::UpgradableGuard l(x_taskQueue); + if (m_taskQueue.empty()) + { + return; + } + + bcos::UpgradeGuard ul(l); + taskPair = m_taskQueue.front(); + m_taskQueue.pop(); + m_parallelism--; + } + + auto task = taskPair.first; + + bcos::Error::Ptr error; + + // add pending task + m_taskState = + m_taskStateFactory->createTaskState(task, std::move(taskPair.second), false, m_config); + m_taskState->setPeerID(getPeerID(task)); + // m_taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), task]() { + // auto pir = self.lock(); + // if (!pir) + // { + // return; + // } + + // pir->noticePeerToFinish(task); + // }); + addPendingTask(m_taskState); + + try + { + auto dataResource = task->selfParty()->dataResource(); + auto role = task->selfParty()->partyIndex(); + + if (role == uint16_t(PartyType::Client)) + { + auto originData = m_taskState->task()->param(); + + PIR_LOG(TRACE) << LOG_DESC("originData") << LOG_KV("originData", originData); + PirTaskMessage taskMessage = parseJson(originData); + PIR_LOG(TRACE) << LOG_DESC("taskMessage") << LOG_KV("requestAgencyDataset", taskMessage.requestAgencyDataset) << LOG_KV("prefixLength", taskMessage.prefixLength) << LOG_KV("searchId", taskMessage.searchId); + auto writer = loadWriter(task->id(), dataResource, m_enableOutputExists); + m_taskState->setWriter(writer); + runSenderGenerateCipher(taskMessage); + } + else if (role == uint16_t(PartyType::Server)) + { + // server接受任务请求，初始化reader + PIR_LOG(TRACE) << LOG_DESC("Server init"); + crypto::ReceiverMessage receiverMessage; + receiverMessage.path = dataResource->desc()->path(); + addReceiver(receiverMessage); + // m_resource = dataResource; + // auto reader = loadReader(task->id(), dataResource, DataSchema::Bytes); + // m_taskState->setReader(reader, -1); + + } + else + { + PIR_LOG(WARNING) << LOG_DESC("undefined task role") << unsigned(role); + onSelfError(task->id(), + std::make_shared( + (int)OTPIRRetCode::UNDEFINED_TASK_ROLE, "undefined task role"), + true); + } + } + catch (bcos::Error const& e) + { + PIR_LOG(WARNING) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + onSelfError( + task->id(), std::make_shared(e.errorCode(), e.errorMessage()), true); + } + catch (const std::exception& e) + { + PIR_LOG(WARNING) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) + << LOG_KV("error", boost::diagnostic_information(e)); + onSelfError(task->id(), + std::make_shared((int)OTPIRRetCode::ON_EXCEPTION, + "exception caught while running task: " + boost::diagnostic_information(e)), + true); + } + + // notify the taskInfo to the front + error = m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + if (error && error->errorCode()) + { + onSelfError(task->id(), error, true); + } +} + +void OtPIRImpl::runSenderGenerateCipher(PirTaskMessage taskMessage) +{ + if (m_taskState->taskDone()) + { + return; + } + crypto::SenderMessage senderMessage = m_ot->senderGenerateCipher(bcos::bytes(taskMessage.searchId.begin(), taskMessage.searchId.end()), taskMessage.prefixLength); + ppctars::SenderMessageParams senderMessageParams; + senderMessageParams.pointX = senderMessage.pointX; + senderMessageParams.pointY = senderMessage.pointY; + senderMessageParams.pointZ = senderMessage.pointZ; + // senderMessageParams.requestAgencyDataset = taskMessage.requestAgencyDataset; + senderMessageParams.sendObfuscatedHash = senderMessage.sendObfuscatedHash; + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR), + uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, + std::make_shared()); + message->setMessageType(uint8_t(OTPIRMessageType::HELLO_RECEIVER)); + ppctars::serialize::encode(senderMessageParams, *message->data()); + addSender(senderMessage); + // PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) << LOG_KV("requestAgencyDataset", senderMessageParams.requestAgencyDataset); + PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) << LOG_KV("sendObfuscatedHash", std::string(senderMessageParams.sendObfuscatedHash.begin(), senderMessageParams.sendObfuscatedHash.end())); + // senderMessageParams.taskId = m_taskID; + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); +} + + +void OtPIRImpl::onReceiverTaskDone(bcos::Error::Ptr _error) +{ + if (m_taskState->taskDone()) + { + return; + } + + PIR_LOG(INFO) << LOG_BADGE("onReceiverTaskDone") LOG_KV("taskID", m_taskID); + + std::string message; + if (_error) + { + message = "\nStatus: FAIL\nMessage: " + _error->errorMessage(); + m_taskResult->setError(std::move(_error)); + } + m_taskState->onTaskFinished(m_taskResult, true); + + PIR_LOG(INFO) << LOG_BADGE("receiverTaskDone") << LOG_KV("taskID", m_taskID) + << LOG_KV("detail", message); +} + + +void OtPIRImpl::saveResults(bcos::bytes result) +{ + PIR_LOG(INFO) << LOG_BADGE("saveResults") LOG_KV("taskID", m_taskID); + try + { + DataBatch::Ptr finalResults = std::make_shared(); + finalResults->append(result); + m_taskState->writeLines(finalResults, DataSchema::Bytes); + } + catch (const std::exception& e) + { + PIR_LOG(WARNING) << LOG_KV("taskID", m_taskID) + << LOG_KV("error", boost::diagnostic_information(e)); + auto error = std::make_shared( + (int)OTPIRRetCode::ON_EXCEPTION, boost::diagnostic_information(e)); + onReceiverTaskDone(error); + } +} + diff --git a/cpp/ppc-pir/src/OtPIRImpl.h b/cpp/ppc-pir/src/OtPIRImpl.h new file mode 100644 index 00000000..5d3fea19 --- /dev/null +++ b/cpp/ppc-pir/src/OtPIRImpl.h @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file AYSService.h + * @author: asherli + * @date 2023-03-13 + */ +#pragma once +#include "Common.h" +// #include "OtPIR.h" +#include "ppc-framework/protocol/DataResource.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include +#include "ppc-framework/protocol/Task.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include +#include +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include "OtPIRConfig.h" +#include +#include "BaseOT.h" + + +using namespace bcos; +using namespace ppc; +using namespace ppc::rpc; +using namespace ppc::tools; +using namespace ppc::task; + +namespace ppc::pir +{ +class OtPIRImpl : public std::enable_shared_from_this, + public bcos::Worker, + public psi::TaskGuarder, // taskGuarder并不一定属于psi-framework + public TaskFrameworkInterface +// class OtPIRImpl : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + // OtPIRImpl() = default; + OtPIRImpl(const OtPIRConfig::Ptr& _config, unsigned _idleTimeMs = 0); + ~OtPIRImpl() override = default; + using OtPIRMsgQueue = bcos::ConcurrentQueue; + using OtPIRMsgQueuePtr = std::shared_ptr; + + // run task + void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + + void start() override; + void stop() override; + + // register to the front to get the message related to ot-pir + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; + + void onReceivedErrorNotification(const std::string& _taskID) override; + void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override; + + // ot-pir main processing function + // for ut to make this function public + void executeWorker() override; + + void handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message); + + void onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message); + // void onReceiveHelloSender(const ppc::front::PPCMessageFace::Ptr& _message); + + void onSnederResults(ppc::front::PPCMessageFace::Ptr _message); + const std::string& taskID() const { return m_taskID; } + + + // void makeBaseOt(Json::Value const& request, Json::Value& response); + // void encryptDataset(Json::Value const& request, Json::Value& response); + + // void setAYSConfig(AYSConfig const& aysConfig); + // std::vector prepareDataset(bcos::bytes sendObfuscatedHash, std::string datasetPath); + +protected: + void asyncRunTask(); + void checkFinishedTask(); + void runSenderGenerateCipher(PirTaskMessage taskMessage); + void saveResults(bcos::bytes result); + + void onReceiverTaskDone(bcos::Error::Ptr _error); + // void runReceiverGenerateMessage(ppctars::SenderMessageParams senderMessageParams); + // void runFinishSender(PirTaskMessage taskMessage); + + void addTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) + { + bcos::WriteGuard l(x_taskQueue); + m_taskQueue.push({std::move(_task), std::move(_onTaskFinished)}); + } + + crypto::ReceiverMessage findReceiver(const std::string& _taskID) + { + bcos::ReadGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + return it->second; + } + return crypto::ReceiverMessage(); + } + void addReceiver(crypto::ReceiverMessage _receiver) + { + bcos::WriteGuard l(x_receivers); + m_receivers[m_taskID] = _receiver; + } + void removeReceiver(const std::string& _taskID) + { + bcos::WriteGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + m_receivers.erase(it); + } + } + + crypto::SenderMessage findSender(const std::string& _taskID) + { + bcos::ReadGuard l(x_senders); + auto it = m_senders.find(_taskID); + if (it != m_senders.end()) + { + return it->second; + } + return crypto::SenderMessage(); + } + void addSender(crypto::SenderMessage _sender) + { + bcos::WriteGuard l(x_senders); + // m_taskID + m_senders[m_taskID] = _sender; + } + void removeSender(const std::string& _taskID) + { + bcos::WriteGuard l(x_senders); + auto it = m_senders.find(_taskID); + if (it != m_senders.end()) + { + m_senders.erase(it); + } + } + // allow the output-path exists, for ut + bool m_enableOutputExists = false; + // 为true时启动时会从配置中加载文件作为匹配源 + bool m_enableMemoryFile = false; + protocol::DataResource m_resource; + + +private: + void wakeupWorker() { m_signal.notify_all(); } + + bool m_started{false}; + std::shared_ptr m_thread; + std::shared_ptr m_ioService; + OtPIRConfig::Ptr m_config; + + std::shared_ptr m_senderThreadPool; + crypto::BaseOT::Ptr m_ot; + std::atomic m_parallelism; + OtPIRMsgQueuePtr m_msgQueue; + + boost::condition_variable m_signal; + + const int c_popWaitMs = 5; + boost::mutex x_signal; + // 查询方需要的数据，包含要发送的和私有保存的 + // crypto::SenderMessage senderMessage; + std::unordered_map m_senders; + mutable bcos::SharedMutex x_senders; + + // 数据方发送的数据，全部要发送 + // crypto::ReceiverMessage receiverMessage; + std::unordered_map m_receivers; + mutable bcos::SharedMutex x_receivers; + + + // TODO: 改为prefix-message的map 预处理的数据集 + std::vector> messageKeypair; + std::queue > m_taskQueue; + mutable bcos::SharedMutex x_taskQueue; + + // // setup的系统参数，分桶大小来决定披露前k个bit 目前版本由TASK先传入 + // const int obfuscation_order = 6; + std::string m_taskID; + TaskState::Ptr m_taskState; + protocol::TaskResult::Ptr m_taskResult; + + + + + void waitSignal() + { + boost::unique_lock l(x_signal); + m_signal.wait_for(l, boost::chrono::milliseconds(5)); + } + +}; +} // namespace ppc::pir \ No newline at end of file diff --git a/cpp/ppc-pir/src/protocol/OtPIR.tars b/cpp/ppc-pir/src/protocol/OtPIR.tars new file mode 100644 index 00000000..664e5ae4 --- /dev/null +++ b/cpp/ppc-pir/src/protocol/OtPIR.tars @@ -0,0 +1,15 @@ +module ppctars { + struct SenderMessageParams { + 1 require vector pointX; + 2 require vector pointY; + 3 require vector pointZ; + 4 require vector sendObfuscatedHash; + 5 require string requestAgencyDataset; + }; + + struct ReceiverMessageParams { + 1 require vector> pointWList; + 2 require vector> encryptMessagePair; + 3 require vector> encryptCipher; + }; +}; diff --git a/cpp/ppc-pir/tests/CMakeLists.txt b/cpp/ppc-pir/tests/CMakeLists.txt new file mode 100644 index 00000000..c7255ab0 --- /dev/null +++ b/cpp/ppc-pir/tests/CMakeLists.txt @@ -0,0 +1,12 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-pir) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) +# target_link_libraries(${PIR_TARGET} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) + +# target_link_libraries(${TEST_BINARY_NAME} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) +target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-ays WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-pir/tests/FakeOtPIRFactory.h b/cpp/ppc-pir/tests/FakeOtPIRFactory.h new file mode 100644 index 00000000..7393a0b8 --- /dev/null +++ b/cpp/ppc-pir/tests/FakeOtPIRFactory.h @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FakeOtPIRFactory.h + * @author: shawnhe + * @date 2022-12-19 + */ +#pragma once + +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-crypto/src/hash/BLAKE2bHash.h" +#include "ppc-crypto/src/hash/Sha512Hash.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "test-utils/FakeFront.h" +#include + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::psi; +using namespace ppc::pir; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; +using namespace ppc::tools; + +namespace ppc::test +{ + +class FakeOtPIRImpl : public OtPIRImpl +{ +public: + using Ptr = std::shared_ptr; + FakeOtPIRImpl(OtPIRConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : OtPIRImpl(_config, _idleTimeMs) + { + m_enableOutputExists = true; + } + ~FakeOtPIRImpl() override = default; +}; + +class FakeOtPIRFactory : public OtPIRFactory +{ +public: + using Ptr = std::shared_ptr; + + FakeOtPIRFactory() + : m_front(std::make_shared()), + m_dataResourceLoader(std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), + m_threadPool(std::make_shared("ot-pir", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared(hashImpl, ppc::protocol::ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } + + ~FakeOtPIRFactory() override = default; + + OtPIRImpl::Ptr createOtPIR(std::string const& _selfParty) + { + auto config = std::make_shared( + _selfParty, m_front, m_cryptoBox, m_threadPool, m_dataResourceLoader, 1); + + return std::make_shared(config); + } + + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + +private: + FakeFront::Ptr m_front; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; + CryptoBox::Ptr m_cryptoBox; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-pir/tests/TestBaseOT.cpp b/cpp/ppc-pir/tests/TestBaseOT.cpp new file mode 100644 index 00000000..7aa264e7 --- /dev/null +++ b/cpp/ppc-pir/tests/TestBaseOT.cpp @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestCEMService.cpp + * @author: asherli + * @date 2023-03-13 + */ +#include "FakeOtPIRFactory.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-pir/src/Common.h" +#include "ppc-pir/src/OtPIRImpl.h" +#include "ppc-pir/src/BaseOT.h" +#include +#include +#include +#include +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-crypto/src/ecc/EccCryptoFactoryImpl.h" +#include "ppc-crypto/src/hash/HashFactoryImpl.h" +#include "test-utils/TaskMock.h" + + +using namespace ppc::pir; +using namespace ppc::crypto; +using namespace ppc::pir; +using namespace bcos; +using namespace bcos::test; + + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(OtPIRest, TestPromptFixture) + + +BOOST_AUTO_TEST_CASE(testBaseOT) +{ + // 统计函数执行时间 + std::cout<< "testBaseOT" << std::endl; + + auto eccFactory = std::make_shared(); + auto hashFactory = std::make_shared(); + auto hash = hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512); + EccCrypto::Ptr ecc = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hash); + // auto aysService = std::make_shared(); + // auto aes = std::make_shared( + // OpenSSLAES::AESType::AES128, + // SymCrypto::OperationMode::CBC, _seed, bcos::bytes()); + + + std::string datasetPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; + // std::cout<< "aysService->prepareDataset" << std::endl; + std::string prefix = "testmsg1"; + bcos::bytes sendObfuscatedHash(prefix.begin(), prefix.end()); + + + uint32_t obfuscatedOrder = 10; + auto baseOT = std::make_shared(ecc, hash); + auto messageKeypair = baseOT->prepareDataset(sendObfuscatedHash, datasetPath); + // for(auto iter = messageKeypair.begin(); iter != messageKeypair.end(); ++iter) + // { + // std::string pairKey(iter->first.begin(), iter->first.end()); + // std::string pairValue(iter->second.begin(), iter->second.end()); + // // for(uint32_t i = 0; i < messageKeypair.size(); ++i) { + // // std::cout<< "pairKey:"<< pairKey << std::endl; + // // std::cout<< "pairValue:"<< pairValue << std::endl; + // } + auto start = std::chrono::high_resolution_clock::now(); + + std::string choice = "testmsg1100"; + // std::cout<< "baseOT->senderGenerateCipher" << std::endl; + auto senderMessage = baseOT->senderGenerateCipher(bcos::bytes(choice.begin(), choice.end()), obfuscatedOrder); + // std::cout<< "baseOT->receiverGenerateMessage" << std::endl; + auto receiverMessage = baseOT->receiverGenerateMessage(senderMessage.pointX, senderMessage.pointY, messageKeypair, senderMessage.pointZ); + + // std::cout<< "baseOT->finishSender" << std::endl; + auto result = baseOT->finishSender(senderMessage.scalarBlidingB, receiverMessage.pointWList, receiverMessage.encryptMessagePair, receiverMessage.encryptCipher); + + auto end = std::chrono::high_resolution_clock::now(); + + auto duration = std::chrono::duration_cast(end - start); + std::cout << "执行时间 time: " << duration.count() << " microseconds" << std::endl; + + if(result.size() == 0){ + std::cout<< "final result: message not found" << std::endl; + } + else { + std::cout<< "final result: " << std::string(result.begin(), result.end()) << std::endl; + } + // for(uint32_t i = 0; i < result.size(); ++i) + // { + // std::cout<< std::string(result[i].begin(), result[i].end()) << std::endl; + // } + + BOOST_CHECK(true); +} + + +void testOTPIR(FakeOtPIRFactory::Ptr _factory, OtPIRImpl::Ptr _sender, + OtPIRImpl::Ptr _receiver, ppc::protocol::Task::ConstPtr _senderPirTask, + ppc::protocol::Task::ConstPtr _receiverPirTask, + std::vector const& _expectedPIRResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + std::atomic flag = 0; + + _sender->asyncRunTask(_senderPirTask, [_senderPirTask, _expectedSuccess, _expectedErrorCode, + &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _senderPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); + } + flag++; + }); + _sender->start(); + + _receiver->asyncRunTask(_receiverPirTask, + [_receiverPirTask, _expectedSuccess, &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _receiverPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + flag++; + }); + _receiver->start(); + + // wait for the task finish and check + while (flag < 2) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + _sender->stop(); + _receiver->stop(); + + // if (_expectedSuccess && !_expectedPIRResult.empty()) + // { + // checkTaskPIRResult(_factory->resourceLoader(), _receiverPirTask, _expectedPIRResult.size(), + // _expectedPIRResult); + // } +} + + +void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, + bool _syncResults, PartyResource::Ptr _senderParty, PartyResource::Ptr _receiverParty, + std::vector const& _expectedPIRResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = "sender"; + auto senderPIR = factory->createOtPIR("sender"); + + // fake the receiver + std::string receiverAgencyName = _receiverParty->id(); + auto receiverPIR = factory->createOtPIR(receiverAgencyName); + + // register the server-pir into the front + factory->front()->registerOTPIR(senderAgencyName, senderPIR); + factory->front()->registerOTPIR(receiverAgencyName, receiverPIR); + + // trigger the pir task + auto senderPIRTask = std::make_shared(senderAgencyName); + senderPIRTask->setId(_taskID); + senderPIRTask->setParam(_params); + senderPIRTask->setSelf(_senderParty); + senderPIRTask->addParty(_receiverParty); + senderPIRTask->setSyncResultToPeer(_syncResults); + senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + auto receiverPIRTask = std::make_shared(receiverAgencyName); + receiverPIRTask->setId(_taskID); + receiverPIRTask->setParam(_params); + receiverPIRTask->setSelf(_receiverParty); + receiverPIRTask->addParty(_senderParty); + receiverPIRTask->setSyncResultToPeer(_syncResults); + receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, + _expectedPIRResult, _expectedSuccess, _expectedErrorCode); +} + +BOOST_AUTO_TEST_CASE(testNormalOtPIRCase) +{ + std::string otSearchPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; + std::string outputPath = "../../../ppc-pir/tests/data/output.csv"; + + uint32_t count = 513; + // prepareInputs(senderPath, count, receiverPath, count, count); + + auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, "sender", + "senderPartyResource", "sender_inputs", protocol::DataResourceType::FILE, ""); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(outputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + auto receiverParty = mockParty((uint16_t)ppc::protocol::PartyType::Server, "receiver", + "receiverPartyResource", "receiver_inputs", DataResourceType::FILE, otSearchPath); + + // auto receiverOutputDesc = std::make_shared(); + // receiverOutputDesc->setPath(outputPath); + // receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + + std::vector expectedResult; + for (uint32_t i = 0; i < count; i++) + { + expectedResult.emplace_back(std::to_string(100000 + i)); + } + + std::string jobParams = "{\"searchId\":\"testmsg1100\",\"requestAgencyId\":\"receiver\",\"prefixLength\":6}"; + + testOTPIRImplFunc( + "0x12345678", jobParams, true, senderParty, receiverParty, expectedResult, true, 0); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-pir/tests/main.cpp b/cpp/ppc-pir/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-pir/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-pir/tests/test.json b/cpp/ppc-pir/tests/test.json new file mode 100644 index 00000000..08df6b4d --- /dev/null +++ b/cpp/ppc-pir/tests/test.json @@ -0,0 +1,6 @@ +{ + "searchId": "testmsg1100", + "requestAgencyId": "1002", + "requestAgencyDataset": "../../../ppc-pir/tests/data/AysPreDataset.csv", + "prefixLength": 6 +} \ No newline at end of file diff --git a/cpp/ppc-protocol/CMakeLists.txt b/cpp/ppc-protocol/CMakeLists.txt new file mode 100644 index 00000000..615270ec --- /dev/null +++ b/cpp/ppc-protocol/CMakeLists.txt @@ -0,0 +1,8 @@ +project(ppc-protocol VERSION ${VERSION}) + +add_subdirectory(src) +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif () diff --git a/cpp/ppc-protocol/src/CMakeLists.txt b/cpp/ppc-protocol/src/CMakeLists.txt new file mode 100644 index 00000000..095eeec4 --- /dev/null +++ b/cpp/ppc-protocol/src/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${PROTOCOL_TARGET} ${SRCS}) + +target_link_libraries(${PROTOCOL_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${CPU_FEATURES_LIB} jsoncpp_static) diff --git a/cpp/ppc-protocol/src/JsonTaskImpl.cpp b/cpp/ppc-protocol/src/JsonTaskImpl.cpp new file mode 100644 index 00000000..5378e6a8 --- /dev/null +++ b/cpp/ppc-protocol/src/JsonTaskImpl.cpp @@ -0,0 +1,431 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JsonTaskImpl.cpp + * @author: yujiechen + * @date 2022-10-18 + */ +#include "JsonTaskImpl.h" +#include +#include + +using namespace ppc; +using namespace bcos; +using namespace ppc::protocol; + + +void JsonTaskImpl::decode(std::string_view _taskData) +{ + Json::Value root; + Json::Reader jsonReader; + + if (!jsonReader.parse(std::string(_taskData), root)) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment( + "The task information must be valid json string.")); + } + decodeJsonValue(root); +} + +void JsonTaskImpl::decodeJsonValue(Json::Value const& root) +{ + // the taskID + if (!root.isMember("taskID")) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the taskID")); + } + checkNull(root["taskID"], "taskID"); + m_id = root["taskID"].asString(); + + // the taskType + if (!root.isMember("type")) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the taskType")); + } + checkNull(root["type"], "taskType"); + m_type = root["type"].asUInt(); + // the taskAlgorithm + if (!root.isMember("algorithm")) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << errinfo_comment("Must specify the taskAlgorithm")); + } + checkNull(root["algorithm"], "taskAlgorithm"); + m_algorithm = root["algorithm"].asUInt(); + + // the parameters + if (root.isMember("params") && !root["params"].isNull()) + { + m_param = root["params"].asString(); + } + // sync result to peer or not + if (root.isMember("syncResult") && !root["syncResult"].isNull()) + { + m_syncResultToPeer = root["syncResult"].asBool(); + } + if (root.isMember("lowBandwidth") && !root["lowBandwidth"].isNull()) + { + m_lowBandwidth = root["lowBandwidth"].asBool(); + } + if (root.isMember("receiverList") && !root["receiverList"].isNull() && + root["receiverList"].isArray()) + { + Json::Value _receivers = root["receiverList"]; + for (const auto& r : _receivers) + { + if (r.isString()) + { + m_receiverLists.push_back(r.asString()); + } + } + } + + checkNull(root["parties"], "parties"); + // the peer-parties-info + if (!root.isMember("parties")) + { + return; + } + auto const& peersPartyEntry = root["parties"]; + for (Json::ArrayIndex i = 0; i < peersPartyEntry.size(); i++) + { + auto partyInfo = std::make_shared(); + decodePartyInfo(partyInfo, peersPartyEntry[i]); + // obtain the self-party + if (partyInfo->id() == m_selfPartyID) + { + m_self = partyInfo; + } + else + { + m_peerParties[partyInfo->id()] = partyInfo; + } + } + if (!m_self) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << errinfo_comment("Must specify the self-party-info")); + } +} + +// decode the party-info +void JsonTaskImpl::decodePartyInfo(PartyResource::Ptr _partyInfo, Json::Value const& _root) +{ + // the partyIndex + if (_root.isMember("partyIndex") && !_root["partyIndex"].isNull()) + { + _partyInfo->setPartyIndex(_root["partyIndex"].asUInt()); + } + // the partyID + if (!_root.isMember("id")) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the partyID")); + } + checkNull(_root["id"], "partyID"); + _partyInfo->setId(_root["id"].asString()); + // the desc + if (_root.isMember("desc")) + { + _partyInfo->setDesc(_root["desc"].asString()); + } + // the data-resource + if (_root.isMember("data") && !_root["data"].isNull()) + { + _partyInfo->setDataResource(decodeDataResource(_root["data"])); + } +} + +// decode the data-resource-info +DataResource::Ptr JsonTaskImpl::decodeDataResource(Json::Value const& _root) +{ + auto dataResource = std::make_shared(); + // the resourceID + if (!_root.isMember("id")) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << errinfo_comment("Must specify the id for the data-resource")); + } + checkNull(_root["id"], "data-resource-id"); + dataResource->setResourceID(_root["id"].asString()); + // the resource-desc + if (_root.isMember("input") && !_root["input"].isNull()) + { + // decode the input + auto const& descEntry = _root["input"]; + auto dataDesc = std::make_shared(); + decodeDataResourceDesc(dataDesc, descEntry); + dataResource->setDesc(dataDesc); + } + // decode the output + if (_root.isMember("output") && !_root["output"].isNull()) + { + auto outputDesc = std::make_shared(); + decodeDataResourceDesc(outputDesc, _root["output"]); + dataResource->setOutputDesc(outputDesc); + } + // decode the rawData + if (_root.isMember("rawData") && !_root["rawData"].isNull()) + { + DataResource::OriginData originData; + for (auto& members : _root["rawData"]) + { + std::vector data; + for (auto& member : members) + { + data.emplace_back(member.asString()); + } + originData.emplace_back(data); + } + dataResource->setRawData(originData); + } + return dataResource; +} + +void JsonTaskImpl::decodeDataResourceDesc(DataResourceDesc::Ptr _desc, Json::Value const& _value) +{ + // the dataResourceType + // Note: the data-resource is not required for the psi-server + if (_value.isMember("type")) + { + _desc->setType(_value["type"].asUInt()); + } + if (_value.isMember("path")) + { + auto path = _value["path"].asString(); + if (_desc->type() == (int)DataResourceType::FILE) + { + if (path.find("..") != std::string::npos) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << errinfo_comment("The \"..\" cannot be in the path")); + } + boost::filesystem::path prePath(m_prePath); + boost::filesystem::path inputPath(path); + boost::filesystem::path filePath(prePath / inputPath); + _desc->setPath(filePath.string()); + } + else + { + _desc->setPath(_value["path"].asString()); + } + } + if (_value.isMember("command")) + { + _desc->setAccessCommand(_value["command"].asString()); + } + if (_value.isMember("fileID")) + { + _desc->setFileID(_value["fileID"].asString()); + } + if (_value.isMember("fileMd5")) + { + _desc->setFileMd5(_value["fileMd5"].asString()); + } + if (_value.isMember("bizSeqNo")) + { + _desc->setBizSeqNo(_value["bizSeqNo"].asString()); + } + + // decode the connection option + if (_desc->type() == (int)DataResourceType::MySQL) + { + if (!_value.isMember("source")) + { + return; + // BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment( + // "The MySQL type resource must define the + // source")); + } + auto const& connectionOption = _value["source"]; + SQLConnectionOption option; + option.host = connectionOption["host"].asString(); + checkNonEmptyField("host", option.host); + + option.port = connectionOption["port"].asUInt(); + option.user = connectionOption["user"].asString(); + checkNonEmptyField("user", option.user); + + option.password = connectionOption["password"].asString(); + checkNonEmptyField("password", option.password); + + option.database = connectionOption["database"].asString(); + checkNonEmptyField("database", option.database); + + _desc->setSQLConnectionOption(std::make_shared(option)); + } + // decode the hdfs + if (_desc->type() == (int)DataResourceType::HDFS) + { + if (!_value.isMember("source")) + { + return; + // BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment( + // "The HDFS type resource must define the + // source")); + } + auto option = std::make_shared(); + auto const& fileStorageOpt = _value["source"]; + option->nameNode = fileStorageOpt["nameNode"].asString(); + checkNonEmptyField("nameNode", option->nameNode); + option->nameNodePort = fileStorageOpt["nameNodePort"].asUInt(); + option->userName = fileStorageOpt["user"].asString(); + checkNonEmptyField("user", option->userName); + + option->token = fileStorageOpt["token"].asString(); + _desc->setFileStorageConnectOption(option); + } +} + +std::string JsonTaskImpl::encode() const +{ + Json::Value taskInfo; + // the taskID + taskInfo["taskID"] = m_id; + // the taskType + taskInfo["type"] = m_type; + // the task-algorithm + taskInfo["algorithm"] = m_algorithm; + // params + taskInfo["params"] = m_param; + // sync-result or not + taskInfo["syncResult"] = m_syncResultToPeer; + taskInfo["lowBandwidth"] = m_lowBandwidth; + + Json::Value receiverList; + for (auto const& it : m_receiverLists) + { + receiverList.append(it); + } + taskInfo["receiverList"] = receiverList; + + Json::Value partiesInfo(Json::arrayValue); + // encode the partyInfo + if (m_self) + { + Json::Value selfPartyInfo; + encodePartyInfo(selfPartyInfo, m_self); + partiesInfo.append(selfPartyInfo); + } + for (auto const& it : m_peerParties) + { + Json::Value partyInfo; + encodePartyInfo(partyInfo, it.second); + partiesInfo.append(partyInfo); + } + taskInfo["parties"] = partiesInfo; + Json::FastWriter fastWriter; + return fastWriter.write(taskInfo); +} + +void JsonTaskImpl::encodePartyInfo(Json::Value& _value, PartyResource::Ptr _party) const +{ + // the partyIndex + _value["partyIndex"] = _party->partyIndex(); + // the partyID + _value["id"] = _party->id(); + // the desc + _value["desc"] = _party->desc(); + // the dataResource + encodeDataResource(_value["data"], _party->dataResource()); +} + +void JsonTaskImpl::encodeDataResource( + Json::Value& _value, DataResource::ConstPtr _dataResource) const +{ + // the dataID + _value["id"] = _dataResource->resourceID(); + // the desc + if (_dataResource->desc()) + { + // the type + _value["input"] = encodeDataResourceDesc(_dataResource->desc()); + } + if (_dataResource->outputDesc()) + { + _value["output"] = encodeDataResourceDesc(_dataResource->outputDesc()); + } + if (!_dataResource->rawData().empty()) + { + Json::Value jsonOriginData; + for (auto& dataVector : _dataResource->rawData()) + { + Json::Value jsonData; + for (auto& data : dataVector) + { + jsonData.append(data); + } + jsonOriginData.append(jsonData); + } + _value["rawData"] = jsonOriginData; + } +} + +Json::Value JsonTaskImpl::encodeDataResourceDesc(DataResourceDesc::ConstPtr _desc) const +{ + Json::Value result; + result["type"] = _desc->type(); + result["path"] = _desc->path(); + result["command"] = _desc->accessCommand(); + // encode the sql connection option + Json::Value connectOption; + if (_desc->sqlConnectionOption()) + { + connectOption["host"] = _desc->sqlConnectionOption()->host; + connectOption["port"] = _desc->sqlConnectionOption()->port; + connectOption["user"] = _desc->sqlConnectionOption()->user; + connectOption["password"] = _desc->sqlConnectionOption()->password; + connectOption["database"] = _desc->sqlConnectionOption()->database; + } + // encode the file-storage-connection-option + auto const& fileStorageOpt = _desc->fileStorageConnectionOption(); + if (fileStorageOpt) + { + connectOption["nameNode"] = fileStorageOpt->nameNode; + connectOption["nameNodePort"] = fileStorageOpt->nameNodePort; + connectOption["user"] = fileStorageOpt->userName; + connectOption["token"] = fileStorageOpt->token; + } + result["source"] = connectOption; + return result; +} + +PartyResource::ConstPtr JsonTaskImpl::getPartyByID(std::string const& _partyID) const +{ + if (m_self && m_self->id() == _partyID) + { + return m_self; + } + if (m_peerParties.count(_partyID)) + { + return m_peerParties.at(_partyID); + } + return nullptr; +} + +PartyResource::ConstPtr JsonTaskImpl::getPartyByIndex(uint16_t _partyIndex) const +{ + if (m_self->partyIndex() == _partyIndex) + { + return m_self; + } + for (const auto& peerParty : m_peerParties) + { + if (peerParty.second->partyIndex() == _partyIndex) + { + return peerParty.second; + } + } + return nullptr; +} diff --git a/cpp/ppc-protocol/src/JsonTaskImpl.h b/cpp/ppc-protocol/src/JsonTaskImpl.h new file mode 100644 index 00000000..dea185b8 --- /dev/null +++ b/cpp/ppc-protocol/src/JsonTaskImpl.h @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JsonTaskImpl.h + * @author: yujiechen + * @date 2022-10-18 + */ +#pragma once +#include "ppc-framework/protocol/Task.h" +#include +#include + +#include + +namespace ppc::protocol +{ +class JsonTaskImpl : public Task +{ +public: + using Ptr = std::shared_ptr; + JsonTaskImpl(std::string const& _selfPartyID, std::string const& _prePath = "data") : m_selfPartyID(_selfPartyID), m_prePath(_prePath) + {} + JsonTaskImpl(std::string const& _selfPartyID, std::string_view _taskData, std::string const& _prePath = "data") : JsonTaskImpl(_selfPartyID, _prePath) + { + decode(_taskData); + } + + JsonTaskImpl(std::string const& _selfPartyID, Json::Value const& _taskJson, std::string const& _prePath = "data") : JsonTaskImpl(_selfPartyID, _prePath) + { + decodeJsonValue(_taskJson); + } + + ~JsonTaskImpl() override = default; + + // the taskID + std::string const& id() const override { return m_id; } + // the taskType, e.g. PSI + uint8_t type() const override { return m_type; } + // the task algorithm, e.g.CM_PSI_2PC/RA_PSI_2PC + uint8_t algorithm() const override { return m_algorithm; } + + // information of the local party + PartyResource::ConstPtr selfParty() const override { return m_self; } + PartyResource::Ptr mutableSelfParty() const override { return m_self; } + + // information of the peer party(not thread-safe) + PartyResource::ConstPtr getPartyByID(std::string const& _partyID) const override; + PartyResource::ConstPtr getPartyByIndex(uint16_t _partyIndex) const override; + // get all peer-parties(not thread-safe) + std::map const& getAllPeerParties() const override + { + return m_peerParties; + } + + std::vector const& getReceiverLists() const override + { + return m_receiverLists; + } + + // params of the task, can be deserialized using json + std::string const& param() const override { return m_param; } + + // sync the psi result to peer or not + bool syncResultToPeer() const override { return m_syncResultToPeer; } + bool lowBandwidth() const override { return m_lowBandwidth; } + + void setId(std::string const& _id) override { m_id = _id; } + void setType(uint8_t _type) override { m_type = _type; } + void setAlgorithm(uint8_t _algorithm) override { m_algorithm = _algorithm; } + void setSelf(PartyResource::Ptr _self) override { m_self = _self; } + + // Note: not thread-safe here + void addParty(PartyResource::Ptr _party) override { m_peerParties[_party->id()] = _party; } + void setParam(std::string const& _param) override { m_param = _param; } + + void setSyncResultToPeer(bool _syncResultToPeer) override + { + m_syncResultToPeer = _syncResultToPeer; + } + void setLowBandwidth(bool _lowBandwidth) override + { + m_lowBandwidth = _lowBandwidth; + } + + // decode the task + void decode(std::string_view _taskData) override; + virtual void decodeJsonValue(Json::Value const& root); + // encode the task into json + std::string encode() const override; + +public: + void decodePartyInfo(PartyResource::Ptr _partyInfo, Json::Value const& _root); + DataResource::Ptr decodeDataResource(Json::Value const& _root); + void decodeDataResourceDesc(DataResourceDesc::Ptr _desc, Json::Value const& _value); + + void encodePartyInfo(Json::Value& _value, PartyResource::Ptr _party) const; + void encodeDataResource(Json::Value& _value, DataResource::ConstPtr _dataResource) const; + Json::Value encodeDataResourceDesc(DataResourceDesc::ConstPtr _desc) const; + + void checkNonEmptyField(std::string const& _field, std::string const& _value) + { + if (_value.empty()) + { + BOOST_THROW_EXCEPTION(bcos::InvalidParameter() << bcos::errinfo_comment( + "The value of " + _field + " must be non-empty")); + } + } + + void checkNull(Json::Value const& _value, std::string const& _field) + { + if (!_value.isNull()) + { + return; + } + BOOST_THROW_EXCEPTION(bcos::InvalidParameter() << bcos::errinfo_comment( + "The value of " + _field + " must be not null!")); + } + +private: + std::string m_selfPartyID; + std::string m_prePath; + std::string m_id; + // the task type + uint8_t m_type; + // the task-algorithm + uint8_t m_algorithm; + PartyResource::Ptr m_self; + // the partyID to PartyResource + std::map m_peerParties; + std::vector m_receiverLists; + // parameters + std::string m_param; + bool m_syncResultToPeer = false; + bool m_lowBandwidth = false; +}; + +class JsonTaskFactory : public TaskFactory +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + JsonTaskFactory(std::string const& _selfPartyID, const std::string& _prePath = "data") + : m_selfPartyID(_selfPartyID), m_prePath(_prePath) + {} + ~JsonTaskFactory() override = default; + + Task::Ptr createTask(std::string_view _taskInfo) override + { + return std::make_shared(m_selfPartyID, _taskInfo, m_prePath); + } + + virtual Task::Ptr createTask(Json::Value const& _taskInfo) + { + return std::make_shared(m_selfPartyID, _taskInfo, m_prePath); + } + + Task::Ptr createTask() override + { + return std::make_shared(m_selfPartyID, m_prePath); + } + +private: + std::string m_selfPartyID; + std::string m_prePath; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-protocol/src/PPCMessage.cpp b/cpp/ppc-protocol/src/PPCMessage.cpp new file mode 100644 index 00000000..3f724e87 --- /dev/null +++ b/cpp/ppc-protocol/src/PPCMessage.cpp @@ -0,0 +1,204 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCMessage.cpp + * @author: shawnhe + * @date 2022-10-19 + */ + +#include "PPCMessage.h" +#include +#include + +using namespace bcos; +using namespace ppc::front; + +void PPCMessage::encode(bytes& _buffer) +{ + _buffer.clear(); + + uint32_t seq = boost::asio::detail::socket_ops::host_to_network_long(m_seq); + uint16_t taskIDLength = boost::asio::detail::socket_ops::host_to_network_short(m_taskID.size()); + uint16_t senderLength = boost::asio::detail::socket_ops::host_to_network_short(m_sender.size()); + uint32_t dataLength = boost::asio::detail::socket_ops::host_to_network_long(m_data->size()); + uint16_t ext = boost::asio::detail::socket_ops::host_to_network_short(m_ext); + + _buffer.insert(_buffer.end(), (byte*)&m_version, (byte*)&m_version + 1); + _buffer.insert(_buffer.end(), (byte*)&m_taskType, (byte*)&m_taskType + 1); + _buffer.insert(_buffer.end(), (byte*)&m_algorithmType, (byte*)&m_algorithmType + 1); + _buffer.insert(_buffer.end(), (byte*)&m_messageType, (byte*)&m_messageType + 1); + _buffer.insert(_buffer.end(), (byte*)&seq, (byte*)&seq + 4); + _buffer.insert(_buffer.end(), (byte*)&taskIDLength, (byte*)&taskIDLength + 2); + _buffer.insert(_buffer.end(), m_taskID.begin(), m_taskID.end()); + _buffer.insert(_buffer.end(), (byte*)&senderLength, (byte*)&senderLength + 2); + _buffer.insert(_buffer.end(), m_sender.begin(), m_sender.end()); + _buffer.insert(_buffer.end(), (byte*)&ext, (byte*)&ext + 2); + // encode the uuid: uuidLen, uuidData + auto uuidLen = m_uuid.size(); + _buffer.insert(_buffer.end(), (byte*)&uuidLen, (byte*)&uuidLen + 1); + if (uuidLen > 0) + { + _buffer.insert(_buffer.end(), m_uuid.begin(), m_uuid.end()); + } + // encode the data: dataLen, dataData + _buffer.insert(_buffer.end(), (byte*)&dataLength, (byte*)&dataLength + 4); + if (dataLength > 0) + { + _buffer.insert(_buffer.end(), m_data->begin(), m_data->end()); + } + _buffer.insert(_buffer.end(), m_header.begin(), m_header.end()); + m_length = _buffer.size(); +} + +int64_t PPCMessage::decode(bcos::bytesPointer _buffer) +{ + return decode(_buffer->size(), _buffer->data()); +} + +int64_t PPCMessage::decode(bytesConstRef _buffer) +{ + return decode(_buffer.size(), (bcos::byte*)_buffer.data()); +} + +int64_t PPCMessage::decode(uint32_t _length, bcos::byte* _data) +{ + size_t minLen = MESSAGE_MIN_LENGTH; + if (_length < minLen) + { + return -1; + } + + m_data->clear(); + auto p = _data; + + // version field + m_version = *((uint8_t*)p); + p += 1; + + // task type field + m_taskType = *((uint8_t*)p); + p += 1; + + // algorithm type field + m_algorithmType = *((uint8_t*)p); + p += 1; + + // message type field + m_messageType = *((uint8_t*)p); + p += 1; + + // seq field + m_seq = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)p)); + p += 4; + + // taskIDLength + uint16_t taskIDLength = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); + p += 2; + minLen += taskIDLength; + if (_length < minLen) + { + return -1; + } + + // taskID field + m_taskID.insert(m_taskID.begin(), p, p + taskIDLength); + p += taskIDLength; + + // senderLength + uint16_t senderLength = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); + p += 2; + minLen += senderLength; + if (_length < minLen) + { + return -1; + } + // sender field + m_sender.insert(m_sender.begin(), p, p + senderLength); + p += senderLength; + + // ext field + m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); + p += 2; + + // decode the uuid + auto uuidLen = *((byte*)p); + p += 1; + minLen += uuidLen; + if (_length < minLen) + { + return -1; + } + if (uuidLen > 0) + { + m_uuid.assign(p, p + uuidLen); + p += uuidLen; + } + + // dataLength + uint32_t dataLength = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)p)); + p += 4; + minLen += dataLength; + if (_length < minLen) + { + return -1; + } + if (dataLength > 0) + { + // data field + m_data->insert(m_data->begin(), p, p + dataLength); + p += dataLength; + } + + if (p) + { + m_header.insert(m_header.begin(), p, _data + _length); + } + m_length = _length; + return _length; +} + +// map -> json(string) +std::string PPCMessage::encodeMap(const std::map& _map) +{ + Json::Value pObj; + for (std::map::const_iterator iter = _map.begin(); iter != _map.end(); + ++iter) + { + pObj[iter->first] = iter->second; + } + return Json::FastWriter().write(pObj); +} + +// json(string) -> map +std::map PPCMessage::decodeMap(const std::string& _encval) +{ + Json::Reader reader; + Json::Value value; + std::map maps; + + if (_encval.length() > 0) + { + if (reader.parse(_encval, value)) + { + Json::Value::Members members = value.getMemberNames(); + for (Json::Value::Members::iterator it = members.begin(); it != members.end(); it++) + { + maps.insert(std::pair(*it, value[*it].asString())); + } + } + } + + return maps; +} \ No newline at end of file diff --git a/cpp/ppc-protocol/src/PPCMessage.h b/cpp/ppc-protocol/src/PPCMessage.h new file mode 100644 index 00000000..c62f890f --- /dev/null +++ b/cpp/ppc-protocol/src/PPCMessage.h @@ -0,0 +1,158 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief ppc network message interface + * @file PPCMessage.h + * @author: shawnhe + * @date 2022-10-19 + */ + +#pragma once + +#include +#include +#include +#include + +#include "ppc-framework/protocol/PPCMessageFace.h" + +namespace ppc +{ +namespace front +{ +// the message format for ppc protocol +class PPCMessage : public PPCMessageFace +{ +public: + // version(1) + taskType(1) + algorithmType(1) + messageType(1) + seq(4) + // + taskIDLength(2) + senderLength(2) + ext(2) + uuidLen(1) + dataLen(4) + data(N) + // + header(M) + const static size_t MESSAGE_MIN_LENGTH = 15; + + using Ptr = std::shared_ptr; + PPCMessage() { m_data = std::make_shared(); } + ~PPCMessage() override = default; + + uint8_t version() const override { return m_version; } + void setVersion(uint8_t _version) override { m_version = _version; } + uint8_t taskType() const override { return m_taskType; } + void setTaskType(uint8_t _taskType) override { m_taskType = _taskType; } + uint8_t algorithmType() const override { return m_algorithmType; } + void setAlgorithmType(uint8_t _algorithmType) override { m_algorithmType = _algorithmType; } + uint8_t messageType() const override { return m_messageType; } + void setMessageType(uint8_t _messageType) override { m_messageType = _messageType; } + uint32_t seq() const override { return m_seq; } + void setSeq(uint32_t _seq) override { m_seq = _seq; } + std::string const& taskID() const override { return m_taskID; } + void setTaskID(std::string const& _taskID) override { m_taskID = _taskID; } + std::string const& sender() const override { return m_sender; } + void setSender(std::string const& _sender) override { m_sender = _sender; } + virtual uint16_t ext() const override { return m_ext; } + virtual void setExt(uint16_t _ext) override { m_ext = _ext; } + std::shared_ptr data() const override { return m_data; } + // Note: here directly use passed-in _data, make-sure _data not changed before send the message + void setData(std::shared_ptr _data) override { m_data = _data; } + std::map header() override + { + return decodeMap(m_header); + } + void setHeader(std::map _header) override + { + m_header = encodeMap(_header); + } + + std::string uuid() const override { return m_uuid; } + void setUuid(std::string const& _uuid) override { m_uuid = _uuid; } + + void encode(bcos::bytes& _buffer) override; + int64_t decode(bcos::bytesPointer _buffer) override; + int64_t decode(bcos::bytesConstRef _buffer) override; + int64_t decode(uint32_t _length, bcos::byte* _data); + + uint32_t length() const override { return m_length; } + + // determine the message is response or not + bool response() const override { return m_ext & MessageExtFlag::ResponseFlag; } + // set the message to be response + void setResponse() override { m_ext |= MessageExtFlag::ResponseFlag; } + +protected: + std::string encodeMap(const std::map& _map); + std::map decodeMap(const std::string& _encval); + +private: + uint8_t m_version = 0; + uint8_t m_taskType = 0; + uint8_t m_algorithmType = 0; + uint8_t m_messageType = 0; + uint32_t m_seq = 0; + std::string m_taskID; + std::string m_sender; + uint16_t m_ext = 0; + // the uuid used to find the response-callback + std::string m_uuid; + std::shared_ptr m_data; + std::string m_header; + + uint32_t m_length = MESSAGE_MIN_LENGTH; +}; + +class PPCMessageFactory : public PPCMessageFaceFactory +{ +public: + using Ptr = std::shared_ptr; + PPCMessageFactory() = default; + virtual ~PPCMessageFactory() {} + +public: + PPCMessageFace::Ptr buildPPCMessage() override { return std::make_shared(); } + + PPCMessageFace::Ptr buildPPCMessage(uint8_t _taskType, uint8_t _algorithmType, + std::string const& _taskID, std::shared_ptr _data) override + { + auto msg = std::make_shared(); + msg->setTaskType(_taskType); + msg->setAlgorithmType(_algorithmType); + msg->setTaskID(_taskID); + msg->setData(std::move(_data)); + return msg; + } + + + PPCMessageFace::Ptr buildPPCMessage(bcos::bytesConstRef _buffer) override + { + auto msg = std::make_shared(); + int64_t length = msg->decode(_buffer); + if (length == -1) + { + return nullptr; + } + return msg; + } + + PPCMessageFace::Ptr buildPPCMessage(bcos::bytesPointer _buffer) override + { + auto msg = std::make_shared(); + int64_t length = msg->decode(std::move(_buffer)); + if (length == -1) + { + return nullptr; + } + return msg; + } +}; + +} // namespace front +} // namespace ppc diff --git a/cpp/ppc-protocol/tests/CMakeLists.txt b/cpp/ppc-protocol/tests/CMakeLists.txt new file mode 100644 index 00000000..4466ace3 --- /dev/null +++ b/cpp/ppc-protocol/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-protocol) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${PROTOCOL_TARGET} TBB::tbb ${BOOST_UNIT_TEST}) +add_test(NAME test-protocol WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-protocol/tests/PPCMessageTest.cpp b/cpp/ppc-protocol/tests/PPCMessageTest.cpp new file mode 100644 index 00000000..316abdcf --- /dev/null +++ b/cpp/ppc-protocol/tests/PPCMessageTest.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCMessageTest.cpp + * @author: shawnhe + * @date 2022-10-28 + */ + +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-framework/protocol/Protocol.h" +#include +#include +#include +#include + +using namespace ppc; +using namespace ppc::front; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(PPCMessageTest, TestPromptFixture) + +void testEncodeAndDecode(PPCMessageFace::Ptr _message) +{ + auto payload = std::make_shared(); + _message->encode(*payload); + + auto messageFactory = std::make_shared(); + auto newMsg = messageFactory->buildPPCMessage(payload); + + BOOST_CHECK(newMsg->version() == _message->version()); + BOOST_CHECK(newMsg->taskType() == _message->taskType()); + BOOST_CHECK(newMsg->algorithmType() == _message->algorithmType()); + BOOST_CHECK(newMsg->messageType() == _message->messageType()); + BOOST_CHECK(newMsg->seq() == _message->seq()); + BOOST_CHECK(newMsg->taskID() == _message->taskID()); + BOOST_CHECK(newMsg->sender() == _message->sender()); + BOOST_CHECK(newMsg->ext() == _message->ext()); + BOOST_CHECK(newMsg->uuid() == _message->uuid()); + BOOST_CHECK(newMsg->data()->size() == _message->data()->size()); + auto newMsgHeader = newMsg->header(); + auto messageHeader = _message->header(); + BOOST_CHECK(newMsgHeader.size() == messageHeader.size()); + BOOST_CHECK(newMsgHeader["x-http-session"] == "111111"); + BOOST_CHECK(messageHeader["x-http-session"] == "111111"); + BOOST_CHECK(newMsgHeader["x-http-request"] == "2222222"); + BOOST_CHECK(messageHeader["x-http-request"] == "2222222"); +} + +BOOST_AUTO_TEST_CASE(test_ppcMesage) +{ + auto messageFactory = std::make_shared(); + auto message = messageFactory->buildPPCMessage(); + message->setVersion(1); + message->setTaskType(uint8_t(protocol::TaskType::PSI)); + message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC)); + message->setMessageType(4); + message->setSeq(5); + message->setTaskID("12345678"); + message->setSender("1001"); + message->setExt(10); + message->setUuid("uuid1245"); + message->setData(std::make_shared(10, 'a')); + std::map head = { + {"x-http-session", "111111"}, {"x-http-request", "2222222"}}; + message->setHeader(head); + testEncodeAndDecode(message); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/ppc-protocol/tests/TestTaskImpl.cpp b/cpp/ppc-protocol/tests/TestTaskImpl.cpp new file mode 100644 index 00000000..33fc5a4c --- /dev/null +++ b/cpp/ppc-protocol/tests/TestTaskImpl.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestTaskImpl.cpp + * @author: yujiechen + * @date 2022-10-19 + */ + +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "test-utils/TaskMock.h" +#include +#include +#include + +using namespace ppc; +using namespace ppc::protocol; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(taskImplTest, TestPromptFixture) + +inline void testTask(TaskFactory::Ptr _taskFactory, std::string _selfPartyID, int _peerPartySize) +{ + auto task = _taskFactory->createTask(); + task->setId("testTask"); + task->setType((uint8_t)ppc::protocol::TaskType::PSI); + task->setParam("taskParam"); + task->setSyncResultToPeer(true); + + // mock self-party + auto party = mockParty((uint16_t)ppc::protocol::PartyType::Server, _selfPartyID, "selfParty", + "dataSelf", ppc::protocol::DataResourceType::FILE, "testPath"); + + std::vector> rawData(2); + for (uint32_t i = 0; i < 10; i++) + { + rawData[0].emplace_back(std::to_string(i)); + rawData[1].emplace_back(std::to_string(i)); + } + party->mutableDataResource()->setRawData(rawData); + + task->setSelf(party); + // mock peers-party + for (int i = 0; i < _peerPartySize; i++) + { + std::string partyID = "party" + std::to_string(i); + party = mockParty((uint16_t)ppc::protocol::PartyType::Server, partyID, partyID, "dataParty", + ppc::protocol::DataResourceType::FILE, "testPath"); + task->addParty(party); + } + // encode + auto encodeData = task->encode(); + // decode + auto decodedTask = _taskFactory->createTask(encodeData); + // check the result + checkTask(decodedTask, task); + + // invalid case + task->setSelf(nullptr); + encodeData = task->encode(); + // decode exception for without selfParty + BOOST_CHECK_THROW(_taskFactory->createTask(encodeData), bcos::InvalidParameter); +} + +BOOST_AUTO_TEST_CASE(testJsonTaskImpl) +{ + std::string selfPartyID = "selfParty"; + auto factory = std::make_shared(selfPartyID); + testTask(factory, selfPartyID, 10); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-protocol/tests/main.cpp b/cpp/ppc-protocol/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-protocol/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-psi/CMakeLists.txt b/cpp/ppc-psi/CMakeLists.txt new file mode 100644 index 00000000..06d0e477 --- /dev/null +++ b/cpp/ppc-psi/CMakeLists.txt @@ -0,0 +1,19 @@ +project(ppc-psi VERSION ${VERSION}) +add_subdirectory(src/psi-framework) +add_subdirectory(src/labeled-psi) +add_subdirectory(src/ra2018-psi) +add_subdirectory(src/ecdh-psi) +add_subdirectory(src/cm2020-psi) +add_subdirectory(src/ecdh-multi-psi) +#add_subdirectory(src/ecdh-conn-psi) + +if(DEMO) + add_subdirectory(demo) +endif() +add_subdirectory(src/bs-ecdh-psi) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() diff --git a/cpp/ppc-psi/demo/CMakeLists.txt b/cpp/ppc-psi/demo/CMakeLists.txt new file mode 100644 index 00000000..6a6508e5 --- /dev/null +++ b/cpp/ppc-psi/demo/CMakeLists.txt @@ -0,0 +1,7 @@ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +set(HASH_COLLISION_BINARY_NAME hash_collision_bench) +add_executable(${HASH_COLLISION_BINARY_NAME} hash_collision_bench.cpp) +target_link_libraries(${HASH_COLLISION_BINARY_NAME} ${CRYPTO_TARGET} ${IO_TARGET}) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) diff --git a/cpp/ppc-psi/demo/hash_collision_bench.cpp b/cpp/ppc-psi/demo/hash_collision_bench.cpp new file mode 100644 index 00000000..9bc55861 --- /dev/null +++ b/cpp/ppc-psi/demo/hash_collision_bench.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file hash_collision_bench.cpp + * @desc: bench for hash collision test + * @author: yujiechen + * @date 2022-12-30 + */ +#include "ppc-crypto-core/src/hash/BitMixMurmurHash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-crypto/src/oprf/RA2018Oprf.h" +#include "ppc-io/src/FileLineReader.h" + + +using namespace ppc::crypto; +using namespace ppc::io; +using namespace bcos; + +void Usage(std::string const& _appName) +{ + std::cout << _appName << "[file1] [file2] [hashBits]" << std::endl; +} + +int main(int argc, char* argv[]) +{ + if (argc < 3) + { + Usage(argv[0]); + return -1; + } + auto file1 = argv[1]; + auto file2 = argv[2]; + int hashBits = 32; + if (argc == 4) + { + hashBits = atoi(argv[3]); + } + if (hashBits != 32 && hashBits != 64) + { + std::cout << "Invalid hashBits, must be 32 or 64" << std::endl; + return -1; + } + std::cout << "hash_collision_bench, file1: " << file1 << std::endl; + std::cout << "hash_collision_bench, file2: " << file2 << std::endl; + auto lineReader1 = std::make_shared(file1); + auto lineReader2 = std::make_shared(file2); + + // load all data + auto dataBatch1 = lineReader1->next(-1, DataSchema::Bytes); + auto dataBatch2 = lineReader2->next(-1, DataSchema::Bytes); + std::cout << "* file1 element size:" << dataBatch1->size() << std::endl; + std::cout << "* file2 element size:" << dataBatch2->size() << std::endl; + + auto hashImpl = std::make_shared(); + auto eccCrypto = + std::make_shared(hashImpl, ppc::protocol::ECCCurve::P256); + auto privateKey = eccCrypto->generateRandomScalar(); + auto oprf = std::make_shared(privateKey, eccCrypto, hashImpl); + // full-evaluate + std::vector result1; + oprf->fullEvaluate(dataBatch1, result1); + std::vector result2; + oprf->fullEvaluate(dataBatch2, result2); + + // calculate the hash + std::set hashData; + auto hasher = std::make_shared(); + for (uint64_t i = 0; i < result1.size(); i++) + { + hashData.insert(hasher->hash(result1.at(i), hashBits)); + } + uint64_t collisionCount = 0; + // check the collision + for (uint64_t i = 0; i < result2.size(); i++) + { + auto hashResult = hasher->hash(result2.at(i), hashBits); + if (hashData.count(hashResult)) + { + collisionCount++; + auto collisionElement = dataBatch2->get(i); + std::cout << "* collision: " + << std::string(collisionElement.begin(), collisionElement.end()) << std::endl; + } + } + std::cout << "* hash_collision_bench test finished!" << std::endl; + std::cout << "* collisionCount: " << collisionCount << std::endl; + std::cout << "* file1 element size: " << dataBatch1->size() << std::endl; + std::cout << "* file2 element size: " << dataBatch2->size() << std::endl; + return 0; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/Common.h b/cpp/ppc-psi/src/Common.h new file mode 100644 index 00000000..b2673d28 --- /dev/null +++ b/cpp/ppc-psi/src/Common.h @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-12-26 + */ +#pragma once +#include +#include +#define PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI") + +namespace ppc::psi +{ +enum class CommonMessageType : uint8_t +{ + ErrorNotification = 0xff, + PingPeer = 0xfe, +}; + +// the common psi packet-type +enum class PSIPacketType : uint32_t +{ + CancelTaskNotification = 0x01, // notify the peer that the task has been canceled for some + // reason + TaskSyncMsg = 0x02, // sync the task information + HandshakeRequest = 0x03, // the handshake request + HandshakeResponse = 0x04, // the handshake response + PSIResultSyncMsg = 0x05, // sync the psi-result to the psi-server in some cases + PSIResultSyncResponse = 0x06, // response for psi-result-sync + HandshakeSuccess = 0x07, // the client response to the server when handshake-success +}; + +// packetType for ra2018-psi +enum class RA2018PacketType : uint32_t +{ + CuckooFilterRequest = 100, // the client request the cuckoofilter + CuckooFilterResponse = 101, // the server response the cuckoofilter + EvaluateRequest = 102, // the client request the server to evaluate-data + EvaluateResponse = 103, // the server response the evaluated-data + MissingCuckooFilterRequest = 104, // request the missed cuckoo-filter + MissingCuckooFilterResponse, // response to the missed-cuckoo-filter-request +}; + +// packetType for ecdh-psi +enum class ECDHPacketType : uint32_t +{ + EvaluateRequest = 100, + EvaluateResponse = 101, + ServerBlindedData = 102, + SyncDataBatchInfo = 103, +}; + +enum PSIRetCode : int +{ + Success = 0, + TaskParamsError = -1000, + PeerNodeDown = -1001, + DuplicatedTask = -1002, + LoadDataFailed = -1003, + PeerNotifyFinish = -1004, + InvalidTaskParamForRA2018 = -1005, + DataResourceOccupied = -1006, + UnsupportedCommand = -1007, + LoadCuckooFilterDataError = -1008, + UnknownPSIPacketType = -1009, + TaskNotFound = -1010, + TaskInProcessing = -1011, + OnlySupportOnePeer = -1012, + UnsupportedPartyType = -1013, + NotSpecifyInputDataResource = -1014, + NotSpecifyOutputDataResource = -1015, + NotOfflineFullEvaluated = -1016, + BlindDataError = -1017, + syncCuckooFilterError = -1018, + NotSpecifyPeerDataResource = -1019, + HandshakeFailed = -1020, + SyncPSIResultFailed = -1021, + RA2018PSIDisabled = -1022, + HandleTaskError = -1023, + TaskNotReady = -1024, + TaskIsNotRunning = -1025, + OnException = -1026, + TaskExists = -1027, + TaskKilled= -1028, + TaskCountReachMax = -1029, + TaskTimeout = -1030 +}; + +enum CacheState +{ + Evaluating = 0, + Finalizing = 1, + Finalized = 2, + IntersectionProgressing = 3, + Intersectioned = 4, + StoreProgressing = 5, + Stored = 6, + Syncing = 7, + Synced = 8, +}; + +inline std::ostream& operator<<(std::ostream& _out, CacheState const& _state) +{ + switch (_state) + { + case CacheState::Evaluating: + _out << "Evaluating"; + break; + case CacheState::Finalizing: + _out << "Finalizing"; + break; + case CacheState::Finalized: + _out << "Finalized"; + break; + case CacheState::IntersectionProgressing: + _out << "IntersectionProgressing"; + break; + case CacheState::Intersectioned: + _out << "Intersectioned"; + break; + case CacheState::StoreProgressing: + _out << "StoreProgressing"; + break; + case CacheState::Stored: + _out << "Stored"; + break; + case CacheState::Syncing: + _out << "Syncing"; + break; + case CacheState::Synced: + _out << "Synced"; + break; + default: + _out << "UnknownCacheState"; + break; + } + return _out; +} +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/PSIConfig.h b/cpp/ppc-psi/src/PSIConfig.h new file mode 100644 index 00000000..41f83430 --- /dev/null +++ b/cpp/ppc-psi/src/PSIConfig.h @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIConfig.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "Common.h" +#include "bcos-utilities/Common.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/protocol/Protocol.h" +#include "psi-framework/interfaces/PSIMessageInterface.h" +#include +#include + +namespace ppc::psi +{ +class PSIConfig +{ +public: + using Ptr = std::shared_ptr; + PSIConfig(ppc::protocol::PSIAlgorithmType _algorithmType, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader) + : m_algorithmType(_algorithmType), m_dataResourceLoader(std::move(_dataResourceLoader)) + {} + + PSIConfig(ppc::protocol::PSIAlgorithmType _algorithmType, const std::string& _selfParty, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes) + : m_algorithmType(_algorithmType), + m_selfParty(_selfParty), + m_front(std::move(_front)), + m_ppcMsgFactory(std::move(_ppcMsgFactory)), + m_dataResourceLoader(std::move(_dataResourceLoader)), + m_networkTimeout(_holdingMessageMinutes * 60 * 1000), + m_taskExpireTime(m_networkTimeout) + { + PSI_LOG(INFO) << LOG_DESC("create PSIConfig") << LOG_KV("algorithmType", m_algorithmType) + << LOG_KV("holdingMessageMinutes", _holdingMessageMinutes) + << LOG_KV("networkTimeout", m_networkTimeout) + << LOG_KV("taskExpireTime", m_taskExpireTime); + } + + virtual ~PSIConfig() = default; + + ppc::front::FrontInterface::Ptr const& front() const { return m_front; } + std::string selfParty() const { return m_selfParty; } + + ppc::protocol::PSIAlgorithmType algorithmType() const { return m_algorithmType; } + ppc::front::PPCMessageFaceFactory::Ptr const& ppcMsgFactory() const { return m_ppcMsgFactory; } + int networkTimeout() const { return m_networkTimeout; } + + bcos::Error::Ptr sendMessage(std::string const& _receiver, front::PPCMessageFace::Ptr _message) + { + std::promise completedFuture; + m_front->asyncSendMessage( + _receiver, std::move(_message), m_networkTimeout, + [&completedFuture]( + const bcos::Error::Ptr& _error) { completedFuture.set_value(_error); }, + nullptr); + + return completedFuture.get_future().get(); + } + + void generateAndSendPPCMessage(std::string const& _peerID, std::string const& _taskID, + PSIMessageInterface::Ptr const& _msg, std::function _callback, + uint32_t _seq = 0, ppc::front::CallbackFunc _responseCallback = nullptr) + { + auto ppcMsg = generatePPCMsg(_taskID, _msg, _seq); + PSI_LOG(TRACE) << LOG_DESC("generateAndSendPPCMessage") << LOG_KV("peer", _peerID) + << printPSIMessage(_msg) << LOG_KV("msgType", (int)_msg->packetType()) + << LOG_KV("seq", _seq); + m_front->asyncSendMessage( + _peerID, ppcMsg, m_networkTimeout, + [_callback](bcos::Error::Ptr _error) { + if (_callback) + { + _callback(std::move(_error)); + } + }, + _responseCallback); + } + + void asyncSendResponse(std::string const& _peerID, std::string const& _taskID, + std::string const& _uuid, PSIMessageInterface::Ptr const& _msg, + ppc::front::ErrorCallbackFunc _callback, uint32_t _seq = 0) + { + auto ppcMsg = generatePPCMsg(_taskID, _msg, _seq); + PSI_LOG(TRACE) << LOG_DESC("sendResponse") << LOG_KV("peer", _peerID) << printPPCMsg(ppcMsg) + << LOG_KV("msgType", (int)_msg->packetType()) << LOG_KV("uuid", _uuid); + m_front->asyncSendResponse(_peerID, _uuid, ppcMsg, _callback); + } + + ppc::io::DataResourceLoader::Ptr const& dataResourceLoader() const + { + return m_dataResourceLoader; + } + + int taskExpireTime() const { return m_taskExpireTime; } + void setTaskExpireTime(int _taskExpireTime) { m_taskExpireTime = _taskExpireTime; } + + void updateAgenyList(std::vector const& _agencyList) + { + bcos::UpgradableGuard l(x_agencyList); + if (m_agencyList != _agencyList) + { + bcos::UpgradeGuard ul(l); + m_agencyList = _agencyList; + } + } + + std::vector agencyList() const + { + bcos::ReadGuard l(x_agencyList); + return m_agencyList; + } + +protected: + ppc::front::PPCMessageFace::Ptr generatePPCMsg( + std::string const& _taskID, PSIMessageInterface::Ptr const& _msg, uint32_t _seq) + { + if (_msg->partyID().empty()) + { + _msg->setPartyID(m_selfParty); + } + auto encodedData = _msg->encode(); + auto ppcMsg = m_ppcMsgFactory->buildPPCMessage(); + ppcMsg->setTaskType((uint8_t)ppc::protocol::TaskType::PSI); + ppcMsg->setAlgorithmType((uint8_t)m_algorithmType); + ppcMsg->setTaskID(_taskID); + ppcMsg->setSender(m_selfParty); + ppcMsg->setData(encodedData); + ppcMsg->setSeq(_seq); + return ppcMsg; + } + +protected: + // the psi-alogrithm-type + ppc::protocol::PSIAlgorithmType m_algorithmType; + std::string m_selfParty; + ppc::front::FrontInterface::Ptr m_front; + // the front message factory + ppc::front::PPCMessageFaceFactory::Ptr m_ppcMsgFactory; + ppc::io::DataResourceLoader::Ptr m_dataResourceLoader; + + // the network-timeout, default 300s + int m_networkTimeout = 300000; + + // the task-expire time + int m_taskExpireTime = 10000; + + // the agency list, for task-sync + std::vector m_agencyList; + mutable bcos::SharedMutex x_agencyList; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h new file mode 100644 index 00000000..08c3bb30 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhPSIFactory.h + * @author: shawnhe + * @date 2023-09-20 + */ + +#pragma once +#include "BsEcdhPSIFactory.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-psi/src/PSIConfig.h" +#include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h" + +namespace ppc::psi +{ +class BsEcdhPSIFactory +{ +public: + using Ptr = std::shared_ptr; + BsEcdhPSIFactory() = default; + virtual ~BsEcdhPSIFactory() = default; + + virtual BsEcdhPSIImpl::Ptr buildBsEcdhPSI(bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _timeoutMinutes) + { + auto config = std::make_shared( + ppc::protocol::PSIAlgorithmType::BS_ECDH_PSI, std::move(_dataResourceLoader)); + return std::make_shared( + std::move(config), std::move(_threadPool), _timeoutMinutes); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp new file mode 100644 index 00000000..1985ddcb --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp @@ -0,0 +1,384 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhPSIImpl.cpp + * @author: shawnhe + * @date 2023-09-20 + */ + +#include "BsEcdhPSIImpl.h" + +using namespace bcos; +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; + +BsEcdhResult::Ptr BsEcdhPSIImpl::getTaskStatus(GetTaskStatusRequest::Ptr _request) +{ + ReadGuard l(x_tasks); + auto state = m_taskStates.find(_request->taskID); + if (state == m_taskStates.end()) + { + auto result = std::make_shared(_request->taskID); + result->setError(std::make_shared(TaskNotFound, "Task not found.")); + return result; + } + + if (isNotExecutable(state->second->status())) + { + return state->second->result(); + } + + return prepareResultWithoutLock(_request->taskID, toString(state->second->status())); +} + +BsEcdhResult::Ptr BsEcdhPSIImpl::updateTaskStatus(UpdateTaskStatusRequest::Ptr _request) +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("update task status") << LOG_KV("taskID", _request->taskID) + << LOG_KV("status", _request->status); + + WriteGuard l(x_tasks); + auto state = m_taskStates.find(_request->taskID); + if (state == m_taskStates.end()) + { + auto result = std::make_shared(_request->taskID); + result->setError(std::make_shared(TaskNotFound, "Task not found.")); + return result; + } + + auto status = fromString(_request->status); + if (status == TaskStatus::RUNNING) + { + state->second->restartTask(); + } + else if (status == TaskStatus::PAUSING) + { + state->second->pauseTask(); + } + + if (isNotExecutable(state->second->status())) + { + return state->second->result(); + } + + return prepareResultWithoutLock(_request->taskID, toString(state->second->status())); +} + + +BsEcdhResult::Ptr BsEcdhPSIImpl::killTask(KillTaskRequest::Ptr _request) +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("kill task") << LOG_KV("taskID", _request->taskID); + + WriteGuard l(x_tasks); + auto result = std::make_shared(_request->taskID); + auto state = m_taskStates.find(_request->taskID); + if (state == m_taskStates.end()) + { + result->setError(std::make_shared(TaskNotFound, "Task not found.")); + return result; + } + + if (isExecutable(state->second->status())) + { + GetTaskStatusResponse response; + response.taskID = _request->taskID; + response.status = toString(TaskStatus::FAILED); + auto ecdhResult = std::make_shared(_request->taskID, response.serialize()); + ecdhResult->setError(std::make_shared(TaskKilled, "Task has been killed.")); + state->second->setResult(ecdhResult); + state->second->updateStatus(TaskStatus::FAILED); + } + + auto cache = m_taskCaches.find(state->second->taskID()); + if (cache != m_taskCaches.end()) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("clean finished task cache") + << LOG_KV("taskID", cache->second->taskID()); + m_taskCaches.erase(cache); + } + + return std::make_shared(_request->taskID); +} + +void BsEcdhPSIImpl::asyncRunTask( + RunTaskRequest::Ptr _request, std::function&& _onTaskFinished) +{ + try + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("receive a task") + << LOG_KV("request", _request->toString()); + auto state = findTaskState(_request->taskID); + if (state) + { + BS_ECDH_PSI_LOG(WARNING) + << LOG_DESC("asyncRunTask, task exists.") << LOG_KV("taskID", _request->taskID); + if (_onTaskFinished) + { + _onTaskFinished(std::make_shared(_request->taskID)); + } + return; + } + + if (taskCount() >= MAX_TASK_COUNT) + { + BS_ECDH_PSI_LOG(WARNING) + << LOG_DESC("task count reach max.") << LOG_KV("taskID", _request->taskID); + auto result = std::make_shared(_request->taskID); + result->setError(std::make_shared(TaskCountReachMax, "Task count reaches max.")); + if (_onTaskFinished) + { + _onTaskFinished(std::move(result)); + } + return; + } + + // init task state + auto taskState = std::make_shared( + _request->taskID, TaskStatus::PENDING, m_timeoutMinutes); + + // create ecdh cache + auto cache = std::make_shared( + _request->taskID, m_threadPool, m_taskGuarder, _request->dataResource, + _request->enableAudit, m_enableOutputExists, + [self = weak_from_this(), taskID = _request->taskID] { + auto psi = self.lock(); + if (psi) + { + psi->onSelfCiphersReady(taskID); + } + }, + [self = weak_from_this(), taskID = _request->taskID] { + auto psi = self.lock(); + if (psi) + { + psi->onAllCiphersReady(taskID); + } + }, + [self = weak_from_this(), taskID = _request->taskID]( + protocol::TaskStatus _status, BsEcdhResult::Ptr _result) { + auto psi = self.lock(); + if (psi) + { + psi->onTaskFinished(taskID, _status, std::move(_result)); + } + }, + _request->partnerInputsSize); + cache->start(); + + // add task + addTask(std::move(taskState), std::move(cache)); + + if (_onTaskFinished) + { + _onTaskFinished(std::make_shared(_request->taskID)); + } + } + catch (std::exception const& e) + { + BS_ECDH_PSI_LOG(ERROR) << LOG_DESC("asyncRunTask") + << LOG_KV("exception", boost::diagnostic_information(e)); + auto result = std::make_shared(_request->taskID); + result->setError(std::make_shared( + TaskParamsError, "Init task error: " + boost::diagnostic_information(e))); + if (_onTaskFinished) + { + _onTaskFinished(std::move(result)); + } + } +} + +BsEcdhResult::Ptr BsEcdhPSIImpl::fetchCipher(FetchCipherRequest::Ptr _request) +{ + auto result = checkTaskRequest(_request->taskID); + if (result.first) + { + return result.first; + } + return result.second->fetchCipher(std::move(_request)); +} + +BsEcdhResult::Ptr BsEcdhPSIImpl::sendEcdhCipher(SendEcdhCipherRequest::Ptr _request) +{ + auto result = checkTaskRequest(_request->taskID); + if (result.first) + { + return result.first; + } + return result.second->onEcdhCipherReceived(std::move(_request)); +} + +BsEcdhResult::Ptr BsEcdhPSIImpl::sendPartnerCipher(SendPartnerCipherRequest::Ptr _request) +{ + auto result = checkTaskRequest(_request->taskID); + if (result.first) + { + return result.first; + } + return result.second->onPartnerCipherReceived(std::move(_request)); +} + +std::pair BsEcdhPSIImpl::checkTaskRequest( + const std::string& _taskID) +{ + auto result = std::make_shared(_taskID); + + auto state = findTaskState(_taskID); + if (!state) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("checkTaskRequest, task not found.") + << LOG_KV("taskID", _taskID); + result->setError(std::make_shared(TaskNotFound, "Task not found.")); + return {result, nullptr}; + } + + // Even if the task is completed, the browser can continue to send ciphers. + if (state->status() != TaskStatus::RUNNING && state->status() != TaskStatus::PAUSING && + state->status() != TaskStatus::COMPLETED) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("checkTaskRequest, task is not running.") + << LOG_KV("taskID", _taskID); + result->setError(std::make_shared(TaskIsNotRunning, "Task is not running.")); + return {result, nullptr}; + } + + auto cache = findTaskCache(_taskID); + if (!cache) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("checkTaskRequest, task not found.") + << LOG_KV("taskID", _taskID); + result->setError(std::make_shared(TaskNotFound, "Task cache not found.")); + return {result, nullptr}; + } + + state->active(); + return {nullptr, cache}; +} + +void BsEcdhPSIImpl::start() +{ + if (m_started) + { + BS_ECDH_PSI_LOG(ERROR) << LOG_DESC("The BS-ECDH-PSI has already been started"); + return; + } + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("Start the BS-ECDH-PSI"); + m_started = true; + + if (m_taskCleaner) + { + m_taskCleaner->registerTimeoutHandler([self = weak_from_this()] { + auto psi = self.lock(); + if (psi) + { + psi->checkAndCleanTask(); + } + }); + m_taskCleaner->start(); + } +} + + +void BsEcdhPSIImpl::stop() +{ + if (!m_started) + { + return; + } + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("Stop BS-ECDH-PSI"); + m_started = false; + + if (m_taskCleaner) + { + m_taskCleaner->stop(); + } +} + +void BsEcdhPSIImpl::checkAndCleanTask() +{ + try + { + WriteGuard l(x_tasks); + for (auto state = m_taskStates.begin(); state != m_taskStates.end();) + { + state->second->autoPauseChecking(); + if (isNotExecutable(state->second->status()) || state->second->isTimeout() || + state->second->isExpired()) + { + auto cache = m_taskCaches.find(state->second->taskID()); + if (cache != m_taskCaches.end()) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("clean finished task cache") + << LOG_KV("taskID", cache->second->taskID()); + m_taskCaches.erase(cache); + } + } + + if (state->second->isExpired()) + { + BS_ECDH_PSI_LOG(INFO) + << LOG_DESC("clean expired task") << LOG_KV("taskID", state->second->taskID()); + state = m_taskStates.erase(state); + } + else + { + ++state; + } + } + } + catch (std::exception const& e) + { + BS_ECDH_PSI_LOG(ERROR) << LOG_DESC("checkAndCleanTask") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + m_taskCleaner->restart(); +} + +void BsEcdhPSIImpl::onSelfCiphersReady(const std::string& _taskID) +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onSelfCiphersReady") << LOG_KV("taskID", _taskID); + WriteGuard l(x_tasks); + auto state = m_taskStates.find(_taskID); + if (state != m_taskStates.end()) + { + state->second->setupAutoPause(); + } +} + +void BsEcdhPSIImpl::onAllCiphersReady(const std::string& _taskID) +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("allCiphersReady") << LOG_KV("taskID", _taskID); + WriteGuard l(x_tasks); + auto state = m_taskStates.find(_taskID); + if (state != m_taskStates.end()) + { + state->second->cancelAutoPause(); + } +} + +void BsEcdhPSIImpl::onTaskFinished( + const std::string& _taskID, protocol::TaskStatus _status, BsEcdhResult::Ptr _result) +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("taskID", _taskID) + << LOG_KV("status", toString(_status)); + { + WriteGuard l(x_tasks); + auto state = m_taskStates.find(_taskID); + if (state != m_taskStates.end()) + { + state->second->updateStatus(_status); + state->second->setResult(std::move(_result)); + } + } +} diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h new file mode 100644 index 00000000..55c7f671 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h @@ -0,0 +1,165 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhPSIImpl.h + * @author: shawnhe + * @date 2023-09-20 + */ + +#pragma once + +#include "BsEcdhPSIInterface.h" +#include "Common.h" +#include "core/BsEcdhCache.h" +#include "core/BsEcdhIoHandler.h" +#include "core/BsEcdhTaskState.h" +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::psi +{ + +class BsEcdhPSIImpl : public BsEcdhPSIInterface, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + BsEcdhPSIImpl( + PSIConfig::Ptr _config, bcos::ThreadPool::Ptr _threadPool, uint32_t _timeoutMinutes) + : m_config(std::move(_config)), + m_threadPool(std::move(_threadPool)), + m_taskCleaner(std::make_shared(PAUSE_THRESHOLD, "BsEcdhTaskCleaner")), + m_timeoutMinutes(_timeoutMinutes) + { + m_taskGuarder = std::make_shared(m_config); + } + + virtual ~BsEcdhPSIImpl() = default; + + BsEcdhResult::Ptr getTaskStatus(GetTaskStatusRequest::Ptr _request) override; + + virtual BsEcdhResult::Ptr updateTaskStatus(UpdateTaskStatusRequest::Ptr _request) override; + + virtual BsEcdhResult::Ptr killTask(KillTaskRequest::Ptr _request) override; + + void asyncRunTask(RunTaskRequest::Ptr _request, + std::function&& _onTaskFinished) override; + + BsEcdhResult::Ptr fetchCipher(FetchCipherRequest::Ptr _request) override; + + BsEcdhResult::Ptr sendEcdhCipher(SendEcdhCipherRequest::Ptr _request) override; + + BsEcdhResult::Ptr sendPartnerCipher(SendPartnerCipherRequest::Ptr _request) override; + + void start() override; + + void stop() override; + + // allow the output-path exists, for ut + void enableOutputExists() { m_enableOutputExists = true; } + +protected: + void checkAndCleanTask(); + std::pair checkTaskRequest(const std::string& _taskID); + void onSelfCiphersReady(const std::string& _taskID); + void onAllCiphersReady(const std::string& _taskID); + void onTaskFinished( + const std::string& _taskID, protocol::TaskStatus _status, BsEcdhResult::Ptr _result); + + void addTask(BsEcdhTaskState::Ptr _state, BsEcdhCache::Ptr _cache) + { + bcos::WriteGuard l(x_tasks); + m_taskStates[_state->taskID()] = std::move(_state); + m_taskCaches[_cache->taskID()] = std::move(_cache); + } + + BsEcdhCache::Ptr findTaskCache(const std::string& _taskID) + { + bcos::ReadGuard l(x_tasks); + auto cache = m_taskCaches.find(_taskID); + if (cache == m_taskCaches.end()) + { + return nullptr; + } + + return cache->second; + } + + BsEcdhTaskState::Ptr findTaskState(const std::string& _taskID) + { + bcos::ReadGuard l(x_tasks); + auto state = m_taskStates.find(_taskID); + if (state == m_taskStates.end()) + { + return nullptr; + } + + return state->second; + } + + uint32_t taskCount() + { + bcos::ReadGuard l(x_tasks); + uint32_t count = 0; + for (auto& state : m_taskStates) + { + if (state.second->status() != protocol::TaskStatus::COMPLETED && + state.second->status() != protocol::TaskStatus::FAILED) + { + count++; + } + } + return count; + } + + BsEcdhResult::Ptr prepareResultWithoutLock( + const std::string& _taskID, const std::string& _status) + { + GetTaskStatusResponse response; + response.taskID = _taskID; + response.status = _status; + + auto cache = m_taskCaches.find(_taskID); + if (cache != m_taskCaches.end()) + { + response.step = cache->second->step(); + response.index = cache->second->index(); + response.progress = cache->second->progress(); + } + return std::make_shared(_taskID, response.serialize()); + } + +private: + bool m_enableOutputExists = false; + + PSIConfig::Ptr m_config; + bcos::ThreadPool::Ptr m_threadPool; + std::shared_ptr m_taskCleaner; + uint32_t m_timeoutMinutes; + TaskGuarder::Ptr m_taskGuarder; + + std::unordered_map m_taskCaches; + std::unordered_map m_taskStates; + mutable bcos::SharedMutex x_tasks; + + std::atomic m_started{false}; +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h new file mode 100644 index 00000000..7c21bd95 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BSECDHPSIInterface.h + * @author: shawnhe + * @date 2023-09-20 + */ +#pragma once +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h" +#include "ppc-psi/src/bs-ecdh-psi/protocol/Message.h" +#include +#include + +namespace ppc::psi +{ + +class BsEcdhPSIInterface +{ +public: + using Ptr = std::shared_ptr; + + BsEcdhPSIInterface() = default; + + virtual ~BsEcdhPSIInterface() = default; + + virtual BsEcdhResult::Ptr getTaskStatus(GetTaskStatusRequest::Ptr _request) = 0; + + virtual BsEcdhResult::Ptr updateTaskStatus(UpdateTaskStatusRequest::Ptr _request) = 0; + + virtual BsEcdhResult::Ptr killTask(KillTaskRequest::Ptr _request) = 0; + + virtual void asyncRunTask(RunTaskRequest::Ptr _request, + std::function&& _onTaskFinished) = 0; + + virtual BsEcdhResult::Ptr fetchCipher(FetchCipherRequest::Ptr _request) = 0; + + virtual BsEcdhResult::Ptr sendEcdhCipher(SendEcdhCipherRequest::Ptr _request) = 0; + + virtual BsEcdhResult::Ptr sendPartnerCipher(SendPartnerCipherRequest::Ptr _request) = 0; + + virtual void start() = 0; + + virtual void stop() = 0; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt b/cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt new file mode 100644 index 00000000..ed1d5866 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${BS_ECDH_PSI_TARGET} ${SRCS}) + +target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb TCMalloc) diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/Common.h b/cpp/ppc-psi/src/bs-ecdh-psi/Common.h new file mode 100644 index 00000000..4e45004e --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/Common.h @@ -0,0 +1,41 @@ +/** +* Copyright (C) 2023 WeDPR. +* SPDX-License-Identifier: Apache-2.0 +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* @file Common.h +* @author: shawnhe +* @date 2023-09-22 +*/ + +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/PPCMessageFace.h" + +#include +#include + + +namespace ppc::psi +{ + +DERIVE_PPC_EXCEPTION(BsEcdhException); + +#define BS_ECDH_PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: BS-ECDH-PSI") +#define INDEX_FILE_SUFFIX ".index" +#define AUDIT_FILE_SUFFIX ".evidence" +#define MAX_TASK_COUNT 32 +#define PAUSE_THRESHOLD 60000 + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp new file mode 100644 index 00000000..6b89ed32 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp @@ -0,0 +1,555 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhCache.cpp + * @author: shawnhe + * @date 2023-09-20 + */ + +#include "BsEcdhCache.h" + +using namespace bcos; +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; + +void BsEcdhCache::start() +{ + m_threadPool->enqueue([self = weak_from_this()]() { + auto cache = self.lock(); + if (!cache) + { + return; + } + cache->prepareCipher(); + }); +} + +void BsEcdhCache::generateKey() +{ + m_key.resize(SCALAR_SIZE + 1); + COutputBuffer outputBuffer{(char*)m_key.data(), SCALAR_SIZE}; + auto ret = wedpr_random_scalar(&outputBuffer); + if (ret != WEDPR_SUCCESS) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)PSIRetCode::OnException, "Generate key error, code: " + std::to_string(ret))); + } + m_key.resize(outputBuffer.len); +} + +std::string BsEcdhCache::genCipherWithBase64(const std::string& _input) +{ + std::string hash(POINT_SIZE + 1, 0); + CInputBuffer message{(char*)_input.data(), _input.size()}; + COutputBuffer point{(char*)hash.data(), POINT_SIZE}; + auto ret = wedpr_hash_to_curve(&message, &point); + if (ret != WEDPR_SUCCESS) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)PSIRetCode::OnException, "Hash to curve error, code: " + std::to_string(ret))); + } + + CInputBuffer key{(char*)m_key.data(), m_key.size()}; + std::string cipher(POINT_SIZE + 1, 0); + CInputBuffer hashPoint{(char*)point.data, point.len}; + COutputBuffer cipherPoint{(char*)cipher.data(), cipher.size()}; + ret = wedpr_point_scalar_multi(&hashPoint, &key, &cipherPoint); + if (ret != WEDPR_SUCCESS) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PSIRetCode::OnException, + "Point multiply by scalar, code: " + std::to_string(ret))); + } + return base64Encode(bytesConstRef((byte*)cipherPoint.data, cipherPoint.len)); +} + +std::string BsEcdhCache::genEcdhCipherWithBase64(const std::string& _point) +{ + auto point = base64Decode(_point); + CInputBuffer rawPoint{(char*)point.data(), point.size()}; + CInputBuffer key{(char*)m_key.data(), m_key.size()}; + std::string ecdhCipher(POINT_SIZE + 1, 0); + COutputBuffer ecdhCipherPoint{(char*)ecdhCipher.data(), ecdhCipher.size()}; + auto ret = wedpr_point_scalar_multi(&rawPoint, &key, &ecdhCipherPoint); + if (ret != WEDPR_SUCCESS) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PSIRetCode::OnException, + "Point multiply by scalar, code: " + std::to_string(ret))); + } + return base64Encode(bytesConstRef((byte*)ecdhCipherPoint.data, ecdhCipherPoint.len)); +} + +void BsEcdhCache::prepareIoHandler() +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start preparing io handler") << LOG_KV("taskID", m_taskID); + + auto reader = m_taskGuarder->loadReader(m_taskID, m_dataResource, DataSchema::String); + + auto resultWriter = m_taskGuarder->loadWriter(m_taskID, m_dataResource, m_enableOutputExists); + + auto outputPath = m_dataResource->outputDesc()->path(); + auto indexPath = outputPath + INDEX_FILE_SUFFIX; + m_dataResource->mutableOutputDesc()->setPath(indexPath); + auto indexWriter = m_taskGuarder->loadWriter(m_taskID, m_dataResource, m_enableOutputExists); + + LineWriter::Ptr evidenceWriter = nullptr; + if (m_enableAudit) + { + auto evidencePath = outputPath + AUDIT_FILE_SUFFIX; + m_dataResource->mutableOutputDesc()->setPath(evidencePath); + evidenceWriter = m_taskGuarder->loadWriter(m_taskID, m_dataResource, m_enableOutputExists); + } + + m_ioHandler = std::make_shared( + m_taskID, reader, resultWriter, indexWriter, evidenceWriter); + + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish preparing io handler") << LOG_KV("taskID", m_taskID); +} + +void BsEcdhCache::prepareCipher() +{ + try + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start preparing cipher") << LOG_KV("taskID", m_taskID); + + prepareIoHandler(); + + // generate ecc sk + generateKey(); + // write key into evidence file + m_ioHandler->appendEvidence( + "WB KEY", base64Encode(bytesConstRef(m_key.data(), m_key.size()))); + + m_originInputs = m_ioHandler->loadInputs(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PSIRetCode::OnException, "Data is empty.")); + } + m_inputsSize = m_originInputs->size(); + m_ciphers.reserve(m_inputsSize); + m_ciphers.resize(m_inputsSize); + m_ecdhCiphers.reserve(m_inputsSize); + m_ecdhCiphers.resize(m_inputsSize); + m_ecdhCiphersMap.reserve(m_inputsSize); + m_ecdhCipherFlags.assign(m_inputsSize, false); + + tbb::parallel_for(tbb::blocked_range(0U, m_inputsSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + m_ciphers[i] = std::move(genCipherWithBase64(m_originInputs->get(i))); + } + }); + + // write ciphers into evidence file + m_ioHandler->appendEvidences("WB CIPHERS", m_ciphers); + + if (m_onSelfCiphersReady) + { + m_step = ProcessingSelfCiphers; + m_onSelfCiphersReady(); + } + + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish preparing cipher") << LOG_KV("taskID", m_taskID); + } + catch (const std::exception& e) + { + onSelfException("prepareCipher", e); + } +} + +uint32_t BsEcdhCache::findCurrentIndex( + const std::vector& _flags, uint32_t _offset, uint32_t _total) +{ + uint32_t final = _offset; + do + { + final++; + } while (_flags[final] && (final < _total)); + + return final; +} + +BsEcdhResult::Ptr BsEcdhCache::fetchCipher(FetchCipherRequest::Ptr _request) +{ + try + { + ReadGuard l(x_ecdhCiphers); + if (m_selfEcdhCiphersReady) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)PSIRetCode::OnException, "WB ciphers hava been processed.")); + } + + auto response = std::make_shared(); + response->taskID = m_taskID; + response->offset = _request->offset; + response->total = m_inputsSize; + response->ciphers.clear(); + + uint32_t end = _request->offset + _request->size >= m_inputsSize ? + m_inputsSize : + _request->offset + _request->size; + for (uint32_t i = _request->offset; i < end; i++) + { + response->ciphers.push_back(m_ciphers[i]); + } + response->size = response->ciphers.size(); + + return std::make_shared(m_taskID, response->serialize()); + } + catch (const std::exception& e) + { + return onRequestException("fetchCipher", e); + } +} + +BsEcdhResult::Ptr BsEcdhCache::onEcdhCipherReceived(SendEcdhCipherRequest::Ptr _request) +{ + try + { + WriteGuard l(x_ecdhCiphers); + if (m_selfEcdhCiphersReady) + { + return std::make_shared(m_taskID); + } + + // save ecdh ciphers + uint32_t end = _request->offset + _request->ecdhCiphers.size() >= m_inputsSize ? + m_inputsSize : + _request->offset + _request->ecdhCiphers.size(); + for (uint32_t i = _request->offset; i < end; i++) + { + if (!m_ecdhCipherFlags[i]) + { + m_ecdhCipherFlags[i] = true; + m_receivedEcdhCipherCount++; + } + m_ecdhCiphers[i] = _request->ecdhCiphers[i - _request->offset]; + } + auto total = + m_inputsSize + m_partnerInputsSize == 0 ? 1 : m_inputsSize + m_partnerInputsSize; + uint64_t currentCount = m_receivedPartnerCipherCount + m_receivedEcdhCipherCount; + m_progress = currentCount * 100 / total; + if (m_progress % 10 == 0) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onEcdhCipherReceived") + << LOG_KV("totalReceived", currentCount) + << LOG_KV("progress", m_progress) << LOG_KV("taskID", m_taskID); + } + + if (_request->offset == m_selfIndex) + { + m_selfIndex = findCurrentIndex(m_ecdhCipherFlags, _request->offset, m_inputsSize); + } + + if (m_receivedEcdhCipherCount == m_inputsSize) + { + m_step = ProcessingPartnerCiphers; + m_selfEcdhCiphersReady.exchange(true); + m_threadPool->enqueue([self = weak_from_this()]() { + auto cache = self.lock(); + if (!cache) + { + return; + } + cache->onAllSelfEcdhCiphersReady(); + }); + } + + return std::make_shared(m_taskID); + } + catch (const std::exception& e) + { + return onRequestException("onEcdhCipherReceived", e); + } +} + +BsEcdhResult::Ptr BsEcdhCache::onPartnerCipherReceived(SendPartnerCipherRequest::Ptr _request) +{ + try + { + WriteGuard l(x_partnerCiphers); + if (m_partnerEcdhCiphersReady) + { + return std::make_shared(m_taskID); + } + + if (m_partnerCiphers.empty()) + { + if (m_partnerInputsSize == 0) + { + // compatible with older versions + m_partnerInputsSize = _request->total; + } + m_partnerCiphers.reserve(m_partnerInputsSize); + m_partnerCiphers.resize(m_partnerInputsSize); + m_partnerEcdhCiphers.reserve(m_partnerInputsSize); + m_partnerEcdhCiphers.resize(m_partnerInputsSize); + m_partnerCipherFlags.assign(m_partnerInputsSize, false); + } + + // save partner ciphers + uint32_t end = _request->offset + _request->partnerCiphers.size() >= m_partnerInputsSize ? + m_partnerInputsSize : + _request->offset + _request->partnerCiphers.size(); + for (uint32_t i = _request->offset; i < end; i++) + { + if (!m_partnerCipherFlags[i]) + { + m_partnerCipherFlags[i] = true; + m_receivedPartnerCipherCount++; + } + + m_partnerCiphers[i] = _request->partnerCiphers[i - _request->offset]; + } + + auto total = + m_inputsSize + m_partnerInputsSize == 0 ? 1 : m_inputsSize + m_partnerInputsSize; + uint64_t currentCount = m_receivedPartnerCipherCount + m_receivedEcdhCipherCount; + m_progress = currentCount * 100 / total; + if (m_progress % 10 == 0) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onPartnerCipherReceived") + << LOG_KV("totalReceived", currentCount) + << LOG_KV("progress", m_progress) << LOG_KV("taskID", m_taskID); + } + + bool error = false; + // compute partner ecdh ciphers + tbb::parallel_for( + tbb::blocked_range(0U, end - _request->offset), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto index = _request->offset + i; + try + { + m_partnerEcdhCiphers[index] = + std::move(genEcdhCipherWithBase64(_request->partnerCiphers[i])); + } + catch (const std::exception& e) + { + error = true; + BS_ECDH_PSI_LOG(WARNING) + << LOG_DESC("genEcdhCipherWithBase64") + << LOG_KV("cipher", _request->partnerCiphers[i]) + << LOG_KV("exception", boost::diagnostic_information(e)); + } + } + }); + + if (error) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)PSIRetCode::OnException, "Compute partner ecdh ciphers error")); + } + + if (_request->offset == m_partnerIndex) + { + m_partnerIndex = + findCurrentIndex(m_partnerCipherFlags, _request->offset, m_partnerInputsSize); + } + + if (m_receivedPartnerCipherCount == m_partnerInputsSize) + { + m_partnerEcdhCiphersReady.exchange(true); + m_threadPool->enqueue([self = weak_from_this()]() { + auto cache = self.lock(); + if (!cache) + { + return; + } + cache->onAllPartnerEcdhCiphersReady(); + }); + } + + return std::make_shared(m_taskID); + } + catch (const std::exception& e) + { + return onRequestException("onPartnerCipherReceived", e); + } +} + +void BsEcdhCache::onAllSelfEcdhCiphersReady() +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onAllSelfEcdhCiphersReady") << LOG_KV("taskID", m_taskID); + + for (uint32_t i = 0; i < m_inputsSize; i++) + { + m_ecdhCiphersMap[m_ecdhCiphers[i]] = i; + } + + // write ecdh ciphers into evidence file + m_ioHandler->appendEvidences("WB ECDH CIPHERS", m_ecdhCiphers); + + // release memory + std::vector().swap(m_ciphers); + std::vector().swap(m_ecdhCiphers); + std::vector().swap(m_ecdhCipherFlags); + MallocExtension::instance()->ReleaseFreeMemory(); + + if (m_partnerEcdhCiphersReady) + { + onAllEcdhCiphersReady(); + } +} + +void BsEcdhCache::onAllPartnerEcdhCiphersReady() +{ + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onAllPartnerEcdhCiphersReady") << LOG_KV("taskID", m_taskID); + + if (m_selfEcdhCiphersReady) + { + onAllEcdhCiphersReady(); + } +} + +void BsEcdhCache::onAllEcdhCiphersReady() +{ + if (m_allCiphersReady.exchange(true)) + { + return; + } + + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("onAllCiphersReady") << LOG_KV("taskID", m_taskID); + if (m_onAllCiphersReady) + { + m_onAllCiphersReady(); + } + + m_step = ComputingResults; + + try + { + // write partner ciphers into evidence file at the end + m_ioHandler->appendEvidences("PARTNER CIPHERS", m_partnerCiphers); + + // release memory + std::vector().swap(m_partnerCiphers); + std::vector().swap(m_partnerCipherFlags); + MallocExtension::instance()->ReleaseFreeMemory(); + + m_ioHandler->uploadEvidences(); + + // find intersections + DataBatch::Ptr partnerIndexes = std::make_shared(); + std::unordered_set intersections; + uint32_t dumpSize = 0; + for (uint32_t i = 0; i < m_partnerInputsSize; i++) + { + if (m_ecdhCiphersMap.contains(m_partnerEcdhCiphers[i])) + { + auto index = m_ecdhCiphersMap[m_partnerEcdhCiphers[i]]; + auto data = m_originInputs->get(index); + if (!intersections.contains(data)) + { + intersections.insert(data); + partnerIndexes->append(std::to_string(i)); + } + else + { + dumpSize++; + } + } + } + + // release memory + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + std::unordered_map().swap(m_ecdhCiphersMap); + std::vector().swap(m_partnerEcdhCiphers); + MallocExtension::instance()->ReleaseFreeMemory(); + + auto resultCount = intersections.size(); + m_ioHandler->saveResults(intersections); + m_ioHandler->saveIndexes(partnerIndexes); + + std::unordered_set().swap(intersections); + partnerIndexes->setData(std::vector()); + MallocExtension::instance()->ReleaseFreeMemory(); + + BS_ECDH_PSI_LOG(INFO) << LOG_DESC( + "finish computing results and start constructing response") + << LOG_KV("taskID", m_taskID); + + // construct response + auto response = std::make_shared(); + response->taskID = m_taskID; + response->status = toString(TaskStatus::COMPLETED); + response->intersections = resultCount; + response->party0Size = m_inputsSize; + response->party1Size = m_partnerInputsSize; + auto timeCosts = std::to_string(utcSteadyTime() - m_startTime) + "ms"; + response->timeCost = timeCosts; + response->step = DownloadIndex; + response->progress = 100; + + auto result = std::make_shared(m_taskID, response->serialize()); + if (m_onTaskFinished) + { + m_onTaskFinished(TaskStatus::COMPLETED, result); + } + + BS_ECDH_PSI_LOG(INFO) << LOG_BADGE("BsEcdhPsiTaskDone") << LOG_KV("taskID", m_taskID) + << LOG_KV("inputsSize", m_inputsSize) + << LOG_KV("partnerInputsSize", m_partnerInputsSize) + << LOG_KV("resultCount", resultCount) << LOG_KV("dumpSize", dumpSize) + << LOG_KV("timeCost", timeCosts) + << LOG_KV("resultFileID", response->resultFileID) + << LOG_KV("resultFileMd5", response->resultFileMd5) + << LOG_KV("partnerIndexFileID", response->partnerIndexFileID) + << LOG_KV("partnerIndexFileMd5", response->partnerIndexFileMd5) + << LOG_KV("evidenceFileID", response->evidenceFileID) + << LOG_KV("evidenceFileMd5", response->evidenceFileMd5); + } + catch (const std::exception& e) + { + return onSelfException("onAllCiphersReady", e); + } +} + +void BsEcdhCache::onSelfException(const std::string& _module, const std::exception& e) +{ + BS_ECDH_PSI_LOG(ERROR) << LOG_DESC("onSelfException") << LOG_KV("taskID", m_taskID) + << LOG_KV("module", _module) + << LOG_KV("exception", boost::diagnostic_information(e)); + + GetTaskStatusResponse response; + response.taskID = m_taskID; + response.status = toString(TaskStatus::FAILED); + auto result = std::make_shared(m_taskID, response.serialize()); + result->setError(std::make_shared( + (int)PSIRetCode::OnException, "Task failed: " + boost::diagnostic_information(e))); + + if (m_onTaskFinished) + { + m_onTaskFinished(TaskStatus::FAILED, result); + } +} + +BsEcdhResult::Ptr BsEcdhCache::onRequestException( + const std::string& _module, const std::exception& e) +{ + BS_ECDH_PSI_LOG(WARNING) << LOG_DESC("onRequestException") << LOG_KV("taskID", m_taskID) + << LOG_KV("module", _module) + << LOG_KV("exception", boost::diagnostic_information(e)); + + auto result = std::make_shared(m_taskID); + result->setError(std::make_shared( + (int)PSIRetCode::OnException, "Task failed: " + boost::diagnostic_information(e))); + return result; +} diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h new file mode 100644 index 00000000..a939f8f2 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h @@ -0,0 +1,190 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhCache.h + * @author: shawnhe + * @date 2023-09-20 + */ + +#pragma once + +#include "BsEcdhIoHandler.h" +#include "ppc-psi/src/Common.h" +#include "ppc-psi/src/bs-ecdh-psi/Common.h" +#include "ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h" +#include "ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h" +#include "ppc-psi/src/bs-ecdh-psi/protocol/Message.h" +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include +#include +#include +#include + +namespace ppc::psi +{ + +enum TaskStep : int +{ + Initializing = 1, + ProcessingSelfCiphers, + ProcessingPartnerCiphers, + ComputingResults, + // execution in ppcs-adm + DownloadIndex, +}; + +class BsEcdhCache : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + BsEcdhCache(std::string _taskID, bcos::ThreadPool::Ptr _threadPool, + TaskGuarder::Ptr _taskGuarder, protocol::DataResource::Ptr _dataResource, bool _enableAudit, + bool _enableOutputExists, std::function&& _onSelfCiphersReady, + std::function&& _onAllCiphersReady, + std::function&& _onTaskFinished, + uint32_t _partnerInputsSize) + : m_taskID(std::move(_taskID)), + m_threadPool(std::move(_threadPool)), + m_taskGuarder(std::move(_taskGuarder)), + m_dataResource(std::move(_dataResource)), + m_enableAudit(_enableAudit), + m_enableOutputExists(_enableOutputExists), + m_onSelfCiphersReady(std::move(_onSelfCiphersReady)), + m_onAllCiphersReady(std::move(_onAllCiphersReady)), + m_onTaskFinished(std::move(_onTaskFinished)), + m_partnerInputsSize(_partnerInputsSize) + { + m_startTime = bcos::utcSteadyTime(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("create BsEcdhCache") + << LOG_KV("partnerInputsSize", m_partnerInputsSize) + << LOG_KV("taskID", m_taskID); + } + ~BsEcdhCache() + { + if (m_ioHandler) + { + m_ioHandler->clean(); + } + // release the memory to os + std::vector().swap(m_ciphers); + std::vector().swap(m_ecdhCiphers); + std::vector().swap(m_ecdhCipherFlags); + std::vector().swap(m_partnerCiphers); + std::vector().swap(m_partnerCipherFlags); + std::unordered_map().swap(m_ecdhCiphersMap); + std::vector().swap(m_partnerEcdhCiphers); + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + MallocExtension::instance()->ReleaseFreeMemory(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("the BsEcdhCache destroyed") + << LOG_KV("taskID", m_taskID); + } + + void start(); + + [[nodiscard]] std::string taskID() const { return m_taskID; } + + [[nodiscard]] uint32_t step() const { return m_step; } + + [[nodiscard]] uint32_t index() const + { + if (m_step == ProcessingSelfCiphers) + { + return m_selfIndex; + } + else if (m_step == ProcessingPartnerCiphers) + { + return m_partnerIndex; + } + else + { + return 0; + } + } + + [[nodiscard]] int progress() const { return m_progress; } + + BsEcdhResult::Ptr fetchCipher(FetchCipherRequest::Ptr _request); + + BsEcdhResult::Ptr onEcdhCipherReceived(SendEcdhCipherRequest::Ptr _request); + + BsEcdhResult::Ptr onPartnerCipherReceived(SendPartnerCipherRequest::Ptr _request); + +public: + // for unit test + bcos::bytes key() { return m_key; } + void setKey(const bcos::bytes& _key) { m_key.assign(_key.begin(), _key.end()); } + + void generateKey(); + std::string genCipherWithBase64(const std::string& _input); + std::string genEcdhCipherWithBase64(const std::string& _point); + +private: + void prepareIoHandler(); + void prepareCipher(); + static uint32_t findCurrentIndex( + const std::vector& _flags, uint32_t _offset, uint32_t _total); + void onAllSelfEcdhCiphersReady(); + void onAllPartnerEcdhCiphersReady(); + void onAllEcdhCiphersReady(); + void onSelfException(const std::string& _module, const std::exception& e); + BsEcdhResult::Ptr onRequestException(const std::string& _module, const std::exception& e); + +private: + std::string m_taskID; + bcos::ThreadPool::Ptr m_threadPool; + TaskGuarder::Ptr m_taskGuarder; + protocol::DataResource::Ptr m_dataResource; + bool m_enableAudit; + bool m_enableOutputExists; + BsEcdhIoHandler::Ptr m_ioHandler; + std::function m_onSelfCiphersReady; + std::function m_onAllCiphersReady; + std::function m_onTaskFinished; + + ppc::io::DataBatch::Ptr m_originInputs; + uint32_t m_inputsSize{0}; + uint32_t m_partnerInputsSize; + uint64_t m_startTime; + + bcos::bytes m_key; + std::vector m_ciphers; + + mutable bcos::SharedMutex x_ecdhCiphers; + std::vector m_ecdhCiphers; + std::unordered_map m_ecdhCiphersMap; + std::vector m_ecdhCipherFlags; + uint32_t m_receivedEcdhCipherCount{0}; + std::atomic m_selfEcdhCiphersReady{false}; + + mutable bcos::SharedMutex x_partnerCiphers; + std::vector m_partnerCiphers; + std::vector m_partnerCipherFlags; + uint32_t m_receivedPartnerCipherCount{0}; + + std::vector m_partnerEcdhCiphers; + std::atomic m_partnerEcdhCiphersReady{false}; + + std::atomic m_step{Initializing}; + std::atomic m_selfIndex{0}; + std::atomic m_partnerIndex{0}; + std::atomic m_progress{0}; + + std::atomic m_allCiphersReady{false}; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h new file mode 100644 index 00000000..07b198ea --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h @@ -0,0 +1,202 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhIoHandler.h + * @author: shawnhe + * @date 2023-09-22 + */ + +#pragma once + +#include + +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/io/LineReader.h" +#include "ppc-psi/src/bs-ecdh-psi/Common.h" + +namespace ppc::psi +{ +class BsEcdhIoHandler +{ +public: + using Ptr = std::shared_ptr; + + BsEcdhIoHandler(std::string _taskID, ppc::io::LineReader::Ptr _reader, + ppc::io::LineWriter::Ptr _resultWriter, ppc::io::LineWriter::Ptr _indexWriter, + ppc::io::LineWriter::Ptr _evidenceWriter) + : m_taskID(std::move(_taskID)), + m_reader(std::move(_reader)), + m_resultWriter(std::move(_resultWriter)), + m_indexWriter(std::move(_indexWriter)), + m_evidenceWriter(std::move(_evidenceWriter)) + {} + + virtual ~BsEcdhIoHandler() = default; + + [[nodiscard]] std::string taskID() const { return m_taskID; } + + [[nodiscard]] ppc::io::LineReader::Ptr reader() const { return m_reader; } + + [[nodiscard]] ppc::io::LineWriter::Ptr resultWriter() const { return m_resultWriter; } + + [[nodiscard]] ppc::io::LineWriter::Ptr indexWriter() const { return m_indexWriter; } + + [[nodiscard]] ppc::io::LineWriter::Ptr evidenceWriter() const { return m_evidenceWriter; } + + io::DataBatch::Ptr loadInputs() + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start loading inputs") << LOG_KV("taskID", m_taskID); + int64_t nextParam = m_reader->type() == protocol::DataResourceType::MySQL ? 0 : -1; + auto data = m_reader->next(nextParam, io::DataSchema::String); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish loading inputs") << LOG_KV("taskID", m_taskID) + << LOG_KV("inputsSize", data->size()); + return data; + } + + protocol::FileInfo::Ptr saveResults(const std::unordered_set& _results) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start saving results") << LOG_KV("taskID", m_taskID) + << LOG_KV("resultsSize", _results.size()); + writeByDataBatch(m_resultWriter, _results); + m_resultWriter->flush(); + m_resultWriter->close(); + // upload + m_resultWriter->upload(); + m_resultWriter->clean(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish saving results") << LOG_KV("taskID", m_taskID) + << LOG_KV("resultsSize", _results.size()); + return m_resultWriter->fileInfo(); + } + + protocol::FileInfo::Ptr saveIndexes(const io::DataBatch::Ptr& _indexes) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start saving indexes") << LOG_KV("taskID", m_taskID) + << LOG_KV("indexesSize", _indexes->size()); + m_indexWriter->writeLine(_indexes, io::DataSchema::String, "\n"); + + m_indexWriter->flush(); + m_indexWriter->close(); + // upload + m_indexWriter->upload(); + m_indexWriter->clean(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish saving indexes") << LOG_KV("taskID", m_taskID) + << LOG_KV("indexesSize", _indexes->size()); + return m_indexWriter->fileInfo(); + } + + void appendEvidence(const std::string& _tag, const std::string& _evidence) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("append evidence") << LOG_KV("taskID", m_taskID) + << LOG_KV("tag", _tag); + if (!m_evidenceWriter) + { + return; + } + io::DataBatch::Ptr dataBatch = std::make_shared(); + dataBatch->append(_tag); + dataBatch->append(_evidence); + m_evidenceWriter->writeLine(dataBatch, io::DataSchema::String, "\n"); + m_evidenceWriter->flush(); + } + + void appendEvidences(const std::string& _tag, const std::vector& _evidences) + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start appending evidences") << LOG_KV("taskID", m_taskID) + << LOG_KV("tag", _tag) << LOG_KV("evidencesSize", _evidences.size()); + if (!m_evidenceWriter) + { + return; + } + io::DataBatch::Ptr dataBatch = std::make_shared(); + dataBatch->append(_tag); + m_evidenceWriter->writeLine(dataBatch, io::DataSchema::String, "\n"); + m_evidenceWriter->flush(); + writeByDataBatch(m_evidenceWriter, _evidences); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish appending evidences") + << LOG_KV("taskID", m_taskID) << LOG_KV("tag", _tag) + << LOG_KV("evidencesSize", _evidences.size()); + } + + protocol::FileInfo::Ptr uploadEvidences() + { + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("start uploading evidences") + << LOG_KV("taskID", m_taskID); + if (!m_evidenceWriter) + { + return nullptr; + } + m_evidenceWriter->flush(); + m_evidenceWriter->close(); + // upload + m_evidenceWriter->upload(); + m_evidenceWriter->clean(); + return m_evidenceWriter->fileInfo(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("finish uploading evidences") + << LOG_KV("taskID", m_taskID); + } + + void clean() + { + if (m_reader) + { + m_reader->clean(); + } + if (m_resultWriter) + { + m_resultWriter->clean(); + } + if (m_indexWriter) + { + m_indexWriter->clean(); + } + if (m_evidenceWriter) + { + m_evidenceWriter->clean(); + } + } + + template + static void writeByDataBatch(const ppc::io::LineWriter::Ptr& _writer, const Container& _data) + { + io::DataBatch::Ptr dataBatch = std::make_shared(); + uint32_t count = 0; + for (const auto& line : _data) + { + dataBatch->append(line); + count++; + if (count == 1000000) + { + _writer->writeLine(dataBatch, io::DataSchema::String, "\n"); + _writer->flush(); + dataBatch = std::make_shared(); + count = 0; + } + } + if (count > 0) + { + _writer->writeLine(dataBatch, io::DataSchema::String, "\n"); + _writer->flush(); + } + } + +private: + std::string m_taskID; + ppc::io::LineReader::Ptr m_reader; + ppc::io::LineWriter::Ptr m_resultWriter; + ppc::io::LineWriter::Ptr m_indexWriter; + ppc::io::LineWriter::Ptr m_evidenceWriter; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h new file mode 100644 index 00000000..9ae29a01 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h @@ -0,0 +1,184 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsEcdhTaskState.h + * @author: shawnhe + * @date 2023-09-22 + */ + +#pragma once + +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/io/LineReader.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-psi/src/bs-ecdh-psi/Common.h" +#include "ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h" +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include +#include +#include + +namespace ppc::psi +{ +#define BS_VALIDITY_TERM 86400000 +#define MIN_BS_ACTIVE_COUNT 3 + +class BsEcdhTaskState +{ +public: + using Ptr = std::shared_ptr; + + BsEcdhTaskState(std::string _taskID, protocol::TaskStatus _status, uint32_t _timeoutMinutes) + : m_taskID(std::move(_taskID)), m_status(_status), m_timeoutMinutes(_timeoutMinutes) + { + m_latestActiveTime = bcos::utcSteadyTime(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("new BsEcdhTaskState") << LOG_KV("taskID", m_taskID) + << LOG_KV("createTime", m_latestActiveTime) + << LOG_KV("timeoutMinutes", m_timeoutMinutes); + } + + virtual ~BsEcdhTaskState() = default; + + [[nodiscard]] std::string taskID() const { return m_taskID; } + + [[nodiscard]] protocol::TaskStatus status() const + { + bcos::ReadGuard l(x_state); + return m_status; + } + void updateStatus(protocol::TaskStatus _status) + { + bcos::WriteGuard l(x_state); + m_status = _status; + } + + [[nodiscard]] BsEcdhResult::Ptr result() const + { + bcos::ReadGuard l(x_state); + return m_result; + } + void setResult(BsEcdhResult::Ptr _result) + { + bcos::WriteGuard l(x_state); + m_result = std::move(_result); + } + + void autoPauseChecking() + { + bcos::WriteGuard l(x_state); + // trigger automatic pause + if (m_status == protocol::TaskStatus::RUNNING && + m_latestActiveTime + m_autoPauseThreshold <= bcos::utcSteadyTime()) + { + turnToPausing(); + BS_ECDH_PSI_LOG(INFO) << LOG_DESC("task is pausing") << LOG_KV("taskID", m_taskID); + } + } + + [[nodiscard]] bool isTimeout() + { + bcos::WriteGuard l(x_state); + auto timeout = + isExecutable(m_status) && + m_latestActiveTime + uint64_t(m_timeoutMinutes * 60 * 1000) <= bcos::utcSteadyTime(); + if (timeout) + { + BS_ECDH_PSI_LOG(WARNING) << LOG_DESC("task is timeout") << LOG_KV("taskID", m_taskID); + GetTaskStatusResponse response; + response.taskID = m_taskID; + response.status = toString(protocol::TaskStatus::FAILED); + m_result = std::make_shared(m_taskID, response.serialize()); + m_result->setError( + std::make_shared((int)PSIRetCode::TaskTimeout, "Task is timeout.")); + m_status = protocol::TaskStatus::FAILED; + } + return timeout; + } + + [[nodiscard]] bool isExpired() const + { + bcos::ReadGuard l(x_state); + return m_latestActiveTime + BS_VALIDITY_TERM <= bcos::utcSteadyTime(); + } + + void active() + { + bcos::WriteGuard l(x_state); + + // The task is activated + if (m_status == protocol::TaskStatus::PAUSING && ++activeCount == MIN_BS_ACTIVE_COUNT) + { + activeCount = 0; + turnToRunning(); + } + + m_latestActiveTime = bcos::utcSteadyTime(); + } + + void setupAutoPause() + { + bcos::WriteGuard l(x_state); + turnToRunning(); + } + + void cancelAutoPause() + { + bcos::WriteGuard l(x_state); + m_autoPauseThreshold = BS_VALIDITY_TERM; + } + + void pauseTask() + { + bcos::WriteGuard l(x_state); + if (m_status == protocol::TaskStatus::RUNNING) + { + turnToPausing(); + } + } + + void restartTask() + { + bcos::WriteGuard l(x_state); + if (m_status == protocol::TaskStatus::PAUSING) + { + turnToRunning(); + } + } + +private: + void turnToRunning() + { + m_status = protocol::TaskStatus::RUNNING; + m_autoPauseThreshold = PAUSE_THRESHOLD; + m_latestActiveTime = bcos::utcSteadyTime(); + } + + void turnToPausing() + { + m_status = protocol::TaskStatus::PAUSING; + m_autoPauseThreshold = BS_VALIDITY_TERM; + } + +private: + mutable bcos::SharedMutex x_state; + std::string m_taskID; + protocol::TaskStatus m_status; + BsEcdhResult::Ptr m_result; + std::atomic m_latestActiveTime; + uint64_t m_timeoutMinutes; + uint64_t m_autoPauseThreshold{BS_VALIDITY_TERM}; + uint64_t activeCount{0}; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h b/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h new file mode 100644 index 00000000..8427bebe --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h @@ -0,0 +1,49 @@ +/** +* Copyright (C) 2023 WeDPR. +* SPDX-License-Identifier: Apache-2.0 +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* @file wedpr_ffi_edwards25519.h +* @author: shawnhe +* @date 2023-10-10 + */ + + +#ifndef _WEDPR_FFI_EDWARDS25519_H_ +#define _WEDPR_FFI_EDWARDS25519_H_ + +#include +#include +#include + +#include "wedpr_utilities.h" + +extern "C" { +/** + * C interface for 'generate random scalar'. + */ +int8_t wedpr_random_scalar(COutputBuffer* random); + +/** + * C interface for 'hash to curve'. + */ +int8_t wedpr_hash_to_curve(const CInputBuffer* message, COutputBuffer* point); + +/** + * C interface for 'point multiply by scalar'. + */ +int8_t wedpr_point_scalar_multi( + const CInputBuffer* raw_point, const CInputBuffer* scalar, COutputBuffer* point); +} + +#endif diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h b/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h new file mode 100644 index 00000000..c477d117 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h @@ -0,0 +1,51 @@ +/** +* Copyright (C) 2023 WeDPR. +* SPDX-License-Identifier: Apache-2.0 +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* @file wedpr_utilities.h +* @author: shawnhe +* @date 2023-10-10 +*/ + +#ifndef _WEDPR_UTILITIES_H_ +#define _WEDPR_UTILITIES_H_ + +#include +#include +#include +#include +#include + +#define POINT_SIZE 32 +#define SCALAR_SIZE 32 + +extern "C" { + +struct CInputBuffer +{ + const char* data; + uintptr_t len; +}; + +struct COutputBuffer +{ + char* data; + uintptr_t len; +}; + +const int8_t WEDPR_ERROR = -1; +const int8_t WEDPR_SUCCESS = 0; +} // extern "C" + +#endif diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h new file mode 100644 index 00000000..e132f198 --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file BsModeResult.h + * @author: shawnhe + * @date 2023-09-21 + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "ppc-framework/protocol/Task.h" + +namespace ppc::psi +{ +class BsEcdhResult : public protocol::TaskResult +{ +public: + using Ptr = std::shared_ptr; + BsEcdhResult(std::string const& _taskID, Json::Value _data = Json::Value{}) + : TaskResult(_taskID), m_data(std::move(_data)) + {} + ~BsEcdhResult() override = default; + + [[nodiscard]] Json::Value const& data() const { return m_data; } + + // serialize the taskResult to json + [[nodiscard]] Json::Value serializeToJson() const override + { + Json::Value response; + if (m_error && error()->errorCode()) + { + response["code"] = error()->errorCode(); + response["message"] = error()->errorMessage(); + } + else + { + response["code"] = 0; + response["message"] = "success"; + } + if (!m_data.empty()) + { + response["data"] = m_data; + } + return response; + } + +private: + Json::Value m_data; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h new file mode 100644 index 00000000..e01179be --- /dev/null +++ b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h @@ -0,0 +1,409 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Message.h + * @author: shawnhe + * @date 2023-09-21 + */ + +#pragma once + +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/rpc/RpcTypeDef.h" +#include "ppc-protocol/src/JsonTaskImpl.h" +#include +#include +#include +#include +#include + +namespace ppc::psi +{ + +struct GetTaskStatusRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + + GetTaskStatusRequest() = default; + ~GetTaskStatusRequest() = default; + + explicit GetTaskStatusRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + } +}; + +struct GetTaskStatusResponse +{ + using Ptr = std::shared_ptr; + std::string taskID; + std::string status; + uint32_t step = 0; + uint32_t index = 0; + uint progress = 0; + uint32_t intersections; + uint32_t party0Size; + uint32_t party1Size; + std::string timeCost; + std::string resultFileMd5; + std::string resultFileID; + std::string partnerIndexFileMd5; + std::string partnerIndexFileID; + std::string evidenceFileMd5; + std::string evidenceFileID; + + [[nodiscard]] Json::Value serialize() const + { + Json::Value root; + root["taskID"] = taskID; + root["status"] = status; + root["step"] = step; + root["index"] = index; + root["progress"] = progress; + if (status == protocol::toString(protocol::TaskStatus::COMPLETED)) + { + root["timeCost"] = timeCost; + root["intersections"] = intersections; + root["party0Size"] = party0Size; + root["party1Size"] = party1Size; + if (!resultFileID.empty()) + { + root["resultFileID"] = resultFileID; + root["resultFileMd5"] = resultFileMd5; + } + if (!partnerIndexFileID.empty()) + { + root["partnerIndexFileID"] = partnerIndexFileID; + root["partnerIndexFileMd5"] = partnerIndexFileMd5; + } + if (!evidenceFileID.empty()) + { + root["evidenceFileID"] = evidenceFileID; + root["evidenceFileMd5"] = evidenceFileMd5; + } + } + return root; + } +}; + +struct KillTaskRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + + KillTaskRequest() = default; + ~KillTaskRequest() = default; + + explicit KillTaskRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + } +}; + +struct UpdateTaskStatusRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + std::string status; + + UpdateTaskStatusRequest() = default; + ~UpdateTaskStatusRequest() = default; + + explicit UpdateTaskStatusRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + + if (!_params.isMember("status")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the status field")); + } + status = _params["status"].asString(); + } +}; + +struct RunTaskRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + bool enableAudit{true}; + uint32_t partnerInputsSize{0}; + protocol::DataResource::Ptr dataResource; + + RunTaskRequest() = default; + ~RunTaskRequest() = default; + + explicit RunTaskRequest(const Json::Value& _params, const std::string& _prePath) + { + deserialize(_params, _prePath); + } + + void deserialize(const Json::Value& _params, const std::string& _prePath) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + + if (_params.isMember("enableAudit")) + { + enableAudit = _params["enableAudit"].asBool(); + } + else + { + enableAudit = true; + } + if (_params.isMember("partnerInputsSize")) + { + partnerInputsSize = _params["partnerInputsSize"].asUInt(); + } + if (!_params.isMember("data")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the data field")); + } + auto jsonTask = std::make_shared("", _prePath); + dataResource = jsonTask->decodeDataResource(_params["data"]); + } + + [[nodiscard]] std::string toString() const + { + std::stringstream ss; + ss << "taskID: " << taskID << ", enableAudit: " << enableAudit << ", " + << protocol::printDataResourceInfo(dataResource) << "\n"; + return ss.str(); + } +}; + +struct FetchCipherRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + uint32_t offset; + uint32_t size; + + FetchCipherRequest() = default; + ~FetchCipherRequest() = default; + + explicit FetchCipherRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + + if (!_params.isMember("offset")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the offset field")); + } + offset = _params["offset"].asUInt(); + + if (!_params.isMember("size")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must define the size field")); + } + size = _params["size"].asUInt(); + } +}; + +struct FetchCipherResponse +{ + using Ptr = std::shared_ptr; + std::string taskID; + uint32_t offset; + uint32_t size; + uint32_t total; + std::vector ciphers; + + [[nodiscard]] Json::Value serialize() const + { + Json::Value root; + root["taskID"] = taskID; + root["offset"] = offset; + root["size"] = size; + root["total"] = total; + + Json::Value cipherArray(Json::arrayValue); + for (const std::string& cipher : ciphers) + { + cipherArray.append(cipher); + } + root["ciphers"] = cipherArray; + + return root; + } +}; + +struct SendEcdhCipherRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + uint32_t offset; + uint32_t size; + std::vector ecdhCiphers; + + SendEcdhCipherRequest() = default; + ~SendEcdhCipherRequest() = default; + + explicit SendEcdhCipherRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + + if (!_params.isMember("offset")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the offset field")); + } + offset = _params["offset"].asUInt(); + + if (!_params.isMember("size")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must define the size field")); + } + size = _params["size"].asUInt(); + + if (!_params.isMember("ecdhCiphers")) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)rpc::RpcError::InvalidRequest, "Must set the ecdhCiphers field")); + } + + const Json::Value& ecdhCiphersJson = _params["ecdhCiphers"]; + if (!ecdhCiphersJson.isArray()) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "ecdhCiphers must be an array")); + } + + for (const Json::Value& cipher : ecdhCiphersJson) + { + if (!cipher.isString() || cipher.asString().empty()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)rpc::RpcError::InvalidRequest, "Invalid ecdhCipher format")); + } + ecdhCiphers.push_back(cipher.asString()); + } + } +}; + +struct SendPartnerCipherRequest +{ + using Ptr = std::shared_ptr; + std::string taskID; + uint32_t offset{}; + uint32_t size{}; + uint32_t total{}; + std::vector partnerCiphers; + + SendPartnerCipherRequest() = default; + ~SendPartnerCipherRequest() = default; + + explicit SendPartnerCipherRequest(const Json::Value& _params) { deserialize(_params); } + + void deserialize(const Json::Value& _params) + { + if (!_params.isMember("taskID")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the taskID field")); + } + taskID = _params["taskID"].asString(); + + if (!_params.isMember("offset")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must set the offset field")); + } + offset = _params["offset"].asUInt(); + + if (!_params.isMember("size")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must define the size field")); + } + size = _params["size"].asUInt(); + + if (!_params.isMember("total")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)rpc::RpcError::InvalidRequest, "Must define the total field")); + } + total = _params["total"].asUInt(); + + if (!_params.isMember("partnerCiphers")) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)rpc::RpcError::InvalidRequest, "Must set the partnerCiphers field")); + } + + const Json::Value& partnerCiphersJson = _params["partnerCiphers"]; + if (!partnerCiphersJson.isArray()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)rpc::RpcError::InvalidRequest, "partnerCiphers must be an array")); + } + + for (const Json::Value& cipher : partnerCiphersJson) + { + if (!cipher.isString() || cipher.asString().empty()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)rpc::RpcError::InvalidRequest, "Invalid partnerCiphers format")); + } + partnerCiphers.push_back(cipher.asString()); + } + } +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h new file mode 100644 index 00000000..59a74e84 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIConfig.h + * @author: shawnhe + * @date 2022-12-7 + */ + +#pragma once +#include + +#include + +#include "Common.h" +#include "ppc-crypto/src/oprf/EcdhOprf.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/crypto/Oprf.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/PSIConfig.h" + +namespace ppc::psi +{ +class CM2020PSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + + CM2020PSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front, + ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes, + uint16_t _parallelism = 3, + const front::PPCMessageFactory::Ptr& _msgFactory = + std::make_shared()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::CM_PSI_2PC, _selfPartyID, std::move(_front), + _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes), + m_cryptoBox(std::move(_cryptoBox)), + m_threadPool(std::move(_threadPool)), + m_parallelism(_parallelism) + {} + + virtual ~CM2020PSIConfig() = default; + +public: + crypto::Hash::Ptr const& hash() const { return m_cryptoBox->hashImpl(); } + crypto::EccCrypto::Ptr const& eccCrypto() const { return m_cryptoBox->eccCrypto(); } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + uint16_t parallelism() const { return m_parallelism; } + +private: + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + bcos::ThreadPool::Ptr m_threadPool; + uint16_t m_parallelism; +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h new file mode 100644 index 00000000..4e1efc2e --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIFactory.h + * @author: shawnhe + * @date 2022-12-7 + */ +#pragma once +#include "CM2020PSIImpl.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::psi +{ +class CM2020PSIFactory +{ +public: + using Ptr = std::shared_ptr; + CM2020PSIFactory() = default; + virtual ~CM2020PSIFactory() = default; + + virtual CM2020PSIImpl::Ptr buildCM2020PSI(std::string const& _selfParty, + ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader, + int _holdingMessageMinutes, uint16_t _parallelism) + { + auto config = std::make_shared(_selfParty, std::move(_front), + std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), + _holdingMessageMinutes, _parallelism); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp new file mode 100644 index 00000000..a1aa8a62 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp @@ -0,0 +1,588 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIImpl.cpp + * @author: shawnhe + * @date 2022-12-7 + */ + +#include "CM2020PSIImpl.h" +#include "Common.h" +#include "ppc-tools/src/common/TransTools.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::tools; +using namespace ppc::crypto; +using namespace ppc::io; + +CM2020PSIImpl::CM2020PSIImpl(const CM2020PSIConfig::Ptr& _config, unsigned _idleTimeMs) + : Worker("CM2020-PSI", _idleTimeMs), + TaskGuarder(_config, PSIAlgorithmType::CM_PSI_2PC, "CM2020-PSI-Timer"), + m_config(_config), + m_msgQueue(std::make_shared()), + m_ioService(std::make_shared()), + m_parallelism(m_config->parallelism()) +{ + m_threadPool = std::make_shared( + "CM2020-PSI-ThreadPool", static_cast(std::thread::hardware_concurrency() * 0.75)); + m_ot = std::make_shared(m_config->eccCrypto(), m_config->hash()); +} + +void CM2020PSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +{ + CM2020_PSI_LOG(INFO) << LOG_DESC("receive a task") << LOG_KV("taskID", _task->id()); + addTask(_task, [self = weak_from_this(), taskID = _task->id(), _onTaskFinished]( + ppc::protocol::TaskResult::Ptr&& _result) { + try + { + auto result = std::move(_result); + _onTaskFinished(std::move(result)); + CM2020_PSI_LOG(INFO) << LOG_DESC("finish a task") << LOG_KV("taskID", taskID); + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->m_parallelism++; + + auto timer = std::make_shared( + *(psi->m_ioService), boost::posix_time::minutes(WAITING_PEER_FINISH_M)); + timer->async_wait([self, timer, taskID](boost::system::error_code) { + auto psi = self.lock(); + if (!psi) + { + return; + } + // erase the taskInfo from the gateway + CM2020_PSI_LOG(INFO) << LOG_DESC("erase task info") << LOG_KV("taskID", taskID); + psi->m_config->front()->eraseTaskInfo(taskID); + }); + } + catch (std::exception& e) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("handle callback error after finishing task") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + }); + + m_threadPool->enqueue([self = weak_from_this()]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + psi->asyncRunTask(); + } + catch (std::exception& e) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("asyncRunTask error") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + }); +} + +// run task +void CM2020PSIImpl::asyncRunTask() +{ + CM2020_PSI_LOG(INFO) << LOG_DESC("asyncRunTask") << LOG_KV("current semaphore", m_parallelism); + + if (m_parallelism <= 0) + { + return; + } + + std::pair taskPair; + { + bcos::UpgradableGuard l(x_taskQueue); + if (m_taskQueue.empty()) + { + return; + } + + bcos::UpgradeGuard ul(l); + taskPair = m_taskQueue.front(); + m_taskQueue.pop(); + m_parallelism--; + } + + auto task = taskPair.first; + + // check task + auto error = checkTask(task, 2, true, task->syncResultToPeer(), false); + if (error) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(task); + // notice peer to cancel task + noticePeerToFinish(task); + auto result = std::make_shared(task->id()); + result->setError(std::move(error)); + taskPair.second(std::move(result)); + + // mark this taskID as occupied + m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + return; + } + + // add pending task + auto taskState = + m_taskStateFactory->createTaskState(task, std::move(taskPair.second), false, m_config); + taskState->setPeerID(getPeerID(task)); + taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), task]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + + psi->noticePeerToFinish(task); + }); + addPendingTask(taskState); + + try + { + // prepare reader and writer + auto dataResource = task->selfParty()->dataResource(); + auto reader = loadReader(task->id(), dataResource, DataSchema::Bytes); + taskState->setReader(reader, -1); + + auto role = task->selfParty()->partyIndex(); + if (role == uint16_t(PartyType::Client) || task->syncResultToPeer()) + { + auto writer = loadWriter(task->id(), dataResource, m_enableOutputExists); + taskState->setWriter(writer); + } + + if (role == uint16_t(PartyType::Client)) + { + auto receiver = std::make_shared(m_config, taskState, m_ot); + receiver->asyncRunTask(); + addReceiver(std::move(receiver)); + } + else if (role == uint16_t(PartyType::Server)) + { + auto sender = std::make_shared( + m_config, taskState, m_ioService, m_ot, m_threadPool); + sender->asyncRunTask(); + addSender(std::move(sender)); + } + else + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("undefined task role") << unsigned(role); + onSelfError(task->id(), + std::make_shared( + (int)CM2020PSIRetCode::UNDEFINED_TASK_ROLE, "undefined task role"), + true); + } + } + catch (bcos::Error const& e) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + onSelfError( + task->id(), std::make_shared(e.errorCode(), e.errorMessage()), true); + } + catch (const std::exception& e) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) + << LOG_KV("exception", boost::diagnostic_information(e)); + onSelfError(task->id(), + std::make_shared((int)CM2020PSIRetCode::ON_EXCEPTION, + "exception caught while running task: " + boost::diagnostic_information(e)), + true); + } + + // notify the taskInfo to the front + error = m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + if (error && error->errorCode()) + { + onSelfError(task->id(), error, true); + } +} + +// register to the front to get the message related to cm2020-psi +void CM2020PSIImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) +{ + try + { + m_msgQueue->push(_msg); + + // notify to handle the message + wakeupWorker(); + } + catch (std::exception const& e) + { + CM2020_PSI_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("exception", boost::diagnostic_information(e)); + } +} + +void CM2020PSIImpl::start() +{ + if (m_started) + { + CM2020_PSI_LOG(ERROR) << LOG_DESC("The CM2020-PSI has already been started"); + return; + } + CM2020_PSI_LOG(INFO) << LOG_DESC("Start the CM2020-PSI"); + m_started = true; + + // start a thread to execute task + startWorking(); + + m_thread = std::make_shared([&] { + bcos::pthread_setThreadName("cm2020_io_service"); + while (m_started) + { + try + { + m_ioService->run(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (m_started && m_ioService->stopped()) + { + m_ioService->restart(); + } + } + catch (std::exception& e) + { + FRONT_LOG(WARNING) << LOG_DESC("Exception in CM2020-PSI Thread:") + << boost::diagnostic_information(e); + } + } + CM2020_PSI_LOG(INFO) << "CM2020-PSI exit"; + }); + + startPingTimer(); +} + +void CM2020PSIImpl::stop() +{ + if (!m_started) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_DESC("Stop CM2020-PSI"); + m_started = false; + + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } + + if (m_threadPool) + { + m_threadPool->stop(); + } + if (m_ioService) + { + m_ioService->stop(); + } + // stop the thread + if (m_thread->get_id() != std::this_thread::get_id()) + { + m_thread->join(); + } + else + { + m_thread->detach(); + } + stopPingTimer(); + + CM2020_PSI_LOG(INFO) << LOG_DESC("CM2020-PSI stopped"); +} + +void CM2020PSIImpl::onReceivedErrorNotification(const std::string& _taskID) +{ + // finish the task while the peer is failed + auto taskState = findPendingTask(_taskID); + if (taskState) + { + taskState->onPeerNotifyFinish(); + + wakeupWorker(); + } +} + +void CM2020PSIImpl::onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +{ + auto taskState = findPendingTask(_taskID); + if (!taskState) + { + return; + } + + CM2020_PSI_LOG(ERROR) << LOG_DESC("onSelfError") << LOG_KV("task", _taskID) + << LOG_KV("exception", _error->errorMessage()) + << LOG_KV("noticePeer", _noticePeer); + + auto result = std::make_shared(taskState->task()->id()); + result->setError(std::move(_error)); + taskState->onTaskFinished(result, _noticePeer); + + wakeupWorker(); +} + +// cm2020-psi main processing function +// for ut to make this function public +void CM2020PSIImpl::executeWorker() +{ + checkFinishedTask(); + auto result = m_msgQueue->tryPop(c_popWaitMs); + if (result.first) + { + handleReceivedMessage(result.second); + return; + } + waitSignal(); +} + +void CM2020PSIImpl::checkFinishedTask() +{ + std::set finishedTask; + { + bcos::WriteGuard l(x_pendingTasks); + if (m_pendingTasks.empty()) + { + return; + } + + for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();) + { + auto task = it->second; + if (task->finished()) + { + finishedTask.insert(it->first); + } + it++; + } + } + for (auto& taskID : finishedTask) + { + removeReceiver(taskID); + removeSender(taskID); + removePendingTask(taskID); + asyncRunTask(); + } +} + +void CM2020PSIImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message) +{ + CM2020_PSI_LOG(TRACE) << LOG_DESC("handleReceivedMessage") << printPPCMsg(_message); + + m_config->threadPool()->enqueue([self = weak_from_this(), _message]() { + try + { + auto psi = self.lock(); + if (!psi) + { + return; + } + switch (int(_message->messageType())) + { + case int(CommonMessageType::ErrorNotification): + { + psi->onReceivedErrorNotification(_message->taskID()); + break; + } + case int(CommonMessageType::PingPeer): + { + break; + } + case int(CM2020PSIMessageType::HELLO_RECEIVER): + { + psi->onReceiveHelloReceiver(_message); + break; + } + case int(CM2020PSIMessageType::HELLO_SENDER): + { + psi->onReceiveHelloSender(_message); + break; + } + case int(CM2020PSIMessageType::RECEIVER_SIZE): + { + psi->onReceiveReceiverSize(_message); + break; + } + case int(CM2020PSIMessageType::SENDER_SIZE): + { + psi->onReceiveSenderSize(_message); + break; + } + case int(CM2020PSIMessageType::POINT_A): + { + psi->onReceivePointA(_message); + break; + } + case int(CM2020PSIMessageType::POINT_B_ARRAY): + { + psi->onReceiveBatchPointB(_message); + break; + } + case int(CM2020PSIMessageType::MATRIX): + { + psi->onReceiveMatrix(_message); + break; + } + case int(CM2020PSIMessageType::DO_NEXT_ROUND): + { + psi->onReceiveDoNextRound(_message); + break; + } + case int(CM2020PSIMessageType::HASHES): + { + psi->onReceiveHashes(_message); + break; + } + case int(CM2020PSIMessageType::RESULTS_SIZE): + { + psi->onReceiveResultCount(_message); + break; + } + case int(CM2020PSIMessageType::RESULTS): + { + psi->onReceiveResults(_message); + break; + } + default: + { + CM2020_PSI_LOG(WARNING) + << LOG_DESC("unsupported messageType ") << unsigned(_message->messageType()); + break; + } + } + } + catch (std::exception const& e) + { + CM2020_PSI_LOG(WARNING) + << LOG_DESC("handleReceivedMessage exception") + << LOG_KV("type", unsigned(_message->messageType())) << printPPCMsg(_message) + << LOG_KV("exception", boost::diagnostic_information(e)); + } + }); +} + + +void CM2020PSIImpl::onReceiveHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->onHandshakeDone(); + } +} + +void CM2020PSIImpl::onReceiveHelloSender(const ppc::front::PPCMessageFace::Ptr& _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onHandshakeDone(_message); + } +} + +void CM2020PSIImpl::onReceiveReceiverSize(ppc::front::PPCMessageFace::Ptr _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onReceiverSizeReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveSenderSize(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->onSenderSizeReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceivePointA(ppc::front::PPCMessageFace::Ptr _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onPointAReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveBatchPointB(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->onBatchPointBReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveMatrix(ppc::front::PPCMessageFace::Ptr _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onMatrixColumnReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveDoNextRound(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->onDoNextRoundReceived(); + } +} + +void CM2020PSIImpl::onReceiveHashes(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->onHashesReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveResultCount(ppc::front::PPCMessageFace::Ptr _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onResultCountReceived(std::move(_message)); + } +} + +void CM2020PSIImpl::onReceiveResults(ppc::front::PPCMessageFace::Ptr _message) +{ + auto sender = findSender(_message->taskID()); + if (sender) + { + sender->onResultReceived(std::move(_message)); + } +} diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h new file mode 100644 index 00000000..06012237 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h @@ -0,0 +1,187 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIImpl.h + * @author: shawnhe + * @date 2022-12-7 + */ + +#pragma once +#include +#include +#include +#include +#include + +#include "../psi-framework/TaskGuarder.h" +#include "../psi-framework/TaskState.h" +#include "CM2020PSIConfig.h" +#include "Common.h" +#include "core/CM2020PSIReceiver.h" +#include "core/CM2020PSISender.h" +#include "ppc-crypto/src/randomot/SimplestOT.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-front/ppc-front/PPCChannel.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-tools/src/common/TransTools.h" + +namespace ppc::psi +{ +class CM2020PSIImpl : public bcos::Worker, + public TaskGuarder, + public TaskFrameworkInterface, + public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + using CM2020PSIMsgQueue = bcos::ConcurrentQueue; + using CM2020PSIMsgQueuePtr = std::shared_ptr; + + CM2020PSIImpl(const CM2020PSIConfig::Ptr& _config, unsigned _idleTimeMs = 0); + + ~CM2020PSIImpl() override = default; + + void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + + // register to the front to get the message related to cm2020-psi + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; + + void start() override; + void stop() override; + void onReceivedErrorNotification(const std::string& _taskID) override; + void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override; + + // cm2020-psi main processing function + // for ut to make this function public + void executeWorker() override; + +protected: + void asyncRunTask(); + void checkFinishedTask(); + + void handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message); + + void onReceiveHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message); + void onReceiveHelloSender(const ppc::front::PPCMessageFace::Ptr& _message); + + void onReceiveReceiverSize(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveSenderSize(ppc::front::PPCMessageFace::Ptr _message); + + void onReceivePointA(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveBatchPointB(ppc::front::PPCMessageFace::Ptr _message); + + void onReceiveMatrix(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveDoNextRound(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveHashes(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveResultCount(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveResults(ppc::front::PPCMessageFace::Ptr _message); + + void addTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) + { + bcos::WriteGuard l(x_taskQueue); + m_taskQueue.push({std::move(_task), std::move(_onTaskFinished)}); + } + + CM2020PSIReceiver::Ptr findReceiver(const std::string& _taskID) + { + bcos::ReadGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + return it->second; + } + return nullptr; + } + void addReceiver(CM2020PSIReceiver::Ptr _receiver) + { + bcos::WriteGuard l(x_receivers); + m_receivers[_receiver->taskID()] = _receiver; + } + void removeReceiver(const std::string& _taskID) + { + bcos::WriteGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + m_receivers.erase(it); + } + } + + CM2020PSISender::Ptr findSender(const std::string& _taskID) + { + bcos::ReadGuard l(x_senders); + auto it = m_senders.find(_taskID); + if (it != m_senders.end()) + { + return it->second; + } + return nullptr; + } + void addSender(CM2020PSISender::Ptr _sender) + { + bcos::WriteGuard l(x_senders); + m_senders[_sender->taskID()] = _sender; + } + void removeSender(const std::string& _taskID) + { + bcos::WriteGuard l(x_senders); + auto it = m_senders.find(_taskID); + if (it != m_senders.end()) + { + m_senders.erase(it); + } + } + +protected: + // allow the output-path exists, for ut + bool m_enableOutputExists = false; + +private: + void waitSignal() + { + boost::unique_lock l(x_signal); + m_signal.wait_for(l, boost::chrono::milliseconds(5)); + } + + void wakeupWorker() { m_signal.notify_all(); } + + CM2020PSIConfig::Ptr m_config; + CM2020PSIMsgQueuePtr m_msgQueue; + std::shared_ptr m_ioService; + std::shared_ptr m_threadPool; + crypto::SimplestOT::Ptr m_ot; + + std::shared_ptr m_thread; + + std::atomic m_parallelism; + std::queue > m_taskQueue; + mutable bcos::SharedMutex x_taskQueue; + + std::unordered_map m_receivers; + mutable bcos::SharedMutex x_receivers; + + std::unordered_map m_senders; + mutable bcos::SharedMutex x_senders; + + bool m_started{false}; + boost::condition_variable m_signal; + boost::mutex x_signal; + + const int c_popWaitMs = 5; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt b/cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt new file mode 100644 index 00000000..58810e52 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt @@ -0,0 +1,31 @@ +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-psi/cm2020-psi/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") + +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () + +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${CM2020_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST}) +target_include_directories(${CM2020_PSI_TARGET} PUBLIC + $) + +target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc) diff --git a/cpp/ppc-psi/src/cm2020-psi/Common.h b/cpp/ppc-psi/src/cm2020-psi/Common.h new file mode 100644 index 00000000..52de3fb2 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/Common.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-12-7 + */ +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/PPCMessageFace.h" + +#include +#include + + +namespace ppc::psi +{ + +DERIVE_PPC_EXCEPTION(CM2020Exception); + +#define CM2020_PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: CM2020-PSI") +#define HIGH_PERFORMANCE_BUCKET_NUMBER (256) +#define DEFAULT_BUCKET_NUMBER (320) +#define MAX_BUCKET_NUMBER (400) +#define DEFAULT_HANDLE_WIDTH_POWER (24) +#define MIN_HANDLE_WIDTH (8) +#define MAX_SEND_BUFFER_LENGTH (8 * 1024 * 1024) +#define MIN_BUCKET_SIZE (8 * 1024) +#define MAX_COMPUTE_HASH_SIZE (1 << 26) +#define ENCODE_RATE (2) +#define RESULT_LEN_BYTE (16) +#define WAITING_PEER_FINISH_M (10) + +enum class CM2020PSIMessageType : uint8_t +{ + HELLO_RECEIVER = 0x01, + HELLO_SENDER = 0x02, + RECEIVER_SIZE = 0x03, + SENDER_SIZE = 0x04, + POINT_A = 0x05, + POINT_B_ARRAY = 0x06, + MATRIX = 0x07, + DO_NEXT_ROUND = 0x08, + HASHES = 0x09, + RESULTS_SIZE = 0x0a, + RESULTS = 0x0b +}; + +enum class CM2020PSIRetCode : int +{ + ON_EXCEPTION = -3000, + UNDEFINED_TASK_ROLE = -3001, + INVALID_TASK_PARAM = -3002 +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp new file mode 100644 index 00000000..f8b5d3dd --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp @@ -0,0 +1,981 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIReceiver.cpp + * @author: shawnhe + * @date 2022-12-7 + */ + +#include "CM2020PSIReceiver.h" +#include "CM2020PSI.h" +#include "openssl/rand.h" +#include "ppc-crypto/src/prng/AESPRNG.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "ppc-tools/src/common/TransTools.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::tools; + +CM2020PSIReceiver::CM2020PSIReceiver( + CM2020PSIConfig::Ptr _config, TaskState::Ptr _taskState, SimplestOT::Ptr _ot) + : m_config(std::move(_config)), m_taskState(std::move(_taskState)), m_ot(std::move(_ot)) +{ + auto task = m_taskState->task(); + m_taskID = task->id(); + m_params = std::make_shared(task->param()); + m_params->setSyncResults(task->syncResultToPeer()); + m_params->setLowBandwidth(task->lowBandwidth()); + m_cm2020Result = std::make_shared(m_taskID); + m_startTimePoint = std::chrono::high_resolution_clock::now(); + m_costs.resize(4); + m_progress = std::make_shared(m_config->threadPool()); +} + +void CM2020PSIReceiver::asyncRunTask() +{ + CM2020_PSI_LOG(INFO) << LOG_DESC("asyncRunTask as receiver") << LOG_KV("taskID", m_taskID); + + m_progress->reset(3, [self = weak_from_this()]() { + /** + * 1. prepare inputs done + * 2. run random OT done + * 3. sender size received + */ + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->preprocessInputs(); + receiver->doOprf(); + }); + + m_config->threadPool()->enqueue([self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->runReceiver(); + }); +} + +void CM2020PSIReceiver::runReceiver() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("runReceiver") << LOG_KV("taskID", m_taskID) + << LOG_KV("syncResults", m_params->enableSyncResults()) + << LOG_KV("bucketNumber", m_params->bucketNumber()); + try + { + ppctars::CM2020Params cm2020Params; + cm2020Params.bucketNumber = m_params->bucketNumber(); + cm2020Params.enableSyncResults = m_params->enableSyncResults(); + cm2020Params.lowBandwidth = m_params->lowBandwidth(); + cm2020Params.seed.resize(16); + if (RAND_bytes(cm2020Params.seed.data(), 16) != 1) + { + BOOST_THROW_EXCEPTION(CM2020Exception()); + } + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, + std::make_shared()); + message->setMessageType(uint8_t(CM2020PSIMessageType::HELLO_SENDER)); + ppctars::serialize::encode(cm2020Params, *message->data()); + CM2020_PSI_LOG(INFO) << LOG_BADGE("runReceiver") << LOG_KV("taskID", m_taskID) + << LOG_KV("syncResults", m_params->enableSyncResults()) + << LOG_KV("message data:", *(bcos::toHexString(*(message->data())))) + << LOG_KV("message size:", message->data()->size()) + << LOG_KV("bucketNumber", m_params->bucketNumber()); + // shake hands with each other + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); + + prepareInputs(); + } + catch (const std::exception& e) + { + onReceiverException("runReceiver", e); + } +} + + +void CM2020PSIReceiver::onHandshakeDone() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onHandshakeDone") << LOG_KV("taskID", m_taskID); + try + { + m_startTimePoint = std::chrono::high_resolution_clock::now(); + runRandomOT(); + } + catch (const std::exception& e) + { + onReceiverException("runRandomOT", e); + } +} + + +void CM2020PSIReceiver::prepareInputs() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("prepareInputs") << LOG_KV("taskID", m_taskID); + try + { + auto originData = m_taskState->task()->selfParty()->dataResource()->rawData(); + if (!originData.empty()) + { + m_ioByInterface = true; + } + + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(CM2020Exception() << bcos::errinfo_comment("data is empty")); + } + + m_rInputSize = m_originInputs->size(); + m_originLocations.reserve(m_rInputSize); + m_originLocations.resize(m_rInputSize); + m_oprfOutputs.reserve(m_rInputSize); + m_oprfOutputs.resize(m_rInputSize); + + syncInputsSize(); + } + catch (const std::exception& e) + { + onReceiverException("prepareInputs", e); + } +} + +void CM2020PSIReceiver::syncInputsSize() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("syncInputsSize") << LOG_KV("taskID", m_taskID); + + auto data = std::make_shared(); + encodeUnsignedNum(data, m_rInputSize); + CM2020_PSI_LOG(TRACE) << LOG_BADGE("syncInputsSize") << LOG_KV("taskID", m_taskID) + << LOG_KV("data", *bcos::toHexString(*data)) + << LOG_KV("m_rInputSize", m_rInputSize); + auto message = m_config->ppcMsgFactory()->buildPPCMessage( + uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, data); + message->setMessageType(uint8_t(CM2020PSIMessageType::RECEIVER_SIZE)); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); + + m_progress->mark("PREPARE_INPUTS"); +} + +void CM2020PSIReceiver::runRandomOT() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("runOT as sender") << LOG_KV("taskID", m_taskID); + m_otState = m_ot->senderGeneratePointA(); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, m_otState.second); + message->setMessageType(uint8_t(CM2020PSIMessageType::POINT_A)); + + // send point A to ot receiver + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); +} + +void CM2020PSIReceiver::onBatchPointBReceived(PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("handleBatchPointB") << LOG_KV("taskID", m_taskID); + + try + { + m_senderKeys = m_ot->finishSender(m_otState.first, m_otState.second, _message->data()); + m_progress->mark("RANDOM_OT"); + } + catch (const std::exception& e) + { + onReceiverException("onBatchPointBReceived", e); + } +} + +void CM2020PSIReceiver::onSenderSizeReceived(front::PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onSenderSizeReceived") << LOG_KV("taskID", m_taskID); + try + { + decodeUnsignedNum(m_sInputSize, _message->data()); + m_progress->mark("RECEIVE_SIZE"); + } + catch (const std::exception& e) + { + onReceiverException("onSenderSizeReceived", e); + } +} + + +void CM2020PSIReceiver::preprocessInputs() +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + CM2020_PSI_LOG(INFO) << LOG_BADGE("preprocessInputs") << LOG_KV("taskID", m_taskID); + bcos::bytesConstRef seed(m_taskID); + auto hash = m_config->hash(); + + tbb::parallel_for(tbb::blocked_range(0U, m_rInputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto state = hash->init(); + hash->update(state, seed); + bcos::bytes data = m_originInputs->getBytes(i); + hash->update(state, bcos::ref(bcos::bytes(data.begin(), data.end()))); + auto res = hash->final(state); + memcpy((bcos::byte*)m_originLocations[i].data(), res.data(), 8 * sizeof(uint32_t)); + } + }); + } + catch (const std::exception& e) + { + onReceiverException("preprocessInputs", e); + } +} + +// considering the memory occupation, we handle the global matrix by 'm_handleWidth' column per +// round, when and only when this round ends, the next round will be made +void CM2020PSIReceiver::doOprf() +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + if (m_oprfRound == 0) + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of rot and first three steps + m_costs[0] = + std::chrono::duration_cast(end - m_startTimePoint) + .count(); + + initMatrixParams(); + } + + CM2020_PSI_LOG(INFO) << LOG_BADGE("doOprf") << LOG_KV("taskID", m_taskID) + << LOG_KV("round", m_oprfRound); + + uint32_t offset = m_oprfRound * m_handleWidth; + if (offset >= m_params->bucketNumber()) + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of oprf + m_costs[1] = + std::chrono::duration_cast(end - m_startTimePoint) + .count(); + + // after negotiating matrix and computing oprf, + // we can receive hash from sender and compute intersection + m_config->threadPool()->enqueue([self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->doPsi(); + }); + } + else + { + m_currentWidth = offset + m_handleWidth < m_params->bucketNumber() ? + m_handleWidth : + m_params->bucketNumber() - offset; + + m_progress->reset(m_currentWidth + 1 + 1, [self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->increaseOprfRound(); + receiver->doOprf(); + }); + + constructMatrices(offset, m_currentWidth); + + for (uint32_t i = 0; i < m_currentWidth; ++i) + { + m_config->threadPool()->enqueue([self = weak_from_this(), i, offset]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + + // encrypt matrix A ^ matrix Delta by column, and send it to sender + receiver->negotiateMatrix(offset + i, i); + receiver->progress()->mark("MATRIX" + std::to_string(i)); + }); + } + + // when we get matrix A, we can compute the inputs for final hash and negotiate matrix + // with counterparty in parallel + computeOprfOutputs(offset, m_currentWidth); + m_progress->mark("OPRF"); + } + } + catch (const std::exception& e) + { + onReceiverException("doOprf", e); + } +} + +void CM2020PSIReceiver::initMatrixParams() +{ + if (m_taskState->taskDone()) + { + return; + } + uint32_t maxInputSize = std::max(m_rInputSize, m_sInputSize); + + m_handleWidth = m_params->bucketNumber(); + + if (maxInputSize > (1 << DEFAULT_HANDLE_WIDTH_POWER)) + { + // while handling mass data, we do oprf block by block to prevent OOM + m_handleWidth = m_params->bucketNumber() / + (1 << int(std::ceil(log2(maxInputSize) - DEFAULT_HANDLE_WIDTH_POWER))); + m_handleWidth = std::max(m_handleWidth, (uint32_t)MIN_HANDLE_WIDTH); + } + + m_bucketSizeInBytes = std::max( + uint32_t(MIN_BUCKET_SIZE / 8), uint32_t(((long double)maxInputSize * ENCODE_RATE + 7) / 8)); + m_mask = m_bucketSizeInBytes * 8; + + CM2020_PSI_LOG(INFO) << LOG_BADGE("initMatrixParams") << LOG_KV("taskID", m_taskID) + << LOG_KV("rInputSize", m_rInputSize) + << LOG_KV("bucketNumber", m_params->bucketNumber()) + << LOG_KV("handleWidth", m_handleWidth) + << LOG_KV("bucketSizeInBytes", m_bucketSizeInBytes) + << LOG_KV("mask", m_mask); +} + +void CM2020PSIReceiver::constructMatrices(uint32_t _offset, uint32_t _width) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("constructMatrices") << LOG_KV("offset", _offset) + << LOG_KV("width", _width) << LOG_KV("taskID", m_taskID); + + if (_offset == 0) + { + m_matrixA.reserve(m_handleWidth); + m_matrixA.resize(m_handleWidth); + + m_matrixDelta.reserve(m_handleWidth); + m_matrixDelta.resize(m_handleWidth); + } + + tbb::parallel_for(tbb::blocked_range(0U, _width), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + // construct matrix A + if (_offset == 0) + { + m_matrixA[i].reserve(m_bucketSizeInBytes); + m_matrixA[i].resize(m_bucketSizeInBytes); + m_matrixDelta[i].reserve(m_bucketSizeInBytes); + m_matrixDelta[i].resize(m_bucketSizeInBytes); + } + AESPRNG prng(m_senderKeys[_offset + i][0]); + prng.generate(m_matrixA[i].data(), m_bucketSizeInBytes); + + // int matrix D + memset(m_matrixDelta[i].data(), 255, m_bucketSizeInBytes); + } + }); +} + + +void CM2020PSIReceiver::negotiateMatrix(uint32_t _bucketIndex, uint32_t _matrixIndex) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(DEBUG) << LOG_BADGE("negotiateMatrix") << LOG_KV("bucketIndex", _bucketIndex) + << LOG_KV("taskID", m_taskID); + + for (uint32_t j = 0; j < m_rInputSize; j++) + { + uint32_t location = (m_originLocations[j][(_bucketIndex) % 4] * (_bucketIndex) + + m_originLocations[j][4 + (_bucketIndex) % 4]) % + m_mask; + m_matrixDelta[_matrixIndex][location >> 3] &= ~(1 << (location & 7)); + } + + auto buffer = std::make_shared(m_bucketSizeInBytes); + + // use key[1] to encrypt matrixA ^ matrixDelta + AESPRNG prng(m_senderKeys[_bucketIndex][1]); + prng.generate(buffer->data(), m_bucketSizeInBytes); + + tbb::parallel_for(tbb::blocked_range(0U, m_bucketSizeInBytes), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + buffer->at(i) ^= m_matrixA[_matrixIndex][i]; + buffer->at(i) ^= m_matrixDelta[_matrixIndex][i]; + } + }); + + // handle m_bucketSizeInBytes > 4M + uint32_t totalRound = + (m_bucketSizeInBytes + MAX_SEND_BUFFER_LENGTH - 1) / MAX_SEND_BUFFER_LENGTH; + for (uint32_t round = 0; round < totalRound; round++) + { + uint32_t rLen = round * MAX_SEND_BUFFER_LENGTH; + uint32_t currentLen = uint32_t((round + 1) * MAX_SEND_BUFFER_LENGTH) > m_bucketSizeInBytes ? + m_bucketSizeInBytes - rLen : + MAX_SEND_BUFFER_LENGTH; + + auto sendBuffer = std::make_shared( + buffer->begin() + rLen, buffer->begin() + rLen + currentLen); + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, sendBuffer); + message->setSeq(_bucketIndex * totalRound + round); + message->setMessageType(uint8_t(CM2020PSIMessageType::MATRIX)); + + if (m_params->lowBandwidth()) + { + auto error = m_config->sendMessage(m_taskState->peerID(), message); + if (error && error->errorCode()) + { + onReceiverTaskDone(std::move(error)); + } + } + else + { + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); + } + } +} + +void CM2020PSIReceiver::computeOprfOutputs(uint32_t _offset, uint32_t _width) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("computeOprfOutputs") << LOG_KV("offset", _offset) + << LOG_KV("width", _width) << LOG_KV("taskID", m_taskID); + try + { + tbb::parallel_for(tbb::blocked_range(0U, m_rInputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto newSize = (_offset + _width + 7) / 8; + m_oprfOutputs[i].reserve(newSize); + m_oprfOutputs[i].resize(newSize); + for (uint32_t j = 0; j < _width; j++) + { + uint32_t location = (m_originLocations[i][(_offset + j) % 4] * (_offset + j) + + m_originLocations[i][4 + (_offset + j) % 4]) % + m_mask; + + m_oprfOutputs[i][(_offset + j) >> 3] |= + (uint8_t)((bool)(m_matrixA[j][location >> 3] & (1 << (location & 7)))) + << ((_offset + j) & 7); + } + } + }); + } + catch (const std::exception& e) + { + onReceiverException("computeOprfOutputs", e); + } +} + +void CM2020PSIReceiver::onDoNextRoundReceived() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onDoNextRoundReceived") << LOG_KV("taskID", m_taskID); + m_progress->mark("NEXT"); +} + +void CM2020PSIReceiver::doPsi() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("doPsi") << LOG_KV("taskID", m_taskID); + + try + { + // receive 2^18 hashes per round + uint32_t number = MAX_SEND_BUFFER_LENGTH / RESULT_LEN_BYTE; + m_maxHashSeq = (m_sInputSize + number - 1) / number; + + m_progress->reset(m_maxHashSeq, [self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->finishPsi(); + }); + + clearOprfMemory(); + computeHash(); + + m_rResults = std::make_shared>(m_rInputSize, 0); + if (m_params->enableSyncResults()) + { + m_sResults = std::make_shared>(m_sInputSize, 0); + } + + bcos::WriteGuard l(x_receivedHash); + m_hashReady.exchange(true); + + tryComputeIntersection(); + } + catch (const std::exception& e) + { + onReceiverException("doPsi", e); + } +} + +void CM2020PSIReceiver::clearOprfMemory() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("clearOprfMemory") << LOG_KV("taskID", m_taskID); + + std::vector>().swap(m_originLocations); + std::vector().swap(m_matrixA); + std::vector().swap(m_matrixDelta); + MallocExtension::instance()->ReleaseFreeMemory(); +} + +void CM2020PSIReceiver::computeHash() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("computeHash") << LOG_KV("taskID", m_taskID); + + // avoid OOM + uint32_t totalRound = (m_rInputSize + MAX_COMPUTE_HASH_SIZE - 1) / MAX_COMPUTE_HASH_SIZE; + + auto hash = m_config->hash(); + for (uint32_t round = 0; round < totalRound; round++) + { + uint32_t offset = round * MAX_COMPUTE_HASH_SIZE; + uint32_t currentNum = (round + 1) * MAX_COMPUTE_HASH_SIZE > m_rInputSize ? + m_rInputSize - offset : + MAX_COMPUTE_HASH_SIZE; + + std::vector hashes; + hashes.reserve(currentNum); + hashes.resize(currentNum); + + tbb::parallel_for(tbb::blocked_range(0U, currentNum), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto res = hash->hash(bcos::ref(m_oprfOutputs[offset + i])); + memcpy((bcos::byte*)&hashes[i], res.data(), RESULT_LEN_BYTE); + } + }); + + for (uint32_t i = 0; i < currentNum; i++) + { + m_hashes[hashes[i]] = offset + i; + } + } + + // clear oprf outputs + std::vector>().swap(m_oprfOutputs); + MallocExtension::instance()->ReleaseFreeMemory(); +} + + +void CM2020PSIReceiver::tryComputeIntersection() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("tryComputeIntersection") << LOG_KV("taskID", m_taskID); + for (auto& it : m_receivedHash) + { + uint32_t round = it.first; + bcos::bytesPointer buffer = it.second; + m_config->threadPool()->enqueue([round, buffer, self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->computeIntersection(round, buffer); + }); + } +} + +void CM2020PSIReceiver::onHashesReceived(PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(DEBUG) << LOG_BADGE("handleReceivedHash") << LOG_KV("taskID", m_taskID) + << LOG_KV("seq", _message->seq()); + + try + { + { + bcos::WriteGuard l(x_receivedHash); + if (!m_hashReady) + { + m_receivedHash[_message->seq()] = _message->data(); + return; + } + } + computeIntersection(_message->seq(), _message->data()); + } + catch (const std::exception& e) + { + onReceiverException("onHashesReceived", e); + } +} + +void CM2020PSIReceiver::computeIntersection(uint32_t _round, bcos::bytesPointer _buffer) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(DEBUG) << LOG_BADGE("computeIntersection") << LOG_KV("taskID", m_taskID) + << LOG_KV("seq", _round); + try + { + uint32_t hashNum = _buffer->size() / RESULT_LEN_BYTE; + uint32_t handleNum = MAX_SEND_BUFFER_LENGTH / RESULT_LEN_BYTE; + for (uint32_t i = 0; i < hashNum; i++) + { + u128 key = *(u128*)(_buffer->data() + i * RESULT_LEN_BYTE); + if (m_hashes.find(key) != m_hashes.end()) + { + m_rResults->at(m_hashes[key]) = 1; + if (m_params->enableSyncResults()) + { + m_sResults->at(_round * handleNum + i) = 1; + } + m_resultCount++; + } + } + + m_progress->mark((uint64_t)_round); + } + catch (const std::exception& e) + { + onReceiverException("computeIntersection", e); + } +} + +void CM2020PSIReceiver::finishPsi() +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of psi + m_costs[2] = + std::chrono::duration_cast(end - m_startTimePoint).count(); + + CM2020_PSI_LOG(INFO) << LOG_BADGE("finishPsi") LOG_KV("taskID", m_taskID); + if (m_params->enableSyncResults()) + { + syncResults(m_resultCount); + } + + if (m_resultCount > 0) + { + // save results + saveResults(); + } + + onReceiverTaskDone(nullptr); + } + catch (const std::exception& e) + { + onReceiverException("finishPsi", e); + } +} + +void CM2020PSIReceiver::syncResults(uint32_t _count) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("syncResults") << LOG_KV("taskID", m_taskID) + << LOG_KV("count", _count); + + // send the count of results + auto countData = std::make_shared(); + encodeUnsignedNum(countData, uint32_t(_count)); + auto message = m_config->ppcMsgFactory()->buildPPCMessage( + uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, countData); + message->setMessageType(uint8_t(CM2020PSIMessageType::RESULTS_SIZE)); + + auto error = m_config->sendMessage(m_taskState->peerID(), message); + if (error && error->errorCode()) + { + onReceiverTaskDone(std::move(error)); + } + + if (_count == 0) + { + return; + } + + // send 2^20 indexes per round + uint32_t number = MAX_SEND_BUFFER_LENGTH / sizeof(uint32_t); + uint32_t count = 0, round = 0, totalCount = 0; + uint32_t currentLen = (round + 1) * number > _count ? _count - round * number : number; + auto buffer = std::make_shared(currentLen * sizeof(uint32_t)); + + for (uint32_t index = 0; index < m_sInputSize; index++) + { + if (m_sResults->at(index)) + { + encodeUnsignedNum(buffer->data() + count * sizeof(uint32_t), index); + count++, totalCount++; + if (count == currentLen) + { + auto resMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, buffer); + resMessage->setMessageType(uint8_t(CM2020PSIMessageType::RESULTS)); + resMessage->setSeq(round); + + if (m_params->lowBandwidth()) + { + error = m_config->sendMessage(m_taskState->peerID(), resMessage); + if (error && error->errorCode()) + { + onReceiverTaskDone(std::move(error)); + } + } + else + { + m_config->front()->asyncSendMessage( + m_taskState->peerID(), resMessage, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); + } + + if (totalCount < _count) + { + count = 0, round++; + currentLen = (round + 1) * number > _count ? _count - round * number : number; + buffer = std::make_shared(currentLen * sizeof(uint32_t)); + } + } + } + } +} + +void CM2020PSIReceiver::saveResults() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("saveResults") << LOG_KV("taskID", m_taskID); + try + { + if (m_ioByInterface) + { + std::vector results; + for (uint32_t index = 0; index < m_rInputSize; index++) + { + if (m_rResults->at(index)) + { + auto line = m_originInputs->getBytes(index); + results.emplace_back(std::string(line.begin(), line.end())); + } + } + m_cm2020Result->m_outputs.emplace_back(std::move(results)); + } + else + { + DataBatch::Ptr finalResults = std::make_shared(); + for (uint32_t index = 0; index < m_rInputSize; index++) + { + if (m_rResults->at(index)) + { + finalResults->append(m_originInputs->getBytes(index)); + } + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) + << LOG_KV("originCount", finalResults->size()); + m_resultCount = finalResults->dedup(); + CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) + << LOG_KV("resultCount", m_resultCount); + m_taskState->writeLines(finalResults, DataSchema::Bytes); + } + } + catch (const std::exception& e) + { + onReceiverException("saveResults", e); + } +} + +void CM2020PSIReceiver::onReceiverException(const std::string& _module, const std::exception& e) +{ + CM2020_PSI_LOG(ERROR) << LOG_BADGE(_module) LOG_KV("taskID", m_taskID) + << LOG_KV("exception", boost::diagnostic_information(e)); + auto error = std::make_shared( + (int)CM2020PSIRetCode::ON_EXCEPTION, boost::diagnostic_information(e)); + onReceiverTaskDone(error); +} + +void CM2020PSIReceiver::onReceiverTaskDone(bcos::Error::Ptr _error) +{ + if (m_taskState->taskDone()) + { + return; + } + + CM2020_PSI_LOG(INFO) << LOG_BADGE("onReceiverTaskDone") LOG_KV("taskID", m_taskID); + + std::string message; + if (_error) + { + message = "\nStatus: FAIL\nMessage: " + _error->errorMessage(); + m_cm2020Result->setError(std::move(_error)); + } + else + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of aftermath + m_costs[3] = + std::chrono::duration_cast(end - m_startTimePoint).count(); + + uint64_t communication = + (uint64_t)m_bucketSizeInBytes * (uint64_t)m_params->bucketNumber() + + (uint64_t)m_sInputSize * (uint64_t)sizeof(u128); + + if (m_params->enableSyncResults()) + { + communication += (uint64_t)m_resultCount * (uint64_t)sizeof(uint32_t); + } + + communication /= (uint64_t)(1024 * 1024); + + m_cm2020Result->m_enableSyncResults = m_params->enableSyncResults(); + m_cm2020Result->m_bucketNumber = m_params->bucketNumber(); + m_cm2020Result->m_communication = std::to_string(communication) + "MB"; + m_cm2020Result->m_intersections = m_resultCount; + m_cm2020Result->m_party0Size = m_rInputSize; + m_cm2020Result->m_party1Size = m_sInputSize; + + message = "\nStatus: SUCCESS\nOrigin Inputs Number: " + std::to_string(m_rInputSize) + + "\nIntersections Number: " + std::to_string(m_resultCount) + + "\nCommunication: " + std::to_string(communication) + + "MB\nTotal Costs: " + std::to_string(m_costs[3]) + "ms\n"; + } + m_taskState->onTaskFinished(m_cm2020Result, true); + + CM2020_PSI_LOG(INFO) << LOG_BADGE("receiverTaskDone") << LOG_KV("taskID", m_taskID) + << LOG_KV("detail", message); +} diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h new file mode 100644 index 00000000..8bd33468 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h @@ -0,0 +1,156 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIReceiver.h + * @author: shawnhe + * @date 2022-12-7 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "../protocol/CM2020PSIResult.h" +#include "TaskParams.h" +#include "ppc-crypto/src/randomot/SimplestOT.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-psi/src/cm2020-psi/CM2020PSIConfig.h" +#include "ppc-tools/src/common/Progress.h" +#include +#include + +namespace ppc::psi +{ + +class CM2020PSIReceiver : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + CM2020PSIReceiver( + CM2020PSIConfig::Ptr m_config, TaskState::Ptr _taskState, crypto::SimplestOT::Ptr _ot); + + ~CM2020PSIReceiver() + { + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + std::vector>().swap(m_originLocations); + std::vector>().swap(m_oprfOutputs); + std::vector().swap(m_matrixA); + std::vector().swap(m_matrixDelta); + std::unordered_map().swap(m_hashes); + std::unordered_map().swap(m_receivedHash); + std::shared_ptr>().swap(m_rResults); + std::shared_ptr>().swap(m_sResults); + // release the memory to os + MallocExtension::instance()->ReleaseFreeMemory(); + CM2020_PSI_LOG(INFO) << LOG_DESC("the receiver destroyed") << LOG_KV("taskID", m_taskID); + } + + void asyncRunTask(); + + void onHandshakeDone(); + + void onSenderSizeReceived(front::PPCMessageFace::Ptr _message); + + void onBatchPointBReceived(front::PPCMessageFace::Ptr _message); + + void onDoNextRoundReceived(); + + void onHashesReceived(front::PPCMessageFace::Ptr _message); + + const std::string& taskID() const { return m_taskID; } + TaskParams::Ptr params() { return m_params; } + +protected: + void runReceiver(); + void prepareInputs(); + void syncInputsSize(); + void runRandomOT(); + + void preprocessInputs(); + void doOprf(); + void initMatrixParams(); + void constructMatrices(uint32_t _offset, uint32_t _width); + void computeOprfOutputs(uint32_t _offset, uint32_t _width); + void negotiateMatrix(uint32_t _bucketIndex, uint32_t _matrixIndex); + void increaseOprfRound() { m_oprfRound++; } + void clearOprfMemory(); + + void doPsi(); + void computeHash(); + void tryComputeIntersection(); + void computeIntersection(uint32_t _round, bcos::bytesPointer _buffer); + + void finishPsi(); + void syncResults(uint32_t _count); + void saveResults(); + + void onReceiverException(const std::string& _module, const std::exception& e); + void onReceiverTaskDone(bcos::Error::Ptr _error); + + tools::Progress::Ptr progress() { return m_progress; } + +private: + CM2020PSIConfig::Ptr m_config; + TaskState::Ptr m_taskState; + crypto::SimplestOT::Ptr m_ot; + + std::string m_taskID; + TaskParams::Ptr m_params; + CM2020PSIResult::Ptr m_cm2020Result; + + ppc::io::DataBatch::Ptr m_originInputs; + uint32_t m_rInputSize; + uint32_t m_sInputSize; + bool m_ioByInterface{false}; + + std::pair m_otState; + std::vector> m_senderKeys; + + std::vector> m_originLocations; + std::vector> m_oprfOutputs; + + // width of per round processing + uint32_t m_handleWidth; + + // size of each bucket (bytes) + // the bucket size is also the height of matrix + uint32_t m_bucketSizeInBytes; + uint32_t m_mask; + + std::atomic m_oprfRound{0}; + uint32_t m_currentWidth; + std::vector m_matrixA; + std::vector m_matrixDelta; + + // key: the hash of OPRF output, value: the index of input + std::unordered_map m_hashes; + + std::atomic m_hashReady{false}; + uint32_t m_maxHashSeq; + mutable bcos::SharedMutex x_receivedHash; + std::unordered_map m_receivedHash; + + std::atomic m_resultCount; + std::shared_ptr> m_rResults; + std::shared_ptr> m_sResults; + + std::chrono::time_point m_startTimePoint; + std::vector m_costs; + + tools::Progress::Ptr m_progress; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp new file mode 100644 index 00000000..7fa95bb1 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp @@ -0,0 +1,890 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSISender.cpp + * @author: shawnhe + * @date 2022-12-7 + */ + +#include "CM2020PSISender.h" +#include "CM2020PSI.h" +#include "openssl/rand.h" +#include "ppc-crypto/src/prng/AESPRNG.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "ppc-tools/src/common/TransTools.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::tools; + +CM2020PSISender::CM2020PSISender(CM2020PSIConfig::Ptr _config, TaskState::Ptr _taskState, + std::shared_ptr _ioService, SimplestOT::Ptr _ot, + std::shared_ptr _threadPool) + : m_config(std::move(_config)), m_taskState(std::move(_taskState)), m_ot(std::move(_ot)) +{ + auto task = m_taskState->task(); + m_taskID = task->id(); + m_params = std::make_shared(task->param()); + m_params->setSyncResults(task->syncResultToPeer()); + m_cm2020Result = std::make_shared(m_taskID); + m_channelManager = std::make_shared(std::move(_ioService), nullptr); + m_channelManager->setThreadPool(std::move(_threadPool)); + m_channel = m_channelManager->buildChannelForTask(m_taskID); + m_startTimePoint = std::chrono::high_resolution_clock::now(); + m_costs.resize(4); + m_matrixProgress = std::make_shared(m_config->threadPool()); + m_progress = std::make_shared(m_config->threadPool()); +} + + +void CM2020PSISender::asyncRunTask() +{ + CM2020_PSI_LOG(INFO) << LOG_DESC("asyncRunTask as sender") << LOG_KV("taskID", m_taskID); + + m_progress->reset(4, [self = weak_from_this()]() { + /** + * 1. prepare inputs done + * 2. run random OT done + * 3. receiver size received + * 4. handshake done + */ + auto sender = self.lock(); + if (!sender) + { + return; + } + sender->preprocessInputs(); + sender->doOprf(); + }); + + m_config->threadPool()->enqueue([self = weak_from_this()]() { + auto sender = self.lock(); + if (!sender) + { + return; + } + sender->runSender(); + }); +} + +void CM2020PSISender::runSender() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("runSender") << LOG_KV("taskID", m_taskID) + << LOG_KV("syncResults", m_params->enableSyncResults()) + << LOG_KV("bucketNumber", m_params->bucketNumber()); + + try + { + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, + std::make_shared()); + message->setMessageType(uint8_t(CM2020PSIMessageType::HELLO_RECEIVER)); + + // shake hands with each other + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + }, + nullptr); + + prepareInputs(); + } + catch (const std::exception& e) + { + onSenderException("runSender", e); + } +} + +void CM2020PSISender::onHandshakeDone(front::PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onHandshakeDone") << LOG_KV("taskID", m_taskID); + m_startTimePoint = std::chrono::high_resolution_clock::now(); + ppctars::CM2020Params tarsParams; + ppctars::serialize::decode(*_message->data(), tarsParams); + m_params->setSyncResults(tarsParams.enableSyncResults); + m_params->setBucketNumber(uint16_t(tarsParams.bucketNumber)); + m_params->setSeed(tarsParams.seed); + m_params->setLowBandwidth(tarsParams.lowBandwidth); + + m_progress->mark("HANDSHAKE"); +} + +void CM2020PSISender::prepareInputs() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("prepareInputs") << LOG_KV("taskID", m_taskID); + try + { + auto originData = m_taskState->task()->selfParty()->dataResource()->rawData(); + if (!originData.empty()) + { + m_ioByInterface = true; + } + + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(CM2020Exception() << bcos::errinfo_comment("data is empty")); + } + + m_sInputSize = m_originInputs->size(); + m_originLocations.reserve(m_sInputSize); + m_originLocations.resize(m_sInputSize); + m_oprfOutputs.reserve(m_sInputSize); + m_oprfOutputs.resize(m_sInputSize); + + syncInputsSize(); + } + catch (const std::exception& e) + { + onSenderException("prepareInputs", e); + } +} + +void CM2020PSISender::syncInputsSize() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("syncInputsSize") << LOG_KV("taskID", m_taskID); + + auto data = std::make_shared(); + encodeUnsignedNum(data, m_sInputSize); + auto message = m_config->ppcMsgFactory()->buildPPCMessage( + uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, data); + message->setMessageType(uint8_t(CM2020PSIMessageType::SENDER_SIZE)); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + }, + nullptr); + + m_progress->mark("PREPARE_INPUTS"); +} + +void CM2020PSISender::onReceiverSizeReceived(front::PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + + CM2020_PSI_LOG(INFO) << LOG_BADGE("onReceiverSizeReceived") << LOG_KV("taskID", m_taskID); + try + { + decodeUnsignedNum(m_rInputSize, _message->data()); + m_progress->mark("RECEIVE_SIZE"); + } + catch (const std::exception& e) + { + onSenderException("onReceiverSizeReceived", e); + } +} + +void CM2020PSISender::onPointAReceived(front::PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onPointAReceived") << LOG_KV("taskID", m_taskID); + try + { + auto pointA = _message->data(); + + bcos::bytes seed(16); + if (RAND_bytes(seed.data(), 16) != 1) + { + BOOST_THROW_EXCEPTION(CM2020Exception()); + } + AESPRNG::Ptr prng = std::make_shared(seed); + m_otChoices = std::make_shared(); + m_otChoices->randomize(prng, (m_params->bucketNumber() + 7) / 8); + + auto retPair = m_ot->receiverGeneratePointsB(m_otChoices, pointA); + + // send batch point B to ot sender + auto message = m_config->ppcMsgFactory()->buildPPCMessage( + uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, retPair.first); + message->setMessageType(uint8_t(CM2020PSIMessageType::POINT_B_ARRAY)); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + }, + nullptr); + + // finish ot receiver + m_receiverKeys = m_ot->finishReceiver(pointA, retPair.second); + m_progress->mark("RANDOM_OT"); + } + catch (const std::exception& e) + { + onSenderException("onPointAReceived", e); + } +} + +void CM2020PSISender::preprocessInputs() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("preprocessInputs") << LOG_KV("taskID", m_taskID); + try + { + bcos::bytesConstRef seed(m_taskID); + auto hash = m_config->hash(); + tbb::parallel_for(tbb::blocked_range(0U, m_sInputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto state = hash->init(); + hash->update(state, seed); + auto data = m_originInputs->getBytes(i); + hash->update(state, bcos::ref(bcos::bytes(data.begin(), data.end()))); + auto res = hash->final(state); + memcpy((bcos::byte*)m_originLocations[i].data(), res.data(), 8 * sizeof(uint32_t)); + } + }); + } + catch (const std::exception& e) + { + onSenderException("preprocessInputs", e); + } +} + +void CM2020PSISender::doOprf() +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + if (m_oprfRound == 0) + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of rot and prepare inputs + m_costs[0] = + std::chrono::duration_cast(end - m_startTimePoint) + .count(); + + initMatrixParams(); + } + + CM2020_PSI_LOG(INFO) << LOG_BADGE("doOprf") << LOG_KV("taskID", m_taskID) + << LOG_KV("round", m_oprfRound); + + uint32_t offset = m_oprfRound * m_handleWidth; + if (offset >= m_params->bucketNumber()) + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of oprf + m_costs[1] = + std::chrono::duration_cast(end - m_startTimePoint) + .count(); + + // after negotiating matrix and computing oprf, + // we can receive hash from sender and compute intersection + m_config->threadPool()->enqueue([self = weak_from_this()]() { + auto sender = self.lock(); + if (!sender) + { + return; + } + sender->doPsi(); + }); + } + else + { + uint32_t currentWidth = offset + m_handleWidth < m_params->bucketNumber() ? + m_handleWidth : + m_params->bucketNumber() - offset; + uint32_t totalRound = + (m_bucketSizeInBytes + MAX_SEND_BUFFER_LENGTH - 1) / MAX_SEND_BUFFER_LENGTH; + + constructMatrices(offset, currentWidth); + m_progress->reset( + currentWidth * totalRound, [self = weak_from_this(), offset, currentWidth]() { + auto sender = self.lock(); + if (!sender) + { + return; + } + sender->computeOprfOutputs(offset, currentWidth); + sender->increaseOprfRound(); + sender->doOprf(); + }); + + m_matrixProgress->reset(currentWidth * totalRound, [self = weak_from_this()]() { + auto sender = self.lock(); + if (!sender) + { + return; + } + sender->noticeReceiverDoNextRound(); + }); + + for (uint32_t col = 0; col < currentWidth; col++) + { + for (uint32_t round = 0; round < totalRound; round++) + { + m_channel->asyncReceiveMessage(uint8_t(CM2020PSIMessageType::MATRIX), + (offset + col) * totalRound + round, 30 * 60, + [self = weak_from_this(), offset, col, totalRound, round]( + bcos::Error::Ptr _error, front::PPCMessageFace::Ptr _message) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + else + { + sender->matrixProgress()->mark( + (offset + col) * totalRound + round); + + sender->handleMatrixColumnReceived( + offset, col, round, _message->data()); + + sender->progress()->mark( + (offset + col) * totalRound + round); + } + }); + } + } + } + } + catch (const std::exception& e) + { + onSenderException("doOprf", e); + } +} + +void CM2020PSISender::noticeReceiverDoNextRound() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("noticeReceiverDoNextRound") << LOG_KV("taskID", m_taskID) + << LOG_KV("round", m_oprfRound); + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, std::make_shared()); + message->setMessageType(uint8_t(CM2020PSIMessageType::DO_NEXT_ROUND)); + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + }, + nullptr); +} + +void CM2020PSISender::initMatrixParams() +{ + if (m_taskState->taskDone()) + { + return; + } + uint32_t maxInputSize = std::max(m_rInputSize, m_sInputSize); + + m_handleWidth = m_params->bucketNumber(); + + if (maxInputSize > (1 << DEFAULT_HANDLE_WIDTH_POWER)) + { + // while handling mass data, we do oprf block by block to prevent OOM + m_handleWidth = m_params->bucketNumber() / + (1 << int(std::ceil(log2(maxInputSize) - DEFAULT_HANDLE_WIDTH_POWER))); + m_handleWidth = std::max(m_handleWidth, (uint32_t)MIN_HANDLE_WIDTH); + } + + m_bucketSizeInBytes = std::max( + uint32_t(MIN_BUCKET_SIZE / 8), uint32_t(((long double)maxInputSize * ENCODE_RATE + 7) / 8)); + m_mask = m_bucketSizeInBytes * 8; + + CM2020_PSI_LOG(INFO) << LOG_BADGE("initMatrixParams") << LOG_KV("taskID", m_taskID) + << LOG_KV("sInputSize", m_sInputSize) + << LOG_KV("bucketNumber", m_params->bucketNumber()) + << LOG_KV("handleWidth", m_handleWidth) + << LOG_KV("bucketSizeInBytes", m_bucketSizeInBytes) + << LOG_KV("mask", m_mask); +} + +void CM2020PSISender::constructMatrices(uint32_t _offset, uint32_t _width) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("constructMatrices") << LOG_KV("offset", _offset) + << LOG_KV("width", _width) << LOG_KV("taskID", m_taskID); + + if (_offset == 0) + { + m_matrixC.reserve(_width); + m_matrixC.resize(_width); + } + + tbb::parallel_for(tbb::blocked_range(0U, _width), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + // construct matrix A + if (_offset == 0) + { + m_matrixC[i].reserve(m_bucketSizeInBytes); + m_matrixC[i].resize(m_bucketSizeInBytes); + } + + AESPRNG prng(m_receiverKeys[_offset + i]); + prng.generate(m_matrixC[i].data(), m_bucketSizeInBytes); + } + }); +} + +void CM2020PSISender::onMatrixColumnReceived(PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("onMatrixColumnReceived") << LOG_KV("taskID", m_taskID) + << LOG_KV("seq", _message->seq()); + try + { + m_channel->onMessageArrived(uint8_t(CM2020PSIMessageType::MATRIX), _message); + } + catch (const std::exception& e) + { + onSenderException("onMatrixColumnReceived", e); + } +} + +void CM2020PSISender::handleMatrixColumnReceived( + uint32_t _offset, uint32_t _col, uint32_t _round, bcos::bytesPointer _buffer) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(DEBUG) << LOG_BADGE("handleMatrixColumnReceived") << LOG_KV("taskID", m_taskID) + << LOG_KV("offset", _offset) << LOG_KV("col", _col) + << LOG_KV("round", _round); + try + { + if (m_otChoices->get(_offset + _col)) + { + uint32_t length = _buffer->size(); + uint32_t start = _round * MAX_SEND_BUFFER_LENGTH; + for (uint32_t i = 0; i < length; i++) + { + m_matrixC[_col][start + i] ^= _buffer->at(i); + } + } + } + catch (const std::exception& e) + { + onSenderException("handleMatrixColumnReceived", e); + } +} + +void CM2020PSISender::computeOprfOutputs(uint32_t _offset, uint32_t _width) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("computeOprfOutputs") << LOG_KV("offset", _offset) + << LOG_KV("width", _width) << LOG_KV("taskID", m_taskID); + try + { + tbb::parallel_for(tbb::blocked_range(0U, m_sInputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto newSize = (_offset + _width + 7) / 8; + m_oprfOutputs[i].reserve(newSize); + m_oprfOutputs[i].resize(newSize); + for (uint32_t j = 0; j < _width; j++) + { + uint32_t location = (m_originLocations[i][(_offset + j) % 4] * (_offset + j) + + m_originLocations[i][4 + (_offset + j) % 4]) % + m_mask; + + m_oprfOutputs[i][(_offset + j) >> 3] |= + (uint8_t)((bool)(m_matrixC[j][location >> 3] & (1 << (location & 7)))) + << ((_offset + j) & 7); + } + } + }); + } + catch (const std::exception& e) + { + onSenderException("computeOprfOutputs", e); + } +} + +void CM2020PSISender::clearOprfMemory() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("clearOprfMemory") << LOG_KV("taskID", m_taskID); + std::vector>().swap(m_originLocations); + std::vector().swap(m_matrixC); + MallocExtension::instance()->ReleaseFreeMemory(); +} + +void CM2020PSISender::doPsi() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("doPsi") << LOG_KV("taskID", m_taskID); + + try + { + m_progress->reset(-1, nullptr); + clearOprfMemory(); + if (m_params->enableSyncResults()) + { + m_sResults = std::make_shared>(m_sInputSize, 0); + } + computeAndSendHash(); + finishPsi(); + } + catch (const std::exception& e) + { + onSenderException("doPsi", e); + } +} + +void CM2020PSISender::computeAndSendHash() +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("computeAndSendHash") << LOG_KV("taskID", m_taskID); + + // send 2^18 hashes per round + uint32_t sendNum = MAX_SEND_BUFFER_LENGTH / RESULT_LEN_BYTE; + uint32_t totalRound = (m_sInputSize + sendNum - 1) / sendNum; + + auto hash = m_config->hash(); + + tbb::parallel_for(tbb::blocked_range(0U, totalRound), [&](auto const& range) { + for (auto round = range.begin(); round < range.end(); round++) + { + uint32_t currentLen = + sendNum * (round + 1) > m_sInputSize ? m_sInputSize - sendNum * round : sendNum; + auto buffer = std::make_shared(currentLen * RESULT_LEN_BYTE); + uint32_t offset = round * sendNum; + for (uint32_t i = 0; i < currentLen; i++) + { + auto res = hash->hash(bcos::ref(m_oprfOutputs[offset + i])); + memcpy(buffer->data() + i * RESULT_LEN_BYTE, res.data(), RESULT_LEN_BYTE); + } + + auto message = m_config->ppcMsgFactory()->buildPPCMessage( + uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, buffer); + message->setMessageType(uint8_t(CM2020PSIMessageType::HASHES)); + message->setSeq(round); + + if (m_params->lowBandwidth()) + { + auto error = m_config->sendMessage(m_taskState->peerID(), message); + if (error && error->errorCode()) + { + onSenderTaskDone(std::move(error)); + } + } + else + { + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(std::move(_error)); + } + }, + nullptr); + } + } + }); + + // clear oprf outputs + std::vector>().swap(m_oprfOutputs); + MallocExtension::instance()->ReleaseFreeMemory(); +} + +void CM2020PSISender::finishPsi() +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + if (m_params->enableSyncResults()) + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of psi + m_costs[2] = + std::chrono::duration_cast(end - m_startTimePoint) + .count(); + + CM2020_PSI_LOG(INFO) << LOG_BADGE("waiting results") << LOG_KV("taskID", m_taskID); + } + else + { + onSenderTaskDone(nullptr); + } + } + catch (const std::exception& e) + { + onSenderException("finishPsi", e); + } +} + +void CM2020PSISender::onResultCountReceived(PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + try + { + decodeUnsignedNum(m_resultCount, _message->data()); + m_resultCountReceived.exchange(true); + + CM2020_PSI_LOG(INFO) << LOG_BADGE("onResultCountReceived") << LOG_KV("taskID", m_taskID) + << LOG_KV("count", m_resultCount); + + uint32_t number = MAX_SEND_BUFFER_LENGTH / sizeof(uint32_t); + uint32_t totalRound = (m_resultCount + number - 1) / number + 1; + + if (m_progress->mark(-1) == totalRound) + { + if (m_resultCount > 0) + { + saveResults(); + } + onSenderTaskDone(nullptr); + } + } + catch (const std::exception& e) + { + onSenderException("onResultCountReceived", e); + } +} + +void CM2020PSISender::onResultReceived(PPCMessageFace::Ptr _message) +{ + if (m_taskState->taskDone()) + { + return; + } + CM2020_PSI_LOG(DEBUG) << LOG_BADGE("onResultReceived") << LOG_KV("taskID", m_taskID) + << LOG_KV("seq", _message->seq()); + try + { + auto buffer = _message->data(); + uint32_t resultNum = buffer->size() / sizeof(uint32_t); + for (uint32_t i = 0; i < resultNum; i++) + { + uint32_t index; + decodeUnsignedNum(index, buffer->data() + i * sizeof(uint32_t)); + m_sResults->at(index) = 1; + } + + if (m_resultCountReceived) + { + // receive 2^20 index per round + uint32_t number = MAX_SEND_BUFFER_LENGTH / sizeof(uint32_t); + uint32_t totalRound = (m_resultCount + number - 1) / number + 1; + + if (m_progress->mark(_message->seq()) == totalRound) + { + if (m_resultCount > 0) + { + saveResults(); + } + + onSenderTaskDone(nullptr); + } + } + else + { + m_progress->mark(_message->seq()); + } + } + catch (const std::exception& e) + { + onSenderException("onResultReceived", e); + } +} + +void CM2020PSISender::saveResults() +{ + CM2020_PSI_LOG(INFO) << LOG_BADGE("saveResults") LOG_KV("taskID", m_taskID); + try + { + if (m_ioByInterface) + { + std::vector results; + for (uint32_t index = 0; index < m_sInputSize; index++) + { + if (m_sResults->at(index)) + { + auto line = m_originInputs->getBytes(index); + results.emplace_back(std::string(line.begin(), line.end())); + } + } + m_cm2020Result->m_outputs.emplace_back(std::move(results)); + } + else + { + DataBatch::Ptr finalResults = std::make_shared(); + for (uint32_t index = 0; index < m_sInputSize; index++) + { + if (m_sResults->at(index)) + { + finalResults->append(m_originInputs->getBytes(index)); + } + } + CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) + << LOG_KV("originCount", finalResults->size()); + m_resultCount = finalResults->dedup(); + CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) + << LOG_KV("resultCount", m_resultCount); + m_taskState->writeLines(finalResults, DataSchema::Bytes); + } + } + catch (const std::exception& e) + { + onSenderException("saveResults", e); + } +} + +void CM2020PSISender::onSenderException(const std::string& _message, const std::exception& e) +{ + CM2020_PSI_LOG(ERROR) << LOG_BADGE(_message) << LOG_KV("taskID", m_taskID) + << LOG_KV("exception", boost::diagnostic_information(e)); + auto error = std::make_shared( + (int)CM2020PSIRetCode::ON_EXCEPTION, boost::diagnostic_information(e)); + onSenderTaskDone(error); +} + +void CM2020PSISender::onSenderTaskDone(bcos::Error::Ptr _error) +{ + if (m_taskState->taskDone()) + { + return; + } + + CM2020_PSI_LOG(INFO) << LOG_BADGE("onSenderTaskDone") << LOG_KV("taskID", m_taskID); + m_channelManager->removeChannelByTask(m_taskID); + + std::string message; + + if (_error) + { + message = "\nStatus: FAIL\nMessage: " + _error->errorMessage(); + m_cm2020Result->setError(std::move(_error)); + } + else + { + auto end = std::chrono::high_resolution_clock::now(); + // costs of aftermath + m_costs[3] = + std::chrono::duration_cast(end - m_startTimePoint).count(); + + uint64_t communication = + (uint64_t)m_bucketSizeInBytes * (uint64_t)m_params->bucketNumber() + + (uint64_t)m_sInputSize * (uint64_t)sizeof(u128); + + if (m_resultCount) + { + communication += (uint64_t)m_resultCount * (uint64_t)sizeof(uint32_t); + } + + communication /= (uint64_t)(1024 * 1024); + + // message = "\nStatus: SUCCESS\nOrigin Inputs Number: " + + // std::to_string(m_sInputSize) + + // "\nCounterparty Inputs Number: " + std::to_string(m_rInputSize) + + // "\nIntersections Number: " + std::to_string(m_resultCount) + + // "\nCommunication: " + std::to_string(communication) + + // "MB\nTotal Costs: " + std::to_string(m_costs[3]) + "ms\n"; + message = "\nStatus: SUCCESS\nOrigin Inputs Number: " + std::to_string(m_sInputSize) + + "\nIntersections Number: " + std::to_string(m_resultCount) + + "\nCommunication: " + std::to_string(communication) + + "MB\nTotal Costs: " + std::to_string(m_costs[3]) + "ms\n"; + + m_cm2020Result->m_enableSyncResults = m_params->enableSyncResults(); + m_cm2020Result->m_bucketNumber = m_params->bucketNumber(); + m_cm2020Result->m_communication = std::to_string(communication) + "MB"; + m_cm2020Result->m_intersections = m_resultCount; + m_cm2020Result->m_party0Size = m_rInputSize; + m_cm2020Result->m_party1Size = m_sInputSize; + } + + m_taskState->onTaskFinished(m_cm2020Result, true); + + CM2020_PSI_LOG(INFO) << LOG_BADGE("onSenderTaskDone") << LOG_KV("taskID", m_taskID) + << LOG_KV("result", message); +} diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h new file mode 100644 index 00000000..71f5bb40 --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h @@ -0,0 +1,152 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSISender.h + * @author: shawnhe + * @date 2022-12-7 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "../protocol/CM2020PSIResult.h" +#include "TaskParams.h" +#include "ppc-crypto/src/randomot/SimplestOT.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-front/ppc-front/PPCChannelManager.h" +#include "ppc-psi/src/cm2020-psi/CM2020PSIConfig.h" +#include "ppc-tools/src/common/Progress.h" +#include +#include + + +namespace ppc::psi +{ +class CM2020PSISender : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + CM2020PSISender(CM2020PSIConfig::Ptr _config, TaskState::Ptr _taskState, + std::shared_ptr _ioService, crypto::SimplestOT::Ptr _ot, + std::shared_ptr _threadPool); + + ~CM2020PSISender() + { + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + std::vector>().swap(m_originLocations); + std::vector>().swap(m_oprfOutputs); + std::vector().swap(m_matrixC); + std::shared_ptr>().swap(m_sResults); + // release the memory to os + MallocExtension::instance()->ReleaseFreeMemory(); + CM2020_PSI_LOG(INFO) << LOG_DESC("the sender destroyed") << LOG_KV("taskID", m_taskID); + } + + void asyncRunTask(); + + void onHandshakeDone(front::PPCMessageFace::Ptr _message); + + void onReceiverSizeReceived(front::PPCMessageFace::Ptr _message); + + void onPointAReceived(front::PPCMessageFace::Ptr _message); + + void onMatrixColumnReceived(front::PPCMessageFace::Ptr _message); + + void onResultCountReceived(front::PPCMessageFace::Ptr _message); + + void onResultReceived(front::PPCMessageFace::Ptr _message); + + const std::string& taskID() const { return m_taskID; } + TaskParams::Ptr params() { return m_params; } + +protected: + void runSender(); + void prepareInputs(); + void syncInputsSize(); + void preprocessInputs(); + + void doOprf(); + void initMatrixParams(); + void constructMatrices(uint32_t _offset, uint32_t _width); + + void handleMatrixColumnReceived( + uint32_t _offset, uint32_t _col, uint32_t _round, bcos::bytesPointer _buffer); + void noticeReceiverDoNextRound(); + void computeOprfOutputs(uint32_t _offset, uint32_t _width); + void clearOprfMemory(); + + void doPsi(); + void computeAndSendHash(); + + void finishPsi(); + void saveResults(); + + void onSenderException(const std::string& _module, const std::exception& e); + void onSenderTaskDone(bcos::Error::Ptr _error); + + void increaseOprfRound() { m_oprfRound++; } + + tools::Progress::Ptr progress() { return m_progress; } + tools::Progress::Ptr matrixProgress() { return m_matrixProgress; } + +private: + CM2020PSIConfig::Ptr m_config; + TaskState::Ptr m_taskState; + crypto::SimplestOT::Ptr m_ot; + + std::string m_taskID; + TaskParams::Ptr m_params; + CM2020PSIResult::Ptr m_cm2020Result; + + ppc::io::DataBatch::Ptr m_originInputs; + uint32_t m_rInputSize; + uint32_t m_sInputSize; + bool m_ioByInterface{false}; + + // width of per round processing + uint32_t m_handleWidth; + + // size of each bucket (bytes) + // the bucket size is also the height of matrix + uint32_t m_bucketSizeInBytes; + uint32_t m_mask; + + crypto::BitVector::Ptr m_otChoices; + std::vector m_receiverKeys; + + std::vector> m_originLocations; + std::vector> m_oprfOutputs; + + front::PPCChannelManager::Ptr m_channelManager; + front::Channel::Ptr m_channel; + + std::vector m_matrixC; + + std::shared_ptr> m_sResults; + std::atomic m_resultCountReceived{false}; + uint32_t m_resultCount{0}; + + uint32_t m_oprfRound{0}; + + std::chrono::time_point m_startTimePoint; + + std::vector m_costs; + + tools::Progress::Ptr m_progress; + tools::Progress::Ptr m_matrixProgress; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h b/cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h new file mode 100644 index 00000000..efa62ddb --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskParams.h + * @author: shawnhe + * @date 2022-12-9 + */ + +#pragma once +#include "../Common.h" +#include +#include + +namespace ppc::psi +{ +class TaskParams +{ +public: + using Ptr = std::shared_ptr; + TaskParams(std::string_view _param) + { + if (_param.empty()) + { + return; + } + + Json::Reader reader; + Json::Value result; + if (!reader.parse(_param.begin(), _param.end(), result)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int)CM2020PSIRetCode::INVALID_TASK_PARAM, "invalid task param: invalid json")); + } + if (!result.isArray()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)CM2020PSIRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param must be array")); + } + if (!result.empty()) + { + int enableHighPerformance = result[0].asInt(); + m_enableHighPerformance = enableHighPerformance > 0; + m_bucketNumber = HIGH_PERFORMANCE_BUCKET_NUMBER; + } + if (result.size() > 1) + { + uint16_t bucketNumber = result[1].asInt(); + if (bucketNumber >= HIGH_PERFORMANCE_BUCKET_NUMBER && bucketNumber <= MAX_BUCKET_NUMBER) + { + m_bucketNumber = bucketNumber; + } + } + } + + void setSyncResults(bool _syncResults) { m_enableSyncResults = _syncResults; } + [[nodiscard]] bool enableSyncResults() const { return m_enableSyncResults; } + + void setBucketNumber(uint16_t _bucketNumber) { m_bucketNumber = _bucketNumber; } + [[nodiscard]] uint16_t bucketNumber() const { return m_bucketNumber; } + + void setSeed(bcos::bytes _seed) { m_seed = std::move(_seed); } + [[nodiscard]] bcos::bytes const& seed() const { return m_seed; } + + void setLowBandwidth(bool _lowBandwidth) { m_lowBandwidth = _lowBandwidth; } + [[nodiscard]] bool lowBandwidth() const { return m_lowBandwidth; } + +private: + // whether the results need to be synchronized to the counterparty + bool m_enableSyncResults{false}; + + // we can also run oprf with high performance just reducing the width of matrix + // note that: it may cause oprf outputs conflict with very small probability + bool m_enableHighPerformance{false}; + + // one input will be mapped to multiple buckets + // the number of buckets is also the width of matrix + uint16_t m_bucketNumber{DEFAULT_BUCKET_NUMBER}; + + bcos::bytes m_seed; + + bool m_lowBandwidth; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars b/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars new file mode 100644 index 00000000..840fb78d --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars @@ -0,0 +1,8 @@ +module ppctars { + struct CM2020Params { + 1 require bool enableSyncResults; + 2 require unsigned int bucketNumber; + 3 require vector seed; + 4 require bool lowBandwidth; + }; +}; diff --git a/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h b/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h new file mode 100644 index 00000000..b690883e --- /dev/null +++ b/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CM2020PSIResult.h + * @author: shawnhe + * @date 2022-12-13 + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "ppc-framework/protocol/Task.h" + +namespace ppc::psi +{ +// Note: can extend on demand +class CM2020PSIResult : public protocol::TaskResult +{ +public: + using Ptr = std::shared_ptr; + CM2020PSIResult(std::string const& _taskID) : TaskResult(_taskID) {} + ~CM2020PSIResult() override = default; + + void setOutputs(std::vector>&& _outputs) + { + m_outputs = std::move(_outputs); + } + + // serialize the taskResult to json + [[nodiscard]] Json::Value serializeToJson() const override + { + Json::Value response; + response["taskID"] = taskID(); + if (m_timeCost) + { + response["timeCost"] = std::to_string(m_timeCost) + "ms"; + } + if (m_fileInfo && !m_fileInfo->bizSeqNo.empty()) + { + response["bizSeqNo"] = m_fileInfo->bizSeqNo; + response["fileID"] = m_fileInfo->fileID; + response["fileMd5"] = m_fileInfo->fileMd5; + } + + if (m_error && error()->errorCode()) + { + response["code"] = error()->errorCode(); + response["message"] = error()->errorMessage(); + response["status"] = protocol::toString(protocol::TaskStatus::FAILED); + } + else + { + response["code"] = 0; + response["message"] = "success"; + response["status"] = protocol::toString(protocol::TaskStatus::COMPLETED); + + Json::Value jsonData; + jsonData["bucketNumber"] = m_bucketNumber; + jsonData["communication"] = m_communication; + jsonData["syncResult"] = m_enableSyncResults; + jsonData["intersections"] = m_intersections; +// jsonData["party0Size"] = m_party0Size; +// jsonData["party1Size"] = m_party1Size; + + if (!m_outputs.empty()) + { + Json::Value jsonOutputs; + for (auto& outputVector : m_outputs) + { + Json::Value jsonOutput; + for (auto& output : outputVector) + { + jsonOutput.append(output); + } + jsonOutputs.append(jsonOutput); + } + jsonData["outputs"] = jsonOutputs; + } + + response["data"] = jsonData; + } + return response; + } + + + std::vector> m_outputs; + uint16_t m_bucketNumber; + std::string m_communication; + bool m_enableSyncResults; + uint32_t m_intersections; + uint32_t m_party0Size; + uint32_t m_party1Size; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt b/cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt new file mode 100644 index 00000000..9915c66d --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt @@ -0,0 +1,31 @@ +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-psi/ecdh-conn-psi/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") + +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () + +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${ECDH_CONN_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST}) +target_include_directories(${ECDH_CONN_PSI_TARGET} PUBLIC + $) + +target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb TCMalloc) \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/Common.h b/cpp/ppc-psi/src/ecdh-conn-psi/Common.h new file mode 100644 index 00000000..0bdc1768 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/Common.h @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: zachma + * @date 2023-8-28 + */ + +#pragma once +#include "ppc-framework/Common.h" +#include + +namespace ppc::psi +{ +#define ECDH_CONN_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-CONN-PSI") +DERIVE_PPC_EXCEPTION(ECDHCONNException); + +enum class EcdhConnProcess : int8_t +{ + HandShakeProcess = 1, + CipherProcess = 2, + END = 3, +}; + +enum class EcdhConnSubProcess : int8_t +{ + CipherSecondProcess = 1 +}; + +const std::string VERSION_HEAD = "x-ptp-version"; +const std::string TECH_PROVIDER_CODE_HEAD = "x-ptp-tech-provider-code"; +const std::string TRACE_ID_HEAD = "x-ptp-trace-id"; +const std::string TOKEN_HEAD = "x-ptp-token"; +const std::string SESSION_ID_HEAD = "x-ptp-session-id"; +const std::string TOPIC_HEAD = "x-ptp-topic"; + +const std::string SOURCE_NODE_HEAD = "x-ptp-source-node-id"; +const std::string SOURCE_INST_HEAD = "x-ptp-source-inst-id"; +const std::string TARGET_NODE_HEAD = "x-ptp-target-node-id"; +const std::string TARGET_INST_HEAD = "x-ptp-target-inst-id"; + +inline std::ostream& operator<<(std::ostream& _out, EcdhConnProcess const& _type) +{ + switch (_type) + { + case EcdhConnProcess::HandShakeProcess: + _out << "HandShakeProcess"; + break; + case EcdhConnProcess::CipherProcess: + _out << "CipherProcess"; + break; + default: + _out << "UnknownTaskType"; + break; + } + return _out; +} + +inline std::ostream& operator<<(std::ostream& _out, EcdhConnSubProcess const& _type) +{ + switch (_type) + { + case EcdhConnSubProcess::CipherSecondProcess: + _out << "CipherSecondProcess"; + break; + default: + _out << "UnknownTaskType"; + break; + } + return _out; +} +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h new file mode 100644 index 00000000..63d09b3b --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIConfig.h + * @author: zachma + * @date 2023-7-17 + */ +#pragma once +#include "../PSIConfig.h" +#include "EcdhConnPSIMessageFactory.h" +#include "ppc-framework/crypto/ECDHCrypto.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +#include + +namespace ppc::psi +{ +class EcdhConnPSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + EcdhConnPSIConfig(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + EcdhConnPSIMessageFactory::Ptr _psiMsgFactory, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader, + bcos::ThreadPool::Ptr _threadPool) + : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN, _ppcConfig->agencyID(), _front, + _ppcMsgFactory, _dataResourceLoader, _ppcConfig->holdingMessageMinutes()), + m_msgFactory(_psiMsgFactory), + m_threadPool(_threadPool), + m_ecdhCryptoFactory(_ecdhCryptoFactory), + m_dataBatchSize(_ppcConfig->ecdhConnPSIConfig().dataBatchSize), + m_config(_ppcConfig) + {} + + virtual ~EcdhConnPSIConfig() = default; + + EcdhConnPSIMessageFactory::Ptr const& msgFactory() { return m_msgFactory; } + ppc::crypto::ECDHCryptoFactory::Ptr const& ecdhCryptoFactory() const + { + return m_ecdhCryptoFactory; + } + + ppc::tools::PPCConfig::Ptr const ppcConfig() { return m_config; } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + + uint32_t dataBatchSize() const { return m_dataBatchSize; } + +private: + EcdhConnPSIMessageFactory::Ptr m_msgFactory; + bcos::ThreadPool::Ptr m_threadPool; + ppc::crypto::ECDHCryptoFactory::Ptr m_ecdhCryptoFactory; + ppc::tools::PPCConfig::Ptr m_config; + + uint32_t m_dataBatchSize = 10000; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h new file mode 100644 index 00000000..5b991fa7 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIFactory.h + * @author: zachma + * @date 2023-7-18 + */ + +#pragma once +#include "EcdhConnPSIConfig.h" +#include "EcdhConnPSIImpl.h" +#include "EcdhConnPSIMessageFactory.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include + +namespace ppc::psi +{ +class EcdhConnPSIFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhConnPSIFactory() = default; + virtual ~EcdhConnPSIFactory() = default; + + virtual EcdhConnPSIImpl::Ptr createEcdhConnPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader) + { + auto psiMsgFactory = std::make_shared(); + auto config = std::make_shared(_ppcConfig, _ecdhCryptoFactory, _front, + _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, _threadPool); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp new file mode 100644 index 00000000..70bfa486 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp @@ -0,0 +1,311 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIImpl.cpp + * @author: zachma + * @date 2023-8-28 + */ + +#include "EcdhConnPSIImpl.h" +#include + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::tools; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace bcos; + +EcdhConnPSIImpl::EcdhConnPSIImpl(const EcdhConnPSIConfig::Ptr& _config, unsigned _idleTimeMs) + : m_config(std::move(_config)), + m_msgPool(std::make_shared()), + TaskGuarder(_config, PSIAlgorithmType::ECDH_PSI_CONN, "ECDH-CONN-PSI-Timer"), + m_ecdhConnTaskStateFactory(std::make_shared()) +{} + +void EcdhConnPSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished) +{ + ECDH_CONN_LOG(INFO) << LOG_DESC("Start a Conn asyncRunTask"); + auto ecdhTaskState = m_ecdhConnTaskStateFactory->createConnTaskState( + _task, std::move(_onTaskFinished), true, m_config); + auto dataResource = _task->selfParty()->dataResource(); + auto reader = loadReader(_task->id(), dataResource, DataSchema::Bytes); + ecdhTaskState->setReader(reader, -1); + auto role = _task->selfParty()->partyIndex(); + checkAndSetPeerInfo(ecdhTaskState); + // init process + triggleProcess((uint8_t)EcdhConnProcess::HandShakeProcess, -1); + // notify the taskInfo to the front + m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + if (role == uint16_t(PartyType::Client)) + { + ECDH_CONN_LOG(INFO) << LOG_DESC("Client do the Task") << LOG_KV("taskID", _task->id()); + auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists); + ecdhTaskState->setWriter(writer); + auto client = std::make_shared(m_config, ecdhTaskState); + addClient(client); + client->asyncStartRunTask(_task); + } + else if (role == uint16_t(PartyType::Server)) + { + ECDH_CONN_LOG(INFO) << LOG_DESC("Server do the Task") << LOG_KV("taskID", _task->id()); + if (_task->syncResultToPeer()) + { + auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists); + ecdhTaskState->setWriter(writer); + } + auto server = std::make_shared(m_config, ecdhTaskState); + addServer(server); + server->asyncStartRunTask(_task); + } + else + { + auto taskResult = std::make_shared(_task->id()); + auto error = BCOS_ERROR_PTR((int)TaskParamsError, + "The party index of the ecdh-conn-psi must be client(0) or server(1)"); + ECDH_CONN_LOG(WARNING) << LOG_DESC("response task result") << LOG_KV("task", _task) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + taskResult->setError(std::move(error)); + _onTaskFinished(std::move(taskResult)); + return; + } +} + +void EcdhConnPSIImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) +{ + try + { + auto payLoad = _msg->data(); + auto psiConnMsg = m_config->msgFactory()->decodePSIConnMessage(*payLoad); + if (psiConnMsg == nullptr) + { + return; + } + psiConnMsg->setTaskID(_msg->taskID()); + auto type = typeProcess(psiConnMsg->mainProcess(), psiConnMsg->subProcess()); + m_msgPool->insert(type, psiConnMsg); + wakeupWorker(); + } + catch (std::exception const& e) + { + ECDH_CONN_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + + +void EcdhConnPSIImpl::start() +{ + startWorking(); +} + +void EcdhConnPSIImpl::stop() +{ + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } +} + +void EcdhConnPSIImpl::onReceivedErrorNotification(const std::string& _taskID) {} + +void EcdhConnPSIImpl::onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +{} + +void EcdhConnPSIImpl::checkAndSetPeerInfo(TaskState::Ptr const& _taskState) +{ + auto task = _taskState->task(); + // check the peer + auto peerParties = task->getAllPeerParties(); + if (peerParties.size() == 1) + { + auto peerParty = peerParties.begin()->second; + _taskState->setPeerID(peerParty->id()); + ECDH_CONN_LOG(INFO) << LOG_DESC("checkAndSetPeerInfo success"); + } +} + +void EcdhConnPSIImpl::executeWorker() +{ + // auto _msg = m_msgQueue->tryPop(c_popWaitMs); + auto _msg = m_msgPool->tryPop(process(), c_popWaitMs); + while (_msg.first && process() != typeProcess((uint8_t)EcdhConnProcess::END, -2)) + { + try + { + handlerPSIReceiveMessage(_msg.second); + _msg = m_msgPool->tryPop(process(), c_popWaitMs); + } + catch (std::exception const& e) + { + ECDH_CONN_LOG(WARNING) << LOG_DESC("executeWorker exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + waitSignal(); + } +} + +void EcdhConnPSIImpl::handlerPSIReceiveMessage(PSIConnMessage::Ptr _msg) +{ + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, _msg]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + auto mainProcess = _msg->mainProcess(); + auto subProcess = _msg->subProcess(); + auto receiver = _msg->receiver(); + auto _taskId = _msg->taskID(); + if (mainProcess == int(EcdhConnProcess::HandShakeProcess) && subProcess == uint8_t(-1)) + { + if (receiver == int(PartyType::Client)) // PART2 : server -> client handshake + { + ECDH_CONN_LOG(INFO) << LOG_DESC(" PART2 : server -> client handshake ") + << LOG_KV("PartyType::Client ", _msg->value().data()); + psi->onHandShakeResponseHandler(_taskId, _msg->value()); + } + else if (receiver == int(PartyType::Server)) // PART1 : client -> server handshake + { + ECDH_CONN_LOG(INFO) << LOG_DESC(" PART1 : client -> server handshake ") + << LOG_KV("PartyType::Server ", _msg->value().data()); + psi->onHandShakeRequestHandler(_taskId, _msg->value()); + } + psi->triggleProcess((uint8_t)EcdhConnProcess::CipherProcess, -1); + } + else if (mainProcess == int(EcdhConnProcess::CipherProcess) && + subProcess == uint8_t(-1)) + { + if (receiver == int(PartyType::Client)) // PART3 : server -> client cipher + { + ECDH_CONN_LOG(INFO) << LOG_DESC(" PART3 : server -> client cipher ") + << LOG_KV("PartyType::Client ", _msg->value().data()); + psi->onCipherServerToClient(_taskId, _msg->value()); + } + else if (receiver == int(PartyType::Server)) // PART4 : client -> server cipher + { + ECDH_CONN_LOG(INFO) << LOG_DESC(" PART4 : client -> server cipher ") + << LOG_KV("PartyType::Server ", _msg->value().data()); + psi->onCipherClientToServer(_taskId, _msg->value()); + } + psi->triggleProcess((uint8_t)EcdhConnSubProcess::CipherSecondProcess, 0); + } + else if (mainProcess == int(EcdhConnSubProcess::CipherSecondProcess) && + subProcess == uint8_t(0)) + { + if (receiver == int(PartyType::Client)) // PART6 : server -> client cipher second + { + ECDH_CONN_LOG(INFO) + << LOG_DESC(" PART5 : server -> client cipher second ") + << LOG_KV("PartyType::Client ", *toHexString(_msg->value())); + psi->onSecondCipherServerToClient(_taskId, _msg->value()); + } + else if (receiver == int(PartyType::Server)) // PART5 : client -> server cipher + // second + { + ECDH_CONN_LOG(INFO) + << LOG_DESC(" PART6 : client -> server cipher second ") + << LOG_KV("PartyType::Server ", *toHexString(_msg->value())); + psi->onSecondCipherClientToServer(_taskId, _msg->value()); + } + psi->triggleProcess((uint8_t)EcdhConnProcess::END, -2); + } + } + catch (std::exception const& e) + { + ECDH_CONN_LOG(WARNING) + << LOG_DESC("handlePSIMsg exception") << LOG_KV("mainProcess", _msg->mainProcess()) + << printPSIConnMessage(_msg) << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + +// Http PART1: client -> server : handshake request +void EcdhConnPSIImpl::onHandShakeRequestHandler(const std::string& _taskId, const bcos::bytes& _msg) +{ + auto server = findServer(_taskId); + if (server) + { + server->onHandShakeRequestHandler(_msg); + } +} + +// Http PART2: server -> client : handshake response +void EcdhConnPSIImpl::onHandShakeResponseHandler( + const std::string& _taskId, const bcos::bytes& _msg) +{ + auto client = findClient(_taskId); + if (client) + { + client->onHandShakeResponseHandler(_msg); + } +} + +// Http Part3: client -> server : cipher +void EcdhConnPSIImpl::onCipherClientToServer(const std::string& _taskId, const bcos::bytes& _msg) +{ + auto server = findServer(_taskId); + if (server) + { + server->onCipherClientToServer(_msg); + } +} + +// Http Part4: server -> client : cipher +void EcdhConnPSIImpl::onCipherServerToClient(const std::string& _taskId, const bcos::bytes& _msg) +{ + auto client = findClient(_taskId); + if (client) + { + client->onCipherServerToClient(_msg); + } +} + +// Http Part5: client -> server : cipher second +void EcdhConnPSIImpl::onSecondCipherClientToServer( + const std::string& _taskId, const bcos::bytes& _msg) +{ + auto server = findServer(_taskId); + if (server) + { + server->onSecondCipherClientToServer(_msg); + } +} + +// Http Part6: server -> client : cipher second +void EcdhConnPSIImpl::onSecondCipherServerToClient( + const std::string& _taskId, const bcos::bytes& _msg) +{ + auto client = findClient(_taskId); + if (client) + { + client->onSecondCipherServerToClient(_msg); + } +} diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h new file mode 100644 index 00000000..3806491e --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIImpl.h + * @author: zachma + * @date 2023-7-18 + */ + +#pragma once +#include "../psi-framework/TaskGuarder.h" +#include "Common.h" +#include "EcdhConnPSIConfig.h" +#include "EcdhConnTaskState.h" +#include "core/EcdhConnPSIClient.h" +#include "core/EcdhConnPSIServer.h" +#include "ppc-framework/rpc/RpcInterface.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-tools/src/common/ConcurrentPool.h" +#include +#include +#include + +#include +#include + +namespace ppc::psi +{ +class EcdhConnPSIImpl : public std::enable_shared_from_this, + public ppc::task::TaskFrameworkInterface, + public TaskGuarder, + public bcos::Worker +{ +public: + using Ptr = std::shared_ptr; + using EcdhConnPSIMsgPool = ppc::tools::ConcurrentPool; + using EcdhConnPSIMsgPoolPtr = std::shared_ptr; + + EcdhConnPSIImpl(const EcdhConnPSIConfig::Ptr& _config, unsigned _idleTimeMs = 0); + virtual ~EcdhConnPSIImpl() = default; + + void asyncRunTask(ppc::protocol::Task::ConstPtr _task, + ppc::task::TaskResponseCallback&& _onTaskFinished) override; + + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; + + + void start() override; + void stop() override; + + void onReceivedErrorNotification(const std::string& _taskID) override; + void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override; + void executeWorker() override; + +protected: + bool m_enableOutputExists = false; + + void handlerPSIReceiveMessage(PSIConnMessage::Ptr _msg); + void onHandShakeRequestHandler(const std::string& _taskId, const bcos::bytes& _msg); + void onHandShakeResponseHandler(const std::string& _taskId, const bcos::bytes& _msg); + void onCipherClientToServer(const std::string& _taskId, const bcos::bytes& _msg); + void onCipherServerToClient(const std::string& _taskId, const bcos::bytes& _msg); + void onSecondCipherClientToServer(const std::string& _taskId, const bcos::bytes& _msg); + void onSecondCipherServerToClient(const std::string& _taskId, const bcos::bytes& _msg); + + void checkAndSetPeerInfo(TaskState::Ptr const& _taskState); + + void addClient(EcdhConnPSIClient::Ptr _client) + { + bcos::WriteGuard l(x_clients); + m_clients[_client->taskID()] = _client; + } + + EcdhConnPSIClient::Ptr findClient(const std::string& _taskID) + { + bcos::ReadGuard l(x_clients); + auto it = m_clients.find(_taskID); + if (it != m_clients.end()) + { + return it->second; + } + return nullptr; + } + + void addServer(EcdhConnPSIServer::Ptr _server) + { + bcos::WriteGuard l(x_servers); + m_servers[_server->taskID()] = _server; + } + + EcdhConnPSIServer::Ptr findServer(const std::string& _taskID) + { + bcos::ReadGuard l(x_servers); + auto it = m_servers.find(_taskID); + if (it != m_servers.end()) + { + return it->second; + } + return nullptr; + } + + void triggleProcess(uint8_t _mainProcess, uint8_t _subProcess) + { + bcos::WriteGuard l(x_process); + m_process = typeProcess(_mainProcess, _subProcess); + } + + uint16_t process() + { + bcos::ReadGuard l(x_process); + return m_process; + } + + uint16_t typeProcess(uint8_t _mainProcess, uint8_t _subProcess) + { + return (_subProcess << 8) | _mainProcess; + } + +private: + void waitSignal() + { + boost::unique_lock l(x_signal); + m_signal.wait_for(l, boost::chrono::milliseconds(5)); + } + + void wakeupWorker() { m_signal.notify_all(); } + + const int c_popWaitMs = 5; + uint16_t m_process; + mutable bcos::SharedMutex x_process; + + EcdhConnTaskStateFactory::Ptr m_ecdhConnTaskStateFactory; + EcdhConnPSIConfig::Ptr m_config; + EcdhConnTaskState::Ptr m_ecdhTaskState; + EcdhConnPSIMsgPoolPtr m_msgPool; + + boost::condition_variable m_signal; + boost::mutex x_signal; + + std::unordered_map m_clients; + mutable bcos::SharedMutex x_clients; + + std::unordered_map m_servers; + mutable bcos::SharedMutex x_servers; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h new file mode 100644 index 00000000..fd9123c8 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIMessageFactory.h + * @author: zachma + * @date 2023-8-23 + */ + +#pragma once +#include "Common.h" +#include "protocol/PSIConnMessage.h" +#include "protocol/transport.pb.h" + +namespace ppc::psi +{ +class EcdhConnPSIMessageFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhConnPSIMessageFactory() = default; + ~EcdhConnPSIMessageFactory() = default; + + PSIConnMessage::Ptr decodePSIConnMessage(bcos::bytes _data) + { + std::string _msgStr = ""; + _msgStr.assign(_data.begin(), _data.end()); + + // parse PushRequest + org::interconnection::link::PushRequest* pushRequest = + new org::interconnection::link::PushRequest(); + pushRequest->ParseFromString(_msgStr); + auto key = pushRequest->key(); + int sender = 0; + int mainProcess = 0; + int subProcess = -1; + int receiver = 0; + if (parseKey(key, &sender, &receiver, &mainProcess, &subProcess)) + { + auto value = pushRequest->value(); + + ECDH_CONN_LOG(INFO) << LOG_DESC("decodePSIConnMessage start") + << LOG_KV("sender", sender) << LOG_KV("receiver", receiver) + << LOG_KV("mainProcess", mainProcess) + << LOG_KV("subProcess", subProcess) + << LOG_KV("value: ", value.size()); + auto _psiConn = std::make_shared(key); + _psiConn->setValue(bcos::bytes(value.begin(), value.end())); + _psiConn->setSender(sender); + _psiConn->setReceiver(receiver); + _psiConn->setMainProcess(mainProcess); + _psiConn->setSubProcess(subProcess); + return _psiConn; + } + + return nullptr; + } + +protected: + // main: root:P2P-1:1->0{'\x01', '\x02'}[1] + // sub: root-0:P2P-1:1->0{'\x01', '\x02'}[1] + bool parseKey( + const std::string& _key, int* _from, int* _to, int* _mainProcess, int* _subProcess) + { + if (NULL == _to || NULL == _mainProcess || NULL == _subProcess || NULL == _from) + { + return false; + } + + std::vector fieldNames; + boost::split(fieldNames, _key, boost::is_any_of(":")); + if (fieldNames.size() < 3) + { + return false; + } + // root-0 + std::string _sub_process_p2p = fieldNames.at(0); + *_subProcess = _sub_process_p2p.back() - 48; + if (*_subProcess < 0 || *_subProcess > 9) + { + *_subProcess = -1; + } + // P2P-1 + std::string _process_p2p = fieldNames.at(1); + *_mainProcess = _process_p2p.back() - 48; + + // 1->0{'\x01', '\x02'}[1] + std::string _to_p2p = fieldNames.at(2); + std::vector fieldNames2; + boost::split(fieldNames2, _to_p2p, boost::is_any_of(std::string{'\x01', '\x02'})); + // *_to = fieldNames2.at(0).back() - 48; + // *_from = fieldNames2.at(0).front() - 48; + *_to = 49 - fieldNames2.at(0).back(); + *_from = 49 - fieldNames2.at(0).front(); + return true; + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h new file mode 100644 index 00000000..831356d9 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnTaskState.h + * @author: zachma + * @date 2023-7-20 + */ + +#pragma once +#include "../psi-framework/TaskState.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/crypto/ECDHCrypto.h" + +namespace ppc::psi +{ +class EcdhConnTaskState : public TaskState +{ +public: + using Ptr = std::shared_ptr; + EcdhConnTaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, + bool _onlySelfRun = false) + : TaskState(_task, std::move(_callback), _onlySelfRun) + {} + ~EcdhConnTaskState() = default; + + ppc::crypto::CryptoBox::Ptr cryptoBox() const + { + // bcos::ReadGuard l(x_cryptoBox); + return m_cryptoBox; + } + + void setCryptoBox(ppc::crypto::CryptoBox::Ptr const& _cryptoBox) + { + // bcos::WriteGuard l(x_cryptoBox); + m_cryptoBox = _cryptoBox; + } + + // return copy here to ensure thread-safe + ppc::crypto::ECDHCrypto::Ptr ecdhCrypto() const + { + // bcos::ReadGuard l(x_ecdhCrypto); + return m_ecdhCrypto; + } + // Note: here must lock, in-case of multiple tasks with the same task-id processed at almost the + // same time + void setEcdhCrypto(ppc::crypto::ECDHCrypto::Ptr const& _ecdhCrypto) + { + // bcos::WriteGuard l(x_ecdhCrypto); + m_ecdhCrypto = _ecdhCrypto; + } + +private: + ppc::crypto::ECDHCrypto::Ptr m_ecdhCrypto; + // mutable bcos::SharedMutex x_ecdhCrypto; + + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + // mutable bcos::SharedMutex x_cryptoBox; +}; + +class EcdhConnTaskStateFactory : public TaskStateFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhConnTaskStateFactory() : TaskStateFactory() {} + ~EcdhConnTaskStateFactory() override = default; + + EcdhConnTaskState::Ptr createConnTaskState(ppc::protocol::Task::ConstPtr const& _task, + TaskResponseCallback&& _callback, bool _onlySelfRun = false, + PSIConfig::Ptr _config = nullptr) + { + return std::make_shared(_task, std::move(_callback), false); + } +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp new file mode 100644 index 00000000..7c21878c --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp @@ -0,0 +1,325 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIClient.cpp + * @author: zachma + * @date 2023-7-18 + */ +#include "EcdhConnPSIClient.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace bcos; + +EcdhConnPSIClient::EcdhConnPSIClient( + EcdhConnPSIConfig::Ptr _config, EcdhConnTaskState::Ptr _taskState) + : m_config(_config), m_taskState(_taskState), m_send_count(0) +{ + auto task = m_taskState->task(); + m_taskID = task->id(); + m_msgFactory = std::make_shared(); +} + +void EcdhConnPSIClient::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task) +{ + ECDH_CONN_LOG(INFO) << LOG_DESC("asyncStartRunTask: ") << printTaskInfo(_task); + m_header = initHeader(_task->id()); + sendHandshakeRequest(m_taskState); +} + +void EcdhConnPSIClient::onSecondCipherServerToClient(const bcos::bytes& _msg) +{ + // 接收Server发来的M_Q2 + auto cipherExVo = m_msgFactory->parseCipherExchange(_msg); + m_Q2 = cipherExVo->cipherText(); + + auto intersectionVec = tryIntersection(m_Q2, m_P2_Client); + m_taskState->storePSIResult(m_config->dataResourceLoader(), intersectionVec); + + // 把m_P2_Client发送到Server + // create CipherExchange + auto cipherBatchVo = std::make_shared("dual.enc", 1, true); + cipherBatchVo->setCount(m_P2_Client.size()); + cipherBatchVo->setCipherText(m_P2_Client); + auto cipherBytes = m_msgFactory->createCipherExchange(cipherBatchVo); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest( + cipherBytes, generateKey((int)EcdhConnSubProcess::CipherSecondProcess, 0)); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(m_taskState->task()->type(), + m_taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(m_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [this, self = weak_from_this()](bcos::Error::Ptr _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error) + { + m_taskState->onTaskFinished(); + return; + } + }, + nullptr); +} + +void EcdhConnPSIClient::onCipherServerToClient(const bcos::bytes& _msg) +{ + // parse EcdhPsiCipherBatch to vector + auto cipherExVo = m_msgFactory->parseCipherExchange(_msg); + m_P1_Client = cipherExVo->cipherText(); + ECDH_CONN_LOG(INFO) << LOG_DESC("onCipherServerToClient: ") + << LOG_KV("m_P1_Client SIZE: ", m_P1_Client.size()); + + computeAndEncryptSetSecond(m_P1_Client, std::make_shared(m_randomA)); + + + auto cipherBatchVo = std::make_shared("enc", 0, true); + + cipherBatchVo->setCount(m_Q1.size()); + cipherBatchVo->setCipherText(m_Q1); + auto cipherBytes = m_msgFactory->createCipherExchange(cipherBatchVo); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest( + cipherBytes, generateKey((int)EcdhConnProcess::CipherProcess, -1)); + + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(m_taskState->task()->type(), + m_taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(m_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [this, self = weak_from_this()](bcos::Error::Ptr _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error) + { + return; + } + }, + nullptr); +} + +void EcdhConnPSIClient::onHandShakeResponseHandler(const bcos::bytes& _msg) +{ + ECDH_CONN_LOG(INFO) << LOG_DESC("onHandShakeResponseHandler start"); + auto handShakeResponseVo = m_msgFactory->parseHandshakeResponse(_msg); + auto errCode = handShakeResponseVo->GetErrorCode(); + auto errMsg = handShakeResponseVo->GetErrorMsg(); + + if (errCode) + { + auto errorMsg = "onHandShakeResponseHandler failed for no errCode/errMsg selected"; + ECDH_CONN_LOG(WARNING) << LOG_DESC(errorMsg) << LOG_KV("errCode", errCode) + << LOG_KV("errMsg", errMsg); + auto result = std::make_shared(m_taskState->task()->id()); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandshakeFailed, errorMsg); + result->setError(std::move(error)); + m_taskState->onTaskFinished(result, false); + return; + } + + auto curve = handShakeResponseVo->GetCurve(); + auto hashType = handShakeResponseVo->GetHash(); +#ifdef ENABLE_CONN + auto cryptoBox = m_config->ecdhCryptoFactory()->createCryptoBox( + (int)ppc::protocol::ECCCurve::SM2, (int)ppc::protocol::HashImplName::SHA256); +#else + auto cryptoBox = + m_config->ecdhCryptoFactory()->createCryptoBox((int8_t)curve, (int8_t)hashType); +#endif + m_taskState->setCryptoBox(cryptoBox); + ECDH_CONN_LOG(INFO) << LOG_DESC("onHandShakeResponseHandler success"); + // CREATE RANDOM + m_randomA = m_taskState->cryptoBox()->eccCrypto()->generateRandomScalar(); + computeAndEncryptSet(std::make_shared(m_randomA)); +} + +void EcdhConnPSIClient::computeAndEncryptSet(bcos::bytesPointer _randA) +{ + auto inputSize = m_originInputs->size(); + if (!m_originInputs || inputSize == 0) + { + ECDH_CONN_LOG(ERROR) << LOG_DESC("Client computeAndEncryptSet") + << LOG_DESC("data size is empty!"); + auto result = std::make_shared(m_taskState->task()->id()); + auto err = std::make_shared(-12222, "canceled by other peer"); + result->setError(std::move(err)); + m_taskState->onTaskFinished(result, false); + } + std::vector _res_vectors; + auto hash = m_taskState->cryptoBox()->hashImpl(); + for (int i = 0; i < inputSize; i++) + { + auto data = m_originInputs->getBytes(i); + auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_taskState->cryptoBox()->eccCrypto()->hashToCurve(hashData); + auto hashSet = m_taskState->cryptoBox()->eccCrypto()->ecMultiply(point, *_randA); + _res_vectors.push_back(hashSet); + } + + m_Q1 = _res_vectors; + ECDH_CONN_LOG(INFO) << LOG_DESC("computeAndEncryptSet success") + << LOG_KV("m_Q1 size: ", m_Q1.size()); +} + +void EcdhConnPSIClient::computeAndEncryptSetSecond( + const std::vector& _input, bcos::bytesPointer _randA) +{ + std::vector _res_vectors; + auto inputSize = _input.size(); + for (int i = 0; i < inputSize; i++) + { + auto inputData = _input.at(i); + auto encryptSet = m_taskState->cryptoBox()->eccCrypto()->ecMultiply(inputData, *_randA); + _res_vectors.push_back(encryptSet); + } + + ECDH_CONN_LOG(INFO) << LOG_DESC("onCipherServerToClient: computeAndEncryptSetSecond: ") + << LOG_KV("m_P1_Client SIZE: ", m_P2_Client.size()); + + m_P2_Client = _res_vectors; +} + + +void EcdhConnPSIClient::sendHandshakeRequest(TaskState::Ptr _taskState) +{ + if (_taskState->peerID().empty()) + { + return; + } + ECDH_CONN_LOG(TRACE) << LOG_DESC("sendHandshakeRequest start"); + m_originInputs = _taskState->loadAllData(); + auto item_count = m_originInputs->size(); + auto ecdhConnPSIConfig = m_config->ppcConfig(); + auto handReqVo = std::make_shared(); + std::set _curve{(int32_t)ECCCurve::P256}; + std::set _hash{(int32_t)HashImplName::SHA256}; + handReqVo->SetCurve(_curve); + handReqVo->SetHash(_hash); + handReqVo->SetItemCount(item_count); + auto handshakeBytes = m_msgFactory->createHandshakeRequest(handReqVo); + auto key = generateKey((int)EcdhConnProcess::HandShakeProcess, -1); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest(handshakeBytes, key); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(_taskState->task()->type(), + _taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + nullptr); +} + +// input1 ∩ input2 -> +std::vector EcdhConnPSIClient::tryIntersection(const std::vector& _input1, + const std::vector& _input2, int _compareSuffixBitSize) +{ + std::vector input1; + for (auto const& _in : _input1) + { + bcos::bytes::const_iterator head = _in.begin() + _in.size() - _compareSuffixBitSize; + bcos::bytes input_temp(head, _in.end()); + input1.push_back(input_temp); + ECDH_CONN_LOG(TRACE) << LOG_KV("input_temp VALUE:", *toHexString(input_temp)) + << LOG_KV("input_temp size:", input_temp.size()); + } + + std::vector input2; + for (auto const& _in : _input2) + { + bcos::bytes::const_iterator head = _in.begin() + _in.size() - _compareSuffixBitSize; + bcos::bytes input_temp(head, _in.end()); + input2.push_back(input_temp); + ECDH_CONN_LOG(TRACE) << LOG_KV("input_temp VALUE:", *toHexString(input_temp)) + << LOG_KV("input_temp size:", input_temp.size()); + } + + int index = 0; + std::set result_set; + for (auto const& _in : input1) + { + if (std::find(input2.begin(), input2.end(), _in) != input2.end()) + { + result_set.insert(m_originInputs->getBytes(index)); + } + index++; + } + std::vector vec; + vec.assign(result_set.begin(), result_set.end()); + return vec; +} + +std::string EcdhConnPSIClient::generateKey(int _mainProcess, int _subProcess) +{ + m_send_count++; + std::string keys = ""; + if (_subProcess == -1) + { + // main process + keys = "root:P2P-" + std::to_string(_mainProcess) + ":" + + std::to_string(1 - (int)PartyType::Client) + "->" + + std::to_string(1 - (int)PartyType::Server) + std::string{'\x01', '\x02'} + + std::to_string(m_send_count); + } + else + { + keys = "root-" + std::to_string(_subProcess) + ":P2P-" + std::to_string(_mainProcess) + + ":" + std::to_string(1 - (int)PartyType::Client) + "->" + + std::to_string(1 - (int)PartyType::Server) + std::string{'\x01', '\x02'} + + std::to_string(m_send_count); + } + return keys; +} + +std::map EcdhConnPSIClient::initHeader(const std::string& _taskId) +{ + std::map header; + header[SESSION_ID_HEAD] = _taskId; + header[VERSION_HEAD] = "v1.0"; + header[TRACE_ID_HEAD] = "trace_" + _taskId; + header[TOKEN_HEAD] = "token_" + _taskId; + + header[SOURCE_NODE_HEAD] = "HOST"; + header[TARGET_NODE_HEAD] = "GUEST"; + header[SOURCE_INST_HEAD] = "HOST"; + header[TARGET_INST_HEAD] = "GUEST"; + header[TECH_PROVIDER_CODE_HEAD] = "WEBANK"; + header[TOPIC_HEAD] = "host.0-guest.0"; + + return header; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h new file mode 100644 index 00000000..35b40fef --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIClient.h + * @author: zachma + * @date 2023-7-18 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "../EcdhConnPSIConfig.h" +#include "../EcdhConnTaskState.h" +#include "../protocol/PSIConnMessageFactory.h" +#include +#include + +namespace ppc::psi +{ +class EcdhConnPSIClient : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + EcdhConnPSIClient(EcdhConnPSIConfig::Ptr _config, EcdhConnTaskState::Ptr _taskState); + virtual ~EcdhConnPSIClient() = default; + const std::string& taskID() const { return m_taskID; } + + + virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task); + + virtual void sendHandshakeRequest(TaskState::Ptr _taskState); + virtual void onSecondCipherServerToClient(const bcos::bytes& _msg); + virtual void onCipherServerToClient(const bcos::bytes& _msg); + virtual void onHandShakeResponseHandler(const bcos::bytes& _msg); + +protected: + virtual void computeAndEncryptSet(bcos::bytesPointer _randA); + virtual void computeAndEncryptSetSecond( + const std::vector& _input, bcos::bytesPointer _randA); + virtual std::vector tryIntersection(const std::vector& _input1, + const std::vector& _input2, int _compareSuffixBitSize = 12); + +private: + int m_send_count = 0; + ppc::io::DataBatch::Ptr m_originInputs; + PSIConnMessageFactory::Ptr m_connMsgFactory; + EcdhConnPSIConfig::Ptr m_config; + std::string m_taskID; + EcdhConnTaskState::Ptr m_taskState; + PSIConnMessageFactory::Ptr m_msgFactory; + + std::map m_header; + // 保存最终结果 + bcos::bytes m_randomA; + + std::vector m_Q2; + std::vector m_Q1; + std::vector m_P2_Client; + std::vector m_P1_Client; + +private: + virtual std::map initHeader(const std::string& _taskId); + virtual std::string generateKey(int _mainProcess, int _subProcess); +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp new file mode 100644 index 00000000..9d342c9b --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp @@ -0,0 +1,349 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIServer.cpp + * @author: zachma + * @date 2023-7-18 + */ +#include "EcdhConnPSIServer.h" +#include "ppc-framework/protocol/GlobalConfig.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace bcos; + +EcdhConnPSIServer::EcdhConnPSIServer( + EcdhConnPSIConfig::Ptr _config, EcdhConnTaskState::Ptr _taskState) + : m_config(_config), m_taskState(_taskState) +{ + auto task = m_taskState->task(); + m_taskID = task->id(); + m_msgFactory = std::make_shared(); +} + +void EcdhConnPSIServer::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task) +{ + ECDH_CONN_LOG(INFO) << LOG_DESC("asyncStartRunTask: ") << printTaskInfo(_task); + m_header = initHeader(_task->id()); +} + + +void EcdhConnPSIServer::onHandShakeRequestHandler(const bcos::bytes& _msg) +{ + auto handRequestVo = m_msgFactory->parseHandshakeRequest(_msg); + auto clientCurves = handRequestVo->GetCurve(); + auto clientHashList = handRequestVo->GetHash(); + auto supportedCurves = g_PPCConfig.supportedCurves( + (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC); + auto supportedHashList = g_PPCConfig.supportedHashList( + (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC); + std::set localCurves(supportedCurves.begin(), supportedCurves.end()); + std::set localHashTypes(supportedHashList.begin(), supportedHashList.end()); + bool curveSelected = false; + bool hashTypeSelected = false; + // sm-crypto enabled + auto handShakeResponse = std::make_shared(); + if (g_PPCConfig.smCrypto()) + { + // try to select SM2 + if (clientCurves.count((int32_t)ppc::protocol::ECCCurve::SM2) && + localCurves.count((int32_t)ppc::protocol::ECCCurve::SM2)) + { + handShakeResponse->SetCurve((int32_t)ppc::protocol::ECCCurve::SM2); + curveSelected = true; + } + // try to select SM3 + if (clientHashList.count((int32_t)ppc::protocol::HashImplName::SM3) && + localHashTypes.count((int32_t)ppc::protocol::HashImplName::SM3)) + { + handShakeResponse->SetHash((int32_t)ppc::protocol::HashImplName::SM3); + hashTypeSelected = true; + } + } + + if (!curveSelected) + { +#ifdef ENABLE_CONN + handShakeResponse->SetCurve((int32_t)ppc::protocol::ECCCurve::SM2); +#else + handShakeResponse->SetCurve(selectCryptoAlgorithm(clientCurves, localCurves)); +#endif + } + if (!hashTypeSelected) + { +#ifdef ENABLE_CONN + handShakeResponse->SetHash((int32_t)ppc::protocol::HashImplName::SHA256); +#else + handShakeResponse->SetHash(selectCryptoAlgorithm(clientHashList, localHashTypes)); +#endif + } + + auto SelectedCurve = handShakeResponse->GetCurve(); + auto SelectedHashType = handShakeResponse->GetHash(); + if (SelectedCurve != int32_t(-1) && SelectedHashType != int32_t(-1)) + { + auto cryptoBox = + m_config->ecdhCryptoFactory()->createCryptoBox(SelectedCurve, SelectedHashType); + m_taskState->setCryptoBox(cryptoBox); + m_randomB = m_taskState->cryptoBox()->eccCrypto()->generateRandomScalar(); + } + else + { + auto errorMsg = "onHandshakeRequest failed for no curve/hashType selected"; + ECDH_CONN_LOG(WARNING) << LOG_DESC(errorMsg) << LOG_KV("selectedCurve", SelectedCurve) + << LOG_KV("selectedHashType", SelectedHashType); + auto result = std::make_shared(m_taskID); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandshakeFailed, errorMsg); + result->setError(std::move(error)); + m_taskState->onTaskFinished(result, false); + return; + } + + // send message to response + auto handshakeResponseBytes = m_msgFactory->createHandshakeResponse(handShakeResponse); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest( + handshakeResponseBytes, generateKey((int)EcdhConnProcess::HandShakeProcess, -1)); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(m_taskState->task()->type(), + m_taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(m_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [this, self = weak_from_this()](bcos::Error::Ptr _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error) + { + psi->computeAndEncryptSet(std::make_shared(m_randomB)); + } + }, + nullptr); +} +void EcdhConnPSIServer::onCipherClientToServer(const bcos::bytes& _msg) +{ + auto cipherExVo = m_msgFactory->parseCipherExchange(_msg); + m_Q1_server = cipherExVo->cipherText(); + // 收到Q1 计算 Q2 + computeAndEncryptSetSecond(m_Q1_server, std::make_shared(m_randomB)); +} +void EcdhConnPSIServer::onSecondCipherClientToServer(const bcos::bytes& _msg) +{ + // 接收client发来的m_P2 + auto cipherExPb = m_msgFactory->parseCipherExchange(_msg); + m_P2 = cipherExPb->cipherText(); + auto intersectionVec = tryIntersection(m_P2, m_Q2_server); + bool synced = m_taskState->task()->syncResultToPeer(); + if (synced) + { + m_taskState->storePSIResult(m_config->dataResourceLoader(), intersectionVec); + } + m_taskState->onTaskFinished(); +} + +void EcdhConnPSIServer::computeAndEncryptSet(bcos::bytesPointer _randB) +{ + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + ECDH_CONN_LOG(INFO) << LOG_DESC("Server computeAndEncryptSet") + << LOG_DESC("data size is empty!"); + auto result = std::make_shared(m_taskState->task()->id()); + auto err = std::make_shared(-12222, "canceled by other peer"); + result->setError(std::move(err)); + m_taskState->onTaskFinished(result, false); + } + auto inputSize = m_originInputs->size(); + std::vector _res_vectors; + auto hash = m_taskState->cryptoBox()->hashImpl(); + for (int i = 0; i < inputSize; i++) + { + auto data = m_originInputs->getBytes(i); + auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_taskState->cryptoBox()->eccCrypto()->hashToCurve(hashData); + auto hashSet = m_taskState->cryptoBox()->eccCrypto()->ecMultiply(point, *_randB); + _res_vectors.push_back(hashSet); + } + + m_P1 = _res_vectors; + + auto cipherBatchVo = std::make_shared("enc", 0, true); + cipherBatchVo->setCount(m_P1.size()); + cipherBatchVo->setCipherText(m_P1); + auto cipherBytes = m_msgFactory->createCipherExchange(cipherBatchVo); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest( + cipherBytes, generateKey((int)EcdhConnProcess::CipherProcess, -1)); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(m_taskState->task()->type(), + m_taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(m_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + // shake hands with each other + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [this, self = weak_from_this()](bcos::Error::Ptr _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error) + { + return; + } + }, + nullptr); +} + +void EcdhConnPSIServer::computeAndEncryptSetSecond( + const std::vector& _input, bcos::bytesPointer _randB) +{ + std::vector _res_vectors; + auto inputSize = _input.size(); + for (int i = 0; i < inputSize; i++) + { + auto inputData = _input.at(i); + auto encryptSet = m_taskState->cryptoBox()->eccCrypto()->ecMultiply(inputData, *_randB); + _res_vectors.push_back(encryptSet); + } + m_Q2_server = _res_vectors; + + auto cipherBatchVo = std::make_shared("dual.enc", 1, true); + cipherBatchVo->setCount(m_Q2_server.size()); + cipherBatchVo->setCipherText(m_Q2_server); + auto cipherBytes = m_msgFactory->createCipherExchange(cipherBatchVo); + auto requestBytes = m_msgFactory->createPSIConnMessageRequest( + cipherBytes, generateKey((int)EcdhConnSubProcess::CipherSecondProcess, 0)); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(m_taskState->task()->type(), + m_taskState->task()->algorithm(), m_taskID, std::make_shared(requestBytes)); + message->setSender(m_taskState->task()->selfParty()->id()); + message->setHeader(m_header); + + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [this, self = weak_from_this()](bcos::Error::Ptr _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error) + { + return; + } + }, + nullptr); +} + +// input1 ∩ input2 -> +std::vector EcdhConnPSIServer::tryIntersection(const std::vector& _input1, + const std::vector& _input2, int _compareSuffixBitSize) +{ + std::vector input1; + for (auto const& _in : _input1) + { + bcos::bytes::const_iterator head = _in.begin() + _in.size() - _compareSuffixBitSize; + bcos::bytes input_temp(head, _in.end()); + input1.push_back(input_temp); + ECDH_CONN_LOG(TRACE) << LOG_KV("input_temp VALUE:", *toHexString(input_temp)) + << LOG_KV("input_temp size:", input_temp.size()); + } + + std::vector input2; + for (auto const& _in : _input2) + { + bcos::bytes::const_iterator head = _in.begin() + _in.size() - _compareSuffixBitSize; + bcos::bytes input_temp(head, _in.end()); + input2.push_back(input_temp); + ECDH_CONN_LOG(TRACE) << LOG_KV("input_temp VALUE:", *toHexString(input_temp)) + << LOG_KV("input_temp size:", input_temp.size()); + } + + int index = 0; + std::set result_set; + for (auto const& _in : input1) + { + if (std::find(input2.begin(), input2.end(), _in) != input2.end()) + { + result_set.insert(m_originInputs->getBytes(index)); + } + index++; + } + std::vector vec; + vec.assign(result_set.begin(), result_set.end()); + return vec; +} + +std::string EcdhConnPSIServer::generateKey(int _mainProcess, int _subProcess) +{ + m_send_count++; + std::string keys = ""; + if (_subProcess == -1) + { + // main process + keys = "root:P2P-" + std::to_string(_mainProcess) + ":" + + std::to_string(1 - (int)PartyType::Server) + "->" + + std::to_string(1 - (int)PartyType::Client) + std::string{'\x01', '\x02'} + + std::to_string(m_send_count); + } + else + { + keys = "root-" + std::to_string(_subProcess) + ":P2P-" + std::to_string(_mainProcess) + + ":" + std::to_string(1 - (int)PartyType::Server) + "->" + + std::to_string(1 - (int)PartyType::Client) + std::string{'\x01', '\x02'} + + std::to_string(m_send_count); + } + return keys; +} + +int32_t EcdhConnPSIServer::selectCryptoAlgorithm( + std::set _clientSupportedAlgorithms, std::set _localSupportedAlgorithms) +{ + for (auto const& algorithm : _clientSupportedAlgorithms) + { + if (_localSupportedAlgorithms.count(algorithm)) + { + return algorithm; + } + } + // Note: -1 means select algorithm failed + return int32_t(-1); +} + +std::map EcdhConnPSIServer::initHeader(const std::string& _taskId) +{ + std::map header; + header[SESSION_ID_HEAD] = _taskId; + header[VERSION_HEAD] = "v1.0"; + header[TRACE_ID_HEAD] = "trace_" + _taskId; + header[TOKEN_HEAD] = "token_" + _taskId; + + header[SOURCE_NODE_HEAD] = "GUEST"; + header[TARGET_NODE_HEAD] = "HOST"; + header[SOURCE_INST_HEAD] = "GUEST"; + header[TARGET_INST_HEAD] = "HOST"; + header[TECH_PROVIDER_CODE_HEAD] = "WEBANK"; + header[TOPIC_HEAD] = "host.0-guest.0"; + + return header; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h new file mode 100644 index 00000000..cbc6de12 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhConnPSIServer.h + * @author: zachma + * @date 2023-7-18 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "../EcdhConnPSIConfig.h" +#include "../EcdhConnTaskState.h" +#include "../protocol/PSIConnMessageFactory.h" +#include +#include + +namespace ppc::psi +{ +class EcdhConnPSIServer : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + EcdhConnPSIServer(EcdhConnPSIConfig::Ptr _config, EcdhConnTaskState::Ptr _taskState); + virtual ~EcdhConnPSIServer() = default; + const std::string& taskID() const { return m_taskID; } + + virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task); + + virtual void onHandShakeRequestHandler(const bcos::bytes& _msg); + virtual void onCipherClientToServer(const bcos::bytes& _msg); + virtual void onSecondCipherClientToServer(const bcos::bytes& _msg); + +protected: + virtual void computeAndEncryptSet(bcos::bytesPointer _randB); + virtual void computeAndEncryptSetSecond( + const std::vector& _input, bcos::bytesPointer _randB); + virtual std::vector tryIntersection(const std::vector& _input1, + const std::vector& _input2, int _compareSuffixBitSize = 12); + virtual int32_t selectCryptoAlgorithm( + std::set _clientSupportedAlgorithms, std::set _localSupportedAlgorithms); + +private: + int m_send_count = 0; + bcos::bytes m_randomB; + ppc::io::DataBatch::Ptr m_originInputs; + EcdhConnPSIConfig::Ptr m_config; + std::string m_taskID; + EcdhConnTaskState::Ptr m_taskState; + PSIConnMessageFactory::Ptr m_msgFactory; + + std::map m_header; + + + // 保存最终结果 + std::vector m_Q1_server; + std::vector m_Q2_server; + + std::vector m_P2; + std::vector m_P1; + +private: + virtual std::string generateKey(int _mainProcess, int _subProcess); + virtual std::map initHeader(const std::string& _taskId); +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h new file mode 100644 index 00000000..bef1b62c --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIConnMessage.h + * @author: zachma + * @date 2023-8-23 + */ +#pragma once +#include "ppc-framework/io/DataBatch.h" +#include +#include + +namespace ppc::psi +{ +class PSIConnMessage +{ +public: + using Ptr = std::shared_ptr; + PSIConnMessage(const std::string& _key) : m_key(_key){}; + ~PSIConnMessage() = default; + + // the key + std::string const& key() const { return m_key; } + void setKey(const std::string& _key) { m_key = _key; } + + // the value + bcos::bytes value() { return m_value; } + void setValue(bcos::bytes _value) { m_value = _value; } + + // the packetType + uint8_t packetType() { return m_packetType; } + void setPacketType(uint8_t _packetType) { m_packetType = _packetType; } + + // the sender + uint16_t sender() { return m_sender; } + void setSender(uint16_t _sender) { m_sender = _sender; } + + // the receiver + uint16_t receiver() { return m_receiver; } + void setReceiver(uint16_t _receiver) { m_receiver = _receiver; } + + // the MainProcess + uint8_t mainProcess() { return m_mainProcess; } + void setMainProcess(uint8_t _mainProcess) { m_mainProcess = _mainProcess; } + + // the SubProcess + uint8_t subProcess() { return m_subProcess; } + void setSubProcess(uint8_t _subProcess) { m_subProcess = _subProcess; } + + // the TaskId + std::string const& taskID() const { return m_taskId; } + void setTaskID(const std::string& _taskId) { m_taskId = _taskId; } + +private: + std::string m_key; + std::string m_taskId; + bcos::bytes m_value; + uint8_t m_packetType; + uint16_t m_sender; + uint16_t m_receiver; + uint8_t m_subProcess; + uint8_t m_mainProcess; +}; + +class HandShakeRequestVo +{ +public: + using Ptr = std::shared_ptr; + HandShakeRequestVo() = default; + HandShakeRequestVo(std::set curve, std::set hash, int32_t protocol_families, + int32_t item_count) + : m_curve(curve), + m_hash(hash), + m_protocol_families(protocol_families), + m_item_count(item_count) + {} + + ~HandShakeRequestVo() = default; + void SetCurve(const std::set& curve) { m_curve = curve; } + + void SetHash(const std::set& hash) { m_hash = hash; } + + void SetProtocolFamilies(int32_t protocol_families) { m_protocol_families = protocol_families; } + + void SetItemCount(int32_t item_count) { m_item_count = item_count; } + + std::set GetCurve() const { return m_curve; } + + std::set GetHash() const { return m_hash; } + + int32_t GetProtocolFamilies() const { return m_protocol_families; } + + int32_t GetItemCount() const { return m_item_count; } + +private: + std::set m_curve; + std::set m_hash; + int32_t m_protocol_families; + int32_t m_item_count; +}; + +class HandShakeResponseVo +{ +public: + using Ptr = std::shared_ptr; + HandShakeResponseVo() = default; + HandShakeResponseVo(int32_t protocol_families, int32_t curve, int32_t hash) + : m_protocol_families(protocol_families), m_curve(curve), m_hash(hash) + {} + + virtual ~HandShakeResponseVo() = default; + + void SetCurve(int32_t curve) { m_curve = curve; } + + void SetHash(int32_t hash) { m_hash = hash; } + + void SetProtocolFamilies(int32_t protocol_families) { m_protocol_families = protocol_families; } + + void SetErrorCode(int32_t error_code) { m_error_code = error_code; } + + void SetErrorMessage(const std::string& msg) { m_error_msg = msg; } + + int32_t GetCurve() const { return m_curve; } + + int32_t GetHash() const { return m_hash; } + + int32_t GetProtocolFamilies() const { return m_protocol_families; } + + int32_t GetErrorCode() const { return m_error_code; } + + std::string GetErrorMsg() const { return m_error_msg; } + +private: + int32_t m_curve; + int32_t m_hash; + int32_t m_protocol_families; + int32_t m_error_code; + std::string m_error_msg; +}; + +class CipherBatchVo +{ +public: + using Ptr = std::shared_ptr; + CipherBatchVo() = default; + CipherBatchVo(std::string _type, int32_t _batch_index, bool _is_last_batch) + : m_type(_type), m_batch_index(_batch_index), m_is_last_batch(_is_last_batch) + {} + virtual ~CipherBatchVo() = default; + + void setType(const std::string& _type) { m_type = _type; } + + void setBatchIndex(int32_t _batch_index) { m_batch_index = _batch_index; } + + void setIsLastBatch(bool _is_last_batch) { m_is_last_batch = _is_last_batch; } + + void setCount(int32_t _count) { m_count = _count; } + + void setCipherText(const std::vector& _cipherText) { m_ciphertext = _cipherText; } + + std::string type() const { return m_type; } + + int32_t batch_index() const { return m_batch_index; } + + bool is_last_batch() const { return m_is_last_batch; } + + int32_t count() const { return m_count; } + + std::vector cipherText() const { return m_ciphertext; } + +private: + std::string m_type; + int32_t m_batch_index; + bool m_is_last_batch; + int32_t m_count; + std::vector m_ciphertext; +}; + + +inline std::string printPSIConnMessage(PSIConnMessage::Ptr _msg) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("sender", _msg->sender()) << LOG_KV("key", _msg->key()) + << LOG_KV("receive", _msg->receiver()) << LOG_KV("taskId", _msg->taskID()) + << LOG_KV("message type", _msg->packetType()) + << LOG_KV("main process", _msg->mainProcess()) + << LOG_KV("sub process", _msg->subProcess()); + return stringstream.str(); +} +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp new file mode 100644 index 00000000..ec5ceb22 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIConnMessageFactory.cpp + * @author: zachma + * @date 2023-8-23 + */ + +#pragma once +#include "PSIConnMessageFactory.h" +#include "ecc.pb.h" +#include "ecdh_psi.pb.h" +#include "entry.pb.h" +#include "header.pb.h" +#include "psi.pb.h" +#include "transport.pb.h" +#include + +using namespace ppc::psi; + +bcos::bytes PSIConnMessageFactory::createHandshakeRequest( + HandShakeRequestVo::Ptr handShakeRequestVo) +{ + org::interconnection::v2::HandshakeRequest handShakeRequest; + handShakeRequest.set_version(2); + handShakeRequest.set_requester_rank((int32_t)EcdhConnProcess::HandShakeProcess); + handShakeRequest.add_supported_algos(org::interconnection::v2::AlgoType::ALGO_TYPE_ECDH_PSI); + handShakeRequest.add_protocol_families( + org::interconnection::v2::ProtocolFamily::PROTOCOL_FAMILY_ECC); + + // create eccProtocol + org::interconnection::v2::protocol::EccProtocolProposal eccProtocol; + eccProtocol.add_supported_versions(2); + eccProtocol.add_point_octet_formats( + org::interconnection::v2::protocol::PointOctetFormat::POINT_OCTET_FORMAT_X962_COMPRESSED); + eccProtocol.set_support_point_truncation(false); + + // create eccSuit + org::interconnection::v2::protocol::EcSuit* eccSuit = eccProtocol.add_ec_suits(); + auto _curve = handShakeRequestVo->GetCurve().cbegin(); + auto _hash = handShakeRequestVo->GetHash().cbegin(); + +#ifdef ENABLE_CONN + eccSuit->set_curve(2); // SM2 + eccSuit->set_hash(11); // SHA256 +#else + eccSuit->set_curve(*_curve); + eccSuit->set_hash(*_hash); +#endif + + eccSuit->set_hash2curve_strategy(org::interconnection::v2::protocol::HashToCurveStrategy:: + HASH_TO_CURVE_STRATEGY_TRY_AND_REHASH); + + // google::protobuf::Any protocol_families_params_any; + handShakeRequest.add_protocol_family_params()->PackFrom(eccProtocol); + + // create PsiDataIoProposal + org::interconnection::v2::algos::PsiDataIoProposal psiDataIo; + psiDataIo.add_supported_versions(2); + psiDataIo.set_result_to_rank(-1); + psiDataIo.set_item_num(handShakeRequestVo->GetItemCount()); + + handShakeRequest.mutable_io_param()->PackFrom(psiDataIo); + bcos::bytes result; + result.resize(handShakeRequest.ByteSize()); + handShakeRequest.SerializeToArray(result.data(), handShakeRequest.ByteSize()); + return result; +} + +HandShakeRequestVo::Ptr PSIConnMessageFactory::parseHandshakeRequest(const bcos::bytes& _value) +{ + std::string msgStr = ""; + msgStr.assign(_value.begin(), _value.end()); + + // parse HandshakeRequest + org::interconnection::v2::HandshakeRequest handShakeRequest; + handShakeRequest.ParseFromString(msgStr); + + auto supported_algos = handShakeRequest.supported_algos(0); + auto protocol_families = handShakeRequest.protocol_families(0); + auto protocol_family_params = handShakeRequest.protocol_family_params(0); + std::set curves; + std::set hashes; + int32_t item_num = -1; + if (protocol_family_params.Is()) + { + org::interconnection::v2::protocol::EccProtocolProposal eccProtocol; + protocol_family_params.UnpackTo(&eccProtocol); + auto ecc_suits = eccProtocol.ec_suits(); + for (auto const& suit : ecc_suits) + { + curves.insert(suit.curve()); + hashes.insert(suit.hash()); + } + } + + auto io_param = handShakeRequest.io_param(); + if (io_param.Is()) + { + org::interconnection::v2::algos::PsiDataIoProposal psiDataIo; + io_param.UnpackTo(&psiDataIo); + item_num = psiDataIo.item_num(); + } + + return std::make_shared(curves, hashes, protocol_families, item_num); +} + +bcos::bytes PSIConnMessageFactory::createHandshakeResponse( + HandShakeResponseVo::Ptr _handShakeResponseVo) +{ + std::string result; + + // create response header + org::interconnection::ResponseHeader* responseHeader = + new org::interconnection::ResponseHeader(); + responseHeader->set_error_code(_handShakeResponseVo->GetErrorCode()); + responseHeader->set_allocated_error_msg(new std::string(_handShakeResponseVo->GetErrorMsg())); + + // create handShakeResponse + org::interconnection::v2::HandshakeResponse handShakeResponse; + handShakeResponse.set_allocated_header(responseHeader); + handShakeResponse.set_algo(org::interconnection::v2::AlgoType::ALGO_TYPE_ECDH_PSI); + handShakeResponse.add_protocol_families( + org::interconnection::v2::ProtocolFamily::PROTOCOL_FAMILY_ECC); + + // create eccSuit + org::interconnection::v2::protocol::EcSuit* eccSuit = + new org::interconnection::v2::protocol::EcSuit(); + eccSuit->set_curve(_handShakeResponseVo->GetCurve()); + eccSuit->set_hash(_handShakeResponseVo->GetHash()); + eccSuit->set_hash2curve_strategy(3); + + // create EccProtocolResult + org::interconnection::v2::protocol::EccProtocolResult eccProtocol; + eccProtocol.set_version(1); + eccProtocol.set_point_octet_format(1); + eccProtocol.set_bit_length_after_truncated(-1); + eccProtocol.set_allocated_ec_suit(eccSuit); + + google::protobuf::Any protocol_families_params_any; + protocol_families_params_any.PackFrom(eccProtocol); + handShakeResponse.add_protocol_family_params()->CopyFrom(protocol_families_params_any); + + // create PsiDataIoResult + org::interconnection::v2::algos::PsiDataIoResult psiDataIo; + psiDataIo.set_version(1); + psiDataIo.set_result_to_rank(-1); + + + google::protobuf::Any* io_param_any = new google::protobuf::Any(); + io_param_any->PackFrom(psiDataIo); + handShakeResponse.set_allocated_io_param(io_param_any); + + handShakeResponse.SerializeToString(&result); + return bcos::bytes(result.begin(), result.end()); +} + +HandShakeResponseVo::Ptr PSIConnMessageFactory::parseHandshakeResponse(const bcos::bytes& _value) +{ + std::string msgStr = ""; + msgStr.assign(_value.begin(), _value.end()); + + org::interconnection::v2::HandshakeResponse handShakeResponse; + handShakeResponse.ParseFromString(msgStr); + + auto responseHeader = handShakeResponse.header(); + + auto errCode = responseHeader.error_code(); + auto errMsg = responseHeader.error_msg(); + auto algo = handShakeResponse.algo(); + auto protocol_families = handShakeResponse.protocol_families(0); + auto protocol_family_params = handShakeResponse.protocol_family_params(0); + int32_t curve; + int32_t hashtype; + if (protocol_family_params.Is()) + { + org::interconnection::v2::protocol::EccProtocolResult eccProtocol; + protocol_family_params.UnpackTo(&eccProtocol); + auto ecc_suit = eccProtocol.ec_suit(); + curve = ecc_suit.curve(); + hashtype = ecc_suit.hash(); + } + auto handRespVo = std::make_shared(); + handRespVo->SetCurve(curve); + handRespVo->SetHash(hashtype); + handRespVo->SetErrorCode(errCode); + handRespVo->SetErrorMessage(errMsg); + handRespVo->SetProtocolFamilies(protocol_families); + + return handRespVo; +} + +bcos::bytes PSIConnMessageFactory::createCipherExchange(CipherBatchVo::Ptr _cipherBatchVo) +{ + org::interconnection::v2::runtime::EcdhPsiCipherBatch ecdhPsiCipherBatch; + ecdhPsiCipherBatch.set_allocated_type(new std::string(_cipherBatchVo->type())); + ecdhPsiCipherBatch.set_is_last_batch(_cipherBatchVo->is_last_batch()); + ecdhPsiCipherBatch.set_batch_index(_cipherBatchVo->batch_index()); + ecdhPsiCipherBatch.set_count(_cipherBatchVo->count()); + ecdhPsiCipherBatch.set_allocated_ciphertext( + new std::string(encodeVectorBytesToString(_cipherBatchVo->cipherText()))); + + std::string result; + ecdhPsiCipherBatch.SerializeToString(&result); + + return bcos::bytes(result.begin(), result.end()); +} + +CipherBatchVo::Ptr PSIConnMessageFactory::parseCipherExchange(const bcos::bytes& _value) +{ + std::string msgStr = ""; + msgStr.assign(_value.begin(), _value.end()); + + org::interconnection::v2::runtime::EcdhPsiCipherBatch ecdhPsiCipherBatch; + ecdhPsiCipherBatch.ParseFromString(msgStr); + + auto cipherBatchVo = std::make_shared(); + cipherBatchVo->setType(ecdhPsiCipherBatch.type()); + cipherBatchVo->setBatchIndex(ecdhPsiCipherBatch.batch_index()); + cipherBatchVo->setIsLastBatch(ecdhPsiCipherBatch.is_last_batch()); + cipherBatchVo->setCount(ecdhPsiCipherBatch.count()); + cipherBatchVo->setCipherText( + decodeStringToVectorBytes(ecdhPsiCipherBatch.ciphertext(), ecdhPsiCipherBatch.count())); + return cipherBatchVo; +} + +bcos::bytes PSIConnMessageFactory::createPSIConnMessageRequest( + const bcos::bytes& _value, const std::string& _key) +{ + org::interconnection::link::PushRequest pushRequest; + pushRequest.set_sender_rank(1); + pushRequest.set_key(_key); + pushRequest.set_trans_type(org::interconnection::link::TransType::MONO); + pushRequest.set_value(_value.data(), _value.size()); + + bcos::bytes result; + result.resize(pushRequest.ByteSize()); + pushRequest.SerializeToArray(result.data(), pushRequest.ByteSize()); + + return result; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h new file mode 100644 index 00000000..58280282 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIConnMessageFactory.h + * @author: zachma + * @date 2023-8-23 + */ + +#pragma once +#include "../Common.h" +#include "PSIConnMessage.h" +#include "ppc-framework/Common.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::psi +{ +#define ECDH_CONN_PB_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-CONN-PSI-PB") +class PSIConnMessageFactory +{ +public: + using Ptr = std::shared_ptr; + PSIConnMessageFactory() = default; + virtual ~PSIConnMessageFactory() = default; + + bcos::bytes createHandshakeRequest(HandShakeRequestVo::Ptr handShakeRequestVo); + HandShakeRequestVo::Ptr parseHandshakeRequest(const bcos::bytes& _value); + + bcos::bytes createHandshakeResponse(HandShakeResponseVo::Ptr _handShakeResponseVo); + HandShakeResponseVo::Ptr parseHandshakeResponse(const bcos::bytes& _value); + + bcos::bytes createCipherExchange(CipherBatchVo::Ptr _cipherBatchVo); + CipherBatchVo::Ptr parseCipherExchange(const bcos::bytes& _value); + + bcos::bytes createPSIConnMessageRequest(const bcos::bytes& _value, const std::string& _key); + +protected: + std::string encodeVectorBytesToString(const std::vector& _cipherBytes) + { + std::string _temp = ""; + for (auto const& _cipher_byte : _cipherBytes) + { + _temp += std::string(_cipher_byte.begin(), _cipher_byte.end()); + } + return _temp; + } + + std::vector decodeStringToVectorBytes(const std::string& _cipherStr, int count) + { + std::vector result; + if (count == 0 || _cipherStr == "") + { + return result; + } + auto _size = _cipherStr.length(); + auto batch = _size / count; + int start = 0; + for (int i = 0; i < count; i++) + { + std::string temp = _cipherStr.substr(start, batch); + bcos::bytes temp_bytes(temp.begin(), temp.end()); + result.push_back(temp_bytes); + start += batch; + } + return result; + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt b/cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt new file mode 100644 index 00000000..a8d547c2 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt @@ -0,0 +1,31 @@ +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-psi/ecdh-multi-psi/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") + +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () + +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${ECDH_MULTI_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST}) +target_include_directories(${ECDH_MULTI_PSI_TARGET} PUBLIC + $) + +target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc) \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/Common.h b/cpp/ppc-psi/src/ecdh-multi-psi/Common.h new file mode 100644 index 00000000..957579c6 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/Common.h @@ -0,0 +1,19 @@ +#pragma once +#include "ppc-framework/Common.h" +#include + +namespace ppc::psi +{ +#define ECDH_MULTI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-MULTI-PSI") +DERIVE_PPC_EXCEPTION(ECDHMULTIException); +enum class EcdhMultiPSIMessageType : uint8_t +{ + GENERATE_RANDOM_TO_PARTNER = 0x01, + SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR = 0x02, + SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER = 0x03, + SEND_ENCRYPTED_SET_TO_CALCULATOR = 0x04, + SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR = 0x05, + RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER = 0x06, + SYNC_FINAL_RESULT_TO_ALL = 0x07, +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h new file mode 100644 index 00000000..0dd01f07 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h @@ -0,0 +1,247 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhCache.h + * @author: zachma + * @date 2023-5-28 + */ +#pragma once +#include "Common.h" +#include "ppc-psi/src/psi-framework/TaskState.h" +#include +#include + +namespace ppc::psi +{ +/// the master data-cache +class MasterCipherDataCache +{ +public: + using Ptr = std::shared_ptr; + MasterCipherDataCache() {} + virtual ~MasterCipherDataCache() + { + m_masterFinalIntersectionCipherData.clear(); + m_masterTaskPeersFinishedList.clear(); + m_masterCipherDataFromCalculator.clear(); + m_masterCipherDataFromPartner.clear(); + m_CipherDataFromCalculatorSubSeq.clear(); + m_CipherDataFromPartnerSubSeq.clear(); + std::map().swap(m_masterFinalIntersectionCipherData); + std::set().swap(m_masterTaskPeersFinishedList); + std::map().swap(m_masterCipherDataFromCalculator); + std::map>().swap(m_masterCipherDataFromPartner); + std::set().swap(m_CipherDataFromCalculatorSubSeq); + std::map>().swap(m_CipherDataFromPartnerSubSeq); + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("the master cipher datacache destroyed ") + << LOG_KV("taskID", m_taskID); + } + + void appendMasterCipherDataFromCalculator(std::string _peerId, + std::map&& _cipherData, uint32_t seq, uint32_t needSendTimes) + { + try + { + bcos::WriteGuard lock(x_peerTasks); + m_masterCipherDataFromCalculator.insert(_cipherData.begin(), _cipherData.end()); + m_CipherDataFromCalculatorSubSeq.insert(seq); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part1-C:Master Receive H(X)*A Size: ", m_CipherDataFromCalculatorSubSeq.size()); + if (m_CipherDataFromCalculatorSubSeq.size() == needSendTimes) + { + m_masterTaskPeersFinishedList.insert(_peerId); + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(ERROR) << LOG_DESC("appendMasterCipherDataFromCalculator Exception:") + << boost::diagnostic_information(e); + } + } + + void appendMasterCipherDataFromPartner(std::string _peerId, + std::vector&& _cipherData, uint32_t seq, uint32_t needSendTimes) + { + try + { + bcos::WriteGuard lock(x_peerTasks); + m_masterCipherDataFromPartner[_peerId].insert( + m_masterCipherDataFromPartner[_peerId].end(), _cipherData.begin(), + _cipherData.end()); + + m_CipherDataFromPartnerSubSeq[_peerId].insert(seq); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part1-P:Master Receive H(Y)*A Size: ", m_CipherDataFromPartnerSubSeq.size()); + if (m_CipherDataFromPartnerSubSeq[_peerId].size() == needSendTimes) + { + m_masterTaskPeersFinishedList.insert(_peerId); + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(ERROR) << LOG_DESC("appendMasterCipherDataFromPartner Exception:") + << boost::diagnostic_information(e); + } + } + + std::set const& masterTaskPeersFinishedList() + { + return m_masterTaskPeersFinishedList; + } + + std::map const& masterFinalIntersectionCipherData() + { + return m_masterFinalIntersectionCipherData; + } + + void tryToGetCipherDataIntersection() + { + // partner intersection _masterCipherDataIntersectionFromPartner + std::vector _masterCipherDataIntersectionFromPartnerVector; + std::map cnt; + uint32_t n = m_masterCipherDataFromPartner.size(); + for (auto& data : m_masterCipherDataFromPartner) + { + std::set _temp(data.second.begin(), data.second.end()); + for (auto& v : _temp) + { + cnt[v]++; + if (cnt[v] == n) + { + _masterCipherDataIntersectionFromPartnerVector.push_back(v); + } + } + } + + // calculator intersection partners m_masterFinalIntersectionCipherData + m_masterFinalIntersectionCipherData.clear(); + std::map temp_out; + for (auto& data : m_masterCipherDataFromCalculator) + { + // temp_out.insert(std::make_pair(data.second, data.first)); + temp_out.emplace(std::make_pair(data.second, data.first)); + } + + for (const auto& data : _masterCipherDataIntersectionFromPartnerVector) + { + auto it = temp_out.find(data); + if (it != temp_out.end()) + { + // m_masterFinalIntersectionCipherData.insert(std::make_pair(it->second, + // it->first)); + m_masterFinalIntersectionCipherData.emplace(std::make_pair(it->second, it->first)); + } + } + } + +private: + std::string m_taskID; + + // store the cipher-data of the master + std::map m_masterFinalIntersectionCipherData; + std::set m_masterTaskPeersFinishedList; + std::map m_masterCipherDataFromCalculator; + std::map> m_masterCipherDataFromPartner; + // std::vector m_masterCipherDataIntersectionFromPartner; + std::set m_CipherDataFromCalculatorSubSeq; + std::map> m_CipherDataFromPartnerSubSeq; + bcos::SharedMutex x_peerTasks; +}; + +class CalculatorCipherDataCache +{ +public: + using Ptr = std::shared_ptr; + CalculatorCipherDataCache() {} + virtual ~CalculatorCipherDataCache() + { + m_CipherDataFromCalculatorSubSeq.clear(); + m_calculatorIntersectionSubSeq.clear(); + m_calculatorCipherDataSubSeq.clear(); + m_calculatorCipherData.clear(); + m_calculatorIntersectionCipherDataMap.clear(); + m_calculatorIntersectionCipherDataFinalMap.clear(); + std::set().swap(m_CipherDataFromCalculatorSubSeq); + std::set().swap(m_calculatorIntersectionSubSeq); + std::set().swap(m_calculatorCipherDataSubSeq); + std::vector().swap(m_calculatorCipherData); + std::map().swap(m_calculatorIntersectionCipherDataMap); + std::map().swap(m_calculatorIntersectionCipherDataFinalMap); + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("the calculator cipher datacache destroyed") + << LOG_KV("taskID", m_taskID); + } + + void tryToGetCipherDataIntersection() + { + m_calculatorIntersectionCipherDataFinalMap.clear(); + std::map temp_out; + for (auto& data : m_calculatorIntersectionCipherDataMap) + { + // temp_out.insert(std::make_pair(data.second, data.first)); + temp_out.emplace(std::make_pair(data.second, data.first)); + } + + for (const auto& data : m_calculatorCipherData) + { + auto it = temp_out.find(data); + if (it != temp_out.end()) + { + m_calculatorIntersectionCipherDataFinalMap.emplace( + std::make_pair(it->second, it->first)); + } + } + } + + bool setCalculatorCipherData( + std::vector&& _cipherData, uint32_t seq, uint32_t needSendTimes) + { + bcos::WriteGuard lock(x_setCalculatorCipherData); + m_calculatorCipherData.insert( + m_calculatorCipherData.end(), _cipherData.begin(), _cipherData.end()); + m_calculatorCipherDataSubSeq.insert(seq); + return m_calculatorCipherDataSubSeq.size() == needSendTimes; + } + + bool setCalculatorIntersectionCipherDataMap( + std::map&& _cipherData, uint32_t seq, uint32_t needSendTimes) + { + bcos::WriteGuard lock(x_setCalculatorIntersectionCipherData); + m_calculatorIntersectionCipherDataMap.insert(_cipherData.begin(), _cipherData.end()); + m_calculatorIntersectionSubSeq.insert(seq); + return m_calculatorIntersectionSubSeq.size() == needSendTimes; + } + + std::map const& calculatorIntersectionCipherDataFinalMap() + { + return m_calculatorIntersectionCipherDataFinalMap; + } + +private: + std::string m_taskID; + + // store the cipher-data of the calculator + std::set m_CipherDataFromCalculatorSubSeq; + std::set m_calculatorIntersectionSubSeq; + std::set m_calculatorCipherDataSubSeq; + std::vector m_calculatorCipherData; + std::map m_calculatorIntersectionCipherDataMap; + std::map m_calculatorIntersectionCipherDataFinalMap; + + mutable boost::shared_mutex x_setCalculatorCipherData; + mutable boost::shared_mutex x_setCalculatorIntersectionCipherData; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h new file mode 100644 index 00000000..3f5b1bd7 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h @@ -0,0 +1,51 @@ +#pragma once +#include "EcdhMultiPSIMessageFactory.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/PSIConfig.h" +#include +#include +#include + +namespace ppc::psi +{ +class EcdhMultiPSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + EcdhMultiPSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front, + ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _dataBatchSize, + int _holdingMessageMinutes, EcdhMultiPSIMessageFactory::Ptr const& _psiMsgFactory, + const front::PPCMessageFactory::Ptr& _ppcMsgFactory = + std::make_shared()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI, _selfPartyID, std::move(_front), + _ppcMsgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes), + m_threadPool(std::move(_threadPool)), + m_cryptoBox(std::move(_cryptoBox)), + m_psiMsgFactory(std::move(_psiMsgFactory)), + m_dataBatchSize(_dataBatchSize) + {} + + virtual ~EcdhMultiPSIConfig() { + if (m_threadPool) + { + m_threadPool->stop(); + } + }; + + EcdhMultiPSIMessageFactory::Ptr const& psiMsgFactory() const { return m_psiMsgFactory; } + crypto::Hash::Ptr const& hash() const { return m_cryptoBox->hashImpl(); } + crypto::EccCrypto::Ptr const& eccCrypto() const { return m_cryptoBox->eccCrypto(); } + crypto::CryptoBox::Ptr const& cryptoBox() const { return m_cryptoBox; } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + uint32_t dataBatchSize() const { return m_dataBatchSize; } + +private: + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + bcos::ThreadPool::Ptr m_threadPool; + EcdhMultiPSIMessageFactory::Ptr m_psiMsgFactory; + uint32_t m_dataBatchSize = 10000; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h new file mode 100644 index 00000000..f6d0090b --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h @@ -0,0 +1,31 @@ +#pragma once +#include "EcdhMultiPSIImpl.h" +#include "EcdhMultiPSIMessageFactory.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::psi +{ +class EcdhMultiPSIFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhMultiPSIFactory() = default; + virtual ~EcdhMultiPSIFactory() = default; + + virtual EcdhMultiPSIImpl::Ptr createEcdhMultiPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader) + { + auto psiMsgFactory = std::make_shared(); + auto const& ecdhParam = _ppcConfig->ecdhMultiPSIConfig(); + auto _selfParty = _ppcConfig->agencyID(); + int _holdingMessageMinutes = _ppcConfig->holdingMessageMinutes(); + auto config = std::make_shared(_selfParty, std::move(_front), + std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), + ecdhParam.dataBatchSize, _holdingMessageMinutes, psiMsgFactory); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp new file mode 100644 index 00000000..d881405d --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp @@ -0,0 +1,379 @@ +#include "EcdhMultiPSIImpl.h" +#include "Common.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace bcos; + + +EcdhMultiPSIImpl::EcdhMultiPSIImpl(const EcdhMultiPSIConfig::Ptr& _config, unsigned _idleTimeMs) + : m_config(std::move(_config)), + m_msgQueue(std::make_shared()), + TaskGuarder(_config, PSIAlgorithmType::ECDH_PSI_MULTI, "ECDH-MULTI-PSI-Timer") +{} + +void EcdhMultiPSIImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) +{ + try + { + m_msgQueue->push(_msg); + wakeupWorker(); + } + catch (std::exception const& e) + { + ECDH_MULTI_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + +void EcdhMultiPSIImpl::handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg) +{ + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, _msg]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + switch (_msg->packetType()) + { + case (uint32_t)EcdhMultiPSIMessageType::GENERATE_RANDOM_TO_PARTNER: + { + // calculator -> partner (A) + psi->onComputeAndEncryptSet(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR: + { + // calculator -> master H(X)*A + psi->onHandlerIntersectEncryptSetFromCalculator(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER: + { + // patner -> master H(Y)*A + psi->onHandlerIntersectEncryptSetFromPartner(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR: + { + psi->onHandlerIntersectEncryptSetToCalculator(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR: + { + psi->onHandlerEncryptSetToCalculator(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType:: + RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER: + { + psi->onHandlerEncryptIntersectionSetFromCalculatorToMaster(_msg); + break; + } + case (uint32_t)EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL: + { + psi->onHandlerSyncFinalResultToAllPeer(_msg); + break; + } + default: + { + ECDH_MULTI_LOG(WARNING) + << LOG_DESC("Unsupported packetType ") << (int)_msg->packetType(); + break; + } + } + } + catch (std::exception const& e) + { + ECDH_MULTI_LOG(WARNING) + << LOG_DESC("handlePSIMsg exception") << LOG_KV("packetType", _msg->packetType()) + << printPSIMessage(_msg) << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + + +void EcdhMultiPSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished) +{ + auto taskState = + m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), false, m_config); + taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->noticePeerToFinish(_task); + }); + addPendingTask(taskState); + + try + { + auto dataResource = _task->selfParty()->dataResource(); + auto reader = loadReader(_task->id(), dataResource, DataSchema::Bytes); + taskState->setReader(reader, -1); + auto role = _task->selfParty()->partyIndex(); + auto receivers = _task->getReceiverLists(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Start a asyncRunTask ") << LOG_KV("taskID", _task->id()) + << LOG_KV("roleId", role); + if (role == uint16_t(PartiesType::Calculator)) + { + auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists); + taskState->setWriter(writer); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator do the Task") + << LOG_KV("taskID", _task->id()); + auto calculator = std::make_shared(m_config, taskState); + calculator->asyncStartRunTask(_task); + addCalculator(std::move(calculator)); + } + else if (role == uint16_t(PartiesType::Partner)) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner do the Task") + << LOG_KV("taskID", _task->id()); + if (_task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), + m_config->selfParty()) != receivers.end()) + { + auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists); + taskState->setWriter(writer); + } + auto partner = std::make_shared(m_config, taskState); + partner->asyncStartRunTask(_task); + addPartner(std::move(partner)); + } + else if (role == uint16_t(PartiesType::Master)) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Master do the Task") << LOG_KV("taskID", _task->id()); + if (_task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), + m_config->selfParty()) != receivers.end()) + { + auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists); + taskState->setWriter(writer); + } + auto master = std::make_shared(m_config, taskState); + master->asyncStartRunTask(_task); + addMaster(std::move(master)); + } + else + { + BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment( + "The party index of the ecdh-multi-psi must be calculator(0) " + "or partner(1) or master(2)!")); + } + + // notify the taskInfo to the front + m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + } + catch (bcos::Error const& e) + { + ECDH_MULTI_LOG(ERROR) << LOG_DESC("asyncRunTask exception") << printTaskInfo(_task) + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + onSelfError( + _task->id(), std::make_shared(e.errorCode(), e.errorMessage()), true); + } + catch (std::exception& e) + { + auto error = BCOS_ERROR_PTR((int)TaskParamsError, boost::diagnostic_information(e)); + onSelfError(_task->id(), error, true); + } +} + +void EcdhMultiPSIImpl::start() +{ + startWorking(); + startPingTimer(); +} + +void EcdhMultiPSIImpl::stop() +{ + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } + stopPingTimer(); +} + +void EcdhMultiPSIImpl::checkFinishedTask() +{ + std::set finishedTask; + { + bcos::WriteGuard l(x_pendingTasks); + if (m_pendingTasks.empty()) + { + return; + } + + for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();) + { + auto task = it->second; + if (task->finished()) + { + finishedTask.insert(it->first); + } + it++; + } + } + for (auto& taskID : finishedTask) + { + removeCalculator(taskID); + removeMaster(taskID); + removePartner(taskID); + removePendingTask(taskID); + } +} + +void EcdhMultiPSIImpl::onReceivedErrorNotification(const std::string& _taskID) +{ + // finish the task while the peer is failed + auto taskState = findPendingTask(_taskID); + if (taskState) + { + taskState->onPeerNotifyFinish(); + + wakeupWorker(); + } +} + +void EcdhMultiPSIImpl::onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +{ + auto taskState = findPendingTask(_taskID); + if (!taskState) + { + return; + } + + ECDH_MULTI_LOG(ERROR) << LOG_DESC("onSelfError") << LOG_KV("task", _taskID) + << LOG_KV("exception", _error->errorMessage()) + << LOG_KV("noticePeer", _noticePeer); + + auto result = std::make_shared(taskState->task()->id()); + result->setError(std::move(_error)); + taskState->onTaskFinished(result, _noticePeer); + + wakeupWorker(); +} + +void EcdhMultiPSIImpl::executeWorker() +{ + checkFinishedTask(); + auto _msg = m_msgQueue->tryPop(c_popWaitMs); + if (_msg.first) + { + auto pop_msg = _msg.second; + if (pop_msg->messageType() == uint8_t(CommonMessageType::ErrorNotification)) + { + onReceivedErrorNotification(pop_msg->taskID()); + return; + } + else if (pop_msg->messageType() == uint8_t(CommonMessageType::PingPeer)) + { + return; + } + + // decode the psi message + auto payLoad = pop_msg->data(); + auto psiMsg = m_config->psiMsgFactory()->decodePSIMessage( + bcos::bytesConstRef(payLoad->data(), payLoad->size())); + psiMsg->setFrom(pop_msg->sender()); + psiMsg->setTaskID(pop_msg->taskID()); + psiMsg->setSeq(pop_msg->seq()); + psiMsg->setUUID(pop_msg->uuid()); + ECDH_MULTI_LOG(TRACE) << LOG_DESC("onReceiveMessage") << printPSIMessage(psiMsg) + << LOG_KV("uuid", psiMsg->uuid()); + handlerPSIReceiveMessage(psiMsg); + return; + } + waitSignal(); +} + +void EcdhMultiPSIImpl::onComputeAndEncryptSet(PSIMessageInterface::Ptr _msg) +{ + auto partner = findPartner(_msg->taskID()); + if (partner) + { + if (_msg->takeData().size() == 1) + { + auto msgData = _msg->getData(0); + partner->oncomputeAndEncryptSet( + std::make_shared(bcos::bytes(msgData.begin(), msgData.end()))); + } + } +} + +void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg) +{ + auto master = findMaster(_msg->taskID()); + if (master) + { + master->onHandlerIntersectEncryptSetFromCalculator(_msg); + } +} + +void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg) +{ + auto master = findMaster(_msg->taskID()); + if (master) + { + master->onHandlerIntersectEncryptSetFromPartner(_msg); + } +} + +void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg) +{ + auto calculator = findCalculator(_msg->taskID()); + if (calculator) + { + calculator->onHandlerIntersectEncryptSetToCalculator(_msg); + } +} + +void EcdhMultiPSIImpl::onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg) +{ + auto calculator = findCalculator(_msg->taskID()); + if (calculator) + { + calculator->onHandlerEncryptSetToCalculator(_msg); + } +} + +void EcdhMultiPSIImpl::onHandlerEncryptIntersectionSetFromCalculatorToMaster( + PSIMessageInterface::Ptr _msg) +{ + auto master = findMaster(_msg->taskID()); + if (master) + { + master->onHandlerEncryptIntersectionSetFromCalculatorToMaster(_msg); + } +} + +void EcdhMultiPSIImpl::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg) +{ + auto master = findMaster(_msg->taskID()); + if (master) + { + master->onHandlerSyncFinalResultToAllPeer(_msg); + return; + } + + auto partner = findPartner(_msg->taskID()); + if (partner) + { + partner->onHandlerSyncFinalResultToAllPeer(_msg); + return; + } +} diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h new file mode 100644 index 00000000..91f898a5 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h @@ -0,0 +1,167 @@ +#pragma once +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include "Common.h" +#include "EcdhMultiPSIConfig.h" +#include "core/EcdhMultiPSICalculator.h" +#include "core/EcdhMultiPSIMaster.h" +#include "core/EcdhMultiPSIPartner.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-protocol/src/PPCMessage.h" +#include +#include +#include + +#include +#include + + +namespace ppc::psi +{ +class EcdhMultiPSIImpl : public std::enable_shared_from_this, + public ppc::task::TaskFrameworkInterface, + public TaskGuarder, + public bcos::Worker +{ +public: + using Ptr = std::shared_ptr; + using EcdhMultiPSIMsgQueue = bcos::ConcurrentQueue; + using EcdhMultiPSIMsgQueuePtr = std::shared_ptr; + + EcdhMultiPSIImpl(const EcdhMultiPSIConfig::Ptr& _config, unsigned _idleTimeMs = 0); + + ~EcdhMultiPSIImpl() override = default; + + void asyncRunTask(ppc::protocol::Task::ConstPtr _task, + ppc::task::TaskResponseCallback&& _onTaskFinished) override; + + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; + + void start() override; + void stop() override; + + void checkFinishedTask(); + void onReceivedErrorNotification(const std::string& _taskID) override; + void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override; + void executeWorker() override; + + +protected: + bool m_enableOutputExists = false; + virtual void onComputeAndEncryptSet(PSIMessageInterface::Ptr _msg); + virtual void onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg); + virtual void handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg); + virtual void onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg); + virtual void onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg); + virtual void onHandlerEncryptIntersectionSetFromCalculatorToMaster( + PSIMessageInterface::Ptr _msg); + virtual void onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg); + virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg); + + EcdhMultiPSICalculator::Ptr findCalculator(const std::string& _taskID) + { + bcos::ReadGuard l(x_calculators); + auto it = m_calculators.find(_taskID); + if (it != m_calculators.end()) + { + return it->second; + } + return nullptr; + } + + void addCalculator(EcdhMultiPSICalculator::Ptr _calculator) + { + bcos::WriteGuard l(x_calculators); + m_calculators[_calculator->taskID()] = _calculator; + } + + void removeCalculator(const std::string& _taskID) + { + bcos::WriteGuard l(x_calculators); + auto it = m_calculators.find(_taskID); + if (it != m_calculators.end()) + { + m_calculators.erase(it); + } + } + + EcdhMultiPSIPartner::Ptr findPartner(const std::string& _taskID) + { + bcos::ReadGuard l(x_partners); + auto it = m_partners.find(_taskID); + if (it != m_partners.end()) + { + return it->second; + } + return nullptr; + } + + void addPartner(EcdhMultiPSIPartner::Ptr _partner) + { + bcos::WriteGuard l(x_partners); + m_partners[_partner->taskID()] = _partner; + } + + void removePartner(const std::string& _taskID) + { + bcos::WriteGuard l(x_partners); + auto it = m_partners.find(_taskID); + if (it != m_partners.end()) + { + m_partners.erase(it); + } + } + + EcdhMultiPSIMaster::Ptr findMaster(const std::string& _taskID) + { + bcos::ReadGuard l(x_masters); + auto it = m_masters.find(_taskID); + if (it != m_masters.end()) + { + return it->second; + } + return nullptr; + } + + void addMaster(EcdhMultiPSIMaster::Ptr _master) + { + bcos::WriteGuard l(x_masters); + m_masters[_master->taskID()] = _master; + } + + void removeMaster(const std::string& _taskID) + { + bcos::WriteGuard l(x_partners); + auto it = m_masters.find(_taskID); + if (it != m_masters.end()) + { + m_masters.erase(it); + } + } + +private: + void waitSignal() + { + boost::unique_lock l(x_signal); + m_signal.wait_for(l, boost::chrono::milliseconds(5)); + } + + void wakeupWorker() { m_signal.notify_all(); } + + const int c_popWaitMs = 5; + EcdhMultiPSIConfig::Ptr m_config; + TaskState::Ptr m_taskMultiState; + EcdhMultiPSIMsgQueuePtr m_msgQueue; + boost::condition_variable m_signal; + boost::mutex x_signal; + + std::unordered_map m_calculators; + mutable bcos::SharedMutex x_calculators; + + std::unordered_map m_partners; + mutable bcos::SharedMutex x_partners; + + std::unordered_map m_masters; + mutable bcos::SharedMutex x_masters; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h new file mode 100644 index 00000000..a51bd0b3 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h @@ -0,0 +1,35 @@ +#pragma once +#include "ppc-psi/src/psi-framework/protocol/PSIMessage.h" +#include "Common.h" +namespace ppc::psi +{ +class EcdhMultiPSIMessageFactory : public PSIMessageFactoryImpl +{ +public: + using Ptr = std::shared_ptr; + EcdhMultiPSIMessageFactory() = default; + ~EcdhMultiPSIMessageFactory() override = default; + + PSIMessageInterface::Ptr decodePSIMessage(bcos::bytesConstRef _data) override + { + auto inner = [inner = ppctars::PSIMessage()]() mutable { return &inner; }; + tars::TarsInputStream input; + input.setBuffer((const char*)_data.data(), _data.size()); + inner()->readFrom(input); + switch (inner()->packetType) + { + case (uint32_t)EcdhMultiPSIMessageType::GENERATE_RANDOM_TO_PARTNER: + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR: + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER: + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR: + case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR: + case (uint32_t) + EcdhMultiPSIMessageType::RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER: + case (uint32_t)EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL: + return std::make_shared(inner); + default: + return decodePSIBaseMessage(inner); + } + } +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp new file mode 100644 index 00000000..3427edde --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp @@ -0,0 +1,433 @@ +#include "EcdhMultiPSICalculator.h" +#include "ppc-psi/src/ecdh-multi-psi/Common.h" +#include +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace bcos; + +EcdhMultiPSICalculator::EcdhMultiPSICalculator( + EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState) + : m_config(std::move(_config)), m_taskState(std::move(_taskState)) +{ + auto task = m_taskState->task(); + auto receivers = task->getReceiverLists(); + m_taskID = task->id(); + m_calculatorCipherDataCache = std::make_shared(); + m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), + m_config->selfParty()) != receivers.end()); +} + +void EcdhMultiPSICalculator::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task) +{ + InitAsyncTask(_task); + auto randA = generateRandomA(_task->id()); + m_randomA = randA; + m_config->threadPool()->enqueue([self = weak_from_this(), _task, randA]() { + auto calculator = self.lock(); + if (!calculator) + { + return; + } + ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator asyncStartRunTask as calculator"); + calculator->computeAndEncryptSet(_task->id(), randA); + }); +} + +// PART1: Calculator -> Partners (A) +bcos::bytes EcdhMultiPSICalculator::generateRandomA(std::string _taskID) +{ + ECDH_MULTI_LOG(INFO) << LOG_KV("PART1: Calculator Start New Task :", _taskID); + auto A = m_config->eccCrypto()->generateRandomScalar(); + // send to all partners + auto message = m_config->psiMsgFactory()->createPSIMessage( + uint32_t(EcdhMultiPSIMessageType::GENERATE_RANDOM_TO_PARTNER)); + message->setData(std::vector{A}); + message->setDataBatchCount(A.size()); + message->setFrom(m_taskState->task()->selfParty()->id()); + for (auto& partner : m_partnerParties) + { + ECDH_MULTI_LOG(INFO) << LOG_KV("PART1: Calculator generateRandomA to ", partner.first) + << LOG_KV(" Random A ", *toHexString(A)); + m_config->generateAndSendPPCMessage( + partner.first, _taskID, message, + [self = weak_from_this(), partner](bcos::Error::Ptr&& _error) { + if (!_error) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV("PART1: Calculator generateRandomA success to ", partner.first); + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + 0); + } + return A; +} + +// PART2: Calculator -> Master H(X)*A +void EcdhMultiPSICalculator::computeAndEncryptSet(std::string _taskID, bcos::bytes _randA) +{ + ECDH_MULTI_LOG(INFO) << LOG_KV( + "PART2:Calculator send to Master Use RandomA: ", *toHexString(_randA)); + auto startT = utcSteadyTime(); + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty")); + } + auto inputSize = m_originInputs->size(); + auto batchSize = m_config->dataBatchSize(); + auto needSendTimes = 0; + + // send counts + if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + ECDH_MULTI_LOG(INFO) << LOG_KV("PART2:Calculator Send Data Batch Times: ", needSendTimes); + try + { + auto hash = m_config->hash(); + uint32_t readStart = 0, readCount = 0; + tbb::concurrent_map encryptedHashMap; + tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto data = m_originInputs->getBytes(i); + auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_config->eccCrypto()->hashToCurve(hashData); + auto hashSet = m_config->eccCrypto()->ecMultiply(point, _randA); + // encryptedHashMap.insert(std::make_pair(i, hashSet)); + encryptedHashMap.emplace(std::make_pair(i, hashSet)); + } + }); + + while (readCount < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + std::map cHashMap; + ConcurrentSTLToCommon(encryptedHashMap, readStart, readCount, cHashMap); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "PART2:Calculator compute the H(X)*A encryptedHashMap success size: ", + cHashMap.size()); + for (auto& master : m_masterParties) + { + auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t( + EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR)); + message->setDataMap(std::move(cHashMap)); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + master.first, _taskID, message, + [self = weak_from_this(), master](bcos::Error::Ptr&& _error) { + if (!_error) + { + ECDH_MULTI_LOG(INFO) << LOG_KV( + "PART2:Send the EncryptSet success to Master: ", master.first); + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in computeAndEncryptSet:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + + +void EcdhMultiPSICalculator::InitAsyncTask(ppc::protocol::Task::ConstPtr _task) +{ + // Init all Roles from all Peers + auto peerParties = _task->getAllPeerParties(); + for (auto& party : peerParties) + { + auto partyId = party.first; + auto partySource = party.second; + if (partySource->partyIndex() == uint16_t(PartiesType::Calculator)) + { + m_calculatorParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Partner)) + { + m_partnerParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Master)) + { + m_masterParties[partyId] = partySource; + } + } +} + +// Part3: Calculator store Intersection_XY^b <- Master (response) +void EcdhMultiPSICalculator::onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg) +{ + auto encryptedMap = _msg->takeDataMap(); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part3: Calculator Receive Intersection_XY^b onHandlerIntersectEncryptSetToCalculator " + "Received Dataset size: ", + encryptedMap.size()); + try + { + auto seq = _msg->seq(); + auto needSendTimes = _msg->dataBatchCount(); + bool finished = m_calculatorCipherDataCache->setCalculatorIntersectionCipherDataMap( + std::move(encryptedMap), seq, needSendTimes); + if (finished == false) + { + return; + } + for (auto& master : m_masterParties) + { + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part3: onHandlerIntersectEncryptSetToCalculator Send the Response to Master: ", + master.first); + auto message = + m_config->psiMsgFactory()->createPSIMessage(uint32_t(EcdhMultiPSIMessageType:: + RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER)); + message->setFrom(m_taskState->task()->selfParty()->id()); + m_config->generateAndSendPPCMessage( + master.first, m_taskID, message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + 0); + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetToCalculator:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + +// Part4 : Intersection_XY^b ∩ H(Z)^b^a +void EcdhMultiPSICalculator::onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg) +{ + auto encryptedSet = _msg->takeData(); + auto inputSize = encryptedSet.size(); + ECDH_MULTI_LOG(INFO) << LOG_KV( + " Part4: Master onHandlerEncryptSetToCalculator Received H(Z)*B Dataset size: ", inputSize); + std::vector encryptedHashSet; + encryptedHashSet.reserve(inputSize); + encryptedHashSet.resize(inputSize); + try + { + tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto data = encryptedSet.at(i); + if (data.data()) + { + auto hashSet = m_config->eccCrypto()->ecMultiply(data, m_randomA); + encryptedHashSet[i] = hashSet; + } + } + }); + + auto needTimes = _msg->dataBatchCount(); + auto seq = _msg->seq(); + bool finished = m_calculatorCipherDataCache->setCalculatorCipherData( + std::move(encryptedHashSet), seq, needTimes); + if (finished == false) + { + return; + } + m_calculatorCipherDataCache->tryToGetCipherDataIntersection(); + auto cipherDataResult = + m_calculatorCipherDataCache->calculatorIntersectionCipherDataFinalMap(); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part4: Master onHandlerEncryptSetToCalculator " + "Intersection_XY^b ∩ H(Z)^b^a Dataset " + "size: ", + cipherDataResult.size()); + + if (!m_originInputs) + { + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION( + ECDHMULTIException() << bcos::errinfo_comment("data is empty")); + } + } + + m_finalResults.clear(); + if (cipherDataResult.size() > 0) + { + for (auto& _res : cipherDataResult) + { + m_finalResults.push_back(m_originInputs->getBytes(_res.first)); + } + } + + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part4: onHandlerEncryptSetToCalculator Store " + "Intersection_XY^b ∩ H(Z)^b^a Success " + "Dataset size: ", + cipherDataResult.size()); + // sync Result or status to All peers + syncResultToAllPeers(); + m_taskState->storePSIResult(m_config->dataResourceLoader(), m_finalResults); + + // return the rpc + m_taskState->setFinished(true); + m_taskState->onTaskFinished(); + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerEncryptSetToCalculator:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + + +void EcdhMultiPSICalculator::syncResultToAllPeers() +{ + auto all_peers = m_taskState->task()->getAllPeerParties(); + ECDH_MULTI_LOG(INFO) << LOG_KV("Calculator: Calculator isSyncedResult:", m_syncResult); + if (!m_syncResult) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator:No Need To SyncResultToAllPeers"); + for (auto& _peer : all_peers) + { + auto message = m_config->psiMsgFactory()->createPSIMessage( + uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL)); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setVersion(-1); + m_config->generateAndSendPPCMessage( + _peer.first, m_taskID, message, + [self = weak_from_this(), _peer](bcos::Error::Ptr&& _error) { + if (!_error) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV("Calculator:Calculator sync no Need Sync To Peer Success:", + _peer.first); + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + 0); + } + return; + } + auto batchSize = m_config->dataBatchSize(); + auto inputSize = m_finalResults.size(); + auto needSendTimes = 0; + // send counts + if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Final:Calculator Send Final Intersection Batch Times: ", needSendTimes); + uint32_t readStart = 0, readCount = 0; + while (readCount < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + + std::vector chunk_final_result; + splitVector(m_finalResults, readStart, readCount, chunk_final_result); + + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Final:Calculator Send Final Intersection Chunk Size: ", chunk_final_result.size()); + for (auto& _peer : all_peers) + { + auto message = m_config->psiMsgFactory()->createPSIMessage( + uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL)); + message->setData(chunk_final_result); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setVersion(0); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + _peer.first, m_taskID, message, + [self = weak_from_this(), _peer](bcos::Error::Ptr&& _error) { + if (!_error) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV("Final:Calculator sync result To Peer Success:", _peer.first); + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } +} + +void EcdhMultiPSICalculator::onTaskError(std::string&& _error) +{ + auto result = std::make_shared(m_taskState->task()->id()); + auto err = std::make_shared(-12222, _error); + result->setError(std::move(err)); + m_taskState->onTaskFinished(result, true); +} diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h new file mode 100644 index 00000000..ea2459ad --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h @@ -0,0 +1,101 @@ +#pragma once +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" +#include "ppc-psi/src/psi-framework/TaskState.h" + +namespace ppc::psi +{ +class EcdhMultiPSICalculator : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + EcdhMultiPSICalculator(EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState); + + virtual ~EcdhMultiPSICalculator() + { + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("the calculator destroyed") << LOG_KV("taskID", m_taskID); + } + + virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task); + virtual void onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg); + virtual void onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg); + + const std::string& taskID() const { return m_taskID; } + +protected: + virtual bcos::bytes generateRandomA(std::string _taskID); + virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task); + virtual void computeAndEncryptSet(std::string _taskID, bcos::bytes _randA); + virtual void syncResultToAllPeers(); + virtual void onTaskError(std::string&& _error); + void ConcurrentSTLToCommon( + tbb::concurrent_map _cMap, std::map& result) + { + ConcurrentSTLToCommon(_cMap, 0, _cMap.size(), result); + }; + + void ConcurrentSTLToCommon(tbb::concurrent_map _cMap, + uint32_t _startIndex, uint32_t _endIndex, std::map& result) + { + std::mutex mutex; + tbb::concurrent_map::const_iterator iter; + uint32_t index = 0; + for (iter = _cMap.begin(); iter != _cMap.end(); iter++) + { + std::lock_guard lock(mutex); + if (index < _startIndex) + { + index++; + continue; + } + else if (index >= _endIndex) + { + break; + } + result.emplace(std::make_pair(iter->first, iter->second)); + // result.insert(std::make_pair(iter->first, iter->second)); + index++; + } + }; + + virtual void splitVector(std::vector& _vectors, uint32_t _start, uint32_t _end, + std::vector& _outVecs) + { + uint32_t index = 0; + for (auto vec : _vectors) + { + if (index < _start) + { + index++; + continue; + } + else if (index >= _end) + { + break; + } + _outVecs.push_back(vec); + index++; + } + }; + + +private: + bool m_syncResult{false}; + ppc::io::DataBatch::Ptr m_originInputs; + EcdhMultiPSIConfig::Ptr m_config; + bcos::bytes m_randomA; + std::string m_taskID; + TaskState::Ptr m_taskState; + std::map m_calculatorParties; + std::map m_partnerParties; + std::map m_masterParties; + std::vector m_finalResults; + CalculatorCipherDataCache::Ptr m_calculatorCipherDataCache; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp new file mode 100644 index 00000000..690738f2 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp @@ -0,0 +1,463 @@ +#include "EcdhMultiPSIMaster.h" +#include "ppc-psi/src/ecdh-multi-psi/Common.h" +#include +#include + + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace bcos; + +EcdhMultiPSIMaster::EcdhMultiPSIMaster(EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState) + : m_config(std::move(_config)), m_taskState(std::move(_taskState)) +{ + auto task = m_taskState->task(); + auto receivers = task->getReceiverLists(); + m_taskID = task->id(); + m_masterCipherDataCache = std::make_shared(); + m_final_counts[m_taskID] = 0; + m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), m_config->selfParty()) != receivers.end()); +} + +void EcdhMultiPSIMaster::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task) +{ + InitAsyncTask(_task); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Master asyncStartRunTask") << printTaskInfo(_task); + auto B = m_config->eccCrypto()->generateRandomScalar(); + m_randomB = std::make_shared(B); +} + +void EcdhMultiPSIMaster::InitAsyncTask(ppc::protocol::Task::ConstPtr _task) +{ + // Init all Roles from all Peers + auto peerParties = _task->getAllPeerParties(); + for (auto& party : peerParties) + { + auto partyId = party.first; + auto partySource = party.second; + if (partySource->partyIndex() == uint16_t(PartiesType::Calculator)) + { + m_calculatorParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Partner)) + { + m_partnerParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Master)) + { + m_masterParties[partyId] = partySource; + } + } +} + +// Part1-C,2-C: Master -> Calculator [H(X)*A ∩ H(Y)*A]*B +void EcdhMultiPSIMaster::onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg) +{ + try + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Part1-C: Master Receive H(X)*A ") + << LOG_KV(" PSIMessageFace Size: ", _msg->takeDataMap().size()) + << LOG_KV(" PSIMessageFace SEQ: ", _msg->seq()) + << LOG_KV(" PSIMessageFace dataBatchCount: ", _msg->dataBatchCount()); + WriteGuard lock(x_appendMasterCipherDataFromCalculator); + m_masterCipherDataCache->appendMasterCipherDataFromCalculator( + _msg->from(), _msg->takeDataMap(), _msg->seq(), _msg->dataBatchCount()); + if (loadCipherDataFinished()) + { + m_masterCipherDataCache->tryToGetCipherDataIntersection(); + auto cipherMaps = m_masterCipherDataCache->masterFinalIntersectionCipherData(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Part2-C:Master [H(X)*A ∩ H(Y)*A]") + << LOG_KV( + " Received Cipher Set Success Size: ", cipherMaps.size()); + tbb::concurrent_map encryptedInterHashMap; + tbb::parallel_for_each(cipherMaps.begin(), cipherMaps.end(), [&](auto const& _pair) { + auto index = _pair.first; + auto value = _pair.second; + if (value.data()) + { + auto hashSet = m_config->eccCrypto()->ecMultiply(value, *m_randomB); + // encryptedInterHashMap.insert(std::make_pair(index, hashSet)); + encryptedInterHashMap.emplace(std::make_pair(index, hashSet)); + } + }); + + auto inputSize = encryptedInterHashMap.size(); + auto batchSize = m_config->dataBatchSize(); + auto needSendTimes = 0; + // send counts + if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + uint32_t readStart = 0, readCount = 0; + while (readStart < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + std::map cHashMap; + ConcurrentSTLToCommon(encryptedInterHashMap, readStart, readCount, cHashMap); + + for (auto& calcultor : m_calculatorParties) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV( + "Part2-C:Master Send the [H(X)*A ∩ H(Y)*A]*B success to " + "Calculator: ", + calcultor.first) + << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", cHashMap.size()); + auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t( + EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR)); + message->setDataMap(std::move(cHashMap)); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + calcultor.first, m_taskID, message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetFromCalculator:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + +// Part1-P,2-P: Partner -> Master [H(X)*A ∩ H(Y)*A]*B +void EcdhMultiPSIMaster::onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg) +{ + try + { + WriteGuard lock(x_appendMasterCipherDataFromPartner); + m_masterCipherDataCache->appendMasterCipherDataFromPartner( + _msg->from(), _msg->takeData(), _msg->seq(), _msg->dataBatchCount()); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Part1-P: Master Receive H(Y)*A") + << LOG_KV(" PSIMessageFace Size: ", _msg->takeData().size()) + << LOG_KV(" PSIMessageFace SEQ: ", _msg->seq()) + << LOG_KV(" PSIMessageFace dataBatchCount: ", _msg->dataBatchCount()); + + if (loadCipherDataFinished()) + { + m_masterCipherDataCache->tryToGetCipherDataIntersection(); + auto cipherMaps = m_masterCipherDataCache->masterFinalIntersectionCipherData(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Part2-P:Master Receive [H(X)*A ∩ H(Y)*A]") + << LOG_KV(" Receive [H(X)*A ∩ H(Y)*A] Success Size: ", + cipherMaps.size()); + tbb::concurrent_map encryptedInterHashMap; + + tbb::parallel_for_each(cipherMaps.begin(), cipherMaps.end(), [&](auto const& _pair) { + auto index = _pair.first; + auto value = _pair.second; + if (value.data()) + { + auto hashSet = m_config->eccCrypto()->ecMultiply(value, *m_randomB); + // encryptedInterHashMap.insert(std::make_pair(index, hashSet)); + encryptedInterHashMap.emplace(std::make_pair(index, hashSet)); + } + }); + + auto inputSize = encryptedInterHashMap.size(); + auto batchSize = m_config->dataBatchSize(); + auto needSendTimes = 0; + // send counts + if (inputSize == 0) + { + needSendTimes = 1; + for (auto& calcultor : m_calculatorParties) + { + std::map cHashMap; + ECDH_MULTI_LOG(INFO) + << LOG_KV( + "Part2-P: Master Send the [H(X)*A ∩ H(Y)*A]*B success to " + "Calculator: ", + calcultor.first) + << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", 0); + auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t( + EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR)); + message->setDataMap(std::move(cHashMap)); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + calcultor.first, m_taskID, message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + 0); + } + return; + } + else if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + uint32_t readStart = 0, readCount = 0; + while (readCount < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + std::map cHashMap; + ConcurrentSTLToCommon(encryptedInterHashMap, readStart, readCount, cHashMap); + + for (auto& calcultor : m_calculatorParties) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV( + "Part2-P: Master Send the [H(X)*A ∩ H(Y)*A]*B success to " + "Calculator: ", + calcultor.first) + << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", cHashMap.size()); + auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t( + EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR)); + message->setDataMap(std::move(cHashMap)); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + calcultor.first, m_taskID, message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetFromPartner:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + +// Part3: Master -> Calculator H(Z)*B +void EcdhMultiPSIMaster::onHandlerEncryptIntersectionSetFromCalculatorToMaster( + PSIMessageInterface::Ptr _msg) +{ + try + { + m_originInputs = m_taskState->loadAllData(); + if (!m_originInputs || m_originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty")); + } + auto inputSize = m_originInputs->size(); + auto batchSize = m_config->dataBatchSize(); + auto needSendTimes = 0; + // send counts + if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + ECDH_MULTI_LOG(INFO) << LOG_KV("Part3: Master computeAndEncryptSet RandomB: ", + m_randomB->data()) + << LOG_KV(" Load All Data Size: ", inputSize); + auto hash = m_config->hash(); + uint32_t readStart = 0, readCount = 0; + std::vector encryptedHashSet; + encryptedHashSet.reserve(inputSize); + encryptedHashSet.resize(inputSize); + tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto data = m_originInputs->getBytes(i); + auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_config->eccCrypto()->hashToCurve(hashData); + auto hashSet = m_config->eccCrypto()->ecMultiply(point, *m_randomB); + encryptedHashSet[i] = hashSet; + } + }); + while (readCount < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + + std::vector encryptedHashSetSplit; + splitVector(encryptedHashSet, readStart, readCount, encryptedHashSetSplit); + + ECDH_MULTI_LOG(INFO) + << LOG_KV("Part3: Master compute the H(Z)*B encryptedHashSet success size: ", + encryptedHashSet.size()) + << LOG_KV("Part3: Master compute the H(Z)*B encryptedHashSetSplit success size: ", + encryptedHashSetSplit.size()); + + for (auto& calcultor : m_calculatorParties) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV( + " Send the EncryptSet H(Z)*B success to Calculator: ", calcultor.first) + << LOG_KV("EncryptSet Sample: ", encryptedHashSet[0].data()); + auto message = m_config->psiMsgFactory()->createPSIMessage( + uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR)); + message->setData(encryptedHashSetSplit); + message->setDataBatchCount(needSendTimes); + message->setFrom(m_taskState->task()->selfParty()->id()); + m_config->generateAndSendPPCMessage( + calcultor.first, m_taskID, message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) + << LOG_DESC("Exception in onHandlerEncryptIntersectionSetFromCalculatorToMaster:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + +void EcdhMultiPSIMaster::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg) +{ + ECDH_MULTI_LOG(INFO) + << LOG_KV("Final: Master Get SyncFinalResultToAllPeer From Calculator : ", + _msg->takeData().size()) + << LOG_KV("Final: Master Get SyncFinalResultToAllPeer From Calculator count: ", + m_final_counts[m_taskID]) + << LOG_KV("Final: Master isSyncedResult:", m_syncResult); + + if (m_syncResult) + { + auto needTimes = _msg->dataBatchCount(); + auto res = _msg->takeData(); + + trimVector(res); + bcos::WriteGuard l(x_final_count); + m_final_vectors.insert(m_final_vectors.end(), res.begin(), res.end()); + m_final_counts[m_taskID]++; + if (m_final_counts[m_taskID] < needTimes) + { + return; + } + + m_taskState->storePSIResult(m_config->dataResourceLoader(), m_final_vectors); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Final: Master onHandlerSyncFinalResultToAllPeer Store Intersection_XY^b ∩ H(Z)^b^a " + "Success" + "Dataset size: ", + m_final_vectors.size()); + } + else + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Master:No Need To ReceiveResultFromCalculator"); + } + + m_taskState->setFinished(true); + m_taskState->onTaskFinished(); +} + +bool EcdhMultiPSIMaster::loadCipherDataFinished() +{ + auto allPeerParties = m_taskState->task()->getAllPeerParties(); + auto finishedPeers = m_masterCipherDataCache->masterTaskPeersFinishedList(); + if (allPeerParties.size() == finishedPeers.size()) + { + for (auto& _peer : allPeerParties) + { + if (!finishedPeers.contains(_peer.first)) + { + return false; + } + } + return true; + } + else + { + return false; + } +} + +void EcdhMultiPSIMaster::onTaskError(std::string&& _error) +{ + auto result = std::make_shared(m_taskState->task()->id()); + auto err = std::make_shared(-12222, _error); + result->setError(std::move(err)); + m_taskState->onTaskFinished(result, true); +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h new file mode 100644 index 00000000..4a3076a6 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h @@ -0,0 +1,116 @@ +#pragma once +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" +#include "ppc-psi/src/psi-framework/TaskState.h" + +namespace ppc::psi +{ +class EcdhMultiPSIMaster : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + EcdhMultiPSIMaster(EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState); + virtual ~EcdhMultiPSIMaster() + { + if (m_originInputs) + { + m_originInputs->setData(std::vector()); + } + std::vector().swap(m_final_vectors); + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("the master destroyed") << LOG_KV("taskID", m_taskID); + } + virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task); + virtual void onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg); + virtual void onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg); + virtual void onHandlerEncryptIntersectionSetFromCalculatorToMaster( + PSIMessageInterface::Ptr _msg); + virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg); + + const std::string& taskID() const { return m_taskID; } + +protected: + virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task); + virtual bool loadCipherDataFinished(); + virtual void onTaskError(std::string&& _error); + void ConcurrentSTLToCommon( + tbb::concurrent_map _cMap, std::map& result) + { + ConcurrentSTLToCommon(_cMap, 0, _cMap.size(), result); + }; + + void ConcurrentSTLToCommon(tbb::concurrent_map _cMap, + uint32_t _startIndex, uint32_t _endIndex, std::map& result) + { + std::mutex mutex; + tbb::concurrent_map::const_iterator iter; + uint32_t index = 0; + for (iter = _cMap.begin(); iter != _cMap.end(); iter++) + { + std::lock_guard lock(mutex); + if (index < _startIndex) + { + index++; + continue; + } + else if (index >= _endIndex) + { + break; + } + // result.insert(std::make_pair(iter->first, iter->second)); + result.emplace(std::make_pair(iter->first, iter->second)); + index++; + } + }; + + virtual void trimVector(std::vector& _vectors) + { + for (auto it = _vectors.begin(); it != _vectors.end();) + { + if (!it->data()) + it = _vectors.erase(it); + else + ++it; + } + } + + virtual void splitVector(std::vector& _vectors, uint32_t _start, uint32_t _end, + std::vector& _outVecs) + { + uint32_t index = 0; + for (auto vec : _vectors) + { + if (index < _start) + { + index++; + continue; + } + else if (index >= _end) + { + break; + } + _outVecs.push_back(vec); + index++; + } + }; + +private: + bool m_syncResult{false}; + ppc::io::DataBatch::Ptr m_originInputs; + std::map m_calculatorParties; + std::map m_partnerParties; + std::map m_masterParties; + std::string m_taskID; + TaskState::Ptr m_taskState; + EcdhMultiPSIConfig::Ptr m_config; + bcos::bytesPointer m_randomB; + MasterCipherDataCache::Ptr m_masterCipherDataCache; + + mutable boost::shared_mutex x_appendMasterCipherDataFromPartner; + mutable boost::shared_mutex x_appendMasterCipherDataFromCalculator; + + std::vector m_final_vectors; + std::map m_final_counts; + mutable bcos::SharedMutex x_final_count; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp new file mode 100644 index 00000000..8c4845f8 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp @@ -0,0 +1,209 @@ +#include "EcdhMultiPSIPartner.h" +#include "ppc-psi/src/ecdh-multi-psi/Common.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace bcos; + +EcdhMultiPSIPartner::EcdhMultiPSIPartner(EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState) + : m_config(std::move(_config)), m_taskState(std::move(_taskState)) +{ + auto task = m_taskState->task(); + auto receivers = task->getReceiverLists(); + m_taskID = task->id(); + m_final_counts[m_taskID] = 0; + m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), + m_config->selfParty()) != receivers.end()); +} + +void EcdhMultiPSIPartner::InitAsyncTask(ppc::protocol::Task::ConstPtr _task) +{ + // Init all Roles from all Peers + auto peerParties = _task->getAllPeerParties(); + for (auto& party : peerParties) + { + auto partyId = party.first; + auto partySource = party.second; + if (partySource->partyIndex() == uint16_t(PartiesType::Calculator)) + { + m_calculatorParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Partner)) + { + m_partnerParties[partyId] = partySource; + } + if (partySource->partyIndex() == uint16_t(PartiesType::Master)) + { + m_masterParties[partyId] = partySource; + } + } +} + +// PART1: Partner -> Master H(Y)*A +void EcdhMultiPSIPartner::oncomputeAndEncryptSet(bcos::bytesPointer _randA) +{ + try + { + ECDH_MULTI_LOG(INFO) << LOG_KV("Part1:Partner Receive RandomA: ", _randA->data()); + auto originInputs = m_taskState->loadAllData(); + if (!originInputs || originInputs->size() == 0) + { + BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty")); + } + + auto inputSize = originInputs->size(); + auto batchSize = m_config->dataBatchSize(); + auto needSendTimes = 0; + + // send counts + if (inputSize % batchSize == 0) + { + needSendTimes = inputSize / batchSize; + } + else + { + needSendTimes = inputSize / batchSize + 1; + } + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Part1: Partner load the resource success size: ", inputSize); + auto hash = m_config->hash(); + std::vector encryptedHashSet; + encryptedHashSet.reserve(inputSize); + encryptedHashSet.resize(inputSize); + tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + auto data = originInputs->getBytes(i); + auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size())); + auto point = m_config->eccCrypto()->hashToCurve(hashData); + auto hashSet = m_config->eccCrypto()->ecMultiply(point, *_randA); + encryptedHashSet[i] = hashSet; + } + }); + uint32_t readStart = 0, readCount = 0; + + while (readCount < inputSize) + { + if (inputSize < batchSize) + { + readCount = batchSize; + } + else if (readCount + batchSize < inputSize) + { + readCount += batchSize; + } + else + { + readCount = inputSize; + } + + std::vector encryptedHashSetSplit; + splitVector(encryptedHashSet, readStart, readCount, encryptedHashSetSplit); + + ECDH_MULTI_LOG(INFO) + << LOG_KV("Part1: Partner compute the EncryptSet success encryptedHashSet size: ", + encryptedHashSet.size()) + << LOG_KV( + "Part1: Partner compute the EncryptSet success encryptedHashSetSplit " + "readStart: ", + readStart) + << LOG_KV( + "Part1: Partner compute the EncryptSet success encryptedHashSetSplit " + "readCount: ", + readCount) + << LOG_KV( + "Part1: Partner compute the EncryptSet success encryptedHashSetSplit " + "size: ", + encryptedHashSetSplit.size()); + + // generate and send encryptedHashSet + for (auto& master : m_masterParties) + { + auto message = m_config->psiMsgFactory()->createPSIMessage( + uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER)); + message->setData(encryptedHashSetSplit); + message->setFrom(m_taskState->task()->selfParty()->id()); + message->setDataBatchCount(needSendTimes); + m_config->generateAndSendPPCMessage( + master.first, m_taskState->task()->id(), message, + [self = weak_from_this()](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + }, + readCount); + } + readStart += batchSize; + } + } + catch (std::exception& e) + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in oncomputeAndEncryptSet:") + << boost::diagnostic_information(e); + onTaskError(boost::diagnostic_information(e)); + } +} + +void EcdhMultiPSIPartner::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg) +{ + if (m_syncResult) + { + ECDH_MULTI_LOG(INFO) + << LOG_KV("Final: Partner Get SyncFinalResultToAllPeer From Calculator : ", + _msg->takeData().size()) + << LOG_KV("Final: Partner Get SyncFinalResultToAllPeer From Calculator count: ", + m_final_counts[m_taskID]) + << LOG_KV("Final: Parter isSyncedResult:", m_syncResult); + auto needTimes = _msg->dataBatchCount(); + auto res = _msg->takeData(); + trimVector(res); + bcos::WriteGuard l(x_final_count); + m_final_vectors.insert(m_final_vectors.end(), res.begin(), res.end()); + m_final_counts[m_taskID]++; + if (m_final_counts[m_taskID] < needTimes) + { + return; + } + + m_taskState->storePSIResult(m_config->dataResourceLoader(), m_final_vectors); + ECDH_MULTI_LOG(INFO) << LOG_KV( + "Final: Partner onHandlerSyncFinalResultToAllPeer Store Intersection_XY^b ∩ H(Z)^b^a " + "Success" + "Dataset size: ", + m_final_vectors.size()); + } + else + { + ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner:No Need To ReceiveResultFromCalculator"); + } + + m_taskState->setFinished(true); + m_taskState->onTaskFinished(); +} + + +void EcdhMultiPSIPartner::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task) +{ + InitAsyncTask(_task); + ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner asyncStartRunTask as partner") + << printTaskInfo(_task); +} + + +void EcdhMultiPSIPartner::onTaskError(std::string&& _error) +{ + auto result = std::make_shared(m_taskState->task()->id()); + auto err = std::make_shared(-12222, _error); + result->setError(std::move(err)); + m_taskState->onTaskFinished(result, true); +} diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h new file mode 100644 index 00000000..f3e5f4a8 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h @@ -0,0 +1,76 @@ +#pragma once +#include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" +#include "ppc-psi/src/psi-framework/TaskState.h" +#include + +namespace ppc::psi +{ +class EcdhMultiPSIPartner : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + EcdhMultiPSIPartner(EcdhMultiPSIConfig::Ptr _config, TaskState::Ptr _taskState); + + virtual ~EcdhMultiPSIPartner() + { + std::vector().swap(m_final_vectors); + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_MULTI_LOG(INFO) << LOG_DESC("the partner destroyed") << LOG_KV("taskID", m_taskID); + } + + virtual void oncomputeAndEncryptSet(bcos::bytesPointer _randA); + virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task); + virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg); + + const std::string& taskID() const { return m_taskID; } + +protected: + virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task); + virtual void onTaskError(std::string&& _error); + + virtual void trimVector(std::vector& _vectors) + { + for (auto it = _vectors.begin(); it != _vectors.end();) + { + if (!it->data()) + it = _vectors.erase(it); + else + ++it; + } + } + + virtual void splitVector(std::vector& _vectors, uint32_t _start, uint32_t _end, + std::vector& _outVecs) + { + uint32_t index = 0; + for (auto vec : _vectors) + { + if (index < _start) + { + index++; + continue; + } + else if (index >= _end) + { + break; + } + _outVecs.push_back(vec); + index++; + } + }; + +private: + bool m_syncResult{false}; + EcdhMultiPSIConfig::Ptr m_config; + TaskState::Ptr m_taskState; + std::string m_taskID; + std::map m_calculatorParties; + std::map m_partnerParties; + std::map m_masterParties; + + mutable bcos::SharedMutex x_final_count; + std::vector m_final_vectors; + std::map m_final_counts; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt b/cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt new file mode 100644 index 00000000..261ee37e --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${ECDH_2PC_PSI_TARGET} ${SRCS}) +target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET} TCMalloc) \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/Common.h b/cpp/ppc-psi/src/ecdh-psi/Common.h new file mode 100644 index 00000000..a183cec4 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/Common.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-12-27 + */ +#pragma once +#include "../Common.h" +#include "ppc-framework/Common.h" + +#define ECDH_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-PSI") \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp b/cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp new file mode 100644 index 00000000..b20cc01d --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp @@ -0,0 +1,304 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhCache.cpp + * @author: yujiechen + * @date 2023-1-6 + */ +#include "EcdhCache.h" +#include "gperftools/malloc_extension.h" + +using namespace ppc::psi; +using namespace ppc::io; + +void DataCache::setClientCipherData(std::vector&& _cipherData) +{ + if (m_state != CacheState::Evaluating) + { + return; + } + bcos::WriteGuard l(x_clientCipherData); + m_clientCipherData = std::move(_cipherData); + m_state = CacheState::Finalized; +} + +bool DataCache::tryToIntersectionAndStoreResult() +{ + try + { + bcos::RecursiveGuard l(m_mutex); + auto startT = bcos::utcSteadyTime(); + if (m_state != CacheState::Finalized) + { + return false; + } + // the server data has not been load finished + if (!m_serverCipherData->serverDataLoadFinished()) + { + return false; + } + m_state = CacheState::IntersectionProgressing; + auto const& serverCipherData = m_serverCipherData->serverCipherData(); + uint64_t offset = 0; + for (auto const& data : m_clientCipherData) + { + if (serverCipherData.count(data)) + { + m_intersectionData.emplace_back(m_plainData->getBytes(offset)); + } + offset++; + } + m_state = CacheState::Intersectioned; + ECDH_LOG(INFO) << LOG_DESC("tryToIntersectionAndStoreResult success") + << LOG_KV("intersections", m_intersectionData.size()) << printCurrentState() + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + // try to store psi-result + storePSIResult(); + return true; + } + catch (std::exception const& e) + { + ECDH_LOG(WARNING) << LOG_DESC("tryToIntersectionAndStoreResult exception") + << printCurrentState() << LOG_KV("msg", boost::diagnostic_information(e)); + m_taskState->onTaskException(boost::diagnostic_information(e)); + return false; + } +} + +void DataCache::storePSIResult() +{ + if (m_state != CacheState::Intersectioned) + { + return; + } + m_state = CacheState::Stored; + ECDH_LOG(INFO) << LOG_DESC("storePSIResult") << printCurrentState(); + m_taskState->storePSIResult(m_config->dataResourceLoader(), m_intersectionData); + if (m_taskState->task()->syncResultToPeer()) + { + syncPSIResult(); + return; + } + // update the taskState + m_taskState->eraseFinishedTaskSeq(m_seq, true); + m_state = CacheState::Synced; +} + +void DataCache::syncPSIResult() +{ + // sync the psi-result to the psi-server by dataBatchSize + auto self = weak_from_this(); + auto syncPSIResultMsg = + m_config->msgFactory()->createPSIMessage((uint64_t)PSIPacketType::PSIResultSyncMsg); + syncPSIResultMsg->setData(std::move(m_intersectionData)); + ECDH_LOG(INFO) << LOG_DESC("syncPSIResult") << printCurrentState(); + m_config->generateAndSendPPCMessage( + m_taskState->peerID(), m_taskState->task()->id(), syncPSIResultMsg, + [self](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + auto cache = self.lock(); + if (!cache) + { + return; + } + ECDH_LOG(WARNING) << LOG_DESC("syncPSIResult to peer error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()) + << printTaskInfo(cache->m_taskState->task()); + }, + m_seq, + [self](bcos::Error::Ptr _error, std::string const& _agencyID, + ppc::front::PPCMessageFace::Ptr _msg, ppc::front::ResponseFunc) { + bool success = true; + if (_error) + { + success = false; + ECDH_LOG(WARNING) << LOG_DESC("syncPSIResult to peer error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + } + ECDH_LOG(INFO) << LOG_DESC("syncPSIResult: receive response") + << LOG_KV("seq", _msg->seq()); + auto cache = self.lock(); + if (!cache) + { + return; + } + cache->m_state = CacheState::Synced; + cache->m_taskState->eraseFinishedTaskSeq(cache->m_seq, success); + }); +} + +ServerCipherDataCache::Ptr EcdhCache::insertServerCipherCache( + std::string const& _taskID, TaskState::Ptr const& _taskState) +{ + bcos::UpgradableGuard l(x_serverDataStore); + auto it = m_serverDataStore.find(_taskID); + if (it != m_serverDataStore.end()) + { + return it->second; + } + bcos::UpgradeGuard ul(l); + auto cache = std::make_shared(_taskID); + m_serverDataStore[_taskID] = cache; + return cache; +} + + +ServerCipherDataCache::Ptr EcdhCache::getServerCipherDataCache(std::string const& _taskID) +{ + bcos::ReadGuard l(x_serverDataStore); + auto it = m_serverDataStore.find(_taskID); + if (it != m_serverDataStore.end()) + { + return it->second; + } + return nullptr; +} + +DataCache::Ptr EcdhCache::insertSubTaskCache(std::string const& _taskID, uint32_t _seq, + TaskState::Ptr const& _taskState, ServerCipherDataCache::Ptr const& _serverCipherData, + ppc::io::DataBatch::Ptr&& _plainData) +{ + bcos::UpgradableGuard l(x_taskCache); + if (!m_taskCache.count(_taskID)) + { + std::map subTaskCache; + m_taskCache[_taskID] = subTaskCache; + } + auto it = m_taskCache.at(_taskID).find(_seq); + if (it != m_taskCache.at(_taskID).end()) + { + return it->second; + } + + auto cache = std::make_shared( + m_config, _taskID, _seq, _taskState, _serverCipherData, std::move(_plainData)); + bcos::UpgradeGuard ul(l); + m_taskCache[_taskID][_seq] = cache; + m_capacity += cache->capacity(); + return cache; +} + +DataCache::Ptr EcdhCache::getSubTaskCache(std::string const& _taskID, uint32_t _seq) const +{ + bcos::ReadGuard l(x_taskCache); + if (m_taskCache.count(_taskID) && m_taskCache.at(_taskID).count(_seq)) + { + return m_taskCache.at(_taskID).at(_seq); + } + return nullptr; +} + +void EcdhCache::tryToIntersectionAndStoreResult() +{ + bcos::WriteGuard l(x_taskCache); + for (auto it = m_taskCache.begin(); it != m_taskCache.end();) + { + auto& clientCipherCache = it->second; + for (auto pclientCipherCache = clientCipherCache.begin(); + pclientCipherCache != clientCipherCache.end();) + { + // Note: the cache must copy here in case of pclientCipherCache->second been erased + auto cache = pclientCipherCache->second; + // update the capacity and erase the finished sub-task + if (cache->state() == CacheState::Synced) + { + pclientCipherCache = clientCipherCache.erase(pclientCipherCache); + ECDH_LOG(INFO) << LOG_DESC( + "tryToIntersectionAndStoreResult: erase finished sub-task") + << cache->printCurrentState(); + if (m_capacity >= cache->capacity()) + { + m_capacity -= cache->capacity(); + } + else + { + m_capacity = 0; + } + continue; + } + if (cache->state() == CacheState::Finalized) + { + m_config->threadPool()->enqueue([cache]() { + // Note: the tryToIntersectionAndStoreResult should not throw exception + cache->tryToIntersectionAndStoreResult(); + }); + pclientCipherCache++; + continue; + } + pclientCipherCache++; + } + if (clientCipherCache.empty()) + { + it = m_taskCache.erase(it); + continue; + } + it++; + } +} + +void EcdhCache::eraseTaskCache(std::string const& _taskID) +{ + // erase the taskCache + // Note: here must using writeLock in case of the capacity been updated multiple times + bcos::WriteGuard l(x_taskCache); + if (!m_taskCache.count(_taskID)) + { + return; + } + ECDH_LOG(INFO) << LOG_DESC("eraseCache: erase the taskCache") << LOG_KV("taskID", _taskID); + // update the capacity + auto const& subTasks = m_taskCache.at(_taskID); + for (auto const& it : subTasks) + { + auto const& cache = it.second; + if (m_capacity >= cache->capacity()) + { + m_capacity -= cache->capacity(); + } + else + { + m_capacity = 0; + } + } + // erase the subTasks + m_taskCache.erase(_taskID); +} + +// Note: only when the task-finished can erase the cache +void EcdhCache::eraseCache(std::string const& _taskID) +{ + auto startT = bcos::utcSteadyTime(); + // Note: since the every DataCache references m_serverDataStore + // the m_serverDataStore only can be released after all the DataCache have been released + eraseTaskCache(_taskID); + // erase the server cipher data cache + bcos::UpgradableGuard l(x_serverDataStore); + if (!m_serverDataStore.count(_taskID)) + { + return; + } + ECDH_LOG(INFO) << LOG_DESC("eraseCache: erase the serverDataStore") + << LOG_KV("taskID", _taskID); + bcos::UpgradeGuard ul(l); + m_serverDataStore.erase(_taskID); + MallocExtension::instance()->ReleaseFreeMemory(); + ECDH_LOG(INFO) << LOG_DESC("eraseCache") << LOG_KV("taskID", _taskID) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); +} diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhCache.h b/cpp/ppc-psi/src/ecdh-psi/EcdhCache.h new file mode 100644 index 00000000..5fd7010a --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhCache.h @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhCache.h + * @author: yujiechen + * @date 2022-12-28 + */ +#pragma once +#include "../psi-framework/TaskState.h" +#include "Common.h" +#include "EcdhPSIConfig.h" +#include +#include +namespace ppc::psi +{ +/// the server data-cache +class ServerCipherDataCache +{ +public: + using Ptr = std::shared_ptr; + ServerCipherDataCache(std::string const& _taskID) : m_taskID(_taskID) {} + virtual ~ServerCipherDataCache() + { + // release the memory + std::set().swap(m_serverCipherData); + } + + void setServerDataBatchCount(int64_t _serverDataBatchCount) + { + m_serverDataBatchCount = _serverDataBatchCount; + if ((int64_t)m_serverDataSeqList.size() == m_serverDataBatchCount) + { + m_serverDataLoadFinished = true; + } + } + + virtual void appendServerCipherData(uint32_t _seq, std::vector&& _cipherData) + { + bcos::WriteGuard l(x_serverCipherData); + if (m_serverDataSeqList.count(_seq)) + { + return; + } + for (auto& it : _cipherData) + { + m_serverCipherData.insert(std::move(it)); + } + m_serverDataSeqList.insert(_seq); + if ((int64_t)m_serverDataSeqList.size() == m_serverDataBatchCount) + { + m_serverDataLoadFinished = true; + } + } + + bool serverDataLoadFinished() { return m_serverDataLoadFinished; } + std::set const& serverCipherData() const { return m_serverCipherData; } + +private: + std::string m_taskID; + bool m_serverDataLoadFinished = false; + + // store the cipher-data of the server + // TODO: replace with unordered_set + std::set m_serverCipherData; + std::set m_serverDataSeqList; + bcos::SharedMutex x_serverCipherData; + std::atomic m_serverDataBatchCount = {-1}; +}; + +class DataCache : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + // set the plainData here for obtain the plain intersection + DataCache(EcdhPSIConfig::Ptr const& _config, std::string const& _taskID, uint32_t _seq, + TaskState::Ptr const& _taskState, ServerCipherDataCache::Ptr const& _serverCipherData, + ppc::io::DataBatch::Ptr&& _plainData) + : m_config(_config), + m_taskID(_taskID), + m_seq(_seq), + m_taskState(_taskState), + m_state(CacheState::Evaluating), + m_serverCipherData(_serverCipherData), + m_plainData(std::move(_plainData)) + { + m_capacity = m_plainData->capacityBytes(); + // for m_clientCipherData + m_capacity += m_plainData->size() * 32; + } + virtual ~DataCache() + { + // release the memory + m_plainData = nullptr; + std::vector().swap(m_clientCipherData); + std::vector().swap(m_intersectionData); + } + virtual void setClientCipherData(std::vector&& _cipherData); + virtual bool tryToIntersectionAndStoreResult(); + std::string const& taskID() const { return m_taskID; } + + CacheState state() const { return m_state; } + + uint64_t capacity() const { return m_capacity; } + + inline std::string printCurrentState() + { + std::ostringstream stringstream; + stringstream << LOG_KV("task", m_taskID) << LOG_KV("seq", m_seq) + << LOG_KV("plainData", m_plainData ? m_plainData->size() : 0) + << LOG_KV("clientCipherData", m_clientCipherData.size()) + << LOG_KV("intersectionData", m_intersectionData.size()) + << LOG_KV("state", m_state); + return stringstream.str(); + } + +private: + void storePSIResult(); + void syncPSIResult(); + +private: + EcdhPSIConfig::Ptr m_config; + std::string m_taskID; + // the sub-task seq + uint32_t m_seq; + TaskState::Ptr m_taskState; + + CacheState m_state; + + uint64_t m_capacity; + + // the server cipher data cache + ServerCipherDataCache::Ptr m_serverCipherData; + + // the plainData + ppc::io::DataBatch::Ptr m_plainData; + // the cipher-data of the client + std::vector m_clientCipherData; + bcos::SharedMutex x_clientCipherData; + + // the intersection data + std::vector m_intersectionData; + + mutable bcos::RecursiveMutex m_mutex; +}; + +class EcdhCache : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + EcdhCache(EcdhPSIConfig::Ptr const& _config, uint64_t _maxCacheSize = 1024 * 1024 * 1024) + : m_config(_config), m_maxTaskCacheSize(_maxCacheSize) + {} + virtual ~EcdhCache() = default; + + // insert the server-cipher-data-cache + virtual ServerCipherDataCache::Ptr insertServerCipherCache( + std::string const& _taskID, TaskState::Ptr const& _taskState); + // get the server cipher data cache + virtual ServerCipherDataCache::Ptr getServerCipherDataCache(std::string const& _taskID); + + // insert data-cache into m_taskCache + virtual DataCache::Ptr insertSubTaskCache(std::string const& _taskID, uint32_t _seq, + TaskState::Ptr const& _taskState, ServerCipherDataCache::Ptr const& _serverCipherData, + ppc::io::DataBatch::Ptr&& _plainData); + + DataCache::Ptr getSubTaskCache(std::string const& _taskID, uint32_t _seq) const; + // Note: only when the task-finished can erase the cache + virtual void eraseCache(std::string const& _taskID); + virtual void tryToIntersectionAndStoreResult(); + + bool isFull() { return m_capacity >= m_maxTaskCacheSize; } + + uint64_t capacity() const { return m_capacity; } + +private: + void eraseTaskCache(std::string const& _taskID); + +private: + EcdhPSIConfig::Ptr m_config; + + uint64_t m_maxTaskCacheSize; + // the capacity for sub-task + // Note: the intersection-data and cipherDataStore may occupy some memory + std::atomic m_capacity = {0}; + + // taskID => seq => sub-task-data + std::map> m_taskCache; + mutable bcos::SharedMutex x_taskCache; + + // taskID => the server cipher data + std::map m_serverDataStore; + mutable bcos::SharedMutex x_serverDataStore; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h new file mode 100644 index 00000000..41affc04 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIConfig.h + * @author: yujiechen + * @date 2022-12-26 + */ +#pragma once +#include "../PSIConfig.h" +#include "../psi-framework/interfaces/PSIMessageFactory.h" +#include "ppc-framework/crypto/ECDHCrypto.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +namespace ppc::psi +{ +class EcdhPSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + EcdhPSIConfig(std::string const& _selfParty, + ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + PSIMessageFactory::Ptr const& _msgFactory, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader, uint32_t _dataBatchSize, + int _holdingMessageMinutes, bcos::ThreadPool::Ptr const& _threadPool = nullptr, + int _threadPoolSize = std::thread::hardware_concurrency()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC, _selfParty, _front, _ppcMsgFactory, + _dataResourceLoader, _holdingMessageMinutes), + m_msgFactory(_msgFactory), + m_threadPool(_threadPool), + m_ecdhCryptoFactory(_ecdhCryptoFactory), + m_dataBatchSize(_dataBatchSize) + { + if (!m_threadPool) + { + m_threadPool = std::make_shared("ecdh-pool", _threadPoolSize); + } + } + ~EcdhPSIConfig() override + { + if (m_threadPool) + { + m_threadPool->stop(); + } + } + + PSIMessageFactory::Ptr const& msgFactory() const { return m_msgFactory; } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + ppc::crypto::ECDHCryptoFactory::Ptr const& ecdhCryptoFactory() const + { + return m_ecdhCryptoFactory; + } + + uint32_t dataBatchSize() const { return m_dataBatchSize; } + +private: + PSIMessageFactory::Ptr m_msgFactory; + bcos::ThreadPool::Ptr m_threadPool; + ppc::crypto::ECDHCryptoFactory::Ptr m_ecdhCryptoFactory; + + uint32_t m_dataBatchSize = 10000; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h new file mode 100644 index 00000000..8513524a --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIFactory.h + * @author: yujiechen + * @date 2022-12-28 + */ +#pragma once +#include "EcdhPSIImpl.h" +#include "EcdhPSIMessageFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +namespace ppc::psi +{ +class EcdhPSIFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhPSIFactory() = default; + virtual ~EcdhPSIFactory() = default; + + virtual EcdhPSIImpl::Ptr createEcdhPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader) + { + auto psiMsgFactory = std::make_shared(); + auto const& ecdhParam = _ppcConfig->ecdhPSIConfig(); + auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory, + _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize, + _ppcConfig->holdingMessageMinutes(), _threadPool); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp new file mode 100644 index 00000000..4f79196b --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp @@ -0,0 +1,780 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIImpl.cpp + * @author: yujiechen + * @date 2022-12-27 + */ +#include "EcdhPSIImpl.h" +#include "ppc-framework/protocol/GlobalConfig.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace bcos; + +void EcdhPSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +{ + try + { + // check the partyIndex + auto partyIndex = _task->selfParty()->partyIndex(); + if (partyIndex != (int)PartyType::Client && partyIndex != (int)PartyType::Server) + { + auto taskResult = std::make_shared(_task->id()); + auto error = BCOS_ERROR_PTR((int)TaskParamsError, + "The party index of the ecdh-psi must be client(0) or server(1)!"); + ECDH_LOG(WARNING) << LOG_DESC("response task result") << LOG_KV("task", _task) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + taskResult->setError(std::move(error)); + _onTaskFinished(std::move(taskResult)); + return; + } + // create the taskState + auto self = weak_from_this(); + auto taskState = m_taskStateFactory->createTaskState( + _task, [self, _task, _onTaskFinished](TaskResult::Ptr&& _result) { + auto result = std::move(_result); + _onTaskFinished(std::move(result)); + ECDH_LOG(INFO) << LOG_DESC("asyncRunTask finished") << printTaskInfo(_task); + }); + auto const& taskID = _task->id(); + taskState->registerFinalizeHandler([self, taskID]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + // erase the cache + psi->m_cache->eraseCache(taskID); + // erase the taskInfo from the gateway + psi->m_config->front()->eraseTaskInfo(taskID); + }); + // Note: ecdh-psi with only run-psi command + auto error = lockResourceAndRecordTaskState(0, taskState); + if (error) + { + // trigger the callback + auto taskResult = std::make_shared(_task->id()); + ECDH_LOG(WARNING) << LOG_DESC("response task result") << LOG_KV("task", _task) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + taskResult->setError(std::move(error)); + _onTaskFinished(std::move(taskResult)); + return; + } + // set the peerID + auto peerParty = checkAndSetPeerInfo(taskState, false); + if (!peerParty) + { + return; + } + auto const& peerID = peerParty->id(); + // check the dataResource + bool server = (partyIndex == (int)PartyType::Server) ? true : false; + if (!checkDataResourceForSelf(taskState, peerID, !server)) + { + return; + } + taskState->registerSubTaskFinishedHandler([self]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + // tryToIntersectionAndStoreResult will erase the finished task, update the capacity and + // trigger next-round data-blinding + psi->m_cache->tryToIntersectionAndStoreResult(); + psi->wakeupWorker(); + }); + auto ret = initTaskState(taskState); + if (ret) + { + runPSI(taskState); + } + } + catch (std::exception const& e) + { + ECDH_LOG(WARNING) << LOG_DESC("asyncRunTask exception, cancel the task") + << printTaskInfo(_task) + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = std::make_shared( + -1, "asyncRunTask " + _task->id() + " exception: " + boost::diagnostic_information(e)); + cancelTask(std::move(error), _task->id()); + } +} + +void EcdhPSIImpl::runPSI(TaskState::Ptr const& _taskState) +{ + ECDH_LOG(INFO) << LOG_DESC("runPSI") << printTaskInfo(_taskState->task()); + // notify the taskInfo to the front + m_config->front()->notifyTaskInfo(std::make_shared(_taskState->task()->id())); + // the psi-client send handshake request to the server + if (_taskState->task()->selfParty()->partyIndex() == (int)PartyType::Client) + { + sendHandshakeRequest(_taskState); + } +} + +// handle the psi message +void EcdhPSIImpl::handlePSIMsg(PSIMessageInterface::Ptr _msg) +{ + if (!checkPSIMsg(_msg)) + { + return; + } + // prioritize lightweight messages + if (handlePSIFrameworkMsg(_msg)) + { + return; + } + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, _msg]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + switch (_msg->packetType()) + { + case (uint32_t)ECDHPacketType::EvaluateRequest: + { + psi->handleEvaluateRequest(_msg); + break; + } + case (uint32_t)ECDHPacketType::EvaluateResponse: + { + psi->handleEvaluateResponse(_msg); + break; + } + case (uint32_t)ECDHPacketType::ServerBlindedData: + { + psi->handleServerBlindData(_msg); + break; + } + case (uint32_t)ECDHPacketType::SyncDataBatchInfo: + { + psi->onRecvSyncDataInfoMsg(_msg); + break; + } + default: + { + ECDH_LOG(ERROR) << LOG_DESC("Unsupported packetType ") << (int)_msg->packetType(); + break; + } + } + } + catch (std::exception const& e) + { + ECDH_LOG(ERROR) << LOG_DESC("handlePSIMsg exception") + << LOG_KV("packetType", _msg->packetType()) << printPSIMessage(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandleTaskError, + "Task " + _msg->taskID() + " error for " + boost::diagnostic_information(e)); + psi->onTaskError("handlePSIMsg", std::move(error), _msg->from(), _msg->taskID(), + "emptyDataResource"); + } + }); +} + +bool EcdhPSIImpl::initTaskState(TaskState::Ptr const& _taskState) +{ + auto task = _taskState->task(); + auto dataResource = task->selfParty()->dataResource(); + bool server = (task->selfParty()->partyIndex() == (int)PartyType::Server) ? true : false; + try + { + // the client + if (!server) + { + m_cache->insertServerCipherCache(task->id(), _taskState); + if (!m_enableOutputExists) + { + // Note: if the output-resource already exists, will throw exception + m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc()); + } + else + { + m_config->dataResourceLoader()->loadWriter(dataResource->outputDesc(), true); + } + } + else if (_taskState->task()->syncResultToPeer()) + { + // check the output resource existence when syncResultToPeer + _taskState->tryToGenerateDefaultOutputDesc(); + m_config->dataResourceLoader()->checkResourceExists( + task->selfParty()->dataResource()->outputDesc()); + } + // load the data + auto reader = loadData(m_config->dataResourceLoader(), task->id(), dataResource); + // Note: the sql read all data at one time + auto sqlReader = (reader->type() == ppc::protocol::DataResourceType::MySQL); + // fileReader: load dataBatchSize() elements from the given file + // sqlReader: load the first column + auto nextParam = sqlReader ? 0 : m_config->dataBatchSize(); + _taskState->setReader(std::move(reader), nextParam); + ECDH_LOG(INFO) << LOG_DESC("initTaskState") << printTaskInfo(task); + return true; + } + catch (std::exception const& e) + { + ECDH_LOG(WARNING) << LOG_DESC("initTaskState exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandleTaskError, + "init task error for " + boost::diagnostic_information(e)); + onTaskError("initTaskState", std::move(error), _taskState->peerID(), task->id(), + dataResource ? dataResource->resourceID() : "emptyDataResource"); + return false; + } +} + +// the psi-server and psi-client load data from DataResource and blind the plainData into cipher +void EcdhPSIImpl::blindData(TaskState::Ptr const& _taskState) +{ + auto startT = utcSteadyTime(); + auto task = _taskState->task(); + bool server = (task->selfParty()->partyIndex() == (int)PartyType::Server) ? true : false; + // the client cache is full + if (!server && m_cache->isFull()) + { + return; + } + auto dataResource = task->selfParty()->dataResource(); + uint32_t packetType = (int32_t)ECDHPacketType::EvaluateRequest; + if (server) + { + packetType = (int32_t)ECDHPacketType::ServerBlindedData; + } + // load all data from the given resource + uint64_t batchDataCount = 0; + auto self = weak_from_this(); + auto const& taskID = task->id(); + auto serverCipherDataCache = m_cache->getServerCipherDataCache(taskID); + do + { + // the task has been canceled + if (!getTaskByID(taskID)) + { + // erase the cache and return + m_cache->eraseCache(taskID); + return; + } + // not to blind when the cache is Full + if (!server && m_cache->isFull()) + { + ECDH_LOG(TRACE) << LOG_DESC("blindData return for the cache is full") + << LOG_KV("capacity", m_cache->capacity()); + break; + } + if (_taskState->loadFinished()) + { + break; + } + int32_t seq = 0; + DataBatch::Ptr dataBatch = nullptr; + { + bcos::Guard l(m_mutex); + // Note: next is not thread-safe + dataBatch = _taskState->reader()->next(_taskState->readerParam(), DataSchema::Bytes); + if (!dataBatch) + { + _taskState->setFinished(true); + break; + } + // allocate seq + seq = _taskState->allocateSeq(); + if (_taskState->sqlReader()) + { + _taskState->setFinished(true); + } + } + auto ecdhTaskState = std::dynamic_pointer_cast(_taskState); + auto batchPk = ecdhTaskState->ecdhCrypto()->batchGetPublicKey(dataBatch); + + auto evaluateRequest = m_config->msgFactory()->createPSIMessage(packetType); + evaluateRequest->setData(std::move(batchPk)); + evaluateRequest->setResourceID(dataResource->resourceID()); + auto dataBatchSize = dataBatch->size(); + // the client should append the plainData into cache for get the plainData later + if (!server) + { + m_cache->insertSubTaskCache( + taskID, seq, _taskState, serverCipherDataCache, std::move(dataBatch)); + } + ECDH_LOG(INFO) << LOG_DESC("blindData") << LOG_KV("resource", dataResource->resourceID()) + << LOG_KV("seq", seq) << LOG_KV("size", dataBatchSize) + << LOG_KV("cacheCapacity", m_cache->capacity()) + << LOG_KV("task", _taskState->task()->id()); + auto sendT = utcSteadyTime(); + m_config->generateAndSendPPCMessage( + _taskState->peerID(), _taskState->task()->id(), evaluateRequest, + [self, seq, _taskState, sendT](Error::Ptr&& _error) { + if (!_error) + { + ECDH_LOG(INFO) << LOG_DESC("blindData: send evaluate request success") + << LOG_KV("seq", seq); + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + ECDH_LOG(WARNING) << LOG_DESC("blindData: send evaluate request error") + << LOG_KV("seq", seq) << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()) + << LOG_KV("timecost", (utcSteadyTime() - sendT)); + psi->cancelTask(std::move(_error), _taskState->task()->id()); + }, + seq); + batchDataCount++; + } while (m_started && !_taskState->sqlReader()); + if (!server) + { + return; + } + // the server send dataBatchCount to client + ECDH_LOG(INFO) << LOG_DESC("blindData: sync the server-data-batch-count") + << LOG_KV("batchDataCount", batchDataCount); + auto req = + m_config->msgFactory()->createPSIMessage((uint32_t)ECDHPacketType::SyncDataBatchInfo); + req->setDataBatchCount(batchDataCount); + req->setResourceID(dataResource->resourceID()); + m_config->generateAndSendPPCMessage(_taskState->peerID(), _taskState->task()->id(), req, + [self, _taskState](Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + ECDH_LOG(WARNING) << LOG_DESC("blindData: send SyncDataBatchInfo request error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + // cancel the task(TODO:retry) + psi->cancelTask(std::move(_error), _taskState->task()->id()); + }); + ECDH_LOG(INFO) << LOG_DESC("blindData success") << printTaskInfo(_taskState->task()) + << LOG_KV("timecost", (utcSteadyTime() - startT)); +} + +// receive the blinded-data from client, the psi-server response the evaluated data +void EcdhPSIImpl::handleEvaluateRequest(PSIMessageInterface::Ptr _msg) +{ + auto ret = checkAndObtainTaskState(_msg); + if (!ret.first || !ret.second) + { + return; + } + auto taskState = ret.first; + auto ecdhCrypto = ret.second; + auto startT = utcSteadyTime(); + ECDH_LOG(INFO) << LOG_DESC("handleEvaluateRequest") << printPSIMessage(_msg); + auto sharedPk = ecdhCrypto->batchGetSharedPublicKey(_msg->takeData()); + // response the evaluated result + auto evaluateResponse = + m_config->msgFactory()->createPSIMessage((uint32_t)ECDHPacketType::EvaluateResponse); + evaluateResponse->setData(std::move(sharedPk)); + auto self = weak_from_this(); + auto startSendT = utcSteadyTime(); + m_config->generateAndSendPPCMessage( + _msg->from(), _msg->taskID(), evaluateResponse, + [self, _msg, startSendT](Error::Ptr&& _error) { + if (!_error) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + ECDH_LOG(WARNING) << LOG_DESC("handleEvaluateRequest: send evaluate response error") + << printPSIMessage(_msg) << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()) + << LOG_KV("timecost", utcSteadyTime() - startSendT); + psi->onTaskError("handleEvaluateRequest: response evaluate data", std::move(_error), + _msg->from(), _msg->taskID(), _msg->resourceID()); + }, + _msg->seq()); + ECDH_LOG(INFO) << LOG_DESC("handleEvaluateRequest success") << printPSIMessage(_msg) + << LOG_KV("timecost", (utcSteadyTime() - startT)); +} + +std::pair EcdhPSIImpl::checkAndObtainTaskState( + PSIMessageInterface::Ptr _msg) +{ + auto taskState = getTaskByID(_msg->taskID()); + if (!taskState) + { + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskNotFound, + "Task " + _msg->taskID() + " not found when handle server blinded-data!"); + onTaskError("handleServerBlindData", std::move(error), _msg->from(), _msg->taskID(), + "emptyDataResource"); + return std::make_pair(nullptr, nullptr); + } + auto ecdhTaskState = std::dynamic_pointer_cast(taskState); + // here the client maybe not ready for the ecdhCrypto when submit the multiple-tasks with same + // taskID and some tasks failed + if (!ecdhTaskState->ecdhCrypto()) + { + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskNotReady, + "The ecdh-crypto-suite is not ready for task " + _msg->taskID() + + " not found when handle server blinded-data!"); + onTaskError("handleServerBlindData", std::move(error), _msg->from(), _msg->taskID(), + "emptyDataResource"); + return std::make_pair(nullptr, nullptr); + } + return std::make_pair(taskState, ecdhTaskState->ecdhCrypto()); +} + +void EcdhPSIImpl::handleServerBlindData(PSIMessageInterface::Ptr _msg) +{ + auto startT = utcSteadyTime(); + ECDH_LOG(INFO) << LOG_DESC("handleServerBlindData") << printPSIMessage(_msg); + auto ret = checkAndObtainTaskState(_msg); + if (!ret.first || !ret.second) + { + return; + } + auto taskState = ret.first; + auto ecdhCrypto = ret.second; + auto sharedPk = ecdhCrypto->batchGetSharedPublicKey(_msg->takeData()); + + // append the evaluated data into cache + auto cache = m_cache->getServerCipherDataCache(_msg->taskID()); + if (!cache) + { + auto taskState = getTaskByID(_msg->taskID()); + if (!taskState) + { + auto error = BCOS_ERROR_PTR(PSIRetCode::TaskNotFound, + m_config->selfParty() + " handleServerBlindData failed for task not found!"); + ECDH_LOG(WARNING) << LOG_DESC("handleServerBlindData failed") + << LOG_KV("error", error->errorCode()) + << LOG_KV("msg", error->errorMessage()) << printPSIMessage(_msg); + onTaskError("handleServerBlindData: Task not found", std::move(error), _msg->from(), + _msg->taskID(), _msg->resourceID()); + return; + } + cache = m_cache->insertServerCipherCache(_msg->taskID(), taskState); + } + cache->appendServerCipherData(_msg->seq(), std::move(sharedPk)); + m_cache->tryToIntersectionAndStoreResult(); + ECDH_LOG(INFO) << LOG_DESC("handleServerBlindData success") << printPSIMessage(_msg) + << LOG_KV("timecost", (utcSteadyTime() - startT)); +} + +void EcdhPSIImpl::handleEvaluateResponse(PSIMessageInterface::Ptr _msg) +{ + auto startT = utcSteadyTime(); + auto evaluateData = _msg->takeData(); + ECDH_LOG(INFO) << LOG_DESC("handleEvaluateResponse") << printPSIMessage(_msg) + << LOG_KV("dataSize", evaluateData.size()); + + // append the evaluate response into the cache + auto cache = m_cache->getSubTaskCache(_msg->taskID(), _msg->seq()); + // if the cache is empty, this means the blindData for the (taskID, seq) has not been triggered + // maybe this is evaluate response of the last-canceled task + if (!cache) + { + return; + } + cache->setClientCipherData(std::move(evaluateData)); + m_cache->tryToIntersectionAndStoreResult(); + ECDH_LOG(INFO) << LOG_DESC("handleEvaluateResponse success") << printPSIMessage(_msg) + << LOG_KV("timecost", (utcSteadyTime() - startT)); +} + +void EcdhPSIImpl::onRecvSyncDataInfoMsg(PSIMessageInterface::Ptr const& _msg) +{ + ECDH_LOG(INFO) << LOG_DESC("onRecvSyncDataInfoMsg") << printPSIMessage(_msg) + << LOG_KV("dataBatchCount", _msg->dataBatchCount()); + auto cache = m_cache->getServerCipherDataCache(_msg->taskID()); + if (!cache) + { + auto error = BCOS_ERROR_PTR( + PSIRetCode::TaskNotFound, "onRecvSyncDataInfoMsg failed for task not found!"); + onTaskError("onRecvSyncDataInfoMsg: Task not found", std::move(error), _msg->from(), + _msg->taskID(), _msg->resourceID()); + return; + } + cache->setServerDataBatchCount(_msg->dataBatchCount()); + // try to intersection and store the psi result + m_cache->tryToIntersectionAndStoreResult(); +} + +// the client receive the handshake response +void EcdhPSIImpl::onHandshakeResponse(PSIMessageInterface::Ptr const& _msg) +{ + auto handshakeResponse = std::dynamic_pointer_cast(_msg); + ECDH_LOG(INFO) << LOG_DESC("onHandshakeResponse") << LOG_KV("curve", handshakeResponse->curve()) + << LOG_KV("hashImpl", handshakeResponse->hashType()) << printPSIMessage(_msg) + << LOG_KV("uuid", _msg->uuid()); + auto taskState = getTaskByID(_msg->taskID()); + if (!taskState) + { + ECDH_LOG(INFO) << LOG_DESC("onHandshakeResponse return for the task not found") + << printPSIMessage(_msg); + auto error = + BCOS_ERROR_PTR((int)PSIRetCode::TaskNotFound, "Task " + _msg->taskID() + " not found!"); + onTaskError("onHandshakeResponse", std::move(error), _msg->from(), _msg->taskID(), + "emptyDataResource"); + return; + } + if (handshakeResponse->curve() == -1 || handshakeResponse->hashType() == -1) + { + auto errorMsg = "onHandshakeResponse return for not find the matching curve/hashType"; + ECDH_LOG(WARNING) << LOG_DESC(errorMsg) << printPSIMessage(handshakeResponse); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandshakeFailed, errorMsg); + auto dataResource = taskState->task()->selfParty()->dataResource(); + onTaskError("onHandshakeResponse", std::move(error), taskState->peerID(), + taskState->task()->id(), dataResource ? dataResource->resourceID() : "empty"); + return; + } + // create and set the handshaked ecdh-crypto + auto ecdhCrypto = m_config->ecdhCryptoFactory()->createECDHCrypto( + handshakeResponse->curve(), handshakeResponse->hashType()); + auto ecdhTaskState = std::dynamic_pointer_cast(taskState); + ecdhTaskState->setEcdhCrypto(ecdhCrypto); + + // response to the server, the server can blindData + auto psiMsg = + m_msgFactory->createTaskNotificationMessage((uint32_t)PSIPacketType::HandshakeSuccess); + psiMsg->setErrorCode(0); + psiMsg->setErrorMessage("success"); + auto startT = bcos::utcSteadyTime(); + m_config->asyncSendResponse(taskState->peerID(), taskState->task()->id(), _msg->uuid(), psiMsg, + [this, startT, _msg](bcos::Error::Ptr _error) { + if (!_error || _error->errorCode() == 0) + { + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("asyncSendResponse success") + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)) + << printPSIMessage(_msg) << LOG_KV("uuid", _msg->uuid()); + return; + } + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("onHandshakeResponse: response-handshake result error") + << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + }); + + // blindData using specified ecc-curve and hash-algorithm + triggerDataBlind(taskState); + + // Note: the sql will load all data at-one-time, no-need blindData repeatedly + if (!taskState->sqlReader()) + { + auto weakTaskState = std::weak_ptr(taskState); + auto self = weak_from_this(); + taskState->setWorker([weakTaskState, self]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + auto state = weakTaskState.lock(); + if (!state) + { + return; + } + psi->triggerDataBlind(state); + }); + } +} + +void EcdhPSIImpl::triggerDataBlind(TaskState::Ptr const& _taskState) +{ + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, _taskState]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + psi->blindData(_taskState); + } + catch (std::exception const& e) + { + ECDH_LOG(INFO) << LOG_DESC("runPSI exception") + << LOG_KV("error", boost::diagnostic_information(e)) + << printTaskInfo(_taskState->task()); + auto dataResource = _taskState->task()->selfParty()->dataResource(); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::BlindDataError, + "ecdh-psi blindData error, msg: " + std::string(boost::diagnostic_information(e))); + psi->onTaskError("runPSI-blindData", std::move(error), _taskState->peerID(), + _taskState->task()->id(), dataResource ? dataResource->resourceID() : "empty"); + } + }); +} + +// the server receive the handshake request and response +void EcdhPSIImpl::onHandshakeRequest(PSIMessageInterface::Ptr const& _msg) +{ + auto taskState = getTaskByID(_msg->taskID()); + if (!taskState) + { + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskNotFound, + "task " + _msg->taskID() + " not found when receiving handshake request"); + onTaskError( + "onHandshakeRequest", std::move(error), _msg->from(), _msg->taskID(), "emptyResource"); + return; + } + auto handshakeReq = std::dynamic_pointer_cast(_msg); + auto const& curves = handshakeReq->curves(); + auto const& hashList = handshakeReq->hashList(); + ECDH_LOG(INFO) << LOG_DESC("onHandshakeRequest") << LOG_KV("supportedCurves", curves.size()) + << LOG_KV("supportedHashList", hashList.size()) << printPSIMessage(_msg); + std::set clientCurves(curves.begin(), curves.end()); + std::set clientHashList(hashList.begin(), hashList.end()); + + auto supportedCurves = g_PPCConfig.supportedCurves( + (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC); + auto supportedHashList = g_PPCConfig.supportedHashList( + (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC); + std::set localCurves(supportedCurves.begin(), supportedCurves.end()); + std::set localHashTypes(supportedHashList.begin(), supportedHashList.end()); + auto handshakeResponse = + m_msgFactory->createHandshakeResponse((uint32_t)PSIPacketType::HandshakeResponse); + bool curveSelected = false; + bool hashTypeSelected = false; + // sm-crypto enabled + if (g_PPCConfig.smCrypto()) + { + // try to select SM2 + if (clientCurves.count((int)ppc::protocol::ECCCurve::SM2) && + localCurves.count((int)ppc::protocol::ECCCurve::SM2)) + { + handshakeResponse->setCurve((int)ppc::protocol::ECCCurve::SM2); + curveSelected = true; + } + // try to select SM3 + if (clientHashList.count((int)ppc::protocol::HashImplName::SM3) && + localHashTypes.count((int)ppc::protocol::HashImplName::SM3)) + { + handshakeResponse->setHashType((int)ppc::protocol::HashImplName::SM3); + hashTypeSelected = true; + } + } + if (!curveSelected) + { + handshakeResponse->setCurve(selectCryptoAlgorithm(handshakeReq->curves(), localCurves)); + } + if (!hashTypeSelected) + { + handshakeResponse->setHashType( + selectCryptoAlgorithm(handshakeReq->hashList(), localHashTypes)); + } + auto ecdhTaskState = std::dynamic_pointer_cast(taskState); + auto selectedCurve = handshakeResponse->curve(); + auto selectedHashType = handshakeResponse->hashType(); + + auto dataResource = taskState->task()->selfParty()->dataResource(); + if (selectedCurve != -1 && selectedHashType != -1) + { + ECDH_LOG(INFO) << LOG_DESC("onHandshakeRequest success") + << LOG_KV("selectedCurve", (ppc::protocol::ECCCurve)selectedCurve) + << LOG_KV("selectedHashType", (ppc::protocol::HashImplName)selectedHashType); + auto ecdhCrypto = + m_config->ecdhCryptoFactory()->createECDHCrypto(selectedCurve, selectedHashType); + ecdhTaskState->setEcdhCrypto(ecdhCrypto); + } + else + { + auto errorMsg = "onHandshakeRequest failed for no curve/hashType selected"; + ECDH_LOG(WARNING) << LOG_DESC(errorMsg) << LOG_KV("selectedCurve", selectedCurve) + << LOG_KV("selectedHashType", selectedHashType); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandshakeFailed, errorMsg); + onTaskError("onHandshakeRequest", std::move(error), _msg->from(), _msg->taskID(), + dataResource ? dataResource->resourceID() : "emptyResource"); + return; + } + auto self = weak_from_this(); + auto startT = bcos::utcSteadyTime(); + m_config->generateAndSendPPCMessage( + _msg->from(), _msg->taskID(), handshakeResponse, + [self, _msg, taskState, dataResource](Error::Ptr&& _error) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error || _error->errorCode() == 0) + { + return; + } + // response failed + ECDH_LOG(ERROR) << LOG_DESC("onHandshakeRequest: send handshake response error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()) << printPSIMessage(_msg); + psi->onTaskError("onHandshakeRequest", std::move(_error), _msg->from(), _msg->taskID(), + dataResource ? dataResource->resourceID() : "emptyResource"); + }, + 0, + [self, _msg, startT, taskState, dataResource](bcos::Error::Ptr _error, + std::string const& _agencyID, ppc::front::PPCMessageFace::Ptr _ppcMsg, + ppc::front::ResponseFunc) { + auto psi = self.lock(); + if (!psi) + { + return; + } + if (!_error || _error->errorCode() == 0) + { + ECDH_LOG(INFO) << LOG_DESC( + "onHandshakeRequest: send handshake response and receive the " + "client-ack, begin triggerDataBlind") + << printTaskInfo(taskState->task()) + << LOG_KV("uuid", _ppcMsg->uuid()); + // triggerDataBlind when receiving the client-ack + psi->triggerDataBlind(taskState); + return; + } + // response failed + ECDH_LOG(ERROR) << LOG_DESC( + "onHandshakeRequest: send handshake response and receive the " + "client-ack error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)) + << LOG_KV("msg", _error->errorMessage()) << printPSIMessage(_msg); + psi->onTaskError("onHandshakeRequest", std::move(_error), _msg->from(), _msg->taskID(), + dataResource ? dataResource->resourceID() : "emptyResource"); + }); +} + +int EcdhPSIImpl::selectCryptoAlgorithm( + std::vector _clientSupportedAlgorithms, std::set _localSupportedAlgorithms) +{ + for (auto const& algorithm : _clientSupportedAlgorithms) + { + if (_localSupportedAlgorithms.count(algorithm)) + { + return algorithm; + } + } + // Note: -1 means select algorithm failed + return -1; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h new file mode 100644 index 00000000..4e8dad12 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIImpl.h + * @author: yujiechen + * @date 2022-12-27 + */ +#pragma once +#include "../psi-framework/PSIFramework.h" +#include "EcdhCache.h" +#include "EcdhPSIConfig.h" +#include "EcdhTaskState.h" + +namespace ppc::psi +{ +class EcdhPSIImpl : public PSIFramework, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + EcdhPSIImpl(EcdhPSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : PSIFramework( + _config->msgFactory(), _config->dataResourceLoader(), _config, "ecdh-psi", _idleTimeMs), + m_config(_config) + { + m_taskStateFactory = std::make_shared(); + m_cache = std::make_shared(_config); + } + + ~EcdhPSIImpl() override = default; + + void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + + // handle the psi message + void handlePSIMsg(PSIMessageInterface::Ptr _msg) override; + // should lock the party resource or not + bool needLockResource(int, int _partyType) override + { + return _partyType == (int)ppc::protocol::PartyType::Client; + } + +protected: + // receive the handshake response + void onHandshakeResponse(PSIMessageInterface::Ptr const& _msg) override; + // receive the handshake request + void onHandshakeRequest(PSIMessageInterface::Ptr const& _msg) override; + + // the psi-server and psi-client load data from DataResource and blind the plainData into cipher + virtual void blindData(TaskState::Ptr const& _taskState); + virtual void triggerDataBlind(TaskState::Ptr const& _taskState); + + // receive the blinded-data from client, the psi-server response the evaluated data + virtual void handleEvaluateRequest(PSIMessageInterface::Ptr _msg); + // receive the evaluate-response from the sever + virtual void handleEvaluateResponse(PSIMessageInterface::Ptr _msg); + + // receive the blinded-data from the server + virtual void handleServerBlindData(PSIMessageInterface::Ptr _msg); + // receive the SyncDataBatchCount request from the server + virtual void onRecvSyncDataInfoMsg(PSIMessageInterface::Ptr const& _msg); + + void runPSI(TaskState::Ptr const& _taskState); + + int selectCryptoAlgorithm( + std::vector _clientSupportedAlgorithms, std::set _localSupportedAlgorithms); + + // init the task-state + bool initTaskState(TaskState::Ptr const& _taskState); + + std::pair checkAndObtainTaskState( + PSIMessageInterface::Ptr _msg); + +protected: + EcdhPSIConfig::Ptr m_config; + EcdhCache::Ptr m_cache; + + // allow the output-path exists, for ut + bool m_enableOutputExists = false; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h new file mode 100644 index 00000000..0230c133 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIMessageFactory.h + * @author: yujiechen + * @date 2022-12-29 + */ +#pragma once +#include "../psi-framework/protocol/PSIMessage.h" +namespace ppc::psi +{ +class EcdhPSIMessageFactory : public PSIMessageFactoryImpl +{ +public: + using Ptr = std::shared_ptr; + EcdhPSIMessageFactory() = default; + ~EcdhPSIMessageFactory() override = default; + + PSIMessageInterface::Ptr decodePSIMessage(bcos::bytesConstRef _data) override + { + auto inner = [inner = ppctars::PSIMessage()]() mutable { return &inner; }; + tars::TarsInputStream input; + input.setBuffer((const char*)_data.data(), _data.size()); + inner()->readFrom(input); + switch (inner()->packetType) + { + case (uint32_t)ECDHPacketType::EvaluateRequest: + case (uint32_t)ECDHPacketType::EvaluateResponse: + case (uint32_t)ECDHPacketType::ServerBlindedData: + case (uint32_t)ECDHPacketType::SyncDataBatchInfo: + return std::make_shared(inner); + default: + return decodePSIBaseMessage(inner); + } + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h b/cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h new file mode 100644 index 00000000..1d1f0998 --- /dev/null +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhTaskState.h + * @author: yujiechen + * @date 2023-1-3 + */ +#pragma once +#include "../psi-framework/TaskState.h" +#include "ppc-framework/crypto/ECDHCrypto.h" +namespace ppc::psi +{ +class EcdhTaskState : public TaskState +{ +public: + using Ptr = std::shared_ptr; + EcdhTaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, + bool _onlySelfRun = false) + : TaskState(_task, std::move(_callback), _onlySelfRun) + {} + ~EcdhTaskState() override = default; + + // return copy here to ensure thread-safe + ppc::crypto::ECDHCrypto::Ptr ecdhCrypto() const + { + bcos::ReadGuard l(x_ecdhCrypto); + return m_ecdhCrypto; + } + // Note: here must lock, in-case of multiple tasks with the same task-id processed at almost the + // same time + void setEcdhCrypto(ppc::crypto::ECDHCrypto::Ptr const& _ecdhCrypto) + { + bcos::WriteGuard l(x_ecdhCrypto); + m_ecdhCrypto = _ecdhCrypto; + } + +private: + ppc::crypto::ECDHCrypto::Ptr m_ecdhCrypto; + mutable bcos::SharedMutex x_ecdhCrypto; +}; + +class EcdhTaskStateFactory : public TaskStateFactory +{ +public: + using Ptr = std::shared_ptr; + EcdhTaskStateFactory() : TaskStateFactory() {} + ~EcdhTaskStateFactory() override = default; + + TaskState::Ptr createTaskState(ppc::protocol::Task::ConstPtr const& _task, + TaskResponseCallback&& _callback, bool _onlySelfRun = false, + PSIConfig::Ptr _config = nullptr) override + { + return std::make_shared(_task, std::move(_callback), false); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/CMakeLists.txt b/cpp/ppc-psi/src/labeled-psi/CMakeLists.txt new file mode 100644 index 00000000..a0a3c07e --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/CMakeLists.txt @@ -0,0 +1,30 @@ +# for tars generator +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-psi/labeled-psi/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${LABELED_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST}) +target_include_directories(${LABELED_PSI_TARGET} PUBLIC + $) + +target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb TCMalloc) diff --git a/cpp/ppc-psi/src/labeled-psi/Common.h b/cpp/ppc-psi/src/labeled-psi/Common.h new file mode 100644 index 00000000..78ab15e3 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/Common.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-11-4 + */ +#pragma once + +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/PPCMessageFace.h" + +#include +#include + + +namespace ppc::psi +{ + +DERIVE_PPC_EXCEPTION(ConfigPowersDagException); +DERIVE_PPC_EXCEPTION(TooManyItemsException); +DERIVE_PPC_EXCEPTION(ResultPackageException); +DERIVE_PPC_EXCEPTION(OverwriteItemException); +DERIVE_PPC_EXCEPTION(InsertItemException); +DERIVE_PPC_EXCEPTION(RemoveItemException); +DERIVE_PPC_EXCEPTION(LabelExceedsSizeException); +DERIVE_PPC_EXCEPTION(NonceExceedsSizeException); +DERIVE_PPC_EXCEPTION(RetrieveItemException); +DERIVE_PPC_EXCEPTION(RetrieveLabelException); +DERIVE_PPC_EXCEPTION(ExtracteRelinKeysException); +DERIVE_PPC_EXCEPTION(ExtracteCiphertextException); +DERIVE_PPC_EXCEPTION(QueryPowersException); +DERIVE_PPC_EXCEPTION(PowerDagException); + +#define LABELED_PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: Labeled-PSI") + +#define CUCKOO_TABLE_INSERT_ATTEMPTS 500 +#define MAX_LABEL_BYTE 1024 +#define MAX_QUERY_SIZE 8 + +constexpr static std::string_view CMD_SETUP_SENDER_DB = "setup_sender_db"; +constexpr static std::string_view CMD_RUN_LABELED_PSI = "run_labeled_psi"; +constexpr static std::string_view CMD_SAVE_SENDER_CACHE = "save_sender_cache"; +constexpr static std::string_view CMD_LOAD_SENDER_CACHE = "load_sender_cache"; +constexpr static std::string_view CMD_ADD_ITEMS = "add_items"; +constexpr static std::string_view CMD_UPDATE_LABELS = "update_labels"; +constexpr static std::string_view CMD_DELETE_ITEMS = "delete_items"; + +enum class LabeledPSIMessageType : uint8_t +{ + PARAMS_REQUEST = 0x01, + PARAMS_RESPONSE = 0x02, + OPRF_BLINDED_ITEMS = 0x03, + OPRF_EVALUATED_ITEMS = 0x04, + QUERY = 0x05, + RESPONSE = 0x06 +}; + +enum class LabeledPSIRetCode : int +{ + ON_EXCEPTION = -2000, + SENDER_NOT_READY = -2001, + CUCKOO_HASH_ERROR = -2002, + MISSING_LABEL_DATA = -2003, + LACK_LABEL_DATA = -2004, + UNDEFINED_TASK_ROLE = -2005, + UNDEFINED_COMMAND = -2006, + DATA_FORMAT_ERROR = -2007, + INVALID_TASK_PARAM = -2008, + SETUP_SENDER_ERROR = -2009, + LOAD_SENDER_CACHE_ERROR = -2010 +}; + +enum class LabeledPSICommand : int +{ + SETUP_SENDER_DB, + RUN_LABELED_PSI, + SAVE_SENDER_CACHE, + LOAD_SENDER_CACHE, + ADD_ITEMS, + UPDATE_LABELS, + DELETE_ITEMS +}; + +template +bool hasNZeros(T* ptr, size_t count) +{ + return std::all_of(ptr, ptr + count, [](auto a) { return a == T(0); }); +} + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h b/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h new file mode 100644 index 00000000..fc2524bc --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIConfig.h + * @author: shawnhe + * @date 2022-11-3 + */ + +#pragma once + +#include "Common.h" +#include "ppc-crypto/src/oprf/EcdhOprf.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/crypto/Oprf.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/PSIConfig.h" + +#include +#include + +namespace ppc::psi +{ +class LabeledPSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + + LabeledPSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front, + ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes, + front::PPCMessageFactory::Ptr _msgFactory = std::make_shared()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC, _selfPartyID, std::move(_front), + _msgFactory, _dataResourceLoader, _holdingMessageMinutes), + m_cryptoBox(std::move(_cryptoBox)), + m_threadPool(std::move(_threadPool)), + m_messageFactory(_msgFactory) + { + m_oprfServer = std::make_shared( + sizeof(apsi::Item::value_type) + sizeof(apsi::LabelKey), m_cryptoBox->hashImpl(), + m_cryptoBox->eccCrypto()); + } + + virtual ~LabeledPSIConfig() = default; + +public: + crypto::Hash::Ptr const& hash() const { return m_cryptoBox->hashImpl(); } + crypto::EccCrypto::Ptr const& eccCrypto() const { return m_cryptoBox->eccCrypto(); } + bcos::ThreadPool::Ptr const& threadPool() const { return m_threadPool; } + crypto::OprfServer::Ptr const& oprfServer() const { return m_oprfServer; } + +private: + ppc::crypto::CryptoBox::Ptr m_cryptoBox; + bcos::ThreadPool::Ptr m_threadPool; + + crypto::OprfServer::Ptr m_oprfServer; + front::PPCMessageFactory::Ptr m_messageFactory; +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h b/cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h new file mode 100644 index 00000000..ba8d20ee --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIFactory.h + * @author: shawnhe + * @date 2022-11-14 + */ +#pragma once +#include "LabeledPSIImpl.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::psi +{ +class LabeledPSIFactory +{ +public: + using Ptr = std::shared_ptr; + LabeledPSIFactory() = default; + virtual ~LabeledPSIFactory() = default; + + virtual LabeledPSIImpl::Ptr buildLabeledPSI(std::string const& _selfParty, + ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader, + int _holdingMessageMinutes) + { + auto config = + std::make_shared(_selfParty, std::move(_front), std::move(_cryptoBox), + std::move(_threadPool), std::move(_dataResourceLoader), _holdingMessageMinutes); + return std::make_shared(config); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp new file mode 100644 index 00000000..bac2fa70 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp @@ -0,0 +1,794 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIImpl.cpp + * @author: shawnhe + * @date 2022-11-7 + */ + +#include "LabeledPSIImpl.h" +#include "Common.h" +#include "LabeledPSI.h" +#include "core/LabeledPSIParams.h" +#include "core/SenderDB.h" +#include "core/TaskCommand.h" +#include "ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::tools; +using namespace ppc::crypto; +using namespace ppc::io; + +LabeledPSIImpl::LabeledPSIImpl(LabeledPSIConfig::Ptr _config, unsigned _idleTimeMs) + : Worker("LabeledPSI", _idleTimeMs), + TaskGuarder(_config, PSIAlgorithmType::LABELED_PSI_2PC, "Labeled-PSI-Timer"), + m_config(std::move(_config)), + m_msgQueue(std::make_shared()), + m_worker(std::make_shared("senderDB-worker", 1)) +{ + m_sender = make_shared( + m_config, [&](const std::string& _taskID, bcos::Error::Ptr _error) { + if (_error && _error->errorCode()) + { + onSelfError(_taskID, std::move(_error), true); + } + else + { + onSenderTaskDone(_taskID); + } + }); +} + +// run task +void LabeledPSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +{ + auto onTaskFinished = [self = weak_from_this(), taskID = _task->id(), _onTaskFinished]( + ppc::protocol::TaskResult::Ptr&& _result) { + try + { + auto result = std::move(_result); + _onTaskFinished(std::move(result)); + LABELED_PSI_LOG(INFO) << LOG_DESC("finish a task") << LOG_KV("taskID", taskID); + auto psi = self.lock(); + if (!psi) + { + return; + } + + // erase the taskInfo from the gateway + psi->m_config->front()->eraseTaskInfo(taskID); + } + catch (std::exception& e) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("handle callback error after finishing task") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + }; + + + // there is always a self-help party + auto role = _task->selfParty()->partyIndex(); + if (role == uint16_t(PartyType::Client)) + { + auto error = checkTask(_task, 2, true, true, false); + if (error) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(_task); + // notice peer to cancel task + noticePeerToFinish(_task); + + auto result = std::make_shared(_task->id()); + result->setError(std::move(error)); + onTaskFinished(std::move(result)); + return; + } + + // add pending task + auto taskState = m_taskStateFactory->createTaskState(_task, std::move(onTaskFinished)); + taskState->setPeerID(getPeerID(_task)); + taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + + psi->noticePeerToFinish(_task); + }); + addPendingTask(taskState); + + auto oprfClient = std::make_shared( + sizeof(apsi::Item::value_type) + sizeof(apsi::LabelKey), m_config->hash(), + m_config->eccCrypto()); + auto receiver = std::make_shared(m_config, taskState, oprfClient); + receiver->asyncRunTask(); + addReceiver(receiver); + + // notify the taskInfo to the front + error = m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + if (error && error->errorCode()) + { + onSelfError(_task->id(), error, true); + } + } + else if (role == uint16_t(PartyType::Server)) + { + asyncRunSenderTask(_task, std::move(onTaskFinished)); + } + else + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("undefined task role") << unsigned(role); + // notice peer to cancel task + noticePeerToFinish(_task); + + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::UNDEFINED_TASK_ROLE, "undefined task role")); + onTaskFinished(std::move(result)); + noticePeerToFinish(_task); + return; + } +} + +// register to the front to get the message related to labeled-psi +void LabeledPSIImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) +{ + try + { + m_msgQueue->push(_msg); + + // notify to handle the message + wakeupWorker(); + } + catch (std::exception const& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + +void LabeledPSIImpl::start() +{ + if (m_started) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("The LabeledPSI has already been started"); + return; + } + LABELED_PSI_LOG(INFO) << LOG_DESC("Start the LabeledPSI"); + m_started = true; + + // start a thread to execute task + startWorking(); + + startPingTimer(); +} + +void LabeledPSIImpl::stop() +{ + if (!m_started) + { + return; + } + LABELED_PSI_LOG(INFO) << LOG_DESC("Stop LabeledPSI"); + m_started = false; + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } + stopPingTimer(); + + LABELED_PSI_LOG(INFO) << LOG_DESC("LabeledPSI stopped"); +} + +void LabeledPSIImpl::onReceivedErrorNotification(const std::string& _taskID) +{ + // finish the task while the peer is failed + auto taskState = findPendingTask(_taskID); + if (taskState) + { + taskState->onPeerNotifyFinish(); + } +} + +void LabeledPSIImpl::onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +{ + LABELED_PSI_LOG(WARNING) << LOG_DESC("onSelfError") << LOG_KV("task", _taskID) + << LOG_KV("error", _error->errorMessage()) + << LOG_KV("noticePeer", _noticePeer); + try + { + auto taskState = findPendingTask(_taskID); + if (!taskState) + { + return; + } + + auto result = std::make_shared(taskState->task()->id()); + result->setError(std::move(_error)); + taskState->onTaskFinished(result, _noticePeer); + + wakeupWorker(); + } + catch (std::exception& e) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("onSelfError") + << LOG_KV("exception", boost::diagnostic_information(e)); + } +} + +void LabeledPSIImpl::asyncRunSenderTask( + const ppc::protocol::Task::ConstPtr& _task, TaskResponseCallback&& _onTaskFinished) +{ + TaskCommand::Ptr taskCommand; + try + { + // parse the task command + taskCommand = std::make_shared(_task->param()); + } + catch (const std::exception& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("asyncRunSenderTask exception") << printTaskInfo(_task) + << LOG_KV("error", boost::diagnostic_information(e)); + + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while asyncRunSenderTask: " + boost::diagnostic_information(e))); + _onTaskFinished(std::move(result)); + noticePeerToFinish(_task); + return; + } + + switch (taskCommand->command()) + { + case (int)LabeledPSICommand::SETUP_SENDER_DB: + { + if (m_senderReady) + { + LABELED_PSI_LOG(WARNING) + << LOG_DESC("sender has been set up: ") << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::SETUP_SENDER_ERROR, "sender has been set up")); + _onTaskFinished(std::move(result)); + return; + } + + auto error = checkTask(_task, 1, true, false, false); + if (error) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::move(error)); + _onTaskFinished(std::move(result)); + return; + } + + auto taskState = + m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), true); + addPendingTask(taskState); + + m_worker->enqueue([self = weak_from_this(), _task, taskCommand]() { + auto psi = self.lock(); + if (psi) + { + auto labelByteCount = taskCommand->args()[0]; + psi->setupSenderDB(_task, labelByteCount); + } + }); + break; + } + case (int)LabeledPSICommand::RUN_LABELED_PSI: + { + if (!m_senderReady) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("sender not ready: ") << printTaskInfo(_task); + + // notice peer to cancel task + noticePeerToFinish(_task); + + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::SENDER_NOT_READY, "sender not ready")); + _onTaskFinished(std::move(result)); + return; + } + + auto error = checkTask(_task, 2, false, false, false, false); + if (error) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(_task); + // notice peer to cancel task + noticePeerToFinish(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::move(error)); + _onTaskFinished(std::move(result)); + return; + } + + auto taskState = m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished)); + taskState->setPeerID(getPeerID(_task)); + taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + + psi->noticePeerToFinish(_task); + }); + addPendingTask(taskState); + + // notify the taskInfo to the front + error = m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + if (error && error->errorCode()) + { + onSelfError(_task->id(), error, true); + } + break; + } + case (int)LabeledPSICommand::SAVE_SENDER_CACHE: + { + if (!m_senderReady) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("sender not ready: ") << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::SENDER_NOT_READY, "sender not ready")); + _onTaskFinished(std::move(result)); + return; + } + + auto error = checkTask(_task, 1, false, true, false); + if (error) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::move(error)); + _onTaskFinished(std::move(result)); + noticePeerToFinish(_task); + return; + } + + auto taskState = + m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), true); + addPendingTask(taskState); + + m_worker->enqueue([self = weak_from_this(), _task]() { + auto psi = self.lock(); + if (psi) + { + psi->saveSenderCache(_task); + } + }); + break; + } + case (int)LabeledPSICommand::LOAD_SENDER_CACHE: + { + if (m_senderReady) + { + LABELED_PSI_LOG(WARNING) + << LOG_DESC("sender has been set up: ") << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::LOAD_SENDER_CACHE_ERROR, "sender has been set up")); + _onTaskFinished(std::move(result)); + return; + } + + auto error = checkTask(_task, 1, true, false, false); + if (error) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage()) + << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::move(error)); + _onTaskFinished(std::move(result)); + noticePeerToFinish(_task); + return; + } + + auto taskState = + m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), true); + addPendingTask(taskState); + m_worker->enqueue([self = weak_from_this(), _task]() { + auto psi = self.lock(); + if (psi) + { + psi->loadSenderCache(_task); + } + }); + break; + } + default: + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("unsupported command: ") << printTaskInfo(_task); + auto result = std::make_shared(_task->id()); + result->setError(std::make_shared( + (int)LabeledPSIRetCode::UNDEFINED_COMMAND, "unsupported command")); + _onTaskFinished(std::move(result)); + noticePeerToFinish(_task); + break; + } + } +} + +void LabeledPSIImpl::setupSenderDB(const ppc::protocol::Task::ConstPtr& _task, int _labelByteCount) +{ + try + { + auto dataResource = _task->selfParty()->dataResource(); + auto reader = loadReader(_task->id(), dataResource, DataSchema::Bytes, 2); + + DataBatch::Ptr items, labels; + if (reader->type() == ppc::protocol::DataResourceType::MySQL) + { + items = reader->next(0, DataSchema::Bytes); + labels = reader->next(1, DataSchema::Bytes); + } + else + { + auto lines = reader->next(-1, DataSchema::Bytes); + items = std::make_shared(); + items->setDataSchema(ppc::io::DataSchema::Bytes); + labels = std::make_shared(); + labels->setDataSchema(ppc::io::DataSchema::Bytes); + for (uint32_t i = 0; i < lines->size(); ++i) + { + auto line = lines->getBytes(i); + std::vector data; + boost::split(data, line, boost::is_any_of(",")); + if (data.size() != 2) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)LabeledPSIRetCode::DATA_FORMAT_ERROR, + "the data format should be: key,value")); + } + + items->append(bcos::bytes(data[0].begin(), data[0].end())); + labels->append(bcos::bytes(data[1].begin(), data[1].end())); + } + } + + if (items->size() != labels->size()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)LabeledPSIRetCode::DATA_FORMAT_ERROR, + "the number of items and labels is not the same")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("finish loading item-label") + << LOG_KV("count", items->size()); + + auto psiParams = getPsiParams(items->size()); + + m_senderDB = std::make_shared( + psiParams, m_config->oprfServer(), _labelByteCount, 16, false); + + std::set tmp; + for (uint32_t i = 0; i < items->size(); i++) + { + tmp.insert(items->getBytes(i)); + } + if (tmp.size() < items->size()) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int)LabeledPSIRetCode::DATA_FORMAT_ERROR, "duplicate key found")); + } + + m_senderDB->setData(items, labels); + + m_sender->setSenderDB(m_senderDB); + m_senderReady.exchange(true); + + LABELED_PSI_LOG(INFO) << LOG_DESC("setupSenderDB done") << printTaskInfo(_task); + + onSenderTaskDone(_task->id()); + } + catch (bcos::Error const& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("setupSenderDB exception") << printTaskInfo(_task) + << LOG_KV("code", e.errorCode()) + << LOG_KV("msg", e.errorMessage()); + onSelfError( + _task->id(), std::make_shared(e.errorCode(), e.errorMessage()), false); + } + catch (const std::exception& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("setupSenderDB exception") << printTaskInfo(_task) + << LOG_KV("error", boost::diagnostic_information(e)); + + onSelfError(_task->id(), + std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while setupSenderDB: " + boost::diagnostic_information(e)), + false); + } +} + +void LabeledPSIImpl::saveSenderCache(const ppc::protocol::Task::ConstPtr& _task) +{ + try + { + auto dataResource = _task->selfParty()->dataResource(); + + LineWriter::Ptr writer; + if (!m_enableOutputExists) + { + // Note: if the output-resource already exists, will throw exception + m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc()); + } + writer = m_config->dataResourceLoader()->loadWriter(dataResource->outputDesc(), true); + + bcos::bytes out; + m_senderDB->saveToBytes(out); + + writer->writeBytes(bcos::ref(out)); + writer->flush(); + writer->close(); + + onSenderTaskDone(_task->id()); + LABELED_PSI_LOG(INFO) << LOG_DESC("saveSenderCache done") << printTaskInfo(_task); + } + catch (bcos::Error const& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("saveSenderCache exception") << printTaskInfo(_task) + << LOG_KV("code", e.errorCode()) + << LOG_KV("msg", e.errorMessage()); + onSelfError( + _task->id(), std::make_shared(e.errorCode(), e.errorMessage()), false); + } + catch (const std::exception& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("saveSenderCache exception") << printTaskInfo(_task) + << LOG_KV("error", boost::diagnostic_information(e)); + + onSelfError(_task->id(), + std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while saveSenderCache: " + boost::diagnostic_information(e)), + false); + } +} + +void LabeledPSIImpl::loadSenderCache(const ppc::protocol::Task::ConstPtr& _task) +{ + try + { + auto dataResource = _task->selfParty()->dataResource(); + auto reader = m_config->dataResourceLoader()->loadReader( + dataResource->desc(), ppc::io::DataSchema::Bytes, false); + + auto cache = reader->readBytes(); + m_senderDB = SenderDB::loadFromBytes(m_config->oprfServer(), cache); + + m_senderReady.exchange(true); + m_sender->setSenderDB(m_senderDB); + onSenderTaskDone(_task->id()); + LABELED_PSI_LOG(INFO) << LOG_DESC("loadSenderCache done") << printTaskInfo(_task); + } + catch (bcos::Error const& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("loadSenderCache exception") << printTaskInfo(_task) + << LOG_KV("code", e.errorCode()) + << LOG_KV("msg", e.errorMessage()); + onSelfError( + _task->id(), std::make_shared(e.errorCode(), e.errorMessage()), false); + } + catch (const std::exception& e) + { + LABELED_PSI_LOG(WARNING) << LOG_DESC("loadSenderCache exception") << printTaskInfo(_task) + << LOG_KV("error", boost::diagnostic_information(e)); + + onSelfError(_task->id(), + std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while loadSenderCache: " + boost::diagnostic_information(e)), + false); + } +} + +// labeled-psi main processing function +// for ut to make this function public +void LabeledPSIImpl::executeWorker() +{ + checkFinishedTask(); + auto result = m_msgQueue->tryPop(c_popWaitMs); + if (result.first) + { + handleReceivedMessage(result.second); + return; + } + waitSignal(); +} + +void LabeledPSIImpl::checkFinishedTask() +{ + std::set finishedTask; + { + bcos::WriteGuard l(x_pendingTasks); + if (m_pendingTasks.empty()) + { + return; + } + + for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();) + { + auto task = it->second; + if (task->finished()) + { + finishedTask.insert(it->first); + } + it++; + } + } + for (auto& taskID : finishedTask) + { + removeReceiver(taskID); + removePendingTask(taskID); + } +} + +void LabeledPSIImpl::onSenderTaskDone(const std::string& _taskID) +{ + auto taskState = findPendingTask(_taskID); + if (taskState) + { + taskState->onTaskFinished(std::make_shared(_taskID), false); + } +} + +void LabeledPSIImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message) +{ + m_config->threadPool()->enqueue([self = weak_from_this(), _message]() { + try + { + auto psi = self.lock(); + if (!psi) + { + return; + } + switch (int(_message->messageType())) + { + case int(CommonMessageType::ErrorNotification): + { + psi->onReceivedErrorNotification(_message->taskID()); + break; + } + case int(CommonMessageType::PingPeer): + { + break; + } + case int(LabeledPSIMessageType::PARAMS_REQUEST): + { + psi->onReceivePsiParamsRequest(_message); + break; + } + case int(LabeledPSIMessageType::PARAMS_RESPONSE): + { + psi->onReceivePsiParamsResponse(_message); + break; + } + case int(LabeledPSIMessageType::OPRF_BLINDED_ITEMS): + { + psi->onReceiveBlindedItems(_message); + break; + } + case int(LabeledPSIMessageType::OPRF_EVALUATED_ITEMS): + { + psi->onReceiveEvaluatedItems(_message); + break; + } + case int(LabeledPSIMessageType::QUERY): + { + psi->onReceiveQuery(_message); + break; + } + case int(LabeledPSIMessageType::RESPONSE): + { + psi->onReceiveResponse(_message); + break; + } + default: + { + LABELED_PSI_LOG(WARNING) + << LOG_DESC("unsupported messageType ") << unsigned(_message->messageType()); + break; + } + } + } + catch (std::exception const& e) + { + LABELED_PSI_LOG(WARNING) + << LOG_DESC("handleReceivedMessage exception") + << LOG_KV("type", unsigned(_message->messageType())) << printPPCMsg(_message) + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + + +void LabeledPSIImpl::onReceivePsiParamsRequest(ppc::front::PPCMessageFace::Ptr _message) +{ + if (!m_senderReady) + { + noticePeerToFinish(_message->taskID(), _message->sender()); + } + else + { + auto taskID = _message->taskID(); + if (findPendingTask(taskID)) + { + m_sender->handlePsiParamsRequest(std::move(_message)); + } + } +} + +void LabeledPSIImpl::onReceivePsiParamsResponse(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->handlePsiParams(std::move(_message)); + } +} + +void LabeledPSIImpl::onReceiveBlindedItems(ppc::front::PPCMessageFace::Ptr _message) +{ + auto taskID = _message->taskID(); + if (findPendingTask(taskID)) + { + m_sender->handleBlindedItems(std::move(_message)); + } +} + +void LabeledPSIImpl::onReceiveEvaluatedItems(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->handleEvaluatedItems(std::move(_message)); + } +} + +void LabeledPSIImpl::onReceiveQuery(ppc::front::PPCMessageFace::Ptr _message) +{ + auto taskID = _message->taskID(); + if (findPendingTask(taskID)) + { + m_sender->handleQuery(std::move(_message)); + } +} + + +void LabeledPSIImpl::onReceiveResponse(ppc::front::PPCMessageFace::Ptr _message) +{ + auto receiver = findReceiver(_message->taskID()); + if (receiver) + { + receiver->handleOneResponse(std::move(_message)); + } +} diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h new file mode 100644 index 00000000..bafa00c1 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIImpl.h + * @author: shawnhe + * @date 2022-11-7 + */ + +#pragma once +#include +#include +#include +#include +#include + +#include "../psi-framework/TaskGuarder.h" +#include "Common.h" +#include "LabeledPSIConfig.h" +#include "core/LabeledPSIReceiver.h" +#include "core/LabeledPSISender.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-front/ppc-front/PPCChannel.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-tools/src/common/TransTools.h" + +namespace ppc::psi +{ +class LabeledPSIImpl : public bcos::Worker, + public TaskGuarder, + public TaskFrameworkInterface, + public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + using LabeledPSIMsgQueue = bcos::ConcurrentQueue; + using LabeledPSIMsgQueuePtr = std::shared_ptr; + + LabeledPSIImpl(LabeledPSIConfig::Ptr _config, unsigned _idleTimeMs = 0); + + virtual ~LabeledPSIImpl() = default; + + // run task + void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + + // register to the front to get the message related to labeled-psi + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; + + void start() override; + void stop() override; + + void onReceivedErrorNotification(const std::string& _taskID) override; + void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override; + + // labeled-psi main processing function + // for ut to make this function public + void executeWorker() override; + +protected: + void asyncRunSenderTask( + const ppc::protocol::Task::ConstPtr& _task, TaskResponseCallback&& _onTaskFinished); + + void checkFinishedTask(); + + void onSenderTaskDone(const std::string& _taskID); + + // init senderDB + void setupSenderDB(const ppc::protocol::Task::ConstPtr& _task, int _labelByteCount); + void saveSenderCache(const ppc::protocol::Task::ConstPtr& _task); + void loadSenderCache(const ppc::protocol::Task::ConstPtr& _task); + + void handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message); + void onReceivePsiParamsRequest(ppc::front::PPCMessageFace::Ptr _message); + void onReceivePsiParamsResponse(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveBlindedItems(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveEvaluatedItems(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveQuery(ppc::front::PPCMessageFace::Ptr _message); + void onReceiveResponse(ppc::front::PPCMessageFace::Ptr _message); + + LabeledPSIReceiver::Ptr findReceiver(const std::string& _taskID) + { + bcos::ReadGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + return it->second; + } + return nullptr; + } + void addReceiver(LabeledPSIReceiver::Ptr _receiver) + { + bcos::WriteGuard l(x_receivers); + m_receivers[_receiver->taskID()] = _receiver; + } + void removeReceiver(const std::string& _taskID) + { + bcos::WriteGuard l(x_receivers); + auto it = m_receivers.find(_taskID); + if (it != m_receivers.end()) + { + m_receivers.erase(it); + } + } + +protected: + // allow the output-path exists, for ut + bool m_enableOutputExists = false; + +private: + void waitSignal() + { + boost::unique_lock l(x_signal); + m_signal.wait_for(l, boost::chrono::milliseconds(5)); + } + + void wakeupWorker() { m_signal.notify_all(); } + + LabeledPSIConfig::Ptr m_config; + LabeledPSIMsgQueuePtr m_msgQueue; + bcos::ThreadPool::Ptr m_worker; + + LabeledPSISender::Ptr m_sender; + std::atomic m_senderReady{false}; + + std::unordered_map m_receivers; + mutable bcos::SharedMutex x_receivers; + + SenderDB::Ptr m_senderDB; + + bool m_started = false; + boost::condition_variable m_signal; + boost::mutex x_signal; + + const int c_popWaitMs = 5; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp b/cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp new file mode 100644 index 00000000..2cbb977b --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp @@ -0,0 +1,1287 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @reference https://github.com/microsoft/APSI/sender/bin_bundle.cpp + * @license MIT license + * @change change the implementation of load and save + * + * @file Binbundle.cpp + * @author: shawnhe + * @date 2022-12-18 + * + */ + +#include "BinBundle.h" + +// STD +#include +#include +#include +#include +#include + +// APSI +#include "apsi/bin_bundle.h" +#include "apsi/thread_pool_mgr.h" +#include "apsi/util/interpolate.h" +#include "apsi/util/utils.h" + +// SEAL +#include "seal/util/defines.h" + +using namespace std; +using namespace seal; +using namespace seal::util; +using namespace apsi; +using namespace apsi::sender::util; +using namespace ppc::psi; + +/** +Helper function. Determines if a field element is present in a bin. +*/ +bool is_present(const vector& bin, felt_t element) +{ + return bin.end() != find(bin.begin(), bin.end(), element); +} + +/** +Helper function. Determines if a field element is present in a bin. +*/ +bool is_present(const vector& bin, const CuckooFilter& filter, felt_t element) +{ + // Check if the key is already in the current bin. + if (filter.contains(element)) + { + // Perform a linear search to determine true/false positives + return is_present(bin, element); + } + + return false; +} + +/** +Helper function. Returns an iterator pointing to the given field element in the bin if +found and bin.end() otherwise. +*/ +template +auto get_iterator(BinT& bin, const CuckooFilter& filter, felt_t element) +{ + if (filter.contains(element)) + { + return find(bin.begin(), bin.end(), element); + } + + return bin.end(); +} + +void try_clear_irrelevant_bits(const EncryptionParameters& parms, Ciphertext& ciphertext) +{ + // If the parameter set has only one prime, we can compress the ciphertext by + // setting low-order bits to zero. This effectively maxes out the noise, but that + // doesn't matter as long as we don't use quite all noise budget. + if (parms.coeff_modulus().size() == 1) + { + // The number of data bits we need to have left in each ciphertext coefficient + int compr_coeff_bit_count = parms.plain_modulus().bit_count() + + get_significant_bit_count(parms.poly_modulus_degree()) + // Being pretty aggressive here + - 1; + + int coeff_mod_bit_count = parms.coeff_modulus()[0].bit_count(); + + // The number of bits to set to zero + int irrelevant_bit_count = coeff_mod_bit_count - compr_coeff_bit_count; + + // Can compression achieve anything? + if (irrelevant_bit_count > 0) + { + // Mask for zeroing out the irrelevant bits + uint64_t mask = ~((uint64_t(1) << irrelevant_bit_count) - 1); + seal_for_each_n(iter(ciphertext), ciphertext.size(), [&](auto&& I) { + // We only have a single RNS component so dereference once more + seal_for_each_n(*I, parms.poly_modulus_degree(), [&](auto& J) { J &= mask; }); + }); + } + } +} + +/** +Evaluates the polynomial on the given ciphertext. We don't compute the powers of the input +ciphertext C ourselves. Instead we assume they've been precomputed and accept the powers: +(C, C², C³, ...) as input. The number of powers provided MUST be at least +plaintext_polyn_coeffs_.size()-1. +*/ +Ciphertext BatchedPlaintextPolyn::eval( + const vector& ciphertext_powers, MemoryPoolHandle& pool) const +{ +#ifdef SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT + static_assert(false, "SEAL must be built with SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF"); +#endif + // We need to have enough ciphertext powers to evaluate this polynomial + if (ciphertext_powers.size() < max(batched_coeffs.size(), 2)) + { + throw invalid_argument("not enough ciphertext powers available"); + } + + auto seal_context = crypto_context.seal_context(); + auto evaluator = crypto_context.evaluator(); + + // We know now that ciphertext_powers is non-empty so read the parms_id from the first + // one; they should all be the same. + const auto& encode_parms_id = ciphertext_powers[1].parms_id(); + + // Lowest degree terms are stored in the lowest index positions in vectors. + // Specifically, ciphertext_powers[1] is the first power of the ciphertext data, but + // batched_coeffs[0] is the constant coefficient. + // + // Because the plaintexts in batched_coeffs can be identically zero, SEAL should be + // built with SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF. We create a result ciphertext + // that is identically zero and set its NTT form flag to true so the additions below + // will work. + Ciphertext result(pool); + result.resize(*seal_context, encode_parms_id, 2); + result.is_ntt_form() = true; + Ciphertext temp(pool); + Plaintext coeff(pool); + for (size_t deg = 1; deg < batched_coeffs.size(); deg++) + { + coeff.unsafe_load(*seal_context, + reinterpret_cast(batched_coeffs[deg].data()), + batched_coeffs[deg].size()); + evaluator->multiply_plain(ciphertext_powers[deg], coeff, temp, pool); + evaluator->add_inplace(result, temp); + } + + // Need to transform back from NTT form before we can add the constant coefficient. The + // constant coefficient is specifically not in NTT form so this can work. + evaluator->transform_from_ntt_inplace(result); + coeff.unsafe_load(*seal_context, reinterpret_cast(batched_coeffs[0].data()), + batched_coeffs[0].size()); + evaluator->add_plain_inplace(result, coeff); + + // Make the result as small as possible by modulus switching and possibly clearing + // irrelevant bits. + while (result.parms_id() != seal_context->last_parms_id()) + { + evaluator->mod_switch_to_next_inplace(result, pool); + } + try_clear_irrelevant_bits(seal_context->last_context_data()->parms(), result); + + return result; +} + +/** +Evaluates the polynomial on the given ciphertext using the Paterson-Stockmeyer algorithm, +as long as it requires less computation than the standard evaluation function above. +The algorithm computes h+1 inner polynomials on low powers (C^1 to C^{l-1}). +Each inner polynomial is then multiplied by the corresponding high power. +The parameters l and h are determined according to the degree of the polynomial and the +number of splits in order to minimize the computation complexity. + +Evaluated polynomial a_0 + a_1*C + a_2*C^2 + ... + C^degree + +Inner polynomials: a_{l*i} + a_{l*i+1}*C + ... + a_{l*i+l-1}*C^{l-1} (for i=0,...,h-1) + and: a_{l*h} + a_{l*h+1}*C + ... + a_{l*h+degree%l}*C^{degree%l} (for i=h) + +Low powers: C^{1}, ..., C^{l-1} +High powers: C^{1*l}, ..., C^{l*h} +*/ +Ciphertext BatchedPlaintextPolyn::eval_patstock(const CryptoContext& eval_crypto_context, + const vector& ciphertext_powers, size_t ps_low_degree, MemoryPoolHandle& pool) const +{ +#ifdef SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT + static_assert(false, "SEAL must be built with SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF"); +#endif + // We need to have enough ciphertext powers to evaluate this polynomial + if (ciphertext_powers.size() < max(batched_coeffs.size(), 2)) + { + throw invalid_argument("not enough ciphertext powers available"); + } + + // This function should not be called when the low-degree is 1 + size_t degree = batched_coeffs.size() - 1; + if (ps_low_degree <= 1 || ps_low_degree >= degree) + { + throw invalid_argument( + "ps_low_degree must be greater than 1 and less than the " + "size of batched_coeffs"); + } + + auto seal_context = eval_crypto_context.seal_context(); + auto evaluator = eval_crypto_context.evaluator(); + auto relin_keys = eval_crypto_context.relin_keys(); + bool relinearize = eval_crypto_context.seal_context()->using_keyswitching(); + + auto high_powers_parms_id = get_parms_id_for_chain_idx(*crypto_context.seal_context(), 1); + + // This is the number of high-degree powers we have: the first high-degree is + // ps_low_degree + 1 and the rest are multiples of that up to (but not exceeding) the + // total degree. + size_t ps_high_degree = ps_low_degree + 1; + size_t ps_high_degree_powers = degree / ps_high_degree; + + // Lowest degree terms are stored in the lowest index positions in vectors. + // Specifically, ciphertext_powers[1] is the first power of the ciphertext data, but + // batched_coeffs[0] is the constant coefficient. + // + // Because the plaintexts in batched_coeffs can be identically zero, SEAL should be + // built with SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF. We create a result ciphertext + // that is identically zero and set its NTT form flag to true so the additions below + // will work. The ciphertext here will have three components; we relinearize only at the + // end. + Ciphertext result(pool); + result.resize(*seal_context, high_powers_parms_id, 3); + result.is_ntt_form() = false; + + // Temporary variables + Ciphertext temp(pool); + Ciphertext temp_in(pool); + Plaintext coeff(pool); + + // Calculate polynomial for i=1,...,ps_high_degree_powers-1 + for (size_t i = 1; i < ps_high_degree_powers; i++) + { + // Evaluate inner polynomial. The free term is left out and added later on. + // The evaluation result is stored in temp_in. + for (size_t j = 1; j < ps_high_degree; j++) + { + coeff.unsafe_load(*seal_context, + reinterpret_cast(batched_coeffs[i * ps_high_degree + j].data()), + batched_coeffs[i * ps_high_degree + j].size()); + + evaluator->multiply_plain(ciphertext_powers[j], coeff, temp, pool); + + if (j == 1) + { + temp_in = temp; + } + else + { + evaluator->add_inplace(temp_in, temp); + } + } + + // Transform inner polynomial to coefficient form + evaluator->transform_from_ntt_inplace(temp_in); + evaluator->mod_switch_to_inplace(temp_in, high_powers_parms_id); + + // The high powers are already in coefficient form + evaluator->multiply_inplace(temp_in, ciphertext_powers[i * ps_high_degree], pool); + evaluator->add_inplace(result, temp_in); + } + + // Calculate polynomial for i=ps_high_degree_powers. + // Done separately because here the degree of the inner poly is degree % ps_high_degree. + // Once again, the free term will only be added later on. + if (degree % ps_high_degree > 0) + { + for (size_t j = 1; j <= degree % ps_high_degree; j++) + { + coeff.unsafe_load(*seal_context, + reinterpret_cast( + batched_coeffs[ps_high_degree_powers * ps_high_degree + j].data()), + batched_coeffs[ps_high_degree_powers * ps_high_degree + j].size()); + + evaluator->multiply_plain(ciphertext_powers[j], coeff, temp, pool); + + if (j == 1) + { + temp_in = temp; + } + else + { + evaluator->add_inplace(temp_in, temp); + } + } + + // Transform inner polynomial to coefficient form + evaluator->transform_from_ntt_inplace(temp_in); + evaluator->mod_switch_to_inplace(temp_in, high_powers_parms_id); + + // The high powers are already in coefficient form + evaluator->multiply_inplace( + temp_in, ciphertext_powers[ps_high_degree_powers * ps_high_degree], pool); + evaluator->add_inplace(result, temp_in); + } + + // Relinearize sum of ciphertext-ciphertext products if relinearization is supported by + // the parameters. + if (relinearize) + { + evaluator->relinearize_inplace(result, *relin_keys, pool); + } + + // Calculate inner polynomial for i=0. + // Done separately since there is no multiplication with a power of high-degree + for (size_t j = 1; j < ps_high_degree; j++) + { + coeff.unsafe_load(*seal_context, + reinterpret_cast(batched_coeffs[j].data()), batched_coeffs[j].size()); + + evaluator->multiply_plain(ciphertext_powers[j], coeff, temp, pool); + evaluator->transform_from_ntt_inplace(temp); + evaluator->mod_switch_to_inplace(temp, high_powers_parms_id); + evaluator->add_inplace(result, temp); + } + + // Add the constant coefficients of the inner polynomials multiplied by the respective + // powers of high-degree + for (size_t i = 1; i < ps_high_degree_powers + 1; i++) + { + coeff.unsafe_load(*seal_context, + reinterpret_cast(batched_coeffs[i * ps_high_degree].data()), + batched_coeffs[i * ps_high_degree].size()); + + evaluator->multiply_plain(ciphertext_powers[i * ps_high_degree], coeff, temp, pool); + evaluator->mod_switch_to_inplace(temp, high_powers_parms_id); + evaluator->add_inplace(result, temp); + } + + // Add the constant coefficient + coeff.unsafe_load(*seal_context, reinterpret_cast(batched_coeffs[0].data()), + batched_coeffs[0].size()); + + evaluator->add_plain_inplace(result, coeff); + + // Make the result as small as possible by modulus switching and possibly clearing + // irrelevant bits. + while (result.parms_id() != seal_context->last_parms_id()) + { + evaluator->mod_switch_to_next_inplace(result, pool); + } + try_clear_irrelevant_bits(seal_context->last_context_data()->parms(), result); + + return result; +} + +/** +Constructs a batched Plaintext polynomial from a list of polynomials. Takes an evaluator and +batch encoder to do encoding and NTT ops. +*/ +BatchedPlaintextPolyn::BatchedPlaintextPolyn( + const vector& polyns, CryptoContext context, uint32_t ps_low_degree, bool compressed) + : crypto_context(std::move(context)) +{ + // compr_mode_type compr_mode = compressed ? compr_mode_type::zstd : + // compr_mode_type::none; + compr_mode_type compr_mode = compr_mode_type::none; + // Find the highest degree polynomial in the list. The max degree determines how many + // Plaintexts we need to make + size_t max_deg = 0; + for (const FEltPolyn& p : polyns) + { + // Degree = number of coefficients - 1 + max_deg = max(p.size(), max_deg + 1) - 1; + } + + // We will encode with parameters that leave one or two levels, depending on whether + // Paterson-Stockmeyer is used. + size_t plain_coeffs_chain_idx = min( + crypto_context.seal_context()->first_context_data()->chain_index(), ps_low_degree ? 2 : 1); + auto encode_parms_id = + get_parms_id_for_chain_idx(*crypto_context.seal_context(), plain_coeffs_chain_idx); + + // Now make the Plaintexts. We let Plaintext i contain all bin coefficients of degree i. + size_t num_polyns = polyns.size(); + for (size_t i = 0; i < max_deg + 1; i++) + { + // Go through all the bins, collecting the coefficients at degree i + vector coeffs_of_deg_i; + coeffs_of_deg_i.reserve(num_polyns); + for (const FEltPolyn& p : polyns) + { + // Get the coefficient if it's set. Otherwise it's zero + felt_t coeff = 0; + if (i < p.size()) + { + coeff = p[i]; + } + + coeffs_of_deg_i.push_back(coeff); + } + + // Now let pt be the Plaintext consisting of all those degree i coefficients + Plaintext pt; + crypto_context.encoder()->encode(coeffs_of_deg_i, pt); + + // When evaluating the match and interpolation polynomials on encrypted query data, + // we multiply each power of the encrypted query with a plaintext (pt here) + // corresponding to the polynomial coefficient, and add the results together. The + // constant coefficient is handled by simply adding to the result, which requires + // that the plaintext is not in NTT form. When Paterson-Stockmeyer is used, this + // applies also to the constant coefficients for all inner polynomials, i.e., with + // i a multiple of the ps_high_degree == ps_low_degree + 1. + if ((!ps_low_degree && (i != 0)) || (ps_low_degree && (i % (ps_low_degree + 1)))) + { + crypto_context.evaluator()->transform_to_ntt_inplace(pt, encode_parms_id); + } + + // Push the new Plaintext + vector pt_data; + pt_data.resize(safe_cast(pt.save_size(compr_mode))); + size_t size = static_cast( + pt.save(reinterpret_cast(pt_data.data()), pt_data.size(), compr_mode)); + pt_data.resize(size); + batched_coeffs.push_back(std::move(pt_data)); + } +} + +BinBundleCache::BinBundleCache(const CryptoContext& crypto_context, size_t label_size) + : batched_matching_polyn(crypto_context) +{ + batched_interp_polyns.reserve(label_size); + for (size_t label_idx = 0; label_idx < label_size; label_idx++) + { + batched_interp_polyns.emplace_back(crypto_context); + } +} + +BinBundle::BinBundle(const CryptoContext& crypto_context, size_t label_size, size_t max_bin_size, + size_t ps_low_degree, size_t num_bins, bool compressed, bool stripped) + : cache_invalid_(true), + crypto_context_(crypto_context), + compressed_(compressed), + label_size_(label_size), + max_bin_size_(max_bin_size), + ps_low_degree_(ps_low_degree), + num_bins_(num_bins), + cache_(crypto_context_, label_size_) +{ + if (!crypto_context_.evaluator()) + { + throw invalid_argument("evaluator is not set in crypto_context"); + } + if (ps_low_degree > max_bin_size) + { + throw invalid_argument("ps_low_degree cannot be larger than max_bin_size"); + } + if (!num_bins) + { + throw invalid_argument("num_bins cannot be zero"); + } + + // Set up internal data structures + clear(stripped); +} + +/** +Returns the modulus that defines the finite field that we're working in +*/ +const Modulus& BinBundle::field_mod() const +{ + const auto& context_data = crypto_context_.seal_context()->first_context_data(); + return context_data->parms().plain_modulus(); +} + +template <> +int32_t BinBundle::multi_insert(const vector& items, size_t start_bin_idx, bool dry_run) +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot insert data to a stripped BinBundle"); + throw logic_error("failed to insert data"); + } + if (items.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item data to insert"); + return -1; + } + + // We are inserting items only; no labels. This BinBundle cannot have a non-zero label + // size. + if (get_label_size()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC( + "Attempted to insert unlabeled data in a labeled BinBundle"); + throw logic_error("failed to insert data"); + } + + // Return -1 if there isn't enough room in the BinBundle to insert at the given location + if (start_bin_idx >= get_num_bins() || items.size() > get_num_bins() - start_bin_idx) + { + return -1; + } + + // If we're here, that means we can insert in all bins + size_t max_bin_size = 0; + size_t curr_bin_idx = start_bin_idx; + for (felt_t curr_item : items) + { + vector& curr_bin = item_bins_[curr_bin_idx]; + + // Compare the would-be bin size here to the running max + if (max_bin_size < curr_bin.size() + 1) + { + max_bin_size = curr_bin.size() + 1; + } + + // Insert if not dry run + if (!dry_run) + { + // Insert the new item + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + curr_bin.push_back(curr_item); + curr_filter.add(curr_item); + + // Indicate that the polynomials need to be recomputed + cache_invalid_ = true; + } + + curr_bin_idx++; + } + + return safe_cast(max_bin_size); +} + +template <> +int32_t BinBundle::multi_insert( + const vector>>& item_labels, size_t start_bin_idx, bool dry_run) +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot insert data to a stripped BinBundle"); + throw logic_error("failed to insert data"); + } + if (item_labels.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item or label data to insert"); + return -1; + } + + // We are inserting item-labels. This BinBundle cannot have a zero label size. + if (!get_label_size()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC( + "Attempted to insert labeled data in an unlabeled BinBundle"); + throw logic_error("failed to insert data"); + } + + // Check that item_labels has correct size + size_t label_size = get_label_size(); + for (const auto& curr_item_label : item_labels) + { + size_t curr_label_size = curr_item_label.second.size(); + if (curr_label_size != label_size) + { + LABELED_PSI_LOG(ERROR) + << LOG_DESC("Attempted to insert item-label with incorrect label size ") + << LOG_KV("curr_label_size", curr_label_size) << LOG_KV("expected", label_size); + throw invalid_argument("failed to insert data"); + } + } + + // Return -1 if there isn't enough room in the BinBundle to insert at the given location + if (start_bin_idx >= get_num_bins() || item_labels.size() > get_num_bins() - start_bin_idx) + { + return -1; + } + + // Do we have a non-zero label size? In that case we cannot have repeated item parts in + // bins + if (get_label_size()) + { + // For each key, check that we can insert into the corresponding bin. If the answer + // is "no" at any point, return -1. + size_t curr_bin_idx = start_bin_idx; + for (auto& curr_item_label : item_labels) + { + felt_t curr_item = curr_item_label.first; + vector& curr_bin = item_bins_[curr_bin_idx]; + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + + // Check if the key is already in the current bin. If so, that's an insertion + // error + if (is_present(curr_bin, curr_filter, curr_item)) + { + return -1; + } + + curr_bin_idx++; + } + } + + // If we're here, that means we can insert in all bins + size_t max_bin_size = 0; + size_t curr_bin_idx = start_bin_idx; + for (auto& curr_item_label : item_labels) + { + felt_t curr_item = curr_item_label.first; + vector& curr_bin = item_bins_[curr_bin_idx]; + + // Compare the would-be bin size here to the running max + if (max_bin_size < curr_bin.size() + 1) + { + max_bin_size = curr_bin.size() + 1; + } + + // Insert if not dry run + if (!dry_run) + { + // Insert the new item + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + curr_bin.push_back(curr_item); + curr_filter.add(curr_item); + + // Insert the new label; loop over each label part + for (size_t label_idx = 0; label_idx < get_label_size(); label_idx++) + { + // Add this label part to the matching bin + felt_t curr_label = curr_item_label.second[label_idx]; + label_bins_[label_idx][curr_bin_idx].push_back(curr_label); + } + + // Indicate that the polynomials need to be recomputed + cache_invalid_ = true; + } + + curr_bin_idx++; + } + + return safe_cast(max_bin_size); +} + +template <> +bool BinBundle::try_multi_overwrite(const vector& items, size_t start_bin_idx) +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot overwrite data in a stripped BinBundle"); + throw logic_error("failed to overwrite data"); + } + if (items.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item data to insert"); + return false; + } + + // This function may have been called accidentally; no label data is given, so nothing + // will be overwritten. This is equivalent to searching for the presence of the felt + // items in this BinBundle and forcing the cache to be recomputed. + // Return false if there isn't enough room in the BinBundle to insert at the given + // location + if (start_bin_idx >= get_num_bins() || items.size() > get_num_bins() - start_bin_idx) + { + return false; + } + + // Check that all the item components appear sequentially in this BinBundle + size_t curr_bin_idx = start_bin_idx; + for (felt_t curr_item : items) + { + vector& curr_bin = item_bins_[curr_bin_idx]; + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + + // A non-match was found; the item is not here. + if (!is_present(curr_bin, curr_filter, curr_item)) + { + return false; + } + + curr_bin_idx++; + } + + // Nothing was done, but mark the cache as dirty anyway + cache_invalid_ = true; + + return true; +} + +template <> +bool BinBundle::try_multi_overwrite( + const vector>>& item_labels, size_t start_bin_idx) +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot overwrite data in a stripped BinBundle"); + throw logic_error("failed to overwrite data"); + } + if (item_labels.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item or label data to insert"); + return false; + } + + // Check that item_labels has correct size + size_t label_size = get_label_size(); + for (const auto& curr_item_label : item_labels) + { + size_t curr_label_size = curr_item_label.second.size(); + if (curr_label_size != label_size) + { + LABELED_PSI_LOG(ERROR) + << LOG_DESC("Attempted to overwrite item-label with incorrect label size ") + << LOG_KV("curr_label_size", curr_label_size) << LOG_KV("expected", label_size); + + throw invalid_argument("failed to overwrite data"); + } + } + + // Return false if there isn't enough room in the BinBundle to insert at the given + // location + if (start_bin_idx >= get_num_bins() || item_labels.size() > get_num_bins() - start_bin_idx) + { + return false; + } + + // Check that all the item components appear sequentially in this BinBundle + size_t curr_bin_idx = start_bin_idx; + for (auto& curr_item_label : item_labels) + { + felt_t curr_item = curr_item_label.first; + vector& curr_bin = item_bins_[curr_bin_idx]; + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + + // A non-match was found; the item is not here. + if (!is_present(curr_bin, curr_filter, curr_item)) + { + return false; + } + + curr_bin_idx++; + } + + // If we're here, that means we can overwrite the labels + curr_bin_idx = start_bin_idx; + for (auto& curr_item_label : item_labels) + { + felt_t curr_item = curr_item_label.first; + + // Overwrite the label in the bin + vector& curr_bin = item_bins_[curr_bin_idx]; + + // No point in using cuckoo filters here for look-up: we know the item exists so do + // linear search + auto found_pos = find(curr_bin.begin(), curr_bin.end(), curr_item); + + // From the earlier check we know that found_pos is not the end-iterator. Check this + // again to be sure. + if (found_pos == curr_bin.end()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC( + "Attempted to overwrite item-label, but the item could no longer be found; " + "the internal state of this BinBundle has been corrupted"); + throw runtime_error("failed to overwrite data"); + } + + // Compute the location in the curr_bin so we know how to index into the label bins + auto item_idx_in_bin = static_cast(distance(curr_bin.begin(), found_pos)); + + // Write the new label; loop over each label part + for (size_t label_idx = 0; label_idx < get_label_size(); label_idx++) + { + // Overwrite this label part in the matching bin + felt_t curr_label = curr_item_label.second[label_idx]; + label_bins_[label_idx][curr_bin_idx][item_idx_in_bin] = curr_label; + } + + // Indicate that the polynomials need to be recomputed + cache_invalid_ = true; + + curr_bin_idx++; + } + + return true; +} + +bool BinBundle::try_multi_remove(const vector& items, size_t start_bin_idx) +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot remove data from a stripped BinBundle"); + throw logic_error("failed to remove data"); + } + if (items.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item data to remove"); + return false; + } + + // Return false if there isn't enough room in the BinBundle at the given location + if (start_bin_idx >= get_num_bins() || items.size() > get_num_bins() - start_bin_idx) + { + return false; + } + + // Go through all the items. If any item doesn't appear, we scrap the whole computation + // and return false. + size_t curr_bin_idx = start_bin_idx; + vector::iterator> to_remove_item_its; + vector::iterator>> to_remove_label_its(get_label_size()); + + for (auto& item : items) + { + vector& curr_bin = item_bins_[curr_bin_idx]; + CuckooFilter& curr_filter = filters_[curr_bin_idx]; + + auto to_remove_item_it = get_iterator(curr_bin, curr_filter, item); + if (curr_bin.end() == to_remove_item_it) + { + // One of the items isn't there; return false; + return false; + } + else + { + // Found the item; mark it for removal + to_remove_item_its.push_back(to_remove_item_it); + + // We need to also mark the corresponding labels for removal + auto item_loc_in_bin = distance(curr_bin.begin(), to_remove_item_it); + for (size_t label_idx = 0; label_idx < get_label_size(); label_idx++) + { + auto to_remove_label_it = + label_bins_[label_idx][curr_bin_idx].begin() + item_loc_in_bin; + to_remove_label_its[label_idx].push_back(to_remove_label_it); + } + } + + curr_bin_idx++; + } + + // We got to this point, so all of the items were found. Now just erase them. + curr_bin_idx = start_bin_idx; + for (auto to_remove_item_it : to_remove_item_its) + { + // Remove the item + filters_[curr_bin_idx].remove(*to_remove_item_it); + item_bins_[curr_bin_idx].erase(to_remove_item_it); + + // Indicate that the polynomials need to be recomputed + cache_invalid_ = true; + + curr_bin_idx++; + } + + // Finally erase the label parts + for (size_t label_idx = 0; label_idx < get_label_size(); label_idx++) + { + curr_bin_idx = start_bin_idx; + for (auto to_remove_label_it : to_remove_label_its[label_idx]) + { + // Remove the label + label_bins_[label_idx][curr_bin_idx].erase(to_remove_label_it); + + curr_bin_idx++; + } + } + + return true; +} + +bool BinBundle::try_get_multi_label( + const vector& items, size_t start_bin_idx, vector& labels) const +{ + if (stripped_) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("Cannot retrieve labels from a stripped BinBundle"); + throw logic_error("failed to retrieve labels"); + } + if (items.empty()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("No item data to search for"); + return false; + } + + // Return false if there isn't enough room in the BinBundle at the given location + if (start_bin_idx >= get_num_bins() || items.size() > get_num_bins() - start_bin_idx) + { + return false; + } + + // Resize the labels vector to expected size; we will write in a non-linear order + labels.clear(); + labels.resize(items.size() * get_label_size()); + + // Go through all the items. If the item appears, find its label and write to labels. If + // any item doesn't appear, we scrap the whole computation and return false. + size_t curr_bin_idx = start_bin_idx; + for (size_t item_idx = 0; item_idx < items.size(); item_idx++) + { + const vector& curr_bin = item_bins_[curr_bin_idx]; + const CuckooFilter& curr_filter = filters_[curr_bin_idx]; + + // Find the item if present in this bin + auto item_it = get_iterator(curr_bin, curr_filter, items[item_idx]); + + if (curr_bin.end() == item_it) + { + // One of the items isn't there. No label to fetch. Clear the labels and return + // early. + labels.clear(); + return false; + } + + // Found the (felt) item. Next collect the label parts for this and write to label. + size_t item_idx_in_bin = static_cast(distance(curr_bin.begin(), item_it)); + for (size_t label_idx = 0; label_idx < get_label_size(); label_idx++) + { + // Need to reorder the felts + labels[items.size() * label_idx + item_idx] = + label_bins_[label_idx][curr_bin_idx][item_idx_in_bin]; + } + + curr_bin_idx++; + } + + return true; +} + +void BinBundle::clear(bool stripped) +{ + // Set the stripped flag + stripped_ = stripped; + + // Clear item data + item_bins_.clear(); + if (!stripped_) + { + item_bins_.resize(num_bins_); + } + + // Clear label data + label_bins_.clear(); + if (!stripped_) + { + label_bins_.reserve(label_size_); + for (size_t i = 0; i < label_size_; i++) + { + label_bins_.emplace_back(num_bins_); + } + } + + // Clear filters + filters_.clear(); + if (!stripped_) + { + filters_.reserve(num_bins_); + for (size_t i = 0; i < num_bins_; i++) + { + filters_.emplace_back(max_bin_size_, /* bits per tag */ 12); + } + } + + // Clear the cache + clear_cache(); +} + +void BinBundle::clear_cache() +{ + cache_.felt_matching_polyns.clear(); + cache_.batched_matching_polyn = crypto_context_; + + cache_.felt_interp_polyns.clear(); + cache_.batched_interp_polyns.clear(); + + cache_invalid_ = true; +} + +const BinBundleCache& BinBundle::get_cache() const +{ + if (cache_invalid_) + { + throw logic_error("tried to retrieve stale cache"); + } + + return cache_; +} + +void BinBundle::regen_plaintexts() +{ + // This function assumes that BinBundle::clear_cache and BinBundle::regen_polyns have + // been called and the polynomials have not been modified since then. + + // Allocate memory for the batched "label polynomials" + cache_.batched_interp_polyns.resize(get_label_size()); + + ThreadPoolMgr tpm; + + vector> futures; + futures.push_back(tpm.thread_pool().enqueue([&]() { + // Compute and cache the batched "matching polynomials". They're computed in both + // labeled and unlabeled PSI. + BatchedPlaintextPolyn bmp(cache_.felt_matching_polyns, crypto_context_, + static_cast(ps_low_degree_), compressed_); + cache_.batched_matching_polyn = std::move(bmp); + })); + + for (size_t label_idx = 0; label_idx < cache_.felt_interp_polyns.size(); label_idx++) + { + futures.push_back(tpm.thread_pool().enqueue([&, label_idx]() { + // Compute and cache the batched Newton interpolation polynomials + const auto& interp_polyn = cache_.felt_interp_polyns[label_idx]; + BatchedPlaintextPolyn bip( + interp_polyn, crypto_context_, static_cast(ps_low_degree_), compressed_); + cache_.batched_interp_polyns[label_idx] = std::move(bip); + })); + } + + // Wait for the tasks to finish + for (auto& f : futures) + { + f.get(); + } +} + +void BinBundle::regen_polyns() +{ + // This function assumes that BinBundle::clear_cache has been called and the polynomials + // have not been modified since then. Specifically, it assumes that item_bins_ is empty + // and and label_bins_ is set to the correct size. + + // Get the field modulus. We need this for polynomial calculations + const Modulus& mod = field_mod(); + + size_t num_bins = get_num_bins(); + size_t label_size = get_label_size(); + cache_.felt_matching_polyns.resize(num_bins); + cache_.felt_interp_polyns.resize(label_size); + for (auto& fips : cache_.felt_interp_polyns) + { + fips.resize(num_bins); + } + + ThreadPoolMgr tpm; + + vector> futures; + // For each bin in the bundle, compute and cache the corresponding "matching + // polynomial" + for (size_t bin_idx = 0; bin_idx < num_bins; bin_idx++) + { + futures.push_back(tpm.thread_pool().enqueue([&, bin_idx]() { + // Compute and cache the matching polynomial + FEltPolyn fmp = polyn_with_roots(item_bins_[bin_idx], mod); + cache_.felt_matching_polyns[bin_idx] = std::move(fmp); + })); + } + + // For each bin in the bundle, compute and cache the corresponding "label polynomials" + for (size_t label_idx = 0; label_idx < label_size; label_idx++) + { + for (size_t bin_idx = 0; bin_idx < num_bins; bin_idx++) + { + futures.push_back(tpm.thread_pool().enqueue([&, label_idx, bin_idx]() { + // Compute and cache the matching polynomial + FEltPolyn fip = newton_interpolate_polyn( + item_bins_[bin_idx], label_bins_[label_idx][bin_idx], mod); + cache_.felt_interp_polyns[label_idx][bin_idx] = std::move(fip); + })); + } + } + + // Wait for the tasks to finish + for (auto& f : futures) + { + f.get(); + } +} + +void BinBundle::regen_cache() +{ + // Only recompute the cache if it needs to be recomputed + if (cache_invalid_) + { + clear_cache(); + regen_polyns(); + regen_plaintexts(); + cache_invalid_ = false; + } +} + +bool BinBundle::empty() const +{ + return all_of(item_bins_.begin(), item_bins_.end(), [](auto& b) { return b.empty(); }); +} + +void BinBundle::strip() +{ + // Ensure the cache is valid + regen_cache(); + + stripped_ = true; + + item_bins_.clear(); + label_bins_.clear(); + filters_.clear(); + + cache_.felt_matching_polyns.clear(); + cache_.felt_interp_polyns.clear(); +} + +ppctars::BinBundle BinBundle::saveToTarsBinBundle() const +{ + ppctars::BinBundle binBundle; + + // write the items and labels + for (auto& bins : item_bins_) + { + std::vector tarsBins; + tarsBins.reserve(bins.size()); + for (auto& item : bins) + { + tarsBins.emplace_back(item); + } + binBundle.itemBins.emplace_back(tarsBins); + } + for (auto& bins : label_bins_) + { + std::vector> tarsBins; + for (auto& labels : bins) + { + std::vector tarsLabels; + tarsLabels.reserve(labels.size()); + for (auto& label : labels) + { + tarsLabels.emplace_back(label); + } + tarsBins.emplace_back(tarsLabels); + } + binBundle.labelBins.emplace_back(tarsBins); + } + + // write BinBundleCache + ppctars::BinBundleCache binBundleCache; + + for (auto& polyns : cache_.felt_matching_polyns) + { + std::vector tarsPolyns; + tarsPolyns.reserve(polyns.size()); + for (auto& polyn : polyns) + { + tarsPolyns.emplace_back(polyn); + } + binBundleCache.feltMatchingPolyns.emplace_back(tarsPolyns); + } + for (auto& interpPolyns : cache_.felt_interp_polyns) + { + std::vector> tarsInterpPolyns; + for (auto& polyns : interpPolyns) + { + std::vector tarsPolyns; + tarsPolyns.reserve(polyns.size()); + for (auto& polyn : polyns) + { + tarsPolyns.emplace_back(polyn); + } + tarsInterpPolyns.emplace_back(tarsPolyns); + } + binBundleCache.feltInterpPolyns.emplace_back(tarsInterpPolyns); + } + binBundleCache.batchedMatchingPolyn.batchedCoeffs = + cache_.batched_matching_polyn.batched_coeffs; + for (auto& polyns : cache_.batched_interp_polyns) + { + ppctars::BatchedPlaintextPolyn batchedPlaintextPolyn; + batchedPlaintextPolyn.batchedCoeffs = polyns.batched_coeffs; + binBundleCache.batchedInterpPolyns.emplace_back(batchedPlaintextPolyn); + } + binBundle.cache = binBundleCache; + + binBundle.cacheInvalid = cache_invalid_; + binBundle.compressed = compressed_; + binBundle.stripped = stripped_; + binBundle.labelSize = label_size_; + binBundle.maxBinSize = max_bin_size_; + binBundle.psLowDegree = ps_low_degree_; + binBundle.numBins = num_bins_; + + return binBundle; +} + +BinBundle BinBundle::loadFromTarsBinBundle( + const CryptoContext& _cryptoContext, const ppctars::BinBundle& _tarsBinBundle) +{ + BinBundle binBundle(_cryptoContext, _tarsBinBundle.labelSize, _tarsBinBundle.maxBinSize, + _tarsBinBundle.psLowDegree, _tarsBinBundle.numBins, _tarsBinBundle.compressed, + _tarsBinBundle.stripped); + binBundle.cache_invalid_ = _tarsBinBundle.cacheInvalid; + + // load the items and labels + binBundle.item_bins_.clear(); + for (auto& tarsBins : _tarsBinBundle.itemBins) + { + std::vector bins; + bins.reserve(tarsBins.size()); + for (auto& item : tarsBins) + { + bins.emplace_back(item); + } + binBundle.item_bins_.emplace_back(bins); + } + + binBundle.label_bins_.clear(); + for (auto& tarsBins : _tarsBinBundle.labelBins) + { + std::vector> bins; + for (auto& tarsLabels : tarsBins) + { + std::vector labels; + labels.reserve(tarsLabels.size()); + for (auto& label : tarsLabels) + { + labels.emplace_back(label); + } + bins.emplace_back(labels); + } + binBundle.label_bins_.emplace_back(bins); + } + + // init CuckooFilter + auto binSize = binBundle.item_bins_.size(); + for (size_t binIdx = 0; !binBundle.stripped_ && (binIdx < binSize); binIdx++) + { + auto& items = binBundle.item_bins_[binIdx]; + for (auto& felt : items) + { + binBundle.filters_[binIdx].add(felt); + } + } + + // init BinBundleCache + for (auto& tarsPolyns : _tarsBinBundle.cache.feltMatchingPolyns) + { + std::vector polyns; + polyns.reserve(tarsPolyns.size()); + for (auto& polyn : tarsPolyns) + { + polyns.emplace_back(polyn); + } + binBundle.cache_.felt_matching_polyns.emplace_back(polyns); + } + for (auto& tarsInterpPolyns : _tarsBinBundle.cache.feltInterpPolyns) + { + std::vector> interpPolyns; + for (auto& tarsPolyns : tarsInterpPolyns) + { + std::vector polyns; + polyns.reserve(tarsPolyns.size()); + for (auto& polyn : tarsPolyns) + { + polyns.emplace_back(polyn); + } + interpPolyns.emplace_back(polyns); + } + binBundle.cache_.felt_interp_polyns.emplace_back(interpPolyns); + } + binBundle.cache_.batched_matching_polyn.batched_coeffs = + _tarsBinBundle.cache.batchedMatchingPolyn.batchedCoeffs; + for (auto& polyn : _tarsBinBundle.cache.batchedInterpPolyns) + { + BatchedPlaintextPolyn batchedPlaintextPolyn; + batchedPlaintextPolyn.batched_coeffs = polyn.batchedCoeffs; + batchedPlaintextPolyn.crypto_context = _cryptoContext; + binBundle.cache_.batched_interp_polyns.emplace_back(std::move(batchedPlaintextPolyn)); + } + + return binBundle; +} diff --git a/cpp/ppc-psi/src/labeled-psi/core/BinBundle.h b/cpp/ppc-psi/src/labeled-psi/core/BinBundle.h new file mode 100644 index 00000000..e2273f3a --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/BinBundle.h @@ -0,0 +1,430 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @reference https://github.com/microsoft/APSI/sender/bin_bundle.h + * @license MIT license + * @change change the implementation of load and save + * + * @file Binbundle.h + * @author: shawnhe + * @date 2022-12-18 + * + */ + +#pragma once + +// STD +#include +#include +#include + +// APSI +#include "apsi/crypto_context.h" +#include "apsi/util/cuckoo_filter.h" +#include "apsi/util/db_encoding.h" + +// SEAL +#include "seal/util/uintarithsmallmod.h" +#include "seal/util/uintcore.h" + +// GSL +#include "gsl/span" + +#include + +#include "../Common.h" +#include "SenderCache.h" +#include + +using namespace apsi::util; + +namespace ppc::psi +{ +/** +Represents a polynomial with coefficients that are field elements. Coefficients are stored +in degree-increasing order, so, for example, the constant term is at index 0. +*/ +using FEltPolyn = std::vector; + +/** +A bunch of polynomials represented using a sequence of batched SEAL Plaintexts. + +Example: Suppose we have 3 polynomials, + + 3x⁵ + 7x⁴ + x³ + 9x² + 4x + 2 + 8x³ + 5x² + + 1 + 9x⁴ + 2x³ + + x + 8 + +To represent them as a BatchedPlaintextPolyn, we would make a Plaintext for every column of +coefficients. Suppose each Plaintext has 3 slots. Let Plaintext #i holds all the +coefficients of degree i. So then the plaintexts P₀, ..., P₅ would be + + |P₅|P₄|P₃|P₂|P₁|P₀| + |--|--|--|--|--|--| + | 3| 7| 1| 9| 4| 2| + | 0| 0| 8| 5| 0| 1| + | 0| 9| 2| 0| 1| 8| +*/ +struct BatchedPlaintextPolyn +{ + /** + A sequence of coefficients represented as batched plaintexts. The length of this vector + is the degree of the highest-degree polynomial in the sequence. + */ + std::vector> batched_coeffs; + + /** + We need this to compute eval() + */ + apsi::CryptoContext crypto_context; + + BatchedPlaintextPolyn(const BatchedPlaintextPolyn& copy) = delete; + + BatchedPlaintextPolyn(BatchedPlaintextPolyn&& source) = default; + + BatchedPlaintextPolyn& operator=(const BatchedPlaintextPolyn& assign) = delete; + + BatchedPlaintextPolyn& operator=(BatchedPlaintextPolyn&& assign) = default; + + /** + Construct and empty BatchedPlaintextPolyn instance. + */ + BatchedPlaintextPolyn() = default; + + /** + Constructs a batched Plaintext polynomial from a list of polynomials. Takes an evaluator + and batch encoder to do encoding and NTT ops. + */ + BatchedPlaintextPolyn(const std::vector& polyns, apsi::CryptoContext context, + std::uint32_t ps_low_degree, bool compressed); + + /** + Constructs an uninitialized Plaintext polynomial using the given crypto context + */ + BatchedPlaintextPolyn(apsi::CryptoContext context) : crypto_context(std::move(context)) {} + + /** + Evaluates the polynomial on the given ciphertext. We don't compute the powers of the + input ciphertext C ourselves. Instead we assume they've been precomputed and accept the + powers: (C, C², C³, ...) as input. The number of powers provided MUST be equal to + plaintext_polyn_coeffs_.size()-1. + */ + seal::Ciphertext eval( + const std::vector& ciphertext_powers, seal::MemoryPoolHandle& pool) const; + + /** + Evaluates the polynomial on the given ciphertext using the Paterson-Stockmeyer + algorithm, as long as it requires less computation than the standard evaluation function + above. The algorithm computes h+1 inner polynomials on low powers (C¹ to C^{l-1}). Each + inner polynomial is then multiplied by the corresponding high power. The parameters l + and h are determined according to the degree of the polynomial and the number of splits + in order to minimize the computation. + + Evaluated polynomial a_0 + a_1*C + a_2*C^2 + ... + C^degree + + Inner polys: a_{l*i} + a_{l*i+1}*C + ... + a_{l*i+l-1}*C^{l-1} (for i=0,...,h-1) + and: a_{l*h} + a_{l*h+1}*C + ... + a_{l*h+degree%l}*C^{degree%l} (for i=h) + + Low powers: C^{1}, ..., C^{l-1} + High powers: C^{1*l}, ..., C^{l*h} + */ + seal::Ciphertext eval_patstock(const apsi::CryptoContext& eval_crypto_context, + const std::vector& ciphertext_powers, std::size_t ps_low_degree, + seal::MemoryPoolHandle& pool) const; + + /** + Returns whether this polynomial has non-zero size. + */ + explicit operator bool() const noexcept { return batched_coeffs.size(); } +}; + +// A cache of all the polynomial and plaintext computations on a single BinBundle +struct BinBundleCache +{ + BinBundleCache(const BinBundleCache& copy) = delete; + + BinBundleCache(BinBundleCache&& source) = default; + + BinBundleCache& operator=(const BinBundleCache& assign) = delete; + + BinBundleCache& operator=(BinBundleCache&& assign) = default; + + BinBundleCache(const apsi::CryptoContext& crypto_context, std::size_t label_size); + + /** + For each bin, stores the "matching polynomial", i.e., unique monic polynomial whose + roots are precisely the items in the bin. + */ + std::vector felt_matching_polyns; + + /** + For each bin, stores the Newton intepolation polynomial whose value at each item in the + bin equals the item's corresponding label. Note that this field is empty when doing + unlabeled PSI. + */ + std::vector> felt_interp_polyns; + + /** + Cached seal::Plaintext representation of the "matching" polynomial of this BinBundle. + */ + BatchedPlaintextPolyn batched_matching_polyn; + + /** + Cached seal::Plaintext representation of the interpolation polynomial of this BinBundle. + Note that this field is empty when doing unlabeled PSI. + */ + std::vector batched_interp_polyns; +}; // struct BinBundleCache + +/** +Represents a specific bin bundle and stores the associated data. The type parameter L +represents the label type. This is either a field element (in the case of labeled PSI), or +an element of the unit type (in the case of unlabeled PSI). +*/ +class BinBundle +{ +private: + /** + This is true iff cache_ needs to be regenerated + */ + bool cache_invalid_; + + /** + We need this to make Plaintexts + */ + apsi::CryptoContext crypto_context_; + + /** + Items (decomposed into field elements) for each bin in the BinBundle. The dimensions + are, in order: + - Bins in the BinBundle + - Field elements in the bin + */ + std::vector> item_bins_; + + /** + Item-size chunks of the label (decomposed into field elements) for each bin in the + BinBundle. The dimensions are, in order: + - Components of the label + - Bins in the BinBundle + - Field elements in the bin + */ + std::vector>> label_bins_; + + /** + Each bin in the BinBundle has a CuckooFilter that helps quickly determine whether a + field element is contained. + */ + std::vector filters_; + + /** + Indicates whether SEAL plaintexts are compressed in memory. + */ + bool compressed_; + + /** + Indicates whether the BinBundle has been stripped of all information not needed for + serving a query. + */ + bool stripped_; + + /** + The size of the labels in multiples of item length. + */ + std::size_t label_size_; + + /** + Maximum size of the bins. + */ + std::size_t max_bin_size_; + + /** + Holds the Paterson-Stockmeyer low-degree for this BinBundle. + */ + std::size_t ps_low_degree_; + + /** + The number of bins in the BinBundle. + */ + std::size_t num_bins_; + + /** + A cache of all the computations we can do on the bins. This is empty by default. + */ + BinBundleCache cache_; + + /** + Returns the modulus that defines the finite field that we're working in + */ + const seal::Modulus& field_mod() const; + + /** + Computes and caches the appropriate polynomials of each bin. For unlabeled PSI, this is + just the "matching" polynomial. For labeled PSI, this is the "matching" polynomial and + the Newton interpolation polynomial. Resulting values are stored in cache_. + */ + void regen_polyns(); + + /** + Batches this BinBundle's polynomials into SEAL Plaintexts. Resulting values are stored + in cache_. + */ + void regen_plaintexts(); + +public: + BinBundle(const apsi::CryptoContext& crypto_context, std::size_t label_size, + std::size_t max_bin_size, std::size_t ps_low_degree, std::size_t num_bins, bool compressed, + bool stripped); + + BinBundle(const BinBundle& copy) = delete; + + BinBundle(BinBundle&& source) = default; + + BinBundle& operator=(const BinBundle& assign) = delete; + + BinBundle& operator=(BinBundle&& assign) = default; + + /** + Inserts item-label pairs into sequential bins, beginning at start_bin_idx. If dry_run is + specified, no change is made to the BinBundle. On success, returns the size of the + largest bin bins in the modified range, after insertion has taken place. On failed + insertion, returns -1. On failure, no modification is made to the BinBundle. + */ + template + std::int32_t multi_insert( + const std::vector& item_labels, std::size_t start_bin_idx, bool dry_run); + + /** + Does a dry-run insertion of item-label pairs into sequential bins, beginning at + start_bin_idx. This does not mutate the BinBundle. On success, returns the size of the + largest bin bins in the modified range, after insertion has taken place. On failed + insertion, returns -1. + */ + template + std::int32_t multi_insert_dry_run(const std::vector& item_labels, std::size_t start_bin_idx) + { + return multi_insert(item_labels, start_bin_idx, true); + } + + /** + Inserts item-label pairs into sequential bins, beginning at start_bin_idx. On success, + returns the size of the largest bin bins in the modified range, after insertion has + taken place. On failed insertion, returns -1. On failure, no modification is made to the + BinBundle. + */ + template + std::int32_t multi_insert_for_real(const std::vector& item_labels, std::size_t start_bin_idx) + { + return multi_insert(item_labels, start_bin_idx, false); + } + + /** + Attempts to overwrite the stored items' labels with the given labels. Returns true iff + it found a contiguous sequence of given items. If no such sequence was found, this + BinBundle is not mutated. This function can be called on a + BinBundle> but it won't do anything except force the cache to get + recomputed, so don't bother. The labeled case has T equal to std::pair>. + */ + template + bool try_multi_overwrite(const std::vector& item_labels, std::size_t start_bin_idx); + + /** + Attempts to remove the stored items and labels. Returns true iff it found a contiguous + sequence of given items and the data was successfully removed. If no such sequence was + found, this BinBundle is not mutated. + */ + bool try_multi_remove(const std::vector& items, std::size_t start_bin_idx); + + /** + Sets the given labels to the set of labels associated with the sequence of items in this + BinBundle, starting at start_idx. If any item is not present in its respective bin, this + returns false and clears the given labels vector. Returns true on success. + */ + bool try_get_multi_label(const std::vector& items, std::size_t start_bin_idx, + std::vector& labels) const; + + /** + Clears the contents of the BinBundle and wipes out the cache. + */ + void clear(bool stripped = false); + + /** + Wipes out the cache of the BinBundle + */ + void clear_cache(); + + /** + Returns whether this BinBundle's cache needs to be recomputed + */ + bool cache_invalid() const noexcept { return cache_invalid_; } + + /** + Gets a constant reference to this BinBundle's cache. This will throw an exception if the + cache is invalid. Check the cache before you wreck the cache. + */ + const BinBundleCache& get_cache() const; + + /** + Generates and caches all the polynomials and plaintexts that this BinBundle requires + */ + void regen_cache(); + + /** + Returns a constant reference to the items in this BinBundle. + */ + const std::vector>& get_item_bins() const noexcept { return item_bins_; } + + /** + Returns the size of the label in multiples of the item size. + */ + std::size_t get_label_size() const noexcept { return label_size_; } + + /** + Returns the number of bins. + */ + std::size_t get_num_bins() const noexcept { return num_bins_; } + + /** + Returns a constant reference to the label parts in this BinBundle. + */ + const std::vector>>& get_label_bins() const noexcept + { + return label_bins_; + } + + /** + Returns whether this BinBundle is empty. + */ + bool empty() const; + + /** + Indicates whether the BinBundle has been stripped of all information not needed for + serving a query. + */ + bool is_stripped() const { return stripped_; } + + /** + Strips the BinBundle of all information not needed for serving a query. + */ + void strip(); + + ppctars::BinBundle saveToTarsBinBundle() const; + + static BinBundle loadFromTarsBinBundle( + const apsi::CryptoContext& _cryptoContext, const ppctars::BinBundle& _tarsBinBundle); +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h new file mode 100644 index 00000000..9cbf3b97 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h @@ -0,0 +1,170 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @reference https://github.com/secretflow/spu + * @license Apache license + * + * @file LabeledPSIParams.h + * @author: shawnhe + * @date 2022-11-14 + * + */ + +#pragma once + +#include "LabeledPSI.h" +#include "ppc-psi/src/labeled-psi/Common.h" + +#include +#include +#include +#include +#include + +namespace ppc::psi +{ + +struct BaseParams +{ + size_t maxItemsPerBin; + size_t polyModulusDegree; + // plain_modulus or plain_modulus_bits + size_t plainModulus = 0; + size_t plainModulusBits = 0; + std::vector coeffModulusBits; + + + size_t getPlainModulusBits() + { + // plain_modulus_bits + if (plainModulusBits > 0) + { + return plainModulusBits; + } + else if (plainModulus > 0) + { + // get plain_modulus_bits by plain_modulus + return std::floor(std::log2(plainModulus)); + } + return 0; + } +}; + +inline size_t getHashTruncateSize(size_t _senderSize, size_t _statsSecParams = 40) +{ + // reference: Fast Private Set Intersection from Homomorphic Encryption, section 4.2 + // url: https://eprint.iacr.org/2017/299 + size_t tSize = + std::ceil(std::log2(_senderSize)) + std::ceil(std::log2(MAX_QUERY_SIZE)) + _statsSecParams; + return std::max(static_cast(80), tSize); +} + +inline apsi::PSIParams getPsiParams(size_t _senderSize) +{ + // reference: microsoft APSI example parameter sets + // url: https://github.com/microsoft/APSI/tree/main/parameters + std::vector kBaseParams = { + {42, 2048, 65537, 0, {48}}, // 0 100k-1-16 + {228, 8192, 65537, 0, {56, 48, 48}}, // 1 1M-1-32 + {782, 8192, 0, 21, {56, 56, 56, 50}}, // 2 16M-1-32 + {8100, 8192, 0, 22, {56, 56, 56, 32}}, // 3 256M-1 + }; + + BaseParams baseParams; + if (_senderSize <= (1 << 18)) + { + baseParams = kBaseParams[0]; + } + else if (_senderSize <= (1 << 20)) + { + baseParams = kBaseParams[1]; + } + else if (_senderSize <= (1 << 24)) + { + baseParams = kBaseParams[2]; + } + else + { + baseParams = kBaseParams[3]; + } + + // item param + apsi::PSIParams::ItemParams itemParams{}; + size_t hash_size = getHashTruncateSize(_senderSize); + itemParams.felts_per_item = + std::ceil(static_cast(hash_size) / (baseParams.getPlainModulusBits() - 1)); + + // table param + apsi::PSIParams::TableParams tableParams{}; + tableParams.max_items_per_bin = baseParams.maxItemsPerBin; + tableParams.hash_func_count = 1; + + size_t polyItemCount = baseParams.polyModulusDegree / itemParams.felts_per_item; + tableParams.table_size = polyItemCount; + + size_t cuckooTableSize = std::ceil(4 * MAX_QUERY_SIZE); + while (tableParams.table_size < cuckooTableSize) + { + tableParams.table_size += polyItemCount; + } + + // query param + apsi::PSIParams::QueryParams queryParams; + if (tableParams.max_items_per_bin == 42) + { + queryParams.ps_low_degree = 0; + queryParams.query_powers = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42}; + } + else if (tableParams.max_items_per_bin == 228) + { + queryParams.ps_low_degree = 0; + queryParams.query_powers = {1, 3, 8, 19, 33, 39, 92, 102}; + } + else if (tableParams.max_items_per_bin == 782) + { + queryParams.ps_low_degree = 26; + queryParams.query_powers = {1, 5, 8, 27, 135}; + } + else + { + queryParams.ps_low_degree = 310; + queryParams.query_powers = {1, 4, 10, 11, 28, 33, 78, 118, 143, 311, 1555}; + } + + // seal param + apsi::PSIParams::SEALParams apsiSealParams; + apsiSealParams.set_poly_modulus_degree(baseParams.polyModulusDegree); + + if (baseParams.plainModulusBits > 0) + { + apsiSealParams.set_plain_modulus(seal::PlainModulus::Batching( + baseParams.polyModulusDegree, baseParams.plainModulusBits)); + } + else + { + apsiSealParams.set_plain_modulus(baseParams.plainModulus); + } + + apsiSealParams.set_coeff_modulus( + seal::CoeffModulus::Create(baseParams.polyModulusDegree, baseParams.coeffModulusBits)); + + apsi::PSIParams psi_params(itemParams, tableParams, queryParams, apsiSealParams); + + return psi_params; +} + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp new file mode 100644 index 00000000..f9b3ebe7 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp @@ -0,0 +1,657 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIReceiver.cpp + * @author: shawnhe + * @date 2022-11-3 + */ + +#include +#include +#include +#include +#include + +#include "LabeledPSI.h" +#include "LabeledPSIReceiver.h" +#include "ResultPackage.h" +#include "bcos-utilities/DataConvertUtility.h" +#include "ppc-psi/src/labeled-psi/Common.h" +#include "ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::tools; + +LabeledPSIReceiver::LabeledPSIReceiver( + LabeledPSIConfig::Ptr _config, TaskState::Ptr _taskState, OprfClient::Ptr _oprfClient) + : m_config(std::move(_config)), + m_taskState(std::move(_taskState)), + m_oprfClient(std::move(_oprfClient)) +{ + m_taskID = m_taskState->task()->id(); + auto originData = m_taskState->task()->selfParty()->dataResource()->rawData(); + if (originData.empty() || originData[0].size() > MAX_QUERY_SIZE) + { + BOOST_THROW_EXCEPTION( + TooManyItemsException() << bcos::errinfo_comment( + "Items count per query should be (0, " + std::to_string(MAX_QUERY_SIZE) + "]")); + } + m_items = std::move(originData[0]); +} + +void LabeledPSIReceiver::asyncRunTask() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("asyncRunTask as receiver") << LOG_KV("taskID", m_taskID); + m_config->threadPool()->enqueue([self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + try + { + receiver->runReceiver(); + } + catch (const std::exception& e) + { + receiver->onReceiverException("runTask", e); + } + }); +} + +void LabeledPSIReceiver::handlePsiParams(const front::PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("handlePsiParams") << LOG_KV("taskID", taskID); + + try + { + ppctars::PsiParams tarsParams; + ppctars::serialize::decode(*_message->data(), tarsParams); + m_psiParams = std::make_shared(toPSIParams(tarsParams)); + m_responseCount = tarsParams.binBundleCount; + + m_progress = std::make_shared(m_config->threadPool()); + m_progress->reset(m_responseCount, [self = weak_from_this()]() { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + receiver->onReceiverTaskDone(nullptr); + }); + + // initialize receiver by params + initializeByParams(); + + // do next step + runOprfAsClient(); + } + catch (const std::exception& e) + { + onReceiverException("handlePsiParams", e); + } +} + +void LabeledPSIReceiver::handleEvaluatedItems(const front::PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("handleEvaluatedItems") << LOG_KV("taskID", taskID); + + try + { + ppctars::OprfData evaluatedData; + ppctars::serialize::decode(*_message->data(), evaluatedData); + + // finish oprf + doOprfFinalize(evaluatedData.data); + + // do next step + requestQuery(); + } + catch (const std::exception& e) + { + onReceiverException("handleEvaluatedItems", e); + } +} + +void LabeledPSIReceiver::handleOneResponse(const front::PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("handleOneResponse") << LOG_KV("taskID", taskID); + + try + { + ppctars::QueryResponse queryResponse; + ppctars::serialize::decode(*_message->data(), queryResponse); + + // handle response + decryptResults(queryResponse); + + m_progress->mark(_message->seq()); + } + catch (const std::exception& e) + { + onReceiverException("handleOneResponse", e); + } +} + +void LabeledPSIReceiver::runReceiver() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("request psi params from sender") + << LOG_KV("taskID", m_taskID) << LOG_KV("itemsSize", m_items.size()); + + auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(protocol::PSIAlgorithmType::LABELED_PSI_2PC), m_taskID, + std::make_shared()); + message->setMessageType(uint8_t(LabeledPSIMessageType::PARAMS_REQUEST)); + + // send request + m_config->front()->asyncSendMessage( + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](const bcos::Error::Ptr& _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(_error); + } + }, + nullptr); +} + +void LabeledPSIReceiver::initializeByParams() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("receiver do initialize") << LOG_KV("taskID", m_taskID) + << LOG_KV("psiParams", m_psiParams->to_string()); + + // initialize the CryptoContext with a new SEALContext + m_cryptoContext = apsi::CryptoContext(*m_psiParams); + + // create new keys + generateKeys(); + + // set up the PowersDag + computePowersDag((*m_psiParams).query_params().query_powers); +} + +void LabeledPSIReceiver::generateKeys() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("receiver generateKeys") << LOG_KV("taskID", m_taskID); + + // generate new keys + seal::KeyGenerator generator(*getSealContext()); + + // set the symmetric key, encryptor, and decryptor + m_cryptoContext.set_secret(generator.secret_key()); + + // create Serializable and move to m_relinKeys for storage + m_relinKeys.clear(); + if (getSealContext()->using_keyswitching()) + { + seal::Serializable relinKeys(generator.create_relin_keys()); + m_relinKeys.set(std::move(relinKeys)); + } +} + +std::uint32_t LabeledPSIReceiver::computePowersDag(const std::set& _sourcePowers) +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("receiver do compute PowersDag") + << LOG_KV("taskID", m_taskID); + // first compute the target powers + std::set targetPowers = apsi::util::create_powers_set( + m_psiParams->query_params().ps_low_degree, m_psiParams->table_params().max_items_per_bin); + + // configure the PowersDag + m_powersDag.configure(_sourcePowers, targetPowers); + + // check that the PowersDag is valid + if (!m_powersDag.is_configured()) + { + LABELED_PSI_LOG(ERROR) << LOG_DESC("failed to configure PowersDag") + << LOG_KV("sourcePowers", apsi::util::to_string(_sourcePowers)) + << LOG_KV("targetPowers", apsi::util::to_string(targetPowers)); + + BOOST_THROW_EXCEPTION( + ConfigPowersDagException() << bcos::errinfo_comment("failed to configure PowersDag")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("finish computing PowersDag") + << LOG_KV("PowersDagDepth", m_powersDag.depth()); + return m_powersDag.depth(); +} + +void LabeledPSIReceiver::runOprfAsClient() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("run oprf as client with sender") + << LOG_KV("taskID", m_taskID) << LOG_KV("itemsSize", m_items.size()); + + // blind items + ppctars::OprfData oprfData; + oprfData.data = m_oprfClient->blind(m_items); + + auto blindedMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared()); + blindedMessage->setMessageType(uint8_t(LabeledPSIMessageType::OPRF_BLINDED_ITEMS)); + ppctars::serialize::encode(oprfData, *blindedMessage->data()); + + // send blinded items to sender + m_config->front()->asyncSendMessage( + m_taskState->peerID(), blindedMessage, m_config->networkTimeout(), + [self = weak_from_this()](const bcos::Error::Ptr& _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(_error); + } + }, + nullptr); +} + +void LabeledPSIReceiver::doOprfFinalize(const std::vector& _evaluatedItems) +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("do oprf finalize") << LOG_KV("taskID", m_taskID) + << LOG_KV("itemsSize", m_items.size()); + + auto output = m_oprfClient->finalize(m_items, _evaluatedItems); + + m_oprfOutputs.first.resize(m_items.size()); + m_oprfOutputs.second.resize(m_items.size()); + + auto hashLen = m_oprfOutputs.first[0].value().size(); + auto keyLen = m_oprfOutputs.second[0].size(); + + for (uint32_t idx = 0; idx < m_items.size(); ++idx) + { + std::memcpy(m_oprfOutputs.first[idx].value().data(), output[idx].data(), hashLen); + std::memcpy(m_oprfOutputs.second[idx].data(), output[idx].data() + hashLen, keyLen); + } +} + +void LabeledPSIReceiver::prepareQueryData() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("prepare query data") << LOG_KV("taskID", m_taskID) + << LOG_KV("itemsSize", m_items.size()); + doCuckooHashing(); + + computeEncryptedPower(); +} + +void LabeledPSIReceiver::doCuckooHashing() +{ + kuku::KukuTable cuckoo(m_psiParams->table_params().table_size, // Size of the hash table + 0, // Not using a stash + m_psiParams->table_params().hash_func_count, // Number of hash functions + {0, 0}, // Hardcoded { 0, 0 } as the seed + CUCKOO_TABLE_INSERT_ATTEMPTS, // The number of insertion attempts + {0, 0}); + + LABELED_PSI_LOG(INFO) << LOG_BADGE("runCuckooHash") << LOG_KV("taskID", m_taskID) + << LOG_KV("tableSize", cuckoo.table_size()); + + // Hash the data into a cuckoo hash table + for (size_t idx = 0; idx < m_oprfOutputs.first.size(); ++idx) + { + const auto& item = m_oprfOutputs.first[idx]; + if (!cuckoo.insert(item.get_as().front())) + { + // Insertion can fail for two reasons: + // + // (1) The item was already in the table, in which case the + // "leftover item" is empty; (2) Cuckoo hashing failed due to too + // small table or too few hash functions. + // + // In case (1) simply move on to the next item and log this issue. Case + // (2) is a critical issue so throw an exception. + if (cuckoo.is_empty_item(cuckoo.leftover_item())) + { + LABELED_PSI_LOG(DEBUG) << LOG_DESC("skipping repeated insertion of items") + << LOG_KV("index", idx) << LOG_KV("item", item.to_string()); + } + else + { + LABELED_PSI_LOG(ERROR) + << LOG_DESC("failed to insert item into cuckoo table") << LOG_KV("index", idx) + << LOG_KV("item", item.to_string()) << LOG_KV("fill-rate", cuckoo.fill_rate()); + + onReceiverTaskDone( + std::make_shared((int)LabeledPSIRetCode::CUCKOO_HASH_ERROR, + "failed to insert item into cuckoo table, index: " + std::to_string(idx))); + } + } + } + LABELED_PSI_LOG(INFO) << LOG_BADGE("finishCuckooHash") << LOG_KV("taskID", m_taskID) + << LOG_KV("tableSize", cuckoo.table_size()) + << LOG_KV("fill-rate", cuckoo.fill_rate()); + + // once the table is filled, fill the table_idx_to_item_idx map + m_itt.item_count_ = m_oprfOutputs.first.size(); + for (size_t idx = 0; idx < m_oprfOutputs.first.size(); ++idx) + { + auto item_loc = cuckoo.query(m_oprfOutputs.first[idx].get_as().front()); + m_itt.table_idx_to_item_idx_[item_loc.location()] = idx; + } + + m_kukuData = cuckoo.table(); + + // clear hashed items + std::vector().swap(m_oprfOutputs.first); + MallocExtension::instance()->ReleaseFreeMemory(); +} + +void LabeledPSIReceiver::computeEncryptedPower() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("compute encrypted powers for items") + << LOG_KV("taskID", m_taskID) << LOG_KV("itemsSize", m_items.size()); + + // set up unencrypted query data + std::vector plainPowers; + for (uint32_t idx = 0; idx < m_psiParams->bundle_idx_count(); ++idx) + { + // first, find the items for this bundle index + gsl::span bundleItems( + m_kukuData.data() + idx * m_psiParams->items_per_bundle(), + m_psiParams->items_per_bundle()); + + std::vector algItems; + for (auto& item : bundleItems) + { + // now set up a BitstringView to this item + gsl::span itemBytes( + reinterpret_cast(item.data()), sizeof(item)); + apsi::BitstringView itemBits(itemBytes, m_psiParams->item_bit_count()); + + // create an algebraic item by breaking up the item into modulo plain_modulus parts + std::vector algItem = apsi::util::bits_to_field_elts( + itemBits, m_psiParams->seal_params().plain_modulus()); + copy(algItem.cbegin(), algItem.cend(), back_inserter(algItems)); + } + + // now that we have the algebraized items for this bundle index, we create a PlaintextPowers + // object that computes all necessary powers of the algebraized items. + plainPowers.emplace_back(std::move(algItems), *m_psiParams, m_powersDag); + } + + // the very last thing to do is encrypt the plainPowers and consolidate the matching + // powers for different bundle indices + LABELED_PSI_LOG(INFO) << LOG_DESC("start encrypt powers") << LOG_KV("taskID", m_taskID) + << LOG_KV("bundleCount", m_psiParams->bundle_idx_count()); + + for (uint32_t idx = 0; idx < m_psiParams->bundle_idx_count(); ++idx) + { + // encrypt the data for this power + auto encryptedPowers(plainPowers[idx].encrypt(m_cryptoContext)); + + // move the encrypted data to encrypted_powers + for (auto& ep : encryptedPowers) + { + m_encryptedPowers[ep.first].emplace_back(std::move(ep.second)); + } + } + + // clear kuku items + std::vector().swap(m_kukuData); + MallocExtension::instance()->ReleaseFreeMemory(); +} + +void LabeledPSIReceiver::requestQuery() +{ + prepareQueryData(); + + // construct tars struct for request + std::vector keys; + if (getSealContext()->using_keyswitching()) + { + keys.resize(m_relinKeys.save_size(m_comprMode)); + } + m_relinKeys.save(keys, m_comprMode); + + ppctars::QueryRequest queryRequest; + queryRequest.relinKeys = std::move(keys); + + for (const auto& ep : m_encryptedPowers) + { + ppctars::EncryptedPowers tarsEps; + tarsEps.power = ep.first; + for (const auto& ct : ep.second) + { + std::vector cipher; + cipher.resize(ct.save_size(m_comprMode)); + auto size = ct.save(cipher, m_comprMode); + cipher.resize(size); + tarsEps.ciphertexts.emplace_back(std::move(cipher)); + } + queryRequest.encryptedPowers.emplace_back(std::move(tarsEps)); + } + + auto queryMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared()); + queryMessage->setMessageType(uint8_t(LabeledPSIMessageType::QUERY)); + ppctars::serialize::encode(queryRequest, *queryMessage->data()); + + LABELED_PSI_LOG(INFO) << LOG_DESC("send query request") << LOG_KV("taskID", m_taskID) + << LOG_KV("dataSize", queryMessage->data()->size()); + + // send request + m_config->front()->asyncSendMessage( + m_taskState->peerID(), queryMessage, m_config->networkTimeout(), + [self = weak_from_this()](const bcos::Error::Ptr& _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(_error); + } + }, + nullptr); + + // clear encrypted powers + std::unordered_map>>().swap( + m_encryptedPowers); + MallocExtension::instance()->ReleaseFreeMemory(); + + m_results.resize(m_items.size()); +} + +void LabeledPSIReceiver::decryptResults(const ppctars::QueryResponse& _queryResult) +{ + size_t bundleIdx = _queryResult.bundleIdx; + + LABELED_PSI_LOG(DEBUG) << LOG_DESC("decryptResults") << LOG_KV("taskID", m_taskID) + << LOG_KV("index", bundleIdx); + + // construct ResultPackage + ResultPackage resultPackage; + resultPackage.bundleIdx = bundleIdx; + resultPackage.labelByteCount = _queryResult.labelByteCount; + resultPackage.nonceByteCount = _queryResult.nonceByteCount; + + gsl::span cipherSpan( + reinterpret_cast(_queryResult.ciphertext.data()), + _queryResult.ciphertext.size()); + resultPackage.psiCipherResult.load(getSealContext(), cipherSpan); + + for (const auto& labelResult : _queryResult.labelResults) + { + gsl::span labelSpan( + reinterpret_cast(labelResult.data()), labelResult.size()); + apsi::SEALObject cipher; + cipher.load(getSealContext(), labelSpan); + resultPackage.labelCipherResults.emplace_back(std::move(cipher)); + } + + // decrypt and decode the result; the result vector will have full batch size + PlainResultPackage plainRp = resultPackage.extract(m_cryptoContext); + + // check if we are supposed to have label data present but don't have for some reason + auto labelByteCount = seal::util::safe_cast(plainRp.labelByteCount); + if (labelByteCount && plainRp.labelResults.empty()) + { + onReceiverTaskDone(std::make_shared( + (int)LabeledPSIRetCode::MISSING_LABEL_DATA, "missing label data")); + return; + } + + // read the nonce byte count and compute the effective label byte count; set the nonce byte + // count to zero if no label is expected anyway + size_t nonceByteCount = + labelByteCount ? seal::util::safe_cast(plainRp.nonceByteCount) : 0; + size_t effectiveLabelByteCount = seal::util::add_safe(nonceByteCount, labelByteCount); + + // how much label data did we actually receive? + size_t receivedLabelBitCount = seal::util::mul_safe( + seal::util::safe_cast(m_psiParams->item_bit_count()), plainRp.labelResults.size()); + + // compute the received label byte count and check that it is not less than what was expected + size_t receivedLabelByteCount = receivedLabelBitCount / 8; + + if (receivedLabelByteCount < nonceByteCount) + { + onReceiverTaskDone(std::make_shared((int)LabeledPSIRetCode::LACK_LABEL_DATA, + "nonce bytes, expected: " + std::to_string(nonceByteCount) + + ", received: " + std::to_string(receivedLabelByteCount))); + return; + } + else if (receivedLabelByteCount < effectiveLabelByteCount) + { + onReceiverTaskDone(std::make_shared((int)LabeledPSIRetCode::LACK_LABEL_DATA, + "label bytes, expected: " + std::to_string(labelByteCount) + + ", received: " + std::to_string(receivedLabelByteCount - nonceByteCount))); + return; + } + + auto itemsPerBundle = seal::util::safe_cast(m_psiParams->items_per_bundle()); + size_t bundleStart = + seal::util::mul_safe(seal::util::safe_cast(bundleIdx), itemsPerBundle); + + LABELED_PSI_LOG(DEBUG) << LOG_DESC("decryptResults") << LOG_KV("taskID", m_taskID) + << LOG_KV("bundleStart", bundleStart) << LOG_KV("bundleIdx", bundleIdx); + + auto feltsPerItem = seal::util::safe_cast(m_psiParams->item_params().felts_per_item); + seal::util::StrideIter plainRpIter(plainRp.psiResult.data(), feltsPerItem); + + seal::util::seal_for_each_n( + seal::util::iter(plainRpIter, size_t(0)), itemsPerBundle, [&](auto&& I) { + // find feltsPerItem consecutive zeros + bool match = hasNZeros(std::get<0>(I).ptr(), feltsPerItem); + if (!match) + { + return; + } + + // compute the cuckoo table index for this item. then find the corresponding index + // in the input items vector, so we know where to place the result. + size_t tableIdx = seal::util::add_safe(size_t(std::get<1>(I)), bundleStart); + auto itemIdx = m_itt.find_item_idx(tableIdx); + + if (match) + { + LABELED_PSI_LOG(DEBUG) << LOG_KV("match", match) << LOG_KV("tableIdx", tableIdx) + << LOG_KV("itemIdx", itemIdx); + } + + // ff this tableIdx doesn't match any itemIdx, ignore the result no matter what it is + if (itemIdx == m_itt.item_count()) + { + return; + } + + apsi::Label label; + if (labelByteCount) + { + // collect the entire label into this vector + apsi::util::AlgLabel algLabel; + + size_t labelOffset = seal::util::mul_safe(size_t(std::get<1>(I)), feltsPerItem); + for (auto& labelResult : plainRp.labelResults) + { + gsl::span labelSpan( + labelResult.data() + labelOffset, feltsPerItem); + std::copy(labelSpan.begin(), labelSpan.end(), back_inserter(algLabel)); + } + + // create the label + apsi::EncryptedLabel encryptedLabel = apsi::util::dealgebraize_label( + algLabel, receivedLabelBitCount, m_psiParams->seal_params().plain_modulus()); + + // resize down to the effective byte count + encryptedLabel.resize(effectiveLabelByteCount); + + // decrypt the label + label = apsi::util::decrypt_label( + encryptedLabel, m_oprfOutputs.second[itemIdx], nonceByteCount); + } + + m_results[itemIdx] = + std::make_pair(m_items[itemIdx], std::string(label.begin(), label.end())); + }); +} + +void LabeledPSIReceiver::onReceiverException(const std::string& _module, const std::exception& _e) +{ + LABELED_PSI_LOG(ERROR) << LOG_BADGE("onReceiverException") << LOG_KV("taskID", m_taskID) + << LOG_KV("module", _module) + << LOG_KV("error", boost::diagnostic_information(_e)); + auto error = std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while do " + _module + ", message: " + boost::diagnostic_information(_e)); + onReceiverTaskDone(error); +} + +void LabeledPSIReceiver::onReceiverTaskDone(bcos::Error::Ptr _error) +{ + if (m_taskState->taskDone()) + { + return; + } + LABELED_PSI_LOG(INFO) << LOG_BADGE("onReceiverDone") << LOG_KV("taskID", m_taskID) + << LOG_KV("success", _error == nullptr); + auto result = std::make_shared(m_taskID); + if (_error) + { + result->setError(std::move(_error)); + } + else + { + std::vector> outputs(2); + for (auto& kv : m_results) + { + if (!kv.first.empty()) + { + outputs[0].emplace_back(kv.first); + std::string label = kv.second; + label.erase( + std::remove_if(label.begin(), label.end(), [](char ch) { return ch == '\0'; }), + label.end()); + outputs[1].emplace_back(label); + } + } + result->setOutputs(std::move(outputs)); + } + m_taskState->onTaskFinished(result, true); +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h new file mode 100644 index 00000000..7e392d19 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIReceiver.h + * @author: shawnhe + * @date 2022-11-3 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "ppc-framework/crypto/Oprf.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include "ppc-front/ppc-front/PPCChannel.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/labeled-psi/LabeledPSIConfig.h" +#include "ppc-psi/src/labeled-psi/protocol/Protocol.h" +#include "ppc-tools/src/common/Progress.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace ppc::psi +{ +class LabeledPSIReceiver : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + // construct after calling asyncRequestParams + LabeledPSIReceiver( + LabeledPSIConfig::Ptr _config, TaskState::Ptr _taskState, crypto::OprfClient::Ptr _client); + + ~LabeledPSIReceiver() + { + m_powersDag.reset(); + std::vector().swap(m_items); + std::pair, std::vector>().swap(m_oprfOutputs); + std::vector().swap(m_kukuData); + std::unordered_map>>().swap( + m_encryptedPowers); + std::vector>().swap(m_results); + // release the memory to os + MallocExtension::instance()->ReleaseFreeMemory(); + LABELED_PSI_LOG(INFO) << LOG_DESC("the sender destroyed") << LOG_KV("taskID", m_taskID); + } + + void asyncRunTask(); + + // init receiver by psi params + void handlePsiParams(const front::PPCMessageFace::Ptr& _message); + + // finish oprf and run query + void handleEvaluatedItems(const front::PPCMessageFace::Ptr& _message); + + // handle one response package + void handleOneResponse(const front::PPCMessageFace::Ptr& _message); + + std::shared_ptr getSealContext() const + { + return m_cryptoContext.seal_context(); + } + + const std::string& taskID() const { return m_taskID; } + +protected: + void runReceiver(); + + // init receiver + void initializeByParams(); + + // generates a new set of keys to use for queries + void generateKeys(); + + /** + *Computes the PowersDag. The function returns the depth of the + *PowersDag. In some cases the receiver may want to ensure that the depth of + *the powers computation will be as expected (PowersDag::depth), and + *otherwise attempt to reconfigure the PowersDag. + */ + std::uint32_t computePowersDag(const std::set& _sourcePowers); + + // run oprf as receiver + void runOprfAsClient(); + void doOprfFinalize(const std::vector& _evaluatedItems); + + // process oprf results and send query quest + void requestQuery(); + void prepareQueryData(); + void doCuckooHashing(); + void computeEncryptedPower(); + + // decrypt psi results and labels, handled by per bundle + void decryptResults(const ppctars::QueryResponse& _queryResponse); + + void onReceiverException(const std::string& _module, const std::exception& e); + void onReceiverTaskDone(bcos::Error::Ptr _error); + +private: + LabeledPSIConfig::Ptr m_config; + TaskState::Ptr m_taskState; + crypto::OprfClient::Ptr m_oprfClient; + + std::string m_taskID; + + std::shared_ptr m_psiParams; + + apsi::CryptoContext m_cryptoContext; + apsi::PowersDag m_powersDag; + apsi::SEALObject m_relinKeys; + seal::compr_mode_type m_comprMode{seal::Serialization::compr_mode_default}; + + std::vector m_items; + std::pair, std::vector> m_oprfOutputs; + apsi::receiver::IndexTranslationTable m_itt; + std::vector m_kukuData; + std::unordered_map>> m_encryptedPowers; + + uint32_t m_responseCount{0}; + std::vector> m_results; + + tools::Progress::Ptr m_progress; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp new file mode 100644 index 00000000..6cf086f5 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp @@ -0,0 +1,461 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSISender.cpp + * @author: shawnhe + * @date 2022-11-6 + */ + + +#include +#include + +#include "LabeledPSIParams.h" +#include "LabeledPSISender.h" +#include "QueryPackage.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "ppc-tools/src/common/TransTools.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace ppc::front; +using namespace ppc::tools; +using namespace ppc::crypto; + +LabeledPSISender::LabeledPSISender(LabeledPSIConfig::Ptr _config, + std::function _taskEndTrigger) + : m_config(std::move(_config)), m_taskEndTrigger(std::move(_taskEndTrigger)) +{ + m_messageFactory = std::make_shared(); +} + +void LabeledPSISender::handlePsiParamsRequest(const PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("handlePsiParamsRequest") << LOG_KV("taskID", taskID); + + try + { + auto tarsParams = fromPSIParams(m_senderDB->getParams(), m_senderDB->getBinBundleCount()); + auto paramsMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared()); + paramsMessage->setMessageType(uint8_t(LabeledPSIMessageType::PARAMS_RESPONSE)); + ppctars::serialize::encode(tarsParams, *paramsMessage->data()); + + // send psi params response + m_config->front()->asyncSendMessage( + _message->sender(), paramsMessage, m_config->networkTimeout(), + [self = weak_from_this(), taskID](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(taskID, std::move(_error)); + } + }, + nullptr); + } + catch (const std::exception& e) + { + onSenderException(taskID, "handlePsiParamsRequest", e); + } +} + +void LabeledPSISender::handleBlindedItems(const front::PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("handleBlindedItems") << LOG_KV("taskID", taskID); + + try + { + ppctars::OprfData blindedData; + ppctars::serialize::decode(*_message->data(), blindedData); + + // evaluate items + ppctars::OprfData evaluatedData; + evaluatedData.data = m_config->oprfServer()->evaluate(blindedData.data); + + auto evaluatedMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared()); + evaluatedMessage->setMessageType(uint8_t(LabeledPSIMessageType::OPRF_EVALUATED_ITEMS)); + ppctars::serialize::encode(evaluatedData, *evaluatedMessage->data()); + + // send evaluated items + m_config->front()->asyncSendMessage( + _message->sender(), evaluatedMessage, m_config->networkTimeout(), + [self = weak_from_this(), taskID](bcos::Error::Ptr _error) { + auto sender = self.lock(); + if (!sender) + { + return; + } + if (_error && _error->errorCode()) + { + sender->onSenderTaskDone(taskID, std::move(_error)); + } + }, + nullptr); + } + catch (const std::exception& e) + { + onSenderException(taskID, "handleBlindedItems", e); + } +} + +// receive encrypted powers and send polynomial ciphertext response +void LabeledPSISender::handleQuery(const front::PPCMessageFace::Ptr& _message) +{ + auto taskID = _message->taskID(); + LABELED_PSI_LOG(INFO) << LOG_BADGE("onReceiveQueryRequest") << LOG_KV("taskID", taskID); + + try + { + ppctars::QueryRequest queryRequest; + ppctars::serialize::decode(*(_message->data()), queryRequest); + + // construct queryPackage + QueryPackage queryPackage(queryRequest, m_senderDB); + + // acquire read lock on SenderDB + auto senderDBlock = m_senderDB->getReaderLock(); + + LABELED_PSI_LOG(INFO) << LOG_DESC("start processing query request on database") + << LOG_KV("taskID", taskID) + << LOG_KV("itemSize", m_senderDB->getItemCount()); + + // Copy over the CryptoContext from SenderDB; set the Evaluator for this local + // instance. Relinearization keys may not have been included in the query. In + // that case query.relin_keys() simply holds an empty seal::RelinKeys + // instance. There is no problem with the below call to CryptoContext::set_evaluator. + apsi::CryptoContext cryptoContext(m_senderDB->getCryptoContext()); + cryptoContext.set_evaluator(queryPackage.relinKeys()); + + uint32_t bundleIdxCount = m_senderDB->getParams().bundle_idx_count(); + uint32_t maxItemsPerBin = m_senderDB->getParams().table_params().max_items_per_bin; + + // For each bundle index i, we need a vector of powers of the query Qᵢ. We + // need powers all the way up to Qᵢ^max_items_per_bin. We don't store the + // zeroth power. If Paterson-Stockmeyer is used, then only a subset of the + // powers will be populated. + auto allPowers = + std::make_shared>>(bundleIdxCount); + + // we use a custom SEAL memory that is freed after the query is done + auto memoryPool = std::make_shared( + seal::MemoryManager::GetPool(seal::mm_force_new)); + + // Initialize powers + for (auto& powers : *allPowers) + { + // The + 1 is because we index by power. The 0th power is a dummy value. I + // promise this makes things easier to read. + size_t powersSize = static_cast(maxItemsPerBin) + 1; + powers.reserve(powersSize); + for (size_t i = 0; i < powersSize; i++) + { + powers.emplace_back(*memoryPool); + } + } + + // Load inputs provided in the query + for (auto& powerIt : queryPackage.powers()) + { + // The exponent of all the query powers we're about to iterate through + auto exponent = static_cast(powerIt.first); + + // Load Qᵢᵉ for all bundle indices i, where e is the exponent specified + // above + for (size_t bundleIdx = 0; bundleIdx < bundleIdxCount; ++bundleIdx) + { + // Load input^power to allPowers[bundle_idx][exponent] + LABELED_PSI_LOG(DEBUG) + << LOG_DESC("extracting query ciphertext") << LOG_KV("power", exponent) + << LOG_KV("bundleIndex", bundleIdx); + + (*allPowers)[bundleIdx][exponent] = powerIt.second[bundleIdx]; + } + } + + auto receiverID = _message->sender(); + + // Compute query powers for the bundle indexes + for (size_t bundleIdx = 0; bundleIdx < bundleIdxCount; ++bundleIdx) + { + computePowers(taskID, receiverID, cryptoContext, allPowers, memoryPool, + queryPackage.powersDag(), static_cast(bundleIdx)); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start processing bin bundle caches") + << LOG_KV("taskID", taskID); + uint32_t seq = 0; + auto current = std::make_shared>(0); + uint32_t total = m_senderDB->getBinBundleCount(); + for (size_t bundleIdx = 0; bundleIdx < bundleIdxCount; ++bundleIdx) + { + auto bundleCaches = m_senderDB->getCacheAt(static_cast(bundleIdx)); + for (auto& cache : bundleCaches) + { + m_config->threadPool()->enqueue([self = weak_from_this(), taskID, receiverID, + allPowers, memoryPool, cryptoContext, cache, + bundleIdx, seq, current, total]() { + auto sender = self.lock(); + if (!sender) + { + return; + } + + try + { + sender->processBinBundleCache(taskID, receiverID, cryptoContext, allPowers, + memoryPool, cache, bundleIdx, seq); + + auto currentStep = current->fetch_add(1); + if (currentStep + 1 == total) + { + // current task is done for sender + sender->onSenderTaskDone(taskID, nullptr); + } + } + catch (const std::exception& e) + { + sender->onSenderException(taskID, + "processBinBundleCache, bundleIdx: " + std::to_string(bundleIdx), e); + } + }); + seq++; + } + } + } + catch (const std::exception& e) + { + onSenderException(taskID, "handleBlindedItems", e); + } +} + + +void LabeledPSISender::computePowers(const std::string& _taskID, const std::string& _receiverID, + const apsi::CryptoContext& _cryptoContext, + std::shared_ptr>> _allPowers, + std::shared_ptr _memoryPool, const apsi::PowersDag& _powersDag, + uint32_t _bundleIdx) +{ + auto bundleCaches = m_senderDB->getCacheAt(_bundleIdx); + if (bundleCaches.empty()) + { + return; + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start computing powers") << LOG_KV("taskID", _taskID) + << LOG_KV("bundleIdx", _bundleIdx); + + auto evaluator = _cryptoContext.evaluator(); + auto relinKeys = _cryptoContext.relin_keys(); + bool relinearize = _cryptoContext.seal_context()->using_keyswitching(); + + _powersDag.parallel_apply([&](const apsi::PowersDag::PowersNode& node) { + if (!node.is_source()) + { + auto parents = node.parents; + seal::Ciphertext prod(*_memoryPool); + if (parents.first == parents.second) + { + evaluator->square((*_allPowers)[_bundleIdx][parents.first], prod, *_memoryPool); + } + else + { + evaluator->multiply((*_allPowers)[_bundleIdx][parents.first], + (*_allPowers)[_bundleIdx][parents.second], prod, *_memoryPool); + } + if (relinearize) + { + evaluator->relinearize_inplace(prod, *relinKeys, *_memoryPool); + } + (*_allPowers)[_bundleIdx][node.power] = std::move(prod); + } + }); + + // Now that all powers of the ciphertext have been computed, we need to + // transform them to NTT form. This will substantially improve the polynomial + // evaluation, because the plaintext polynomials are already in NTT + // transformed form, and the ciphertexts are used repeatedly for each bin + // bundle at this index. This computation is separate from the graph + // processing above, because the multiplications must all be done before + // transforming to NTT form. We omit the first ciphertext in the vector, + // because it corresponds to the zeroth power of the query and is included + // only for convenience of the indexing; the ciphertext is actually not set or + // valid for use. + + // After computing all powers we do modules switch down to parameters that + // one more level for low powers than for high powers; same choice must be + // used when encoding/NTT transforming the SenderDB data. + auto highPowersParmsId = + apsi::util::get_parms_id_for_chain_idx(*_cryptoContext.seal_context(), 1); + auto lowPowersParmsId = + apsi::util::get_parms_id_for_chain_idx(*_cryptoContext.seal_context(), 2); + + uint32_t psLowDegree = m_senderDB->getParams().query_params().ps_low_degree; + + std::vector targetPowers; + for (auto& power : _powersDag.target_powers()) + { + targetPowers.emplace_back(power); + } + + tbb::parallel_for(tbb::blocked_range(0U, targetPowers.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + uint32_t power = targetPowers[i]; + + if (!psLowDegree) + { + // Only one ciphertext-plaintext multiplication is needed after this + evaluator->mod_switch_to_inplace( + (*_allPowers)[_bundleIdx][power], highPowersParmsId, *_memoryPool); + + // All powers must be in NTT form + evaluator->transform_to_ntt_inplace((*_allPowers)[_bundleIdx][power]); + } + else + { + if (power <= psLowDegree) + { + // Low powers must be at a higher level than high powers + evaluator->mod_switch_to_inplace( + (*_allPowers)[_bundleIdx][power], lowPowersParmsId, *_memoryPool); + + // Low powers must be in NTT form + evaluator->transform_to_ntt_inplace((*_allPowers)[_bundleIdx][power]); + } + else + { + // High powers are only modulus switched + evaluator->mod_switch_to_inplace( + (*_allPowers)[_bundleIdx][power], highPowersParmsId, *_memoryPool); + } + } + } + }); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finish computing powers") << LOG_KV("taskID", _taskID) + << LOG_KV("bundleIdx", _bundleIdx); +} + +void LabeledPSISender::processBinBundleCache(const std::string& _taskID, + const std::string& _receiverID, const apsi::CryptoContext& _cryptoContext, + const std::shared_ptr>>& _allPowers, + const std::shared_ptr& _memoryPool, + std::reference_wrapper _cache, uint32_t _bundleIdx, uint32_t _seq) +{ + LABELED_PSI_LOG(DEBUG) << LOG_DESC("processBinBundleCache") << LOG_KV("taskID", _taskID) + << LOG_KV("bundleIdx", _bundleIdx); + + // Package for the result data + ResultPackage resultPackage; + resultPackage.bundleIdx = _bundleIdx; + resultPackage.nonceByteCount = seal::util::safe_cast(m_senderDB->getNonceByteCount()); + resultPackage.labelByteCount = seal::util::safe_cast(m_senderDB->getLabelByteCount()); + + // Compute the matching result and move to rp + const BatchedPlaintextPolyn& matchingPolyn = _cache.get().batched_matching_polyn; + + // Determine if we use Paterson-Stockmeyer or not + uint32_t psLowDegree = m_senderDB->getParams().query_params().ps_low_degree; + uint32_t degree = seal::util::safe_cast(matchingPolyn.batched_coeffs.size()) - 1; + + bool usingPs = (psLowDegree > 1) && (psLowDegree < degree); + if (usingPs) + { + resultPackage.psiCipherResult = matchingPolyn.eval_patstock(_cryptoContext, + (*_allPowers)[_bundleIdx], seal::util::safe_cast(psLowDegree), *_memoryPool); + } + else + { + resultPackage.psiCipherResult = matchingPolyn.eval((*_allPowers)[_bundleIdx], *_memoryPool); + } + + for (const auto& interpPolyn : _cache.get().batched_interp_polyns) + { + // Compute the label result and move to rp + degree = seal::util::safe_cast(interpPolyn.batched_coeffs.size()) - 1; + usingPs = (psLowDegree > 1) && (psLowDegree < degree); + if (usingPs) + { + resultPackage.labelCipherResults.emplace_back(interpPolyn.eval_patstock( + _cryptoContext, (*_allPowers)[_bundleIdx], psLowDegree, *_memoryPool)); + } + else + { + resultPackage.labelCipherResults.emplace_back( + interpPolyn.eval((*_allPowers)[_bundleIdx], *_memoryPool)); + } + } + + // send result package to receiver + sendResultPackage(_taskID, _receiverID, resultPackage, _seq); +} + +void LabeledPSISender::sendResultPackage(const std::string& _taskID, const std::string& _receiverID, + const ResultPackage& _resultPackage, uint32_t _seq) +{ + LABELED_PSI_LOG(DEBUG) << LOG_DESC("send result package") << LOG_KV("taskID", _taskID) + << LOG_KV("seq", _seq); + + ppctars::QueryResponse queryResponse; + queryResponse.labelByteCount = _resultPackage.labelByteCount; + queryResponse.nonceByteCount = _resultPackage.nonceByteCount; + queryResponse.bundleIdx = _resultPackage.bundleIdx; + + std::vector psiCipher(_resultPackage.psiCipherResult.save_size(m_comprMode)); + _resultPackage.psiCipherResult.save(psiCipher, m_comprMode); + queryResponse.ciphertext = std::move(psiCipher); + + for (auto& result : _resultPackage.labelCipherResults) + { + std::vector labelCipher(result.save_size(m_comprMode)); + result.save(labelCipher, m_comprMode); + queryResponse.labelResults.emplace_back(std::move(labelCipher)); + } + + auto resultMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI), + uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), _taskID, std::make_shared()); + resultMessage->setMessageType(uint8_t(LabeledPSIMessageType::RESPONSE)); + resultMessage->setSeq(_seq); + ppctars::serialize::encode(queryResponse, *resultMessage->data()); + + auto error = m_config->sendMessage(_receiverID, resultMessage); + if (error && error->errorCode()) + { + onSenderTaskDone(_taskID, std::move(error)); + } +} + +void LabeledPSISender::onSenderException( + const std::string& _taskID, const std::string& _module, const std::exception& _e) +{ + LABELED_PSI_LOG(ERROR) << LOG_BADGE("onSenderException") << LOG_KV("taskID", _taskID) + << LOG_KV("module", _module) + << LOG_KV("error", boost::diagnostic_information(_e)); + auto error = std::make_shared((int)LabeledPSIRetCode::ON_EXCEPTION, + "exception caught while do " + _module + ", message: " + boost::diagnostic_information(_e)); + m_taskEndTrigger(_taskID, error); +} + +void LabeledPSISender::onSenderTaskDone(const std::string& _taskID, bcos::Error::Ptr _error) +{ + m_taskEndTrigger(_taskID, std::move(_error)); +} diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h new file mode 100644 index 00000000..32497157 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h @@ -0,0 +1,93 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSISender.h + * @author: shawnhe + * @date 2022-11-6 + */ + + +#pragma once + +#include + +#include +#include +#include +#include + +#include "ResultPackage.h" +#include "SenderDB.h" +#include "ppc-framework/crypto/Oprf.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-front/ppc-front/PPCChannel.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/labeled-psi/LabeledPSIConfig.h" +#include "ppc-psi/src/labeled-psi/protocol/Protocol.h" + +namespace ppc::psi +{ +class LabeledPSISender : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + virtual ~LabeledPSISender() = default; + + LabeledPSISender(LabeledPSIConfig::Ptr _config, + std::function _taskEndTrigger); + +public: + // send psi params for some task + void handlePsiParamsRequest(const front::PPCMessageFace::Ptr& _message); + + // receive blinded items and send evaluated items + void handleBlindedItems(const front::PPCMessageFace::Ptr& _message); + + // receive encrypted powers and send polynomial ciphertext response + void handleQuery(const front::PPCMessageFace::Ptr& _message); + + void setSenderDB(SenderDB::Ptr _senderDB) { m_senderDB = std::move(_senderDB); } + +protected: + void computePowers(const std::string& _taskID, const std::string& _receiverID, + const apsi::CryptoContext& _cryptoContext, + std::shared_ptr>> _allPowers, + std::shared_ptr _memoryPool, const apsi::PowersDag& _powersDag, + uint32_t _bundleIdx); + void processBinBundleCache(const std::string& _taskID, const std::string& _receiverID, + const apsi::CryptoContext& _cryptoContext, + const std::shared_ptr>>& _allPowers, + const std::shared_ptr& _memoryPool, + std::reference_wrapper _cache, uint32_t _bundleIdx, uint32_t _seq); + void sendResultPackage(const std::string& _taskID, const std::string& _receiverID, + const ResultPackage& _resultPackage, uint32_t _seq); + + void onSenderException( + const std::string& _taskID, const std::string& _module, const std::exception& _e); + void onSenderTaskDone(const std::string& _taskID, bcos::Error::Ptr _error); + +private: + LabeledPSIConfig::Ptr m_config; + + std::function m_taskEndTrigger; + + front::PPCMessageFactory::Ptr m_messageFactory; + + // set after initialization of sender + SenderDB::Ptr m_senderDB; + + seal::compr_mode_type m_comprMode{seal::Serialization::compr_mode_default}; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp b/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp new file mode 100644 index 00000000..c1e86738 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp @@ -0,0 +1,128 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file QueryPackage.cpp + * @author: shawnhe + * @date 2022-11-14 + * + */ + +#include +#include +#include + +#include "QueryPackage.h" +#include "ppc-psi/src/labeled-psi/Common.h" + +using namespace ppc::psi; + +QueryPackage::QueryPackage(ppctars::QueryRequest _queryQuest, SenderDB::Ptr _senderDB) +{ + auto sealContext = _senderDB->getSealContext(); + + // extract and validate relinearization keys + apsi::SEALObject relinKeys; + if (sealContext->using_keyswitching()) + { + gsl::span relinKeysSpan( + reinterpret_cast(_queryQuest.relinKeys.data()), + _queryQuest.relinKeys.size()); + relinKeys.load(sealContext, relinKeysSpan); + m_relinKeys = relinKeys.extract(sealContext); + if (!is_valid_for(m_relinKeys, *sealContext)) + { + BOOST_THROW_EXCEPTION(ExtracteRelinKeysException() << bcos::errinfo_comment( + "Extracted RelinKeys are invalid for SEALContext")); + } + } + + // extract and validate query ciphertexts + for (auto& encryptedPowers : _queryQuest.encryptedPowers) + { + std::vector ciphertexts; + ciphertexts.reserve(encryptedPowers.ciphertexts.size()); + for (auto& ct : encryptedPowers.ciphertexts) + { + gsl::span ctSpan(reinterpret_cast(ct.data()), ct.size()); + apsi::SEALObject temp; + temp.load(sealContext, ctSpan); + ciphertexts.emplace_back(temp.extract(sealContext)); + if (!is_valid_for(ciphertexts.back(), *sealContext)) + { + BOOST_THROW_EXCEPTION(ExtracteCiphertextException() << bcos::errinfo_comment( + "Extracted ciphertext is invalid for SEALContext")); + } + } + m_powers.emplace(encryptedPowers.power, std::move(ciphertexts)); + } + + uint32_t psLowDegree = _senderDB->getParams().query_params().ps_low_degree; + uint32_t maxItemsPerBin = _senderDB->getParams().table_params().max_items_per_bin; + + std::set targetPowers = create_powers_set(psLowDegree, maxItemsPerBin); + const std::set& queryPowers = _senderDB->getParams().query_params().query_powers; + + // create the PowersDag + m_powersDag.configure(queryPowers, targetPowers); + + // check that the PowersDag is valid + if (!m_powersDag.is_configured()) + { + BOOST_THROW_EXCEPTION(PowerDagException() << bcos::errinfo_comment( + "Failed to configure PowersDag, queryPowers: " + + apsi::util::to_string(queryPowers) + + ", targetPowers: " + apsi::util::to_string(targetPowers))); + } + LABELED_PSI_LOG(INFO) << LOG_DESC("finished configuring PowersDag") + << LOG_KV("depth", m_powersDag.depth()); + + // check that the query data size matches the PSIParams + + if (m_powers.size() != queryPowers.size()) + { + BOOST_THROW_EXCEPTION( + QueryPowersException() << bcos::errinfo_comment( + "Extracted query data is incompatible with PSI parameters, powderDag contains " + + std::to_string(m_powers.size()) + + " ciphertext powers which does not match with the size of queryPowers (" + + std::to_string(queryPowers.size()) + ")")); + } + uint32_t bundleIdxCount = _senderDB->getParams().bundle_idx_count(); + for (auto& powers : m_powers) + { + // check that powers in the query data match source nodes in the PowersDag + if (powers.second.size() != bundleIdxCount) + { + BOOST_THROW_EXCEPTION( + QueryPowersException() << bcos::errinfo_comment( + "Extracted query data is incompatible with PSI parameters, queryPower " + + std::to_string(powers.first) + " contains " + + std::to_string(powers.second.size()) + + " ciphertexts which does not match with bundle_idx_count (" + + std::to_string(bundleIdxCount) + ")")); + } + + auto where = find_if(queryPowers.cbegin(), queryPowers.cend(), + [&powers](auto n) { return n == powers.first; }); + if (where == queryPowers.cend()) + { + BOOST_THROW_EXCEPTION( + QueryPowersException() << bcos::errinfo_comment( + "Extracted query data is incompatible with PowersDag, queryPower " + + std::to_string(powers.first) + + " does not match with a source node in PowersDag")); + } + } +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h b/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h new file mode 100644 index 00000000..87579bfc --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file QueryPackage.h + * @author: shawnhe + * @date 2022-11-14 + * + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include "LabeledPSI.h" +#include "SenderDB.h" +#include "ppc-psi/src/labeled-psi/Common.h" + + +namespace ppc::psi +{ +class QueryPackage +{ +public: + QueryPackage() = default; + + QueryPackage(ppctars::QueryRequest _queryQuest, SenderDB::Ptr _senderDB); + + const seal::RelinKeys& relinKeys() { return m_relinKeys; } + + const std::unordered_map>& powers() + { + return m_powers; + } + + const apsi::PowersDag& powersDag() const noexcept { return m_powersDag; } + +private: + seal::RelinKeys m_relinKeys; + + std::unordered_map> m_powers; + + apsi::PowersDag m_powersDag; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp new file mode 100644 index 00000000..d425f734 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ResultPackage.cpp + * @author: shawnhe + * @date 2022-11-6 + */ + +#include "ResultPackage.h" +#include "ppc-psi/src/labeled-psi/Common.h" + +using namespace ppc::psi; + +PlainResultPackage ResultPackage::extract(const apsi::CryptoContext& _cryptoContext) +{ + if (!_cryptoContext.decryptor()) + { + BOOST_THROW_EXCEPTION(ResultPackageException() << bcos::errinfo_comment( + "Decryptor is not configured in CryptoContext")); + } + + PlainResultPackage plainRp; + plainRp.bundleIdx = bundleIdx; + plainRp.labelByteCount = labelByteCount; + plainRp.nonceByteCount = nonceByteCount; + + // psi + seal::Plaintext psiPlain; + seal::Ciphertext psiCipher = psiCipherResult.extract(_cryptoContext.seal_context()); + _cryptoContext.decryptor()->decrypt(psiCipher, psiPlain); + _cryptoContext.encoder()->decode(psiPlain, plainRp.psiResult); + + // labels + for (auto& ct : labelCipherResults) + { + seal::Plaintext labelPlain; + seal::Ciphertext labelCipher = ct.extract(_cryptoContext.seal_context()); + _cryptoContext.decryptor()->decrypt(labelCipher, labelPlain); + + std::vector labelResultData; + _cryptoContext.encoder()->decode(labelPlain, labelResultData); + plainRp.labelResults.emplace_back(std::move(labelResultData)); + } + + // clear the label data + std::vector>().swap(labelCipherResults); + + return plainRp; +} diff --git a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h new file mode 100644 index 00000000..7e0d32f0 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ResultPackage.h + * @author: shawnhe + * @date 2022-11-6 + */ + +#pragma once +#include +#include + +#include +#include +#include +#include + +namespace ppc::psi +{ + +/** +Stores a decrypted and decoded PSI response and optionally a labeled PSI response. +*/ +struct PlainResultPackage +{ + std::uint32_t bundleIdx; + + std::vector psiResult; + + std::uint32_t labelByteCount; + + std::uint32_t nonceByteCount; + + std::vector> labelResults; +}; + +/** +Stores a PSI response and optionally labeled PSI response ciphertexts. +*/ +struct ResultPackage +{ +public: + PlainResultPackage extract(const apsi::CryptoContext& crypto_context); + + std::uint32_t bundleIdx; + + apsi::SEALObject psiCipherResult; + + std::uint32_t labelByteCount; + + std::uint32_t nonceByteCount; + + std::vector> labelCipherResults; +}; + + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp new file mode 100644 index 00000000..567176f5 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp @@ -0,0 +1,1184 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @reference https://github.com/microsoft/APSI/sender/sender_db.cpp + * @license MIT license + * @change optimize the implementation of some functions & replace the instance of oprf + * + * @file SenderDB.h + * @author: shawnhe + * @date 2022-11-7 + * + */ + +// APSI +#include +#include +#include +#include + +// Kuku +#include "kuku/locfunc.h" + +// SEAL +#include "seal/util/common.h" +#include "seal/util/streambuf.h" + +#include +#include +#include + +#include "SenderDB.h" +#include "ppc-psi/src/labeled-psi/Common.h" + +#include "../protocol/Protocol.h" + +using namespace ppc; +using namespace ppc::crypto; +using namespace ppc::psi; + +/** +Creates and returns the vector of hash functions similarly to how Kuku 2.x sets +them internally. +*/ +inline std::vector hashFunctions(const apsi::PSIParams& params) +{ + std::vector result; + for (uint32_t i = 0; i < params.table_params().hash_func_count; i++) + { + result.emplace_back(params.table_params().table_size, kuku::make_item(i, 0)); + } + + return result; +} + +/** +Computes all cuckoo hash table locations for a given item. +*/ +inline std::unordered_set allLocations( + const std::vector& hash_funcs, const apsi::HashedItem& item) +{ + std::unordered_set result; + for (auto& hf : hash_funcs) + { + result.emplace(hf(item.get_as().front())); + } + + return result; +} + +/** +Compute the label size in multiples of item-size chunks. +*/ +inline size_t computeLabelSize(size_t label_byte_count, const apsi::PSIParams& params) +{ + return (label_byte_count * 8 + params.item_bit_count() - 1) / params.item_bit_count(); +} + +/** +Unpacks a cuckoo idx into its bin and bundle indices +*/ +inline std::pair unpackCuckooIdx(size_t cuckoo_idx, size_t bins_per_bundle) +{ + // Recall that bin indices are relative to the bundle index. That is, the + // first bin index of a bundle at bundle index 5 is 0. A cuckoo index is + // similar, except it is not relative to the bundle index. It just keeps + // counting past bundle boundaries. So in order to get the bin index from the + // cuckoo index, just compute cuckoo_idx (mod bins_per_bundle). + size_t bin_idx = cuckoo_idx % bins_per_bundle; + + // Compute which bundle index this cuckoo index belongs to + size_t bundle_idx = (cuckoo_idx - bin_idx) / bins_per_bundle; + + return {bin_idx, bundle_idx}; +} + +/** +Compute oprf hash for items and encrypt labels +*/ +inline std::vector> preprocessItemsAndLabels( + ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels, size_t _labelByteCount, + size_t _nonceByteCount, crypto::OprfServer::Ptr _oprfServer) +{ + // do oprf + size_t dataSize = _items->size(); + std::vector oprfOut; + oprfOut.reserve(dataSize); + oprfOut.resize(dataSize); + + tbb::parallel_for(tbb::blocked_range(0U, dataSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + oprfOut[i] = _oprfServer->fullEvaluate(_items->getBytes(i)); + } + }); + + std::vector> results(dataSize); + tbb::parallel_for(tbb::blocked_range(0U, dataSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); ++i) + { + // the left half is item hash + apsi::HashedItem hashedItem; + std::memcpy(hashedItem.value().data(), oprfOut[i].data(), hashedItem.value().size()); + + // the right half is key for label encryption + apsi::LabelKey key; + std::memcpy(key.data(), oprfOut[i].data() + hashedItem.value().size(), key.size()); + + // encrypt labels + apsi::EncryptedLabel encryptedLabel = + encrypt_label(_labels->getBytes(i), key, _labelByteCount, _nonceByteCount); + + results[i] = std::make_pair(hashedItem, encryptedLabel); + } + }); + + return results; +} + +inline std::vector> preprocessItemsAndLabels( + const std::vector>& _pairs, size_t _labelByteCount, + size_t _nonceByteCount, crypto::OprfServer::Ptr _oprfServer) +{ + // do oprf + size_t dataSize = _pairs.size(); + std::vector oprfOut; + oprfOut.reserve(dataSize); + oprfOut.resize(dataSize); + + tbb::parallel_for(tbb::blocked_range(0U, dataSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + oprfOut[i] = _oprfServer->fullEvaluate( + bcos::bytes(_pairs[i].first.value().begin(), _pairs[i].first.value().end())); + } + }); + + std::vector> results(dataSize); + + tbb::parallel_for(tbb::blocked_range(0U, dataSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); ++i) + { + // the left half is item hash + apsi::HashedItem hashedItem; + std::memcpy(hashedItem.value().data(), oprfOut[i].data(), hashedItem.value().size()); + + // the right half is key for label encryption + apsi::LabelKey key; + std::memcpy(key.data(), oprfOut[i].data() + hashedItem.value().size(), key.size()); + + // encrypt labels + apsi::EncryptedLabel encryptedLabel = + encrypt_label(_pairs[i].second, key, _labelByteCount, _nonceByteCount); + + results[i] = std::make_pair(hashedItem, encryptedLabel); + } + }); + + return results; +} + +/** +Compute oprf hash for items +*/ +inline std::vector preprocessItems( + const std::vector& _items, crypto::OprfServer::Ptr _oprfServer) +{ + // do oprf + size_t dataSize = _items.size(); + std::vector oprfOut; + oprfOut.reserve(dataSize); + oprfOut.resize(dataSize); + tbb::parallel_for(tbb::blocked_range(0U, dataSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); i++) + { + std::string itemStr(_items[i].value().size(), 0); + std::memcpy(itemStr.data(), _items[i].value().data(), _items[i].value().size()); + oprfOut[i] = _oprfServer->fullEvaluate(itemStr); + } + }); + + std::vector results; + for (auto& hash : oprfOut) + { + // truncate hash + apsi::Item::value_type value{}; + std::memcpy(value.data(), hash.data(), value.size()); + results.emplace_back(value); + } + + return results; +} + +/** +Converts each given Item-Label pair in between the given iterators into its +algebraic form, i.e., a sequence of felt-felt pairs. Also computes each Item's +cuckoo index. +*/ +std::vector> transformLabeledData( + const std::vector>::const_iterator begin, + const std::vector>::const_iterator end, + const apsi::PSIParams& params) +{ + LABELED_PSI_LOG(DEBUG) << LOG_DESC("start transforming labeled data") + << LOG_KV("count", distance(begin, end)); + + // Some variables we'll need + size_t bins_per_item = params.item_params().felts_per_item; + size_t item_bit_count = params.item_bit_count(); + + // Set up Kuku hash functions + auto hash_funcs = hashFunctions(params); + + // Calculate the cuckoo indices for each item. Store every pair of + // (item-label, cuckoo_idx) in a vector. Later, we're gonna sort this vector + // by cuckoo_idx and use the result to parallelize the work of inserting the + // items into BinBundles. + std::vector> data_with_indices; + for (auto it = begin; it != end; it++) + { + const std::pair& item_label_pair = *it; + + // Serialize the data into field elements + const apsi::HashedItem& item = item_label_pair.first; + const apsi::EncryptedLabel& label = item_label_pair.second; + apsi::util::AlgItemLabel alg_item_label = algebraize_item_label( + item, label, item_bit_count, params.seal_params().plain_modulus()); + + // Get the cuckoo table locations for this item and add to data_with_indices + for (auto location : allLocations(hash_funcs, item)) + { + // The current hash value is an index into a table of Items. In reality + // our BinBundles are tables of bins, which contain chunks of items. How + // many chunks? bins_per_item many chunks + size_t bin_idx = location * bins_per_item; + + // Store the data along with its index + data_with_indices.emplace_back(alg_item_label, bin_idx); + } + } + + LABELED_PSI_LOG(DEBUG) << LOG_DESC("finished transforming labeled data") + << LOG_KV("count", distance(begin, end)); + + return data_with_indices; +} + +/** +Converts each given Item into its algebraic form, i.e., a sequence of +felt-monostate pairs. Also computes each Item's cuckoo index. +*/ +std::vector> transformUnlabeledData( + const std::vector::const_iterator begin, + const std::vector::const_iterator end, const apsi::PSIParams& params) +{ + LABELED_PSI_LOG(DEBUG) << LOG_DESC("start transforming unlabeled data") + << LOG_KV("count", distance(begin, end)); + + // Some variables we'll need + size_t bins_per_item = params.item_params().felts_per_item; + size_t item_bit_count = params.item_bit_count(); + + // Set up Kuku hash functions + auto hash_funcs = hashFunctions(params); + + // Calculate the cuckoo indices for each item. Store every pair of + // (item-label, cuckoo_idx) in a vector. Later, we're gonna sort this vector + // by cuckoo_idx and use the result to parallelize the work of inserting the + // items into BinBundles. + std::vector> data_with_indices; + for (auto it = begin; it != end; it++) + { + const apsi::HashedItem& item = *it; + + // Serialize the data into field elements + apsi::util::AlgItem alg_item = + algebraize_item(item, item_bit_count, params.seal_params().plain_modulus()); + + // Get the cuckoo table locations for this item and add to data_with_indices + for (auto location : allLocations(hash_funcs, item)) + { + // The current hash value is an index into a table of Items. In reality + // our BinBundles are tables of bins, which contain chunks of items. How + // many chunks? bins_per_item many chunks + size_t bin_idx = location * bins_per_item; + + // Store the data along with its index + data_with_indices.emplace_back(std::make_pair(alg_item, bin_idx)); + } + } + + LABELED_PSI_LOG(DEBUG) << LOG_DESC("finished transforming unlabeled data") + << LOG_KV("count", distance(begin, end)); + + return data_with_indices; +} + +/** +Converts given Item into its algebraic form, i.e., a sequence of felt-monostate +pairs. Also computes the Item's cuckoo index. +*/ +std::vector> transformUnlabeledData( + const apsi::HashedItem& item, const apsi::PSIParams& params) +{ + std::vector item_singleton{item}; + return transformUnlabeledData(item_singleton.begin(), item_singleton.end(), params); +} + +/** +Inserts the given items and corresponding labels into bin_bundles at their +respective cuckoo indices. It will only insert the data with bundle index in the +half-open range range indicated by work_range. If inserting into a BinBundle +would make the number of items in a bin larger than max_bin_size, this function +will create and insert a new BinBundle. If overwrite is set, this will overwrite +the labels if it finds an AlgItemLabel that matches the input perfectly. +*/ +template +void insertOrAssignWorker(std::vector& bin_bundles, + const std::vector>& data_with_indices, + const apsi::CryptoContext& crypto_context, uint32_t bundle_index, size_t label_size, + const apsi::PSIParams& _params, bool overwrite, bool compressed) +{ + LABELED_PSI_LOG(DEBUG) << LOG_DESC("start insert-or-assign worker") + << LOG_KV("bundleIndex", bundle_index) << LOG_KV("overwrite", overwrite); + + // Iteratively insert each item-label pair at the given cuckoo index + for (auto& data_with_idx : data_with_indices) + { + const T& data = data_with_idx.first; + + // Get the bundle index + size_t cuckoo_idx = data_with_idx.second; + size_t bin_idx, bundle_idx; + std::tie(bin_idx, bundle_idx) = unpackCuckooIdx(cuckoo_idx, _params.bins_per_bundle()); + + // If the bundle_idx isn't in the prescribed range, don't try to insert this + // data + if (bundle_idx != bundle_index) + { + // Dealing with this bundle index is not our job + continue; + } + + // Try to insert or overwrite these field elements in an existing BinBundle + // at this bundle index. Keep track of whether we succeed or not. + bool written = false; + for (auto bundle_it = bin_bundles.rbegin(); bundle_it != bin_bundles.rend(); bundle_it++) + { + // If we're supposed to overwrite, try to overwrite. One of these + // BinBundles has to have the data we're trying to overwrite. + if (overwrite) + { + // If we successfully overwrote, we're done with this bundle + written = bundle_it->try_multi_overwrite(data, bin_idx); + if (written) + { + break; + } + } + + // Do a dry-run insertion and see if the new largest bin size in the range + // exceeds the limit + int32_t new_largest_bin_size = bundle_it->multi_insert_dry_run(data, bin_idx); + + // Check if inserting would violate the max bin size constraint + if (new_largest_bin_size > 0 && seal::util::safe_cast(new_largest_bin_size) < + _params.table_params().max_items_per_bin) + { + // All good + bundle_it->multi_insert_for_real(data, bin_idx); + written = true; + break; + } + } + + // We tried to overwrite an item that doesn't exist. This should never + // happen + if (overwrite && !written) + { + BOOST_THROW_EXCEPTION(OverwriteItemException() + << bcos::errinfo_comment("Tried to overwrite non-existent item")); + } + + // If we had conflicts everywhere when trying to insert, then we need to + // make a new BinBundle and insert the data there + if (!written) + { + // Make a fresh BinBundle and insert + BinBundle new_bin_bundle(crypto_context, label_size, + _params.table_params().max_items_per_bin, _params.query_params().ps_low_degree, + _params.bins_per_bundle(), compressed, false); + int res = new_bin_bundle.multi_insert_for_real(data, bin_idx); + + // If even that failed, I don't know what could've happened + if (res < 0) + { + BOOST_THROW_EXCEPTION(InsertItemException() << bcos::errinfo_comment( + "Failed to insert item into a new BinBundle with index " + + std::to_string(bundle_idx))); + } + + // Push a new BinBundle to the set of BinBundles at this bundle index + bin_bundles.push_back(std::move(new_bin_bundle)); + } + } + + LABELED_PSI_LOG(DEBUG) << LOG_DESC("finished insert-or-assign worker") + << LOG_KV("bundleIndex", bundle_index) << LOG_KV("overwrite", overwrite); +} + + +/** +Takes algebraized data to be inserted, splits it up, and distributes it so that +thread_count many threads can all insert in parallel. If overwrite is set, this +will overwrite the labels if it finds an AlgItemLabel that matches the input +perfectly. +*/ +template +void dispatchInsertOrAssign(std::vector>& bin_bundles, + const std::vector>& data_with_indices, + const apsi::CryptoContext& crypto_context, size_t label_size, const apsi::PSIParams& _params, + bool overwrite, bool compressed) +{ + // Collect the bundle indices and partition them into thread_count many + // partitions. By some uniformity assumption, the number of things to insert + // per partition should be roughly the same. Note that the contents of + // bundle_indices is always sorted (increasing order). + std::set bundle_indices_set; + for (auto& data_with_idx : data_with_indices) + { + size_t cuckoo_idx = data_with_idx.second; + size_t bin_idx, bundle_idx; + std::tie(bin_idx, bundle_idx) = unpackCuckooIdx(cuckoo_idx, _params.bins_per_bundle()); + bundle_indices_set.insert(bundle_idx); + } + + std::vector bundle_indices; + bundle_indices.assign(bundle_indices_set.begin(), bundle_indices_set.end()); + + tbb::parallel_for( + tbb::blocked_range(0U, bundle_indices.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); ++i) + { + auto bundleIndex = static_cast(bundle_indices[i]); + insertOrAssignWorker(bin_bundles[bundleIndex], data_with_indices, crypto_context, + bundleIndex, label_size, _params, overwrite, compressed); + } + }); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished all insert-or-assign worker tasks"); +} + + +/** +Removes the given items and corresponding labels from bin_bundles at their +respective cuckoo indices. +*/ +void removeWorker(std::vector& bin_bundles, + const std::vector>& data_with_indices, + uint32_t bundle_index, uint32_t bins_per_bundle) +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("start remove worker") << LOG_KV("bundleIndex", bundle_index); + + + // Iteratively remove each item-label pair at the given cuckoo index + for (auto& data_with_idx : data_with_indices) + { + // Get the bundle index + size_t cuckoo_idx = data_with_idx.second; + size_t bin_idx, bundle_idx; + std::tie(bin_idx, bundle_idx) = unpackCuckooIdx(cuckoo_idx, bins_per_bundle); + + // If the bundle_idx isn't in the prescribed range, don't try to remove this + // data + if (bundle_idx != bundle_index) + { + // Dealing with this bundle index is not our job + continue; + } + + // Try to remove these field elements from an existing BinBundle at this + // bundle index. Keep track of whether we succeed or not. + bool removed = false; + for (BinBundle& bundle : bin_bundles) + { + // If we successfully removed, we're done with this bundle + removed = bundle.try_multi_remove(data_with_idx.first, bin_idx); + if (removed) + { + break; + } + } + + // We may have produced some empty BinBundles so just remove them all + auto rem_it = std::remove_if( + bin_bundles.begin(), bin_bundles.end(), [](auto& bundle) { return bundle.empty(); }); + bin_bundles.erase(rem_it, bin_bundles.end()); + + // We tried to remove an item that doesn't exist. This should never happen + if (!removed) + { + BOOST_THROW_EXCEPTION(RemoveItemException() << bcos::errinfo_comment( + "Failed to remove item that do not exist, bundle index: " + + std::to_string(bundle_idx))); + } + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished removing items") + << LOG_KV("bundleIndex", bundle_index); +} + +/** +Takes algebraized data to be removed, splits it up, and distributes it so that +thread_count many threads can all remove in parallel. +*/ +void dispatchRemove(std::vector>& bin_bundles, + const std::vector>& data_with_indices, + uint32_t bins_per_bundle) +{ + // Collect the bundle indices and partition them into thread_count many + // partitions. By some uniformity assumption, the number of things to remove + // per partition should be roughly the same. Note that the contents of + // bundle_indices is always sorted (increasing order). + std::set bundle_indices_set; + for (auto& data_with_idx : data_with_indices) + { + size_t cuckoo_idx = data_with_idx.second; + size_t bin_idx, bundle_idx; + std::tie(bin_idx, bundle_idx) = unpackCuckooIdx(cuckoo_idx, bins_per_bundle); + bundle_indices_set.insert(bundle_idx); + } + + std::vector bundle_indices; + bundle_indices.assign(bundle_indices_set.begin(), bundle_indices_set.end()); + + tbb::parallel_for( + tbb::blocked_range(0U, bundle_indices.size()), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); ++i) + { + auto bundleIndex = static_cast(bundle_indices[i]); + removeWorker( + bin_bundles[bundleIndex], data_with_indices, bundleIndex, bins_per_bundle); + } + }); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished all remove worker tasks"); +} + +/** +Returns a set of DB cache references corresponding to the bundles in the given +set +*/ +std::vector> collectCaches( + std::vector* bin_bundles) +{ + std::vector> result; + for (const auto& bundle : (*bin_bundles)) + { + result.emplace_back(std::cref(bundle.get_cache())); + } + + return result; +} + +SenderDB::SenderDB(const apsi::PSIParams& params, crypto::OprfServer::Ptr _oprfServer, + std::size_t label_byte_count, std::size_t nonce_byte_count, bool compressed) + : params_(params), + crypto_context_(params_), + label_byte_count_(label_byte_count), + nonce_byte_count_(nonce_byte_count), + item_count_(0), + compressed_(compressed), + m_oprfServer(std::move(_oprfServer)) +{ + // The labels cannot be more than 1 KB. + if (label_byte_count_ > MAX_LABEL_BYTE) + { + BOOST_THROW_EXCEPTION(LabelExceedsSizeException() << bcos::errinfo_comment( + "Requested label byte count exceeds the maximum (1024), count: " + + std::to_string(label_byte_count_))); + } + + if (nonce_byte_count_ > apsi::max_nonce_byte_count) + { + BOOST_THROW_EXCEPTION(NonceExceedsSizeException() << bcos::errinfo_comment( + "Requested nonce byte count exceeds the maximum (16), count: " + + std::to_string(nonce_byte_count_))); + } + + // Set the evaluator. This will be used for BatchedPlaintextPolyn::eval. + crypto_context_.set_evaluator(); + + // Reset the SenderDB data structures + clear(); +} + +SenderDB::SenderDB(SenderDB&& source) + : params_(source.params_), + crypto_context_(source.crypto_context_), + label_byte_count_(source.label_byte_count_), + nonce_byte_count_(source.nonce_byte_count_), + item_count_(source.item_count_), + compressed_(source.compressed_), + stripped_(source.stripped_), + m_oprfServer(std::move(source.m_oprfServer)) +{ + // Lock the source before moving stuff over + auto lock = source.getWriterLock(); + + hashed_items_ = std::move(source.hashed_items_); + bin_bundles_ = std::move(source.bin_bundles_); + + // Reset the source data structures + source.clearInternal(); +} + +SenderDB& SenderDB::operator=(SenderDB&& source) +{ + // Do nothing if moving to self + if (&source == this) + { + return *this; + } + + // Lock the current SenderDB + auto this_lock = getWriterLock(); + + params_ = source.params_; + crypto_context_ = source.crypto_context_; + label_byte_count_ = source.label_byte_count_; + nonce_byte_count_ = source.nonce_byte_count_; + item_count_ = source.item_count_; + compressed_ = source.compressed_; + stripped_ = source.stripped_; + m_oprfServer = std::move(source.m_oprfServer); + + // Lock the source before moving stuff over + auto source_lock = source.getWriterLock(); + + hashed_items_ = std::move(source.hashed_items_); + bin_bundles_ = std::move(source.bin_bundles_); + + // Reset the source data structures + source.clearInternal(); + + return *this; +} + +size_t SenderDB::getBinBundleCount(uint32_t bundle_idx) const +{ + // Lock the database for reading + auto lock = getReaderLock(); + + return bin_bundles_.at(seal::util::safe_cast(bundle_idx)).size(); +} + +size_t SenderDB::getBinBundleCount() const +{ + // Lock the database for reading + auto lock = getReaderLock(); + + // Compute the total number of BinBundles + return std::accumulate(bin_bundles_.cbegin(), bin_bundles_.cend(), size_t(0), + [&](auto a, auto& b) { return a + b.size(); }); +} + +double SenderDB::getPackingRate() const +{ + // Lock the database for reading + auto lock = getReaderLock(); + + uint64_t item_count = seal::util::mul_safe(static_cast(getItemCount()), + static_cast(params_.table_params().hash_func_count)); + uint64_t max_item_count = seal::util::mul_safe(static_cast(getBinBundleCount()), + static_cast(params_.items_per_bundle()), + static_cast(params_.table_params().max_items_per_bin)); + + return max_item_count ? static_cast(item_count) / static_cast(max_item_count) : + 0.0; +} + +void SenderDB::clearInternal() +{ + // Assume the SenderDB is already locked for writing + + // Clear the set of inserted items + hashed_items_.clear(); + item_count_ = 0; + + // Clear the BinBundles + bin_bundles_.clear(); + bin_bundles_.resize(params_.bundle_idx_count()); + + // Reset the stripped_ flag + stripped_ = false; +} + +void SenderDB::clear() +{ + // Lock the database for writing + auto lock = getWriterLock(); + clearInternal(); +} + +void SenderDB::generateCaches() +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("start generating bin bundle caches"); + + for (auto& bundle_idx : bin_bundles_) + { + for (auto& bb : bundle_idx) + { + bb.regen_cache(); + } + } + LABELED_PSI_LOG(INFO) << LOG_DESC("finished generating bin bundle caches"); +} + +std::vector> SenderDB::getCacheAt(uint32_t bundle_idx) +{ + return collectCaches(&(bin_bundles_.at(seal::util::safe_cast(bundle_idx)))); +} + +void SenderDB::strip() +{ + // Lock the database for writing + auto lock = getWriterLock(); + + stripped_ = true; + + hashed_items_.clear(); + + uint32_t bundleSize = bin_bundles_.size(); + + tbb::parallel_for(tbb::blocked_range(0U, bundleSize), [&](auto const& range) { + for (auto i = range.begin(); i < range.end(); ++i) + { + for (auto& bb : bin_bundles_[i]) + { + bb.strip(); + } + } + }); + + LABELED_PSI_LOG(INFO) << LOG_DESC("SenderDB has been stripped"); +} + + +void SenderDB::insertOrAssign(const std::vector>& data) +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION(InsertItemException() << bcos::errinfo_comment( + "Cannot insert data to a stripped SenderDB")); + } + if (!isLabeled()) + { + BOOST_THROW_EXCEPTION( + InsertItemException() << bcos::errinfo_comment( + "Attempted to insert labeled data but this is an unlabeled SenderDB")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start inserting items and labels in SenderDB") + << LOG_KV("count", data.size()); + + auto hashedData = + preprocessItemsAndLabels(data, label_byte_count_, nonce_byte_count_, m_oprfServer); + + doInsertOrAssign(hashedData); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished inserting items and labels in SenderDB"); +} + + +void SenderDB::insertOrAssign(ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels) +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION(InsertItemException() << bcos::errinfo_comment( + "Cannot insert data to a stripped SenderDB")); + } + if (!isLabeled()) + { + BOOST_THROW_EXCEPTION( + InsertItemException() << bcos::errinfo_comment( + "Attempted to insert labeled data but this is an unlabeled SenderDB")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start inserting items and labels in SenderDB") + << LOG_KV("count", _items->size()); + + auto hashedData = preprocessItemsAndLabels( + std::move(_items), std::move(_labels), label_byte_count_, nonce_byte_count_, m_oprfServer); + + doInsertOrAssign(hashedData); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished inserting items and labels in SenderDB"); +} + + +void SenderDB::doInsertOrAssign( + std::vector>& _hashedData) +{ + // Lock the database for writing + auto lock = getWriterLock(); + + std::vector> newData; + std::vector> reWriteData; + + for (auto itemLabelPair : _hashedData) + { + if (hashed_items_.find(itemLabelPair.first) == hashed_items_.end()) + { + hashed_items_.insert(itemLabelPair.first); + item_count_++; + newData.emplace_back(std::move(itemLabelPair)); + } + else + { + reWriteData.emplace_back(std::move(itemLabelPair)); + } + } + + // Compute the label size; this ceil(effective_label_bit_count / item_bit_count) + size_t labelSize = computeLabelSize(nonce_byte_count_ + label_byte_count_, params_); + + auto newItemCount = newData.size(); + auto existingItemCount = reWriteData.size(); + + // Dispatch the insertion, first for the new data, then for the data we're gonna overwrite + + if (existingItemCount) + { + LABELED_PSI_LOG(INFO) << LOG_DESC("found existing items to replace in SenderDB") + << LOG_KV("count", existingItemCount); + + // Break the data into field element representation. Also compute the items' + // cuckoo indices. + std::vector> data_with_indices = + transformLabeledData(reWriteData.begin(), reWriteData.end(), params_); + + dispatchInsertOrAssign(bin_bundles_, data_with_indices, crypto_context_, labelSize, params_, + /* overwrite items */ true, compressed_); + } + if (newItemCount) + { + LABELED_PSI_LOG(INFO) << LOG_DESC("found new items to replace in SenderDB") + << LOG_KV("count", newItemCount); + + // Process and add the new data. Break the data into field element + // representation. Also compute the items' cuckoo indices. + std::vector> data_with_indices = + transformLabeledData(newData.begin(), newData.end(), params_); + + dispatchInsertOrAssign(bin_bundles_, data_with_indices, crypto_context_, labelSize, params_, + /* don't overwrite items */ false, compressed_); + } + + // Generate the BinBundle caches + generateCaches(); +} + +void SenderDB::insertOrAssign(const std::vector& data) +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION(InsertItemException() << bcos::errinfo_comment( + "Cannot insert item to a stripped SenderDB")); + } + if (isLabeled()) + { + BOOST_THROW_EXCEPTION(InsertItemException() << bcos::errinfo_comment( + "Attempted to insert item data but this is an labeled SenderDB")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start inserting items in SenderDB") + << LOG_KV("count", data.size()); + + std::vector hashedData = preprocessItems(data, m_oprfServer); + + // Lock the database for writing + auto lock = getWriterLock(); + + // We are not going to insert items that already appear in the database. + auto newDataEnd = std::remove_if(hashedData.begin(), hashedData.end(), [&](const auto& item) { + bool found = hashed_items_.find(item) != hashed_items_.end(); + if (!found) + { + // Add to hashed_items_ already at this point! + hashed_items_.insert(item); + item_count_++; + } + + // Remove those that were found + return found; + }); + + // Erase the previously existing items from hashed_data; in unlabeled case + // there is nothing to do + hashedData.erase(newDataEnd, hashedData.end()); + + LABELED_PSI_LOG(INFO) << LOG_DESC("found new items to replace in SenderDB") + << LOG_KV("count", hashedData.size()); + + // Break the new data down into its field element representation. Also compute + // the items' cuckoo indices. + std::vector> data_with_indices = + transformUnlabeledData(hashedData.begin(), hashedData.end(), params_); + + // Dispatch the insertion + dispatchInsertOrAssign(bin_bundles_, data_with_indices, crypto_context_, + /* label size */ 0, params_, /* don't overwrite items */ false, compressed_); + + // Generate the BinBundle caches + generateCaches(); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished inserting items in SenderDB"); +} + +void SenderDB::remove(const std::vector& data) +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION(RemoveItemException() << bcos::errinfo_comment( + "Cannot remove data from a stripped SenderDB")); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("start removing items in SenderDB") + << LOG_KV("count", data.size()); + + std::vector hashedData = preprocessItems(data, m_oprfServer); + + // Lock the database for writing + auto lock = getWriterLock(); + + // Remove items that do not exist in the database. + std::remove_if(hashedData.begin(), hashedData.end(), [&](const auto& item) { + bool found = hashed_items_.find(item) != hashed_items_.end(); + if (found) + { + // Remove from hashed_items_ already at this point! + hashed_items_.erase(item); + item_count_--; + } + + // Remove those that were not found + return !found; + }); + + // Break the data down into its field element representation. Also compute the + // items' cuckoo indices. + std::vector> data_with_indices = + transformUnlabeledData(hashedData.begin(), hashedData.end(), params_); + + // Dispatch the removal + uint32_t bins_per_bundle = params_.bins_per_bundle(); + dispatchRemove(bin_bundles_, data_with_indices, bins_per_bundle); + + // Generate the BinBundle caches + generateCaches(); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finished removing items in SenderDB"); +} + +bool SenderDB::hasItem(const apsi::Item& item) const +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION( + RetrieveItemException() << bcos::errinfo_comment( + "Cannot retrieve the presence of an item from a stripped SenderDB")); + } + + std::string itemStr(item.value().size(), 0); + std::memcpy(&itemStr[0], item.value().data(), item.value().size()); + + bcos::bytes oprfOut = m_oprfServer->fullEvaluate(itemStr); + + apsi::HashedItem hashedItem; + std::memcpy(hashedItem.value().data(), oprfOut.data(), hashedItem.value().size()); + + // Lock the database for reading + auto lock = getReaderLock(); + + return hashed_items_.find(hashedItem) != hashed_items_.end(); +} + +apsi::Label SenderDB::getLabel(const apsi::Item& item) const +{ + if (stripped_) + { + BOOST_THROW_EXCEPTION( + RetrieveLabelException() << bcos::errinfo_comment( + "Cannot retrieve the presence of a label from a stripped SenderDB")); + } + if (!isLabeled()) + { + BOOST_THROW_EXCEPTION( + RetrieveLabelException() << bcos::errinfo_comment( + "Attempted to retrieve a label but this is an unlabeled SenderDB")); + } + + std::string itemStr(item.value().size(), 0); + std::memcpy(&itemStr[0], item.value().data(), item.value().size()); + + bcos::bytes oprfOut = m_oprfServer->fullEvaluate(itemStr); + + apsi::HashedItem hashedItem; + std::memcpy(hashedItem.value().data(), oprfOut.data(), hashedItem.value().size()); + + // Lock the database for reading + auto lock = getReaderLock(); + + // Check if this item is in the DB. If not, throw an exception + if (hashed_items_.find(hashedItem) == hashed_items_.end()) + { + BOOST_THROW_EXCEPTION(RetrieveLabelException() << bcos::errinfo_comment( + "Cannot retrieve label for an item that is not in the SenderDB")); + } + + uint32_t bins_per_bundle = params_.bins_per_bundle(); + + // Preprocess a single element. This algebraizes the item and gives back its + // field element representation as well as its cuckoo hash. We only read one + // of the locations because the labels are the same in each location. + apsi::util::AlgItem alg_item; + size_t cuckoo_idx; + std::tie(alg_item, cuckoo_idx) = transformUnlabeledData(hashedItem, params_)[0]; + + // Now figure out where to look to get the label + size_t bin_idx, bundle_idx; + std::tie(bin_idx, bundle_idx) = unpackCuckooIdx(cuckoo_idx, bins_per_bundle); + + // Retrieve the algebraic labels from one of the BinBundles at this index + const std::vector& bundle_set = bin_bundles_[bundle_idx]; + std::vector alg_label; + bool got_labels = false; + for (const BinBundle& bundle : bundle_set) + { + // Try to retrieve the contiguous labels from this BinBundle + if (bundle.try_get_multi_label(alg_item, bin_idx, alg_label)) + { + got_labels = true; + break; + } + } + + // It shouldn't be possible to have items in your set but be unable to + // retrieve the associated label. Throw an exception because something is + // terribly wrong. + if (!got_labels) + { + BOOST_THROW_EXCEPTION( + RetrieveLabelException() << bcos::errinfo_comment( + "Failed to retrieve label for an item that was supposed to be in the SenderDB")); + } + + // All good. Now just reconstruct the big label from its split-up parts + apsi::EncryptedLabel encrypted_label = dealgebraize_label(alg_label, + alg_label.size() * static_cast(params_.item_bit_count_per_felt()), + params_.seal_params().plain_modulus()); + + // Resize down to the effective byte count + encrypted_label.resize(nonce_byte_count_ + label_byte_count_); + + apsi::LabelKey key; + std::memcpy(key.data(), oprfOut.data() + hashedItem.value().size(), key.size()); + + // Decrypt the label + return decrypt_label(encrypted_label, key, nonce_byte_count_); +} + +/** +Writes the SenderDB bytes +*/ +void SenderDB::saveToBytes(bcos::bytes& _out) const +{ + // Lock the database for reading + auto lock = getReaderLock(); + LABELED_PSI_LOG(INFO) << LOG_DESC("start saving SenderDB"); + + ppctars::SenderDB tarsSenderDB; + for (auto& hashedItems : hashed_items_) + { + bcos::bytes item(sizeof(apsi::Item::value_type)); + std::memcpy(item.data(), hashedItems.value().data(), sizeof(apsi::Item::value_type)); + tarsSenderDB.hashedItems.emplace_back(item); + } + tarsSenderDB.psiParams = fromPSIParams(params_, 0); + tarsSenderDB.oprfKey = m_oprfServer->privateKey(); + tarsSenderDB.stripped = stripped_; + tarsSenderDB.compressed = compressed_; + tarsSenderDB.itemCount = item_count_; + tarsSenderDB.nonceByteCount = nonce_byte_count_; + tarsSenderDB.labelByteCount = label_byte_count_; + + for (auto& binBundles : bin_bundles_) + { + std::vector tarsBinBundles; + tarsBinBundles.reserve(binBundles.size()); + for (auto& binBundle : binBundles) + { + tarsBinBundles.emplace_back(binBundle.saveToTarsBinBundle()); + } + tarsSenderDB.binBundles.emplace_back(tarsBinBundles); + } + + _out.clear(); + ppctars::serialize::encode(tarsSenderDB, _out); + + LABELED_PSI_LOG(INFO) << LOG_DESC("finish saving SenderDB"); +} + +/** +Reads the SenderDB from bytes. +*/ +SenderDB::Ptr SenderDB::loadFromBytes(crypto::OprfServer::Ptr _oprfServer, const bcos::bytes& _in) +{ + LABELED_PSI_LOG(INFO) << LOG_DESC("start loading SenderDB"); + + ppctars::SenderDB tarsSenderDB; + ppctars::serialize::decode(_in, tarsSenderDB); + + _oprfServer->setPrivateKey(tarsSenderDB.oprfKey); + SenderDB::Ptr ret = + std::make_shared(toPSIParams(tarsSenderDB.psiParams), std::move(_oprfServer), + tarsSenderDB.labelByteCount, tarsSenderDB.nonceByteCount, tarsSenderDB.compressed); + + ret->stripped_ = tarsSenderDB.stripped; + ret->item_count_ = tarsSenderDB.itemCount; + for (auto& tarsHashedItems : tarsSenderDB.hashedItems) + { + apsi::HashedItem hashedItem; + std::memcpy(hashedItem.value().data(), tarsHashedItems.data(), hashedItem.value().size()); + ret->hashed_items_.insert(hashedItem); + } + + ret->bin_bundles_.clear(); + for (auto& tarsBinBundles : tarsSenderDB.binBundles) + { + std::vector binBundles; + binBundles.reserve(tarsBinBundles.size()); + for (auto& tarsBinBundle : tarsBinBundles) + { + binBundles.emplace_back( + BinBundle::loadFromTarsBinBundle(ret->crypto_context_, tarsBinBundle)); + } + ret->bin_bundles_.emplace_back(std::move(binBundles)); + } + + LABELED_PSI_LOG(INFO) << LOG_DESC("finish loading SenderDB"); + return ret; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h new file mode 100644 index 00000000..c9580b59 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h @@ -0,0 +1,374 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @reference https://github.com/microsoft/APSI/sender/sender_db.h + * @license MIT license + * @change optimize the implementation of some functions & replace the instance of oprf + * + * @file SenderDB.h + * @author: shawnhe + * @date 2022-11-7 + * + */ + +#pragma once + +// STD +#include +#include +#include +#include +#include +#include +#include +#include + +// GSL +#include + +// APSI +#include +#include +#include + +// SEAL +#include "seal/plaintext.h" +#include "seal/util/locks.h" + + +#include "ppc-framework/crypto/Oprf.h" + +#include "ppc-framework/io/DataBatch.h" + +#include "../protocol/Protocol.h" +#include "BinBundle.h" +#include "SenderCache.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" + +namespace ppc::psi +{ +/** +A SenderDB maintains an in-memory representation of the sender's set of items +and labels (in labeled mode). This data is not simply copied into the SenderDB +data structures, but also preprocessed heavily to allow for faster online +computation time. Since inserting a large number of new items into a SenderDB +can take time, it is not recommended to recreate the SenderDB when the database +changes a little bit. Instead, the class supports fast update and deletion +operations that should be preferred: SenderDB::InsertOrAssign and +SenderDB::remove. + +The SenderDB constructor allows the label byte count to be specified; unlabeled +mode is activated by setting the label byte count to zero. It is possible to +optionally specify the size of the nonce used in encrypting the labels, but this +is best left to its default value unless the user is absolutely sure of what +they are doing. + +The SenderDB requires substantially more memory than the raw data would. Part of +that memory can automatically be compressed when it is not in use; this feature +is enabled by default, and can be disabled when constructing the SenderDB. The +downside of in-memory compression is a performance reduction from decompressing +parts of the data when they are used, and recompressing them if they are +updated. +*/ + +class SenderDB : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + virtual ~SenderDB() = default; + + /** + Creates a new SenderDB. + */ + SenderDB(const apsi::PSIParams& params, crypto::OprfServer::Ptr _oprfServer, + std::size_t label_byte_count = 0, std::size_t nonce_byte_count = 16, + bool compressed = true); + /** + Creates a new SenderDB by moving from an existing one. + */ + SenderDB(SenderDB&& source); + + /** + Moves an existing SenderDB to the current one. + */ + SenderDB& operator=(SenderDB&& source); + + /** + Clears the database. Every item and label will be removed. The OPRF key is + unchanged. + */ + void clear(); + + /** + Returns whether this is a labeled SenderDB. + */ + bool isLabeled() const { return 0 != label_byte_count_; } + + /** + Returns the label byte count. A zero value indicates an unlabeled SenderDB. + */ + std::size_t getLabelByteCount() const { return label_byte_count_; } + + /** + Returns the nonce byte count used for encrypting labels. + */ + std::size_t getNonceByteCount() const { return nonce_byte_count_; } + + /** + Indicates whether SEAL plaintexts are compressed in memory. + */ + bool isCompressed() const { return compressed_; } + + /** + Indicates whether the SenderDB has been stripped of all information not needed + for serving a query. + */ + bool isStripped() const { return stripped_; } + + /** + Strips the SenderDB of all information not needed for serving a query. Returns + a copy of the OPRF key and clears it from the SenderDB. + */ + void strip(); + + /** + Inserts the given data into the database. This function can be used only on a + labeled SenderDB instance. If an item already exists in the database, its + label is overwritten with the new label. + */ + void insertOrAssign(const std::vector>& data); + + void insertOrAssign(ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels); + + // hashedData will be cleared + void doInsertOrAssign( + std::vector>& _hashedData); + + /** + Inserts the given (hashed) item-label pair into the database. This function + can be used only on a labeled SenderDB instance. If the item already exists in + the database, its label is overwritten with the new label. + */ + void insertOrAssign(const std::pair& data) + { + std::vector> data_singleton{data}; + insertOrAssign(data_singleton); + } + + /** + Inserts the given data into the database. This function can be used only on an + unlabeled SenderDB instance. + */ + void insertOrAssign(const std::vector& data); + + /** + Inserts the given (hashed) item into the database. This function can be used + only on an unlabeled SenderDB instance. + */ + void insertOrAssign(const apsi::Item& data) + { + std::vector data_singleton{data}; + insertOrAssign(data_singleton); + } + + /** + Clears the database and inserts the given data. This function can be used only + on a labeled SenderDB instance. + */ + void setData(const std::vector>& data) + { + clear(); + insertOrAssign(data); + } + + void setData(ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels) + { + clear(); + insertOrAssign(std::move(_items), std::move(_labels)); + } + + /** + Clears the database and inserts the given data. This function can be used only + on an unlabeled SenderDB instance. + */ + void setData(const std::vector& data) + { + clear(); + insertOrAssign(data); + } + + /** + Removes the given data from the database, using at most thread_count threads. + */ + void remove(const std::vector& data); + + /** + Removes the given (hashed) item from the database. + */ + void remove(const apsi::Item& data) + { + std::vector data_singleton{data}; + remove(data_singleton); + } + + /** + Returns whether the given item has been inserted in the SenderDB. + */ + bool hasItem(const apsi::Item& item) const; + + /** + Returns the label associated to the given item in the database. Throws + std::invalid_argument if the item does not appear in the database. + */ + apsi::Label getLabel(const apsi::Item& item) const; + + /** + Returns a set of cache references corresponding to the bundles at the given + bundle index. Even though this function returns a vector, the order has no + significance. This function is meant for internal use. + */ + auto getCacheAt(std::uint32_t bundle_idx) + -> std::vector>; + + /** + Returns a reference to the PSI parameters for this SenderDB. + */ + const apsi::PSIParams& getParams() const { return params_; } + + /** + Returns a reference to the CryptoContext for this SenderDB. + */ + const apsi::CryptoContext& getCryptoContext() const { return crypto_context_; } + + /** + Returns a reference to the SEALContext for this SenderDB. + */ + std::shared_ptr getSealContext() const + { + return crypto_context_.seal_context(); + } + + /** + Returns a reference to a set of item hashes already existing in the SenderDB. + */ + const std::unordered_set& getHashedItems() const { return hashed_items_; } + + /** + Returns the number of items in this SenderDB. + */ + size_t getItemCount() const { return item_count_; } + + /** + Returns the total number of bin bundles at a specific bundle index. + */ + std::size_t getBinBundleCount(std::uint32_t bundle_idx) const; + + /** + Returns the total number of bin bundles. + */ + std::size_t getBinBundleCount() const; + + /** + Returns how efficiently the SenderDB is packaged. A higher rate indicates + better performance and a lower communication cost in a query execution. + */ + double getPackingRate() const; + + /** + Obtains a scoped lock preventing the SenderDB from being changed. + */ + seal::util::ReaderLock getReaderLock() const { return db_lock_.acquire_read(); } + + /** + Writes the SenderDB bytes + */ + void saveToBytes(bcos::bytes& _out) const; + + /** + Reads the SenderDB from bytes. + */ + static SenderDB::Ptr loadFromBytes(crypto::OprfServer::Ptr _oprfServer, const bcos::bytes& _in); + +private: + SenderDB(const SenderDB& copy) = delete; + + seal::util::WriterLock getWriterLock() { return db_lock_.acquire_write(); } + + void clearInternal(); + + void generateCaches(); + + /** + The set of all items that have been inserted into the database + */ + std::unordered_set hashed_items_; + + /** + The PSI parameters define the SEAL parameters, base field, item size, table + size, etc. + */ + apsi::PSIParams params_; + + /** + Necessary for evaluating polynomials of Plaintexts. + */ + apsi::CryptoContext crypto_context_; + + /** + A read-write lock to protect the database from modification while in use. + */ + mutable seal::util::ReaderWriterLocker db_lock_; + + /** + Indicates the size of the label in bytes. A zero value indicates an unlabeled + SenderDB. + */ + std::size_t label_byte_count_; + + /** + Indicates the number of bytes of the effective label reserved for a randomly + sampled nonce. The effective label byte count is the sum of label_byte_count + and nonce_byte_count. The value can range between 0 and 16. If + label_byte_count is zero, nonce_byte_count has no effect. + */ + std::size_t nonce_byte_count_; + + /** + The number of items currently in the SenderDB. + */ + std::size_t item_count_; + + /** + Indicates whether SEAL plaintexts are compressed in memory. + */ + bool compressed_; + + /** + Indicates whether the SenderDB has been stripped of all information not needed + for serving a query. + */ + bool stripped_; + + /** + All the BinBundles in the database, indexed by bundle index. The set + (represented by a vector internally) at bundle index i contains all the + BinBundles with bundle index i. + */ + std::vector> bin_bundles_; + + crypto::OprfServer::Ptr m_oprfServer; +}; // class SenderDB + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h b/cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h new file mode 100644 index 00000000..dd830896 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskCommand.h + * @author: shawnhe + * @date 2022-11-14 + */ + +#pragma once +#include "../Common.h" +#include +#include + +namespace ppc::psi +{ +class TaskCommand +{ +public: + using Ptr = std::shared_ptr; + TaskCommand(std::string_view _param) + { + Json::Reader reader; + Json::Value result; + if (!reader.parse(_param.begin(), _param.end(), result)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int)LabeledPSIRetCode::INVALID_TASK_PARAM, "invalid task param: invalid json")); + } + if (!result.isArray() || result.empty()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)LabeledPSIRetCode::INVALID_TASK_PARAM, + "invalid task param:: the param must be array")); + } + auto command = result[0].asString(); + if (command == CMD_SETUP_SENDER_DB) + { + m_command = (int)LabeledPSICommand::SETUP_SENDER_DB; + if (result.size() < 2) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)LabeledPSIRetCode::INVALID_TASK_PARAM, + "must set label_byte_count for command: " + command)); + } + m_args.emplace_back(std::stoi(result[1].asString())); + } + else if (command == CMD_RUN_LABELED_PSI) + { + m_command = (int)LabeledPSICommand::RUN_LABELED_PSI; + } + else if (command == CMD_SAVE_SENDER_CACHE) + { + m_command = (int)LabeledPSICommand::SAVE_SENDER_CACHE; + } + else if (command == CMD_LOAD_SENDER_CACHE) + { + m_command = (int)LabeledPSICommand::LOAD_SENDER_CACHE; + } + else if (command == CMD_ADD_ITEMS) + { + m_command = (int)LabeledPSICommand::ADD_ITEMS; + } + else if (command == CMD_DELETE_ITEMS) + { + m_command = (int)LabeledPSICommand::DELETE_ITEMS; + } + else if (command == CMD_UPDATE_LABELS) + { + m_command = (int)LabeledPSICommand::UPDATE_LABELS; + } + else + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int)LabeledPSIRetCode::UNDEFINED_COMMAND, "unsupported command: " + command)); + } + } + + int command() const { return m_command; } + std::vector const& args() const { return m_args; } + +private: + int m_command; + std::vector m_args; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars b/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars new file mode 100644 index 00000000..28fb7935 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars @@ -0,0 +1,45 @@ +module ppctars { + struct SealParams { + 1 require unsigned int polyModulusDegree; + 2 require unsigned int plainModulus; + 3 require vector coeffModulus; + }; + + struct PsiParams + { + 1 require unsigned int hashFuncCount; + 2 require unsigned int tableSize; + 3 require unsigned int maxItemsPerBin; + 4 require unsigned int feltsPerItem; + 5 require unsigned int psLowDegree; + 6 require vector queryPowers; + 7 require SealParams sealParams; + 8 require unsigned int binBundleCount; + }; + + struct OprfData + { + 1 require vector> data; + }; + + struct EncryptedPowers + { + 1 require unsigned int power; + 2 require vector> ciphertexts; + }; + + struct QueryRequest + { + 1 require vector relinKeys; + 2 require vector encryptedPowers; + }; + + struct QueryResponse + { + 1 require unsigned int bundleIdx; + 2 require vector ciphertext; + 3 require unsigned int labelByteCount; + 4 require unsigned int nonceByteCount; + 5 require vector> labelResults; + }; +}; diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h b/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h new file mode 100644 index 00000000..13dd3bf6 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h @@ -0,0 +1,93 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIResult.h + * @author: shawnhe + * @date 2022-11-13 + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "ppc-framework/protocol/Task.h" + +namespace ppc::psi +{ +// Note: can extend on demand +class LabeledPSIResult : public protocol::TaskResult +{ +public: + using Ptr = std::shared_ptr; + LabeledPSIResult(std::string const& _taskID) : TaskResult(_taskID) {} + virtual ~LabeledPSIResult() = default; + + + void setOutputs(std::vector>&& _outputs) + { + m_outputs = std::move(_outputs); + } + + const std::vector>& getOutputs() { return m_outputs; } + + // serialize the taskResult to json + Json::Value serializeToJson() const override + { + Json::Value response; + response["taskID"] = taskID(); + if (m_timeCost) + { + response["timeCost"] = std::to_string(m_timeCost) + "ms"; + } + if (error() && error()->errorCode()) + { + response["code"] = error()->errorCode(); + response["message"] = error()->errorMessage(); + response["status"] = protocol::toString(protocol::TaskStatus::FAILED); + } + else + { + response["code"] = 0; + response["message"] = "success"; + response["status"] = protocol::toString(protocol::TaskStatus::COMPLETED); + } + if (!m_outputs.empty()) + { + Json::Value jsonOutputs; + for (auto& outputVector : m_outputs) + { + Json::Value jsonOutput; + for (auto& output : outputVector) + { + jsonOutput.append(output); + } + jsonOutputs.append(jsonOutput); + } + Json::Value jsonData; + jsonData["outputs"] = jsonOutputs; + response["data"] = jsonData; + } + return response; + } + +private: + std::vector> m_outputs; +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h b/cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h new file mode 100644 index 00000000..4ef3aa02 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h @@ -0,0 +1,102 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Protocol.h + * @author: shawnhe + * @date 2022-11-6 + */ + +#pragma once +#include "LabeledPSI.h" +#include "ppc-framework/Common.h" + +#include +#include + +#include + +namespace ppc::psi +{ +inline apsi::PSIParams toPSIParams(const ppctars::PsiParams& _tarsParams) +{ + apsi::PSIParams::ItemParams itemParams{}; + apsi::PSIParams::TableParams tableParams{}; + apsi::PSIParams::QueryParams queryParams; + apsi::PSIParams::SEALParams sealParams; + + itemParams.felts_per_item = _tarsParams.feltsPerItem; + + tableParams.hash_func_count = _tarsParams.hashFuncCount; + tableParams.table_size = _tarsParams.tableSize; + tableParams.max_items_per_bin = _tarsParams.maxItemsPerBin; + + queryParams.ps_low_degree = _tarsParams.psLowDegree; + + if (!_tarsParams.queryPowers.empty()) + { + for (auto queryPower : _tarsParams.queryPowers) + { + queryParams.query_powers.insert(queryPower); + } + } + else + { + for (uint idx = 1; idx <= tableParams.max_items_per_bin; ++idx) + { + queryParams.query_powers.insert(idx); + } + } + + std::vector coeffModulus; + coeffModulus.insert(coeffModulus.end(), _tarsParams.sealParams.coeffModulus.begin(), + _tarsParams.sealParams.coeffModulus.end()); + + sealParams.set_coeff_modulus(coeffModulus); + sealParams.set_poly_modulus_degree(_tarsParams.sealParams.polyModulusDegree); + sealParams.set_plain_modulus(_tarsParams.sealParams.plainModulus); + + apsi::PSIParams psiParams(itemParams, tableParams, queryParams, sealParams); + return psiParams; +} + +inline ppctars::PsiParams fromPSIParams(const apsi::PSIParams& _psiParams, uint32_t _binBundleCount) +{ + ppctars::PsiParams tarsParams; + tarsParams.hashFuncCount = _psiParams.table_params().hash_func_count; + tarsParams.tableSize = _psiParams.table_params().table_size; + tarsParams.maxItemsPerBin = _psiParams.table_params().max_items_per_bin; + + tarsParams.feltsPerItem = _psiParams.item_params().felts_per_item; + + tarsParams.psLowDegree = _psiParams.query_params().ps_low_degree; + for (auto& power : _psiParams.query_params().query_powers) + { + tarsParams.queryPowers.emplace_back(power); + } + + ppctars::SealParams tarsSealParams; + tarsSealParams.plainModulus = _psiParams.seal_params().plain_modulus().value(); + tarsSealParams.polyModulusDegree = _psiParams.seal_params().poly_modulus_degree(); + for (auto& modulu : _psiParams.seal_params().coeff_modulus()) + { + tarsSealParams.coeffModulus.emplace_back(modulu.value()); + } + tarsParams.sealParams = tarsSealParams; + + tarsParams.binBundleCount = _binBundleCount; + return tarsParams; +} + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars b/cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars new file mode 100644 index 00000000..76e7f5f0 --- /dev/null +++ b/cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars @@ -0,0 +1,42 @@ +#include "LabeledPSI.tars" +module ppctars { + struct BatchedPlaintextPolyn + { + 1 require vector> batchedCoeffs; + }; + + struct BinBundleCache + { + 1 require vector> feltMatchingPolyns; + 2 require vector>> feltInterpPolyns; + 3 require BatchedPlaintextPolyn batchedMatchingPolyn; + 4 require vector batchedInterpPolyns; + }; + + struct BinBundle + { + 1 require bool cacheInvalid; + 2 require bool compressed; + 3 require bool stripped; + 4 require vector> itemBins; + 5 require vector>> labelBins; + 6 require BinBundleCache cache; + 7 require unsigned int labelSize; + 8 require unsigned int maxBinSize; + 9 require unsigned int psLowDegree; + 10 require unsigned int numBins; + }; + + struct SenderDB + { + 1 require vector oprfKey; + 2 require vector> binBundles; + 3 require bool stripped; + 4 require bool compressed; + 5 require unsigned int itemCount; + 6 require unsigned int nonceByteCount; + 7 require unsigned int labelByteCount; + 8 require vector> hashedItems; + 9 require PsiParams psiParams; + }; +}; diff --git a/cpp/ppc-psi/src/psi-framework/CMakeLists.txt b/cpp/ppc-psi/src/psi-framework/CMakeLists.txt new file mode 100644 index 00000000..63d30086 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/CMakeLists.txt @@ -0,0 +1,29 @@ +# for tars generator +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-psi/psi/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") +# generate tars +if(TARS_INPUT) + foreach(TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach() +endif() +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") +include_directories(${TARS_HEADER_DIR}) + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${PSI_FRAMEWORK_TARGET} ${SRCS} ${OUT_TARS_H_LIST}) +target_include_directories(${PSI_FRAMEWORK_TARGET} PUBLIC + $) +target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET}) diff --git a/cpp/ppc-psi/src/psi-framework/PSIFramework.cpp b/cpp/ppc-psi/src/psi-framework/PSIFramework.cpp new file mode 100644 index 00000000..850b2804 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/PSIFramework.cpp @@ -0,0 +1,757 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIFramework.cpp + * @author: yujiechen + * @date 2022-12-26 + */ +#include "PSIFramework.h" +#include "../Common.h" +#include "ppc-framework/protocol/GlobalConfig.h" + +using namespace ppc::psi; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace bcos; +using namespace ppc::front; + +void PSIFramework::start() +{ + if (m_started) + { + PSI_FRAMEWORK_LOG(WARNING) << LOG_DESC("The PSI has already been started"); + return; + } + m_started = true; + m_taskSyncTimer->registerTimeoutHandler([this]() { syncTaskInfo(); }); + m_taskSyncTimer->start(); + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("Start the PSI"); + // start a thread to execute task + startWorking(); +} + +void PSIFramework::stop() +{ + if (m_started == false) + { + return; + } + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("Stop PSI"); + m_started = false; + if (m_taskSyncTimer) + { + m_taskSyncTimer->stop(); + } + finishWorker(); + if (isWorking()) + { + // stop the worker thread + stopWorking(); + terminate(); + } + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("PSI stopped"); +} + +void PSIFramework::executeWorker() +{ + handleLocalTask(); + checkAndNotifyTaskResult(); + auto result = m_msgQueue->tryPop(c_PopWaitMs); + if (result.first) + { + handlePSIMsg(result.second); + return; + } + waitSignal(); +} + +// handle the local task +void PSIFramework::handleLocalTask() +{ + bcos::ReadGuard l(x_pendingTasks); + if (m_pendingTasks.empty()) + { + return; + } + for (auto const& it : m_pendingTasks) + { + if (it.second->finished()) + { + continue; + } + // Note: executeWork should not occupy too-much time + it.second->executeWork(); + } +} + + +// check the task-finished-or-not and response the result of the finished task +void PSIFramework::checkAndNotifyTaskResult() +{ + std::set erasedResourceID; + { + bcos::UpgradableGuard l(x_pendingTasks); + // with-no-task + if (m_pendingTasks.empty()) + { + return; + } + for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();) + { + auto taskState = it->second; + if (!taskState->finished()) + { + it++; + continue; + } + // notify the result to peer + auto dataResource = (taskState->task()->selfParty()->dataResource()); + if (dataResource) + { + erasedResourceID.insert(dataResource->resourceID()); + } + { + bcos::UpgradeGuard ul(l); + it = m_pendingTasks.erase(it); + } + notifyTaskResult(nullptr, taskState->peerID(), taskState->task()->id(), + dataResource ? dataResource->resourceID() : ""); + // notify the result to rpc response + taskState->onTaskFinished(); + } + } + // clear the dataResourceInfo related to the finished-task + batchRemoveDataResource(erasedResourceID); +} + +// Note: we should not block this function too much time +void PSIFramework::onReceiveMessage(PPCMessageFace::Ptr _msg) +{ + try + { + // decode the psi message and put it into to m_msgQueue + auto payLoad = _msg->data(); + auto psiMsg = + m_msgFactory->decodePSIMessage(bcos::bytesConstRef(payLoad->data(), payLoad->size())); + psiMsg->setFrom(_msg->sender()); + psiMsg->setTaskID(_msg->taskID()); + psiMsg->setSeq(_msg->seq()); + psiMsg->setUUID(_msg->uuid()); + m_msgQueue->push(psiMsg); + PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("onReceiveMessage") << printPSIMessage(psiMsg) + << LOG_KV("uuid", _msg->uuid()); + // notify to handle the message + m_signalled.notify_all(); + } + catch (std::exception const& e) + { + PSI_FRAMEWORK_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) + << LOG_KV("error", boost::diagnostic_information(e)) + << LOG_KV("msgSize", _msg->data()->size()); + } +} + +void PSIFramework::onTaskError(std::string const& _desc, bcos::Error::Ptr&& _error, + std::string const& _peerID, std::string const& _taskID, std::string const& _resourceID) +{ + if (!_error) + { + PSI_FRAMEWORK_LOG(DEBUG) << LOG_DESC(_desc) << LOG_KV("peer", _peerID) + << LOG_KV("task", _taskID) << LOG_KV("resource", _resourceID); + return; + } + PSI_FRAMEWORK_LOG(WARNING) << LOG_DESC(_desc) << " error" << LOG_KV("peer", _peerID) + << LOG_KV("task", _taskID) << LOG_KV("resource", _resourceID) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + // notify the peer that the task has been canceled + notifyTaskResult(_error, _peerID, _taskID, _resourceID); + // cancel the task + cancelTask(std::move(_error), _taskID); +} + + +// cancel the task and response to the user when error happens +void PSIFramework::cancelTask(bcos::Error::Ptr&& _error, std::string const& _task) +{ + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("cancelTask") << LOG_KV("task", _task) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + TaskResponseCallback callback = nullptr; + std::function finalizeCallback = nullptr; + TaskState::Ptr taskState = nullptr; + { + // get and delete the pending-tasks + UpgradableGuard l(x_pendingTasks); + auto it = m_pendingTasks.find(_task); + // find in the pending-task + if (it != m_pendingTasks.end()) + { + taskState = it->second; + UpgradeGuard ul(l); + // take the callback in-case-of the callback been called more than once + callback = it->second->takeCallback(); + finalizeCallback = it->second->takeFinalizeHandler(); + // erase from the pending-tasks + m_pendingTasks.erase(it); + } + } + if (taskState && taskState->task()->selfParty()->dataResource()) + { + removeLockingResource(taskState->task()->selfParty()->dataResource()->resourceID()); + } + { + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("cancelTask: response task result") + << LOG_KV("task", _task) << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + auto taskResult = std::make_shared(_task); + taskResult->setError(_error); + try + { + // calls the finalizeCallback to finalize when task failed + // make sure the resource has been released when the task failed + if (finalizeCallback) + { + finalizeCallback(); + } + + if (taskState) + { + auto reader = taskState->reader(); + if (reader) + { + reader->clean(); + } + + auto writer = taskState->writer(); + if (writer) + { + writer->clean(); + } + + if (writer && !_error->errorCode()) + { + writer->upload(); + } + } + } + catch (std::exception const& e) + { + PSI_LOG(WARNING) << LOG_DESC("cancelTask exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = std::make_shared(-1, boost::diagnostic_information(e)); + taskResult->setError(std::move(error)); + } + if (callback) + { + callback(std::move(taskResult)); + } + } +} + +bool PSIFramework::checkDataResourceForSelf( + TaskState::Ptr const& _taskState, std::string const& _peerID, bool _requireOutput) +{ + auto task = _taskState->task(); + auto dataResource = task->selfParty()->dataResource(); + // check the dataResource + if (!dataResource || !dataResource->desc()) + { + std::string errorMessage = "runPSI failed: Must specified the input data-resource"; + auto error = + std::make_shared((int)PSIRetCode::NotSpecifyInputDataResource, errorMessage); + onTaskError("runPSI", std::move(error), _peerID, task->id(), + dataResource ? dataResource->resourceID() : "empty"); + PSI_LOG(WARNING) << LOG_DESC("runPSI error: ") << errorMessage; + return false; + } + if (_requireOutput && !dataResource->outputDesc()) + { + std::string errorMessage = "Must specified the output data-resource"; + auto error = + std::make_shared((int)PSIRetCode::NotSpecifyOutputDataResource, errorMessage); + onTaskError("runPSI", std::move(error), _peerID, task->id(), dataResource->resourceID()); + PSI_LOG(WARNING) << LOG_DESC("runPSI error: ") << errorMessage; + return false; + } + return true; +} + +bcos::Error::Ptr PSIFramework::lockResourceAndRecordTaskState( + int _command, TaskState::Ptr const& _taskState) +{ + // should notify the worker to check the task-status when the sub-task finished + _taskState->registerSubTaskFinishedHandler([this]() { wakeupWorker(); }); + auto const& task = _taskState->task(); + bcos::Error::Ptr error = nullptr; + { + // check the task-in-progressing or not + bcos::UpgradableGuard l(x_pendingTasks); + if (m_pendingTasks.count(task->id())) + { + error = BCOS_ERROR_PTR( + (int)PSIRetCode::TaskInProcessing, "The task " + task->id() + " is in-processing!"); + } + else + { + // insert the task into pending-task-pool + bcos::UpgradeGuard ul(l); + m_pendingTasks[task->id()] = _taskState; + } + } + auto ret = error; + if (error) + { + // trigger the callback + // Note: here can not calls cancel task for cancelTask will cancel the processing task + auto taskResult = std::make_shared(task->id()); + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("lockResourceAndRecordTaskState: cancel task") + << LOG_KV("task", task) << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + taskResult->setError(std::move(error)); + auto callback = _taskState->takeCallback(); + callback(std::move(taskResult)); + return ret; + } + // Note: not all type tasks has resourceID + auto partyIndex = task->selfParty()->partyIndex(); + auto const& dataResource = task->selfParty()->dataResource(); + auto resourceID = dataResource ? dataResource->resourceID() : "empty"; + if (dataResource && needLockResource(_command, partyIndex)) + { + // Note: can't process the same data-resource in the same time + bcos::UpgradableGuard l(x_processingDataResource); + if (m_processingDataResource.count(resourceID)) + { + error = BCOS_ERROR_PTR((int)PSIRetCode::DataResourceOccupied, + "The dataResource " + resourceID + " is in-processing!"); + } + else + { + bcos::UpgradeGuard ul(l); + m_processingDataResource.insert(resourceID); + } + } + // Note: cancelTask will try to occupy lock x_processingDataResource and x_pendingTasks + // we should be careful when use cancelTask with locking x_processingDataResource or + // x_pendingTasks + ret = error; + if (error) + { + cancelTask(std::move(error), task->id()); + return ret; + } + return nullptr; +} + +ppc::protocol::PartyResource::Ptr PSIFramework::checkAndSetPeerInfo( + TaskState::Ptr const& _taskState, bool _enforcePeerResource) +{ + auto const& task = _taskState->task(); + auto const& dataResource = task->selfParty()->dataResource(); + // check the peer + auto const& peerParties = task->getAllPeerParties(); + if (peerParties.size() != 1) + { + std::string errorMessage = + "PSI only support two-party, please limit the party to 1, passed party count " + "is " + + std::to_string(peerParties.size()); + auto error = std::make_shared((int)PSIRetCode::OnlySupportOnePeer, errorMessage); + cancelTask(std::move(error), task->id()); + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("runPSI error for invalid peer ") << errorMessage << printTaskInfo(task); + return nullptr; + } + auto peerParty = peerParties.begin()->second; + _taskState->setPeerID(peerParty->id()); + if (!_enforcePeerResource) + { + return peerParty; + } + // check the peerDataResource + if (!peerParty->dataResource()) + { + std::string errorMessage = "runPSI failed: Must specified the peer data-resource-id"; + auto error = + std::make_shared((int)PSIRetCode::NotSpecifyPeerDataResource, errorMessage); + onTaskError( + "runPSI", std::move(error), peerParty->id(), task->id(), dataResource->resourceID()); + PSI_FRAMEWORK_LOG(WARNING) << LOG_DESC("runPSI error: ") << errorMessage; + return nullptr; + } + return peerParty; +} + +bool PSIFramework::checkPSIMsg(PSIMessageInterface::Ptr const& _msg) +{ + if (_msg->packetType() != (int)PSIPacketType::CancelTaskNotification && + _msg->packetType() != (int)PSIPacketType::TaskSyncMsg) + { + ReadGuard l(x_pendingTasks); + if (!m_pendingTasks.count(_msg->taskID())) + { + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("The task is not in the pendingPool") << printPSIMessage(_msg); + // notify the peer the task canceled + auto error = std::make_shared((int)PSIRetCode::TaskNotFound, + "Task " + _msg->taskID() + " not found in " + m_psiConfig->selfParty() + " !"); + notifyTaskResult(error, _msg->from(), _msg->taskID(), _msg->resourceID()); + return false; + } + } + return true; +} + +// notify the peer the task has been canceled for some error +void PSIFramework::notifyTaskResult(bcos::Error::Ptr const& _error, std::string const& _peerID, + std::string const& _taskID, std::string const& _resourceID) +{ + if (_peerID.empty() || _taskID.empty()) + { + return; + } + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("notifyTaskResult") << LOG_KV("peer", _peerID) + << LOG_KV("taskID", _taskID); + auto taskNotificationMsg = m_msgFactory->createTaskNotificationMessage( + (uint32_t)PSIPacketType::CancelTaskNotification); + taskNotificationMsg->setResourceID(_resourceID); + taskNotificationMsg->setErrorCode(_error ? _error->errorCode() : 0); + taskNotificationMsg->setErrorMessage(_error ? _error->errorMessage() : "success"); + taskNotificationMsg->setTaskID(_taskID); + taskNotificationMsg->setFrom(m_psiConfig->selfParty()); + m_psiConfig->generateAndSendPPCMessage(_peerID, _taskID, taskNotificationMsg, + [this, taskNotificationMsg](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("notifyTaskResult send error") << printPSIMessage(taskNotificationMsg) + << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); + }); +} + + +// broadcast the sync-task-information to all nodes of the given peerID +void PSIFramework::broadcastSyncTaskInfo( + std::string const& _peerID, std::vector const& _taskList) +{ + // Note: the empty task-id means that broadcast the message to all nodes + std::string targetTaskID = ""; + auto taskSyncMsg = m_msgFactory->createTaskInfoMessage((uint32_t)PSIPacketType::TaskSyncMsg); + taskSyncMsg->setTaskList(_taskList); + // the partyID to using to differentiate psi nodes + taskSyncMsg->setPartyID(m_psiConfig->front()->selfEndPoint()); + // broadcast the taskSyncMsg to all nodes + m_psiConfig->generateAndSendPPCMessage( + _peerID, targetTaskID, taskSyncMsg, [this, _peerID, _taskList](bcos::Error::Ptr&& _error) { + if (!_error) + { + return; + } + // PSI_FRAMEWORK_LOG(WARNING) + // << LOG_DESC( + // "broadcastSyncTaskInfo: send message error, cancel all tasks + // related to the " "peer") + // << LOG_KV("peer", _peerID) << LOG_KV("tasks", _taskList.size()) + // << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", + // _error->errorMessage()); + // cancel all the tasks + for (auto const& task : _taskList) + { + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskNotFound, + "Cancel task " + task + " for network error between peer-party!"); + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("syncTaskInfo: cancel tasks for network error") + << LOG_KV("task", task); + cancelTask(std::move(error), task); + } + }); +} + +// sync the task-information to the peers periodically +void PSIFramework::syncTaskInfo() +{ + // get the pendingTasks + std::map> pendingTasks; + std::set syncedPeers; + { + bcos::ReadGuard l(x_pendingTasks); + for (auto const& it : m_pendingTasks) + { + auto const& peer = it.second->peerID(); + if (peer.empty()) + { + continue; + } + pendingTasks[peer].emplace_back(it.first); + syncedPeers.insert(peer); + } + PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("syncTaskInfo") + << LOG_KV("totalPendingTasks", m_pendingTasks.size()); + } + ////// sync the pending-tasks information to the peer + // sync empty taskList to the peers without local-task corresponding-to + auto agencyList = m_psiConfig->agencyList(); + for (auto const& it : agencyList) + { + if (syncedPeers.count(it)) + { + continue; + } + if (it == m_psiConfig->selfParty()) + { + continue; + } + PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("syncTaskInfo: sync empty task") << LOG_KV("peer", it); + std::vector emptyTaskList; + broadcastSyncTaskInfo(it, emptyTaskList); + } + + // sync tasks to peers with pendingTasks corresponding-to + for (auto const& it : pendingTasks) + { + auto peerID = it.first; + if (peerID.empty()) + { + continue; + } + auto const& taskList = it.second; + PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("syncTaskInfo") + << LOG_KV("pendingTasks", taskList.size()) + << LOG_KV("peer", peerID); + broadcastSyncTaskInfo(peerID, taskList); + } + m_taskSyncTimer->restart(); +} + +// receive the taskSync message, delete tasks that no longer exist in-peer +void PSIFramework::handleTaskSyncInfo(PSIMessageInterface::Ptr _msg) +{ + auto taskInfoMsg = std::dynamic_pointer_cast(_msg); + auto const& taskList = taskInfoMsg->taskList(); + std::set peerTaskSet(taskList.begin(), taskList.end()); + updatePeerTasks(_msg->from(), _msg->partyID(), peerTaskSet); + // get the local task list corresponding to the peer + std::vector localExpiredTaskList; + { + bcos::ReadGuard l(x_pendingTasks); + for (auto const& it : m_pendingTasks) + { + auto const& peerID = it.second->peerID(); + if (peerID != _msg->from()) + { + continue; + } + // only add the expired task to the localExpiredTaskList + if (it.second->taskPendingTime() <= (uint64_t)m_psiConfig->taskExpireTime()) + { + continue; + } + localExpiredTaskList.emplace_back(it.first); + } + } + // compare and erase the task + std::vector tasksToCancel; + auto peerTasks = getPeerTasks(_msg->from()); + for (auto const& it : localExpiredTaskList) + { + bool taskExists = false; + for (auto const& taskSet : peerTasks) + { + if (taskSet.second.count(it)) + { + taskExists = true; + break; + } + } + if (!taskExists) + { + tasksToCancel.emplace_back(it); + } + } + PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("handleTaskSyncInfo") << printPSIMessage(_msg) + << LOG_KV("peerTaskSize", taskList.size()) + << LOG_KV("localExpiredTaskList", localExpiredTaskList.size()) + << LOG_KV("tasksToCancel", tasksToCancel.size()); + // cancel the tasks + for (auto const& it : tasksToCancel) + { + PSI_FRAMEWORK_LOG(INFO) + << LOG_DESC("handleTaskSyncInfo: cancel task that no longer exist in-peer") + << LOG_KV("task", it); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskNotFound, + "Cancel task " + it + " for task no longer exists in the peer-party!"); + cancelTask(std::move(error), it); + } +} + +void PSIFramework::handleTaskNotificationMsg(PSIMessageInterface::Ptr _msg) +{ + auto notificationResult = std::dynamic_pointer_cast(_msg); + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("handleTaskNotificationMsg") << printPSIMessage(_msg) + << LOG_KV("code", notificationResult->errorCode()) + << LOG_KV("msg", notificationResult->errorMessage()); + // cancel the local task + auto error = std::make_shared( + notificationResult->errorCode(), notificationResult->errorMessage()); + cancelTask(std::move(error), notificationResult->taskID()); +} + +bool PSIFramework::handlePSIFrameworkMsg(PSIMessageInterface::Ptr _msg) +{ + // Note: exception should been catched by the caller + switch (_msg->packetType()) + { + case (uint32_t)PSIPacketType::CancelTaskNotification: + { + handleTaskNotificationMsg(_msg); + break; + } + case (uint32_t)PSIPacketType::TaskSyncMsg: + { + handleTaskSyncInfo(_msg); + break; + } + case (uint32_t)PSIPacketType::HandshakeResponse: + { + onHandshakeResponse(_msg); + break; + } + case (uint32_t)PSIPacketType::HandshakeRequest: + { + onHandshakeRequest(_msg); + break; + } + case (uint32_t)PSIPacketType::PSIResultSyncMsg: + { + handlePSIResultSyncMsg(_msg); + break; + } + default: + { + return false; + } + } + return true; +} + +LineReader::Ptr PSIFramework::loadData(DataResourceLoader::Ptr _dataResourceLoader, + std::string const& _taskID, DataResource::ConstPtr const& _dataResource) +{ + // load data from the data-source(TODO: loadReader use async-interface) + auto reader = + _dataResourceLoader->loadReader(_dataResource->desc(), ppc::io::DataSchema::Bytes, true); + // only support one-column + if (reader->columnSize() == 0 || reader->columnSize() > 1) + { + auto errorMsg = "load data for task " + _taskID + + "failed, PSI dataSourcemust be on column, current column size is " + + std::to_string(reader->columnSize()); + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::LoadDataFailed, errorMsg)); + } + return reader; +} + +void PSIFramework::sendHandshakeRequest(TaskState::Ptr const& _taskState) +{ + if (_taskState->peerID().empty()) + { + return; + } + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("sendHandshakeRequest") + << printTaskInfo(_taskState->task()); + auto handshakeReq = + m_msgFactory->createHandshakeRequest((uint32_t)PSIPacketType::HandshakeRequest); + handshakeReq->setCurves( + g_PPCConfig.supportedCurves((uint8_t)TaskType::PSI, (uint8_t)m_psiConfig->algorithmType())); + handshakeReq->setHashList(g_PPCConfig.supportedHashList( + (uint8_t)TaskType::PSI, (uint8_t)m_psiConfig->algorithmType())); + handshakeReq->setTaskID(_taskState->task()->id()); + handshakeReq->setFrom(m_psiConfig->selfParty()); + m_psiConfig->generateAndSendPPCMessage(_taskState->peerID(), _taskState->task()->id(), + handshakeReq, [this, _taskState, handshakeReq](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("sendHandshakeRequest error") << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + auto resource = _taskState->task()->selfParty()->dataResource(); + onTaskError("sendHandshakeRequest", std::move(_error), _taskState->peerID(), + _taskState->task()->id(), resource ? resource->resourceID() : "emptyResource"); + }); +} + + +void PSIFramework::responsePSIResultSyncStatus(int32_t _code, std::string const& _msg, + std::string const& _peer, std::string const& _taskID, std::string const& _uuid, uint32_t _seq) +{ + // response to the client + auto psiMsg = + m_msgFactory->createTaskNotificationMessage((uint32_t)PSIPacketType::PSIResultSyncResponse); + psiMsg->setErrorCode(_code); + psiMsg->setErrorMessage(_msg); + m_psiConfig->asyncSendResponse( + _peer, _taskID, _uuid, psiMsg, + [this](bcos::Error::Ptr _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("responsePSIResultSyncStatus error") + << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); + }, + _seq); +} + +void PSIFramework::handlePSIResultSyncMsg(PSIMessageInterface::Ptr _resultSyncMsg) +{ + PSI_FRAMEWORK_LOG(INFO) << LOG_DESC("handlePSIResultSyncMsg") + << printPSIMessage(_resultSyncMsg); + auto taskState = getTaskByID(_resultSyncMsg->taskID()); + if (!taskState) + { + PSI_FRAMEWORK_LOG(WARNING) + << LOG_DESC("handlePSIResultSyncMsg error for the task not found") + << printPSIMessage(_resultSyncMsg); + std::string msg = + "sync psi result for task " + _resultSyncMsg->taskID() + " failed for task not found!"; + responsePSIResultSyncStatus((int32_t)PSIRetCode::TaskNotFound, msg, _resultSyncMsg->from(), + _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), _resultSyncMsg->seq()); + return; + } + try + { + taskState->storePSIResult(m_dataResourceLoader, _resultSyncMsg->takeData()); + responsePSIResultSyncStatus((int32_t)PSIRetCode::Success, "success", _resultSyncMsg->from(), + _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), _resultSyncMsg->seq()); + } + catch (std::exception const& e) + { + PSI_FRAMEWORK_LOG(WARNING) << LOG_DESC("handlePSIResultSyncMsg exception") + << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMessage = "sync psi result for " + _resultSyncMsg->taskID() + + " failed, error: " + std::string(boost::diagnostic_information(e)); + responsePSIResultSyncStatus((int32_t)PSIRetCode::SyncPSIResultFailed, errorMessage, + _resultSyncMsg->from(), _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), + _resultSyncMsg->seq()); + // cancel the task + auto error = BCOS_ERROR_PTR((int32_t)PSIRetCode::SyncPSIResultFailed, errorMessage); + cancelTask(std::move(error), _resultSyncMsg->taskID()); + } +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/PSIFramework.h b/cpp/ppc-psi/src/psi-framework/PSIFramework.h new file mode 100644 index 00000000..9f44414a --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/PSIFramework.h @@ -0,0 +1,250 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIFramework.h + * @author: yujiechen + * @date 2022-12-26 + */ +#pragma once +#include "../PSIConfig.h" +#include "TaskState.h" +#include "interfaces/PSIMessageFactory.h" +#include "interfaces/PSIMessageInterface.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include +#include +#include +#include +#include + +#define PSI_FRAMEWORK_LOG(LEVEL) \ + BCOS_LOG(LEVEL) << LOG_BADGE("PSI") \ + << LOG_BADGE((ppc::protocol::PSIAlgorithmType)m_psiConfig->algorithmType()) +using namespace ppc::task; +namespace ppc::psi +{ +using PSIMsgQueue = bcos::ConcurrentQueue; +using PSIMsgQueuePtr = std::shared_ptr; +class PSIFramework : public bcos::Worker, public TaskFrameworkInterface +{ +public: + using Ptr = std::shared_ptr; + PSIFramework(PSIMessageFactory::Ptr const& _msgFactory, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader, PSIConfig::Ptr const& _config, + std::string const& _workerName = "psi-base", unsigned _idleTimeMs = 0) + : Worker(_workerName, _idleTimeMs), + m_msgFactory(_msgFactory), + m_dataResourceLoader(_dataResourceLoader), + m_psiConfig(_config), + m_taskStateFactory(std::make_shared()), + m_msgQueue(std::make_shared()) + { + // Note: c_taskSyncTimerPeriod is inited after PSIFramework created, so the timer should + // been created after construct PSIFramework + m_taskSyncTimer = std::make_shared(c_taskSyncTimerPeriod, "taskSyncTimer"); + } + virtual ~PSIFramework() = default; + + void start() override; + void stop() override; + + // psi main processing function + // for ut to make this function public + void executeWorker() override; + + // register to the front to get the message related to + void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) override; + + uint64_t lockingResourceSize() const + { + bcos::ReadGuard l(x_processingDataResource); + return m_processingDataResource.size(); + } + + uint64_t pendingTasksSize() const + { + bcos::ReadGuard l(x_pendingTasks); + return m_pendingTasks.size(); + } + + inline void removePendingTask(std::string const& _taskID) + { + bcos::WriteGuard l(x_pendingTasks); + m_pendingTasks.erase(_taskID); + } + + TaskState::Ptr getTaskByID(std::string const& _taskID) + { + bcos::ReadGuard l(x_pendingTasks); + auto it = m_pendingTasks.find(_taskID); + if (it != m_pendingTasks.end()) + { + return it->second; + } + return nullptr; + } + + bool checkPSIMsg(PSIMessageInterface::Ptr const& _msg); + + bool checkDataResourceForSelf( + TaskState::Ptr const& _taskState, std::string const& _peerID, bool _requireOutput = false); + + PSIConfig::Ptr const& psiConfig() const { return m_psiConfig; } + +protected: + // handle the psi message + virtual void handlePSIMsg(PSIMessageInterface::Ptr _msg) = 0; + // should lock the party resource or not + virtual bool needLockResource(int _command, int _partyIndex) = 0; + // receive the handshake response + virtual void onHandshakeResponse(PSIMessageInterface::Ptr const& _msg) = 0; + // receive the handshake request + virtual void onHandshakeRequest(PSIMessageInterface::Ptr const& _msg) = 0; + + // handle the taskNotification and taskInfos related messages + virtual bool handlePSIFrameworkMsg(PSIMessageInterface::Ptr _msg); + + virtual bcos::Error::Ptr lockResourceAndRecordTaskState( + int _command, TaskState::Ptr const& _taskState); + + // notify error-result to the peer and cancel-task + void onTaskError(std::string const& _desc, bcos::Error::Ptr&& _error, + std::string const& _peerID, std::string const& _taskID, std::string const& _resourceID); + // cancel the task and response to the user when error happens + virtual void cancelTask(bcos::Error::Ptr&& _error, std::string const& _task); + + + // handle the local task looply + virtual void handleLocalTask(); + virtual void checkAndNotifyTaskResult(); + + // sync the task information and erase the task already deleted from peer + virtual void syncTaskInfo(); + virtual void handleTaskSyncInfo(PSIMessageInterface::Ptr _msg); + + // receive the task-cancel-msg + virtual void handleTaskNotificationMsg(PSIMessageInterface::Ptr _msg); + + /////// the common function + // notify the peer the task has been canceled for some error + virtual void notifyTaskResult(bcos::Error::Ptr const& _error, std::string const& _peerID, + std::string const& _taskID, std::string const& _resourceID); + + template + inline void batchRemoveDataResource(T const& _resourceList) + { + if (_resourceList.empty()) + { + return; + } + bcos::WriteGuard l(x_processingDataResource); + for (auto const& it : _resourceList) + { + m_processingDataResource.erase(it); + } + } + + virtual void removeLockingResource(std::string const& _resourceID) + { + // update the processing-data-resource + bcos::UpgradableGuard l(x_processingDataResource); + auto it = m_processingDataResource.find(_resourceID); + if (it != m_processingDataResource.end()) + { + bcos::UpgradeGuard ul(l); + m_processingDataResource.erase(it); + } + } + + void wakeupWorker() { m_signalled.notify_all(); } + + virtual ppc::protocol::PartyResource::Ptr checkAndSetPeerInfo( + TaskState::Ptr const& _taskState, bool _enforcePeerResource); + ppc::io::LineReader::Ptr loadData(ppc::io::DataResourceLoader::Ptr _dataResourceLoader, + std::string const& _taskID, ppc::protocol::DataResource::ConstPtr const& _dataResource); + + // the client send the handshakeRequest to the server + virtual void sendHandshakeRequest(TaskState::Ptr const& _taskState); + + // handle the PSIResultSyncMsg + virtual void handlePSIResultSyncMsg(PSIMessageInterface::Ptr _resultSyncMsg); + +private: + // utility functions + void waitSignal() + { + boost::unique_lock l(x_signalled); + m_signalled.wait_for(l, boost::chrono::milliseconds(5)); + } + void responsePSIResultSyncStatus(int32_t _code, std::string const& _msg, + std::string const& _peer, std::string const& _taskID, std::string const& _uuid, + uint32_t _seq); + + void broadcastSyncTaskInfo( + std::string const& _peerID, std::vector const& _taskList); + + void updatePeerTasks(std::string const& _agencyID, std::string const& _partyID, + std::set const& _tasks) + { + bcos::WriteGuard l(x_peerTasks); + m_peerTasks[_agencyID][_partyID] = _tasks; + } + + std::map> getPeerTasks(std::string const& _agencyID) const + { + bcos::ReadGuard l(x_peerTasks); + if (m_peerTasks.count(_agencyID)) + { + return m_peerTasks.at(_agencyID); + } + return std::map>(); + } + +protected: + PSIMessageFactory::Ptr m_msgFactory; + ppc::io::DataResourceLoader::Ptr m_dataResourceLoader; + PSIConfig::Ptr m_psiConfig; + // the task-state factory + TaskStateFactory::Ptr m_taskStateFactory; + + bool m_started = false; + + boost::condition_variable m_signalled; + boost::mutex x_signalled; + + mutable bcos::Mutex m_mutex; + + // the processing data resource + std::set m_processingDataResource; + mutable bcos::SharedMutex x_processingDataResource; + + // record the pending tasks(Note: only psi task recorded here) + std::unordered_map m_pendingTasks; + mutable bcos::SharedMutex x_pendingTasks; + + PSIMsgQueuePtr m_msgQueue; + const unsigned c_PopWaitMs = 5; + + // the timer used to sync the task-information and evict the expired or exceptioned task + // periodically + std::shared_ptr m_taskSyncTimer; + const unsigned int c_taskSyncTimerPeriod = 10000; + + // record the tasks of the peer + std::map>> m_peerTasks; + mutable bcos::SharedMutex x_peerTasks; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/ppc-psi/src/psi-framework/TaskGuarder.h new file mode 100644 index 00000000..a51c48f1 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/TaskGuarder.h @@ -0,0 +1,336 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskGuarder.h + * @author: shawnhe + * @date 2022-01-07 + */ + +#pragma once + +#include "../Common.h" +#include "../PSIConfig.h" +#include "TaskState.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-protocol/src/PPCMessage.h" +#include + +namespace ppc::psi +{ +class TaskGuarder +{ +public: + using Ptr = std::shared_ptr; + explicit TaskGuarder(PSIConfig::Ptr _config) : m_config(std::move(_config)) {} + TaskGuarder( + PSIConfig::Ptr _config, protocol::PSIAlgorithmType _type, const std::string& _timerName) + : m_config(std::move(_config)), + m_type(_type), + m_taskStateFactory(std::make_shared()) + { + m_pingTimer = std::make_shared(c_pingTimerPeriod, _timerName); + } + virtual ~TaskGuarder() = default; + + virtual void onReceivedErrorNotification(const std::string& _taskID){}; + virtual void onSelfError( + const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer){}; + + void startPingTimer() + { + m_pingTimer->registerTimeoutHandler([this]() { checkPeerActivity(); }); + m_pingTimer->start(); + } + void stopPingTimer() + { + if (m_pingTimer) + { + m_pingTimer->stop(); + } + } + + bcos::Error::Ptr checkTask(const ppc::protocol::Task::ConstPtr& _task, uint16_t _partiesCount, + bool _enforceSelfInput, bool _enforceSelfOutput, bool _enforcePeerResource, + bool _enforceSelfResource = true) + { + { + // check task id + bcos::ReadGuard l(x_pendingTasks); + if (m_pendingTasks.contains(_task->id())) + { + return std::make_shared( + (int)PSIRetCode::DuplicatedTask, "task already exists"); + } + } + + // check self party + auto const& selfParty = _task->selfParty(); + if (!selfParty) + { + return std::make_shared( + (int)PSIRetCode::TaskParamsError, "must specify self party info"); + } + if (selfParty->partyIndex() == (uint16_t)protocol::PartyType::Client) + { + _enforceSelfInput = true; + _enforceSelfOutput = true; + } + + // check self data resource + if (_enforceSelfResource) + { + auto const& dataResource = selfParty->dataResource(); + if (!dataResource) + { + return std::make_shared( + (int)PSIRetCode::TaskParamsError, "no data resource specified for self party"); + } + auto originData = dataResource->rawData(); + if (_enforceSelfInput && originData.empty() && !dataResource->desc()) + { + return std::make_shared( + (int)PSIRetCode::TaskParamsError, "no input resource specified for self party"); + } + if (_enforceSelfOutput && originData.empty() && !dataResource->outputDesc()) + { + return std::make_shared((int)PSIRetCode::TaskParamsError, + "no output resource specified for self party"); + } + } + + // check party index + std::set indexSet; + auto index = selfParty->partyIndex(); + if (_partiesCount > 1 && index >= _partiesCount) + { + return std::make_shared( + (int)PSIRetCode::TaskParamsError, "invalid partyIndex: " + std::to_string(index)); + } + indexSet.insert(index); + + // check the peer size + auto const& peerParties = _task->getAllPeerParties(); + if (peerParties.size() != uint(_partiesCount - 1)) + { + std::string errorMessage = "expected parties count: " + std::to_string(_partiesCount) + + ", current is " + std::to_string(peerParties.size()); + return std::make_shared((int)PSIRetCode::TaskParamsError, errorMessage); + } + + // check the party index of peers + for (auto& it : peerParties) + { + index = it.second->partyIndex(); + if (_partiesCount > 1 && index >= _partiesCount) + { + return std::make_shared((int)PSIRetCode::TaskParamsError, + "invalid partyIndex: " + std::to_string(index)); + } + if (indexSet.contains(index)) + { + return std::make_shared((int)PSIRetCode::TaskParamsError, + "repeated party index: " + std::to_string(index)); + } + indexSet.insert(index); + } + + // check the data resource of peer + if (!_enforcePeerResource) + { + return nullptr; + } + for (auto& it : peerParties) + { + if (!it.second->dataResource()) + { + return std::make_shared((int)PSIRetCode::NotSpecifyPeerDataResource, + "must specify the peer data resource"); + } + } + return nullptr; + } + + // this only work for two-party task + static std::string getPeerID(ppc::protocol::Task::ConstPtr _task) + { + auto const& peerParties = _task->getAllPeerParties(); + if (peerParties.empty()) + { + return ""; + } + return peerParties.begin()->second->id(); + } + + void noticePeerToFinish(const ppc::protocol::Task::ConstPtr& _task) + { + auto const& peerParties = _task->getAllPeerParties(); + if (peerParties.empty()) + { + return; + } + + for (const auto& peer : peerParties) + { + noticePeerToFinish(_task->id(), peer.first); + } + } + + void noticePeerToFinish(const std::string& _taskID, const std::string& _peer) + { + PSI_LOG(INFO) << LOG_DESC("noticePeerToFinish") << LOG_KV("task", _taskID) + << LOG_KV("peer", _peer); + try + { + if (_peer.empty()) + { + return; + } + + auto message = + m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(m_type), _taskID, std::make_shared()); + message->setMessageType(uint8_t(CommonMessageType::ErrorNotification)); + m_config->front()->asyncSendMessage( + _peer, message, m_config->networkTimeout(), [](const bcos::Error::Ptr&) {}, + nullptr); + } + catch (std::exception& e) + { + PSI_LOG(ERROR) << LOG_DESC("noticePeerToFinish") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + } + + void checkPeerActivity() + { + bcos::ReadGuard l(x_pendingTasks); + for (auto const& it : m_pendingTasks) + { + if (it.second->onlySelfRun()) + { + // no need to check peer + continue; + } + + auto task = it.second->task(); + auto const& peerParties = task->getAllPeerParties(); + if (peerParties.empty()) + { + continue; + } + + for (const auto& peer : peerParties) + { + auto message = + m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI), + uint8_t(m_type), task->id(), std::make_shared()); + message->setMessageType(uint8_t(CommonMessageType::PingPeer)); + m_config->front()->asyncSendMessage( + peer.first, message, m_config->networkTimeout(), + [this, task, peerID = peer.first](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + onSelfError(task->id(), + std::make_shared( + (int)PSIRetCode::PeerNodeDown, "peer node is down, id: " + peerID), + false); + }, + nullptr); + } + } + m_pingTimer->restart(); + } + + io::LineReader::Ptr loadReader(std::string const& _taskID, + protocol::DataResource::ConstPtr const& _dataResource, io::DataSchema _dataSchema, + uint32_t _columnSize = 1) + { + if (!_dataResource->rawData().empty()) + { + return nullptr; + } + auto reader = + m_config->dataResourceLoader()->loadReader(_dataResource->desc(), _dataSchema, true); + + if (reader->columnSize() == 0 || (reader->type() == protocol::DataResourceType::MySQL && + reader->columnSize() != _columnSize)) + { + auto errorMsg = "load data for task " + _taskID + "failed, expect " + + std::to_string(_columnSize) + " column, current column size is " + + std::to_string(reader->columnSize()); + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::LoadDataFailed, errorMsg)); + } + return reader; + } + + io::LineWriter::Ptr loadWriter(std::string const& _taskID, + protocol::DataResource::ConstPtr const& _dataResource, bool _enableOutputExists) + { + if (!_dataResource->rawData().empty()) + { + return nullptr; + } + if (!_enableOutputExists) + { + // Note: if the output-resource already exists, will throw exception + m_config->dataResourceLoader()->checkResourceExists(_dataResource->outputDesc()); + } + return m_config->dataResourceLoader()->loadWriter(_dataResource->outputDesc(), true); + } + + TaskState::Ptr findPendingTask(const std::string& _taskID) + { + bcos::ReadGuard l(x_pendingTasks); + auto it = m_pendingTasks.find(_taskID); + if (it == m_pendingTasks.end()) + { + return nullptr; + } + return it->second; + } + + void addPendingTask(TaskState::Ptr _taskState) + { + bcos::WriteGuard l(x_pendingTasks); + auto id = _taskState->task()->id(); + m_pendingTasks[id] = std::move(_taskState); + } + + void removePendingTask(const std::string& _taskID) + { + bcos::WriteGuard l(x_pendingTasks); + auto it = m_pendingTasks.find(_taskID); + if (it != m_pendingTasks.end()) + { + m_pendingTasks.erase(it); + } + } + +protected: + PSIConfig::Ptr m_config; + protocol::PSIAlgorithmType m_type; + TaskStateFactory::Ptr m_taskStateFactory; + // the timer used to check the activity of peer node + std::shared_ptr m_pingTimer; + + const unsigned int c_pingTimerPeriod = 60000; + + std::unordered_map m_pendingTasks; + mutable bcos::SharedMutex x_pendingTasks; +}; + +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/TaskState.h b/cpp/ppc-psi/src/psi-framework/TaskState.h new file mode 100644 index 00000000..2552533c --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/TaskState.h @@ -0,0 +1,460 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskState.h + * @author: yujiechen + * @date 2022-11-11 + */ +#pragma once +#include "../Common.h" +#include "../PSIConfig.h" +#include "ppc-framework/io/DataResourceLoader.h" +#include "ppc-framework/io/LineReader.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" +#include +#include +#include +#include +#include +using namespace ppc::task; +namespace ppc::psi +{ +class TaskState : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + TaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, + bool _onlySelfRun = false, PSIConfig::Ptr _config = nullptr) + : m_task(_task), + m_callback(std::move(_callback)), + m_onlySelfRun(_onlySelfRun), + m_config(std::move(_config)) + { + m_taskStartTime = bcos::utcSteadyTime(); + } + + virtual ~TaskState() {} + + TaskResponseCallback const& callback() { return m_callback; } + TaskResponseCallback takeCallback() { return std::move(m_callback); } + bool onlySelfRun() { return m_onlySelfRun; } + void setReader(io::LineReader::Ptr _reader, int64_t _readerParam) + { + m_reader = std::move(_reader); + m_readerParam = _readerParam; + if (m_reader) + { + m_sqlReader = (m_reader->type() == protocol::DataResourceType::MySQL); + } + } + void setWriter(io::LineWriter::Ptr _writer) { m_writer = std::move(_writer); } + protocol::Task::ConstPtr const& task() const { return m_task; } + + io::LineReader::Ptr const& reader() const { return m_reader; } + io::LineWriter::Ptr const& writer() const { return m_writer; } + + int64_t readerParam() const { return m_readerParam; } + bool sqlReader() const { return m_sqlReader; } + + io::DataBatch::Ptr loadAllData() + { + io::DataBatch::Ptr results; + auto originData = m_task->selfParty()->dataResource()->rawData(); + if (!originData.empty()) + { + results = std::make_shared(); + results->setDataSchema(ppc::io::DataSchema::Bytes); + + std::vector data; + for (auto& line : originData[0]) + { + data.emplace_back(bcos::bytes(line.begin(), line.end())); + } + results->setData(std::move(data)); + } + else + { + if (m_reader) + { + int64_t nextParam = m_sqlReader ? 0 : -1; + results = m_reader->next(nextParam, io::DataSchema::Bytes); + } + } + if (!results) + { + results = std::make_shared(); + } + return results; + } + + void writeLines(const io::DataBatch::Ptr& _data, io::DataSchema _schema) + { + if (m_writer) + { + m_writer->writeLine(_data, _schema); + m_writer->flush(); + m_writer->close(); + m_writer->upload(); + m_uploaded = true; + } + } + + void writeBytes(bcos::bytesConstRef _data) + { + if (m_writer) + { + m_writer->writeBytes(_data); + m_writer->flush(); + m_writer->close(); + m_writer->upload(); + m_uploaded = true; + } + } + + int32_t allocateSeq() + { + m_currentSeq.store(m_currentSeq.load() + 1); + { + bcos::WriteGuard l(x_seqList); + m_seqList.insert(m_currentSeq.load()); + } + return m_currentSeq; + } + + void eraseFinishedTaskSeq(uint32_t _seq, bool _success) + { + { + bcos::UpgradableGuard l(x_seqList); + auto it = m_seqList.find(_seq); + if (it == m_seqList.end()) + { + return; + } + bcos::UpgradeGuard ul(l); + m_seqList.erase(it); + if (_success) + { + m_successCount++; + } + else + { + m_failedCount++; + } + } + try + { + // trigger the callback when the sub-task finished + // Note: the subTaskHandler may go wrong + if (m_onSubTaskFinished) + { + m_onSubTaskFinished(); + } + PSI_LOG(INFO) << LOG_DESC("eraseFinishedTaskSeq") << LOG_KV("task", m_task->id()) + << LOG_KV("seq", _seq) << LOG_KV("success", _success) + << LOG_KV("seqs", m_seqList.size()) + << LOG_KV("successCount", m_successCount) + << LOG_KV("failedCount", m_failedCount); + } + catch (std::exception const& e) + { + PSI_LOG(WARNING) << LOG_DESC( + "eraseFinishedTaskSeq error for calls the sub-task-finalize " + "handler exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + onTaskException(boost::diagnostic_information(e)); + } + } + + virtual void setFinished(bool _finished) { m_finished.store(_finished); } + + virtual std::string peerID() const { return m_peerID; } + virtual void setPeerID(std::string const& _peerID) { m_peerID = _peerID; } + + virtual bool taskDone() { return m_taskDone; } + + // represent that the task has been finished or not + virtual bool finished() const + { + bcos::ReadGuard l(x_seqList); + return m_finished.load() && m_seqList.empty(); + } + + // trigger the callback to response + virtual void onTaskFinished() + { + // avoid repeated calls + if (m_taskDone.exchange(true)) + { + return; + } + PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("task", m_task->id()) + << LOG_KV("success", m_successCount) << LOG_KV("failed", m_failedCount) + << LOG_KV("loadFinished", m_finished.load()) + << LOG_KV("callback", m_callback ? "withCallback" : "emptyCallback"); + auto result = std::make_shared(m_task->id()); + try + { + // upload the psi-result + if (m_writer && !m_uploaded) + { + m_writer->upload(); + m_uploaded = true; + } + if (m_finalizeHandler) + { + m_finalizeHandler(); + } + if (m_failedCount > 0) + { + auto error = std::make_shared( + -1, "task " + m_task->id() + " failed for " + + boost::lexical_cast(m_failedCount) + " error!"); + result->setError(std::move(error)); + } + + // clear file + if (m_reader) + { + m_reader->clean(); + } + } + catch (std::exception const& e) + { + PSI_LOG(WARNING) << LOG_DESC("onTaskFinished exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = std::make_shared(-1, boost::diagnostic_information(e)); + result->setError(std::move(error)); + } + if (m_callback) + { + m_callback(std::move(result)); + } + } + + virtual void onTaskFinished(ppc::protocol::TaskResult::Ptr _result, bool _noticePeer) + { + // avoid repeated calls + if (m_taskDone.exchange(true)) + { + return; + } + PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("task", m_task->id()) + << LOG_KV("success", m_successCount) << LOG_KV("onlySelfRun", m_onlySelfRun) + << LOG_KV("finished", m_finished.load()) << LOG_KV("noticePeer", _noticePeer); + if (!_result) + { + _result = std::make_shared(m_task->id()); + } + try + { + // Note: we consider that the task success even if the handler exception + if (_noticePeer && !m_onlySelfRun && _result->error() && + _result->error()->errorCode() && m_notifyPeerFinishHandler) + { + m_notifyPeerFinishHandler(); + } + + if (m_finalizeHandler) + { + m_finalizeHandler(); + } + + m_finished.exchange(true); + + // clear file + if (m_reader) + { + m_reader->clean(); + } + } + catch (std::exception const& e) + { + PSI_LOG(WARNING) << LOG_DESC("onTaskFinished exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = std::make_shared(-1, boost::diagnostic_information(e)); + _result->setError(std::move(error)); + } + if (m_callback) + { + m_callback(std::move(_result)); + } + } + + virtual void onPeerNotifyFinish() + { + PSI_LOG(WARNING) << LOG_BADGE("onReceivePeerError") << LOG_KV("taskID", m_task->id()); + auto tesult = std::make_shared(task()->id()); + tesult->setError(std::make_shared( + (int)PSIRetCode::PeerNotifyFinish, "job participant sent an error")); + onTaskFinished(std::move(tesult), false); + } + + void setWorker(std::function const& _worker) { m_worker = _worker; } + void executeWork() + { + if (!m_worker) + { + return; + } + m_worker(); + } + + // set handler called when the task-finished + // Note: the finalize-handler maybe called no matter task success or failed + void registerFinalizeHandler(std::function const& _finalizeHandler) + { + m_finalizeHandler = _finalizeHandler; + } + + void registerSubTaskFinishedHandler(std::function const& _onSubTaskFinished) + { + m_onSubTaskFinished = _onSubTaskFinished; + } + + void registerNotifyPeerFinishHandler(std::function const& _notifyPeerFinishHandler) + { + m_notifyPeerFinishHandler = _notifyPeerFinishHandler; + } + + // Note: must store the result serially + void storePSIResult(ppc::io::DataResourceLoader::Ptr const& _resourceLoader, + std::vector const& _data) + { + bcos::RecursiveGuard l(m_mutex); + // try to generate-default-output-desc to make sure the server output exists even if not + // specified + tryToGenerateDefaultOutputDesc(); + auto dataResource = m_task->selfParty()->dataResource(); + // load the writer + if (!m_writer) + { + m_writer = _resourceLoader->loadWriter(dataResource->outputDesc()); + } + auto dataBatch = std::make_shared(); + dataBatch->setData(_data); + m_writer->writeLine(dataBatch, ppc::io::DataSchema::Bytes); + m_writer->flush(); + } + + std::function takeFinalizeHandler() { return std::move(m_finalizeHandler); } + + void onTaskException(std::string const& _errorMsg) + { + // set the task finished + setFinished(true); + { + bcos::WriteGuard l(x_seqList); + m_seqList.clear(); + } + if (!m_callback) + { + return; + } + auto taskResult = std::make_shared(m_task->id()); + auto msg = "Task " + m_task->id() + " exception, error : " + _errorMsg; + auto error = std::make_shared(-1, msg); + taskResult->setError(std::move(error)); + m_callback(std::move(taskResult)); + PSI_LOG(WARNING) << LOG_DESC(msg); + } + + bool loadFinished() const { return m_finished.load(); } + + // generate default output-desc for given task + void tryToGenerateDefaultOutputDesc() + { + auto dataResource = m_task->selfParty()->mutableDataResource(); + if (!dataResource) + { + dataResource = std::make_shared(); + m_task->mutableSelfParty()->setDataResource(dataResource); + } + if (dataResource->outputDesc()) + { + return; + } + auto outputDesc = std::make_shared(); + auto dstPath = c_resultPath + "/" + m_task->id() + ".result"; + outputDesc->setPath(dstPath); + outputDesc->setType((uint16_t)(ppc::protocol::DataResourceType::FILE)); + dataResource->setOutputDesc(outputDesc); + PSI_LOG(INFO) << LOG_DESC("GenerateDefaultOutputDesc for the output-desc not specified") + << LOG_KV("task", m_task->id()) << LOG_KV("path", dstPath); + } + + uint64_t taskPendingTime() { return (bcos::utcSteadyTime() - m_taskStartTime); } + +protected: + ppc::protocol::Task::ConstPtr m_task; + TaskResponseCallback m_callback; + bool m_onlySelfRun{false}; + PSIConfig::Ptr m_config; + uint64_t m_taskStartTime = 0; + + // record the task-peer + std::string m_peerID; + + std::function m_worker; + // handler called after the task-finished + std::function m_finalizeHandler; + + // handler called when the sub-task corresponding to the given seq completed + std::function m_onSubTaskFinished; + + std::function m_notifyPeerFinishHandler; + + // the reader + ppc::io::LineReader::Ptr m_reader; + int64_t m_readerParam; + bool m_sqlReader; + io::LineWriter::Ptr m_writer = nullptr; + + // to load logic in segments, prevent memory from filling up + // Note: only file has the segment logic + std::atomic m_currentSeq = {0}; + std::set m_seqList; + mutable bcos::SharedMutex x_seqList; + + uint64_t m_successCount = 0; + uint64_t m_failedCount = 0; + + std::atomic m_taskDone{false}; + std::atomic m_finished = {false}; + + mutable bcos::RecursiveMutex m_mutex; + + const std::string c_resultPath = "result"; + bool m_uploaded = false; +}; + +class TaskStateFactory +{ +public: + using Ptr = std::shared_ptr; + TaskStateFactory() = default; + virtual ~TaskStateFactory() = default; + + virtual TaskState::Ptr createTaskState(ppc::protocol::Task::ConstPtr const& _task, + TaskResponseCallback&& _callback, bool _onlySelfRun = false, + PSIConfig::Ptr _config = nullptr) + { + return std::make_shared( + _task, std::move(_callback), _onlySelfRun, std::move(_config)); + } +}; + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h b/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h new file mode 100644 index 00000000..e573d05f --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIMessageFactory.h + * @author: yujiechen + * @date 2022-11-9 + */ +#pragma once +#include "PSIMessageInterface.h" +#include +namespace ppc::psi +{ +class PSIMessageFactory +{ +public: + using Ptr = std::shared_ptr; + PSIMessageFactory() = default; + virtual ~PSIMessageFactory() = default; + + virtual PSIMessageInterface::Ptr decodePSIMessage(bcos::bytesConstRef _data) = 0; + virtual PSIMessageInterface::Ptr createPSIMessage(uint32_t _packetType) = 0; + + virtual PSIHandshakeRequest::Ptr createHandshakeRequest(uint32_t _packetType) = 0; + virtual PSIHandshakeResponse::Ptr createHandshakeResponse(uint32_t _packetType) = 0; + + virtual PSITaskNotificationMessage::Ptr createTaskNotificationMessage(uint32_t _packetType) = 0; + virtual PSITaskInfoMsg::Ptr createTaskInfoMessage(uint32_t _packetType) = 0; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h b/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h new file mode 100644 index 00000000..c4263477 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIMessageInterface.h + * @author: yujiechen + * @date 2022-11-9 + */ +#pragma once +#include "ppc-framework/io/DataBatch.h" +#include +#include + +namespace ppc::psi +{ +// the base message +class PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSIMessageInterface() = default; + virtual ~PSIMessageInterface() = default; + + // the packet-type + virtual uint32_t packetType() const = 0; + // the partyID(the current version is not used, but this field is added to facilitate other + // things in the future (such as permission control, etc.)) + virtual std::string const& partyID() const = 0; + // the resourceID + virtual std::string const& resourceID() const = 0; + + // for compatibility in the future + virtual int32_t version() const = 0; + // get all the data + virtual std::vector takeData() = 0; + virtual void setDataMap(std::map&& _map) = 0; + virtual std::map takeDataMap() = 0; + // Note: can't call this after takeData + virtual uint64_t dataSize() const = 0; + // get the data item + // Note: must ensure the object will not released before access the function + virtual bcos::bytesConstRef getData(uint64_t _index) = 0; + + virtual void setPartyID(std::string const& _partyID) = 0; + virtual void setResourceID(std::string const& _resourceID) = 0; + virtual void setVersion(int32_t _version) = 0; + virtual void setData(std::vector const& _data) = 0; + virtual void setData(std::vector&& _data) = 0; + // copy [_offset, _offset + _len) into the data-field + virtual void setData( + std::vector const& _data, uint64_t _offset, uint64_t _len) = 0; + + virtual void appendData(bcos::bytes const& _dataItem) = 0; + + virtual void setPacketType(uint32_t _packetType) = 0; + // the dataBatchSize + virtual void setDataBatchCount(uint32_t _dataBatchCount) = 0; + virtual uint32_t dataBatchCount() const = 0; + // encode the PSIMessage + virtual bcos::bytesPointer encode() const = 0; + // decode the PSIMessage + virtual void decode(bcos::bytesConstRef _data) = 0; + + // the field that should not serialized/deserialized, exist only in memory + virtual void setTaskID(std::string const& _taskID) { m_taskID = _taskID; } + virtual void setSeq(uint32_t _seq) { m_seq = _seq; } + virtual void setFrom(std::string const& _from) { m_from = _from; } + + + virtual std::string const& taskID() const { return m_taskID; } + virtual uint32_t seq() const { return m_seq; } + virtual std::string const& from() const { return m_from; } + + virtual std::string const& uuid() const { return m_uuid; } + virtual void setUUID(std::string const& _uuid) { m_uuid = _uuid; } + +private: + std::string m_taskID; + int32_t m_seq; + std::string m_from; + std::string m_uuid; +}; + + +// the psi-task-notification message interface +class PSITaskNotificationMessage : virtual public PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSITaskNotificationMessage() = default; + ~PSITaskNotificationMessage() override = default; + + virtual void setErrorCode(int32_t _code) = 0; + virtual void setErrorMessage(std::string const& _errorMsg) = 0; + + virtual int32_t errorCode() const = 0; + virtual std::string const& errorMessage() const = 0; +}; + +class PSITaskInfoMsg : virtual public PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSITaskInfoMsg() = default; + ~PSITaskInfoMsg() override = default; + + virtual std::vector taskList() const = 0; + virtual void setTaskList(std::vector&& _taskList) = 0; + virtual void setTaskList(std::vector const& _taskList) = 0; +}; + +// the psi handshake-request +class PSIHandshakeRequest : virtual public PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSIHandshakeRequest() = default; + ~PSIHandshakeRequest() override = default; + + virtual void setCurves(std::vector&& _curves) = 0; + virtual void setHashList(std::vector&& _hashList) = 0; + + virtual std::vector const& curves() const = 0; + virtual std::vector const& hashList() const = 0; +}; + +// the psi handshake-response +class PSIHandshakeResponse : virtual public PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSIHandshakeResponse() = default; + ~PSIHandshakeResponse() override = default; + + virtual void setHashType(int _hashType) = 0; + virtual void setCurve(int _curve) = 0; + + virtual int hashType() const = 0; + virtual int curve() const = 0; +}; + +inline std::string printPSIMessage(PSIMessageInterface::Ptr _msg) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("from", _msg->from()) << LOG_KV("party", _msg->partyID()) + << LOG_KV("task", _msg->taskID()) << LOG_KV("seq", _msg->seq()) + << LOG_KV("resource", _msg->resourceID()) << LOG_KV("type", _msg->packetType()); + return stringstream.str(); +} +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSI.tars b/cpp/ppc-psi/src/psi-framework/protocol/PSI.tars new file mode 100644 index 00000000..13645e10 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/protocol/PSI.tars @@ -0,0 +1,28 @@ +module ppctars{ + struct TarsCuckooFilter + { + 1 optional int id; + 2 optional vector hash; + 3 optional vector data; + }; + + struct PSIMessage + { + 1 optional int version; + 2 require int packetType; + 3 optional string partyID; + 4 optional string resourceID; + 5 optional vector> data; + 6 optional map> dataMap; + 7 optional int errorCode; + 8 optional string errorMessage; + 9 optional vector taskList; + 10 optional vector cuckooFilterInfo; + 11 optional int cuckooFilterSize; + 12 optional int dataBatchCount; + 13 optional vector curves; // for PSIHandshakeRequest + 14 optional vector hashList; // for PSIHandshakeRequest + 15 optional int curve; // for PSIHandshakeResponse + 16 optional int hashType; // for PSIHandshakeResponse + }; +}; \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp new file mode 100644 index 00000000..1a93d668 --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIMessage.cpp + * @author: yujiechen + * @date 2022-11-9 + */ +#include "PSIMessage.h" +#include "ppc-tars-protocol/Common.h" + +using namespace ppc::psi; +using namespace bcos; + +// encode the PSIMessage +bytesPointer PSIMessage::encode() const +{ + auto encodedData = std::make_shared(); + tars::TarsOutputStream output; + m_inner()->writeTo(output); + output.getByteBuffer().swap(*encodedData); + return encodedData; +} + +// decode the PSIMessage +void PSIMessage::decode(bytesConstRef _data) +{ + tars::TarsInputStream input; + input.setBuffer((const char*)_data.data(), _data.size()); + m_inner()->readFrom(input); +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h new file mode 100644 index 00000000..8cdeff6d --- /dev/null +++ b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h @@ -0,0 +1,298 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PSIMessage.h + * @author: yujiechen + * @date 2022-11-9 + */ +#pragma once +#include "../../Common.h" +#include "../interfaces/PSIMessageFactory.h" +#include "../interfaces/PSIMessageInterface.h" +#include "PSI.h" +#include + +namespace ppc::psi +{ +// Note: this implementation is not thread-safe +class PSIMessage : virtual public PSIMessageInterface +{ +public: + using Ptr = std::shared_ptr; + PSIMessage() : m_inner([inner = ppctars::PSIMessage()]() mutable { return &inner; }) {} + explicit PSIMessage(uint32_t _type) : PSIMessage() { setPacketType(_type); } + explicit PSIMessage(bcos::bytesConstRef _data) : PSIMessage() { decode(_data); } + explicit PSIMessage(std::function _inner) : m_inner(std::move(_inner)) + {} + ~PSIMessage() {} + + // the packet-type + uint32_t packetType() const override { return m_inner()->packetType; } + // the partyID(the current version is not used, but this field is added to facilitate other + // things in the future (such as permission control, etc.)) + std::string const& partyID() const override { return m_inner()->partyID; } + std::string const& resourceID() const override { return m_inner()->resourceID; } + // for compatibility in the future + int32_t version() const override { return m_inner()->version; } + // get all the data: not recommend to use this method(only used for ut) + uint64_t dataSize() const override { return m_inner()->data.size(); } + + std::vector takeData() override + { + auto dataSize = m_inner()->data.size(); + std::vector result; + result.resize(dataSize); + for (uint64_t i = 0; i < dataSize; i++) + { + std::move(m_inner()->data[i].begin(), m_inner()->data[i].end(), + std::back_inserter(result[i])); + } + return result; + } + + std::map takeDataMap() override + { + std::map result; + for (auto& val : m_inner()->dataMap) + { + if (val.second.data()) + { + result[val.first].assign( + m_inner()->dataMap[val.first].begin(), m_inner()->dataMap[val.first].end()); + } + } + return result; + } + + // Note: must ensure the object will not released before access the function + bcos::bytesConstRef getData(uint64_t _index) override + { + auto const& dataItem = m_inner()->data.at(_index); + return bcos::bytesConstRef((const bcos::byte*)(dataItem.data()), dataItem.size()); + } + void setPartyID(std::string const& _partyID) override { m_inner()->partyID = _partyID; } + void setResourceID(std::string const& _resourceID) override + { + m_inner()->resourceID = _resourceID; + } + + void setVersion(int32_t _version) override { m_inner()->version = _version; } + void setData(std::vector const& _data) override + { + setData(_data, 0, _data.size()); + } + + void setData(std::vector const& _data, uint64_t _offset, uint64_t _len) override + { + m_inner()->data.clear(); + uint64_t endPos = std::min(_offset + _len, (uint64_t)_data.size()); + m_inner()->data.resize((endPos - _offset + 1)); + uint64_t j = 0; + for (uint64_t i = _offset; i < endPos; i++) + { + m_inner()->data[j].assign(_data.at(i).begin(), _data.at(i).end()); + j++; + } + } + + void setData(std::vector&& _data) override + { + auto dataSize = _data.size(); + m_inner()->data.clear(); + m_inner()->data.resize(dataSize); + for (uint64_t i = 0; i < dataSize; i++) + { + std::move(_data[i].begin(), _data[i].end(), std::back_inserter(m_inner()->data[i])); + } + } + + void setDataMap(std::map&& _map) override + { + m_inner()->dataMap.clear(); + for (auto& val : _map) + { + m_inner()->dataMap[val.first].assign(val.second.begin(), val.second.end()); + } + } + + void appendData(bcos::bytes const& _dataItem) override + { + m_inner()->data.emplace_back(_dataItem.begin(), _dataItem.end()); + } + void setPacketType(uint32_t _packetType) override { m_inner()->packetType = _packetType; } + + void setDataBatchCount(uint32_t _dataBatchCount) override + { + m_inner()->dataBatchCount = _dataBatchCount; + } + uint32_t dataBatchCount() const override { return m_inner()->dataBatchCount; } + + // encode the PSIMessage + bcos::bytesPointer encode() const override; + // decode the PSIMessage + void decode(bcos::bytesConstRef _data) override; + +protected: + std::function m_inner; +}; + +class PSITaskNotificationMessageImpl : virtual public PSITaskNotificationMessage, public PSIMessage +{ +public: + using Ptr = std::shared_ptr; + PSITaskNotificationMessageImpl() : PSIMessage() {} + explicit PSITaskNotificationMessageImpl(uint32_t _type) : PSIMessage(_type) {} + explicit PSITaskNotificationMessageImpl(std::function _inner) + : PSIMessage(std::move(_inner)) + {} + + ~PSITaskNotificationMessageImpl() override = default; + + virtual void setErrorCode(int32_t _code) override { m_inner()->errorCode = _code; } + virtual void setErrorMessage(std::string const& _errorMsg) override + { + m_inner()->errorMessage = _errorMsg; + } + + virtual int32_t errorCode() const override { return m_inner()->errorCode; } + virtual std::string const& errorMessage() const override { return m_inner()->errorMessage; } +}; + +class PSITaskInfoMsgImpl : public PSITaskInfoMsg, public PSIMessage +{ +public: + using Ptr = std::shared_ptr; + PSITaskInfoMsgImpl() : PSIMessage() {} + PSITaskInfoMsgImpl(uint32_t _type) : PSIMessage(_type) {} + explicit PSITaskInfoMsgImpl(std::function _inner) + : PSIMessage(std::move(_inner)) + {} + + ~PSITaskInfoMsgImpl() override = default; + + std::vector taskList() const override { return m_inner()->taskList; } + + void setTaskList(std::vector&& _taskList) override + { + m_inner()->taskList = std::move(_taskList); + } + + void setTaskList(std::vector const& _taskList) override + { + m_inner()->taskList = _taskList; + } +}; + +class PSIHandshakeRequestImpl : public PSIHandshakeRequest, public PSIMessage +{ +public: + using Ptr = std::shared_ptr; + PSIHandshakeRequestImpl() : PSIMessage() {} + PSIHandshakeRequestImpl(uint32_t _type) : PSIMessage(_type) {} + PSIHandshakeRequestImpl(std::function _inner) + : PSIMessage(std::move(_inner)) + {} + ~PSIHandshakeRequestImpl() override = default; + + void setCurves(std::vector&& _curves) override { m_inner()->curves = std::move(_curves); } + void setHashList(std::vector&& _hashList) override + { + m_inner()->hashList = std::move(_hashList); + } + + std::vector const& curves() const override { return m_inner()->curves; } + std::vector const& hashList() const override { return m_inner()->hashList; } +}; + +// the psi handshake-response +class PSIHandshakeResponseImpl : public PSIHandshakeResponse, public PSIMessage +{ +public: + using Ptr = std::shared_ptr; + PSIHandshakeResponseImpl() : PSIMessage() {} + PSIHandshakeResponseImpl(uint32_t _type) : PSIMessage(_type) {} + PSIHandshakeResponseImpl(std::function _inner) + : PSIMessage(std::move(_inner)) + {} + + ~PSIHandshakeResponseImpl() override = default; + + void setHashType(int _hashType) override { m_inner()->hashType = _hashType; } + void setCurve(int _curve) override { m_inner()->curve = _curve; } + + // the selected hash algorithm + int hashType() const override { return m_inner()->hashType; } + // the selected curve + int curve() const override { return m_inner()->curve; } +}; + +// the factory implementation +class PSIMessageFactoryImpl : virtual public PSIMessageFactory +{ +public: + using Ptr = std::shared_ptr; + PSIMessageFactoryImpl() = default; + ~PSIMessageFactoryImpl() override {} + + PSIMessageInterface::Ptr createPSIMessage(uint32_t _packetType) override + { + return std::make_shared(_packetType); + } + + PSITaskNotificationMessage::Ptr createTaskNotificationMessage(uint32_t _packetType) override + { + return std::make_shared(_packetType); + } + + PSITaskInfoMsg::Ptr createTaskInfoMessage(uint32_t _packetType) override + { + return std::make_shared(_packetType); + } + + PSIHandshakeRequest::Ptr createHandshakeRequest(uint32_t _packetType) override + { + return std::make_shared(_packetType); + } + + PSIHandshakeResponse::Ptr createHandshakeResponse(uint32_t _packetType) override + { + return std::make_shared(_packetType); + } + +protected: + virtual PSIMessageInterface::Ptr decodePSIBaseMessage( + std::function _inner) + { + switch (_inner()->packetType) + { + case (uint32_t)PSIPacketType::PSIResultSyncMsg: + return std::make_shared(_inner); + case (uint32_t)PSIPacketType::CancelTaskNotification: + case (uint32_t)PSIPacketType::PSIResultSyncResponse: + case (uint32_t)PSIPacketType::HandshakeSuccess: + return std::make_shared(_inner); + case (uint32_t)PSIPacketType::TaskSyncMsg: + return std::make_shared(_inner); + case (uint32_t)PSIPacketType::HandshakeRequest: + return std::make_shared(_inner); + case (uint32_t)PSIPacketType::HandshakeResponse: + return std::make_shared(_inner); + default: + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::UnknownPSIPacketType, + "unknown psi-message-type " + std::to_string(_inner()->packetType))); + } + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt b/cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt new file mode 100644 index 00000000..d38c319b --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${RA2018_PSI_TARGET} ${SRCS}) +target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET}) \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/Common.h b/cpp/ppc-psi/src/ra2018-psi/Common.h new file mode 100644 index 00000000..b5996cdb --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/Common.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-27 + */ +#pragma once +#include "../Common.h" +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-tools/src/cuckoo/Cuckoofilter.h" +#include +#include + +#define RA2018_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: RA2018") + +namespace ppc::psi +{ +using DefaultCukooFilter = + ppc::tools::Cuckoofilter; +using DefaultCukooFilterPtr = + std::shared_ptr>; +// the data-preprocessing command +constexpr static std::string_view DATA_PREPROCESSING_CMD = "data_preprocessing"; +// the psi command +constexpr static std::string_view RUN_PSI_CMD = "ra2018_psi"; + +enum DataPreProcessingOption : int +{ + Insert, + Delete, +}; +inline std::ostream& operator<<(std::ostream& _out, DataPreProcessingOption const& _option) +{ + switch (_option) + { + case DataPreProcessingOption::Insert: + _out << "Insert"; + break; + case DataPreProcessingOption::Delete: + _out << "Delete"; + break; + default: + _out << "UnknownDataPreProcessingOption"; + break; + } + return _out; +} + +enum RA2018Command : int +{ + DATA_PREPROCESSING, + RUN_PSI, +}; +inline std::ostream& operator<<(std::ostream& _out, RA2018Command const& _command) +{ + switch (_command) + { + case RA2018Command::DATA_PREPROCESSING: + _out << "DATA_PREPROCESSING"; + break; + case RA2018Command::RUN_PSI: + _out << "RUN_PSI"; + break; + default: + _out << "UnknownRA2018PSICommand"; + break; + } + return _out; +} +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h new file mode 100644 index 00000000..6954356e --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIConfig.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "../PSIConfig.h" +#include "Common.h" +#include "ppc-framework/crypto/Hash.h" +#include "ppc-framework/crypto/RA2018OprfInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include "ppc-framework/storage/SQLStorage.h" +#include "protocol/RA2018Message.h" +#include + +namespace ppc::psi +{ +class RA2018PSIConfig : public PSIConfig +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIConfig(std::string const& _selfParty, ppc::front::FrontInterface::Ptr _front, + ppc::crypto::RA2018OprfInterface::Ptr _oprf, ppc::crypto::Hash::Ptr _binHashImpl, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + ppc::tools::CuckoofilterOption::Ptr _cuckooFilterOption, bcos::ThreadPool::Ptr _threadPool, + ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes, + std::string const& _dbName = "ra2018", uint64_t _cuckooFilterCacheSize = 255 * 1024 * 1024, + uint64_t _ra2018CacheCapacity = 1024 * 1024 * 1024, int _dataBatchSize = -1, + int _threadPoolSize = std::thread::hardware_concurrency()) + : PSIConfig(ppc::protocol::PSIAlgorithmType::RA_PSI_2PC, _selfParty, _front, _ppcMsgFactory, + _dataResourceLoader, _holdingMessageMinutes), + m_oprf(std::move(_oprf)), + m_binHashImpl(std::move(_binHashImpl)), + m_ra2018MsgFactory(std::make_shared()), + m_threadPool(std::move(_threadPool)), + m_storage(std::move(_storage)), + m_fileStorage(std::move(_fileStorage)), + m_dbName(_dbName), + m_cuckooFilterOption(std::move(_cuckooFilterOption)), + m_cuckooFilterCacheSize(_cuckooFilterCacheSize), + m_ra2018CacheCapacity(_ra2018CacheCapacity), + m_dataBatchSize(_dataBatchSize) + { + if (!m_threadPool) + { + m_threadPool = std::make_shared("ra2018", _threadPoolSize); + } + } + ~RA2018PSIConfig() override = default; + + std::string const& dbName() const { return m_dbName; } + std::string const& cuckooFilterTableName() const { return m_cuckooFilterTableName; } + void setCukooFilterTableName(std::string const& _tableName) + { + m_cuckooFilterTableName = _tableName; + } + + ppc::storage::SQLStorage::Ptr const& storage() { return m_storage; } + ppc::storage::FileStorage::Ptr const& fileStorage() { return m_fileStorage; } + + bcos::ThreadPool::Ptr const& threadPool() { return m_threadPool; } + + ppc::io::DataResourceLoader::Ptr const& dataResourceLoader() { return m_dataResourceLoader; } + ppc::crypto::RA2018OprfInterface::Ptr const& oprf() const { return m_oprf; } + ppc::crypto::Hash::Ptr const& binHashImpl() const { return m_binHashImpl; } + + // the ra2018 message factory + RA2018MessageFactory::Ptr const& ra2018MsgFactory() const { return m_ra2018MsgFactory; } + ppc::tools::CuckoofilterOption::Ptr const& cuckooFilterOption() const + { + return m_cuckooFilterOption; + } + int64_t dataBatchSize() const { return m_dataBatchSize; } + + uint64_t cuckooFilterCacheSize() const { return m_cuckooFilterCacheSize; } + uint64_t ra2018CacheCapacity() const { return m_ra2018CacheCapacity; } + std::string const& dataPath() const { return m_dataPath; } + +protected: + ppc::crypto::RA2018OprfInterface::Ptr m_oprf; + // the hash-impl used to calculate binary hash(for perf consideration, we default choose md5) + ppc::crypto::Hash::Ptr m_binHashImpl; + // the ra2018 message factory + RA2018MessageFactory::Ptr m_ra2018MsgFactory; + + // the threadPool used to computation + bcos::ThreadPool::Ptr m_threadPool; + // the ppc-privacy-service storage + ppc::storage::SQLStorage::Ptr m_storage; + // the file-storage + ppc::storage::FileStorage::Ptr m_fileStorage; + + // the db to store all data for RA2018 + std::string m_dbName = "ra2018"; + // the cuckoo-filter tableName, stores {data_resource-id, cuckoo-filter-id, cuckoo-filter-data} + std::string m_cuckooFilterTableName = "t_cuckoo"; + + // the cuckooFilterOption + ppc::tools::CuckoofilterOption::Ptr m_cuckooFilterOption; + + uint64_t m_cuckooFilterCacheSize; + uint64_t m_ra2018CacheCapacity = 0; + // default load all data + int64_t m_dataBatchSize = -1; + + std::string m_dataPath = "data"; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h new file mode 100644 index 00000000..a75c742a --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIFactory.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "RA2018PSIImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "storage/RA2018PSIStorage.h" +namespace ppc::psi +{ +class RA2018PSIFactory +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIFactory() = default; + virtual ~RA2018PSIFactory() = default; + + virtual RA2018PSIImpl::Ptr createRA2018PSI(std::string const& _selfParty, + ppc::front::FrontInterface::Ptr _front, ppc::tools::PPCConfig::Ptr const& _config, + ppc::crypto::RA2018OprfInterface::Ptr const& _oprf, ppc::crypto::Hash::Ptr _binHashImpl, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage, + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader) + { + auto const& ra2018Config = _config->ra2018PSIConfig(); + + auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl, + _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage, + _dataResourceLoader, _config->holdingMessageMinutes(), ra2018Config.dbName, + ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize, ra2018Config.dataBatchSize); + + RA2018PSIStorage::Ptr psiStorage = nullptr; + if (!_config->disableRA2018()) + { + psiStorage = std::make_shared(config); + // init the storage + psiStorage->init(); + } + return std::make_shared( + config, psiStorage, 0, false, _config->disableRA2018()); + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp new file mode 100644 index 00000000..362857c7 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp @@ -0,0 +1,1077 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIImpl.cpp + * @author: yujiechen + * @date 2022-10-25 + */ +#include "RA2018PSIImpl.h" +#include "core/CuckoofilterAllocator.h" +#include "core/Ra2018TaskParam.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace bcos; +using namespace ppc::front; +using namespace ppc::io; + +RA2018PSIImpl::RA2018PSIImpl(RA2018PSIConfig::Ptr const& _config, + RA2018PSIStorage::Ptr const& _storage, unsigned _idleTimeMs, bool _waitResult, bool _disabled) + : PSIFramework(_config->ra2018MsgFactory(), _config->dataResourceLoader(), _config, "ra2018psi", + _idleTimeMs), + m_disabled(_disabled), + m_config(_config), + m_storage(_storage), + m_server(std::make_shared(_config)), // create the RA2018PSIServer to support + // fullEvaluate/Evaluate + m_client(std::make_shared(_config, + _config->ra2018CacheCapacity())), // create the RA2018PSIClient to support + // blind/finalize/intersection, support multiple-tasks + m_cuckooFiltersCache(std::make_shared( + _storage, _config->cuckooFilterCacheSize())), // the cuckoo-filters-cache + m_worker(std::make_shared("offline-worker", 1)), + m_waitResult(_waitResult) // notify only when the task-finished or exceptioned +{ + m_notifier = m_client->onReady([&]() { wakeupWorker(); }); +} + +void RA2018PSIImpl::start() +{ + RA2018_LOG(INFO) << LOG_DESC("Start the RA2018PSI"); + // start a thread to execute task + PSIFramework::start(); +} + +void RA2018PSIImpl::stop() +{ + RA2018_LOG(INFO) << LOG_DESC("Stop RA2018PSI"); + if (m_config->threadPool()) + { + m_config->threadPool()->stop(); + } + PSIFramework::stop(); + RA2018_LOG(INFO) << LOG_DESC("RA2018PSI stopped"); +} + +void RA2018PSIImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +{ + if (m_disabled) + { + m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + auto taskResult = std::make_shared(_task->id()); + auto error = BCOS_ERROR_PTR( + (int)RA2018PSIDisabled, "The ra2018-psi has been disabled by this node!"); + RA2018_LOG(WARNING) << LOG_DESC("response task result") << LOG_KV("task", _task) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + auto const& peerParties = _task->getAllPeerParties(); + + auto notifiedError = BCOS_ERROR_PTR( + (int)RA2018PSIDisabled, "The ra2018-psi has been disabled by the peer node!"); + for (auto const& it : peerParties) + { + notifyTaskResult(notifiedError, it.second->id(), _task->id(), "emptyResource"); + } + taskResult->setError(std::move(error)); + _onTaskFinished(std::move(taskResult)); + // erase the task id + m_config->front()->eraseTaskInfo(_task->id()); + return; + } + // parse the task command + auto taskParam = std::make_shared(_task->param()); + auto dataResource = _task->selfParty()->dataResource(); + auto resourceID = dataResource ? dataResource->resourceID() : "empty"; + auto self = weak_from_this(); + auto taskState = m_taskStateFactory->createTaskState( + _task, [self, _task, _onTaskFinished](TaskResult::Ptr&& _result) { + // record the task time + auto result = std::move(_result); + _onTaskFinished(std::move(result)); + RA2018_LOG(INFO) << LOG_DESC("asyncRunTask finished") << printTaskInfo(_task); + }); + auto const& taskID = _task->id(); + // Note: the finalizeHandler will be called no matter task success of failed + taskState->registerFinalizeHandler([self, taskID]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + // erase the cache + psi->m_client->eraseTaskCache(taskID); + // erase task-info from the gateway + psi->m_config->front()->eraseTaskInfo(taskID); + }); + try + { + // Note: the psi-server no need to lock the resource and task + auto error = lockResourceAndRecordTaskState(taskParam->command(), taskState); + // response to the sdk + if (error) + { + // trigger the callback + auto taskResult = std::make_shared(_task->id()); + RA2018_LOG(WARNING) << LOG_DESC("response task result") << LOG_KV("task", _task) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + taskResult->setError(std::move(error)); + _onTaskFinished(std::move(taskResult)); + return; + } + taskState->registerSubTaskFinishedHandler([self]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->m_client->checkAndStoreInterSectionResult(); + psi->wakeupWorker(); + }); + + switch (taskParam->command()) + { + // data-preprocessing + case (int)RA2018Command::DATA_PREPROCESSING: + { + asyncRunDataProcessing( + _task->id(), dataResource, taskParam->operation(), _onTaskFinished); + break; + } + case (int)RA2018Command::RUN_PSI: + { + runPSI(taskState); + break; + } + default: + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::UnsupportedCommand, + "UnsupportedCommand " + std::to_string(taskParam->command()))); + break; + } + } + } + catch (bcos::Error const& e) + { + RA2018_LOG(WARNING) << LOG_DESC("asyncRunTask exception, cancel the task") + << printTaskInfo(_task) << LOG_KV("code", e.errorCode()) + << LOG_KV("msg", e.errorMessage()); + auto error = std::make_shared(e.errorCode(), e.errorMessage()); + onTaskError("asyncRunTask", std::move(error), taskState->peerID(), _task->id(), resourceID); + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) << LOG_DESC("asyncRunTask exception, cancel the task") + << printTaskInfo(_task) + << LOG_KV("msg", boost::diagnostic_information(e)); + auto error = std::make_shared(-1, boost::diagnostic_information(e)); + onTaskError("asyncRunTask", std::move(error), taskState->peerID(), _task->id(), resourceID); + } +} + +// run-psi +void RA2018PSIImpl::runPSI(TaskState::Ptr const& _taskState) +{ + auto task = _taskState->task(); + // notify the taskInfo to the front + m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + switch (task->selfParty()->partyIndex()) + { + case (int)ppc::protocol::PartyType::Client: + { + runClientPSI(_taskState); + break; + } + // the server do-nothing + case (int)ppc::protocol::PartyType::Server: + { + auto peerParty = checkAndSetPeerInfo(_taskState, false); + if (!peerParty) + { + return; + } + // check the output resource existence when syncResultToPeer + if (_taskState->task()->syncResultToPeer()) + { + _taskState->tryToGenerateDefaultOutputDesc(); + m_config->dataResourceLoader()->checkResourceExists( + task->selfParty()->dataResource()->outputDesc()); + } + RA2018_LOG(INFO) << LOG_DESC("runServerPSI") << LOG_KV("peer", peerParty->id()) + << printTaskInfo(task); + break; + } + default: + { + auto error = std::make_shared((int)PSIRetCode::UnsupportedPartyType, + "UnsupportedPartyType " + std::to_string(task->selfParty()->partyIndex()) + + ", task: " + task->id()); + cancelTask(std::move(error), task->id()); + break; + } + } +} + +void RA2018PSIImpl::asyncRunDataProcessing(std::string const& _taskID, + DataResource::ConstPtr const& _dataResource, int _operation, + TaskResponseCallback const& _callback) +{ + try + { + auto reader = loadData(m_config->dataResourceLoader(), _taskID, _dataResource); + // Note: consider the offline-full-evaluate task takes-long-time, we resonse the result when + // the task submit success + if (!m_waitResult) + { + auto taskResult = std::make_shared(_taskID); + _callback(std::move(taskResult)); + } + + RA2018_LOG(INFO) << LOG_DESC("asyncRunDataProcessing") << LOG_KV("task", _taskID) + << printDataResourceInfo(_dataResource); + // process offlineFullEvaluate + auto startT = utcSteadyTime(); + // Note: we use one worker to process offline-evaluate task in case of it occupies + // too-much cpu resource, and to release the rpc thread resource + auto self = weak_from_this(); + m_worker->enqueue([self, startT, _taskID, _dataResource, _operation, reader, + _callback]() mutable { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + psi->offlineFullEvaluate(_dataResource->resourceID(), _operation, std::move(reader), + [self, startT, _dataResource, _taskID, _callback](Error::Ptr&& _error) { + if (_error) + { + RA2018_LOG(WARNING) << LOG_DESC("asyncRunDataProcessing error") + << LOG_KV("task", _taskID) + << LOG_KV("data", _dataResource->resourceID()) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + } + auto psi = self.lock(); + if (!psi) + { + return; + } + // remove the dataResource from the processingDataResource pool, so that + // can accept the task related to the data-resource + psi->removeLockingResource(_dataResource->resourceID()); + // erase the task record + psi->removePendingTask(_taskID); + if (psi->m_waitResult) + { + auto taskResult = std::make_shared(_taskID); + _callback(std::move(taskResult)); + } + RA2018_LOG(INFO) + << LOG_DESC("asyncRunDataProcessing success") << LOG_KV("task", _taskID) + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << printDataResourceInfo(_dataResource); + }); + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) + << LOG_DESC("asyncRunDataProcessing exception") + << LOG_KV("error", boost::diagnostic_information(e)) << LOG_KV("task", _taskID) + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << printDataResourceInfo(_dataResource); + auto error = BCOS_ERROR_PTR(-1, "asyncRunDataProcessing exception: " + + std::string(boost::diagnostic_information(e))); + if (psi->m_waitResult) + { + auto taskResult = std::make_shared(_taskID); + taskResult->setError(error); + _callback(std::move(taskResult)); + } + psi->cancelTask(std::move(error), _taskID); + } + }); + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) << LOG_DESC("asyncRunDataProcessing exception") + << LOG_KV("error", boost::diagnostic_information(e)); + auto taskResult = std::make_shared(_taskID); + auto error = + BCOS_ERROR_PTR(-1, "asyncRunDataProcessing: " + boost::diagnostic_information(e)); + taskResult->setError(error); + _callback(std::move(taskResult)); + cancelTask(std::move(error), _taskID); + } +} + + +void RA2018PSIImpl::offlineFullEvaluate(std::string const& _resourceID, int _operation, + LineReader::Ptr _reader, std::function _callback) +{ + switch (_operation) + { + // offline data-insert + case DataPreProcessingOption::Insert: + { + // only obtain the cuckoo-filter not full + auto filterInfos = m_storage->getCuckooFilterInfos(_resourceID, true); + evaluateAndInsert(_resourceID, std::move(_reader), std::move(filterInfos), _callback); + break; + } + // offline data-delete + case DataPreProcessingOption::Delete: + { + // get all the filters specified by _resourceID + auto filterInfos = m_storage->getCuckooFilterInfos(_resourceID, false); + evaluateAndDelete(_resourceID, std::move(_reader), std::move(filterInfos), _callback); + break; + } + default: + { + _callback(BCOS_ERROR_PTR( + -1, "Unsupported data pre-processing operation " + std::to_string(_operation))); + break; + } + } +} + +// delete the full-evaluated-data from the cuckoo-filter +void RA2018PSIImpl::evaluateAndDelete(std::string const& _resourceID, LineReader::Ptr _reader, + CuckooFilterInfoSet&& _filterInfos, std::function _callback) +{ + auto sqlReader = (_reader->type() == ppc::protocol::DataResourceType::MySQL); + DataBatch::Ptr data; + if (sqlReader) + { + // load the first column data from the sql + data = _reader->next(0, DataSchema::Bytes); + } + else + { + // load all data from the file(Note: we assume that the amount of data deleted each time is + // not very large) + data = _reader->next(-1, DataSchema::Bytes); + } + + // the total filters expected to store + auto totalStoreCount = std::make_shared>(_filterInfos.size()); + auto callback = [totalStoreCount, _callback, + calledCount = std::make_shared>(0), + failed = std::make_shared>(0)](bcos::Error::Ptr _error) { + (*calledCount)++; + if (_error) + { + (*failed)++; + RA2018_LOG(WARNING) << LOG_DESC("evaluateAndDelete: update cuckoo-filter error") + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + } + // not all store operations have been finished + if (*calledCount != *totalStoreCount) + { + return; + } + // all the filters stored to backend successfully + if ((*failed) == 0) + { + _callback(nullptr); + return; + } + // with some-filters failed to store to the backend + _callback(BCOS_ERROR_PTR(-1, "evaluateAndDelete failed for update cuckoo-filter error")); + }; + + // evaluate the data + auto result = m_server->fullEvaluate(data); + // delete the result from all the filters + // here load and update cuckoo-filter by batch to decrease the memory overhead + int storeCount = 0; + + uint64_t i = 0; + for (auto const& filterInfo : _filterInfos) + { + auto const& filterID = filterInfo->filterID(); + auto filter = m_storage->loadCuckooFilter(_resourceID, filterID); + if (!filter) + { + i++; + continue; + } + auto orgSize = result.size(); + // delete some elment from the cuckoo-filter + filter->batchDeleteAndGetRemainKeys(result); + // update the modified filter + if (result.size() < orgSize) + { + storeCount++; + // Note: the data has been deleted or the last-cuckoo-filter + if (result.empty() || (i == (_filterInfos.size() - 1))) + { + // reset the totalStoreCount to the real-store-count + // Note: must ensure the real totalStoreCount setted before calls + // asyncUpdateCuckooFilter, otherwise the callback maybe never called + *totalStoreCount = storeCount; + } + auto updatedFilter = std::make_shared(filterID, filterInfo->hash()); + updatedFilter->setCuckooFilter(std::move(filter)); + m_storage->asyncUpdateCuckooFilter(_resourceID, updatedFilter, callback); + } + // all data has been deleted, break + if (result.empty()) + { + break; + } + i++; + } + RA2018_LOG(INFO) << LOG_DESC("evaluateAndDelete finish") << LOG_KV("resource", _resourceID) + << LOG_KV("filterSize", _filterInfos.size()) + << LOG_KV("storeCount", storeCount) << LOG_KV("failedCount", result.size()); +} + +// insert the full-evaluated-data into the cuckoo-filter +void RA2018PSIImpl::evaluateAndInsert(std::string const& _resourceID, LineReader::Ptr _reader, + CuckooFilterInfoSet&& _reusableFilters, std::function _callback) +{ + auto sqlReader = (_reader->type() == ppc::protocol::DataResourceType::MySQL); + // fileReader: load dataBatchSize() elements from the given file + // sqlReader: load the first column + auto nextParam = sqlReader ? 0 : m_config->cuckooFilterOption()->capacity; + // the allocator to create a new cuckoo-filter or load the reusable cuckoo-filter from the + // backend-db + auto allocator = std::make_shared( + m_config, m_storage, _resourceID, _callback, std::move(_reusableFilters)); + RA2018_LOG(INFO) << LOG_DESC("evaluateAndInsert") << LOG_KV("resourceID", _resourceID) + << LOG_KV("nextParam", nextParam); + // Note: should exit when stop + do + { + auto data = _reader->next(nextParam, DataSchema::Bytes); + // all data have been loaded + if (!data) + { + break; + } + // evaluate the data + auto result = m_server->fullEvaluate(data); + uint64_t offset = 0; + // TODO: optimize here without copy + std::set resultSet(result.begin(), result.end()); + // insert and store all the result into cuckoo-filter + do + { + // allocate cuckoo-filter to insert the result + auto filter = allocator->allocate(); + // all elements has been inserted into the cuckoo-filter + if (filter->batchInsert(resultSet)) + { + break; + } + } while (true && m_started); + } while (true && (!sqlReader) && m_started); // Note: the sql only has one-column, the file + // support load by batch + // flush all the memory cuckoo-filter into storage + allocator->flush(); +} + +// sync the cuckoo-filter from the server +// Note: this operation can be enqueue into the threadPool to increase perf +void RA2018PSIImpl::syncCuckooFilter( + std::string const& _peerID, std::string const& _taskID, DataResource::ConstPtr _dataResource) +{ + try + { + // load and init the cuckooFilterState + auto state = m_client->loadCuckooFilterState(_dataResource->resourceID(), true); + state->init(); + + auto filterInfos = m_storage->getCuckooFilterInfos(_dataResource->resourceID(), false); + RA2018_LOG(INFO) << LOG_DESC("syncCuckooFilter") << LOG_KV("task", _taskID) + << LOG_KV("peer", _peerID) << printDataResourceInfo(_dataResource) + << LOG_KV("filterInfos", filterInfos.size()); + requestCuckooFilters(_peerID, _taskID, _dataResource->resourceID(), filterInfos, + (uint32_t)RA2018PacketType::CuckooFilterRequest); + } + catch (std::exception const& e) + { + std::stringstream ss; + ss << LOG_DESC("runPSIClient: syncCuckooFilter exception") << LOG_KV("task", _taskID) + << LOG_KV("peer", _peerID) << printDataResourceInfo(_dataResource) + << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = ss.str(); + auto error = std::make_shared((int)PSIRetCode::syncCuckooFilterError, errorMsg); + RA2018_LOG(WARNING) << errorMsg; + // cancel the task + onTaskError( + "syncCuckooFilter", std::move(error), _peerID, _taskID, _dataResource->resourceID()); + } +} + +// handle the cuckoo-filter response +void RA2018PSIImpl::handleCuckooFilterResponse(PSIMessageInterface::Ptr _msg) +{ + auto ra2018FilterMsg = std::dynamic_pointer_cast(_msg); + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterResponse") << printPSIMessage(ra2018FilterMsg) + << LOG_KV("cuckooFilterSize", ra2018FilterMsg->cuckooFilterSize()); + // Note: if the filter has not been changed, the filter will be nullptr + auto const& filterInfo = ra2018FilterMsg->mutableFilterInfo(); + auto state = m_client->loadCuckooFilterState(ra2018FilterMsg->resourceID(), false); + if (state) + { + state->setCuckooFilterSize(ra2018FilterMsg->cuckooFilterSize()); + } + for (auto const& it : filterInfo) + { + handleCuckooFilterResponseItem(ra2018FilterMsg, state, it); + } + m_client->computeIntersection(); + // try to store the result + m_client->checkAndStoreInterSectionResult(); +} + +// TODO: remove the erased cuckoo-filter of the client +bool RA2018PSIImpl::handleCuckooFilterResponseItem(RA2018FilterMessage::Ptr _msg, + CuckooFilterState::Ptr _cuckooFilterState, CuckooFilterInfo::Ptr const& _cuckooFilterInfo) +{ + auto self = weak_from_this(); + auto onHandleFailed = [self, _msg, _cuckooFilterInfo](std::string const& _desc) { + auto psi = self.lock(); + if (!psi) + { + return; + } + std::stringstream ss; + ss << LOG_DESC("handleCuckooFilterResponse failed: ") << _desc + << LOG_KV("task", _msg->taskID()) << LOG_KV("peer", _msg->partyID()) + << LOG_KV("resource", _msg->resourceID()) << _cuckooFilterInfo->desc(); + // not found the hitted cuckoo-filter + // cancel the task + auto errorMsg = ss.str(); + RA2018_LOG(WARNING) << errorMsg; + auto error = BCOS_ERROR_PTR(PSIRetCode::LoadCuckooFilterDataError, std::move(errorMsg)); + + psi->cancelTask(std::move(error), _msg->taskID()); + }; + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterResponseItem: receive the cuckooFilter") + << printPSIMessage(_msg) << _cuckooFilterInfo->desc() + << LOG_KV("hit", _cuckooFilterInfo->cuckooFilter() ? false : true); + if (!_cuckooFilterState) + { + onHandleFailed("the task cuckoo-filter resources are not prepared"); + return false; + } + auto cuckooFilterInfo = _cuckooFilterInfo; + if (_cuckooFilterInfo->cuckooFilter()) + { + // filter changed, load into the cache + m_cuckooFiltersCache->insert(_msg->resourceID(), _cuckooFilterInfo); + } + else + { + // without change, load from local cuckoo-filter cache + cuckooFilterInfo = + m_cuckooFiltersCache->query(_msg->resourceID(), _cuckooFilterInfo->filterID()); + // request for the missed cuckoo-filter + // Note: if the server miss the requested cuckoo-filters(the msg type is + // MissingCuckooFilterResponse, no need to request again) + if (!cuckooFilterInfo) + { + if (_msg->packetType() != (uint32_t)RA2018PacketType::MissingCuckooFilterResponse) + { + RA2018_LOG(INFO) + << LOG_DESC( + "Query filter from local storage failed, try to request the " + "cuckoo filter from the server") + << LOG_KV("resourceID", _msg->resourceID()) + << LOG_KV("filterID", _cuckooFilterInfo->filterID()) << printPSIMessage(_msg); + std::set filters; + filters.insert(_cuckooFilterInfo); + // request the missed cuckoo-filter with given seq and cuckooFilterSize when + // local-cuckoo-filter miss or broken + requestCuckooFilters(_msg->from(), _msg->taskID(), _msg->resourceID(), filters, + (uint32_t)RA2018PacketType::MissingCuckooFilterRequest, + _msg->cuckooFilterSize(), _msg->seq()); + } + else + { + RA2018_LOG(INFO) << LOG_DESC( + "Query filter from server storage failed, set " + "load-cuckoo-filter finished") + << LOG_KV("resourceID", _msg->resourceID()) + << LOG_KV("filterID", _cuckooFilterInfo->filterID()) + << printPSIMessage(_msg); + _cuckooFilterState->decreaseCuckooFilterSize(1); + } + } + } + if (cuckooFilterInfo) + { + _cuckooFilterState->appendCuckooFilterInfo(_msg->seq(), std::move(cuckooFilterInfo)); + } + return true; +} + +// the client run-psi +void RA2018PSIImpl::runClientPSI(TaskState::Ptr const& _taskState) +{ + // Note: enforce the peer-data-resource + auto peerParty = checkAndSetPeerInfo(_taskState, true); + if (!peerParty) + { + return; + } + auto const& peerID = peerParty->id(); + auto const& task = _taskState->task(); + auto const& dataResource = task->selfParty()->dataResource(); + RA2018_LOG(INFO) << LOG_DESC("runClientPSI") << printTaskInfo(task) << LOG_KV("peer", peerID); + // check the dataResource + if (!checkDataResourceForSelf(_taskState, peerID, true)) + { + return; + } + if (!m_enableOutputExists) + { + // Note: if the output-resource already exists, will throw exception + m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc()); + } + else + { + m_config->dataResourceLoader()->loadWriter(dataResource->outputDesc(), true); + } + // load the data + auto reader = loadData(m_config->dataResourceLoader(), task->id(), dataResource); + auto sqlReader = (reader->type() == ppc::protocol::DataResourceType::MySQL); + // fileReader: load dataBatchSize() elements from the given file + // sqlReader: load the first column + auto nextParam = sqlReader ? 0 : m_config->dataBatchSize(); + _taskState->setReader(std::move(reader), nextParam); + + // sync the cuckooFilter + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, peerID, task, peerParty]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->syncCuckooFilter(peerID, task->id(), peerParty->dataResource()); + }); + // blind data + auto cuckooFilterResourceID = peerParty->dataResource()->resourceID(); + blindData(cuckooFilterResourceID, _taskState); + + if (!_taskState->sqlReader()) + { + // since one call to blindData may not handle all data of a task for cache limit + // add the blindData to taskState worker to execute looply + auto weakTaskState = std::weak_ptr(_taskState); + _taskState->setWorker([weakTaskState, cuckooFilterResourceID, self]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + auto state = weakTaskState.lock(); + if (!state) + { + return; + } + psi->blindData(cuckooFilterResourceID, state); + }); + } +} + +// load data from DataResource and blind the plainData into cipher +// Note: the thread-pool executed function should not throw exception +bool RA2018PSIImpl::blindData( + std::string const& _cuckooFilterResourceID, TaskState::Ptr const& _taskState) +{ + if (m_client->isFull()) + { + return false; + } + auto dataResource = _taskState->task()->selfParty()->dataResource(); + auto self = weak_from_this(); + auto sendCallback = [self, _taskState, dataResource](Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + RA2018_LOG(WARNING) << LOG_DESC("send evaluate request error") + << LOG_KV("peer", _taskState->peerID()) + << printTaskInfo(_taskState->task()) + << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + psi->cancelTask(std::move(_error), _taskState->task()->id()); + }; + // put the task into the thread-pool + m_config->threadPool()->enqueue([self, _cuckooFilterResourceID, _taskState, dataResource, + sendCallback]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + auto task = psi->getTaskByID(_taskState->task()->id()); + // the task has already been canceled + if (!task) + { + return; + } + if (_taskState->loadFinished()) + { + return; + } + do + { + int32_t seq = 0; + DataBatch::Ptr dataBatch = nullptr; + // Note: the setFinished/allocateSeq must be atomic, because they are used to + // judge whether the task is finished + { + bcos::Guard l(psi->m_mutex); + // Note: next is not thread-safe + dataBatch = + _taskState->reader()->next(_taskState->readerParam(), DataSchema::Bytes); + if (!dataBatch) + { + _taskState->setFinished(true); + break; + } + seq = _taskState->allocateSeq(); + if (_taskState->sqlReader()) + { + _taskState->setFinished(true); + } + } + auto dataBatchSize = dataBatch->size(); + auto ret = psi->m_client->insertCache(_cuckooFilterResourceID, + _taskState->task()->id(), _taskState, dataResource, seq, std::move(dataBatch)); + // blind error for insert the cache failed(the cache already exists) + if (!ret.first && ret.second != nullptr) + { + auto errorMsg = "The task " + _taskState->task()->id() + + " is in-processing! Cancel the repeated task!"; + auto error = BCOS_ERROR_PTR((int)PSIRetCode::TaskInProcessing, errorMsg); + psi->onTaskError("blindData", std::move(error), _taskState->peerID(), + _taskState->task()->id(), _cuckooFilterResourceID); + break; + } + auto cacheItem = ret.second; + RA2018_LOG(INFO) << LOG_DESC("blindData") << printTaskInfo(_taskState->task()) + << LOG_KV("seq", seq) << LOG_KV("dataSize", dataBatchSize) + << LOG_KV("cacheCapacity", psi->m_client->capacity()); + // the cache is full + if (!cacheItem) + { + break; + } + // trigger blind operation + auto blindedData = cacheItem->blind(); + // send the evaluate request + auto msg = psi->m_config->ra2018MsgFactory()->createRA2018FilterMessage( + (uint32_t)RA2018PacketType::EvaluateRequest); + RA2018_LOG(INFO) << LOG_DESC("EvaluateRequest") << printTaskInfo(_taskState->task()) + << LOG_KV("seq", seq) << LOG_KV("dataSize", blindedData.size()); + msg->setResourceID(dataResource->resourceID()); + msg->setData(std::move(blindedData)); + psi->m_config->generateAndSendPPCMessage( + _taskState->peerID(), _taskState->task()->id(), msg, sendCallback, seq); + } while (!_taskState->sqlReader() && psi->m_started); + } + catch (std::exception const& e) + { + std::stringstream ss; + ss << LOG_DESC("blindData exception") << printTaskInfo(_taskState->task()) + << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = ss.str(); + auto error = std::make_shared((int)PSIRetCode::BlindDataError, errorMsg); + RA2018_LOG(WARNING) << errorMsg; + // cancel the task + psi->onTaskError("blindData", std::move(error), _taskState->peerID(), + _taskState->task()->id(), + _taskState->task()->selfParty()->dataResource()->resourceID()); + } + }); + return true; +} + +// handle the evaluate data responsed from the server, try to finalize and intersection +void RA2018PSIImpl::handleEvaluateResponse(PSIMessageInterface::Ptr _msg) +{ + auto evaluatedData = _msg->takeData(); + RA2018_LOG(INFO) << LOG_DESC("handleEvaluateResponse") << printPSIMessage(_msg) + << LOG_KV("evaluatedSize", evaluatedData.size()); + auto clientCache = m_client->cache(_msg->taskID(), _msg->seq()); + // the task has been canceled + if (!clientCache) + { + return; + } + // finliaize the evaluated-data + clientCache->finalize(evaluatedData); + // try to intersection + clientCache->computeIntersection(); + // try to store the result + m_client->checkAndStoreInterSectionResult(); +} + +// the cuckoo-filter of the client has been missed or broken +void RA2018PSIImpl::handleMissedCuckooFilterRequest(PSIMessageInterface::Ptr _msg) +{ + RA2018_LOG(INFO) << LOG_DESC("handleMissedCuckooFilterRequest") << printPSIMessage(_msg); + auto cuckooFilterRequestMsg = std::dynamic_pointer_cast(_msg); + auto const& filterInfos = cuckooFilterRequestMsg->filterInfo(); + for (auto const& it : filterInfos) + { + auto filterData = m_storage->loadCuckooFilterData(_msg->resourceID(), it->filterID()); + RA2018_LOG(INFO) + << LOG_DESC("handleMissedCuckooFilterRequest: load cuckoo-filter from the storage") + << LOG_KV("resource", _msg->resourceID()) << it->desc() + << LOG_KV("size", filterData.size()); + it->setCuckooFilterData(std::move(filterData)); + responseCuckooFilter((uint32_t)(RA2018PacketType::MissingCuckooFilterResponse), _msg, it, + cuckooFilterRequestMsg->cuckooFilterSize(), _msg->seq()); + } +} + +// receive cuckoo-filter request, response the delta-cuckoo-filter +void RA2018PSIImpl::handleCuckooFilterRequest(PSIMessageInterface::Ptr _msg) +{ + auto startT = utcSteadyTime(); + auto cuckooFilterRequestMsg = std::dynamic_pointer_cast(_msg); + auto& filterInfo = cuckooFilterRequestMsg->mutableFilterInfo(); + // load the local cuckooFilterInfo + auto localCuckooFilterInfo = m_storage->getCuckooFilterInfos(_msg->resourceID(), false); + // server miss the cukoo-filter + if (localCuckooFilterInfo.empty()) + { + auto errorMessage = + "the server's data has not been processed offline, the intersection cannot be " + "performed at present! Please trigger the offline-data-process firstly!"; + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterRequest:") << errorMessage + << printPSIMessage(_msg) << LOG_KV("clientFilterSize", filterInfo.size()) + << LOG_KV("localCuckooFilter", localCuckooFilterInfo.size()); + auto error = + std::make_shared((int)PSIRetCode::NotOfflineFullEvaluated, errorMessage); + // cancel the task and notify the peer + onTaskError("handleCuckooFilterRequest", std::move(error), _msg->from(), _msg->taskID(), + _msg->resourceID()); + return; + } + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterRequest") << printPSIMessage(_msg) + << LOG_KV("clientFilterSize", filterInfo.size()) + << LOG_KV("localCuckooFilter", localCuckooFilterInfo.size()); + std::vector responseCuckooFilters; + for (auto const& filter : filterInfo) + { + auto it = localCuckooFilterInfo.find(filter); + // the filter is not change + if (it != localCuckooFilterInfo.end()) + { + // put the entry into the repsonse-filter-list with empty cuckoo-filter, which + // represented that the filter is not changed + responseCuckooFilters.emplace_back(*it); + // erase the hitted entry from the local + localCuckooFilterInfo.erase(it); + } + } + RA2018_LOG(INFO) << LOG_DESC( + "handleCuckooFilterRequest: load the missed filter from the storage") + << printPSIMessage(_msg) + << LOG_KV("hittedFilters", responseCuckooFilters.size()) + << LOG_KV("missedFilters", localCuckooFilterInfo.size()); + for (auto const& it : localCuckooFilterInfo) + { + auto filterData = m_storage->loadCuckooFilterData(_msg->resourceID(), it->filterID()); + if (filterData.empty()) + { + continue; + } + RA2018_LOG(INFO) << LOG_DESC( + "handleCuckooFilterRequest: load cuckoo-filter from the storage") + << LOG_KV("resource", _msg->resourceID()) << it->desc() + << LOG_KV("size", filterData.size()); + it->setCuckooFilterData(std::move(filterData)); + // the missed entry with cuckoo-filter-data + responseCuckooFilters.emplace_back(it); + } + // response to client + // Note: for the network-msg-size limit, we send the one-cuckoo-filter per-network-msg + size_t seq = 1; + for (uint64_t i = 0; i < responseCuckooFilters.size(); i++) + { + responseCuckooFilter((uint32_t)RA2018PacketType::CuckooFilterResponse, _msg, + responseCuckooFilters.at(i), responseCuckooFilters.size(), seq); + seq++; + } + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterRequest success") + << LOG_KV("responsedCuckooFilters", responseCuckooFilters.size()) + << LOG_KV("timecost", (utcSteadyTime() - startT)) << printPSIMessage(_msg); +} + +void RA2018PSIImpl::responseCuckooFilter(uint32_t _responseType, + PSIMessageInterface::Ptr const& _msg, CuckooFilterInfo::Ptr const& _filterInfo, + uint64_t _filterInfoSize, size_t _seq) +{ + auto cuckooFilterResponseMsg = + m_config->ra2018MsgFactory()->createRA2018FilterMessage(_responseType); + cuckooFilterResponseMsg->appendFilterInfo(_filterInfo); + cuckooFilterResponseMsg->setResourceID(_msg->resourceID()); + cuckooFilterResponseMsg->setPartyID(m_config->selfParty()); + cuckooFilterResponseMsg->setCuckooFilterSize(_filterInfoSize); + RA2018_LOG(INFO) << LOG_DESC("handleCuckooFilterRequest: response cuckoo-filter") + << printPSIMessage(_msg) << LOG_KV("seq", _seq) << _filterInfo->desc(); + auto self = weak_from_this(); + m_config->generateAndSendPPCMessage( + _msg->from(), _msg->taskID(), cuckooFilterResponseMsg, + [self, _msg](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + // TODO: retry or cancel task + psi->onTaskError("handleCuckooFilterRequest: response cuckoo-filters", + std::move(_error), _msg->from(), _msg->taskID(), _msg->resourceID()); + }, + _seq); +} + +void RA2018PSIImpl::handleEvaluateRequest(PSIMessageInterface::Ptr _msg) +{ + auto startT = utcSteadyTime(); + auto&& blindedData = _msg->takeData(); + RA2018_LOG(INFO) << LOG_DESC("handleEvaluateRequest") << printPSIMessage(_msg) + << LOG_KV("blindDataItem", blindedData.size()); + // evaluate the blindedData + auto evaluatedData = m_server->evaluate(blindedData); + // generate ra2018-message, response the evaluated-data to the client + auto response = m_config->ra2018MsgFactory()->createPSIMessage( + (uint32_t)RA2018PacketType::EvaluateResponse); + response->setData(std::move(evaluatedData)); + response->setResourceID(_msg->resourceID()); + response->setPartyID(m_config->selfParty()); + auto self = weak_from_this(); + // Note: must specified the seq here + m_config->generateAndSendPPCMessage( + _msg->from(), _msg->taskID(), response, + [self, _msg](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->onTaskError("handleEvaluateRequest: response evaluated-data", std::move(_error), + _msg->from(), _msg->taskID(), _msg->resourceID()); + }, + _msg->seq()); + RA2018_LOG(INFO) << LOG_DESC("handleEvaluateRequest success") << printPSIMessage(_msg) + << LOG_KV("timecost", (utcSteadyTime() - startT)); +} + +void RA2018PSIImpl::handlePSIMsg(PSIMessageInterface::Ptr _msg) +{ + if (m_disabled) + { + handlePSIFrameworkMsg(_msg); + return; + } + // check the task status + if (!checkPSIMsg(_msg)) + { + return; + } + // prioritize lightweight messages + if (handlePSIFrameworkMsg(_msg)) + { + return; + } + auto self = weak_from_this(); + m_config->threadPool()->enqueue([self, _msg]() { + auto psi = self.lock(); + if (!psi) + { + return; + } + try + { + switch (_msg->packetType()) + { + case (uint32_t)RA2018PacketType::CuckooFilterRequest: + { + psi->handleCuckooFilterRequest(_msg); + break; + } + case (uint32_t)RA2018PacketType::MissingCuckooFilterRequest: + { + psi->handleMissedCuckooFilterRequest(_msg); + break; + } + case (uint32_t)RA2018PacketType::MissingCuckooFilterResponse: + case (uint32_t)RA2018PacketType::CuckooFilterResponse: + { + psi->handleCuckooFilterResponse(_msg); + break; + } + case (uint32_t)RA2018PacketType::EvaluateRequest: + { + psi->handleEvaluateRequest(_msg); + break; + } + case (uint32_t)RA2018PacketType::EvaluateResponse: + { + psi->handleEvaluateResponse(_msg); + break; + } + default: + { + RA2018_LOG(WARNING) + << LOG_DESC("Unsupported packetType ") << (int)_msg->packetType(); + break; + } + } + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) << LOG_DESC("handlePSIMsg exception") + << LOG_KV("packetType", _msg->packetType()) << printPSIMessage(_msg) + << LOG_KV("error", boost::diagnostic_information(e)); + auto error = BCOS_ERROR_PTR((int)PSIRetCode::HandleTaskError, + "Task " + _msg->taskID() + " error for " + boost::diagnostic_information(e)); + psi->onTaskError("handlePSIMsg", std::move(error), _msg->from(), _msg->taskID(), + "emptyDataResource"); + } + }); +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h new file mode 100644 index 00000000..d8d376b7 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIImpl.h + * @author: yujiechen + * @date 2022-10-25 + */ +// +// server client +// Offline +// FullEvaluate +// CF.insert(evaluated items) +// send CF +// -----------------------> +// ====================================================== +// Online +// Blind +// blinded items(batch_size) +// <---------------------- +// Evaluate +// evaluated items +// -----------------------> +// Finalize +// ======================================================= +// Intersection +#pragma once +#include "../psi-framework/PSIFramework.h" +#include "../psi-framework/TaskState.h" +#include "RA2018PSIConfig.h" +#include "core/CuckooFiltersCache.h" +#include "core/RA2018PSIClient.h" +#include "core/RA2018PSIServer.h" +#include "ppc-framework/protocol/Task.h" +#include "storage/RA2018PSIStorage.h" +#include +#include + +namespace ppc::psi +{ +class RA2018PSIImpl : public PSIFramework, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIImpl(RA2018PSIConfig::Ptr const& _config, RA2018PSIStorage::Ptr const& _storage, + unsigned _idleTimeMs = 0, bool _waitResult = false, bool _disabled = false); + ~RA2018PSIImpl() override = default; + + // run task + void asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + + void start() override; + void stop() override; + +protected: + // run data-processing task to generate and store cuckoo-filter + void asyncRunDataProcessing(std::string const& _taskID, + ppc::protocol::DataResource::ConstPtr const& _dataResource, int _operation, + TaskResponseCallback const& _callback); + // run psi task to generate data-intersection + virtual void runPSI(TaskState::Ptr const& _taskState); + virtual void runClientPSI(TaskState::Ptr const& _taskState); + + virtual void offlineFullEvaluate(std::string const& _resourceID, int _operation, + ppc::io::LineReader::Ptr _reader, std::function _callback); + + virtual void evaluateAndInsert(std::string const& _resourceID, ppc::io::LineReader::Ptr _reader, + CuckooFilterInfoSet&& _reusableFilters, std::function _callback); + virtual void evaluateAndDelete(std::string const& _resourceID, ppc::io::LineReader::Ptr _reader, + CuckooFilterInfoSet&& _filterInfos, std::function _callback); + + /////// the function to handle all ra2018 mesage + void handlePSIMsg(PSIMessageInterface::Ptr _msg) override; + + /////// online PSI: client + // sync the cuckoo-filter from the server + virtual void syncCuckooFilter(std::string const& _peerID, std::string const& _taskID, + ppc::protocol::DataResource::ConstPtr _dataResource); + // handle the cuckoo-filter response + virtual void handleCuckooFilterResponse(PSIMessageInterface::Ptr _msg); + virtual bool handleCuckooFilterResponseItem(RA2018FilterMessage::Ptr _msg, + CuckooFilterState::Ptr _cuckooFilterState, CuckooFilterInfo::Ptr const& _cuckooFilterInfo); + // load data from DataResource and blind the plainData into cipher + virtual bool blindData( + std::string const& _cuckooFilterResourceID, TaskState::Ptr const& _taskState); + // handle the evaluate data responsed from the server, try to finalize and intersection + virtual void handleEvaluateResponse(PSIMessageInterface::Ptr _msg); + + + /////// online PSI: Server + // receive cuckoo-filter request, response the delta-cuckoo-filter + virtual void handleCuckooFilterRequest(PSIMessageInterface::Ptr _msg); + virtual void handleMissedCuckooFilterRequest(PSIMessageInterface::Ptr _msg); + // receive the evaluate request, evaluate and response + virtual void handleEvaluateRequest(PSIMessageInterface::Ptr _msg); + + // TODO: check this logic + bool needLockResource(int _command, int _partyType) override + { + // Note: the psi server no need to lock the resource and record the pendingTasks + if (_command == RA2018Command::RUN_PSI && + _partyType == (int)ppc::protocol::PartyType::Server) + { + return false; + } + return true; + } + + void responseCuckooFilter(uint32_t _responseType, PSIMessageInterface::Ptr const& _msg, + CuckooFilterInfo::Ptr const& _filterInfo, uint64_t _filterInfoSize, size_t _seq); + + template + void requestCuckooFilters(std::string const& _peerID, std::string const& _taskID, + std::string const& _resourceID, T const& _filterInfos, uint32_t _type, + int64_t _cuckooFilterSize = 0, uint32_t _seq = 0) + { + auto ra2018Msg = m_config->ra2018MsgFactory()->createRA2018FilterMessage(_type); + std::vector filterInfoList(_filterInfos.begin(), _filterInfos.end()); + ra2018Msg->setPartyID(m_config->selfParty()); + ra2018Msg->setFilterInfo(std::move(filterInfoList)); + ra2018Msg->setResourceID(_resourceID); + ra2018Msg->setCuckooFilterSize(_cuckooFilterSize); + auto self = weak_from_this(); + m_config->generateAndSendPPCMessage( + _peerID, _taskID, ra2018Msg, + [self, _peerID, _taskID, _resourceID](bcos::Error::Ptr&& _error) { + if (!_error || _error->errorCode() == 0) + { + return; + } + auto psi = self.lock(); + if (!psi) + { + return; + } + psi->onTaskError("requestCuckooFilters: send cuckoo-filter request", + std::move(_error), _peerID, _taskID, _resourceID); + }, + _seq); + } + + // receive the handshake response + void onHandshakeResponse(PSIMessageInterface::Ptr const& _msg) override + { + throw std::runtime_error("RA2018PSIImpl: unimplemented onHandshakeResponse!"); + } + // receive the handshake request + void onHandshakeRequest(PSIMessageInterface::Ptr const& _msg) override + { + throw std::runtime_error("RA2018PSIImpl: unimplemented onHandshakeRequest!"); + } + +protected: + bool m_disabled = false; + RA2018PSIConfig::Ptr m_config; + RA2018PSIStorage::Ptr m_storage; + RA2018PSIServer::Ptr m_server; + RA2018PSIClient::Ptr m_client; + // the cuckoo-filter cache + // cache for psi-client + CuckooFiltersCache::Ptr m_cuckooFiltersCache; + + // the handler triggered when the RA2018PSIClient capacity decreased or some tasks execute + // success + bcos::Handler<> m_notifier; + + bcos::ThreadPool::Ptr m_worker; + // the flag means that response to the sdk once handling the task or after the task completed + bool m_waitResult; + + // allow the output-path exists, for ut + bool m_enableOutputExists = false; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h new file mode 100644 index 00000000..ceea025b --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckooFilterInfo.h + * @author: yujiechen + * @date 2022-11-10 + */ +#pragma once +#include "../Common.h" +#include +#include + +namespace ppc::psi +{ +class CuckooFilterInfo +{ +public: + using Ptr = std::shared_ptr; + CuckooFilterInfo(int32_t _filterID) : m_filterID(_filterID) {} + CuckooFilterInfo(int32_t _filterID, bcos::bytes const& _hash) + : m_filterID(_filterID), m_hash(_hash) + {} + + CuckooFilterInfo( + int32_t _filterID, bcos::bytes const& _hash, DefaultCukooFilterPtr const& _cuckooFilter) + : CuckooFilterInfo(_filterID, _hash) + { + m_cuckooFilter = _cuckooFilter; + } + + int32_t filterID() const { return m_filterID; } + bcos::bytes const& hash() const { return m_hash; } + + DefaultCukooFilterPtr cuckooFilter() const { return m_cuckooFilter; } + + bcos::bytes const& cuckooFilterData() const { return m_cuckooFilterData; } + + bool dirty() const { return m_dirty; } + + void setDirty(bool _dirty) { m_dirty = _dirty; } + void setCuckooFilter(DefaultCukooFilterPtr&& _cuckooFilter) + { + m_cuckooFilter = std::move(_cuckooFilter); + } + void setCuckooFilterData(bcos::bytes&& _data) { m_cuckooFilterData = std::move(_data); } + void setFilterID(int32_t _filterID) { m_filterID = _filterID; } + + void setHash(bcos::bytes const& _hash) { m_hash = _hash; } + void setHash(bcos::bytes&& _hash) { m_hash = std::move(_hash); } + + inline std::string desc() const + { + std::stringstream oss; + oss << LOG_KV("id", m_filterID) << LOG_KV("hash", bcos::toHex(m_hash)); + return oss.str(); + } + +private: + int32_t m_filterID; + bcos::bytes m_hash; + // Note: in some cases, no need to deserialize the cuckoo-filter + DefaultCukooFilterPtr m_cuckooFilter; + bcos::bytes m_cuckooFilterData; + + bool m_dirty = false; + + bcos::bytes m_emptyCuckooFilterData; +}; + +struct CuckooFilterInfoComparator +{ + bool operator()(const CuckooFilterInfo::Ptr& _left, const CuckooFilterInfo::Ptr& _right) const + { + if (_left->filterID() == _right->filterID()) + { + return _left->hash() < _right->hash(); + } + return (_left->filterID() < _right->filterID()); + } +}; +using CuckooFilterInfoSet = std::set; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h new file mode 100644 index 00000000..d2632e11 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckooFilterState.h + * @author: yujiechen + * @date 2022-11-8 + */ +#pragma once +#include "../Common.h" +#include + +namespace ppc::psi +{ +class CuckooFilterState +{ +public: + using Ptr = std::shared_ptr; + CuckooFilterState(std::string const& _resourceID) : m_resourceID(_resourceID) {} + virtual ~CuckooFilterState() = default; + + std::string const& resourceID() const { return m_resourceID; } + // Note: the state will only been accessed by one task with _resourceID + std::map const& cuckooFilterInfos() const + { + bcos::ReadGuard l(x_cuckooFilterInfos); + return m_cuckooFilterInfos; + } + // the resource fetched all cuckoo-filters or not + bool fetchFinish() const + { + if (m_cuckooFilterSize.load() == -1) + { + return false; + } + bcos::ReadGuard l(x_receivedCuckooFilter); + if ((int32_t)m_receivedCuckooFilter.size() == m_cuckooFilterSize.load()) + { + return true; + } + return false; + } + + void decreaseCuckooFilterSize(int32_t _decreasedCuckooFilterSize) + { + m_cuckooFilterSize -= _decreasedCuckooFilterSize; + } + + void init() + { + bcos::WriteGuard lock(x_receivedCuckooFilter); + m_receivedCuckooFilter.clear(); + m_cuckooFilterSize.store(-1); + } + + void appendCuckooFilterInfo(uint32_t _seq, CuckooFilterInfo::Ptr&& _cuckooFilterInfo) + { + { + bcos::WriteGuard lock(x_receivedCuckooFilter); + m_receivedCuckooFilter.insert(_seq); + RA2018_LOG(INFO) << LOG_DESC("appendCuckooFilterInfo") << LOG_KV("seq", _seq) + << LOG_KV("receivedSize", m_receivedCuckooFilter.size()) + << LOG_KV("expectedSize", m_cuckooFilterSize); + } + auto const& filterID = _cuckooFilterInfo->filterID(); + // Note: not update the old-cuckoo-filter when hit + bcos::UpgradableGuard l(x_cuckooFilterInfos); + if (m_cuckooFilterInfos.count(filterID) && + m_cuckooFilterInfos.at(filterID)->hash() == _cuckooFilterInfo->hash()) + { + return; + } + bcos::UpgradeGuard ul(l); + m_cuckooFilterInfos[filterID] = _cuckooFilterInfo; + } + + void setCuckooFilterSize(int32_t _cuckooFilterSize) + { + m_cuckooFilterSize.store(_cuckooFilterSize); + } + +private: + std::string m_resourceID; + // the total-cuckoo-filters of the given resourceID + std::atomic m_cuckooFilterSize = {0}; + + // record the loaded cuckoo-filters, update when the filterID hitted + std::map m_cuckooFilterInfos; + mutable bcos::SharedMutex x_cuckooFilterInfos; + + // record the fetched cuckoo-filter + std::set m_receivedCuckooFilter; + mutable bcos::SharedMutex x_receivedCuckooFilter; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp new file mode 100644 index 00000000..8bed4c14 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp @@ -0,0 +1,223 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckooFiltersCache.cpp + * @author: yujiechen + * @date 2022-11-10 + */ +#include "CuckooFiltersCache.h" + +using namespace ppc::psi; +using namespace bcos; + +bool CuckooFiltersCache::insert( + std::string const& _resourceID, CuckooFilterInfo::Ptr const& _filter) +{ + insertWithoutEvict(_resourceID, std::move(_filter)); + return evict(); +} + +CuckooFilterInfo::Ptr CuckooFiltersCache::query(std::string const& _resourceID, int32_t _filterID) +{ + auto filterInfo = queryWithoutEvict(_resourceID, _filterID); + evict(); + return filterInfo; +} + +// insert a cuckoo-filter into the cache, flush the last-element from m_resourceQueue if cache-full +void CuckooFiltersCache::insertWithoutEvict( + std::string const& _resourceID, CuckooFilterInfo::Ptr const& _filter) +{ + UpgradableGuard l(x_mutex); + auto const& filterID = _filter->filterID(); + auto it = m_resourceToCuckooFilters.find(_resourceID); + if (it != m_resourceToCuckooFilters.end()) + { + auto pCuckooFilter = it->second.find(filterID); + if (pCuckooFilter != it->second.end()) + { + auto currentFilter = pCuckooFilter->second; + if (_filter->hash() == currentFilter->hash()) + { + RA2018_LOG(INFO) + << LOG_DESC("CuckooFiltersCache: insert return directly for hit the cache") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", filterID) + << LOG_KV("capacity", m_capacity); + return; + } + // update the currentFilter if its-hash changed + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache: update the cuckoo-filter") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", filterID) + << LOG_KV("capacity", m_capacity); + _filter->setDirty(true); + // update the capacity + m_capacity -= currentFilter->cuckooFilter()->capacity(); + m_capacity += _filter->cuckooFilter()->capacity(); + UpgradeGuard ul(l); + pCuckooFilter->second = _filter; + flush(_resourceID, filterID, _filter); + return; + } + } + UpgradeGuard ul(l); + // insert new cuckoo-filter + m_resourceToCuckooFilters[_resourceID][filterID] = _filter; + m_resourceQueue.push(std::make_pair(_resourceID, filterID)); + _filter->setDirty(true); + flush(_resourceID, filterID, _filter); + m_capacity += _filter->cuckooFilter()->capacity(); + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache: insert new cuckoo-filter") + << LOG_KV("resource", _resourceID) << LOG_KV("capacity", m_capacity); +} + +// query cuckoo-filter specified by hash, access the storage and update the +// m_resourceToCuckooFilters when miss +CuckooFilterInfo::Ptr CuckooFiltersCache::queryWithoutEvict( + std::string const& _resourceID, int32_t _filterID) +{ + { + ReadGuard l(x_mutex); + auto it = m_resourceToCuckooFilters.find(_resourceID); + if (it != m_resourceToCuckooFilters.end()) + { + auto pCuckooFilter = it->second.find(_filterID); + // hit the cache + if (pCuckooFilter != it->second.end()) + { + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache: query hit the cache") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", _filterID) + << LOG_KV("capacity", m_capacity); + return pCuckooFilter->second; + } + } + } + return loadCuckooFilterFromStorage(_resourceID, _filterID); +} + +CuckooFilterInfo::Ptr CuckooFiltersCache::loadCuckooFilterFromStorage( + std::string const& _resourceID, int32_t _filterID) +{ + try + { + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache: loadCuckooFilterFromStorage") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", _filterID) + << LOG_KV("capacity", m_capacity); + // miss the cache load the cuckoo-filter from the backend + // Note: since the load operation may occupy more time, we remove the lock here + auto filterInfo = m_storage->loadCuckooFilterDataInfo(_resourceID, _filterID, true); + // the filter non-exist + if (!filterInfo) + { + return nullptr; + } + UpgradableGuard l(x_mutex); + // check the existance again, in case of the filter been inserted + if (!m_resourceToCuckooFilters.count(_resourceID) || + !m_resourceToCuckooFilters.at(_resourceID).count(_filterID)) + { + UpgradeGuard ul(l); + filterInfo->setFilterID(_filterID); + m_resourceToCuckooFilters[_resourceID][_filterID] = filterInfo; + m_resourceQueue.push(std::make_pair(_resourceID, _filterID)); + m_capacity += filterInfo->cuckooFilter()->capacity(); + return filterInfo; + } + return m_resourceToCuckooFilters.at(_resourceID).at(_filterID); + } + catch (std::exception const& e) + { + // load cuckoo-filter failed + RA2018_LOG(WARNING) << LOG_DESC("loadCuckooFilterFromStorage exception") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", _filterID) + << LOG_KV("error", boost::diagnostic_information(e)); + return nullptr; + } +} + +bool CuckooFiltersCache::evict() +{ + // the cache is not full + if (m_capacity <= m_maxCapacity) + { + return false; + } + // the cache is full + bcos::WriteGuard l(x_mutex); + while (m_capacity > m_maxCapacity) + { + // find out the element to evict + auto element = m_resourceQueue.back(); + m_resourceQueue.pop(); + auto it = m_resourceToCuckooFilters.find(element.first); + if (it == m_resourceToCuckooFilters.end()) + { + continue; + } + auto pFilter = it->second.find(element.second); + if (pFilter == it->second.end()) + { + continue; + } + auto evictedCuckooFilter = pFilter->second; + it->second.erase(pFilter); + m_capacity -= evictedCuckooFilter->cuckooFilter()->capacity(); + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache evict") + << LOG_KV("resource", element.first) << LOG_KV("filter", element.second) + << LOG_KV("evictedSize", evictedCuckooFilter->cuckooFilter()->capacity()) + << LOG_KV("capacity", m_capacity); + flush(element.first, element.second, evictedCuckooFilter); + } + return true; +} + +void CuckooFiltersCache::flushAll() +{ + ReadGuard l(x_mutex); + for (auto const& it : m_resourceToCuckooFilters) + { + auto const& resourceID = it.first; + for (auto const& cuckooItem : it.second) + { + flush(resourceID, cuckooItem.first, cuckooItem.second); + } + } +} + +void CuckooFiltersCache::flush( + std::string const& _resourceID, int32_t _filterID, CuckooFilterInfo::Ptr _filter) +{ + if (!_filter->dirty()) + { + return; + } + // insert + RA2018_LOG(INFO) << LOG_DESC("CuckooFiltersCache flush: insert new cuckoo-filter") + << LOG_KV("resource", _resourceID) << _filter->desc(); + // Note: since every cuckoo-filter-fetch-process will check the cuckoo-filter between + // server and client, storage failure has no effect + m_storage->asyncInsertCuckooFilter( + _resourceID, _filter->cuckooFilter(), + [_resourceID](Error::Ptr&& _error) { + if (!_error) + { + return; + } + RA2018_LOG(WARNING) + << LOG_DESC("CuckooFiltersCache evict: insert cuckoo-filter to storage failed") + << LOG_KV("resource", _resourceID) << LOG_KV("code", _error->errorCode()) + << LOG_KV("msg", _error->errorMessage()); + }, + _filterID); +} diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h new file mode 100644 index 00000000..359801e7 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckooFiltersCache.h + * @author: yujiechen + * @date 2022-11-10 + */ +#pragma once +#include "../storage/RA2018PSIStorage.h" +#include "CuckooFilterInfo.h" +#include +#include +namespace ppc::psi +{ +// Note: the cuckoo-filter is useful for the client +// we access the storage directly when the save insert/update/load cuckoo-filter +class CuckooFiltersCache +{ +public: + using Ptr = std::shared_ptr; + // default cache size is 255MB + CuckooFiltersCache( + RA2018PSIStorage::Ptr const& _storage, uint64_t _maxCapacity = 255 * 1024 * 1024) + : m_storage(_storage), m_maxCapacity(_maxCapacity) + {} + virtual ~CuckooFiltersCache() {} + + // insert a cuckoo-filter into the cache, flush the last-element from m_resourceQueue if + // cache-full, return the cache is full or not + virtual bool insert(std::string const& _resourceID, CuckooFilterInfo::Ptr const& _filter); + + // query cuckoo-filter specified by hash, access the storage and update the + // m_resourceToCuckooFilters when miss + virtual CuckooFilterInfo::Ptr query(std::string const& _resourceID, int32_t _filterID); + + // flush the updated/dirty cuckoo-filter into the db + // we can call this periodly + virtual void flushAll(); + + uint64_t capacity() const { return m_capacity; } + bool full() const { return m_capacity >= m_maxCapacity; } + +protected: + // Note: only flush the dirty-cuckoo-filter to the backend when evict + virtual bool evict(); + // insert the cuckoo-filter into the db + virtual void insertWithoutEvict( + std::string const& _resourceID, CuckooFilterInfo::Ptr const& _filter); + + // query the cuckoo-filter according to the resourceID and filterID + virtual CuckooFilterInfo::Ptr queryWithoutEvict( + std::string const& _resourceID, int32_t _filterID); + + // load the cuckoo-filter from the backend-storage + virtual CuckooFilterInfo::Ptr loadCuckooFilterFromStorage( + std::string const& _resourceID, int32_t _filterID); + + // flush the dirty/inserted cuckoo-filter to the storage + virtual void flush( + std::string const& _resourceID, int32_t _filterID, CuckooFilterInfo::Ptr _filter); + +private: + RA2018PSIStorage::Ptr m_storage; + uint64_t m_maxCapacity; + std::atomic m_capacity = {0}; + + // for FIFO cache-policy + // (resourceID, filter ID) + std::queue> m_resourceQueue; + // the resourceID to cuckoo-filters + // resourceID => filter id => cuckoo-filter + std::map> m_resourceToCuckooFilters; + mutable bcos::SharedMutex x_mutex; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp b/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp new file mode 100644 index 00000000..a4e072fb --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckoofilterAllocator.cpp + * @author: yujiechen + * @date 2022-11-7 + */ +#include "CuckoofilterAllocator.h" + +using namespace ppc::psi; + +// Note: all allocate call will trigger allocate the used-up cuckoo-filter +DefaultCukooFilterPtr CuckoofilterAllocator::allocate() +{ + // with cuckoo-filter and the cuckoo-filter underload + if (m_currentCuckooFilter) + { + if (!m_currentCuckooFilter->cuckooFilter()->full()) + { + return m_currentCuckooFilter->cuckooFilter(); + } + else + { + // if the cuckoo-filter overload, try to flush to backend + saveOverLoadedCuckooFilter(false); + } + } + // load the reusable-cuckoo-filter from the backend + while (!m_currentCuckooFilter && !m_reusableFilters.empty()) + { + auto const& filterID = m_reusableFilters.back()->filterID(); + try + { + // Note: the cuckoo-filter files maybe deleted which will cause exception + m_currentCuckooFilter = + m_storage->loadCuckooFilterDataInfo(m_resourceID, filterID, true); + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) + << LOG_DESC("CuckoofilterAllocator: loadCuckooFilterInfo error when allocate") + << LOG_KV("error", boost::diagnostic_information(e)) + << LOG_KV("resource", m_resourceID) << LOG_KV("filter", filterID); + } + m_reusableFilters.pop_back(); + } + + // allocate a new cuckoo-filter + if (m_reusableFilters.empty() && !m_currentCuckooFilter) + { + auto allocatedCuckooFilter = + std::make_shared(m_config->cuckooFilterOption()); + // the id of the new cuckoo-filter is -1 + m_currentCuckooFilter = std::make_shared(-1); + m_currentCuckooFilter->setCuckooFilter(std::move(allocatedCuckooFilter)); + RA2018_LOG(INFO) << LOG_DESC("CuckoofilterAllocator: allocate new cuckoofilter"); + } + else + { + RA2018_LOG(INFO) << LOG_DESC("CuckoofilterAllocator: reuse old cuckoofilter: ") + << m_currentCuckooFilter->filterID(); + } + return m_currentCuckooFilter->cuckooFilter(); +} + +void CuckoofilterAllocator::flush() +{ + RA2018_LOG(INFO) << LOG_DESC("CuckoofilterAllocator flush") << LOG_KV("resource", m_resourceID); + // Note: the last cuckoo-filter must store to the backend + saveOverLoadedCuckooFilter(true); +} + +void CuckoofilterAllocator::saveOverLoadedCuckooFilter(bool _flush) +{ + // without cuckoo-filter + if (!m_currentCuckooFilter) + { + if (m_flush) + { + m_callback(nullptr); + } + return; + } + m_total++; + // Note: must set m_total to the real-store-count before trigger the last + // cuckoo-filter-store-operation + if (_flush) + { + m_flush = true; + } + // TODO: check will this cause memory leak here? + auto allocator = shared_from_this(); + auto callback = [allocator](bcos::Error::Ptr _error) { + allocator->m_total--; + if (_error) + { + allocator->m_failed++; + } + RA2018_LOG(INFO) << LOG_DESC("saveOverLoadedCuckooFilter") + << LOG_KV("code", _error ? _error->errorCode() : 0) + << LOG_KV("msg", _error ? _error->errorMessage() : "success") + << LOG_KV("resource", allocator->m_resourceID); + // not all data has been stored into backend + if (allocator->m_total > 0) + { + return; + } + // not calls flush yet + if (!allocator->m_flush) + { + return; + } + // with error + if (allocator->m_failed) + { + allocator->m_callback(std::make_shared(-1, "store cuckoo-filter error!")); + return; + } + // success + allocator->m_callback(nullptr); + }; + // insert a new-overloaded-cuckoo-filter + // Note: the m_currentCuckooFilter will be re-allocated later + auto cuckooFilter = std::move(m_currentCuckooFilter); + if (-1 == cuckooFilter->filterID()) + { + RA2018_LOG(INFO) << LOG_DESC( + "CuckoofilterAllocator: saveOverLoadedCuckooFilter with insert") + << LOG_KV("resourceID", m_resourceID); + m_storage->asyncInsertCuckooFilter(m_resourceID, cuckooFilter->cuckooFilter(), callback); + return; + } + RA2018_LOG(INFO) << LOG_DESC("CuckoofilterAllocator: saveOverLoadedCuckooFilter with update") + << LOG_KV("resourceID", m_resourceID) << cuckooFilter->desc() + << LOG_KV("total", m_total); + // update the old-overloaded-cuckoo-filter + m_storage->asyncUpdateCuckooFilter(m_resourceID, cuckooFilter, callback); +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h b/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h new file mode 100644 index 00000000..38ae5988 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CuckoofilterAllocator.h + * @author: yujiechen + * @date 2022-11-7 + */ +#pragma once +#include "../Common.h" +#include "../RA2018PSIConfig.h" +#include "../storage/RA2018PSIStorage.h" + +namespace ppc::psi +{ +class CuckoofilterAllocator : public std::enable_shared_from_this +{ +public: + // the _reusableFilters record the cuckoo-filter ids that not full + CuckoofilterAllocator(RA2018PSIConfig::Ptr _config, RA2018PSIStorage::Ptr _storage, + std::string const& _resourceID, std::function _callback, + CuckooFilterInfoSet&& _reusableFilters) + : m_config(std::move(_config)), + m_storage(std::move(_storage)), + m_resourceID(_resourceID), + m_callback(std::move(_callback)), + m_reusableFilters(_reusableFilters.begin(), _reusableFilters.end()) + {} + virtual ~CuckoofilterAllocator() = default; + + // allocate the underload cuckoo-filter + virtual DefaultCukooFilterPtr allocate(); + // flush the updated/new cuckoo-filter into the backend-storage + // Note: the callback can'be called only after flush + virtual void flush(); + +private: + void saveOverLoadedCuckooFilter(bool _flush); + +private: + RA2018PSIConfig::Ptr m_config; + RA2018PSIStorage::Ptr m_storage; + std::string m_resourceID; + std::function m_callback; + + std::vector m_reusableFilters; + + // the current allocated cuckoo-filter + CuckooFilterInfo::Ptr m_currentCuckooFilter = nullptr; + + // for determine store the cuckoo-filter success or not + std::atomic m_total = {0}; + std::atomic m_failed = {0}; + std::atomic m_flush = {false}; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp new file mode 100644 index 00000000..54d1ac8d --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIClient.cpp + * @author: yujiechen + * @date 2022-11-8 + */ +#include "RA2018PSIClient.h" +#include + +using namespace ppc::psi; +using namespace ppc::protocol; +using namespace bcos; + +// insert new RA2018PSIClientCache for given task and seq +std::pair RA2018PSIClient::insertCache( + std::string const& _cuckooFilterResourceID, std::string const& _taskID, + TaskState::Ptr const& _taskState, DataResource::ConstPtr const& _dataResource, uint32_t _seq, + ppc::io::DataBatch::Ptr&& _plainData) +{ + if (isFull()) + { + return std::make_pair(false, nullptr); + } + bcos::UpgradableGuard l(x_caches); + auto it = m_caches.find(_taskID); + if (it != m_caches.end()) + { + auto pItem = it->second.find(_seq); + if (pItem != it->second.end()) + { + RA2018_LOG(WARNING) << LOG_DESC("insertCache return for the cache already exists") + << LOG_KV("task", _taskID) << LOG_KV("seq", _seq); + return std::make_pair(false, pItem->second); + } + } + bcos::UpgradeGuard ul(l); + // TODO: Be careful of the memory expansion caused by the evaluated-data and intersection-data + // stored in the RA2018PSIClientCache + m_capacity += _plainData->capacityBytes(); + auto state = loadCuckooFilterState(_cuckooFilterResourceID, false); + auto cacheItem = std::make_shared( + m_config, state, _taskID, _taskState, _dataResource, _seq, std::move(_plainData)); + m_caches[_taskID][_seq] = cacheItem; + RA2018_LOG(INFO) << LOG_DESC("insertCache success") << LOG_KV("task", _taskID) + << LOG_KV("seq", _seq); + return std::make_pair(true, cacheItem); +} + +// obtain the cache according to the taskID and seq +RA2018PSIClientCache::Ptr RA2018PSIClient::cache(std::string const& _taskID, uint32_t _seq) const +{ + bcos::ReadGuard l(x_caches); + if (!m_caches.count(_taskID) || !m_caches.at(_taskID).count(_seq)) + { + return nullptr; + } + return m_caches.at(_taskID).at(_seq); +} + +// Serially write the results of each (task, seq) to the result file +// erase the finished (task, seq) cache from the m_caches +void RA2018PSIClient::checkAndStoreInterSectionResult() +{ + bcos::WriteGuard l(x_caches); + for (auto it = m_caches.begin(); it != m_caches.end();) + { + auto& taskCache = it->second; + for (auto ptaskCache = taskCache.begin(); ptaskCache != taskCache.end();) + { + // commit the psi-result and erase the intersectioned cache + auto state = ptaskCache->second->state(); + if (state == CacheState::Intersectioned) + { + // Note: commit for the same-task must be seq when the output-data-resource is file + commitPSIResult(ptaskCache->second); + auto plainDataSize = ptaskCache->second->plainData()->capacityBytes(); + if (plainDataSize > m_capacity) + { + m_capacity = 0; + } + else + { + m_capacity -= plainDataSize; + } + // notify to wakeup the worker when the capacity decreased + m_onReady(); + } + // Note: only the can erase cache with synced state + if (state == CacheState::Synced) + { + ptaskCache = taskCache.erase(ptaskCache); + continue; + } + ptaskCache++; + } + // all seq-data of the task has been handled finished + if (taskCache.empty()) + { + it = m_caches.erase(it); + continue; + } + it++; + } +} + +// trigger computeIntersection when receive the cuckoo-filter-response +void RA2018PSIClient::computeIntersection() +{ + auto self = weak_from_this(); + ReadGuard l(x_caches); + for (auto const& it : m_caches) + { + auto clientCaches = it.second; + for (auto const& item : clientCaches) + { + auto clientCache = item.second; + auto state = clientCache->state(); + // only trigger intersection when finalized + // trigger psi-store when intersectioned + if (state != CacheState::Finalized && state != CacheState::Intersectioned) + { + continue; + } + m_config->threadPool()->enqueue([self, clientCache]() { + auto client = self.lock(); + if (!client) + { + return; + } + try + { + clientCache->computeIntersection(); + client->checkAndStoreInterSectionResult(); + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) + << LOG_DESC("computeIntersection exception, cancel the task") + << LOG_KV("exception", boost::diagnostic_information(e)) + << printTaskInfo(clientCache->taskState()->task()); + clientCache->taskState()->onTaskException(boost::diagnostic_information(e)); + } + }); + } + } +} + +// TODO: optimize the commit Performance with multiple thread +void RA2018PSIClient::commitPSIResult(RA2018PSIClientCache::Ptr _cache) +{ + // in-case of store the result multiple times + auto self = weak_from_this(); + m_commitWorker->enqueue([_cache, self]() { + auto client = self.lock(); + if (!client) + { + return; + } + try + { + auto startT = utcSteadyTime(); + _cache->storePSIResult(); + // notify to check the task-result + RA2018_LOG(INFO) << LOG_DESC("commitPSIResult success") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("task", _cache->taskID()) << LOG_KV("seq", _cache->seq()); + } + catch (std::exception const& e) + { + // commit error + _cache->taskState()->eraseFinishedTaskSeq(_cache->seq(), false); + RA2018_LOG(WARNING) << LOG_DESC("commitPSIResult error") + << LOG_KV("task", _cache->taskID()) << LOG_KV("seq", _cache->seq()) + << LOG_KV("msg", boost::diagnostic_information(e)); + } + }); +} + +CuckooFilterState::Ptr RA2018PSIClient::loadCuckooFilterState( + std::string const& _resourceID, bool _create) +{ + bcos::UpgradableGuard l(x_resourceCuckooFilterState); + auto it = m_resourceCuckooFilterState.find(_resourceID); + if (it != m_resourceCuckooFilterState.end()) + { + return it->second; + } + if (!_create) + { + return nullptr; + } + // create and insert a new cuckoo-filter-state + auto state = std::make_shared(_resourceID); + RA2018_LOG(INFO) << LOG_DESC("loadCuckooFilterState: create cuckooFilterState") + << LOG_KV("resource", _resourceID); + bcos::UpgradeGuard ul(l); + m_resourceCuckooFilterState[_resourceID] = state; + return state; +} diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h new file mode 100644 index 00000000..fc230bb2 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIClient.h + * @author: yujiechen + * @date 2022-11-8 + */ + +#pragma once +#include "../RA2018PSIConfig.h" +#include "CuckooFilterState.h" +#include "RA2018PSIClientCache.h" +#include +#include +#include +namespace ppc::psi +{ +class RA2018PSIClient : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIClient(RA2018PSIConfig::Ptr const& _config, uint64_t maxCacheSize = 1024 * 1024 * 1024) + : m_config(_config), + m_commitWorker(std::make_shared("committer", 1)), + m_maxCacheSize(maxCacheSize) + {} + virtual ~RA2018PSIClient() = default; + + // insert new RA2018PSIClientCache for given seq + std::pair insertCache( + std::string const& _cuckooFilterResourceID, std::string const& _taskID, + TaskState::Ptr const& _taskState, + ppc::protocol::DataResource::ConstPtr const& _dataResource, uint32_t _seq, + ppc::io::DataBatch::Ptr&& _plainData); + + // erase the cache + void eraseTaskCache(std::string const& _taskID) + { + bcos::WriteGuard l(x_caches); + m_caches.erase(_taskID); + } + // obtain the cache according to the taskID and seq + RA2018PSIClientCache::Ptr cache(std::string const& _taskID, uint32_t _seq) const; + + // check the task has completed or not, and store the intersection-result + virtual void checkAndStoreInterSectionResult(); + // compute-intersection + virtual void computeIntersection(); + + virtual CuckooFilterState::Ptr loadCuckooFilterState( + std::string const& _resourceID, bool _create); + + virtual bool isFull() const { return (m_capacity >= m_maxCacheSize); } + + template + bcos::Handler<> onReady(T const& _t) + { + return m_onReady.add(_t); + } + + uint64_t capacity() const { return m_capacity.load(); } + +private: + void commitPSIResult(RA2018PSIClientCache::Ptr _cache); + +private: + RA2018PSIConfig::Ptr m_config; + + // taskID->seq->RA2018PSIClientCache + using RA2018PSICacheType = + std::unordered_map>; + RA2018PSICacheType m_caches; + mutable bcos::SharedMutex x_caches; + + bcos::ThreadPool::Ptr m_commitWorker; + uint64_t m_maxCacheSize; + + std::atomic m_capacity = {0}; + + // record the cuckoo-filter-state for given resource + // TODO: clear the expired resourceCuckooFilterState + std::map m_resourceCuckooFilterState; + mutable bcos::SharedMutex x_resourceCuckooFilterState; + + // onReady + bcos::CallbackCollectionHandler<> m_onReady; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp new file mode 100644 index 00000000..217d780f --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIClientCache.cpp + * @author: yujiechen + * @date 2022-11-8 + */ +#include "RA2018PSIClientCache.h" + +using namespace ppc::psi; +using namespace bcos; + +// the client blind the input data +std::vector RA2018PSIClientCache::blind() +{ + auto startT = utcSteadyTime(); + bcos::RecursiveGuard l(m_mutex); + std::vector result; + m_config->oprf()->blind(m_plainData, m_privateKey, result); + RA2018_LOG(INFO) << LOG_DESC("blind") << LOG_KV("timecost", (utcSteadyTime() - startT)) + << printCurrentState(); + return result; +} + +// the client finalize the evaluated-data +void RA2018PSIClientCache::finalize(std::vector const& _evaluatedData) +{ + bcos::RecursiveGuard l(m_mutex); + // only can finalize in the evaluating-state + if (m_state != CacheState::Evaluating) + { + return; + } + if (_evaluatedData.size() != m_plainData->size()) + { + RA2018_LOG(WARNING) << LOG_DESC("Invalid evaluated data") + << LOG_KV("evaluatedDataSize", _evaluatedData.size()) + << printCurrentState(); + return; + } + m_state = CacheState::Finalizing; + + auto startT = utcSteadyTime(); + std::vector result; + m_config->oprf()->finalize(_evaluatedData, m_invPrivateKey, m_finalizedData); + m_state = CacheState::Finalized; + RA2018_LOG(INFO) << LOG_DESC("finalize successs") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("size", m_finalizedData.size()) << printCurrentState(); + return; +} + +void RA2018PSIClientCache::computeIntersection() +{ + bcos::RecursiveGuard l(m_mutex); + // not finalized + // already been intersectioned or intersection-in-progress or storeing + if (m_state != CacheState::Finalized) + { + RA2018_LOG(DEBUG) << LOG_DESC( + "computeIntersection return for not in-finalized-state(only can " + "intersection in finalized state)") + << printCurrentState(); + return; + } + // the cuckoo-filters not setted + if (!m_cuckooFilterState->fetchFinish()) + { + RA2018_LOG(DEBUG) << LOG_DESC( + "computeIntersection return for the cuckoo-filter not fetched") + << printCurrentState(); + return; + } + auto startT = utcSteadyTime(); + m_state = CacheState::IntersectionProgressing; + // trigger intersection + auto cuckooFilterInfos = m_cuckooFilterState->cuckooFilterInfos(); + for (uint64_t i = 0; i < m_finalizedData.size(); i++) + { + auto const& key = m_finalizedData.at(i); + // foreach the cuckooFilters to find the intersection + for (auto const& it : cuckooFilterInfos) + { + if (it.second->cuckooFilter()->contains(key)) + { + m_intersectionData.emplace_back(m_plainData->getBytes(i)); + break; + } + } + } + m_state = CacheState::Intersectioned; + RA2018_LOG(INFO) << LOG_DESC("computeIntersection success") + << LOG_KV("timecost", (utcSteadyTime() - startT)) << printCurrentState() + << LOG_KV("intersectionSize", m_intersectionData.size()) + << LOG_KV("cuckooFilterSize", cuckooFilterInfos.size()); + return; +} + +void RA2018PSIClientCache::storePSIResult() +{ + // only can store the intersectioned data + bcos::RecursiveGuard l(m_mutex); + if (m_state != CacheState::Intersectioned) + { + return; + } + m_state = CacheState::StoreProgressing; + RA2018_LOG(INFO) << LOG_DESC("storePSIResult") << printCurrentState() + << printDataResourceInfo(m_dataResource); + // store the psi result + m_taskState->storePSIResult(m_config->dataResourceLoader(), m_intersectionData); + m_state = CacheState::Stored; + // commit success + if (!m_taskState->task()->syncResultToPeer()) + { + m_state = CacheState::Synced; + // No need to sync task to peers, erase the seq from the task-state directly + m_taskState->eraseFinishedTaskSeq(m_seq, true); + return; + } + // sync psi result to server + syncPSIResult(); + + // release the allocated memory + std::vector().swap(m_finalizedData); + std::vector().swap(m_intersectionData); +} + +// sync psi result to server +void RA2018PSIClientCache::syncPSIResult() +{ + if (m_state != CacheState::Stored) + { + return; + } + m_state = CacheState::Syncing; + RA2018_LOG(INFO) << LOG_DESC("syncResultToPeer") << printCurrentState() + << LOG_KV("peer", m_taskState->peerID()); + auto dataSyncMsg = + m_config->ra2018MsgFactory()->createPSIMessage((uint32_t)PSIPacketType::PSIResultSyncMsg); + dataSyncMsg->setData(m_intersectionData); + auto self = weak_from_this(); + m_config->generateAndSendPPCMessage( + m_taskState->peerID(), m_taskState->task()->id(), dataSyncMsg, + [self](Error::Ptr&& _error) { + auto cache = self.lock(); + if (!cache) + { + return; + } + if (!_error || _error->errorCode() == 0) + { + return; + } + RA2018_LOG(WARNING) << LOG_DESC("Sync PSI result error") << LOG_KV("seq", cache->m_seq) + << printTaskInfo(cache->m_taskState->task()); + }, + m_seq, + [self](bcos::Error::Ptr _error, std::string const& _agencyID, + ppc::front::PPCMessageFace::Ptr _msg, ppc::front::ResponseFunc) { + auto cache = self.lock(); + if (!cache) + { + return; + } + if (_error) + { + RA2018_LOG(WARNING) + << LOG_DESC("Sync PSI result error") << LOG_KV("seq", cache->m_seq) + << printTaskInfo(cache->m_taskState->task()); + } + else + { + RA2018_LOG(INFO) << LOG_DESC("Sync PSI result success") + << LOG_KV("seq", cache->m_seq) + << printTaskInfo(cache->m_taskState->task()); + } + cache->m_state = CacheState::Synced; + // erase the seq from task-state only when receive the response from the peer + cache->m_taskState->eraseFinishedTaskSeq(cache->m_seq, true); + }); +} diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h new file mode 100644 index 00000000..4891b617 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIClientCache.h + * @author: yujiechen + * @date 2022-11-8 + */ + +#pragma once +#include "../../psi-framework/TaskState.h" +#include "../Common.h" +#include "../RA2018PSIConfig.h" +#include "CuckooFilterState.h" +#include "ppc-framework/io/DataBatch.h" +#include "ppc-framework/protocol/DataResource.h" +#include +#include +#include + +namespace ppc::psi +{ +class RA2018PSIClientCache : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIClientCache(RA2018PSIConfig::Ptr const& _config, + CuckooFilterState::Ptr _cuckooFilterState, std::string const& _taskID, + TaskState::Ptr const& _taskState, + ppc::protocol::DataResource::ConstPtr const& _dataResource, uint32_t _seq, + ppc::io::DataBatch::Ptr&& _plainData) + : m_config(_config), + m_taskID(_taskID), + m_taskState(_taskState), + m_cuckooFilterState(std::move(_cuckooFilterState)), + m_dataResource(_dataResource), + m_seq(_seq), + m_plainData(std::move(_plainData)) + { + m_privateKey = m_config->oprf()->generatePrivateKey(); + m_invPrivateKey = m_config->oprf()->inv(m_privateKey); + } + virtual ~RA2018PSIClientCache() = default; + + // the client blind the input data + virtual std::vector blind(); + + // the client finalize the evaluated-data + virtual void finalize(std::vector const& _evaluatedData); + // compute the intersection when receive both the finalized data and cuckoo-filter + virtual void computeIntersection(); + // the data has been intersectioned or not + virtual CacheState state() const { return m_state; } + virtual void setState(CacheState _state) { m_state = _state; } + + // store the psi-result after intersectioned + virtual void storePSIResult(); + // the task-seq + uint32_t seq() const { return m_seq; } + // the taskID + std::string const& taskID() const { return m_taskID; } + ppc::io::DataBatch::Ptr const& plainData() const { return m_plainData; } + TaskState::Ptr taskState() const { return m_taskState; } + +private: + inline std::string printCurrentState() + { + std::ostringstream stringstream; + stringstream << LOG_KV("task", m_taskID) << LOG_KV("seq", m_seq) + << LOG_KV("plainDataSize", m_plainData->size()) + << LOG_KV("finalizedData", m_finalizedData.size()) + << LOG_KV("intersectionData", m_intersectionData.size()) + << LOG_KV("state", m_state); + return stringstream.str(); + } + + void syncPSIResult(); + +private: + RA2018PSIConfig::Ptr m_config; + std::string m_taskID; + TaskState::Ptr m_taskState; + // Note the data-resource may have multiple cuckoo-filter + CuckooFilterState::Ptr m_cuckooFilterState; + + ppc::protocol::DataResource::ConstPtr m_dataResource; + uint32_t m_seq; + // the plainData batch for given seq + ppc::io::DataBatch::Ptr m_plainData; + + // the finalized-data + std::vector m_finalizedData; + // the intersection-data + std::vector m_intersectionData; + CacheState m_state = CacheState::Evaluating; + + // the privateKey + bcos::bytes m_privateKey; + bcos::bytes m_invPrivateKey; + + mutable bcos::RecursiveMutex m_mutex; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp new file mode 100644 index 00000000..1e7f3c0d --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIServer.cpp + * @author: yujiechen + * @date 2022-11-8 + */ +#include "RA2018PSIServer.h" +#include + +using namespace ppc::psi; +using namespace ppc::io; +using namespace bcos; + +// full-evaluate the input data +std::vector RA2018PSIServer::fullEvaluate(DataBatch::Ptr const& _data) +{ + auto startT = utcSteadyTime(); + std::vector result; + m_config->oprf()->fullEvaluate(_data, result); + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIServer fullEvaluate finish") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("dataSize", _data->size()); + return result; +} + +std::vector RA2018PSIServer::evaluate(std::vector const& _blindData) +{ + auto startT = utcSteadyTime(); + std::vector result; + m_config->oprf()->evaluate(_blindData, result); + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIServer evaluate finish") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("dataSize", _blindData.size()); + return result; +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h new file mode 100644 index 00000000..f1f0336d --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIServer.h + * @author: yujiechen + * @date 2022-11-8 + */ +#pragma once +#include "../RA2018PSIConfig.h" +#include "ppc-framework/io/DataBatch.h" +#include +namespace ppc::psi +{ +class RA2018PSIServer +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIServer(RA2018PSIConfig::Ptr const& _config) : m_config(_config) {} + virtual ~RA2018PSIServer() = default; + + // the server offline-full-evaluate + virtual std::vector fullEvaluate(ppc::io::DataBatch::Ptr const& _data); + // the server evaluate the blindedData + virtual std::vector evaluate(std::vector const& _blindData); + +private: + RA2018PSIConfig::Ptr m_config; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h b/cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h new file mode 100644 index 00000000..dd05d4a3 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Ra2018TaskParam.h + * @author: yujiechen + * @date 2022-11-7 + */ +#pragma once +#include "../Common.h" +#include +namespace ppc::psi +{ +class Ra2018TaskParam +{ +public: + using Ptr = std::shared_ptr; + Ra2018TaskParam(std::string_view _param) + { + Json::Reader reader; + Json::Value result; + if (!reader.parse(_param.begin(), _param.end(), result)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::InvalidTaskParamForRA2018, + "Invalid Ra2018TaskParam: invalid json")); + } + if (!result.isArray()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::InvalidTaskParamForRA2018, + "Invalid Ra2018TaskParam: the param must be array")); + } + auto command = result[0].asString(); + // the DATA_PREPROCESSING command must specified the operationType(insert(0)/delete(1)) + if (DATA_PREPROCESSING_CMD == command) + { + m_command = RA2018Command::DATA_PREPROCESSING; + if (result.size() < 2) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::InvalidTaskParamForRA2018, + "Must set operation for " + command + ", insert(0)/delete(1)")); + } + m_operation = result[1].asInt(); + } + else if (RUN_PSI_CMD == command) + { + m_command = RA2018Command::RUN_PSI; + } + else + { + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int)PSIRetCode::InvalidTaskParamForRA2018, "Unsupported command " + command)); + } + } + + int command() const { return m_command; } + int operation() const { return m_operation; } + +private: + int m_command; + int m_operation; +}; +} // namespace ppc::psi diff --git a/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h b/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h new file mode 100644 index 00000000..c14d7c81 --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018Message.h + * @author: yujiechen + * @date 2022-11-9 + */ +#pragma once +#include "../../psi-framework/protocol/PSIMessage.h" +#include "../Common.h" +#include "../core/CuckooFilterInfo.h" +#include "ppc-tars-protocol/Common.h" + +namespace ppc::psi +{ +class RA2018FilterMessage : public PSIMessage +{ +public: + using Ptr = std::shared_ptr; + RA2018FilterMessage() : PSIMessage() {} + explicit RA2018FilterMessage(uint32_t _type) : PSIMessage(_type) {} + explicit RA2018FilterMessage(std::function _inner) + : PSIMessage(std::move(_inner)) + { + decodeCuckooFilterInfos(); + } + + explicit RA2018FilterMessage(bcos::bytesConstRef _data) : RA2018FilterMessage() + { + decode(_data); + } + + virtual void setFilterInfo(std::vector&& _filterInfos) + { + m_cuckooFilterInfos = std::move(_filterInfos); + } + + virtual void appendFilterInfo(CuckooFilterInfo::Ptr _filterInfo) + { + m_cuckooFilterInfos.emplace_back(std::move(_filterInfo)); + } + + virtual void setCuckooFilterSize(int32_t _cuckooFilterSize) + { + m_inner()->cuckooFilterSize = _cuckooFilterSize; + } + virtual int32_t cuckooFilterSize() const { return m_inner()->cuckooFilterSize; } + + virtual std::vector& mutableFilterInfo() { return m_cuckooFilterInfos; } + virtual std::vector const& filterInfo() { return m_cuckooFilterInfos; } + + bcos::bytesPointer encode() const override + { + m_inner()->cuckooFilterInfo.clear(); + // encode m_cuckooFilterInfos + for (auto& info : m_cuckooFilterInfos) + { + ppctars::TarsCuckooFilter filter; + filter.id = info->filterID(); + filter.hash = std::vector(info->hash().begin(), info->hash().end()); + if (info->cuckooFilterData().empty() && info->cuckooFilter()) + { + auto encodedData = info->cuckooFilter()->serialize(); + std::move(encodedData.begin(), encodedData.end(), std::back_inserter(filter.data)); + } + else if (!info->cuckooFilterData().empty()) + { + auto data = info->cuckooFilterData(); + std::move(data.begin(), data.end(), std::back_inserter(filter.data)); + } + m_inner()->cuckooFilterInfo.emplace_back(std::move(filter)); + } + return PSIMessage::encode(); + } + + // decode the RA2018FilterMessage + void decode(bcos::bytesConstRef _data) override + { + PSIMessage::decode(_data); + // decode m_cuckooFilterInfos + decodeCuckooFilterInfos(); + } + +protected: + void decodeCuckooFilterInfos() + { + m_cuckooFilterInfos.clear(); + for (auto& filterInfo : m_inner()->cuckooFilterInfo) + { + DefaultCukooFilterPtr filter = nullptr; + // decode the cuckoo-filter + if (!filterInfo.data.empty()) + { + filter = std::make_shared(bcos::bytesConstRef( + (const bcos::byte*)filterInfo.data.data(), filterInfo.data.size())); + } + bcos::bytes hashData(filterInfo.hash.begin(), filterInfo.hash.end()); + auto cuckooFilterInfo = + std::make_shared(filterInfo.id, hashData, filter); + m_cuckooFilterInfos.emplace_back(std::move(cuckooFilterInfo)); + } + } + +private: + mutable std::vector m_cuckooFilterInfos; +}; + +// the factory implementation +class RA2018MessageFactory : public PSIMessageFactoryImpl +{ +public: + using Ptr = std::shared_ptr; + RA2018MessageFactory() : PSIMessageFactoryImpl() {} + ~RA2018MessageFactory() override = default; + + virtual RA2018FilterMessage::Ptr createRA2018FilterMessage(uint32_t _packetType) + { + return std::make_shared(_packetType); + } + + PSIMessageInterface::Ptr decodePSIMessage(bcos::bytesConstRef _data) override + { + auto inner = [inner = ppctars::PSIMessage()]() mutable { return &inner; }; + tars::TarsInputStream input; + input.setBuffer((const char*)_data.data(), _data.size()); + inner()->readFrom(input); + switch (inner()->packetType) + { + case (uint32_t)RA2018PacketType::CuckooFilterRequest: + case (uint32_t)RA2018PacketType::CuckooFilterResponse: + case (uint32_t)RA2018PacketType::MissingCuckooFilterRequest: + case (uint32_t)RA2018PacketType::MissingCuckooFilterResponse: + return std::make_shared(inner); + case (uint32_t)RA2018PacketType::EvaluateRequest: + case (uint32_t)RA2018PacketType::EvaluateResponse: + return std::make_shared(inner); + default: + return decodePSIBaseMessage(inner); + } + } +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp b/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp new file mode 100644 index 00000000..6b2e316d --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp @@ -0,0 +1,423 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIStorage.cpp + * @author: yujiechen + * @date 2022-10-27 + */ +#include "RA2018PSIStorage.h" +#include "../Common.h" +#include "ppc-framework/protocol/Protocol.h" +#include +#include +#include + +using namespace ppc; +using namespace bcos; +using namespace ppc::psi; +using namespace ppc::storage; +using namespace ppc::protocol; +using namespace ppc::io; + +// create the database +void RA2018PSIStorage::createDataBase() +{ + std::string command = "CREATE DATABASE IF NOT EXISTS " + m_config->dbName(); + m_config->storage()->execCommit(command.c_str(), (int)FieldDataType::TERMINATE); + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage: create database success") + << LOG_KV("dbName", m_config->dbName()); + if (m_config->fileStorage()) + { + m_config->fileStorage()->createDirectory(m_config->dbName()); + } + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage: create directory success") + << LOG_KV("dirPath", m_config->dbName()); +} + +// create the cuckoo-filter-table +// TODO: add hash field to the cuckooFilterTable +void RA2018PSIStorage::createCuckooFilterTable() +{ + std::stringstream ss; + ss << "CREATE TABLE IF NOT EXISTS `" << m_config->cuckooFilterTableName() << "` (\n"; + ss << "`" << m_resourceIDField << "` varchar(128),\n"; + ss << "`" << m_filterIDField << "` INT AUTO_INCREMENT,\n"; + // Note: here filter-hash-field use at-most 32Bytes, for perf-consideration, default use md5 + ss << "`" << m_filterHashField << "` TINYBLOB DEFAULT NULL,\n"; + // Note: the mysql will convert boolean to tinyint + ss << "`" << m_loadField << "` BOOLEAN,\n"; + ss << " PRIMARY KEY (`" << m_resourceIDField << "`, `" << m_filterIDField << "`)\n"; + ss << ", key(" << m_filterIDField << ")"; + ss << ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"; + m_config->storage()->execCommit(ss.str().c_str(), FieldDataType::TERMINATE); + RA2018_LOG(INFO) << LOG_DESC("createCuckooFilterTable: create cuckoo-filter-table success") + << LOG_KV("dbName", m_config->cuckooFilterTableName()); +} + +// get the cuckoo-filter-ids according to the _resourceID +CuckooFilterInfoSet RA2018PSIStorage::getCuckooFilterInfos( + std::string const& _resourceID, bool _onlyNotFull) +{ + try + { + auto startT = utcSteadyTime(); + std::stringstream ss; + ss << "select `" << m_filterIDField << "`, `" << m_filterHashField << "` from `" + << m_config->cuckooFilterTableName() << "` where `" << m_resourceIDField << "`='" + << _resourceID << "'"; + if (_onlyNotFull) + { + ss << " and `" << m_loadField << "` = 0"; + } + auto sqlCommand = ss.str(); + auto result = + m_config->storage()->execQuery(true, sqlCommand.c_str(), FieldDataType::TERMINATE); + if (result->size() != 2) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, + "Invalid queried data, expect 2 columns, the result is " + + std::to_string(result->size()) + " columns")); + } + // parse the result + RA2018_LOG(INFO) << LOG_DESC("getCuckooFilterInfos success") + << LOG_KV("metaFieldSize", result->metaData().size()); + // parse the filter data + // int64_t type + auto const& filterIDData = result->data().at(0); + // bytes type + auto const& filterHashData = result->data().at(1); + if (filterIDData->size() != filterHashData->size()) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, + "Invalid queried data, the item size of filterIDs should be equal to the size of " + "filterHashData")); + } + + CuckooFilterInfoSet filterInfos; + for (uint64_t i = 0; i < filterIDData->size(); i++) + { + auto filterID = filterIDData->get(i); + auto hash = filterHashData->get(i); + filterInfos.insert(std::make_shared(filterID, hash, nullptr)); + RA2018_LOG(INFO) << LOG_DESC("getCuckooFilterInfos") << LOG_KV("resource", _resourceID) + << LOG_KV("filterID", filterID) << LOG_KV("hash", toHex(hash)); + } + RA2018_LOG(INFO) << LOG_DESC("getCuckooFilterInfos success") + << LOG_KV("resource", _resourceID) + << LOG_KV("filterSize", filterInfos.size()) + << LOG_KV("timecost", (utcSteadyTime() - startT)); + return filterInfos; + } + catch (std::exception const& e) + { + std::stringstream oss; + oss << "getCuckooFilterInfos error! " << LOG_KV("resource", _resourceID) + << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = oss.str(); + RA2018_LOG(WARNING) << errorMsg; + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, errorMsg)); + } +} + +void RA2018PSIStorage::storeFilterData(std::string const& _desc, std::string const& _resourceID, + bcos::bytes const& _hash, bytesConstRef _data) +{ + auto startT = utcSteadyTime(); + auto hexHash = toHex(_hash); + auto desc = obtainCuckooFilterDesc(_resourceID, hexHash); + auto writer = m_config->dataResourceLoader()->loadWriter(desc, true, m_config->fileStorage()); + writer->writeBytes(_data); + writer->flush(); + RA2018_LOG(INFO) << LOG_BADGE("storeFilterData success") << LOG_DESC(_desc) + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("resource", _resourceID) << LOG_KV("hash", hexHash) + << LOG_KV("dataSize", _data.size()) << LOG_KV("path", desc->path()); +} + +// remove the old filter when updating cuckoo-filter +void RA2018PSIStorage::removeOldFilterData( + std::string const& _desc, std::string const& _resourceID, bcos::bytes const& _hash) +{ + auto hexHash = toHex(_hash); + auto desc = obtainCuckooFilterDesc(_resourceID, hexHash); + try + { + if (desc->type() == (int)ppc::protocol::DataResourceType::FILE) + { + std::remove(desc->path().c_str()); + } + if (desc->type() == (int)ppc::protocol::DataResourceType::HDFS) + { + m_config->fileStorage()->deleteFile(desc->path()); + } + } + catch (std::exception const& e) + { + RA2018_LOG(WARNING) << LOG_DESC("removeOldFilterData error") + << LOG_KV("resource", _resourceID) << LOG_KV("hash", hexHash) + << LOG_KV("path", desc->path()); + } +} + +// store the cuckoo-filter to the storage +void RA2018PSIStorage::asyncInsertCuckooFilter(std::string const& _resourceID, + DefaultCukooFilterPtr _cukooFilter, std::function _callback, + int32_t _filterID) +{ + std::stringstream ss; + auto filterData = _cukooFilter->serialize(); + auto filterDataHash = m_config->binHashImpl()->hash( + bytesConstRef((const bcos::byte*)(filterData.data()), filterData.size())); + storeFilterData("asyncInsertCuckooFilter", _resourceID, filterDataHash, + bytesConstRef((bcos::byte const*)filterData.data(), filterData.size())); + + RA2018_LOG(INFO) << LOG_DESC("asyncInsertCuckooFilter: update meta-info") + << LOG_KV("resource", _resourceID) << LOG_KV("dataSize", filterData.size()); + + auto cuckooFilterFull = _cukooFilter->full() ? 1 : 0; + auto withFilterID = (_filterID == -1) ? false : true; + + ss << "insert into " << m_config->cuckooFilterTableName() << "("; + ss << m_resourceIDField << ", " << m_loadField << ", " << m_filterHashField; + if (withFilterID) + { + ss << "," << m_filterIDField; + } + // resourceID, loadField, filterHash + ss << ") values(?, ?, ?"; + // filterID + if (withFilterID) + { + ss << ",?"; + } + // support update when key-conflict + ss << ")ON DUPLICATE KEY UPDATE `" << m_resourceIDField << "`=?, `" << m_loadField << "`=?, `" + << m_filterHashField << "`=?"; + if (withFilterID) + { + ss << ", `" << m_filterIDField << "` = ?"; + } + // the command + auto sqlCommand = ss.str(); + // resourceID, loadField, filterDataHash + auto statement = + m_config->storage()->generateStatement(sqlCommand.c_str(), FieldDataType::STRING, + _resourceID.c_str(), FieldDataType::UINT, cuckooFilterFull, FieldDataType::BYTES, + filterDataHash.data(), filterDataHash.size(), FieldDataType::TERMINATE); + // filterID + if (withFilterID) + { + m_config->storage()->appendStatement( + statement, FieldDataType::SINT, _filterID, FieldDataType::TERMINATE); + } + m_config->storage()->appendStatement(statement, FieldDataType::STRING, _resourceID.c_str(), + FieldDataType::UINT, cuckooFilterFull, FieldDataType::BYTES, filterDataHash.data(), + filterDataHash.size(), FieldDataType::TERMINATE); + if (withFilterID) + { + m_config->storage()->appendStatement( + statement, FieldDataType::SINT, _filterID, FieldDataType::TERMINATE); + } + asyncCommit("asyncInsertCuckooFilter", _resourceID, statement, _callback); +} + +void RA2018PSIStorage::asyncUpdateCuckooFilter(std::string const& _resourceID, + CuckooFilterInfo::Ptr const& _filterInfo, std::function _callback) +{ + if (!_filterInfo->cuckooFilter()) + { + _callback(BCOS_ERROR_PTR(-1, "asyncUpdateCuckooFilter failed for empty cuckoo-filter")); + return; + } + // the hash not changed + auto filterData = _filterInfo->cuckooFilter()->serialize(); + auto filterDataHash = m_config->binHashImpl()->hash( + bytesConstRef((const bcos::byte*)(filterData.data()), filterData.size())); + + if (filterDataHash == _filterInfo->hash()) + { + RA2018_LOG(INFO) << LOG_DESC("asyncUpdateCuckooFilter return for the filter not changed") + << _filterInfo->desc(); + _callback(nullptr); + return; + } + // remove the old-cuckoo-filter + removeOldFilterData("asyncUpdateCuckooFilter", _resourceID, _filterInfo->hash()); + // store the updated-cuckoo-filter + storeFilterData("asyncUpdateCuckooFilter", _resourceID, filterDataHash, + bytesConstRef((bcos::byte const*)filterData.data(), filterData.size())); + + RA2018_LOG(INFO) << LOG_DESC("asyncUpdateCuckooFilter: updateMetaInfo") + << LOG_KV("resource", _resourceID) << _filterInfo->desc() + << LOG_KV("currentHash", bcos::toHex(filterDataHash)) + << LOG_KV("dataSize", filterData.size()); + std::stringstream ss; + auto cuckooFilterFull = _filterInfo->cuckooFilter()->full() ? 1 : 0; + + ss << "update `" << m_config->cuckooFilterTableName() << "` set "; + ss << m_loadField << " = ?, "; + ss << m_filterHashField << " = ?"; + ss << " where " << m_resourceIDField << " = '" << _resourceID << "'"; + ss << " and " << m_filterIDField << " = " << _filterInfo->filterID(); + auto sqlCommand = ss.str(); + auto statement = m_config->storage()->generateStatement(sqlCommand.c_str(), FieldDataType::UINT, + cuckooFilterFull, FieldDataType::BYTES, filterDataHash.data(), filterDataHash.size(), + FieldDataType::TERMINATE); + asyncCommit("asyncUpdateCuckooFilter", _resourceID, statement, _callback); +} + +void RA2018PSIStorage::asyncCommit(std::string&& _desc, std::string const& _resourceID, + ppc::storage::Statement::Ptr _statement, std::function _callback) +{ + auto self = weak_from_this(); + m_config->threadPool()->enqueue( + [_statement, desc = std::move(_desc), _resourceID, _callback, self]() { + try + { + auto storage = self.lock(); + if (!storage) + { + return; + } + storage->m_config->storage()->execStatement(_statement); + _callback(nullptr); + } + catch (std::exception const& e) + { + std::stringstream oss; + oss << desc << " error !" << LOG_KV("resource", _resourceID) + << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = oss.str(); + RA2018_LOG(WARNING) << errorMsg; + _callback(BCOS_ERROR_PTR((int64_t)PPCRetCode::EXCEPTION, errorMsg)); + } + }); +} + +CuckooFilterInfo::Ptr RA2018PSIStorage::loadCuckooFilterFromFile(std::string const& _resourceID, + int32_t _filterID, bcos::bytes const& _cuckooFilterHash, bool _deserialize) +{ + try + { + auto startT = utcSteadyTime(); + auto cuckooFilterInfo = std::make_shared(_filterID, _cuckooFilterHash); + auto hexHash = toHex(_cuckooFilterHash); + auto desc = obtainCuckooFilterDesc(_resourceID, hexHash); + auto reader = m_config->dataResourceLoader()->loadReader( + desc, DataSchema::Bytes, false, m_config->fileStorage()); + auto cuckooFilterData = reader->readBytes(); + auto dataSize = cuckooFilterData.size(); + // deserialize + if (_deserialize) + { + RA2018_LOG(INFO) << LOG_DESC("deserialize cuckoo-filter") << cuckooFilterInfo->desc() + << LOG_KV("size", cuckooFilterData.size()); + // return the deserialized cuckoo-filter + auto cuckooFilter = std::make_shared( + bcos::bytesConstRef(cuckooFilterData.data(), cuckooFilterData.size())); + cuckooFilterInfo->setCuckooFilter(std::move(cuckooFilter)); + RA2018_LOG(INFO) << LOG_DESC("deserialize cuckoo-filter success") + << cuckooFilterInfo->desc() << LOG_KV("size", cuckooFilterData.size()); + } + else + { + // return the raw data + cuckooFilterInfo->setCuckooFilterData(std::move(cuckooFilterData)); + } + RA2018_LOG(INFO) << LOG_DESC("loadCuckooFilterFromFile success") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("resource", _resourceID) << cuckooFilterInfo->desc() + << LOG_KV("dataSize", dataSize) << LOG_KV("path", desc->path()); + return cuckooFilterInfo; + } + catch (OpenFileFailed const& e) + { + RA2018_LOG(WARNING) << LOG_DESC( + "loadCuckooFilterFromFile exception for OpenFileFailed: delete " + "the meta-info from sql-storage") + << LOG_KV("resource", _resourceID) << LOG_KV("filter", _filterID) + << LOG_KV("error", boost::diagnostic_information(e)); + // delete the meta-info from the storage + deleteCuckooFilter(_resourceID, _filterID); + throw e; + } +} +// load the cuckoo-filter from the storage +CuckooFilterInfo::Ptr RA2018PSIStorage::loadCuckooFilterDataInfo( + std::string const& _resourceID, int32_t _filterID, bool _deserialize) +{ + try + { + auto startT = utcSteadyTime(); + std::stringstream ss; + ss << "select `" << m_filterHashField << "` " + << " from `" << m_config->cuckooFilterTableName() << "`"; + ss << " where `" << m_resourceIDField << "` = '" << _resourceID << "'"; + ss << " and `" << m_filterIDField << "` = " << _filterID; + auto command = ss.str(); + + auto result = + m_config->storage()->execQuery(true, command.c_str(), FieldDataType::TERMINATE); + if (result->data().empty()) + { + return nullptr; + } + if (result->size() != 1) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, + "loadCuckooFilterDataInfo: invalid queried data, expect get 1 columns, real get " + + std::to_string(result->size()) + " columns!")); + } + // Note: only expect obtain one element from the database + auto cuckooFilterHash = result->data().at(0)->get(0); + auto cuckooFilterInfo = + loadCuckooFilterFromFile(_resourceID, _filterID, cuckooFilterHash, _deserialize); + RA2018_LOG(INFO) << LOG_DESC("loadCuckooFilterDataInfo success") + << LOG_KV("timecost", (utcSteadyTime() - startT)) + << LOG_KV("resource", _resourceID) << cuckooFilterInfo->desc(); + return cuckooFilterInfo; + } + catch (std::exception const& e) + { + std::stringstream oss; + oss << "loadCuckooFilterDataInfo error! " << LOG_KV("resource", _resourceID) + << LOG_KV("filter", _filterID) << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = oss.str(); + RA2018_LOG(WARNING) << errorMsg; + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, errorMsg)); + } +} + +void RA2018PSIStorage::deleteCuckooFilter(std::string const& _resourceID, int32_t _filterID) +{ + try + { + std::stringstream oss; + oss << "delete from `" << m_config->cuckooFilterTableName() << "`"; + oss << " where `" << m_resourceIDField << "` = '" << _resourceID << "'"; + oss << " and `" << m_filterIDField << "` = " << _filterID; + auto command = oss.str(); + m_config->storage()->execCommit(command.c_str(), FieldDataType::TERMINATE); + } + catch (std::exception const& e) + { + std::stringstream oss; + oss << "deleteCuckooFilter error! " << LOG_KV("resource", _resourceID) + << LOG_KV("filter", _filterID) << LOG_KV("error", boost::diagnostic_information(e)); + auto errorMsg = oss.str(); + RA2018_LOG(WARNING) << errorMsg; + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)PPCRetCode::EXCEPTION, errorMsg)); + } +} \ No newline at end of file diff --git a/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h b/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h new file mode 100644 index 00000000..edb8158a --- /dev/null +++ b/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIStorage.h + * @author: yujiechen + * @date 2022-10-27 + */ + +#pragma once +#include "../Common.h" +#include "../RA2018PSIConfig.h" +#include "../core/CuckooFilterInfo.h" +#include "ppc-framework/storage/SQLStorage.h" +#include +#include +#include + +namespace ppc::psi +{ +class RA2018PSIStorage : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + RA2018PSIStorage(RA2018PSIConfig::Ptr _config) : m_config(_config) {} + virtual ~RA2018PSIStorage() = default; + + // get the (cuckoo-filter-id, cuckoo-filter-hash according to the _resourceID and loadFactor + // TODO: optimize the perf(use async-client) + virtual CuckooFilterInfoSet getCuckooFilterInfos( + std::string const& _resourceID, bool _onlyNotFull); + + // load the cuckoo-filter from the storage + // TODO: optimize the perf(use async-client) + virtual CuckooFilterInfo::Ptr loadCuckooFilterDataInfo( + std::string const& _resourceID, int32_t _filterID, bool _deserialize); + + virtual bcos::bytes loadCuckooFilterData(std::string const& _resourceID, int32_t _filterID) + { + try + { + // need to deserialize + auto filterInfo = loadCuckooFilterDataInfo(_resourceID, _filterID, false); + if (!filterInfo) + { + return bcos::bytes(); + } + return filterInfo->cuckooFilterData(); + } + catch (std::exception const& e) + { + return bcos::bytes(); + } + } + + virtual DefaultCukooFilterPtr loadCuckooFilter( + std::string const& _resourceID, int32_t _filterID) + { + // need to serialize + auto filterInfo = loadCuckooFilterDataInfo(_resourceID, _filterID, true); + if (!filterInfo) + { + return nullptr; + } + return filterInfo->cuckooFilter(); + } + + // insert a new cuckoo-filter to the storage + // update the entry if key-conflict + virtual void asyncInsertCuckooFilter(std::string const& _resourceID, + DefaultCukooFilterPtr _cukooFilter, std::function _callback, + int32_t _filterID = -1); + // update the cuckoo-filter + // Note: update is more efficient than asyncInsertCuckooFilter + virtual void asyncUpdateCuckooFilter(std::string const& _resourceID, + CuckooFilterInfo::Ptr const& _filterInfo, + std::function _callback); + + virtual void init() + { + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage init the database") + << LOG_KV("dbName", m_config->dbName()); + createDataBase(); + // use the created database + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage use database: ") << m_config->dbName(); + m_config->storage()->useDataBase(m_config->dbName().c_str()); + + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage init the cuckoo-filter-table") + << LOG_KV("dbName", m_config->cuckooFilterTableName()); + createCuckooFilterTable(); + RA2018_LOG(INFO) << LOG_DESC("RA2018PSIStorage init success"); + } + +protected: + // init the database for ra2018-storage + void createDataBase(); + void createCuckooFilterTable(); + + void asyncCommit(std::string&& _desc, std::string const& _resourceID, + ppc::storage::Statement::Ptr _statement, std::function _callback); + + // store the filter data into file or hdfs + void storeFilterData(std::string const& _desc, std::string const& _resourceID, + bcos::bytes const& _hash, bcos::bytesConstRef _filterData); + // remove the old filter when updating cuckoo-filter + void removeOldFilterData( + std::string const& _desc, std::string const& _resourceID, bcos::bytes const& _hash); + + CuckooFilterInfo::Ptr loadCuckooFilterFromFile(std::string const& _resourceID, + int32_t _filterID, bcos::bytes const& _cuckooFilterHash, bool _deserialize); + + ppc::protocol::DataResourceDesc::Ptr obtainCuckooFilterDesc( + std::string const& _resourceID, std::string const& _hexHash) + { + auto desc = std::make_shared(); + auto resourceIDHash = m_config->binHashImpl()->hash( + bcos::bytesConstRef((bcos::byte const*)_resourceID.data(), _resourceID.size())); + + std::string path = m_config->dataPath() + "/" + bcos::toHex(resourceIDHash) + "/" + + _hexHash + c_dataPostFix; + // use file + if (!m_config->fileStorage()) + { + desc->setType((uint16_t)ppc::protocol::DataResourceType::FILE); + } + else + { + // use hdfs + // Note: we use dbName to distinguish files for different-nodes + path = m_config->dbName() + "/" + path; + desc->setType((uint16_t)ppc::protocol::DataResourceType::HDFS); + } + + desc->setPath(path); + return desc; + } + + // delete the given cuckoo-filter + virtual void deleteCuckooFilter(std::string const& _resourceID, int32_t _filterID); + +private: + RA2018PSIConfig::Ptr m_config; + + std::string const m_resourceIDField = "resource_id"; + // the filter-id auto-inc + std::string const m_filterIDField = "filter_id"; + std::string const m_filterHashField = "hash"; + std::string const m_loadField = "full"; + + std::string const c_dataPostFix = ".filter"; +}; +} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/ppc-psi/tests/CMakeLists.txt b/cpp/ppc-psi/tests/CMakeLists.txt new file mode 100644 index 00000000..f2fa68c3 --- /dev/null +++ b/cpp/ppc-psi/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-psi) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) +target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-ppc-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp b/cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp new file mode 100644 index 00000000..82409479 --- /dev/null +++ b/cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestBsEcdhCache.cpp + * @author: shawnhe + * @date 2023-10-10 + */ + +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h" +#include "ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h" +#include "test-utils/FakeFront.h" +#include "test-utils/FileTool.h" +#include "test-utils/TaskMock.h" +#include +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::protocol; +using namespace ppc::io; +using namespace ppc::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(BsEcdhCacheTest, TestPromptFixture) + +void testEcdhFunc(const std::string& _input, BsEcdhCache::Ptr _cache0, BsEcdhCache::Ptr _cache1) +{ + _cache0->generateKey(); + _cache1->generateKey(); + auto cipher0 = _cache0->genCipherWithBase64(_input); + auto cipher1 = _cache1->genCipherWithBase64(_input); + auto ecdhCipher0 = _cache1->genEcdhCipherWithBase64(cipher0); + auto ecdhCipher1 = _cache0->genEcdhCipherWithBase64(cipher1); + BOOST_CHECK(ecdhCipher0 == ecdhCipher1); + BOOST_CHECK(ecdhCipher0.size() > POINT_SIZE); + BOOST_CHECK(ecdhCipher1.size() > POINT_SIZE); +} + +BOOST_AUTO_TEST_CASE(testEcdhEnc) +{ + auto cache0 = std::make_shared( + "test0", nullptr, nullptr, nullptr, false, false, nullptr, nullptr, nullptr, 100); + auto cache1 = std::make_shared( + "test1", nullptr, nullptr, nullptr, false, false, nullptr, nullptr, nullptr, 100); + for (int i = 0; i < 100; i++) + { + testEcdhFunc(std::to_string(i), cache0, cache1); + } + + std::string input = "hello world"; + std::string key0 = bcos::base64Decode("teT9hMCdq8UCskX6CRlJ9bX7bC5TZ4M1O/eQlpHmyws="); + std::string key1 = bcos::base64Decode("Oav3jsBuamDwLyUo33VOYumJc0f/ySo5oLbkBe+fMgo="); + cache0->setKey(bcos::bytes(key0.begin(), key0.end())); + cache1->setKey(bcos::bytes(key1.begin(), key1.end())); + auto cipher0 = cache0->genCipherWithBase64(input); + auto cipher1 = cache1->genCipherWithBase64(input); + auto ecdhCipher0 = cache1->genEcdhCipherWithBase64(cipher0); + auto ecdhCipher1 = cache0->genEcdhCipherWithBase64(cipher1); + + std::cout << cipher0 << std::endl; + std::cout << cipher1 << std::endl; + std::cout << ecdhCipher0 << std::endl; + std::cout << ecdhCipher1 << std::endl; + + BOOST_CHECK(ecdhCipher0 == ecdhCipher1); +} + +BOOST_AUTO_TEST_CASE(testBsEcdh) +{ + auto inputPath = "../../../ppc-psi/tests/bs-ecdh-psi/data/inputs.csv"; + auto outputPath = "../../../ppc-psi/tests/bs-ecdh-psi/data/outputs.csv"; + boost::filesystem::create_directory("../../../ppc-psi/tests/bs-ecdh-psi/data"); + + auto count = 123456; + prepareInputs(inputPath, count); + + auto taskRequest = std::make_shared(); + auto taskID = "0x123456"; + auto dataResource = std::make_shared("id"); + auto dataDesc = std::make_shared(); + dataDesc->setType(0); + dataDesc->setPath(inputPath); + dataResource->setDesc(dataDesc); + auto outDesc = std::make_shared(); + outDesc->setType(0); + outDesc->setPath(outputPath); + dataResource->setOutputDesc(outDesc); + taskRequest->dataResource = dataResource; + taskRequest->taskID = taskID; + taskRequest->enableAudit = true; + + auto dataResourceLoader = std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr); + auto bsEcdhPSIFactory = std::make_shared(); + auto bsEcdhPSI = bsEcdhPSIFactory->buildBsEcdhPSI( + std::make_shared("test_bs-ecdh-psi", std::thread::hardware_concurrency()), + dataResourceLoader, 60); + bsEcdhPSI->enableOutputExists(); + bsEcdhPSI->start(); + bsEcdhPSI->asyncRunTask(taskRequest, nullptr); + + auto getTaskStatusRequest = std::make_shared(); + getTaskStatusRequest->taskID = taskID; + + while (bsEcdhPSI->getTaskStatus(getTaskStatusRequest)->data()["status"].asString() == + protocol::toString(TaskStatus::PENDING)) + { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + BOOST_CHECK(bsEcdhPSI->getTaskStatus(getTaskStatusRequest)->data()["status"].asString() == + protocol::toString(TaskStatus::RUNNING)); + + auto cache = std::make_shared( + "test", nullptr, nullptr, nullptr, false, false, nullptr, nullptr, nullptr, count); + cache->generateKey(); + + uint32_t offset = 0; + uint32_t size = 10000; + uint32_t total = count; + do + { + auto fetchCipherRequest = std::make_shared(); + fetchCipherRequest->taskID = taskID; + fetchCipherRequest->offset = offset; + fetchCipherRequest->size = size; + auto result = bsEcdhPSI->fetchCipher(fetchCipherRequest); + total = result->data()["total"].asInt(); + std::vector ciphers; + for (const Json::Value& cipher : result->data()["ciphers"]) + { + ciphers.push_back(cipher.asString()); + } + + auto sendEcdhCipherRequest = std::make_shared(); + sendEcdhCipherRequest->taskID = taskID; + sendEcdhCipherRequest->offset = offset; + sendEcdhCipherRequest->size = ciphers.size(); + + for (const auto& cipher : ciphers) + { + sendEcdhCipherRequest->ecdhCiphers.push_back(cache->genEcdhCipherWithBase64(cipher)); + } + bsEcdhPSI->sendEcdhCipher(sendEcdhCipherRequest); + bsEcdhPSI->sendEcdhCipher(sendEcdhCipherRequest); + + auto sendPartnerCipherRequest = std::make_shared(); + sendPartnerCipherRequest->taskID = taskID; + sendPartnerCipherRequest->offset = offset; + sendPartnerCipherRequest->size = ciphers.size(); + sendPartnerCipherRequest->total = total; + + for (uint32_t i = offset; i < ciphers.size() + offset; i++) + { + sendPartnerCipherRequest->partnerCiphers.push_back( + cache->genCipherWithBase64(std::to_string(100000 + i))); + } + bsEcdhPSI->sendPartnerCipher(sendPartnerCipherRequest); + bsEcdhPSI->sendPartnerCipher(sendPartnerCipherRequest); + + offset += ciphers.size(); + } while (offset < total); + + while (bsEcdhPSI->getTaskStatus(getTaskStatusRequest)->data()["status"].asString() == + protocol::toString(TaskStatus::RUNNING)) + { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + BOOST_CHECK(bsEcdhPSI->getTaskStatus(getTaskStatusRequest)->data()["status"].asString() == + protocol::toString(TaskStatus::COMPLETED)); +} + + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h b/cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h new file mode 100644 index 00000000..6bf27d64 --- /dev/null +++ b/cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FakeCM2020PSIFactory.h + * @author: shawnhe + * @date 2022-12-19 + */ +#pragma once + +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "test-utils/FakeFront.h" +#include + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; +using namespace ppc::tools; + +namespace ppc::test +{ +class FakeCM2020PSIImpl : public CM2020PSIImpl +{ +public: + using Ptr = std::shared_ptr; + FakeCM2020PSIImpl(CM2020PSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : CM2020PSIImpl(_config, _idleTimeMs) + { + m_enableOutputExists = true; + } + ~FakeCM2020PSIImpl() override = default; +}; + +class FakeCM2020PSIFactory : public CM2020PSIFactory +{ +public: + using Ptr = std::shared_ptr; + + FakeCM2020PSIFactory() + : m_front(std::make_shared()), + m_dataResourceLoader(std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), + m_threadPool(std::make_shared("cm2018-psi", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared(); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } + + ~FakeCM2020PSIFactory() override = default; + + CM2020PSIImpl::Ptr createCM2020PSI(std::string const& _selfParty) + { + auto config = std::make_shared( + _selfParty, m_front, m_cryptoBox, m_threadPool, m_dataResourceLoader, 1); + + return std::make_shared(config); + } + + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + +private: + FakeFront::Ptr m_front; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; + CryptoBox::Ptr m_cryptoBox; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp new file mode 100644 index 00000000..866bcf54 --- /dev/null +++ b/cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp @@ -0,0 +1,326 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestCM2020Impl.cpp + * @author: shawnhe + * @date 2022-12-19 + */ + +#include "FakeCM2020PSIFactory.h" +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "ppc-psi/src/cm2020-psi/CM2020PSIImpl.h" +#include "test-utils/FakeFront.h" +#include "test-utils/FileTool.h" +#include "test-utils/TaskMock.h" +#include +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::tools; +using namespace ppc::protocol; +using namespace ppc::io; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(CM2020ImplTest, TestPromptFixture) + +void checkTaskPSIResult(DataResourceLoader::Ptr _resourceLoader, + ppc::protocol::Task::ConstPtr _task, uint64_t expectedResultSize, + std::vector _expectedPSIResult) +{ + auto outputDesc = _task->selfParty()->dataResource()->outputDesc(); + auto reader = _resourceLoader->loadReader(outputDesc, DataSchema::String, false); + // get all result + auto result = reader->next(-1); + // check the result + std::cout << "### result size:" << result->size() << std::endl; + std::cout << "### expectedResultSize: " << expectedResultSize << std::endl; + BOOST_CHECK(result->size() == expectedResultSize); + if (_expectedPSIResult.empty()) + { + return; + } + for (uint64_t i = 0; i < result->size(); i++) + { + BOOST_CHECK(result->get(i) == _expectedPSIResult.at(i)); + } +} + +void testCM2020PSI(FakeCM2020PSIFactory::Ptr _factory, CM2020PSIImpl::Ptr _sender, + CM2020PSIImpl::Ptr _receiver, ppc::protocol::Task::ConstPtr _senderPsiTask, + ppc::protocol::Task::ConstPtr _receiverPsiTask, + std::vector const& _expectedPSIResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + std::atomic flag = 0; + _sender->asyncRunTask(_senderPsiTask, [_senderPsiTask, _expectedSuccess, _expectedErrorCode, + &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _senderPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); + } + flag++; + }); + + _receiver->asyncRunTask(_receiverPsiTask, + [_receiverPsiTask, _expectedSuccess, &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _receiverPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + flag++; + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + _sender->start(); + _receiver->start(); + + // wait for the task finish and check + while (flag < 2) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + _sender->stop(); + _receiver->stop(); + + if (_expectedSuccess && !_expectedPSIResult.empty()) + { + checkTaskPSIResult(_factory->resourceLoader(), _receiverPsiTask, _expectedPSIResult.size(), + _expectedPSIResult); + } +} + + +void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _params, + bool _syncResults, PartyResource::Ptr _senderParty, PartyResource::Ptr _receiverParty, + std::vector const& _expectedPSIResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = _senderParty->id(); + auto senderPSI = factory->createCM2020PSI(senderAgencyName); + + // fake the receiver + std::string receiverAgencyName = _receiverParty->id(); + auto receiverPSI = factory->createCM2020PSI(receiverAgencyName); + + // register the server-psi into the front + factory->front()->registerCM2020(senderAgencyName, senderPSI); + factory->front()->registerCM2020(receiverAgencyName, receiverPSI); + + // trigger the psi task + auto senderPSITask = std::make_shared(senderAgencyName); + senderPSITask->setId(_taskID); + senderPSITask->setParam(_params); + senderPSITask->setSelf(_senderParty); + senderPSITask->addParty(_receiverParty); + senderPSITask->setSyncResultToPeer(_syncResults); + senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::CM_PSI_2PC); + + auto receiverPSITask = std::make_shared(receiverAgencyName); + receiverPSITask->setId(_taskID); + receiverPSITask->setParam(_params); + receiverPSITask->setSelf(_receiverParty); + receiverPSITask->addParty(_senderParty); + receiverPSITask->setSyncResultToPeer(_syncResults); + receiverPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::CM_PSI_2PC); + + testCM2020PSI(factory, senderPSI, receiverPSI, senderPSITask, receiverPSITask, + _expectedPSIResult, _expectedSuccess, _expectedErrorCode); +} + +BOOST_AUTO_TEST_CASE(testNormalCM2020PSICase) +{ + std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + + uint32_t count = 513; + boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + prepareInputs(senderPath, count, receiverPath, count, count); + + auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Server, "sender", + "senderPartyResource", "sender_inputs", DataResourceType::FILE, senderPath); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(senderOutputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + auto receiverParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, "receiver", + "receiverPartyResource", "receiver_inputs", DataResourceType::FILE, receiverPath); + auto receiverOutputDesc = std::make_shared(); + receiverOutputDesc->setPath(receiverOutputPath); + receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + + std::vector expectedResult; + for (uint32_t i = 0; i < count; i++) + { + expectedResult.emplace_back(std::to_string(100000 + i)); + } + + testCM2020PSIImplFunc( + "0x12345678", "[0]", true, senderParty, receiverParty, expectedResult, true, 0); + testCM2020PSIImplFunc( + "0x12345678", "[1]", true, senderParty, receiverParty, expectedResult, true, 0); + testCM2020PSIImplFunc( + "0x12345678", "[0]", false, senderParty, receiverParty, expectedResult, true, 0); + testCM2020PSIImplFunc( + "0x12345678", "[1]", false, senderParty, receiverParty, expectedResult, true, 0); +} + +BOOST_AUTO_TEST_CASE(testBigDataCM2020PSICase) +{ + std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + + uint32_t count = 1000000, common = 100; + + boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + prepareInputs(senderPath, count, receiverPath, count, common); + + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, senderPath); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(senderOutputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "receiver", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, receiverPath); + auto receiverOutputDesc = std::make_shared(); + receiverOutputDesc->setPath(receiverOutputPath); + receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + + std::vector expectedResult; + for (uint32_t i = 0; i < common; i++) + { + expectedResult.emplace_back(std::to_string(100000 + count - common + i)); + } + + testCM2020PSIImplFunc( + "0x12345678", "[0]", false, senderParty, receiverParty, expectedResult, true, 0); +} + +BOOST_AUTO_TEST_CASE(testErrorCM2020PSICase) +{ + std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + + uint32_t count = 1234; + boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + prepareInputs(senderPath, count, receiverPath, count, count); + + auto senderParty = + mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", "sender_inputs", + DataResourceType::FILE, "../../../ppc-psi/tests/cm2020-psi/data/not_existed.csv"); + + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(senderOutputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "receiver", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, receiverPath); + auto receiverOutputDesc = std::make_shared(); + receiverOutputDesc->setPath(receiverOutputPath); + receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + + testCM2020PSIImplFunc("0x12345678", "[0]", false, senderParty, receiverParty, + std::vector(), false, -3000); +} + +BOOST_AUTO_TEST_CASE(testRawDataCM2020PSICase) +{ + std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + + uint32_t count = 1234; + boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + prepareInputs(senderPath, count, receiverPath, count, count); + + auto reader = std::make_shared(senderPath); + auto data = reader->next(-1); + std::vector> sRawData(1); + for (uint32_t i = 0; i < data->size(); i++) + { + sRawData[0].emplace_back(data->get(i)); + } + + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, senderPath, sRawData); + + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(senderOutputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + reader = std::make_shared(senderPath); + data = reader->next(-1); + std::vector> rRawData(1); + for (uint32_t i = 0; i < data->size(); i++) + { + rRawData[0].emplace_back(data->get(i)); + } + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "receiver", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, receiverPath, rRawData); + auto receiverOutputDesc = std::make_shared(); + receiverOutputDesc->setPath(receiverOutputPath); + receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + + testCM2020PSIImplFunc( + "0x12345678", "[1]", true, senderParty, receiverParty, std::vector(), true, 0); +} + +BOOST_AUTO_TEST_CASE(testPrepareInptsCase) +{ + std::string senderPath = "/tmp/test0.csv"; + std::string receiverPath = "/tmp/test1.csv"; + + uint32_t count = 1000 * 10000; + prepareInputs(senderPath, count, receiverPath, count, 500000); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/labeled-psi/DataTools.h b/cpp/ppc-psi/tests/labeled-psi/DataTools.h new file mode 100644 index 00000000..dbeabf52 --- /dev/null +++ b/cpp/ppc-psi/tests/labeled-psi/DataTools.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file DataTools.cpp + * @author: shawnhe + * @date 2022-11-25 + */ + +#pragma +#include "ppc-framework/io/DataBatch.h" +#include +#include + +namespace ppc::psi +{ + +inline void genItemsLabels( + ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels, uint32_t _size) +{ + std::vector items(_size); + std::vector labels(_size); + for (uint32_t i = 0; i < _size; ++i) + { + std::string item = std::to_string(i); + items[i] = bcos::bytes(item.begin(), item.end()); + labels[i] = bcos::bytes(item.begin(), item.end()); + } + _items->setData(std::move(items)); + _labels->setData(std::move(labels)); +} + +} // namespace ppc::psi diff --git a/cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h b/cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h new file mode 100644 index 00000000..ab863356 --- /dev/null +++ b/cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h @@ -0,0 +1,87 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FakeLabeledPSIFactory.h + * @author: shawnhe + * @date 2022-12-26 + */ +#pragma once + +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "test-utils/FakeFront.h" +#include + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; +using namespace ppc::tools; + +namespace ppc::test +{ +class FakeLabeledPSIImpl : public LabeledPSIImpl +{ +public: + using Ptr = std::shared_ptr; + FakeLabeledPSIImpl(LabeledPSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : LabeledPSIImpl(_config, _idleTimeMs) + { + m_enableOutputExists = true; + } + ~FakeLabeledPSIImpl() override = default; +}; + +class FakeLabeledPSIFactory : public LabeledPSIFactory +{ +public: + using Ptr = std::shared_ptr; + + FakeLabeledPSIFactory() + : m_front(std::make_shared()), + m_dataResourceLoader(std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), + m_threadPool(std::make_shared("labeled-psi", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared(); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } + + ~FakeLabeledPSIFactory() override = default; + + LabeledPSIImpl::Ptr createLabeledPSI(std::string const& _selfParty) + { + auto config = std::make_shared( + _selfParty, m_front, m_cryptoBox, m_threadPool, m_dataResourceLoader, 1); + return std::make_shared(config); + } + + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + +private: + FakeFront::Ptr m_front; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + bcos::ThreadPool::Ptr m_threadPool; + CryptoBox::Ptr m_cryptoBox; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp new file mode 100644 index 00000000..c7ed5e01 --- /dev/null +++ b/cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp @@ -0,0 +1,321 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestCM2020Impl.cpp + * @author: shawnhe + * @date 2022-12-26 + */ + +#include "FakeLabeledPSIFactory.h" +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "ppc-psi/src/labeled-psi/LabeledPSIImpl.h" +#include "ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h" +#include "test-utils/FileTool.h" +#include "test-utils/TaskMock.h" +#include +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::tools; +using namespace ppc::protocol; +using namespace ppc::io; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(LabeledPSIImplTest, TestPromptFixture) + +void runSetup(LabeledPSIImpl::Ptr _sender) +{ + std::string senderPath = "../../../ppc-psi/tests/labeled-psi/data/sender.csv"; + + uint32_t count = 10000; + boost::filesystem::create_directory("../../../ppc-psi/tests/labeled-psi/data"); + prepareItemsAndLabels(senderPath, count); + + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, senderPath); + auto senderOutputDesc = std::make_shared(); + + // fake the sender + std::string senderAgencyName = senderParty->id(); + + auto senderPSITask = std::make_shared(senderAgencyName); + senderPSITask->setId("0x00000000"); + senderPSITask->setParam(R"(["setup_sender_db","32"])"); + senderPSITask->setSelf(senderParty); + senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC); + + std::atomic flag = 0; + _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + flag++; + }); + + while (flag < 1) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +void saveCache(LabeledPSIImpl::Ptr _sender) +{ + std::string cachePath = "../../../ppc-psi/tests/labeled-psi/data/sender_cache.txt"; + boost::filesystem::create_directory("../../../ppc-psi/tests/labeled-psi/data"); + + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, ""); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(cachePath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + + // fake the sender + std::string senderAgencyName = senderParty->id(); + + auto senderPSITask = std::make_shared(senderAgencyName); + senderPSITask->setId("0x00000012"); + senderPSITask->setParam(R"(["save_sender_cache"])"); + senderPSITask->setSelf(senderParty); + senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC); + + std::atomic flag = 0; + _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + flag++; + }); + + while (flag < 1) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +void loadCache(LabeledPSIImpl::Ptr _sender) +{ + std::string cachePath = "../../../ppc-psi/tests/labeled-psi/data/sender_cache.txt"; + + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, cachePath); + + // fake the sender + std::string senderAgencyName = senderParty->id(); + + auto senderPSITask = std::make_shared(senderAgencyName); + senderPSITask->setId("0x00000034"); + senderPSITask->setParam(R"(["load_sender_cache"])"); + senderPSITask->setSelf(senderParty); + senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC); + + std::atomic flag = 0; + _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + flag++; + }); + + while (flag < 1) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +void testLabeledPSI(LabeledPSIImpl::Ptr _sender, LabeledPSIImpl::Ptr _receiver, + ppc::protocol::Task::ConstPtr _senderPsiTask, ppc::protocol::Task::ConstPtr _receiverPsiTask, + bool _expectedSuccess, int _expectedErrorCode = 0, int _hit = 0) +{ + std::atomic flag = 0; + + _receiver->asyncRunTask(_receiverPsiTask, + [_receiverPsiTask, _expectedSuccess, _hit, &flag](protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _receiverPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + + LabeledPSIResult::Ptr labeledPSIResult = + std::dynamic_pointer_cast(_response); + BOOST_CHECK(labeledPSIResult->getOutputs()[0].size() == uint(_hit)); + // std::cout << labeledPSIResult->getOutputs() << std::endl; + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + flag++; + }); + + _sender->asyncRunTask(_senderPsiTask, [_senderPsiTask, _expectedSuccess, _expectedErrorCode, + &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _senderPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); + } + flag++; + }); + + // wait for the task finish and check + while (flag < 2) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +void testLabeledPSIImplFunc(const std::string& _taskID, const std::string& _params, + PartyResource::Ptr _senderParty, PartyResource::Ptr _receiverParty, bool _ready, bool _testLoad, + bool _expectedSuccess, int _expectedErrorCode = 0, int _hit = 0) +{ + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = _senderParty->id(); + auto senderPSI = factory->createLabeledPSI(senderAgencyName); + senderPSI->start(); + if (_ready) + { + if (_testLoad) + { + loadCache(senderPSI); + } + else + { + runSetup(senderPSI); + saveCache(senderPSI); + } + } + + // fake the receiver + std::string receiverAgencyName = _receiverParty->id(); + auto receiverPSI = factory->createLabeledPSI(receiverAgencyName); + receiverPSI->start(); + + // register the server-psi into the front + factory->front()->registerLabeledPSI(senderAgencyName, senderPSI); + factory->front()->registerLabeledPSI(receiverAgencyName, receiverPSI); + + // trigger the psi task + auto senderPSITask = std::make_shared(senderAgencyName); + senderPSITask->setId(_taskID); + senderPSITask->setParam(_params); + senderPSITask->setSelf(_senderParty); + senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC); + senderPSITask->addParty(_receiverParty); + + auto receiverPSITask = std::make_shared(receiverAgencyName); + receiverPSITask->setId(_taskID); + receiverPSITask->setSelf(_receiverParty); + receiverPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC); + receiverPSITask->addParty(_senderParty); + + testLabeledPSI(senderPSI, receiverPSI, senderPSITask, receiverPSITask, _expectedSuccess, + _expectedErrorCode, _hit); + + senderPSI->stop(); + receiverPSI->stop(); +} + + +BOOST_AUTO_TEST_CASE(testSenderSetup) +{ + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = "sender"; + auto senderPSI = factory->createLabeledPSI(senderAgencyName); + + senderPSI->start(); + + runSetup(senderPSI); + + senderPSI->stop(); +} + + +BOOST_AUTO_TEST_CASE(testLabeledPSICase) +{ + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, ""); + + std::vector> rRawData(1); + for (int i = 0; i < MAX_QUERY_SIZE / 4; i++) + { + rRawData[0].emplace_back(std::to_string(100000 + i)); + rRawData[0].emplace_back(std::to_string(100000000 + i)); + } + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, "", rRawData); + + testLabeledPSIImplFunc("0x12345678", "[\"run_labeled_psi\"]", senderParty, receiverParty, true, + false, true, 0, MAX_QUERY_SIZE / 4); +} + +BOOST_AUTO_TEST_CASE(testLabeledPSINotReady) +{ + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, ""); + + std::vector> rRawData(1); + rRawData[0].emplace_back("100018"); + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "receiver", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, "", rRawData); + + testLabeledPSIImplFunc("0x12345678", "[\"run_labeled_psi\"]", senderParty, receiverParty, false, + false, false, -2001); +} + +BOOST_AUTO_TEST_CASE(testLabeledPSILoacdCache) +{ + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, ""); + + std::vector> rRawData(1); + for (int i = 0; i < MAX_QUERY_SIZE / 4; i++) + { + rRawData[0].emplace_back(std::to_string(100000 + i)); + rRawData[0].emplace_back(std::to_string(100000000 + i)); + } + + auto receiverParty = mockParty(uint16_t(PartyType::Client), "", "receiverPartyResource", + "receiver_inputs", DataResourceType::FILE, "", rRawData); + + testLabeledPSIImplFunc("0x12345678", "[\"run_labeled_psi\"]", senderParty, receiverParty, true, + true, true, 0, MAX_QUERY_SIZE / 4); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp b/cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp new file mode 100644 index 00000000..d6c50960 --- /dev/null +++ b/cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestSenderDB.cpp + * @author: shawnhe + * @date 2022-11-25 + */ + +#include "DataTools.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/oprf/EcdhOprf.h" +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "ppc-psi/src/labeled-psi/core/LabeledPSIParams.h" +#include "ppc-psi/src/labeled-psi/core/SenderDB.h" +#include "test-utils/TaskMock.h" +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::protocol; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(SenderDBTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(TestPsiParams) +{ + auto params1 = getPsiParams(1024); + BOOST_CHECK(params1.table_params().max_items_per_bin == 42); + BOOST_CHECK(params1.item_params().felts_per_item == 6); + BOOST_CHECK(params1.seal_params().poly_modulus_degree() == 2048); + BOOST_CHECK(params1.query_params().ps_low_degree == 0); + auto params2 = getPsiParams(360000); + BOOST_CHECK(params2.table_params().max_items_per_bin == 228); + BOOST_CHECK(params2.item_params().felts_per_item == 6); + BOOST_CHECK(params2.seal_params().poly_modulus_degree() == 8192); + BOOST_CHECK(params2.query_params().ps_low_degree == 0); + auto params3 = getPsiParams(2000000); + BOOST_CHECK(params3.table_params().max_items_per_bin == 782); + BOOST_CHECK(params3.item_params().felts_per_item == 4); + BOOST_CHECK(params3.seal_params().poly_modulus_degree() == 8192); + BOOST_CHECK(params3.query_params().ps_low_degree == 26); + auto params4 = getPsiParams(20000000); + BOOST_CHECK(params4.table_params().max_items_per_bin == 8100); + BOOST_CHECK(params4.item_params().felts_per_item == 4); + BOOST_CHECK(params4.seal_params().poly_modulus_degree() == 8192); + BOOST_CHECK(params4.query_params().ps_low_degree == 310); +} + +BOOST_AUTO_TEST_CASE(TestSenderDB) +{ + uint32_t count = (1 << 10); + auto items = std::make_shared(); + items->setDataSchema(ppc::io::DataSchema::Bytes); + auto labels = std::make_shared(); + labels->setDataSchema(ppc::io::DataSchema::Bytes); + genItemsLabels(items, labels, count); + BOOST_CHECK(items->size() == count); + + auto hashImpl = std::make_shared(); + auto eccImpl = std::make_shared(); + auto oprfServer = std::make_shared(256, hashImpl, eccImpl); + auto senderDB = std::make_shared(getPsiParams(count), oprfServer, 16, 16, false); + + senderDB->setData(items, labels); + BOOST_CHECK(senderDB->getItemCount() == count); +} + +BOOST_AUTO_TEST_CASE(TestSenderCache) +{ + uint32_t count = (1 << 15); + auto items = std::make_shared(); + items->setDataSchema(ppc::io::DataSchema::Bytes); + auto labels = std::make_shared(); + labels->setDataSchema(ppc::io::DataSchema::Bytes); + genItemsLabels(items, labels, count); + BOOST_CHECK(items->size() == count); + + auto hashImpl = std::make_shared(); + auto eccImpl = std::make_shared(); + auto oprfServer = std::make_shared(256, hashImpl, eccImpl); + auto senderDB = std::make_shared(getPsiParams(count), oprfServer, 16, 16, false); + + senderDB->setData(items, labels); + + bcos::bytes out; + senderDB->saveToBytes(out); + auto newSenderDB = SenderDB::loadFromBytes(oprfServer, out); + + BOOST_CHECK(newSenderDB->getItemCount() == count); + BOOST_CHECK(senderDB->getParams().to_string() == newSenderDB->getParams().to_string()); + BOOST_CHECK(senderDB->isCompressed() == newSenderDB->isCompressed()); + BOOST_CHECK(senderDB->getItemCount() == newSenderDB->getItemCount()); + BOOST_CHECK(senderDB->getHashedItems().size() == newSenderDB->getHashedItems().size()); + BOOST_CHECK(senderDB->getBinBundleCount() == newSenderDB->getBinBundleCount()); +} + + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/main.cpp b/cpp/ppc-psi/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-psi/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp new file mode 100644 index 00000000..862d57f9 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestEcdhImpl.cpp + * @author: yujiechen + * @date 2022-12-29 + */ +#include "mock/Common.h" +#include "mock/EcdhPSIFixture.h" +// Note: it's better not to depends on the task-impl +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "test-utils/TaskMock.h" +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::tools; +using namespace ppc::protocol; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(EcdhPSIImplTest, TestPromptFixture) +void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataSource, + std::string const& _clientPSIDataSource, std::string const& _resultDataPath, + bool _expectOfflineSuccess, bool _expectPSISuccess, + std::vector& _expectedPSIResult, int _expectedErrorCode = 0, + bool _mismatchedTaskID = false) +{ + auto factory = std::make_shared(); + // the server config, with default setting + auto serverConfig = std::make_shared(); + serverConfig->mutableEcdhPSIConfig().dataBatchSize = _dataBatchSize; + serverConfig->setPrivateKey(factory->cryptoBox()->eccCrypto()->generateRandomScalar()); + + // the client config, with default setting + auto clientConfig = std::make_shared(); + clientConfig->mutableEcdhPSIConfig().dataBatchSize = _dataBatchSize; + clientConfig->setPrivateKey(factory->cryptoBox()->eccCrypto()->generateRandomScalar()); + + // fake the server + std::string serverAgencyName = "server"; + auto serverPSI = factory->createEcdhPSI(serverAgencyName, serverConfig); + + // fake the client + std::string clientAgencyName = "client"; + auto clientPSI = factory->createEcdhPSI(clientAgencyName, clientConfig); + + std::vector agencyList = {serverAgencyName, clientAgencyName}; + serverPSI->psiConfig()->updateAgenyList(agencyList); + clientPSI->psiConfig()->updateAgenyList(agencyList); + + // register the server-psi into the front + factory->front()->registerEcdhPSI(serverAgencyName, serverPSI); + factory->front()->registerEcdhPSI(clientAgencyName, clientPSI); + // the server party + std::string resourceID = "dataResource"; + std::string clientResourceID = resourceID; + std::string serverResourceID = resourceID; + auto serverParty = mockParty((uint16_t)PartyType::Server, serverAgencyName, "serverParty", + serverResourceID, DataResourceType::FILE, _serverPSIDataSource); + // the clientParty + auto clientParty = mockParty((uint16_t)PartyType::Client, clientAgencyName, "clientParty", + clientResourceID, DataResourceType::FILE, _clientPSIDataSource); + auto outputDesc = std::make_shared(); + outputDesc->setPath(_resultDataPath); + clientParty->mutableDataResource()->setOutputDesc(outputDesc); + // generate client task + auto clientPSITask = std::make_shared(clientAgencyName); + std::string taskID = "runPSI"; + clientPSITask->setId(taskID); + clientPSITask->setType((int8_t)TaskType::PSI); + clientPSITask->setAlgorithm((int8_t)PSIAlgorithmType::ECDH_PSI_2PC); + clientPSITask->setSelf(clientParty); + clientPSITask->addParty(serverParty); + + // generate server task + auto serverPSITask = std::make_shared(serverAgencyName); + serverPSITask->setSelf(serverParty); + if (_mismatchedTaskID) + { + serverPSITask->setId(taskID + "-mismatch"); + } + else + { + serverPSITask->setId(taskID); + } + serverPSITask->setType((int8_t)TaskType::PSI); + serverPSITask->setAlgorithm((int8_t)PSIAlgorithmType::ECDH_PSI_2PC); + serverPSITask->addParty(clientParty); + // run psi + testPSI(factory, serverPSI, clientPSI, serverPSITask, clientPSITask, _expectPSISuccess, + _expectedPSIResult, _expectedErrorCode); +} + +void testNormalCase(std::string const& _outputPrefix, int64_t _dataBatchSize) +{ + std::string serverDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string clientDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string outputPath = _outputPrefix + "psiResult.txt"; + std::vector expectedResult; + for (int i = 1; i < 49; i++) + { + expectedResult.emplace_back(std::to_string(i)); + } + testEcdhImplFunc( + _dataBatchSize, serverDataPath, clientDataPath, outputPath, true, true, expectedResult, 0); +} + +BOOST_AUTO_TEST_CASE(testNormalEcdhPSICase) +{ + std::cout << "### testNormalEcdhPSICase, batch-size: " << 1000 << std::endl; + testNormalCase("normal-ecdh-psi-1000-batch-", 1000); + std::cout << "### testNormalEcdhPSICase, batch-size: " << 1000 << " success!" << std::endl; + + std::cout << "### testNormalEcdhPSICase, batch-size: " << 20 << std::endl; + testNormalCase("normal-ecdh-psi-20-batch-", 20); + std::cout << "### testNormalEcdhPSICase, batch-size: " << 20 << " success!" << std::endl; +} + +BOOST_AUTO_TEST_CASE(testECDHMissingResourceDataCase) +{ + std::string serverPSIDataPath = + "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate-missing.txt"; + std::string clientPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string outputPath = "psiResult_MissingResourceData.txt"; + std::vector expectedResult; + testEcdhImplFunc(1000, serverPSIDataPath, clientPSIDataPath, outputPath, true, false, + expectedResult, (int)PSIRetCode::HandleTaskError); +} +BOOST_AUTO_TEST_CASE(testECDHMisMatchTaskID) +{ + std::string serverPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string clientPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string outputPath = "psiResult_MismatchTaskID.txt"; + std::vector expectedResult; + testEcdhImplFunc(1000, serverPSIDataPath, clientPSIDataPath, outputPath, true, false, + expectedResult, (int)PSIRetCode::TaskNotFound, true); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp new file mode 100644 index 00000000..30ba8ff2 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp @@ -0,0 +1,261 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestRA2018Impl.cpp + * @author: yujiechen + * @date 2022-11-16 + */ +#include "mock/Common.h" +#include "mock/RA2018PSIFixture.h" +// Note: it's better not to depends on the task-impl +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "test-utils/TaskMock.h" +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::tools; +using namespace ppc::protocol; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(RA2018ImplTest, TestPromptFixture) +// offline evaluate +void testRA2018FullEvaluate(FakeRA2018PSIFactory::Ptr _factory, RA2018PSIImpl::Ptr _server, + ppc::protocol::Task::ConstPtr _evaluateTask, bool _expectedSuccess) +{ + // test the server offline-evaluate + std::promise completedFuture; + _server->asyncRunTask(_evaluateTask, [_evaluateTask, _expectedSuccess, &completedFuture]( + ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _evaluateTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + completedFuture.set_value(true); + }); + completedFuture.get_future().get(); +} + +void testRA2018PSIImplFunc(int _dataBatchSize, CuckoofilterOption::Ptr option, + std::string const& _offlineDataPath, std::string const& _psiDataSource, + std::string const& _resultDataPath, bool _expectOfflineSuccess, bool _expectPSISuccess, + std::vector& _expectedPSIResult, int _expectedErrorCode = 0, + bool _misMatchDataSource = false, std::string const& _offlineInsertDataPath = "", + std::vector const& _appendIntersectionData = std::vector()) +{ + auto factory = std::make_shared(); + // the server config, with default cache setting + auto serverConfig = std::make_shared(); + auto& mutableServerConfig = serverConfig->mutableRA2018PSIConfig(); + mutableServerConfig.dataBatchSize = _dataBatchSize; + + serverConfig->setPrivateKey(factory->cryptoBox()->eccCrypto()->generateRandomScalar()); + mutableServerConfig.cuckooFilterOption = option; + + // the client config, with default cache setting + auto clientConfig = std::make_shared(); + auto& mutableClientConfig = clientConfig->mutableRA2018PSIConfig(); + mutableClientConfig.cuckooFilterOption = option; + mutableClientConfig.dataBatchSize = _dataBatchSize; + + // fake the server + std::string serverAgencyName = "server"; + auto serverPSI = factory->createRA2018PSI(serverAgencyName, serverConfig); + + // fake the client + std::string clientAgencyName = "client"; + auto clientPSI = factory->createRA2018PSI(clientAgencyName, clientConfig); + // register the server-psi into the front + factory->front()->registerRA2018(serverAgencyName, serverPSI); + factory->front()->registerRA2018(clientAgencyName, clientPSI); + + // generate offline-evaluate task + std::string resourceID = "dataResource"; + std::string clientResourceID = resourceID; + std::string serverResourceID = resourceID; + auto serverParty = mockParty((uint16_t)PartyType::Server, serverAgencyName, "serverParty", + serverResourceID, DataResourceType::FILE, _offlineDataPath); + + auto offlineFullEvaluateTask = std::make_shared(serverAgencyName); + offlineFullEvaluateTask->setId("offlineFullEvaluate"); + offlineFullEvaluateTask->setSelf(serverParty); + // insert operation + std::string param = "[\"data_preprocessing\", 0]"; + offlineFullEvaluateTask->setParam(param); + // the server trigger offline-evaluate + testRA2018FullEvaluate(factory, serverPSI, offlineFullEvaluateTask, _expectOfflineSuccess); + + /// trigger the psi task + // the client task + if (_misMatchDataSource) + { + serverResourceID = "miss-match-" + resourceID; + } + // update the server-resource-id + serverParty->mutableDataResource()->setResourceID(serverResourceID); + auto clientParty = mockParty((uint16_t)PartyType::Client, clientAgencyName, "clientParty", + clientResourceID, DataResourceType::FILE, _psiDataSource); + auto outputDesc = std::make_shared(); + outputDesc->setPath(_resultDataPath); + clientParty->mutableDataResource()->setOutputDesc(outputDesc); + auto clientPSITask = std::make_shared(clientAgencyName); + std::string taskID = "runPSI"; + clientPSITask->setId(taskID); + clientPSITask->setSelf(clientParty); + clientPSITask->addParty(serverParty); + param = "[\"ra2018_psi\"]"; + clientPSITask->setParam(param); + + // the server task + auto serverPSITask = std::make_shared(serverAgencyName); + serverPSITask->setSelf(serverParty); + serverPSITask->setId(taskID); + serverPSITask->setParam(param); + serverPSITask->addParty(clientParty); + testPSI(factory, serverPSI, clientPSI, serverPSITask, clientPSITask, _expectPSISuccess, + _expectedPSIResult, _expectedErrorCode); + + // test insert new data into the cuckoo-filter + if (_offlineInsertDataPath.empty() || _appendIntersectionData.empty()) + { + return; + } + std::cout << "### test offline-fullevaluate-insert" << std::endl; + offlineFullEvaluateTask->selfParty()->mutableDataResource()->mutableDesc()->setPath( + _offlineInsertDataPath); + offlineFullEvaluateTask->setId("insert_" + _offlineDataPath); + // the server trigger offline-evaluate + testRA2018FullEvaluate(factory, serverPSI, offlineFullEvaluateTask, _expectOfflineSuccess); + std::cout << "### test offline-fullevaluate-insert finished" << std::endl; + + // trigger psi task again + std::cout << "### test psi-insert" << std::endl; + taskID = "insert_runPSI"; + serverPSITask->setId(taskID); + clientPSITask->setId(taskID); + auto expectedResult = _expectedPSIResult; + expectedResult.insert( + expectedResult.end(), _appendIntersectionData.begin(), _appendIntersectionData.end()); + testPSI(factory, serverPSI, clientPSI, serverPSITask, clientPSITask, _expectPSISuccess, + expectedResult, _expectedErrorCode); + std::cout << "### test psi-insert success" << std::endl; + + // delete the evaluated data + std::cout << "### test offline-fullevaluate-delete" << std::endl; + param = "[\"data_preprocessing\", 1]"; + offlineFullEvaluateTask->setId("delete-" + _offlineDataPath); + offlineFullEvaluateTask->setParam(param); + // the server trigger offline-evaluate + testRA2018FullEvaluate(factory, serverPSI, offlineFullEvaluateTask, _expectOfflineSuccess); + std::cout << "### test offline-fullevaluate-delete finished" << std::endl; + + // trigger psi after delete + std::cout << "### test psi-delete" << std::endl; + taskID = "delete_runPSI"; + serverPSITask->setId(taskID); + clientPSITask->setId(taskID); + testPSI(factory, serverPSI, clientPSI, serverPSITask, clientPSITask, _expectPSISuccess, + _expectedPSIResult, _expectedErrorCode); + std::cout << "### test psi-delete success" << std::endl; +} + +void testNormalCase(std::string const& _outputPrefix, CuckoofilterOption::Ptr option, + uint64_t _dataBatchSize = 10000) +{ + std::string dataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string appendPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt"; + std::string outputPath = _outputPrefix + "psiResult.txt"; + std::vector expectedResult; + for (int i = 1; i < 49; i++) + { + expectedResult.emplace_back(std::to_string(i)); + } + testRA2018PSIImplFunc(_dataBatchSize, option, dataPath, psiPath, outputPath, true, true, + expectedResult, 0, false, appendPath, std::vector()); + + // with psi2.txt as input + psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi2.txt"; + expectedResult.clear(); + for (int i = 2000; i < 4000; i += 3) + { + expectedResult.emplace_back(std::to_string(i)); + } + outputPath = _outputPrefix + "psiResult2.txt"; + std::vector appendIntersectionData; + std::string prefix = "abcdefg"; + for (int i = 0; i < 4; i++) + { + appendIntersectionData.emplace_back(prefix + std::to_string(i)); + } + appendIntersectionData.emplace_back("abcdefg0"); + testRA2018PSIImplFunc(_dataBatchSize, option, dataPath, psiPath, outputPath, true, true, + expectedResult, 0, false, appendPath, appendIntersectionData); +} +// the cuckoo-filter not hitted in the server +BOOST_AUTO_TEST_CASE(testMisMatchResourceIDCase) +{ + auto option = std::make_shared(); + option->tagBits = 32; + option->maxKickOutCount = 10; + option->capacity = 200000; + std::string dataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string outputPath = "psiResult_MismatchResource.txt"; + std::vector expectedResult; + testRA2018PSIImplFunc(10000, option, dataPath, psiPath, outputPath, true, false, expectedResult, + (int)PSIRetCode::NotOfflineFullEvaluated, true); +} + +BOOST_AUTO_TEST_CASE(testNormalRA2018PSICase) +{ + // singleCuckooFilter + auto option = std::make_shared(); + option->tagBits = 32; + option->maxKickOutCount = 10; + option->capacity = 200000; + testNormalCase("singleCuckooFilter-", option); + + // with data-batch-size = 100 + testNormalCase("singleCuckooFilter-100-dataBatch-", option, 100); +} + +// test normal case with multiple-cuckoo-filter +BOOST_AUTO_TEST_CASE(testNormalRA2018PSICaseWithMultipleCuckooFilter) +{ + auto option = std::make_shared(); + option->tagBits = 32; + option->maxKickOutCount = 100; + option->capacity = 2000; + testNormalCase("multipleCuckooFilter-", option); + + // with data-batch-size = 100 + testNormalCase("multipleCuckooFilter-100-dataBatch-", option, 100); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp b/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp new file mode 100644 index 00000000..59f2b576 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestRA2018Message.cpp + * @author: yujiechen + * @date 2022-11-16 + */ +#include "mock/RA2018MessageFixture.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-psi/src/ra2018-psi/Common.h" +#include +#include + +using namespace ppc::psi; +using namespace bcos; +using namespace bcos::test; +using namespace ppc::crypto; +using namespace ppc::tools; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(RA2018MessageTest, TestPromptFixture) + +BOOST_AUTO_TEST_CASE(testRA2018MessageImpl) +{ + auto msgFactory = std::make_shared(); + std::string partyID = "selfParty"; + std::string resourceID = "testResource"; + int32_t version = 1000123; + // fake the data + std::vector data; + for (int i = 0; i < 100; i++) + { + std::string item = std::to_string(i) + "werwlerklk"; + data.emplace_back(bcos::bytes(item.begin(), item.end())); + } + + // the evaluate message + std::cout << "#### test EvaluateRequest" << std::endl; + auto msg = msgFactory->createPSIMessage((uint32_t)RA2018PacketType::EvaluateRequest); + fakeAndCheckRA2018Message(msg, msgFactory, (uint32_t)RA2018PacketType::EvaluateRequest, partyID, + resourceID, version, data); + std::cout << "#### test EvaluateRequest finish" << std::endl; + // the EvaluateResponse message + std::cout << "#### test EvaluateResponse" << std::endl; + msg = msgFactory->createPSIMessage((uint32_t)RA2018PacketType::EvaluateResponse); + fakeAndCheckRA2018Message(msg, msgFactory, (uint32_t)RA2018PacketType::EvaluateResponse, + partyID, resourceID, version, data); + std::cout << "#### test EvaluateResponse finish" << std::endl; + + // fake the cuckooFilterInfo + std::vector cuckooFilters; + auto hashImpl = std::make_shared(); + auto option = std::make_shared(); + option->tagBits = 32; + option->maxKickOutCount = 10; + option->capacity = 20000; + + for (int i = 0; i < 10; i++) + { + auto filterInfo = fakeCuckooFilterInfo(i, hashImpl, option, 10000 + i); + cuckooFilters.emplace_back(filterInfo); + } + // check the CuckooFilterRequest + std::cout << "#### test CuckooFilterRequest" << std::endl; + msg = msgFactory->createRA2018FilterMessage((uint32_t)RA2018PacketType::CuckooFilterRequest); + fakeAndCheckRA2018FilterMessage(msg, msgFactory, + (uint32_t)RA2018PacketType::CuckooFilterRequest, partyID, resourceID, version, data, + cuckooFilters); + std::cout << "#### test CuckooFilterRequest finish" << std::endl; + + // check the CuckooFilterResponse + std::cout << "#### test CuckooFilterResponse" << std::endl; + msg = msgFactory->createRA2018FilterMessage((uint32_t)RA2018PacketType::CuckooFilterResponse); + fakeAndCheckRA2018FilterMessage(msg, msgFactory, + (uint32_t)RA2018PacketType::CuckooFilterResponse, partyID, resourceID, version, data, + cuckooFilters); + std::cout << "#### test CuckooFilterResponse finish" << std::endl; + + // check the CancelTaskNotification + std::cout << "#### test CancelTaskNotification" << std::endl; + msg = + msgFactory->createTaskNotificationMessage((uint32_t)PSIPacketType::CancelTaskNotification); + fakeAndCheckNotificationMessage(msg, msgFactory, + (uint32_t)PSIPacketType::CancelTaskNotification, partyID, resourceID, version, data, -10999, + "-1000999"); + std::cout << "#### test CancelTaskNotification finish" << std::endl; +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt b/cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt new file mode 100644 index 00000000..247d6104 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt @@ -0,0 +1,16 @@ +abcdefg0 +abcdefg1 +abcdefg2 +abcdefg3 +abcdefg3 +abcdefg3 +abcdefg3 +abcdefg4 +abcdefg4 +abcdefg4 +abcdefg4 +abcdefg5 +abcdefg5 +abcdefg5 +abcdefg5 +abcdefg0 diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt b/cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt new file mode 100644 index 00000000..5bcdcf0d --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt @@ -0,0 +1,10002 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 + + +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +380 +381 +382 +383 +384 +385 +386 +387 +388 +389 +390 +391 +392 +393 +394 +395 +396 +397 +398 +399 +400 +401 +402 +403 +404 +405 +406 +407 +408 +409 +410 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +426 +427 +428 +429 +430 +431 +432 +433 +434 +435 +436 +437 +438 +439 +440 +441 +442 +443 +444 +445 +446 +447 +448 +449 +450 +451 +452 +453 +454 +455 +456 +457 +458 +459 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +470 +471 +472 +473 +474 +475 +476 +477 +478 +479 +480 +481 +482 +483 +484 +485 +486 +487 +488 +489 +490 +491 +492 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +517 +518 +519 +520 +521 +522 +523 +524 +525 +526 +527 +528 +529 +530 +531 +532 +533 +534 +535 +536 +537 +538 +539 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +550 +551 +552 +553 +554 +555 +556 +557 +558 +559 +560 +561 +562 +563 +564 +565 +566 +567 +568 +569 +570 +571 +572 +573 +574 +575 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +592 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +604 +605 +606 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +620 +621 +622 +623 +624 +625 +626 +627 +628 +629 +630 +631 +632 +633 +634 +635 +636 +637 +638 +639 +640 +641 +642 +643 +644 +645 +646 +647 +648 +649 +650 +651 +652 +653 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +664 +665 +666 +667 +668 +669 +670 +671 +672 +673 +674 +675 +676 +677 +678 +679 +680 +681 +682 +683 +684 +685 +686 +687 +688 +689 +690 +691 +692 +693 +694 +695 +696 +697 +698 +699 +700 +701 +702 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +714 +715 +716 +717 +718 +719 +720 +721 +722 +723 +724 +725 +726 +727 +728 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +739 +740 +741 +742 +743 +744 +745 +746 +747 +748 +749 +750 +751 +752 +753 +754 +755 +756 +757 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +769 +770 +771 +772 +773 +774 +775 +776 +777 +778 +779 +780 +781 +782 +783 +784 +785 +786 +787 +788 +789 +790 +791 +792 +793 +794 +795 +796 +797 +798 +799 +800 +801 +802 +803 +804 +805 +806 +807 +808 +809 +810 +811 +812 +813 +814 +815 +816 +817 +818 +819 +820 +821 +822 +823 +824 +825 +826 +827 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +841 +842 +843 +844 +845 +846 +847 +848 +849 +850 +851 +852 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +864 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +879 +880 +881 +882 +883 +884 +885 +886 +887 +888 +889 +890 +891 +892 +893 +894 +895 +896 +897 +898 +899 +900 +901 +902 +903 +904 +905 +906 +907 +908 +909 +910 +911 +912 +913 +914 +915 +916 +917 +918 +919 +920 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +932 +933 +934 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +948 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +967 +968 +969 +970 +971 +972 +973 +974 +975 +976 +977 +978 +979 +980 +981 +982 +983 +984 +985 +986 +987 +988 +989 +990 +991 +992 +993 +994 +995 +996 +997 +998 +999 +1000 +1001 +1002 +1003 +1004 +1005 +1006 +1007 +1008 +1009 +1010 +1011 +1012 +1013 +1014 +1015 +1016 +1017 +1018 +1019 +1020 +1021 +1022 +1023 +1024 +1025 +1026 +1027 +1028 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1037 +1038 +1039 +1040 +1041 +1042 +1043 +1044 +1045 +1046 +1047 +1048 +1049 +1050 +1051 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1059 +1060 +1061 +1062 +1063 +1064 +1065 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1075 +1076 +1077 +1078 +1079 +1080 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1091 +1092 +1093 +1094 +1095 +1096 +1097 +1098 +1099 +1100 +1101 +1102 +1103 +1104 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1116 +1117 +1118 +1119 +1120 +1121 +1122 +1123 +1124 +1125 +1126 +1127 +1128 +1129 +1130 +1131 +1132 +1133 +1134 +1135 +1136 +1137 +1138 +1139 +1140 +1141 +1142 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1153 +1154 +1155 +1156 +1157 +1158 +1159 +1160 +1161 +1162 +1163 +1164 +1165 +1166 +1167 +1168 +1169 +1170 +1171 +1172 +1173 +1174 +1175 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1187 +1188 +1189 +1190 +1191 +1192 +1193 +1194 +1195 +1196 +1197 +1198 +1199 +1200 +1201 +1202 +1203 +1204 +1205 +1206 +1207 +1208 +1209 +1210 +1211 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1221 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1229 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1240 +1241 +1242 +1243 +1244 +1245 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1254 +1255 +1256 +1257 +1258 +1259 +1260 +1261 +1262 +1263 +1264 +1265 +1266 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1275 +1276 +1277 +1278 +1279 +1280 +1281 +1282 +1283 +1284 +1285 +1286 +1287 +1288 +1289 +1290 +1291 +1292 +1293 +1294 +1295 +1296 +1297 +1298 +1299 +1300 +1301 +1302 +1303 +1304 +1305 +1306 +1307 +1308 +1309 +1310 +1311 +1312 +1313 +1314 +1315 +1316 +1317 +1318 +1319 +1320 +1321 +1322 +1323 +1324 +1325 +1326 +1327 +1328 +1329 +1330 +1331 +1332 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 +1341 +1342 +1343 +1344 +1345 +1346 +1347 +1348 +1349 +1350 +1351 +1352 +1353 +1354 +1355 +1356 +1357 +1358 +1359 +1360 +1361 +1362 +1363 +1364 +1365 +1366 +1367 +1368 +1369 +1370 +1371 +1372 +1373 +1374 +1375 +1376 +1377 +1378 +1379 +1380 +1381 +1382 +1383 +1384 +1385 +1386 +1387 +1388 +1389 +1390 +1391 +1392 +1393 +1394 +1395 +1396 +1397 +1398 +1399 +1400 +1401 +1402 +1403 +1404 +1405 +1406 +1407 +1408 +1409 +1410 +1411 +1412 +1413 +1414 +1415 +1416 +1417 +1418 +1419 +1420 +1421 +1422 +1423 +1424 +1425 +1426 +1427 +1428 +1429 +1430 +1431 +1432 +1433 +1434 +1435 +1436 +1437 +1438 +1439 +1440 +1441 +1442 +1443 +1444 +1445 +1446 +1447 +1448 +1449 +1450 +1451 +1452 +1453 +1454 +1455 +1456 +1457 +1458 +1459 +1460 +1461 +1462 +1463 +1464 +1465 +1466 +1467 +1468 +1469 +1470 +1471 +1472 +1473 +1474 +1475 +1476 +1477 +1478 +1479 +1480 +1481 +1482 +1483 +1484 +1485 +1486 +1487 +1488 +1489 +1490 +1491 +1492 +1493 +1494 +1495 +1496 +1497 +1498 +1499 +1500 +1501 +1502 +1503 +1504 +1505 +1506 +1507 +1508 +1509 +1510 +1511 +1512 +1513 +1514 +1515 +1516 +1517 +1518 +1519 +1520 +1521 +1522 +1523 +1524 +1525 +1526 +1527 +1528 +1529 +1530 +1531 +1532 +1533 +1534 +1535 +1536 +1537 +1538 +1539 +1540 +1541 +1542 +1543 +1544 +1545 +1546 +1547 +1548 +1549 +1550 +1551 +1552 +1553 +1554 +1555 +1556 +1557 +1558 +1559 +1560 +1561 +1562 +1563 +1564 +1565 +1566 +1567 +1568 +1569 +1570 +1571 +1572 +1573 +1574 +1575 +1576 +1577 +1578 +1579 +1580 +1581 +1582 +1583 +1584 +1585 +1586 +1587 +1588 +1589 +1590 +1591 +1592 +1593 +1594 +1595 +1596 +1597 +1598 +1599 +1600 +1601 +1602 +1603 +1604 +1605 +1606 +1607 +1608 +1609 +1610 +1611 +1612 +1613 +1614 +1615 +1616 +1617 +1618 +1619 +1620 +1621 +1622 +1623 +1624 +1625 +1626 +1627 +1628 +1629 +1630 +1631 +1632 +1633 +1634 +1635 +1636 +1637 +1638 +1639 +1640 +1641 +1642 +1643 +1644 +1645 +1646 +1647 +1648 +1649 +1650 +1651 +1652 +1653 +1654 +1655 +1656 +1657 +1658 +1659 +1660 +1661 +1662 +1663 +1664 +1665 +1666 +1667 +1668 +1669 +1670 +1671 +1672 +1673 +1674 +1675 +1676 +1677 +1678 +1679 +1680 +1681 +1682 +1683 +1684 +1685 +1686 +1687 +1688 +1689 +1690 +1691 +1692 +1693 +1694 +1695 +1696 +1697 +1698 +1699 +1700 +1701 +1702 +1703 +1704 +1705 +1706 +1707 +1708 +1709 +1710 +1711 +1712 +1713 +1714 +1715 +1716 +1717 +1718 +1719 +1720 +1721 +1722 +1723 +1724 +1725 +1726 +1727 +1728 +1729 +1730 +1731 +1732 +1733 +1734 +1735 +1736 +1737 +1738 +1739 +1740 +1741 +1742 +1743 +1744 +1745 +1746 +1747 +1748 +1749 +1750 +1751 +1752 +1753 +1754 +1755 +1756 +1757 +1758 +1759 +1760 +1761 +1762 +1763 +1764 +1765 +1766 +1767 +1768 +1769 +1770 +1771 +1772 +1773 +1774 +1775 +1776 +1777 +1778 +1779 +1780 +1781 +1782 +1783 +1784 +1785 +1786 +1787 +1788 +1789 +1790 +1791 +1792 +1793 +1794 +1795 +1796 +1797 +1798 +1799 +1800 +1801 +1802 +1803 +1804 +1805 +1806 +1807 +1808 +1809 +1810 +1811 +1812 +1813 +1814 +1815 +1816 +1817 +1818 +1819 +1820 +1821 +1822 +1823 +1824 +1825 +1826 +1827 +1828 +1829 +1830 +1831 +1832 +1833 +1834 +1835 +1836 +1837 +1838 +1839 +1840 +1841 +1842 +1843 +1844 +1845 +1846 +1847 +1848 +1849 +1850 +1851 +1852 +1853 +1854 +1855 +1856 +1857 +1858 +1859 +1860 +1861 +1862 +1863 +1864 +1865 +1866 +1867 +1868 +1869 +1870 +1871 +1872 +1873 +1874 +1875 +1876 +1877 +1878 +1879 +1880 +1881 +1882 +1883 +1884 +1885 +1886 +1887 +1888 +1889 +1890 +1891 +1892 +1893 +1894 +1895 +1896 +1897 +1898 +1899 +1900 +1901 +1902 +1903 +1904 +1905 +1906 +1907 +1908 +1909 +1910 +1911 +1912 +1913 +1914 +1915 +1916 +1917 +1918 +1919 +1920 +1921 +1922 +1923 +1924 +1925 +1926 +1927 +1928 +1929 +1930 +1931 +1932 +1933 +1934 +1935 +1936 +1937 +1938 +1939 +1940 +1941 +1942 +1943 +1944 +1945 +1946 +1947 +1948 +1949 +1950 +1951 +1952 +1953 +1954 +1955 +1956 +1957 +1958 +1959 +1960 +1961 +1962 +1963 +1964 +1965 +1966 +1967 +1968 +1969 +1970 +1971 +1972 +1973 +1974 +1975 +1976 +1977 +1978 +1979 +1980 +1981 +1982 +1983 +1984 +1985 +1986 +1987 +1988 +1989 +1990 +1991 +1992 +1993 +1994 +1995 +1996 +1997 +1998 +1999 +2000 +2001 +2002 +2003 +2004 +2005 +2006 +2007 +2008 +2009 +2010 +2011 +2012 +2013 +2014 +2015 +2016 +2017 +2018 +2019 +2020 +2021 +2022 +2023 +2024 +2025 +2026 +2027 +2028 +2029 +2030 +2031 +2032 +2033 +2034 +2035 +2036 +2037 +2038 +2039 +2040 +2041 +2042 +2043 +2044 +2045 +2046 +2047 +2048 +2049 +2050 +2051 +2052 +2053 +2054 +2055 +2056 +2057 +2058 +2059 +2060 +2061 +2062 +2063 +2064 +2065 +2066 +2067 +2068 +2069 +2070 +2071 +2072 +2073 +2074 +2075 +2076 +2077 +2078 +2079 +2080 +2081 +2082 +2083 +2084 +2085 +2086 +2087 +2088 +2089 +2090 +2091 +2092 +2093 +2094 +2095 +2096 +2097 +2098 +2099 +2100 +2101 +2102 +2103 +2104 +2105 +2106 +2107 +2108 +2109 +2110 +2111 +2112 +2113 +2114 +2115 +2116 +2117 +2118 +2119 +2120 +2121 +2122 +2123 +2124 +2125 +2126 +2127 +2128 +2129 +2130 +2131 +2132 +2133 +2134 +2135 +2136 +2137 +2138 +2139 +2140 +2141 +2142 +2143 +2144 +2145 +2146 +2147 +2148 +2149 +2150 +2151 +2152 +2153 +2154 +2155 +2156 +2157 +2158 +2159 +2160 +2161 +2162 +2163 +2164 +2165 +2166 +2167 +2168 +2169 +2170 +2171 +2172 +2173 +2174 +2175 +2176 +2177 +2178 +2179 +2180 +2181 +2182 +2183 +2184 +2185 +2186 +2187 +2188 +2189 +2190 +2191 +2192 +2193 +2194 +2195 +2196 +2197 +2198 +2199 +2200 +2201 +2202 +2203 +2204 +2205 +2206 +2207 +2208 +2209 +2210 +2211 +2212 +2213 +2214 +2215 +2216 +2217 +2218 +2219 +2220 +2221 +2222 +2223 +2224 +2225 +2226 +2227 +2228 +2229 +2230 +2231 +2232 +2233 +2234 +2235 +2236 +2237 +2238 +2239 +2240 +2241 +2242 +2243 +2244 +2245 +2246 +2247 +2248 +2249 +2250 +2251 +2252 +2253 +2254 +2255 +2256 +2257 +2258 +2259 +2260 +2261 +2262 +2263 +2264 +2265 +2266 +2267 +2268 +2269 +2270 +2271 +2272 +2273 +2274 +2275 +2276 +2277 +2278 +2279 +2280 +2281 +2282 +2283 +2284 +2285 +2286 +2287 +2288 +2289 +2290 +2291 +2292 +2293 +2294 +2295 +2296 +2297 +2298 +2299 +2300 +2301 +2302 +2303 +2304 +2305 +2306 +2307 +2308 +2309 +2310 +2311 +2312 +2313 +2314 +2315 +2316 +2317 +2318 +2319 +2320 +2321 +2322 +2323 +2324 +2325 +2326 +2327 +2328 +2329 +2330 +2331 +2332 +2333 +2334 +2335 +2336 +2337 +2338 +2339 +2340 +2341 +2342 +2343 +2344 +2345 +2346 +2347 +2348 +2349 +2350 +2351 +2352 +2353 +2354 +2355 +2356 +2357 +2358 +2359 +2360 +2361 +2362 +2363 +2364 +2365 +2366 +2367 +2368 +2369 +2370 +2371 +2372 +2373 +2374 +2375 +2376 +2377 +2378 +2379 +2380 +2381 +2382 +2383 +2384 +2385 +2386 +2387 +2388 +2389 +2390 +2391 +2392 +2393 +2394 +2395 +2396 +2397 +2398 +2399 +2400 +2401 +2402 +2403 +2404 +2405 +2406 +2407 +2408 +2409 +2410 +2411 +2412 +2413 +2414 +2415 +2416 +2417 +2418 +2419 +2420 +2421 +2422 +2423 +2424 +2425 +2426 +2427 +2428 +2429 +2430 +2431 +2432 +2433 +2434 +2435 +2436 +2437 +2438 +2439 +2440 +2441 +2442 +2443 +2444 +2445 +2446 +2447 +2448 +2449 +2450 +2451 +2452 +2453 +2454 +2455 +2456 +2457 +2458 +2459 +2460 +2461 +2462 +2463 +2464 +2465 +2466 +2467 +2468 +2469 +2470 +2471 +2472 +2473 +2474 +2475 +2476 +2477 +2478 +2479 +2480 +2481 +2482 +2483 +2484 +2485 +2486 +2487 +2488 +2489 +2490 +2491 +2492 +2493 +2494 +2495 +2496 +2497 +2498 +2499 +2500 +2501 +2502 +2503 +2504 +2505 +2506 +2507 +2508 +2509 +2510 +2511 +2512 +2513 +2514 +2515 +2516 +2517 +2518 +2519 +2520 +2521 +2522 +2523 +2524 +2525 +2526 +2527 +2528 +2529 +2530 +2531 +2532 +2533 +2534 +2535 +2536 +2537 +2538 +2539 +2540 +2541 +2542 +2543 +2544 +2545 +2546 +2547 +2548 +2549 +2550 +2551 +2552 +2553 +2554 +2555 +2556 +2557 +2558 +2559 +2560 +2561 +2562 +2563 +2564 +2565 +2566 +2567 +2568 +2569 +2570 +2571 +2572 +2573 +2574 +2575 +2576 +2577 +2578 +2579 +2580 +2581 +2582 +2583 +2584 +2585 +2586 +2587 +2588 +2589 +2590 +2591 +2592 +2593 +2594 +2595 +2596 +2597 +2598 +2599 +2600 +2601 +2602 +2603 +2604 +2605 +2606 +2607 +2608 +2609 +2610 +2611 +2612 +2613 +2614 +2615 +2616 +2617 +2618 +2619 +2620 +2621 +2622 +2623 +2624 +2625 +2626 +2627 +2628 +2629 +2630 +2631 +2632 +2633 +2634 +2635 +2636 +2637 +2638 +2639 +2640 +2641 +2642 +2643 +2644 +2645 +2646 +2647 +2648 +2649 +2650 +2651 +2652 +2653 +2654 +2655 +2656 +2657 +2658 +2659 +2660 +2661 +2662 +2663 +2664 +2665 +2666 +2667 +2668 +2669 +2670 +2671 +2672 +2673 +2674 +2675 +2676 +2677 +2678 +2679 +2680 +2681 +2682 +2683 +2684 +2685 +2686 +2687 +2688 +2689 +2690 +2691 +2692 +2693 +2694 +2695 +2696 +2697 +2698 +2699 +2700 +2701 +2702 +2703 +2704 +2705 +2706 +2707 +2708 +2709 +2710 +2711 +2712 +2713 +2714 +2715 +2716 +2717 +2718 +2719 +2720 +2721 +2722 +2723 +2724 +2725 +2726 +2727 +2728 +2729 +2730 +2731 +2732 +2733 +2734 +2735 +2736 +2737 +2738 +2739 +2740 +2741 +2742 +2743 +2744 +2745 +2746 +2747 +2748 +2749 +2750 +2751 +2752 +2753 +2754 +2755 +2756 +2757 +2758 +2759 +2760 +2761 +2762 +2763 +2764 +2765 +2766 +2767 +2768 +2769 +2770 +2771 +2772 +2773 +2774 +2775 +2776 +2777 +2778 +2779 +2780 +2781 +2782 +2783 +2784 +2785 +2786 +2787 +2788 +2789 +2790 +2791 +2792 +2793 +2794 +2795 +2796 +2797 +2798 +2799 +2800 +2801 +2802 +2803 +2804 +2805 +2806 +2807 +2808 +2809 +2810 +2811 +2812 +2813 +2814 +2815 +2816 +2817 +2818 +2819 +2820 +2821 +2822 +2823 +2824 +2825 +2826 +2827 +2828 +2829 +2830 +2831 +2832 +2833 +2834 +2835 +2836 +2837 +2838 +2839 +2840 +2841 +2842 +2843 +2844 +2845 +2846 +2847 +2848 +2849 +2850 +2851 +2852 +2853 +2854 +2855 +2856 +2857 +2858 +2859 +2860 +2861 +2862 +2863 +2864 +2865 +2866 +2867 +2868 +2869 +2870 +2871 +2872 +2873 +2874 +2875 +2876 +2877 +2878 +2879 +2880 +2881 +2882 +2883 +2884 +2885 +2886 +2887 +2888 +2889 +2890 +2891 +2892 +2893 +2894 +2895 +2896 +2897 +2898 +2899 +2900 +2901 +2902 +2903 +2904 +2905 +2906 +2907 +2908 +2909 +2910 +2911 +2912 +2913 +2914 +2915 +2916 +2917 +2918 +2919 +2920 +2921 +2922 +2923 +2924 +2925 +2926 +2927 +2928 +2929 +2930 +2931 +2932 +2933 +2934 +2935 +2936 +2937 +2938 +2939 +2940 +2941 +2942 +2943 +2944 +2945 +2946 +2947 +2948 +2949 +2950 +2951 +2952 +2953 +2954 +2955 +2956 +2957 +2958 +2959 +2960 +2961 +2962 +2963 +2964 +2965 +2966 +2967 +2968 +2969 +2970 +2971 +2972 +2973 +2974 +2975 +2976 +2977 +2978 +2979 +2980 +2981 +2982 +2983 +2984 +2985 +2986 +2987 +2988 +2989 +2990 +2991 +2992 +2993 +2994 +2995 +2996 +2997 +2998 +2999 +3000 +3001 +3002 +3003 +3004 +3005 +3006 +3007 +3008 +3009 +3010 +3011 +3012 +3013 +3014 +3015 +3016 +3017 +3018 +3019 +3020 +3021 +3022 +3023 +3024 +3025 +3026 +3027 +3028 +3029 +3030 +3031 +3032 +3033 +3034 +3035 +3036 +3037 +3038 +3039 +3040 +3041 +3042 +3043 +3044 +3045 +3046 +3047 +3048 +3049 +3050 +3051 +3052 +3053 +3054 +3055 +3056 +3057 +3058 +3059 +3060 +3061 +3062 +3063 +3064 +3065 +3066 +3067 +3068 +3069 +3070 +3071 +3072 +3073 +3074 +3075 +3076 +3077 +3078 +3079 +3080 +3081 +3082 +3083 +3084 +3085 +3086 +3087 +3088 +3089 +3090 +3091 +3092 +3093 +3094 +3095 +3096 +3097 +3098 +3099 +3100 +3101 +3102 +3103 +3104 +3105 +3106 +3107 +3108 +3109 +3110 +3111 +3112 +3113 +3114 +3115 +3116 +3117 +3118 +3119 +3120 +3121 +3122 +3123 +3124 +3125 +3126 +3127 +3128 +3129 +3130 +3131 +3132 +3133 +3134 +3135 +3136 +3137 +3138 +3139 +3140 +3141 +3142 +3143 +3144 +3145 +3146 +3147 +3148 +3149 +3150 +3151 +3152 +3153 +3154 +3155 +3156 +3157 +3158 +3159 +3160 +3161 +3162 +3163 +3164 +3165 +3166 +3167 +3168 +3169 +3170 +3171 +3172 +3173 +3174 +3175 +3176 +3177 +3178 +3179 +3180 +3181 +3182 +3183 +3184 +3185 +3186 +3187 +3188 +3189 +3190 +3191 +3192 +3193 +3194 +3195 +3196 +3197 +3198 +3199 +3200 +3201 +3202 +3203 +3204 +3205 +3206 +3207 +3208 +3209 +3210 +3211 +3212 +3213 +3214 +3215 +3216 +3217 +3218 +3219 +3220 +3221 +3222 +3223 +3224 +3225 +3226 +3227 +3228 +3229 +3230 +3231 +3232 +3233 +3234 +3235 +3236 +3237 +3238 +3239 +3240 +3241 +3242 +3243 +3244 +3245 +3246 +3247 +3248 +3249 +3250 +3251 +3252 +3253 +3254 +3255 +3256 +3257 +3258 +3259 +3260 +3261 +3262 +3263 +3264 +3265 +3266 +3267 +3268 +3269 +3270 +3271 +3272 +3273 +3274 +3275 +3276 +3277 +3278 +3279 +3280 +3281 +3282 +3283 +3284 +3285 +3286 +3287 +3288 +3289 +3290 +3291 +3292 +3293 +3294 +3295 +3296 +3297 +3298 +3299 +3300 +3301 +3302 +3303 +3304 +3305 +3306 +3307 +3308 +3309 +3310 +3311 +3312 +3313 +3314 +3315 +3316 +3317 +3318 +3319 +3320 +3321 +3322 +3323 +3324 +3325 +3326 +3327 +3328 +3329 +3330 +3331 +3332 +3333 +3334 +3335 +3336 +3337 +3338 +3339 +3340 +3341 +3342 +3343 +3344 +3345 +3346 +3347 +3348 +3349 +3350 +3351 +3352 +3353 +3354 +3355 +3356 +3357 +3358 +3359 +3360 +3361 +3362 +3363 +3364 +3365 +3366 +3367 +3368 +3369 +3370 +3371 +3372 +3373 +3374 +3375 +3376 +3377 +3378 +3379 +3380 +3381 +3382 +3383 +3384 +3385 +3386 +3387 +3388 +3389 +3390 +3391 +3392 +3393 +3394 +3395 +3396 +3397 +3398 +3399 +3400 +3401 +3402 +3403 +3404 +3405 +3406 +3407 +3408 +3409 +3410 +3411 +3412 +3413 +3414 +3415 +3416 +3417 +3418 +3419 +3420 +3421 +3422 +3423 +3424 +3425 +3426 +3427 +3428 +3429 +3430 +3431 +3432 +3433 +3434 +3435 +3436 +3437 +3438 +3439 +3440 +3441 +3442 +3443 +3444 +3445 +3446 +3447 +3448 +3449 +3450 +3451 +3452 +3453 +3454 +3455 +3456 +3457 +3458 +3459 +3460 +3461 +3462 +3463 +3464 +3465 +3466 +3467 +3468 +3469 +3470 +3471 +3472 +3473 +3474 +3475 +3476 +3477 +3478 +3479 +3480 +3481 +3482 +3483 +3484 +3485 +3486 +3487 +3488 +3489 +3490 +3491 +3492 +3493 +3494 +3495 +3496 +3497 +3498 +3499 +3500 +3501 +3502 +3503 +3504 +3505 +3506 +3507 +3508 +3509 +3510 +3511 +3512 +3513 +3514 +3515 +3516 +3517 +3518 +3519 +3520 +3521 +3522 +3523 +3524 +3525 +3526 +3527 +3528 +3529 +3530 +3531 +3532 +3533 +3534 +3535 +3536 +3537 +3538 +3539 +3540 +3541 +3542 +3543 +3544 +3545 +3546 +3547 +3548 +3549 +3550 +3551 +3552 +3553 +3554 +3555 +3556 +3557 +3558 +3559 +3560 +3561 +3562 +3563 +3564 +3565 +3566 +3567 +3568 +3569 +3570 +3571 +3572 +3573 +3574 +3575 +3576 +3577 +3578 +3579 +3580 +3581 +3582 +3583 +3584 +3585 +3586 +3587 +3588 +3589 +3590 +3591 +3592 +3593 +3594 +3595 +3596 +3597 +3598 +3599 +3600 +3601 +3602 +3603 +3604 +3605 +3606 +3607 +3608 +3609 +3610 +3611 +3612 +3613 +3614 +3615 +3616 +3617 +3618 +3619 +3620 +3621 +3622 +3623 +3624 +3625 +3626 +3627 +3628 +3629 +3630 +3631 +3632 +3633 +3634 +3635 +3636 +3637 +3638 +3639 +3640 +3641 +3642 +3643 +3644 +3645 +3646 +3647 +3648 +3649 +3650 +3651 +3652 +3653 +3654 +3655 +3656 +3657 +3658 +3659 +3660 +3661 +3662 +3663 +3664 +3665 +3666 +3667 +3668 +3669 +3670 +3671 +3672 +3673 +3674 +3675 +3676 +3677 +3678 +3679 +3680 +3681 +3682 +3683 +3684 +3685 +3686 +3687 +3688 +3689 +3690 +3691 +3692 +3693 +3694 +3695 +3696 +3697 +3698 +3699 +3700 +3701 +3702 +3703 +3704 +3705 +3706 +3707 +3708 +3709 +3710 +3711 +3712 +3713 +3714 +3715 +3716 +3717 +3718 +3719 +3720 +3721 +3722 +3723 +3724 +3725 +3726 +3727 +3728 +3729 +3730 +3731 +3732 +3733 +3734 +3735 +3736 +3737 +3738 +3739 +3740 +3741 +3742 +3743 +3744 +3745 +3746 +3747 +3748 +3749 +3750 +3751 +3752 +3753 +3754 +3755 +3756 +3757 +3758 +3759 +3760 +3761 +3762 +3763 +3764 +3765 +3766 +3767 +3768 +3769 +3770 +3771 +3772 +3773 +3774 +3775 +3776 +3777 +3778 +3779 +3780 +3781 +3782 +3783 +3784 +3785 +3786 +3787 +3788 +3789 +3790 +3791 +3792 +3793 +3794 +3795 +3796 +3797 +3798 +3799 +3800 +3801 +3802 +3803 +3804 +3805 +3806 +3807 +3808 +3809 +3810 +3811 +3812 +3813 +3814 +3815 +3816 +3817 +3818 +3819 +3820 +3821 +3822 +3823 +3824 +3825 +3826 +3827 +3828 +3829 +3830 +3831 +3832 +3833 +3834 +3835 +3836 +3837 +3838 +3839 +3840 +3841 +3842 +3843 +3844 +3845 +3846 +3847 +3848 +3849 +3850 +3851 +3852 +3853 +3854 +3855 +3856 +3857 +3858 +3859 +3860 +3861 +3862 +3863 +3864 +3865 +3866 +3867 +3868 +3869 +3870 +3871 +3872 +3873 +3874 +3875 +3876 +3877 +3878 +3879 +3880 +3881 +3882 +3883 +3884 +3885 +3886 +3887 +3888 +3889 +3890 +3891 +3892 +3893 +3894 +3895 +3896 +3897 +3898 +3899 +3900 +3901 +3902 +3903 +3904 +3905 +3906 +3907 +3908 +3909 +3910 +3911 +3912 +3913 +3914 +3915 +3916 +3917 +3918 +3919 +3920 +3921 +3922 +3923 +3924 +3925 +3926 +3927 +3928 +3929 +3930 +3931 +3932 +3933 +3934 +3935 +3936 +3937 +3938 +3939 +3940 +3941 +3942 +3943 +3944 +3945 +3946 +3947 +3948 +3949 +3950 +3951 +3952 +3953 +3954 +3955 +3956 +3957 +3958 +3959 +3960 +3961 +3962 +3963 +3964 +3965 +3966 +3967 +3968 +3969 +3970 +3971 +3972 +3973 +3974 +3975 +3976 +3977 +3978 +3979 +3980 +3981 +3982 +3983 +3984 +3985 +3986 +3987 +3988 +3989 +3990 +3991 +3992 +3993 +3994 +3995 +3996 +3997 +3998 +3999 +4000 +4001 +4002 +4003 +4004 +4005 +4006 +4007 +4008 +4009 +4010 +4011 +4012 +4013 +4014 +4015 +4016 +4017 +4018 +4019 +4020 +4021 +4022 +4023 +4024 +4025 +4026 +4027 +4028 +4029 +4030 +4031 +4032 +4033 +4034 +4035 +4036 +4037 +4038 +4039 +4040 +4041 +4042 +4043 +4044 +4045 +4046 +4047 +4048 +4049 +4050 +4051 +4052 +4053 +4054 +4055 +4056 +4057 +4058 +4059 +4060 +4061 +4062 +4063 +4064 +4065 +4066 +4067 +4068 +4069 +4070 +4071 +4072 +4073 +4074 +4075 +4076 +4077 +4078 +4079 +4080 +4081 +4082 +4083 +4084 +4085 +4086 +4087 +4088 +4089 +4090 +4091 +4092 +4093 +4094 +4095 +4096 +4097 +4098 +4099 +4100 +4101 +4102 +4103 +4104 +4105 +4106 +4107 +4108 +4109 +4110 +4111 +4112 +4113 +4114 +4115 +4116 +4117 +4118 +4119 +4120 +4121 +4122 +4123 +4124 +4125 +4126 +4127 +4128 +4129 +4130 +4131 +4132 +4133 +4134 +4135 +4136 +4137 +4138 +4139 +4140 +4141 +4142 +4143 +4144 +4145 +4146 +4147 +4148 +4149 +4150 +4151 +4152 +4153 +4154 +4155 +4156 +4157 +4158 +4159 +4160 +4161 +4162 +4163 +4164 +4165 +4166 +4167 +4168 +4169 +4170 +4171 +4172 +4173 +4174 +4175 +4176 +4177 +4178 +4179 +4180 +4181 +4182 +4183 +4184 +4185 +4186 +4187 +4188 +4189 +4190 +4191 +4192 +4193 +4194 +4195 +4196 +4197 +4198 +4199 +4200 +4201 +4202 +4203 +4204 +4205 +4206 +4207 +4208 +4209 +4210 +4211 +4212 +4213 +4214 +4215 +4216 +4217 +4218 +4219 +4220 +4221 +4222 +4223 +4224 +4225 +4226 +4227 +4228 +4229 +4230 +4231 +4232 +4233 +4234 +4235 +4236 +4237 +4238 +4239 +4240 +4241 +4242 +4243 +4244 +4245 +4246 +4247 +4248 +4249 +4250 +4251 +4252 +4253 +4254 +4255 +4256 +4257 +4258 +4259 +4260 +4261 +4262 +4263 +4264 +4265 +4266 +4267 +4268 +4269 +4270 +4271 +4272 +4273 +4274 +4275 +4276 +4277 +4278 +4279 +4280 +4281 +4282 +4283 +4284 +4285 +4286 +4287 +4288 +4289 +4290 +4291 +4292 +4293 +4294 +4295 +4296 +4297 +4298 +4299 +4300 +4301 +4302 +4303 +4304 +4305 +4306 +4307 +4308 +4309 +4310 +4311 +4312 +4313 +4314 +4315 +4316 +4317 +4318 +4319 +4320 +4321 +4322 +4323 +4324 +4325 +4326 +4327 +4328 +4329 +4330 +4331 +4332 +4333 +4334 +4335 +4336 +4337 +4338 +4339 +4340 +4341 +4342 +4343 +4344 +4345 +4346 +4347 +4348 +4349 +4350 +4351 +4352 +4353 +4354 +4355 +4356 +4357 +4358 +4359 +4360 +4361 +4362 +4363 +4364 +4365 +4366 +4367 +4368 +4369 +4370 +4371 +4372 +4373 +4374 +4375 +4376 +4377 +4378 +4379 +4380 +4381 +4382 +4383 +4384 +4385 +4386 +4387 +4388 +4389 +4390 +4391 +4392 +4393 +4394 +4395 +4396 +4397 +4398 +4399 +4400 +4401 +4402 +4403 +4404 +4405 +4406 +4407 +4408 +4409 +4410 +4411 +4412 +4413 +4414 +4415 +4416 +4417 +4418 +4419 +4420 +4421 +4422 +4423 +4424 +4425 +4426 +4427 +4428 +4429 +4430 +4431 +4432 +4433 +4434 +4435 +4436 +4437 +4438 +4439 +4440 +4441 +4442 +4443 +4444 +4445 +4446 +4447 +4448 +4449 +4450 +4451 +4452 +4453 +4454 +4455 +4456 +4457 +4458 +4459 +4460 +4461 +4462 +4463 +4464 +4465 +4466 +4467 +4468 +4469 +4470 +4471 +4472 +4473 +4474 +4475 +4476 +4477 +4478 +4479 +4480 +4481 +4482 +4483 +4484 +4485 +4486 +4487 +4488 +4489 +4490 +4491 +4492 +4493 +4494 +4495 +4496 +4497 +4498 +4499 +4500 +4501 +4502 +4503 +4504 +4505 +4506 +4507 +4508 +4509 +4510 +4511 +4512 +4513 +4514 +4515 +4516 +4517 +4518 +4519 +4520 +4521 +4522 +4523 +4524 +4525 +4526 +4527 +4528 +4529 +4530 +4531 +4532 +4533 +4534 +4535 +4536 +4537 +4538 +4539 +4540 +4541 +4542 +4543 +4544 +4545 +4546 +4547 +4548 +4549 +4550 +4551 +4552 +4553 +4554 +4555 +4556 +4557 +4558 +4559 +4560 +4561 +4562 +4563 +4564 +4565 +4566 +4567 +4568 +4569 +4570 +4571 +4572 +4573 +4574 +4575 +4576 +4577 +4578 +4579 +4580 +4581 +4582 +4583 +4584 +4585 +4586 +4587 +4588 +4589 +4590 +4591 +4592 +4593 +4594 +4595 +4596 +4597 +4598 +4599 +4600 +4601 +4602 +4603 +4604 +4605 +4606 +4607 +4608 +4609 +4610 +4611 +4612 +4613 +4614 +4615 +4616 +4617 +4618 +4619 +4620 +4621 +4622 +4623 +4624 +4625 +4626 +4627 +4628 +4629 +4630 +4631 +4632 +4633 +4634 +4635 +4636 +4637 +4638 +4639 +4640 +4641 +4642 +4643 +4644 +4645 +4646 +4647 +4648 +4649 +4650 +4651 +4652 +4653 +4654 +4655 +4656 +4657 +4658 +4659 +4660 +4661 +4662 +4663 +4664 +4665 +4666 +4667 +4668 +4669 +4670 +4671 +4672 +4673 +4674 +4675 +4676 +4677 +4678 +4679 +4680 +4681 +4682 +4683 +4684 +4685 +4686 +4687 +4688 +4689 +4690 +4691 +4692 +4693 +4694 +4695 +4696 +4697 +4698 +4699 +4700 +4701 +4702 +4703 +4704 +4705 +4706 +4707 +4708 +4709 +4710 +4711 +4712 +4713 +4714 +4715 +4716 +4717 +4718 +4719 +4720 +4721 +4722 +4723 +4724 +4725 +4726 +4727 +4728 +4729 +4730 +4731 +4732 +4733 +4734 +4735 +4736 +4737 +4738 +4739 +4740 +4741 +4742 +4743 +4744 +4745 +4746 +4747 +4748 +4749 +4750 +4751 +4752 +4753 +4754 +4755 +4756 +4757 +4758 +4759 +4760 +4761 +4762 +4763 +4764 +4765 +4766 +4767 +4768 +4769 +4770 +4771 +4772 +4773 +4774 +4775 +4776 +4777 +4778 +4779 +4780 +4781 +4782 +4783 +4784 +4785 +4786 +4787 +4788 +4789 +4790 +4791 +4792 +4793 +4794 +4795 +4796 +4797 +4798 +4799 +4800 +4801 +4802 +4803 +4804 +4805 +4806 +4807 +4808 +4809 +4810 +4811 +4812 +4813 +4814 +4815 +4816 +4817 +4818 +4819 +4820 +4821 +4822 +4823 +4824 +4825 +4826 +4827 +4828 +4829 +4830 +4831 +4832 +4833 +4834 +4835 +4836 +4837 +4838 +4839 +4840 +4841 +4842 +4843 +4844 +4845 +4846 +4847 +4848 +4849 +4850 +4851 +4852 +4853 +4854 +4855 +4856 +4857 +4858 +4859 +4860 +4861 +4862 +4863 +4864 +4865 +4866 +4867 +4868 +4869 +4870 +4871 +4872 +4873 +4874 +4875 +4876 +4877 +4878 +4879 +4880 +4881 +4882 +4883 +4884 +4885 +4886 +4887 +4888 +4889 +4890 +4891 +4892 +4893 +4894 +4895 +4896 +4897 +4898 +4899 +4900 +4901 +4902 +4903 +4904 +4905 +4906 +4907 +4908 +4909 +4910 +4911 +4912 +4913 +4914 +4915 +4916 +4917 +4918 +4919 +4920 +4921 +4922 +4923 +4924 +4925 +4926 +4927 +4928 +4929 +4930 +4931 +4932 +4933 +4934 +4935 +4936 +4937 +4938 +4939 +4940 +4941 +4942 +4943 +4944 +4945 +4946 +4947 +4948 +4949 +4950 +4951 +4952 +4953 +4954 +4955 +4956 +4957 +4958 +4959 +4960 +4961 +4962 +4963 +4964 +4965 +4966 +4967 +4968 +4969 +4970 +4971 +4972 +4973 +4974 +4975 +4976 +4977 +4978 +4979 +4980 +4981 +4982 +4983 +4984 +4985 +4986 +4987 +4988 +4989 +4990 +4991 +4992 +4993 +4994 +4995 +4996 +4997 +4998 +4999 +5000 +5001 +5002 +5003 +5004 +5005 +5006 +5007 +5008 +5009 +5010 +5011 +5012 +5013 +5014 +5015 +5016 +5017 +5018 +5019 +5020 +5021 +5022 +5023 +5024 +5025 +5026 +5027 +5028 +5029 +5030 +5031 +5032 +5033 +5034 +5035 +5036 +5037 +5038 +5039 +5040 +5041 +5042 +5043 +5044 +5045 +5046 +5047 +5048 +5049 +5050 +5051 +5052 +5053 +5054 +5055 +5056 +5057 +5058 +5059 +5060 +5061 +5062 +5063 +5064 +5065 +5066 +5067 +5068 +5069 +5070 +5071 +5072 +5073 +5074 +5075 +5076 +5077 +5078 +5079 +5080 +5081 +5082 +5083 +5084 +5085 +5086 +5087 +5088 +5089 +5090 +5091 +5092 +5093 +5094 +5095 +5096 +5097 +5098 +5099 +5100 +5101 +5102 +5103 +5104 +5105 +5106 +5107 +5108 +5109 +5110 +5111 +5112 +5113 +5114 +5115 +5116 +5117 +5118 +5119 +5120 +5121 +5122 +5123 +5124 +5125 +5126 +5127 +5128 +5129 +5130 +5131 +5132 +5133 +5134 +5135 +5136 +5137 +5138 +5139 +5140 +5141 +5142 +5143 +5144 +5145 +5146 +5147 +5148 +5149 +5150 +5151 +5152 +5153 +5154 +5155 +5156 +5157 +5158 +5159 +5160 +5161 +5162 +5163 +5164 +5165 +5166 +5167 +5168 +5169 +5170 +5171 +5172 +5173 +5174 +5175 +5176 +5177 +5178 +5179 +5180 +5181 +5182 +5183 +5184 +5185 +5186 +5187 +5188 +5189 +5190 +5191 +5192 +5193 +5194 +5195 +5196 +5197 +5198 +5199 +5200 +5201 +5202 +5203 +5204 +5205 +5206 +5207 +5208 +5209 +5210 +5211 +5212 +5213 +5214 +5215 +5216 +5217 +5218 +5219 +5220 +5221 +5222 +5223 +5224 +5225 +5226 +5227 +5228 +5229 +5230 +5231 +5232 +5233 +5234 +5235 +5236 +5237 +5238 +5239 +5240 +5241 +5242 +5243 +5244 +5245 +5246 +5247 +5248 +5249 +5250 +5251 +5252 +5253 +5254 +5255 +5256 +5257 +5258 +5259 +5260 +5261 +5262 +5263 +5264 +5265 +5266 +5267 +5268 +5269 +5270 +5271 +5272 +5273 +5274 +5275 +5276 +5277 +5278 +5279 +5280 +5281 +5282 +5283 +5284 +5285 +5286 +5287 +5288 +5289 +5290 +5291 +5292 +5293 +5294 +5295 +5296 +5297 +5298 +5299 +5300 +5301 +5302 +5303 +5304 +5305 +5306 +5307 +5308 +5309 +5310 +5311 +5312 +5313 +5314 +5315 +5316 +5317 +5318 +5319 +5320 +5321 +5322 +5323 +5324 +5325 +5326 +5327 +5328 +5329 +5330 +5331 +5332 +5333 +5334 +5335 +5336 +5337 +5338 +5339 +5340 +5341 +5342 +5343 +5344 +5345 +5346 +5347 +5348 +5349 +5350 +5351 +5352 +5353 +5354 +5355 +5356 +5357 +5358 +5359 +5360 +5361 +5362 +5363 +5364 +5365 +5366 +5367 +5368 +5369 +5370 +5371 +5372 +5373 +5374 +5375 +5376 +5377 +5378 +5379 +5380 +5381 +5382 +5383 +5384 +5385 +5386 +5387 +5388 +5389 +5390 +5391 +5392 +5393 +5394 +5395 +5396 +5397 +5398 +5399 +5400 +5401 +5402 +5403 +5404 +5405 +5406 +5407 +5408 +5409 +5410 +5411 +5412 +5413 +5414 +5415 +5416 +5417 +5418 +5419 +5420 +5421 +5422 +5423 +5424 +5425 +5426 +5427 +5428 +5429 +5430 +5431 +5432 +5433 +5434 +5435 +5436 +5437 +5438 +5439 +5440 +5441 +5442 +5443 +5444 +5445 +5446 +5447 +5448 +5449 +5450 +5451 +5452 +5453 +5454 +5455 +5456 +5457 +5458 +5459 +5460 +5461 +5462 +5463 +5464 +5465 +5466 +5467 +5468 +5469 +5470 +5471 +5472 +5473 +5474 +5475 +5476 +5477 +5478 +5479 +5480 +5481 +5482 +5483 +5484 +5485 +5486 +5487 +5488 +5489 +5490 +5491 +5492 +5493 +5494 +5495 +5496 +5497 +5498 +5499 +5500 +5501 +5502 +5503 +5504 +5505 +5506 +5507 +5508 +5509 +5510 +5511 +5512 +5513 +5514 +5515 +5516 +5517 +5518 +5519 +5520 +5521 +5522 +5523 +5524 +5525 +5526 +5527 +5528 +5529 +5530 +5531 +5532 +5533 +5534 +5535 +5536 +5537 +5538 +5539 +5540 +5541 +5542 +5543 +5544 +5545 +5546 +5547 +5548 +5549 +5550 +5551 +5552 +5553 +5554 +5555 +5556 +5557 +5558 +5559 +5560 +5561 +5562 +5563 +5564 +5565 +5566 +5567 +5568 +5569 +5570 +5571 +5572 +5573 +5574 +5575 +5576 +5577 +5578 +5579 +5580 +5581 +5582 +5583 +5584 +5585 +5586 +5587 +5588 +5589 +5590 +5591 +5592 +5593 +5594 +5595 +5596 +5597 +5598 +5599 +5600 +5601 +5602 +5603 +5604 +5605 +5606 +5607 +5608 +5609 +5610 +5611 +5612 +5613 +5614 +5615 +5616 +5617 +5618 +5619 +5620 +5621 +5622 +5623 +5624 +5625 +5626 +5627 +5628 +5629 +5630 +5631 +5632 +5633 +5634 +5635 +5636 +5637 +5638 +5639 +5640 +5641 +5642 +5643 +5644 +5645 +5646 +5647 +5648 +5649 +5650 +5651 +5652 +5653 +5654 +5655 +5656 +5657 +5658 +5659 +5660 +5661 +5662 +5663 +5664 +5665 +5666 +5667 +5668 +5669 +5670 +5671 +5672 +5673 +5674 +5675 +5676 +5677 +5678 +5679 +5680 +5681 +5682 +5683 +5684 +5685 +5686 +5687 +5688 +5689 +5690 +5691 +5692 +5693 +5694 +5695 +5696 +5697 +5698 +5699 +5700 +5701 +5702 +5703 +5704 +5705 +5706 +5707 +5708 +5709 +5710 +5711 +5712 +5713 +5714 +5715 +5716 +5717 +5718 +5719 +5720 +5721 +5722 +5723 +5724 +5725 +5726 +5727 +5728 +5729 +5730 +5731 +5732 +5733 +5734 +5735 +5736 +5737 +5738 +5739 +5740 +5741 +5742 +5743 +5744 +5745 +5746 +5747 +5748 +5749 +5750 +5751 +5752 +5753 +5754 +5755 +5756 +5757 +5758 +5759 +5760 +5761 +5762 +5763 +5764 +5765 +5766 +5767 +5768 +5769 +5770 +5771 +5772 +5773 +5774 +5775 +5776 +5777 +5778 +5779 +5780 +5781 +5782 +5783 +5784 +5785 +5786 +5787 +5788 +5789 +5790 +5791 +5792 +5793 +5794 +5795 +5796 +5797 +5798 +5799 +5800 +5801 +5802 +5803 +5804 +5805 +5806 +5807 +5808 +5809 +5810 +5811 +5812 +5813 +5814 +5815 +5816 +5817 +5818 +5819 +5820 +5821 +5822 +5823 +5824 +5825 +5826 +5827 +5828 +5829 +5830 +5831 +5832 +5833 +5834 +5835 +5836 +5837 +5838 +5839 +5840 +5841 +5842 +5843 +5844 +5845 +5846 +5847 +5848 +5849 +5850 +5851 +5852 +5853 +5854 +5855 +5856 +5857 +5858 +5859 +5860 +5861 +5862 +5863 +5864 +5865 +5866 +5867 +5868 +5869 +5870 +5871 +5872 +5873 +5874 +5875 +5876 +5877 +5878 +5879 +5880 +5881 +5882 +5883 +5884 +5885 +5886 +5887 +5888 +5889 +5890 +5891 +5892 +5893 +5894 +5895 +5896 +5897 +5898 +5899 +5900 +5901 +5902 +5903 +5904 +5905 +5906 +5907 +5908 +5909 +5910 +5911 +5912 +5913 +5914 +5915 +5916 +5917 +5918 +5919 +5920 +5921 +5922 +5923 +5924 +5925 +5926 +5927 +5928 +5929 +5930 +5931 +5932 +5933 +5934 +5935 +5936 +5937 +5938 +5939 +5940 +5941 +5942 +5943 +5944 +5945 +5946 +5947 +5948 +5949 +5950 +5951 +5952 +5953 +5954 +5955 +5956 +5957 +5958 +5959 +5960 +5961 +5962 +5963 +5964 +5965 +5966 +5967 +5968 +5969 +5970 +5971 +5972 +5973 +5974 +5975 +5976 +5977 +5978 +5979 +5980 +5981 +5982 +5983 +5984 +5985 +5986 +5987 +5988 +5989 +5990 +5991 +5992 +5993 +5994 +5995 +5996 +5997 +5998 +5999 +6000 +6001 +6002 +6003 +6004 +6005 +6006 +6007 +6008 +6009 +6010 +6011 +6012 +6013 +6014 +6015 +6016 +6017 +6018 +6019 +6020 +6021 +6022 +6023 +6024 +6025 +6026 +6027 +6028 +6029 +6030 +6031 +6032 +6033 +6034 +6035 +6036 +6037 +6038 +6039 +6040 +6041 +6042 +6043 +6044 +6045 +6046 +6047 +6048 +6049 +6050 +6051 +6052 +6053 +6054 +6055 +6056 +6057 +6058 +6059 +6060 +6061 +6062 +6063 +6064 +6065 +6066 +6067 +6068 +6069 +6070 +6071 +6072 +6073 +6074 +6075 +6076 +6077 +6078 +6079 +6080 +6081 +6082 +6083 +6084 +6085 +6086 +6087 +6088 +6089 +6090 +6091 +6092 +6093 +6094 +6095 +6096 +6097 +6098 +6099 +6100 +6101 +6102 +6103 +6104 +6105 +6106 +6107 +6108 +6109 +6110 +6111 +6112 +6113 +6114 +6115 +6116 +6117 +6118 +6119 +6120 +6121 +6122 +6123 +6124 +6125 +6126 +6127 +6128 +6129 +6130 +6131 +6132 +6133 +6134 +6135 +6136 +6137 +6138 +6139 +6140 +6141 +6142 +6143 +6144 +6145 +6146 +6147 +6148 +6149 +6150 +6151 +6152 +6153 +6154 +6155 +6156 +6157 +6158 +6159 +6160 +6161 +6162 +6163 +6164 +6165 +6166 +6167 +6168 +6169 +6170 +6171 +6172 +6173 +6174 +6175 +6176 +6177 +6178 +6179 +6180 +6181 +6182 +6183 +6184 +6185 +6186 +6187 +6188 +6189 +6190 +6191 +6192 +6193 +6194 +6195 +6196 +6197 +6198 +6199 +6200 +6201 +6202 +6203 +6204 +6205 +6206 +6207 +6208 +6209 +6210 +6211 +6212 +6213 +6214 +6215 +6216 +6217 +6218 +6219 +6220 +6221 +6222 +6223 +6224 +6225 +6226 +6227 +6228 +6229 +6230 +6231 +6232 +6233 +6234 +6235 +6236 +6237 +6238 +6239 +6240 +6241 +6242 +6243 +6244 +6245 +6246 +6247 +6248 +6249 +6250 +6251 +6252 +6253 +6254 +6255 +6256 +6257 +6258 +6259 +6260 +6261 +6262 +6263 +6264 +6265 +6266 +6267 +6268 +6269 +6270 +6271 +6272 +6273 +6274 +6275 +6276 +6277 +6278 +6279 +6280 +6281 +6282 +6283 +6284 +6285 +6286 +6287 +6288 +6289 +6290 +6291 +6292 +6293 +6294 +6295 +6296 +6297 +6298 +6299 +6300 +6301 +6302 +6303 +6304 +6305 +6306 +6307 +6308 +6309 +6310 +6311 +6312 +6313 +6314 +6315 +6316 +6317 +6318 +6319 +6320 +6321 +6322 +6323 +6324 +6325 +6326 +6327 +6328 +6329 +6330 +6331 +6332 +6333 +6334 +6335 +6336 +6337 +6338 +6339 +6340 +6341 +6342 +6343 +6344 +6345 +6346 +6347 +6348 +6349 +6350 +6351 +6352 +6353 +6354 +6355 +6356 +6357 +6358 +6359 +6360 +6361 +6362 +6363 +6364 +6365 +6366 +6367 +6368 +6369 +6370 +6371 +6372 +6373 +6374 +6375 +6376 +6377 +6378 +6379 +6380 +6381 +6382 +6383 +6384 +6385 +6386 +6387 +6388 +6389 +6390 +6391 +6392 +6393 +6394 +6395 +6396 +6397 +6398 +6399 +6400 +6401 +6402 +6403 +6404 +6405 +6406 +6407 +6408 +6409 +6410 +6411 +6412 +6413 +6414 +6415 +6416 +6417 +6418 +6419 +6420 +6421 +6422 +6423 +6424 +6425 +6426 +6427 +6428 +6429 +6430 +6431 +6432 +6433 +6434 +6435 +6436 +6437 +6438 +6439 +6440 +6441 +6442 +6443 +6444 +6445 +6446 +6447 +6448 +6449 +6450 +6451 +6452 +6453 +6454 +6455 +6456 +6457 +6458 +6459 +6460 +6461 +6462 +6463 +6464 +6465 +6466 +6467 +6468 +6469 +6470 +6471 +6472 +6473 +6474 +6475 +6476 +6477 +6478 +6479 +6480 +6481 +6482 +6483 +6484 +6485 +6486 +6487 +6488 +6489 +6490 +6491 +6492 +6493 +6494 +6495 +6496 +6497 +6498 +6499 +6500 +6501 +6502 +6503 +6504 +6505 +6506 +6507 +6508 +6509 +6510 +6511 +6512 +6513 +6514 +6515 +6516 +6517 +6518 +6519 +6520 +6521 +6522 +6523 +6524 +6525 +6526 +6527 +6528 +6529 +6530 +6531 +6532 +6533 +6534 +6535 +6536 +6537 +6538 +6539 +6540 +6541 +6542 +6543 +6544 +6545 +6546 +6547 +6548 +6549 +6550 +6551 +6552 +6553 +6554 +6555 +6556 +6557 +6558 +6559 +6560 +6561 +6562 +6563 +6564 +6565 +6566 +6567 +6568 +6569 +6570 +6571 +6572 +6573 +6574 +6575 +6576 +6577 +6578 +6579 +6580 +6581 +6582 +6583 +6584 +6585 +6586 +6587 +6588 +6589 +6590 +6591 +6592 +6593 +6594 +6595 +6596 +6597 +6598 +6599 +6600 +6601 +6602 +6603 +6604 +6605 +6606 +6607 +6608 +6609 +6610 +6611 +6612 +6613 +6614 +6615 +6616 +6617 +6618 +6619 +6620 +6621 +6622 +6623 +6624 +6625 +6626 +6627 +6628 +6629 +6630 +6631 +6632 +6633 +6634 +6635 +6636 +6637 +6638 +6639 +6640 +6641 +6642 +6643 +6644 +6645 +6646 +6647 +6648 +6649 +6650 +6651 +6652 +6653 +6654 +6655 +6656 +6657 +6658 +6659 +6660 +6661 +6662 +6663 +6664 +6665 +6666 +6667 +6668 +6669 +6670 +6671 +6672 +6673 +6674 +6675 +6676 +6677 +6678 +6679 +6680 +6681 +6682 +6683 +6684 +6685 +6686 +6687 +6688 +6689 +6690 +6691 +6692 +6693 +6694 +6695 +6696 +6697 +6698 +6699 +6700 +6701 +6702 +6703 +6704 +6705 +6706 +6707 +6708 +6709 +6710 +6711 +6712 +6713 +6714 +6715 +6716 +6717 +6718 +6719 +6720 +6721 +6722 +6723 +6724 +6725 +6726 +6727 +6728 +6729 +6730 +6731 +6732 +6733 +6734 +6735 +6736 +6737 +6738 +6739 +6740 +6741 +6742 +6743 +6744 +6745 +6746 +6747 +6748 +6749 +6750 +6751 +6752 +6753 +6754 +6755 +6756 +6757 +6758 +6759 +6760 +6761 +6762 +6763 +6764 +6765 +6766 +6767 +6768 +6769 +6770 +6771 +6772 +6773 +6774 +6775 +6776 +6777 +6778 +6779 +6780 +6781 +6782 +6783 +6784 +6785 +6786 +6787 +6788 +6789 +6790 +6791 +6792 +6793 +6794 +6795 +6796 +6797 +6798 +6799 +6800 +6801 +6802 +6803 +6804 +6805 +6806 +6807 +6808 +6809 +6810 +6811 +6812 +6813 +6814 +6815 +6816 +6817 +6818 +6819 +6820 +6821 +6822 +6823 +6824 +6825 +6826 +6827 +6828 +6829 +6830 +6831 +6832 +6833 +6834 +6835 +6836 +6837 +6838 +6839 +6840 +6841 +6842 +6843 +6844 +6845 +6846 +6847 +6848 +6849 +6850 +6851 +6852 +6853 +6854 +6855 +6856 +6857 +6858 +6859 +6860 +6861 +6862 +6863 +6864 +6865 +6866 +6867 +6868 +6869 +6870 +6871 +6872 +6873 +6874 +6875 +6876 +6877 +6878 +6879 +6880 +6881 +6882 +6883 +6884 +6885 +6886 +6887 +6888 +6889 +6890 +6891 +6892 +6893 +6894 +6895 +6896 +6897 +6898 +6899 +6900 +6901 +6902 +6903 +6904 +6905 +6906 +6907 +6908 +6909 +6910 +6911 +6912 +6913 +6914 +6915 +6916 +6917 +6918 +6919 +6920 +6921 +6922 +6923 +6924 +6925 +6926 +6927 +6928 +6929 +6930 +6931 +6932 +6933 +6934 +6935 +6936 +6937 +6938 +6939 +6940 +6941 +6942 +6943 +6944 +6945 +6946 +6947 +6948 +6949 +6950 +6951 +6952 +6953 +6954 +6955 +6956 +6957 +6958 +6959 +6960 +6961 +6962 +6963 +6964 +6965 +6966 +6967 +6968 +6969 +6970 +6971 +6972 +6973 +6974 +6975 +6976 +6977 +6978 +6979 +6980 +6981 +6982 +6983 +6984 +6985 +6986 +6987 +6988 +6989 +6990 +6991 +6992 +6993 +6994 +6995 +6996 +6997 +6998 +6999 +7000 +7001 +7002 +7003 +7004 +7005 +7006 +7007 +7008 +7009 +7010 +7011 +7012 +7013 +7014 +7015 +7016 +7017 +7018 +7019 +7020 +7021 +7022 +7023 +7024 +7025 +7026 +7027 +7028 +7029 +7030 +7031 +7032 +7033 +7034 +7035 +7036 +7037 +7038 +7039 +7040 +7041 +7042 +7043 +7044 +7045 +7046 +7047 +7048 +7049 +7050 +7051 +7052 +7053 +7054 +7055 +7056 +7057 +7058 +7059 +7060 +7061 +7062 +7063 +7064 +7065 +7066 +7067 +7068 +7069 +7070 +7071 +7072 +7073 +7074 +7075 +7076 +7077 +7078 +7079 +7080 +7081 +7082 +7083 +7084 +7085 +7086 +7087 +7088 +7089 +7090 +7091 +7092 +7093 +7094 +7095 +7096 +7097 +7098 +7099 +7100 +7101 +7102 +7103 +7104 +7105 +7106 +7107 +7108 +7109 +7110 +7111 +7112 +7113 +7114 +7115 +7116 +7117 +7118 +7119 +7120 +7121 +7122 +7123 +7124 +7125 +7126 +7127 +7128 +7129 +7130 +7131 +7132 +7133 +7134 +7135 +7136 +7137 +7138 +7139 +7140 +7141 +7142 +7143 +7144 +7145 +7146 +7147 +7148 +7149 +7150 +7151 +7152 +7153 +7154 +7155 +7156 +7157 +7158 +7159 +7160 +7161 +7162 +7163 +7164 +7165 +7166 +7167 +7168 +7169 +7170 +7171 +7172 +7173 +7174 +7175 +7176 +7177 +7178 +7179 +7180 +7181 +7182 +7183 +7184 +7185 +7186 +7187 +7188 +7189 +7190 +7191 +7192 +7193 +7194 +7195 +7196 +7197 +7198 +7199 +7200 +7201 +7202 +7203 +7204 +7205 +7206 +7207 +7208 +7209 +7210 +7211 +7212 +7213 +7214 +7215 +7216 +7217 +7218 +7219 +7220 +7221 +7222 +7223 +7224 +7225 +7226 +7227 +7228 +7229 +7230 +7231 +7232 +7233 +7234 +7235 +7236 +7237 +7238 +7239 +7240 +7241 +7242 +7243 +7244 +7245 +7246 +7247 +7248 +7249 +7250 +7251 +7252 +7253 +7254 +7255 +7256 +7257 +7258 +7259 +7260 +7261 +7262 +7263 +7264 +7265 +7266 +7267 +7268 +7269 +7270 +7271 +7272 +7273 +7274 +7275 +7276 +7277 +7278 +7279 +7280 +7281 +7282 +7283 +7284 +7285 +7286 +7287 +7288 +7289 +7290 +7291 +7292 +7293 +7294 +7295 +7296 +7297 +7298 +7299 +7300 +7301 +7302 +7303 +7304 +7305 +7306 +7307 +7308 +7309 +7310 +7311 +7312 +7313 +7314 +7315 +7316 +7317 +7318 +7319 +7320 +7321 +7322 +7323 +7324 +7325 +7326 +7327 +7328 +7329 +7330 +7331 +7332 +7333 +7334 +7335 +7336 +7337 +7338 +7339 +7340 +7341 +7342 +7343 +7344 +7345 +7346 +7347 +7348 +7349 +7350 +7351 +7352 +7353 +7354 +7355 +7356 +7357 +7358 +7359 +7360 +7361 +7362 +7363 +7364 +7365 +7366 +7367 +7368 +7369 +7370 +7371 +7372 +7373 +7374 +7375 +7376 +7377 +7378 +7379 +7380 +7381 +7382 +7383 +7384 +7385 +7386 +7387 +7388 +7389 +7390 +7391 +7392 +7393 +7394 +7395 +7396 +7397 +7398 +7399 +7400 +7401 +7402 +7403 +7404 +7405 +7406 +7407 +7408 +7409 +7410 +7411 +7412 +7413 +7414 +7415 +7416 +7417 +7418 +7419 +7420 +7421 +7422 +7423 +7424 +7425 +7426 +7427 +7428 +7429 +7430 +7431 +7432 +7433 +7434 +7435 +7436 +7437 +7438 +7439 +7440 +7441 +7442 +7443 +7444 +7445 +7446 +7447 +7448 +7449 +7450 +7451 +7452 +7453 +7454 +7455 +7456 +7457 +7458 +7459 +7460 +7461 +7462 +7463 +7464 +7465 +7466 +7467 +7468 +7469 +7470 +7471 +7472 +7473 +7474 +7475 +7476 +7477 +7478 +7479 +7480 +7481 +7482 +7483 +7484 +7485 +7486 +7487 +7488 +7489 +7490 +7491 +7492 +7493 +7494 +7495 +7496 +7497 +7498 +7499 +7500 +7501 +7502 +7503 +7504 +7505 +7506 +7507 +7508 +7509 +7510 +7511 +7512 +7513 +7514 +7515 +7516 +7517 +7518 +7519 +7520 +7521 +7522 +7523 +7524 +7525 +7526 +7527 +7528 +7529 +7530 +7531 +7532 +7533 +7534 +7535 +7536 +7537 +7538 +7539 +7540 +7541 +7542 +7543 +7544 +7545 +7546 +7547 +7548 +7549 +7550 +7551 +7552 +7553 +7554 +7555 +7556 +7557 +7558 +7559 +7560 +7561 +7562 +7563 +7564 +7565 +7566 +7567 +7568 +7569 +7570 +7571 +7572 +7573 +7574 +7575 +7576 +7577 +7578 +7579 +7580 +7581 +7582 +7583 +7584 +7585 +7586 +7587 +7588 +7589 +7590 +7591 +7592 +7593 +7594 +7595 +7596 +7597 +7598 +7599 +7600 +7601 +7602 +7603 +7604 +7605 +7606 +7607 +7608 +7609 +7610 +7611 +7612 +7613 +7614 +7615 +7616 +7617 +7618 +7619 +7620 +7621 +7622 +7623 +7624 +7625 +7626 +7627 +7628 +7629 +7630 +7631 +7632 +7633 +7634 +7635 +7636 +7637 +7638 +7639 +7640 +7641 +7642 +7643 +7644 +7645 +7646 +7647 +7648 +7649 +7650 +7651 +7652 +7653 +7654 +7655 +7656 +7657 +7658 +7659 +7660 +7661 +7662 +7663 +7664 +7665 +7666 +7667 +7668 +7669 +7670 +7671 +7672 +7673 +7674 +7675 +7676 +7677 +7678 +7679 +7680 +7681 +7682 +7683 +7684 +7685 +7686 +7687 +7688 +7689 +7690 +7691 +7692 +7693 +7694 +7695 +7696 +7697 +7698 +7699 +7700 +7701 +7702 +7703 +7704 +7705 +7706 +7707 +7708 +7709 +7710 +7711 +7712 +7713 +7714 +7715 +7716 +7717 +7718 +7719 +7720 +7721 +7722 +7723 +7724 +7725 +7726 +7727 +7728 +7729 +7730 +7731 +7732 +7733 +7734 +7735 +7736 +7737 +7738 +7739 +7740 +7741 +7742 +7743 +7744 +7745 +7746 +7747 +7748 +7749 +7750 +7751 +7752 +7753 +7754 +7755 +7756 +7757 +7758 +7759 +7760 +7761 +7762 +7763 +7764 +7765 +7766 +7767 +7768 +7769 +7770 +7771 +7772 +7773 +7774 +7775 +7776 +7777 +7778 +7779 +7780 +7781 +7782 +7783 +7784 +7785 +7786 +7787 +7788 +7789 +7790 +7791 +7792 +7793 +7794 +7795 +7796 +7797 +7798 +7799 +7800 +7801 +7802 +7803 +7804 +7805 +7806 +7807 +7808 +7809 +7810 +7811 +7812 +7813 +7814 +7815 +7816 +7817 +7818 +7819 +7820 +7821 +7822 +7823 +7824 +7825 +7826 +7827 +7828 +7829 +7830 +7831 +7832 +7833 +7834 +7835 +7836 +7837 +7838 +7839 +7840 +7841 +7842 +7843 +7844 +7845 +7846 +7847 +7848 +7849 +7850 +7851 +7852 +7853 +7854 +7855 +7856 +7857 +7858 +7859 +7860 +7861 +7862 +7863 +7864 +7865 +7866 +7867 +7868 +7869 +7870 +7871 +7872 +7873 +7874 +7875 +7876 +7877 +7878 +7879 +7880 +7881 +7882 +7883 +7884 +7885 +7886 +7887 +7888 +7889 +7890 +7891 +7892 +7893 +7894 +7895 +7896 +7897 +7898 +7899 +7900 +7901 +7902 +7903 +7904 +7905 +7906 +7907 +7908 +7909 +7910 +7911 +7912 +7913 +7914 +7915 +7916 +7917 +7918 +7919 +7920 +7921 +7922 +7923 +7924 +7925 +7926 +7927 +7928 +7929 +7930 +7931 +7932 +7933 +7934 +7935 +7936 +7937 +7938 +7939 +7940 +7941 +7942 +7943 +7944 +7945 +7946 +7947 +7948 +7949 +7950 +7951 +7952 +7953 +7954 +7955 +7956 +7957 +7958 +7959 +7960 +7961 +7962 +7963 +7964 +7965 +7966 +7967 +7968 +7969 +7970 +7971 +7972 +7973 +7974 +7975 +7976 +7977 +7978 +7979 +7980 +7981 +7982 +7983 +7984 +7985 +7986 +7987 +7988 +7989 +7990 +7991 +7992 +7993 +7994 +7995 +7996 +7997 +7998 +7999 +8000 +8001 +8002 +8003 +8004 +8005 +8006 +8007 +8008 +8009 +8010 +8011 +8012 +8013 +8014 +8015 +8016 +8017 +8018 +8019 +8020 +8021 +8022 +8023 +8024 +8025 +8026 +8027 +8028 +8029 +8030 +8031 +8032 +8033 +8034 +8035 +8036 +8037 +8038 +8039 +8040 +8041 +8042 +8043 +8044 +8045 +8046 +8047 +8048 +8049 +8050 +8051 +8052 +8053 +8054 +8055 +8056 +8057 +8058 +8059 +8060 +8061 +8062 +8063 +8064 +8065 +8066 +8067 +8068 +8069 +8070 +8071 +8072 +8073 +8074 +8075 +8076 +8077 +8078 +8079 +8080 +8081 +8082 +8083 +8084 +8085 +8086 +8087 +8088 +8089 +8090 +8091 +8092 +8093 +8094 +8095 +8096 +8097 +8098 +8099 +8100 +8101 +8102 +8103 +8104 +8105 +8106 +8107 +8108 +8109 +8110 +8111 +8112 +8113 +8114 +8115 +8116 +8117 +8118 +8119 +8120 +8121 +8122 +8123 +8124 +8125 +8126 +8127 +8128 +8129 +8130 +8131 +8132 +8133 +8134 +8135 +8136 +8137 +8138 +8139 +8140 +8141 +8142 +8143 +8144 +8145 +8146 +8147 +8148 +8149 +8150 +8151 +8152 +8153 +8154 +8155 +8156 +8157 +8158 +8159 +8160 +8161 +8162 +8163 +8164 +8165 +8166 +8167 +8168 +8169 +8170 +8171 +8172 +8173 +8174 +8175 +8176 +8177 +8178 +8179 +8180 +8181 +8182 +8183 +8184 +8185 +8186 +8187 +8188 +8189 +8190 +8191 +8192 +8193 +8194 +8195 +8196 +8197 +8198 +8199 +8200 +8201 +8202 +8203 +8204 +8205 +8206 +8207 +8208 +8209 +8210 +8211 +8212 +8213 +8214 +8215 +8216 +8217 +8218 +8219 +8220 +8221 +8222 +8223 +8224 +8225 +8226 +8227 +8228 +8229 +8230 +8231 +8232 +8233 +8234 +8235 +8236 +8237 +8238 +8239 +8240 +8241 +8242 +8243 +8244 +8245 +8246 +8247 +8248 +8249 +8250 +8251 +8252 +8253 +8254 +8255 +8256 +8257 +8258 +8259 +8260 +8261 +8262 +8263 +8264 +8265 +8266 +8267 +8268 +8269 +8270 +8271 +8272 +8273 +8274 +8275 +8276 +8277 +8278 +8279 +8280 +8281 +8282 +8283 +8284 +8285 +8286 +8287 +8288 +8289 +8290 +8291 +8292 +8293 +8294 +8295 +8296 +8297 +8298 +8299 +8300 +8301 +8302 +8303 +8304 +8305 +8306 +8307 +8308 +8309 +8310 +8311 +8312 +8313 +8314 +8315 +8316 +8317 +8318 +8319 +8320 +8321 +8322 +8323 +8324 +8325 +8326 +8327 +8328 +8329 +8330 +8331 +8332 +8333 +8334 +8335 +8336 +8337 +8338 +8339 +8340 +8341 +8342 +8343 +8344 +8345 +8346 +8347 +8348 +8349 +8350 +8351 +8352 +8353 +8354 +8355 +8356 +8357 +8358 +8359 +8360 +8361 +8362 +8363 +8364 +8365 +8366 +8367 +8368 +8369 +8370 +8371 +8372 +8373 +8374 +8375 +8376 +8377 +8378 +8379 +8380 +8381 +8382 +8383 +8384 +8385 +8386 +8387 +8388 +8389 +8390 +8391 +8392 +8393 +8394 +8395 +8396 +8397 +8398 +8399 +8400 +8401 +8402 +8403 +8404 +8405 +8406 +8407 +8408 +8409 +8410 +8411 +8412 +8413 +8414 +8415 +8416 +8417 +8418 +8419 +8420 +8421 +8422 +8423 +8424 +8425 +8426 +8427 +8428 +8429 +8430 +8431 +8432 +8433 +8434 +8435 +8436 +8437 +8438 +8439 +8440 +8441 +8442 +8443 +8444 +8445 +8446 +8447 +8448 +8449 +8450 +8451 +8452 +8453 +8454 +8455 +8456 +8457 +8458 +8459 +8460 +8461 +8462 +8463 +8464 +8465 +8466 +8467 +8468 +8469 +8470 +8471 +8472 +8473 +8474 +8475 +8476 +8477 +8478 +8479 +8480 +8481 +8482 +8483 +8484 +8485 +8486 +8487 +8488 +8489 +8490 +8491 +8492 +8493 +8494 +8495 +8496 +8497 +8498 +8499 +8500 +8501 +8502 +8503 +8504 +8505 +8506 +8507 +8508 +8509 +8510 +8511 +8512 +8513 +8514 +8515 +8516 +8517 +8518 +8519 +8520 +8521 +8522 +8523 +8524 +8525 +8526 +8527 +8528 +8529 +8530 +8531 +8532 +8533 +8534 +8535 +8536 +8537 +8538 +8539 +8540 +8541 +8542 +8543 +8544 +8545 +8546 +8547 +8548 +8549 +8550 +8551 +8552 +8553 +8554 +8555 +8556 +8557 +8558 +8559 +8560 +8561 +8562 +8563 +8564 +8565 +8566 +8567 +8568 +8569 +8570 +8571 +8572 +8573 +8574 +8575 +8576 +8577 +8578 +8579 +8580 +8581 +8582 +8583 +8584 +8585 +8586 +8587 +8588 +8589 +8590 +8591 +8592 +8593 +8594 +8595 +8596 +8597 +8598 +8599 +8600 +8601 +8602 +8603 +8604 +8605 +8606 +8607 +8608 +8609 +8610 +8611 +8612 +8613 +8614 +8615 +8616 +8617 +8618 +8619 +8620 +8621 +8622 +8623 +8624 +8625 +8626 +8627 +8628 +8629 +8630 +8631 +8632 +8633 +8634 +8635 +8636 +8637 +8638 +8639 +8640 +8641 +8642 +8643 +8644 +8645 +8646 +8647 +8648 +8649 +8650 +8651 +8652 +8653 +8654 +8655 +8656 +8657 +8658 +8659 +8660 +8661 +8662 +8663 +8664 +8665 +8666 +8667 +8668 +8669 +8670 +8671 +8672 +8673 +8674 +8675 +8676 +8677 +8678 +8679 +8680 +8681 +8682 +8683 +8684 +8685 +8686 +8687 +8688 +8689 +8690 +8691 +8692 +8693 +8694 +8695 +8696 +8697 +8698 +8699 +8700 +8701 +8702 +8703 +8704 +8705 +8706 +8707 +8708 +8709 +8710 +8711 +8712 +8713 +8714 +8715 +8716 +8717 +8718 +8719 +8720 +8721 +8722 +8723 +8724 +8725 +8726 +8727 +8728 +8729 +8730 +8731 +8732 +8733 +8734 +8735 +8736 +8737 +8738 +8739 +8740 +8741 +8742 +8743 +8744 +8745 +8746 +8747 +8748 +8749 +8750 +8751 +8752 +8753 +8754 +8755 +8756 +8757 +8758 +8759 +8760 +8761 +8762 +8763 +8764 +8765 +8766 +8767 +8768 +8769 +8770 +8771 +8772 +8773 +8774 +8775 +8776 +8777 +8778 +8779 +8780 +8781 +8782 +8783 +8784 +8785 +8786 +8787 +8788 +8789 +8790 +8791 +8792 +8793 +8794 +8795 +8796 +8797 +8798 +8799 +8800 +8801 +8802 +8803 +8804 +8805 +8806 +8807 +8808 +8809 +8810 +8811 +8812 +8813 +8814 +8815 +8816 +8817 +8818 +8819 +8820 +8821 +8822 +8823 +8824 +8825 +8826 +8827 +8828 +8829 +8830 +8831 +8832 +8833 +8834 +8835 +8836 +8837 +8838 +8839 +8840 +8841 +8842 +8843 +8844 +8845 +8846 +8847 +8848 +8849 +8850 +8851 +8852 +8853 +8854 +8855 +8856 +8857 +8858 +8859 +8860 +8861 +8862 +8863 +8864 +8865 +8866 +8867 +8868 +8869 +8870 +8871 +8872 +8873 +8874 +8875 +8876 +8877 +8878 +8879 +8880 +8881 +8882 +8883 +8884 +8885 +8886 +8887 +8888 +8889 +8890 +8891 +8892 +8893 +8894 +8895 +8896 +8897 +8898 +8899 +8900 +8901 +8902 +8903 +8904 +8905 +8906 +8907 +8908 +8909 +8910 +8911 +8912 +8913 +8914 +8915 +8916 +8917 +8918 +8919 +8920 +8921 +8922 +8923 +8924 +8925 +8926 +8927 +8928 +8929 +8930 +8931 +8932 +8933 +8934 +8935 +8936 +8937 +8938 +8939 +8940 +8941 +8942 +8943 +8944 +8945 +8946 +8947 +8948 +8949 +8950 +8951 +8952 +8953 +8954 +8955 +8956 +8957 +8958 +8959 +8960 +8961 +8962 +8963 +8964 +8965 +8966 +8967 +8968 +8969 +8970 +8971 +8972 +8973 +8974 +8975 +8976 +8977 +8978 +8979 +8980 +8981 +8982 +8983 +8984 +8985 +8986 +8987 +8988 +8989 +8990 +8991 +8992 +8993 +8994 +8995 +8996 +8997 +8998 +8999 +9000 +9001 +9002 +9003 +9004 +9005 +9006 +9007 +9008 +9009 +9010 +9011 +9012 +9013 +9014 +9015 +9016 +9017 +9018 +9019 +9020 +9021 +9022 +9023 +9024 +9025 +9026 +9027 +9028 +9029 +9030 +9031 +9032 +9033 +9034 +9035 +9036 +9037 +9038 +9039 +9040 +9041 +9042 +9043 +9044 +9045 +9046 +9047 +9048 +9049 +9050 +9051 +9052 +9053 +9054 +9055 +9056 +9057 +9058 +9059 +9060 +9061 +9062 +9063 +9064 +9065 +9066 +9067 +9068 +9069 +9070 +9071 +9072 +9073 +9074 +9075 +9076 +9077 +9078 +9079 +9080 +9081 +9082 +9083 +9084 +9085 +9086 +9087 +9088 +9089 +9090 +9091 +9092 +9093 +9094 +9095 +9096 +9097 +9098 +9099 +9100 +9101 +9102 +9103 +9104 +9105 +9106 +9107 +9108 +9109 +9110 +9111 +9112 +9113 +9114 +9115 +9116 +9117 +9118 +9119 +9120 +9121 +9122 +9123 +9124 +9125 +9126 +9127 +9128 +9129 +9130 +9131 +9132 +9133 +9134 +9135 +9136 +9137 +9138 +9139 +9140 +9141 +9142 +9143 +9144 +9145 +9146 +9147 +9148 +9149 +9150 +9151 +9152 +9153 +9154 +9155 +9156 +9157 +9158 +9159 +9160 +9161 +9162 +9163 +9164 +9165 +9166 +9167 +9168 +9169 +9170 +9171 +9172 +9173 +9174 +9175 +9176 +9177 +9178 +9179 +9180 +9181 +9182 +9183 +9184 +9185 +9186 +9187 +9188 +9189 +9190 +9191 +9192 +9193 +9194 +9195 +9196 +9197 +9198 +9199 +9200 +9201 +9202 +9203 +9204 +9205 +9206 +9207 +9208 +9209 +9210 +9211 +9212 +9213 +9214 +9215 +9216 +9217 +9218 +9219 +9220 +9221 +9222 +9223 +9224 +9225 +9226 +9227 +9228 +9229 +9230 +9231 +9232 +9233 +9234 +9235 +9236 +9237 +9238 +9239 +9240 +9241 +9242 +9243 +9244 +9245 +9246 +9247 +9248 +9249 +9250 +9251 +9252 +9253 +9254 +9255 +9256 +9257 +9258 +9259 +9260 +9261 +9262 +9263 +9264 +9265 +9266 +9267 +9268 +9269 +9270 +9271 +9272 +9273 +9274 +9275 +9276 +9277 +9278 +9279 +9280 +9281 +9282 +9283 +9284 +9285 +9286 +9287 +9288 +9289 +9290 +9291 +9292 +9293 +9294 +9295 +9296 +9297 +9298 +9299 +9300 +9301 +9302 +9303 +9304 +9305 +9306 +9307 +9308 +9309 +9310 +9311 +9312 +9313 +9314 +9315 +9316 +9317 +9318 +9319 +9320 +9321 +9322 +9323 +9324 +9325 +9326 +9327 +9328 +9329 +9330 +9331 +9332 +9333 +9334 +9335 +9336 +9337 +9338 +9339 +9340 +9341 +9342 +9343 +9344 +9345 +9346 +9347 +9348 +9349 +9350 +9351 +9352 +9353 +9354 +9355 +9356 +9357 +9358 +9359 +9360 +9361 +9362 +9363 +9364 +9365 +9366 +9367 +9368 +9369 +9370 +9371 +9372 +9373 +9374 +9375 +9376 +9377 +9378 +9379 +9380 +9381 +9382 +9383 +9384 +9385 +9386 +9387 +9388 +9389 +9390 +9391 +9392 +9393 +9394 +9395 +9396 +9397 +9398 +9399 +9400 +9401 +9402 +9403 +9404 +9405 +9406 +9407 +9408 +9409 +9410 +9411 +9412 +9413 +9414 +9415 +9416 +9417 +9418 +9419 +9420 +9421 +9422 +9423 +9424 +9425 +9426 +9427 +9428 +9429 +9430 +9431 +9432 +9433 +9434 +9435 +9436 +9437 +9438 +9439 +9440 +9441 +9442 +9443 +9444 +9445 +9446 +9447 +9448 +9449 +9450 +9451 +9452 +9453 +9454 +9455 +9456 +9457 +9458 +9459 +9460 +9461 +9462 +9463 +9464 +9465 +9466 +9467 +9468 +9469 +9470 +9471 +9472 +9473 +9474 +9475 +9476 +9477 +9478 +9479 +9480 +9481 +9482 +9483 +9484 +9485 +9486 +9487 +9488 +9489 +9490 +9491 +9492 +9493 +9494 +9495 +9496 +9497 +9498 +9499 +9500 +9501 +9502 +9503 +9504 +9505 +9506 +9507 +9508 +9509 +9510 +9511 +9512 +9513 +9514 +9515 +9516 +9517 +9518 +9519 +9520 +9521 +9522 +9523 +9524 +9525 +9526 +9527 +9528 +9529 +9530 +9531 +9532 +9533 +9534 +9535 +9536 +9537 +9538 +9539 +9540 +9541 +9542 +9543 +9544 +9545 +9546 +9547 +9548 +9549 +9550 +9551 +9552 +9553 +9554 +9555 +9556 +9557 +9558 +9559 +9560 +9561 +9562 +9563 +9564 +9565 +9566 +9567 +9568 +9569 +9570 +9571 +9572 +9573 +9574 +9575 +9576 +9577 +9578 +9579 +9580 +9581 +9582 +9583 +9584 +9585 +9586 +9587 +9588 +9589 +9590 +9591 +9592 +9593 +9594 +9595 +9596 +9597 +9598 +9599 +9600 +9601 +9602 +9603 +9604 +9605 +9606 +9607 +9608 +9609 +9610 +9611 +9612 +9613 +9614 +9615 +9616 +9617 +9618 +9619 +9620 +9621 +9622 +9623 +9624 +9625 +9626 +9627 +9628 +9629 +9630 +9631 +9632 +9633 +9634 +9635 +9636 +9637 +9638 +9639 +9640 +9641 +9642 +9643 +9644 +9645 +9646 +9647 +9648 +9649 +9650 +9651 +9652 +9653 +9654 +9655 +9656 +9657 +9658 +9659 +9660 +9661 +9662 +9663 +9664 +9665 +9666 +9667 +9668 +9669 +9670 +9671 +9672 +9673 +9674 +9675 +9676 +9677 +9678 +9679 +9680 +9681 +9682 +9683 +9684 +9685 +9686 +9687 +9688 +9689 +9690 +9691 +9692 +9693 +9694 +9695 +9696 +9697 +9698 +9699 +9700 +9701 +9702 +9703 +9704 +9705 +9706 +9707 +9708 +9709 +9710 +9711 +9712 +9713 +9714 +9715 +9716 +9717 +9718 +9719 +9720 +9721 +9722 +9723 +9724 +9725 +9726 +9727 +9728 +9729 +9730 +9731 +9732 +9733 +9734 +9735 +9736 +9737 +9738 +9739 +9740 +9741 +9742 +9743 +9744 +9745 +9746 +9747 +9748 +9749 +9750 +9751 +9752 +9753 +9754 +9755 +9756 +9757 +9758 +9759 +9760 +9761 +9762 +9763 +9764 +9765 +9766 +9767 +9768 +9769 +9770 +9771 +9772 +9773 +9774 +9775 +9776 +9777 +9778 +9779 +9780 +9781 +9782 +9783 +9784 +9785 +9786 +9787 +9788 +9789 +9790 +9791 +9792 +9793 +9794 +9795 +9796 +9797 +9798 +9799 +9800 +9801 +9802 +9803 +9804 +9805 +9806 +9807 +9808 +9809 +9810 +9811 +9812 +9813 +9814 +9815 +9816 +9817 +9818 +9819 +9820 +9821 +9822 +9823 +9824 +9825 +9826 +9827 +9828 +9829 +9830 +9831 +9832 +9833 +9834 +9835 +9836 +9837 +9838 +9839 +9840 +9841 +9842 +9843 +9844 +9845 +9846 +9847 +9848 +9849 +9850 +9851 +9852 +9853 +9854 +9855 +9856 +9857 +9858 +9859 +9860 +9861 +9862 +9863 +9864 +9865 +9866 +9867 +9868 +9869 +9870 +9871 +9872 +9873 +9874 +9875 +9876 +9877 +9878 +9879 +9880 +9881 +9882 +9883 +9884 +9885 +9886 +9887 +9888 +9889 +9890 +9891 +9892 +9893 +9894 +9895 +9896 +9897 +9898 +9899 +9900 +9901 +9902 +9903 +9904 +9905 +9906 +9907 +9908 +9909 +9910 +9911 +9912 +9913 +9914 +9915 +9916 +9917 +9918 +9919 +9920 +9921 +9922 +9923 +9924 +9925 +9926 +9927 +9928 +9929 +9930 +9931 +9932 +9933 +9934 +9935 +9936 +9937 +9938 +9939 +9940 +9941 +9942 +9943 +9944 +9945 +9946 +9947 +9948 +9949 +9950 +9951 +9952 +9953 +9954 +9955 +9956 +9957 +9958 +9959 +9960 +9961 +9962 +9963 +9964 +9965 +9966 +9967 +9968 +9969 +9970 +9971 +9972 +9973 +9974 +9975 +9976 +9977 +9978 +9979 +9980 +9981 +9982 +9983 +9984 +9985 +9986 +9987 +9988 +9989 +9990 +9991 +9992 +9993 +9994 +9995 +9996 +9997 +9998 +9999 +10000 diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt b/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt new file mode 100644 index 00000000..d39f4855 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt @@ -0,0 +1,50 @@ +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 + + +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt b/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt new file mode 100644 index 00000000..d1517d07 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt @@ -0,0 +1,672 @@ +2000 +2003 +2006 +2009 +2012 +2015 +2018 +2021 +2024 +2027 +2030 +2033 +2036 +2039 +2042 +2045 +2048 +2051 +2054 +2057 +2060 +2063 +2066 +2069 +2072 +2075 +2078 +2081 +2084 +2087 +2090 +2093 +2096 +2099 +2102 +2105 +2108 +2111 +2114 +2117 +2120 +2123 +2126 +2129 +2132 +2135 +2138 +2141 +2144 +2147 +2150 +2153 +2156 +2159 +2162 +2165 +2168 +2171 +2174 +2177 +2180 +2183 +2186 +2189 +2192 +2195 +2198 +2201 +2204 +2207 +2210 +2213 +2216 +2219 +2222 +2225 +2228 +2231 +2234 +2237 +2240 +2243 +2246 +2249 +2252 +2255 +2258 +2261 +2264 +2267 +2270 +2273 +2276 +2279 +2282 +2285 +2288 +2291 +2294 +2297 +2300 +2303 +2306 +2309 +2312 +2315 +2318 +2321 +2324 +2327 +2330 +2333 +2336 +2339 +2342 +2345 +2348 +2351 +2354 +2357 +2360 +2363 +2366 +2369 +2372 +2375 +2378 +2381 +2384 +2387 +2390 +2393 +2396 +2399 +2402 +2405 +2408 +2411 +2414 +2417 +2420 +2423 +2426 +2429 +2432 +2435 +2438 +2441 +2444 +2447 +2450 +2453 +2456 +2459 +2462 +2465 +2468 +2471 +2474 +2477 +2480 +2483 +2486 +2489 +2492 +2495 +2498 +2501 +2504 +2507 +2510 +2513 +2516 +2519 +2522 +2525 +2528 +2531 +2534 +2537 +2540 +2543 +2546 +2549 +2552 +2555 +2558 +2561 +2564 +2567 +2570 +2573 +2576 +2579 +2582 +2585 +2588 +2591 +2594 +2597 +2600 +2603 +2606 +2609 +2612 +2615 +2618 +2621 +2624 +2627 +2630 +2633 +2636 +2639 +2642 +2645 +2648 +2651 +2654 +2657 +2660 +2663 +2666 +2669 +2672 +2675 +2678 +2681 +2684 +2687 +2690 +2693 +2696 +2699 +2702 +2705 +2708 +2711 +2714 +2717 +2720 +2723 +2726 +2729 +2732 +2735 +2738 +2741 +2744 +2747 +2750 +2753 +2756 +2759 +2762 +2765 +2768 +2771 +2774 +2777 +2780 +2783 +2786 +2789 +2792 +2795 +2798 +2801 +2804 +2807 +2810 +2813 +2816 +2819 +2822 +2825 +2828 +2831 +2834 +2837 +2840 +2843 +2846 +2849 +2852 +2855 +2858 +2861 +2864 +2867 +2870 +2873 +2876 +2879 +2882 +2885 +2888 +2891 +2894 +2897 +2900 +2903 +2906 +2909 +2912 +2915 +2918 +2921 +2924 +2927 +2930 +2933 +2936 +2939 +2942 +2945 +2948 +2951 +2954 +2957 +2960 +2963 +2966 +2969 +2972 +2975 +2978 +2981 +2984 +2987 +2990 +2993 +2996 +2999 +3002 +3005 +3008 +3011 +3014 +3017 +3020 +3023 +3026 +3029 +3032 +3035 +3038 +3041 +3044 +3047 +3050 +3053 +3056 +3059 +3062 +3065 +3068 +3071 +3074 +3077 +3080 +3083 +3086 +3089 +3092 +3095 +3098 +3101 +3104 +3107 +3110 +3113 +3116 +3119 +3122 +3125 +3128 +3131 +3134 +3137 +3140 +3143 +3146 +3149 +3152 +3155 +3158 +3161 +3164 +3167 +3170 +3173 +3176 +3179 +3182 +3185 +3188 +3191 +3194 +3197 +3200 +3203 +3206 +3209 +3212 +3215 +3218 +3221 +3224 +3227 +3230 +3233 +3236 +3239 +3242 +3245 +3248 +3251 +3254 +3257 +3260 +3263 +3266 +3269 +3272 +3275 +3278 +3281 +3284 +3287 +3290 +3293 +3296 +3299 +3302 +3305 +3308 +3311 +3314 +3317 +3320 +3323 +3326 +3329 +3332 +3335 +3338 +3341 +3344 +3347 +3350 +3353 +3356 +3359 +3362 +3365 +3368 +3371 +3374 +3377 +3380 +3383 +3386 +3389 +3392 +3395 +3398 +3401 +3404 +3407 +3410 +3413 +3416 +3419 +3422 +3425 +3428 +3431 +3434 +3437 +3440 +3443 +3446 +3449 +3452 +3455 +3458 +3461 +3464 +3467 +3470 +3473 +3476 +3479 +3482 +3485 +3488 +3491 +3494 +3497 +3500 +3503 +3506 +3509 +3512 +3515 +3518 +3521 +3524 +3527 +3530 +3533 +3536 +3539 +3542 +3545 +3548 +3551 +3554 +3557 +3560 +3563 +3566 +3569 +3572 +3575 +3578 +3581 +3584 +3587 +3590 +3593 +3596 +3599 +3602 +3605 +3608 +3611 +3614 +3617 +3620 +3623 +3626 +3629 +3632 +3635 +3638 +3641 +3644 +3647 +3650 +3653 +3656 +3659 +3662 +3665 +3668 +3671 +3674 +3677 +3680 +3683 +3686 +3689 +3692 +3695 +3698 +3701 +3704 +3707 +3710 +3713 +3716 +3719 +3722 +3725 +3728 +3731 +3734 +3737 +3740 +3743 +3746 +3749 +3752 +3755 +3758 +3761 +3764 +3767 +3770 +3773 +3776 +3779 +3782 +3785 +3788 +3791 +3794 +3797 +3800 +3803 +3806 +3809 +3812 +3815 +3818 +3821 +3824 +3827 +3830 +3833 +3836 +3839 +3842 +3845 +3848 +3851 +3854 +3857 +3860 +3863 +3866 +3869 +3872 +3875 +3878 +3881 +3884 +3887 +3890 +3893 +3896 +3899 +3902 +3905 +3908 +3911 +3914 +3917 +3920 +3923 +3926 +3929 +3932 +3935 +3938 +3941 +3944 +3947 +3950 +3953 +3956 +3959 +3962 +3965 +3968 +3971 +3974 +3977 +3980 +3983 +3986 +3989 +3992 +3995 +3998 +abcdefg0 +abcdefg1 +abcdefg2 +abcdefg3 +abcdefg0 diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/Common.h b/cpp/ppc-psi/tests/ra2018-psi/mock/Common.h new file mode 100644 index 00000000..9786e518 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock/Common.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-12-29 + */ +#pragma once +#include "ppc-protocol/src/JsonTaskImpl.h" +#include "test-utils/TaskMock.h" +#include + +using namespace bcos; +using namespace ppc::protocol; + +namespace ppc::test +{ +template +inline void checkTaskPSIResult(T _factory, ppc::protocol::Task::ConstPtr _task, + uint64_t expectedResultSize, std::vector _expectedPSIResult) +{ + auto outputDesc = _task->selfParty()->dataResource()->outputDesc(); + auto reader = _factory->resourceLoader()->loadReader(outputDesc, DataSchema::String, false); + // get all result + DataBatch::Ptr result = reader->next(-1); + // check the result + std::cout << "### result size:" << (result ? result->size() : 0) << std::endl; + std::cout << "### expectedResultSize: " << expectedResultSize << std::endl; + BOOST_CHECK(result->size() == expectedResultSize); + if (_expectedPSIResult.empty()) + { + return; + } + // Note: the order of the result is not the same with _expectedPSIResult + std::set expectedResult(_expectedPSIResult.begin(), _expectedPSIResult.end()); + for (uint64_t i = 0; i < result->size(); i++) + { + BOOST_CHECK(expectedResult.count(result->get(i))); + } +} + +// online psi +template +void testPSI(T _factory, S _server, S _client, ppc::protocol::Task::ConstPtr _serverPsiTask, + ppc::protocol::Task::ConstPtr _clientPsiTask, bool _expectedSuccess, + std::vector const& _expectedPSIResult, int _expectedErrorCode = 0) +{ + bool serverFinished = false; + bool clientFinished = false; + _server->asyncRunTask(_serverPsiTask, [_serverPsiTask, _expectedSuccess, _expectedErrorCode, + &serverFinished]( + ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _serverPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + std::cout << "### response errorCode: " << _response->error()->errorCode() << std::endl; + std::cout << "### _expectedErrorCode: " << _expectedErrorCode << std::endl; + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); + } + serverFinished = true; + std::cout << "#### testPSI, the server asyncRunTask success" << std::endl; + }); + _client->asyncRunTask(_clientPsiTask, [_clientPsiTask, _expectedSuccess, &clientFinished]( + ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _clientPsiTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + clientFinished = true; + std::cout << "#### testPSI, the client asyncRunTask success" << std::endl; + }); + // wait for the task finish and check + while (!clientFinished || !serverFinished || _server->pendingTasksSize() > 0 || + _client->pendingTasksSize() > 0) + { + _server->executeWorker(); + _client->executeWorker(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + BOOST_CHECK(_client->pendingTasksSize() == 0); + BOOST_CHECK(_client->lockingResourceSize() == 0); + BOOST_CHECK(_server->pendingTasksSize() == 0); + BOOST_CHECK(_server->lockingResourceSize() == 0); + if (_expectedSuccess) + { + checkTaskPSIResult(_factory, _clientPsiTask, _expectedPSIResult.size(), _expectedPSIResult); + } +} + +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h b/cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h new file mode 100644 index 00000000..6b155285 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIFixutre.h + * @author: yujiechen + * @date 2022-12-29 + */ +#pragma once +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "test-utils/FakeFront.h" +#include "test-utils/FakePPCMessage.h" +#include + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; +using namespace ppc::tools; + +namespace ppc::test +{ +class FakeEcdhPSIImpl : public EcdhPSIImpl +{ +public: + using Ptr = std::shared_ptr; + FakeEcdhPSIImpl(EcdhPSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : EcdhPSIImpl(_config, _idleTimeMs) + { + // set the m_started flag to be true + m_started = true; + m_enableOutputExists = true; + m_taskSyncTimer->registerTimeoutHandler([this]() { syncTaskInfo(); }); + m_taskSyncTimer->start(); + } + ~FakeEcdhPSIImpl() override = default; +}; + +class FakeEcdhPSIFactory : public EcdhPSIFactory +{ +public: + using Ptr = std::shared_ptr; + FakeEcdhPSIFactory() + : m_front(std::make_shared()), + m_ppcMsgFactory(std::make_shared()), + m_dataResourceLoader(std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), + m_threadPool(std::make_shared("ra2018-psi", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } + + ~FakeEcdhPSIFactory() override = default; + + EcdhPSIImpl::Ptr createEcdhPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, + ppc::front::FrontInterface::Ptr _front, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader) override + { + auto psiMsgFactory = std::make_shared(); + auto const& ecdhParam = _ppcConfig->ecdhPSIConfig(); + auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory, + _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize, + 10000, _threadPool); + // enforce the taskExpireTime to 3000ms + config->setTaskExpireTime(3000); + return std::make_shared(config); + } + + EcdhPSIImpl::Ptr createEcdhPSI(std::string const& _selfParty, PPCConfig::Ptr const& _config) + { + _config->setAgencyID(_selfParty); + auto const& ra2018Config = _config->ra2018PSIConfig(); + auto ecdhCryptoFactory = std::make_shared(_config->privateKey()); + return createEcdhPSI( + _config, ecdhCryptoFactory, m_front, m_ppcMsgFactory, nullptr, m_dataResourceLoader); + } + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + +private: + FakeFront::Ptr m_front; + CryptoBox::Ptr m_cryptoBox; + FakePPCMessageFactory::Ptr m_ppcMsgFactory; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h b/cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h new file mode 100644 index 00000000..343f2b69 --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FakeRA2018PSIStorage.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h" + +using namespace ppc::psi; +using namespace ppc::crypto; + +namespace ppc::test +{ +class FakeRA2018PSIStorage : public RA2018PSIStorage +{ +public: + FakeRA2018PSIStorage(RA2018PSIConfig::Ptr _config) + : RA2018PSIStorage(_config), m_hashImpl(std::make_shared()) + {} + + ~FakeRA2018PSIStorage() override = default; + + void init() override {} + // get the (cuckoo-filter-id, cuckoo-filter-hash according to the _resourceID and loadFactor + // TODO: optimize the perf(use async-client) + CuckooFilterInfoSet getCuckooFilterInfos( + std::string const& _resourceID, bool _onlyNotFull) override + { + CuckooFilterInfoSet result; + bcos::ReadGuard l(x_cuckooFilterInfo); + if (!m_cuckooFilterInfo.count(_resourceID)) + { + return result; + } + auto const& filters = m_cuckooFilterInfo.at(_resourceID); + for (auto const& it : filters) + { + auto filter = it.second->cuckooFilter(); + if (_onlyNotFull && !filter->full()) + { + result.insert(it.second); + continue; + } + result.insert(it.second); + } + return result; + } + + // load the cuckoo-filter from the storage + CuckooFilterInfo::Ptr loadCuckooFilterDataInfo( + std::string const& _resourceID, int32_t _filterID, bool _deserialize) override + { + bcos::ReadGuard l(x_cuckooFilterInfo); + if (!m_cuckooFilterInfo.count(_resourceID) || + !m_cuckooFilterInfo.at(_resourceID).count(_filterID)) + { + return nullptr; + } + auto filterInfo = m_cuckooFilterInfo[_resourceID][_filterID]; + // with raw-data + if (!_deserialize) + { + auto cuckooFilterInfo = + std::make_shared(filterInfo->filterID(), filterInfo->hash()); + auto encodedData = filterInfo->cuckooFilter()->serialize(); + bcos::bytes data(encodedData.begin(), encodedData.end()); + // set the cuckoo-filterData + cuckooFilterInfo->setCuckooFilterData(std::move(data)); + return cuckooFilterInfo; + } + // with deserialized cuckoo-filter + return filterInfo; + } + + // insert a new cuckoo-filter to the storage + // update the entry if key-conflict + void asyncInsertCuckooFilter(std::string const& _resourceID, + DefaultCukooFilterPtr _cuckooFilter, std::function _callback, + int32_t _filterID = -1) override + { + auto filterID = _filterID; + bcos::WriteGuard l(x_cuckooFilterInfo); + if (filterID == -1) + { + m_filterID++; + filterID = m_filterID; + } + m_cuckooFilterInfo[_resourceID][filterID] = + generateCuckooFilterInfo(filterID, _cuckooFilter); + _callback(nullptr); + } + + // update the cuckoo-filter + // Note: update is more efficient than asyncInsertCuckooFilter + void asyncUpdateCuckooFilter(std::string const& _resourceID, + CuckooFilterInfo::Ptr const& _filterInfo, + std::function _callback) override + { + bcos::WriteGuard l(x_cuckooFilterInfo); + if (!m_cuckooFilterInfo.count(_resourceID) || + !m_cuckooFilterInfo.at(_resourceID).count(_filterInfo->filterID())) + { + _callback(std::make_shared( + -1, "update cuckoo-filter failed for the entry not exist!")); + return; + } + m_cuckooFilterInfo[_resourceID][_filterInfo->filterID()] = + generateCuckooFilterInfo(_filterInfo->filterID(), _filterInfo->cuckooFilter()); + _callback(nullptr); + } + +private: + CuckooFilterInfo::Ptr generateCuckooFilterInfo( + int32_t _filterID, DefaultCukooFilterPtr _cuckooFilter) + { + auto encodedData = _cuckooFilter->serialize(); + auto hashResult = m_hashImpl->hash( + bcos::bytesConstRef((bcos::byte*)encodedData.data(), encodedData.size())); + auto cuckooFilterInfo = std::make_shared(_filterID, hashResult); + cuckooFilterInfo->setCuckooFilter(std::move(_cuckooFilter)); + return cuckooFilterInfo; + } + +private: + Hash::Ptr m_hashImpl; + // resourceID => filterID => cuckooFilter + std::map> m_cuckooFilterInfo; + mutable bcos::SharedMutex x_cuckooFilterInfo; + int32_t m_filterID = 0; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h b/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h new file mode 100644 index 00000000..22c0bbec --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018MessageFixture.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-psi/src/ra2018-psi/protocol/RA2018Message.h" +#include +#include +#include +#include + +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::tools; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +inline CuckooFilterInfo::Ptr fakeCuckooFilterInfo( + int32_t _filterID, Hash::Ptr hashImpl, CuckoofilterOption::Ptr option, uint64_t insertedKeySize) +{ + auto filterInfo = std::make_shared(_filterID); + auto filter = std::make_shared>(option); + std::vector hashList; + for (uint64_t i = 0; i < insertedKeySize; i++) + { + std::string input = std::to_string(i); + bcos::bytes inputData(input.begin(), input.end()); + hashList.emplace_back(hashImpl->hash(ref(inputData))); + } + // insert the hashList and check + auto tmpHashList = hashList; + auto result = filter->batchInsert(tmpHashList); + BOOST_CHECK(tmpHashList.empty() == true); + BOOST_CHECK(result); + // set the hash + auto md5Hash = std::make_shared(); + auto encodedData = filter->serialize(); + auto hashResult = + md5Hash->hash(bytesConstRef((bcos::byte*)encodedData.data(), encodedData.size())); + filterInfo->setHash(hashResult); + // set the cuckoo-filter + filterInfo->setCuckooFilter(std::move(filter)); + return filterInfo; +} + +inline void checkMessage(PSIMessageInterface::Ptr _msg, uint32_t _packetType, + std::string const& _partyID, std::string const& _resourceID, int32_t _version, + std::vector const& _data) +{ + // check the message + BOOST_CHECK(_msg->partyID() == _partyID); + BOOST_CHECK(_msg->resourceID() == _resourceID); + BOOST_CHECK(_msg->version() == _version); + BOOST_CHECK(_data.size() == _msg->dataSize()); + uint64_t i = 0; + for (uint64_t i = 0; i < _msg->dataSize(); i++) + { + auto msgData = _msg->getData(i); + BOOST_CHECK(_data.at(i) == bytes(msgData.begin(), msgData.end())); + } +} + +inline void fakeAndCheckRA2018Message(PSIMessageInterface::Ptr msg, + RA2018MessageFactory::Ptr _msgFactory, uint32_t _packetType, std::string const& _partyID, + std::string const& _resourceID, int32_t _version, std::vector const& _data) +{ + msg->setPacketType(_packetType); + msg->setPartyID(_partyID); + msg->setResourceID(_resourceID); + msg->setVersion(_version); + auto tmpData = _data; + msg->setData(std::move(tmpData)); + + // check the result + checkMessage(msg, _packetType, _partyID, _resourceID, _version, _data); + // encode + auto encodedData = msg->encode(); + // decode + auto decodedMsg = _msgFactory->decodePSIMessage(ref(*encodedData)); + checkMessage(decodedMsg, _packetType, _partyID, _resourceID, _version, _data); + + // check takeData + auto takedData = decodedMsg->takeData(); + uint64_t i = 0; + for (auto const& it : takedData) + { + BOOST_CHECK(_data.at(i) == it); + i++; + } +} + +inline void checkCuckooFilter( + RA2018FilterMessage::Ptr _msg, std::vector _cuckooFilters) +{ + auto& msgCuckooFilters = _msg->mutableFilterInfo(); + BOOST_CHECK(msgCuckooFilters.size() == _cuckooFilters.size()); + auto md5Hash = std::make_shared(); + for (uint64_t i = 0; i < msgCuckooFilters.size(); i++) + { + auto msgCuckooFilter = msgCuckooFilters.at(i); + auto cuckooFilter = _cuckooFilters.at(i); + BOOST_CHECK(msgCuckooFilter->filterID() == cuckooFilter->filterID()); + BOOST_CHECK(msgCuckooFilter->hash() == cuckooFilter->hash()); + if (cuckooFilter->cuckooFilter()) + { + BOOST_CHECK(msgCuckooFilter->cuckooFilter() != nullptr); + auto cuckooFilterEncodedData = cuckooFilter->cuckooFilter()->serialize(); + BOOST_CHECK(msgCuckooFilter->cuckooFilter()->serialize() == cuckooFilterEncodedData); + // check the hash + auto hashResult = md5Hash->hash(bytesConstRef( + (bcos::byte*)(cuckooFilterEncodedData.data()), cuckooFilterEncodedData.size())); + BOOST_CHECK(hashResult == cuckooFilter->hash()); + } + BOOST_CHECK(msgCuckooFilter->cuckooFilterData() == cuckooFilter->cuckooFilterData()); + } +} + +inline void fakeAndCheckRA2018FilterMessage(PSIMessageInterface::Ptr _msg, + RA2018MessageFactory::Ptr _msgFactory, uint32_t _packetType, std::string const& _partyID, + std::string const& _resourceID, int32_t _version, std::vector const& _data, + std::vector _cuckooFilters) +{ + auto msg = std::dynamic_pointer_cast(_msg); + // with empty cuckoo-filter + fakeAndCheckRA2018Message( + msg, _msgFactory, _packetType, _partyID, _resourceID, _version, _data); + + // set the cuckoo-filter + auto cuckooFilters = _cuckooFilters; + msg->setFilterInfo(std::move(_cuckooFilters)); + // check the cuckooFilters + checkCuckooFilter(msg, cuckooFilters); + + // encode + auto encodedData = msg->encode(); + auto decodedMsg = _msgFactory->decodePSIMessage(ref(*encodedData)); + checkMessage(decodedMsg, _packetType, _partyID, _resourceID, _version, _data); + auto decodedFilterMsg = std::dynamic_pointer_cast(decodedMsg); + // check the cuckoo-filters + checkCuckooFilter(decodedFilterMsg, cuckooFilters); +} + +inline void fakeAndCheckNotificationMessage(PSIMessageInterface::Ptr _msg, + RA2018MessageFactory::Ptr _msgFactory, uint32_t _packetType, std::string const& _partyID, + std::string const& _resourceID, int32_t _version, std::vector const& _data, + int32_t _errorCode, std::string const& _errorMsg) +{ + auto msg = std::dynamic_pointer_cast(_msg); + // with default errorCode/errorMessage + fakeAndCheckRA2018Message( + msg, _msgFactory, _packetType, _partyID, _resourceID, _version, _data); + msg->setErrorCode(_errorCode); + msg->setErrorMessage(_errorMsg); + + BOOST_CHECK(msg->errorCode() == _errorCode); + BOOST_CHECK(msg->errorMessage() == _errorMsg); + // encode + auto encodedData = msg->encode(); + // decode + auto decodedMsg = _msgFactory->decodePSIMessage(ref(*encodedData)); + auto decodedNotificationMsg = std::dynamic_pointer_cast(decodedMsg); + // check the errorCode/errorMessage + BOOST_CHECK(decodedNotificationMsg->errorCode() == _errorCode); + BOOST_CHECK(decodedNotificationMsg->errorMessage() == _errorMsg); +} + +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h b/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h new file mode 100644 index 00000000..d5a9d1bc --- /dev/null +++ b/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIFixture.h + * @author: yujiechen + * @date 2022-11-6 + */ +#pragma once +#include "FakeRA2018PSIStorage.h" +#include "ppc-crypto-core/src/hash/MD5Hash.h" +#include "ppc-crypto-core/src/hash/Sha256Hash.h" +#include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" +#include "ppc-crypto/src/oprf/RA2018Oprf.h" +#include "ppc-framework/crypto/CryptoBox.h" +#include "ppc-io/src/DataResourceLoaderImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "test-utils/FakeFront.h" +#include "test-utils/FakePPCMessage.h" +#include + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; +using namespace ppc::tools; + +namespace ppc::test +{ +class FakeRA2018Impl : public RA2018PSIImpl +{ +public: + using Ptr = std::shared_ptr; + FakeRA2018Impl(RA2018PSIConfig::Ptr _config, RA2018PSIStorage::Ptr _storage, + unsigned _idleTimeMs = 0, bool _waitResult = false, bool _disabled = false) + : RA2018PSIImpl(_config, _storage, _idleTimeMs, _waitResult, _disabled) + { + // set the m_started flag to be true + m_started = true; + m_enableOutputExists = true; + m_taskSyncTimer->registerTimeoutHandler([this]() { syncTaskInfo(); }); + m_taskSyncTimer->start(); + } + + ~FakeRA2018Impl() override = default; +}; + +class FakeRA2018PSIFactory : public RA2018PSIFactory +{ +public: + using Ptr = std::shared_ptr; + FakeRA2018PSIFactory() + : m_front(std::make_shared()), + m_ppcMsgFactory(std::make_shared()), + m_dataResourceLoader(std::make_shared( + nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), + m_threadPool(std::make_shared("ra2018-psi", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared(hashImpl, ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + m_binHashImpl = std::make_shared(); + } + + ~FakeRA2018PSIFactory() override = default; + + RA2018PSIImpl::Ptr createRA2018PSI(std::string const& _selfParty, + ppc::front::FrontInterface::Ptr _front, ppc::tools::PPCConfig::Ptr const& _config, + ppc::crypto::RA2018OprfInterface::Ptr const& _oprf, ppc::crypto::Hash::Ptr _binHashImpl, + ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, + ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage, + bcos::ThreadPool::Ptr _threadPool, + ppc::io::DataResourceLoader::Ptr _dataResourceLoader) override + { + auto const& ra2018Config = _config->ra2018PSIConfig(); + auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl, + _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage, + _dataResourceLoader, 10000, ra2018Config.dbName, ra2018Config.cuckooFilterCacheSize, + ra2018Config.cacheSize, ra2018Config.dataBatchSize); + // enforce the taskExpireTime to 3000ms + config->setTaskExpireTime(3000); + // use the FakeRA2018PSIStorage + auto PSIStorage = std::make_shared(config); + // wait-for the task completed or exceptioned to call the notification-callback + return std::make_shared(config, PSIStorage, 0, true, false); + } + + RA2018PSIImpl::Ptr createRA2018PSI(std::string const& _selfParty, PPCConfig::Ptr const& _config) + { + auto const& ra2018Config = _config->ra2018PSIConfig(); + auto oprf = std::make_shared( + _config->privateKey(), m_cryptoBox->eccCrypto(), m_cryptoBox->hashImpl()); + return createRA2018PSI(_selfParty, m_front, _config, oprf, m_binHashImpl, m_ppcMsgFactory, + nullptr, nullptr, m_threadPool, m_dataResourceLoader); + } + + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + +private: + FakeFront::Ptr m_front; + CryptoBox::Ptr m_cryptoBox; + Hash::Ptr m_binHashImpl; + FakePPCMessageFactory::Ptr m_ppcMsgFactory; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-rpc/CMakeLists.txt b/cpp/ppc-rpc/CMakeLists.txt new file mode 100644 index 00000000..f111a675 --- /dev/null +++ b/cpp/ppc-rpc/CMakeLists.txt @@ -0,0 +1,12 @@ +project(ppc-rpc VERSION ${VERSION}) +add_subdirectory(src) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() + +if (DEMO) + add_subdirectory(demo) +endif() \ No newline at end of file diff --git a/cpp/ppc-rpc/demo/CMakeLists.txt b/cpp/ppc-rpc/demo/CMakeLists.txt new file mode 100644 index 00000000..b3b0bfa1 --- /dev/null +++ b/cpp/ppc-rpc/demo/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(BINARY_NAME rpc-demo) + +add_executable(${BINARY_NAME} ${SOURCES}) +target_include_directories(${BINARY_NAME} PRIVATE .) +target_link_libraries(${BINARY_NAME} ${RPC_TARGET} ${HELPER_TARGET} ${PROTOCOL_TARGET} TBB::tbb) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-rpc/demo/config.ini b/cpp/ppc-rpc/demo/config.ini new file mode 100644 index 00000000..2357f4e0 --- /dev/null +++ b/cpp/ppc-rpc/demo/config.ini @@ -0,0 +1,14 @@ +[agency] + id = agency0 + +[rpc] + listen_ip=0.0.0.0 + listen_port=8550 + thread_count=4 + ; ssl or sm ssl + sm_ssl=false + cert_path=conf + ; ssl connection switch, if disable the ssl connection, default: false + disable_ssl=true + ; GB, reject task if available memory is less than min_needed_memory + ; min_needed_memory = 5 diff --git a/cpp/ppc-rpc/demo/rpc_demo.cpp b/cpp/ppc-rpc/demo/rpc_demo.cpp new file mode 100644 index 00000000..f51ecc3c --- /dev/null +++ b/cpp/ppc-rpc/demo/rpc_demo.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file rpc_demo.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "libhelper/CommandHelper.h" +#include "libhelper/ExitHandler.h" +#include "ppc-rpc/src/RpcFactory.h" +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::rpc; +using namespace ppc::tools; + +void registerEchoHandler(RpcInterface::Ptr _rpc) +{ + _rpc->registerHandler("echo", [](Json::Value const& _request, RespFunc _func) { + Json::Value response; + response["request"] = _request; + response["tag"] = "echo"; + _func(nullptr, std::move(response)); + }); +} + +int main(int argc, const char* argv[]) +{ + ExitHandler exitHandler; + signal(SIGTERM, &ExitHandler::exitHandler); + signal(SIGABRT, &ExitHandler::exitHandler); + signal(SIGINT, &ExitHandler::exitHandler); + try + { + auto param = initCommandLine(argc, argv); + auto ppcConfig = std::make_shared(); + // not specify the certPath in air-mode + ppcConfig->loadRpcConfig(nullptr, param.configFilePath); + auto rpcFactory = std::make_shared("selfParty"); + auto rpc = rpcFactory->buildRpc(ppcConfig); + registerEchoHandler(rpc); + // start the rpc + rpc->start(); + while (!exitHandler.shouldExit()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "rpc-demo program exit normally." << std::endl; + } + catch (std::exception const& e) + { + ppc::printVersion(); + std::cout << "[" << bcos::getCurrentDateTime() << "] "; + std::cout << "start rpc-demo failed, error:" << boost::diagnostic_information(e) + << std::endl; + return -1; + } +} \ No newline at end of file diff --git a/cpp/ppc-rpc/src/CMakeLists.txt b/cpp/ppc-rpc/src/CMakeLists.txt new file mode 100644 index 00000000..ecec04a6 --- /dev/null +++ b/cpp/ppc-rpc/src/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${RPC_TARGET} ${SRCS}) + +target_link_libraries(${RPC_TARGET} PUBLIC ${TOOLS_TARGET} ${STORAGE_TARGET} ${PROTOCOL_TARGET} jsoncpp_static ${BCOS_BOOSTSSL_TARGET}) diff --git a/cpp/ppc-rpc/src/JsonRequest.h b/cpp/ppc-rpc/src/JsonRequest.h new file mode 100644 index 00000000..00bdc0d3 --- /dev/null +++ b/cpp/ppc-rpc/src/JsonRequest.h @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JsonRequest.h + * @author: yujiechen + * @date 2022-11-3 + */ +#pragma once +#include "ppc-framework/rpc/RpcTypeDef.h" +#include +#include +#include +#include + +namespace ppc::rpc +{ +class JsonRequest +{ +public: + using Ptr = std::shared_ptr; + JsonRequest() = default; + JsonRequest(std::string_view _data) { deserialize(_data); } + + virtual ~JsonRequest() {} + + virtual void setJsonRpc(std::string const& _jsonrpc) { m_jsonrpc = _jsonrpc; } + virtual void setMethod(std::string const& _method) { m_method = _method; } + virtual void setId(int64_t _id) { m_id = _id; } + virtual void setParams(Json::Value const& _params) { m_params = _params; } + + virtual std::string const& jsonRpc() const { return m_jsonrpc; } + virtual std::string const& token() const { return m_token; } + virtual int64_t id() const { return m_id; } + virtual Json::Value const& params() const { return m_params; } + virtual std::string const& method() const { return m_method; } + +protected: + virtual void deserialize(std::string_view _requestBody) + { + Json::Value root; + Json::Reader reader; + if (!reader.parse(_requestBody.begin(), _requestBody.end(), root)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)RpcError::InvalidRequest, + "invalid request for parse to json object failed")); + } + // parse jsonrpc field + if (!root.isMember("jsonrpc")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::InvalidRequest, "Must set the jsonrpc field")); + } + m_jsonrpc = root["jsonrpc"].asString(); + // parse jsonrpc field + if (!root.isMember("token")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::InvalidRequest, "Must set the token field")); + } + m_token = root["token"].asString(); + // parse method field + if (!root.isMember("method")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::InvalidRequest, "Must define the method field")); + } + m_method = root["method"].asString(); + // parse the id field, optional + if (root.isMember("id")) + { + m_id = root["id"].asInt64(); + } + // parse the params field + if (!root.isMember("params")) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::InvalidRequest, "Must define the params field")); + } + m_params = root["params"]; + } + +private: + std::string m_jsonrpc; + std::string m_token; + std::string m_method; + int64_t m_id; + Json::Value m_params; +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-rpc/src/JsonResponse.h b/cpp/ppc-rpc/src/JsonResponse.h new file mode 100644 index 00000000..d37cd2c0 --- /dev/null +++ b/cpp/ppc-rpc/src/JsonResponse.h @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file JsonResponse.h + * @author: yujiechen + * @date 2022-11-3 + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace ppc::rpc +{ +class JsonSink +{ +public: + typedef char char_type; + typedef boost::iostreams::sink_tag category; + + JsonSink(bcos::bytes& buffer) : m_buffer(buffer) {} + + std::streamsize write(const char* s, std::streamsize n) + { + m_buffer.insert(m_buffer.end(), (bcos::byte*)s, (bcos::byte*)s + n); + return n; + } + + bcos::bytes& m_buffer; +}; +class JsonResponse +{ +public: + using Ptr = std::shared_ptr; + JsonResponse() : m_error(std::make_shared(0, "success")) {} + virtual ~JsonResponse() = default; + + void setJsonRpc(std::string const& _jsonrpc) { m_jsonrpc = _jsonrpc; } + void setId(int64_t _id) { m_id = _id; } + void setError(bcos::Error const& _error) { *m_error = _error; } + void setError(bcos::Error&& _error) { *m_error = std::move(_error); } + void setResult(Json::Value&& _result) { m_result = std::move(_result); } + + bcos::Error::Ptr mutableError() { return m_error; } + + virtual bcos::bytes serialize() + { + Json::Value response; + response["jsonrpc"] = m_jsonrpc; + response["id"] = m_id; + if (m_error && m_error->errorCode()) + { + m_result["code"] = m_error->errorCode(); + m_result["message"] = m_error->errorMessage(); + } + response["result"] = m_result; + bcos::bytes out; + boost::iostreams::stream outputStream(out); + std::unique_ptr writer(Json::StreamWriterBuilder().newStreamWriter()); + writer->write(response, &outputStream); + writer.reset(); + return out; + } + +private: + std::string m_jsonrpc; + int64_t m_id; + bcos::Error::Ptr m_error; + Json::Value m_result; +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-rpc/src/Rpc.cpp b/cpp/ppc-rpc/src/Rpc.cpp new file mode 100644 index 00000000..66f0972c --- /dev/null +++ b/cpp/ppc-rpc/src/Rpc.cpp @@ -0,0 +1,458 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Rpc.cpp + * @author: yujiechen + * @date 2022-11-3 + */ +#include "Rpc.h" +#include "JsonRequest.h" +#include "JsonResponse.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/rpc/RpcTypeDef.h" +#include "ppc-tools/src/common/MemInfo.h" +#include "ppc-tools/src/config/ParamChecker.h" +#include + +using namespace bcos; +using namespace ppc::rpc; +using namespace ppc::tools; +using namespace ppc::protocol; + +Rpc::Rpc(std::shared_ptr _wsService, std::string const& _selfPartyID, + std::string const& _token, std::string const& _prePath) + : m_prePath(_prePath), + m_wsService(std::move(_wsService)), + m_taskFactory(std::make_shared(_selfPartyID, _prePath)), + m_token(_token) +{ + // register handler for wsService + m_wsService->registerMsgHandler((uint16_t)ppc::protocol::MessageType::RpcRequest, + boost::bind(&Rpc::onWsRpcRequest, this, boost::placeholders::_1, boost::placeholders::_2)); + // register handler for httpServer + auto httpServer = m_wsService->httpServer(); + if (httpServer) + { + httpServer->setHttpReqHandler([this](bcos::boostssl::http::HttpRequest&& _request, + std::function _responseHandler) { + this->onRPCRequest(std::move(_request), _responseHandler); + }); + } + // register the handler for RUN_TASK_METHOD + m_methodToHandler[RUN_TASK_METHOD] = + boost::bind(&Rpc::runTask, this, boost::placeholders::_1, boost::placeholders::_2); + // register the handler for ASYNC_RUN_TASK_METHOD + m_methodToHandler[ASYNC_RUN_TASK_METHOD] = + boost::bind(&Rpc::asyncRunTask, this, boost::placeholders::_1, boost::placeholders::_2); + // register the handler for GET_TASK_STATUS + m_methodToHandler[GET_TASK_STATUS] = + boost::bind(&Rpc::getTaskStatus, this, boost::placeholders::_1, boost::placeholders::_2); + // register the handler for REGISTER_GATEWAY_URL + m_methodToHandler[REGISTER_GATEWAY_URL] = boost::bind( + &Rpc::registerGatewayUrl, this, boost::placeholders::_1, boost::placeholders::_2); + + // register ecdh bs mode methods + m_methodToHandler[ASYNC_RUN_BS_MODE_TASK] = boost::bind( + &Rpc::asyncRunBsModeTask, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[FETCH_CIPHER] = + boost::bind(&Rpc::fetchCipher, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[SEND_ECDH_CIPHER] = + boost::bind(&Rpc::sendEcdhCipher, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[SEND_PARTNER_CIPHER] = boost::bind( + &Rpc::sendPartnerCipher, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[GET_BS_MODE_TASK_STATUS] = boost::bind( + &Rpc::getBsModeTaskStatus, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[KILL_BS_MODE_TASK] = + boost::bind(&Rpc::killBsModeTask, this, boost::placeholders::_1, boost::placeholders::_2); + m_methodToHandler[UPDATE_BS_MODE_TASK_STATUS] = boost::bind( + &Rpc::updateBsModeTaskStatus, this, boost::placeholders::_1, boost::placeholders::_2); + + RPC_LOG(INFO) << LOG_DESC("init rpc success") << LOG_KV("selfParty", _selfPartyID); +} + +void Rpc::onRPCRequest( + bcos::boostssl::http::HttpRequest&& _request, std::function _responseHandler) +{ + onRPCRequestImpl(_request.body(), _responseHandler); +} + +void Rpc::onRPCRequestImpl( + std::string_view const& _requestBody, std::function _responseHandler) +{ + auto response = std::make_shared(); + try + { + auto startT = bcos::utcSteadyTime(); + JsonRequest request(_requestBody); + if (request.token() != m_token) + { + RPC_LOG(INFO) << LOG_DESC("onRPCRequest: token not match") + << LOG_KV("interface", request.token()) << LOG_KV("config", m_token); + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::NoPermission, "no permission, please check token")); + } + response->setJsonRpc(request.jsonRpc()); + response->setId(request.id()); + auto it = m_methodToHandler.find(request.method()); + if (it == m_methodToHandler.end()) + { + RPC_LOG(DEBUG) << LOG_DESC("onRPCRequest: method not found") + << LOG_KV("method", request.method()); + BOOST_THROW_EXCEPTION(BCOS_ERROR( + (int64_t)RpcError::MethodNotFound, "The method does not exist/is not available.")); + } + auto const& methodHandler = it->second; + methodHandler( + request.params(), [response, _responseHandler, startT, method = request.method()]( + Error::Ptr _error, Json::Value&& _result) { + auto costs = bcos::utcSteadyTime() - startT; + if (costs >= 300) + { + RPC_LOG(INFO) << LOG_DESC("Slow rpc interface found") + << LOG_KV("method", method) << LOG_KV("costs(ms)", costs); + } + + if (_error && _error->errorCode() != 0) + { + response->setError(std::move(*_error)); + response->setResult(std::move(_result)); + } + else + { + response->setResult(std::move(_result)); + } + _responseHandler(response->serialize()); + }); + } + catch (Error const& e) + { + RPC_LOG(WARNING) << LOG_DESC("onRPCRequest error") << LOG_KV("request", _requestBody) + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + response->setError(e); + _responseHandler(response->serialize()); + } + catch (std::exception const& e) + { + RPC_LOG(WARNING) << LOG_DESC("onRPCRequest error") << LOG_KV("request", _requestBody) + << LOG_KV("msg", boost::diagnostic_information(e)); + response->mutableError()->setErrorCode(PPCRetCode::EXCEPTION); + response->mutableError()->setErrorMessage(boost::diagnostic_information(e)); + _responseHandler(response->serialize()); + } +} + +void Rpc::onWsRpcRequest(std::shared_ptr _msg, + std::shared_ptr _session) +{ + auto buffer = _msg->payload(); + auto request = std::string_view((const char*)buffer->data(), buffer->size()); + onRPCRequestImpl(request, [m_buffer = std::move(buffer), _msg, _session](bcos::bytes resp) { + if (_session && _session->isConnected()) + { + auto buffer = std::make_shared(std::move(resp)); + _msg->setPayload(buffer); + _session->asyncSendMessage(_msg); + return; + } + RPC_LOG(WARNING) + << LOG_DESC("onWsRpcRequest: unable to send response for session has been inactive") + << LOG_KV("req", std::string_view((const char*)m_buffer->data(), m_buffer->size())) + << LOG_KV("resp", std::string_view((const char*)resp.data(), resp.size())) + << LOG_KV("seq", _msg->seq()) + << LOG_KV("endpoint", _session ? _session->endPoint() : std::string("")); + }); +} + +void Rpc::runTask(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_rpcStorage) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set")); + } + checkHostResource(); + + auto task = m_taskFactory->createTask(_req); + auto taskHandler = getTaskHandler(task->type(), task->algorithm()); + // not find the handler + if (!taskHandler) + { + std::stringstream oss; + oss << "The specified task algorithm not been implemented or disabled, type: " + << (ppc::protocol::TaskType)(task->type()) << "(" << std::to_string(task->type()) + << ")"; + oss << ", algorithm: " << (ppc::protocol::PSIAlgorithmType)(task->algorithm()) << "(" + << std::to_string(task->algorithm()) << ")"; + auto errorMsg = oss.str(); + RPC_LOG(WARNING) << errorMsg; + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)RpcError::NotImplemented, errorMsg)); + } + RPC_LOG(DEBUG) << LOG_DESC("runTask") << printTaskInfo(task); + auto startT = utcSteadyTime(); + + // insert task + auto error = m_rpcStorage->insertTask(task); + if (error && error->errorCode()) + { + auto result = std::make_shared(task->id()); + result->setStatus(toString(TaskStatus::RUNNING)); + _respFunc(result->error(), result->serializeToJson()); + return; + } + + taskHandler(task, [&, task, _respFunc, startT](ppc::protocol::TaskResult::Ptr&& _result) { + auto cost = bcos::utcSteadyTime() - startT; + _result->setTimeCost(cost); + RPC_LOG(DEBUG) << LOG_DESC("runTask finish") << LOG_KV("timecost", cost) + << printTaskInfo(task); + + // update status + auto error = m_rpcStorage->updateTaskStatus(_result); + if (error && error->errorCode()) + { + auto result = std::make_shared(task->id()); + result->setError(std::move(error)); + result->setStatus(toString(TaskStatus::FAILED)); + _respFunc(result->error(), result->serializeToJson()); + return; + } + + _respFunc(_result->error(), _result->serializeToJson()); + }); +} + +void Rpc::asyncRunTask(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_rpcStorage) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set")); + } + checkHostResource(); + + auto task = m_taskFactory->createTask(_req); + auto taskHandler = getTaskHandler(task->type(), task->algorithm()); + // not find the handler + if (!taskHandler) + { + std::stringstream oss; + oss << "The specified task algorithm not been implemented or disabled, type: " + << (ppc::protocol::TaskType)(task->type()) << "(" << std::to_string(task->type()) + << ")"; + oss << ", algorithm: " << (ppc::protocol::PSIAlgorithmType)(task->algorithm()) << "(" + << std::to_string(task->algorithm()) << ")"; + auto errorMsg = oss.str(); + RPC_LOG(WARNING) << errorMsg; + BOOST_THROW_EXCEPTION(BCOS_ERROR((int64_t)RpcError::NotImplemented, errorMsg)); + } + RPC_LOG(DEBUG) << LOG_DESC("asyncRunTask") << printTaskInfo(task); + + // insert task + auto error = m_rpcStorage->insertTask(task); + if (error && error->errorCode()) + { + auto result = std::make_shared(task->id()); + result->setStatus(toString(TaskStatus::RUNNING)); + _respFunc(result->error(), result->serializeToJson()); + return; + } + + // return response immediately + auto result = std::make_shared(task->id()); + result->setStatus(toString(TaskStatus::RUNNING)); + _respFunc(result->error(), result->serializeToJson()); + + auto startT = utcSteadyTime(); + taskHandler(task, [&, task, startT](ppc::protocol::TaskResult::Ptr&& _result) { + auto cost = utcSteadyTime() - startT; + _result->setTimeCost(cost); + RPC_LOG(DEBUG) << LOG_DESC("runTask finish") << LOG_KV("timecost", cost) + << printTaskInfo(task); + // update status + auto error = m_rpcStorage->updateTaskStatus(_result); + if (error && error->errorCode()) + { + RPC_LOG(WARNING) << LOG_DESC("failed to updateTaskStatus") << printTaskInfo(task); + } + }); +} + +void Rpc::getTaskStatus(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_rpcStorage) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set")); + } + + if (!_req.isMember("taskID")) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the taskID")); + } + auto taskID = _req["taskID"].asString(); + + auto result = m_rpcStorage->getTaskStatus(taskID); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::registerGatewayUrl(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_rpcStorage) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set")); + } + + if (!_req.isMember("id")) + { + BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the agencyID")); + } + auto agencyID = _req["id"].asString(); + + if (!_req.isMember("url")) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << errinfo_comment("Must specify the gateway url")); + } + auto agencyUrl = _req["url"].asString(); + + std::vector endpoints; + boost::split(endpoints, agencyUrl, boost::is_any_of(",")); + for (auto& endpoint : endpoints) + { + if (!checkEndpoint(endpoint)) + { + BOOST_THROW_EXCEPTION( + InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); + } + } + Json::Value response; + auto error = m_rpcStorage->insertGateway(agencyID, agencyUrl); + if (error && error->errorCode()) + { + response["code"] = error->errorCode(); + response["message"] = error->errorMessage(); + _respFunc(error, std::move(response)); + return; + } + + response["code"] = 0; + response["message"] = "success"; + _respFunc(error, std::move(response)); +} + +void Rpc::asyncRunBsModeTask(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + checkHostResource(); + + auto request = std::make_shared(_req, m_prePath); + m_bsEcdhPSI->asyncRunTask(request, [&](psi::BsEcdhResult::Ptr&& _result) { + _respFunc(_result->error(), _result->serializeToJson()); + }); +} + +void Rpc::fetchCipher(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->fetchCipher(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::sendEcdhCipher(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->sendEcdhCipher(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::sendPartnerCipher(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->sendPartnerCipher(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::getBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->getTaskStatus(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::killBsModeTask(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->killTask(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::updateBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc) +{ + if (!m_bsEcdhPSI) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set")); + } + + auto request = std::make_shared(_req); + auto result = m_bsEcdhPSI->updateTaskStatus(request); + _respFunc(result->error(), result->serializeToJson()); +} + +void Rpc::checkHostResource() +{ +#ifdef __linux__ + if (!hasAvailableMem(m_minNeededMemoryGB)) + { + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int64_t)RpcError::LackOfMemory, "Lack of memory, please try again later.")); + } +#endif +} diff --git a/cpp/ppc-rpc/src/Rpc.h b/cpp/ppc-rpc/src/Rpc.h new file mode 100644 index 00000000..a48a9b2b --- /dev/null +++ b/cpp/ppc-rpc/src/Rpc.h @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Rpc.h + * @author: yujiechen + * @date 2022-11-3 + */ +#pragma once +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/rpc/RpcInterface.h" +#include "ppc-framework/rpc/RpcStatusInterface.h" +#include "ppc-protocol/src/JsonTaskImpl.h" +#include +#include +#include +#include +#include +#include + +namespace ppc::rpc +{ +class Rpc : public RpcInterface +{ +public: + using Ptr = std::shared_ptr; + Rpc(std::shared_ptr _wsService, std::string const& _selfPartyID, + std::string const& _token, std::string const& _prePath = "data"); + ~Rpc() override { stop(); } + void start() override + { + if (m_wsService) + { + m_wsService->start(); + } + if (m_rpcStorage) + { + m_rpcStorage->start(); + } + RPC_LOG(INFO) << LOG_DESC("start RPC"); + } + void stop() override + { + if (m_wsService) + { + m_wsService->stop(); + } + if (m_rpcStorage) + { + m_rpcStorage->stop(); + } + RPC_LOG(INFO) << LOG_DESC("stop RPC"); + } + + void registerHandler(std::string const& _methodName, + std::function _handler) override + { + bcos::UpgradableGuard l(x_methodToHandler); + if (m_methodToHandler.count(_methodName)) + { + RPC_LOG(INFO) << LOG_DESC("registerHandler return for method handler already exists") + << LOG_KV("method", _methodName); + return; + } + bcos::UpgradeGuard ul(l); + m_methodToHandler[_methodName] = _handler; + RPC_LOG(INFO) << LOG_DESC("registerHandler success") << LOG_KV("method", _methodName); + } + + void registerTaskHandler( + ppc::protocol::TaskType _type, uint8_t _algorithm, TaskHandler _handler) override + { + bcos::UpgradableGuard l(x_taskHandlers); + if (m_taskHandlers.count((uint8_t)_type) && + m_taskHandlers.at((uint8_t)_type).count(_algorithm)) + { + RPC_LOG(INFO) << LOG_DESC("registerTaskHandler return for handler already exists") + << LOG_KV("type", _type) << LOG_KV("algorithm", _algorithm); + return; + } + m_taskHandlers[(uint8_t)_type][_algorithm] = _handler; + RPC_LOG(INFO) << LOG_DESC("registerTaskHandler success") << LOG_KV("type", _type) + << LOG_KV("algorithm", (int)_algorithm); + } + + void setRpcStorage(RpcStatusInterface::Ptr _storage) { m_rpcStorage = std::move(_storage); } + + void setBsEcdhPSI(psi::BsEcdhPSIInterface::Ptr _psi) { m_bsEcdhPSI = std::move(_psi); } + + void setMinNeededMemory(uint32_t _minNeededMemory) { m_minNeededMemoryGB = _minNeededMemory; } + +protected: + virtual void onRPCRequest(bcos::boostssl::http::HttpRequest&& _request, + std::function _responseHandler); + + virtual void onRPCRequestImpl( + std::string_view const& _requestBody, std::function _responseHandler); + virtual void onWsRpcRequest(std::shared_ptr _msg, + std::shared_ptr _session); + + virtual TaskHandler getTaskHandler(uint8_t _type, uint8_t _algorithm) const + { + bcos::ReadGuard l(x_taskHandlers); + if (!m_taskHandlers.count(_type) || !m_taskHandlers.at(_type).count(_algorithm)) + { + return nullptr; + } + return m_taskHandlers.at(_type).at(_algorithm); + } + + virtual void runTask(Json::Value const& _req, RespFunc _respFunc); + virtual void asyncRunTask(Json::Value const& _req, RespFunc _respFunc); + virtual void getTaskStatus(Json::Value const& _req, RespFunc _respFunc); + virtual void registerGatewayUrl(Json::Value const& _req, RespFunc _respFunc); + + virtual void asyncRunBsModeTask(Json::Value const& _req, RespFunc _respFunc); + virtual void fetchCipher(Json::Value const& _req, RespFunc _respFunc); + virtual void sendEcdhCipher(Json::Value const& _req, RespFunc _respFunc); + virtual void sendPartnerCipher(Json::Value const& _req, RespFunc _respFunc); + virtual void getBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc); + virtual void killBsModeTask(Json::Value const& _req, RespFunc _respFunc); + virtual void updateBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc); + + void checkHostResource(); + +private: + std::string m_prePath; + std::shared_ptr m_wsService; + RpcStatusInterface::Ptr m_rpcStorage; + + // Note: here use jsonTaskFactory to decrease the overhead to convert json::value to string when + // deserialze the task + ppc::protocol::JsonTaskFactory::Ptr m_taskFactory; + std::string m_token; + + // the method_name to function + std::map> m_methodToHandler; + bcos::SharedMutex x_methodToHandler; + + // the taskHandlers + std::map> m_taskHandlers; + mutable bcos::SharedMutex x_taskHandlers; + + psi::BsEcdhPSIInterface::Ptr m_bsEcdhPSI; + + uint32_t m_minNeededMemoryGB; +}; +} // namespace ppc::rpc diff --git a/cpp/ppc-rpc/src/RpcFactory.cpp b/cpp/ppc-rpc/src/RpcFactory.cpp new file mode 100644 index 00000000..a7e880ef --- /dev/null +++ b/cpp/ppc-rpc/src/RpcFactory.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcFactory.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "RpcFactory.h" +#include "ppc-storage/src/mysql/MySQLStorage.h" +#include +#include + +using namespace bcos; +using namespace ppc::rpc; +using namespace ppc::tools; + +Rpc::Ptr RpcFactory::buildRpc(ppc::tools::PPCConfig::ConstPtr _config) +{ + auto wsConfig = initConfig(_config); + // create the wsConfig + auto wsService = std::make_shared(); + wsService->setTimerFactory(std::make_shared()); + auto initializer = std::make_shared(); + + initializer->setConfig(wsConfig); + initializer->initWsService(wsService); + + auto rpc = std::make_shared( + wsService, m_selfPartyID, _config->rpcConfig().token, _config->dataLocation()); + rpc->setMinNeededMemory(_config->rpcConfig().minNeededMemoryGB); + return rpc; +} + +std::shared_ptr RpcFactory::initConfig( + ppc::tools::PPCConfig::ConstPtr _config) +{ + // init the wsConfig + auto wsConfig = std::make_shared(); + wsConfig->setModel(boostssl::ws::WsModel::Server); + + wsConfig->setListenIP(_config->rpcConfig().listenIp); + wsConfig->setListenPort(_config->rpcConfig().listenPort); + wsConfig->setThreadPoolSize(_config->rpcConfig().threadPoolSize); + wsConfig->setDisableSsl(_config->rpcConfig().disableSsl); + if (_config->rpcConfig().disableSsl) + { + return wsConfig; + } + // non-sm-ssl + auto contextConfig = std::make_shared(); + if (!_config->rpcConfig().enableSM) + { + boostssl::context::ContextConfig::CertConfig certConfig; + certConfig.caCert = _config->rpcConfig().caCertPath; + certConfig.nodeCert = _config->rpcConfig().sslCertPath; + certConfig.nodeKey = _config->rpcConfig().sslKeyPath; + contextConfig->setCertConfig(certConfig); + contextConfig->setSslType("ssl"); + RPC_LOG(INFO) << LOG_DESC("initConfig: rpc work in non-sm-ssl model") + << LOG_KV("caCert", certConfig.caCert) + << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("nodeKey", certConfig.nodeKey); + wsConfig->setContextConfig(contextConfig); + return wsConfig; + } + // sm-ssl + boostssl::context::ContextConfig::SMCertConfig certConfig; + certConfig.caCert = _config->rpcConfig().smCaCertPath; + certConfig.nodeCert = _config->rpcConfig().smSslCertPath; + certConfig.nodeKey = _config->rpcConfig().smSslKeyPath; + certConfig.enNodeCert = _config->rpcConfig().smEnSslCertPath; + certConfig.enNodeKey = _config->rpcConfig().smEnSslKeyPath; + contextConfig->setSmCertConfig(certConfig); + contextConfig->setSslType("sm_ssl"); + + BCOS_LOG(INFO) << LOG_DESC("initConfig") << LOG_DESC("rpc work in sm ssl model") + << LOG_KV("caCert", certConfig.caCert) << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("enNodeCert", certConfig.enNodeCert); + + wsConfig->setContextConfig(contextConfig); + return wsConfig; +} \ No newline at end of file diff --git a/cpp/ppc-rpc/src/RpcFactory.h b/cpp/ppc-rpc/src/RpcFactory.h new file mode 100644 index 00000000..bfa04d9d --- /dev/null +++ b/cpp/ppc-rpc/src/RpcFactory.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcFactory.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include "Rpc.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include +namespace bcos::boostssl::ws +{ +class WsConfig; +} +namespace ppc::rpc +{ +class RpcFactory +{ +public: + using Ptr = std::shared_ptr; + RpcFactory(std::string const& _selfPartyID) : m_selfPartyID(_selfPartyID) {} + virtual ~RpcFactory() = default; + + Rpc::Ptr buildRpc(ppc::tools::PPCConfig::ConstPtr _config); + + +private: + std::shared_ptr initConfig( + ppc::tools::PPCConfig::ConstPtr _config); + +private: + std::string m_selfPartyID; +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-rpc/src/RpcMemory.cpp b/cpp/ppc-rpc/src/RpcMemory.cpp new file mode 100644 index 00000000..c44fd9c7 --- /dev/null +++ b/cpp/ppc-rpc/src/RpcMemory.cpp @@ -0,0 +1,137 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcMemory.cpp + * @author: shawnhe + * @date 2023-03-28 + */ + +#include "RpcMemory.h" +#include "bcos-utilities/Common.h" + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::rpc; +using namespace ppc::storage; + +void RpcMemory::start() +{ + if (m_taskCleaner) + { + m_taskCleaner->registerTimeoutHandler(boost::bind(&RpcMemory::cleanTask, this)); + m_taskCleaner->start(); + } +} + + +void RpcMemory::stop() +{ + if (m_taskCleaner) + { + m_taskCleaner->stop(); + } +} + +void RpcMemory::cleanTask() +{ + try + { + WriteGuard l(x_tasks); + for (auto it = m_tasks.begin(); it != m_tasks.end();) + { + if (it->second.first + VALIDITY_TERM <= utcSteadyTime()) + { + it = m_tasks.erase(it); + } + else + { + ++it; + } + } + } + catch (std::exception const& e) + { + RPC_STATUS_LOG(WARNING) << LOG_DESC("cleanTask exception") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + m_taskCleaner->restart(); +} + +bcos::Error::Ptr RpcMemory::insertTask(protocol::Task::Ptr _task) +{ + WriteGuard l(x_tasks); + if (m_tasks.find(_task->id()) != m_tasks.end()) + { + return std::make_shared(PPCRetCode::WRITE_RPC_STATUS_ERROR, "task exists"); + } + auto taskResult = std::make_shared(_task->id()); + taskResult->setStatus(toString(TaskStatus::RUNNING)); + m_tasks[_task->id()] = {utcSteadyTime(), std::move(taskResult)}; + return nullptr; +} + +bcos::Error::Ptr RpcMemory::updateTaskStatus(protocol::TaskResult::Ptr _taskResult) +{ + WriteGuard l(x_tasks); + if (m_tasks.find(_taskResult->taskID()) == m_tasks.end()) + { + return std::make_shared(PPCRetCode::WRITE_RPC_STATUS_ERROR, "task not found"); + } + m_tasks[_taskResult->taskID()].second = std::move(_taskResult); + return nullptr; +} + +TaskResult::Ptr RpcMemory::getTaskStatus(const std::string& _taskID) +{ + ReadGuard l(x_tasks); + if (m_tasks.find(_taskID) == m_tasks.end()) + { + auto error = + std::make_shared(PPCRetCode::READ_RPC_STATUS_ERROR, "task not found"); + auto result = std::make_shared(_taskID); + result->setStatus(toString(TaskStatus::FAILED)); + result->setError(error); + return result; + } + + return m_tasks[_taskID].second; +} + +bcos::Error::Ptr RpcMemory::insertGateway( + const std::string& _agencyID, const std::string& _endpoint) +{ + try + { + std::vector gatewayList; + gatewayList.push_back({_agencyID, _endpoint}); + m_gateway->registerGateway(gatewayList); + return nullptr; + } + catch (std::exception const& e) + { + return std::make_shared( + PPCRetCode::EXCEPTION, "insertGateway error: " + boost::diagnostic_information(e)); + } +} + +bcos::Error::Ptr RpcMemory::deleteGateway(const std::string& _agencyID) +{ + return nullptr; +} + +std::vector RpcMemory::listGateway() +{ + return {}; +} diff --git a/cpp/ppc-rpc/src/RpcMemory.h b/cpp/ppc-rpc/src/RpcMemory.h new file mode 100644 index 00000000..e275b3ea --- /dev/null +++ b/cpp/ppc-rpc/src/RpcMemory.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RpcMemory.h + * @author: shawnhe + * @date 2023-03-27 + * @desc: 如果无法启用mysqlx协议，则数据保存到内存中 + */ + +#pragma once +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-framework/rpc/RpcStatusInterface.h" +#include +#include + +namespace ppc::rpc +{ + +class RpcMemory : public RpcStatusInterface +{ +public: + using Ptr = std::shared_ptr; + + RpcMemory(ppc::gateway::GatewayInterface::Ptr _gateway) + : m_gateway(std::move(_gateway)), + m_taskCleaner(std::make_shared(60 * 60 * 1000, "taskCleaner")) + {} + ~RpcMemory() override = default; + + void start() override; + void stop() override; + + bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) override; + bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) override; + protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) override; + bcos::Error::Ptr insertGateway( + const std::string& _agencyID, const std::string& _endpoint) override; + bcos::Error::Ptr deleteGateway(const std::string& _agencyID) override; + std::vector listGateway() override; + +protected: + void cleanTask(); + +private: + ppc::gateway::GatewayInterface::Ptr m_gateway; + + mutable bcos::SharedMutex x_tasks; + std::unordered_map> m_tasks; + std::shared_ptr m_taskCleaner; + + const uint64_t VALIDITY_TERM = 24 * 60 * 60 * 1000; // ms +}; +} // namespace ppc::rpc \ No newline at end of file diff --git a/cpp/ppc-rpc/tests/CMakeLists.txt b/cpp/ppc-rpc/tests/CMakeLists.txt new file mode 100644 index 00000000..9654e227 --- /dev/null +++ b/cpp/ppc-rpc/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-rpc) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${RPC_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-rpc WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp b/cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp new file mode 100644 index 00000000..134f9192 --- /dev/null +++ b/cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestRpcRequestResponse.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "ppc-rpc/src/JsonRequest.h" +#include "ppc-rpc/src/JsonResponse.h" +#include +#include +#include + +using namespace ppc::rpc; +using namespace bcos; +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(rpcRequestResponseTest, TestPromptFixture) +BOOST_AUTO_TEST_CASE(testJsonRequest) +{ + // case1: valid request + std::string requestData = + "{\"jsonrpc\":\"2.0\",\"token\":\"test\",\"method\":\"getBlockNumber\",\"params\":[1]," + "\"id\":1}"; + auto jsonRequest = std::make_shared(requestData); + BOOST_CHECK(jsonRequest->jsonRpc() == "2.0"); + BOOST_CHECK(jsonRequest->method() == "getBlockNumber"); + BOOST_CHECK(jsonRequest->id() == 1); + Json::Value params = jsonRequest->params(); + BOOST_CHECK(params.isArray() == true); + for (auto const& param : params) + { + BOOST_CHECK(param.asInt64() == 1); + } + // case2: invalid request without method + requestData = "{\"jsonrpc\":\"2.0\",\"token\":\"test\",\"params\":[1],\"id\":1}"; + BOOST_CHECK_THROW(std::make_shared(requestData), bcos::Error); + // case3: invalid json request + requestData = "abc"; + BOOST_CHECK_THROW(std::make_shared(requestData), bcos::Error); + + // case4: valid request with taskInfo + requestData = + "{\"jsonrpc\":\"2.0\",\"token\":\"test\",\"method\":\"runTask\",\"params\":{\"algorithm\":" + "45,\"params\":" + "\"taskParam\", " + "\"parties\":[{\"data\":{\"desc\":{\"path\":\"testPath\"},\"id\":\"dataSelf\",\"type\":0}," + "\"desc\":\"selfParty\",\"id\":\"selfParty\",\"partyIndex\":0,\"type\":0}]},\"id\":1}"; + std::make_shared(requestData); + BOOST_CHECK_NO_THROW(std::make_shared(requestData)); +} + +BOOST_AUTO_TEST_CASE(testJsonResponse) +{ + auto jsonResponse = std::make_shared(); + jsonResponse->setJsonRpc("3.0"); + jsonResponse->setId(10); + BOOST_CHECK_NO_THROW(jsonResponse->serialize()); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-rpc/tests/main.cpp b/cpp/ppc-rpc/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-rpc/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-storage/CMakeLists.txt b/cpp/ppc-storage/CMakeLists.txt new file mode 100644 index 00000000..36b28b6c --- /dev/null +++ b/cpp/ppc-storage/CMakeLists.txt @@ -0,0 +1,6 @@ +project(ppc-storage VERSION ${VERSION}) +add_subdirectory(src) + +if (DEMO) + add_subdirectory(demo) +endif() \ No newline at end of file diff --git a/cpp/ppc-storage/demo/CMakeLists.txt b/cpp/ppc-storage/demo/CMakeLists.txt new file mode 100644 index 00000000..b3d19832 --- /dev/null +++ b/cpp/ppc-storage/demo/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(STORAGE_DEMO_BINARY_NAME storage_demo) + +add_executable(${STORAGE_DEMO_BINARY_NAME} storage_demo.cpp) + +target_link_libraries(${STORAGE_DEMO_BINARY_NAME} ${STORAGE_TARGET} ${CPU_FEATURES_LIB} ${BOOST_UNIT_TEST}) +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-storage/demo/storage_demo.cpp b/cpp/ppc-storage/demo/storage_demo.cpp new file mode 100644 index 00000000..3d8f92c8 --- /dev/null +++ b/cpp/ppc-storage/demo/storage_demo.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file storage_demo.cpp + * @desc: demo for storage + * @author: yujiechen + * @date 2022-10-24 + */ +#include "ppc-storage/src/CacheStorageFactoryImpl.h" +#include +#include +#include +#include +#include + +using namespace ppc::storage; +using namespace ppc::protocol; +using namespace bcos; +int main(int argc, char* argv[]) +{ + if (argc < 4) + { + std::cout << "Usage: " << argv[0] + << "\nredis: 0 ip port 123456\n proxy obServer cluster user password"; + return -1; + } + try + { + CacheStorageConfig cacheStorageConfig; + + if (atoi(argv[1]) == 0) + { + // redis + cacheStorageConfig.type = CacheType::Redis; + cacheStorageConfig.host = argv[2]; + cacheStorageConfig.port = atoi(argv[3]); + cacheStorageConfig.password = argv[4]; + cacheStorageConfig.database = 1; + } + else + { + std::cout << "Usage: " << argv[0] + << "\nredis: 0 ip port 123456\n proxy obServer cluster user password"; + return -1; + } + + CacheStorageFactoryImpl::Ptr factory = std::make_shared(); + auto cacheClient = factory->createCacheStorage(cacheStorageConfig); + + // set and get data + std::cout << "#### start cache test ####" << std::endl; + auto startT = utcSteadyTime(); + std::cout << "check setValue/getValue/exists..." << std::endl; + for (int i = 0; i < 1000; i++) + { + auto key = std::to_string(i); + auto value = "value" + key; + cacheClient->setValue(key, value); + auto result = cacheClient->getValue(key); + BOOST_CHECK(result.has_value() && result == value); + // check exists + BOOST_CHECK(cacheClient->exists(key)); + } + std::cout << "setValue/getValue/exists success, time: " << (utcSteadyTime() - startT) + << "ms" << std::endl; + std::cout << "check non-exists key..." << std::endl; + // check non-exists data + startT = utcSteadyTime(); + for (int i = 10000; i < 500; i++) + { + auto key = std::to_string(i); + auto result = cacheClient->getValue(key); + BOOST_CHECK(!result.has_value()); + BOOST_CHECK(!cacheClient->exists(key)); + } + std::cout << "check non-exists key success, time: " << (utcSteadyTime() - startT) << "ms" + << std::endl; + // expire all the keys with 1s expiration-time + startT = utcSteadyTime(); + std::cout << "check expireKey key..." << std::endl; + for (int i = 0; i < 1000; i++) + { + auto key = std::to_string(i); + cacheClient->expireKey(key, 1); + } + std::cout << "check expireKey key success, time: " << (utcSteadyTime() - startT) << "ms" + << std::endl; + // sleep 2s + std::cout << "sleep 2 seconds to wait for key-expiration" << std::endl; + boost::this_thread::sleep_for(boost::chrono::milliseconds(2000)); + std::cout << "wait-finished" << std::endl; + // check the key + startT = utcSteadyTime(); + std::cout << "check the key-expiration..." << std::endl; + for (int i = 0; i < 1000; i++) + { + auto key = std::to_string(i); + auto result = cacheClient->getValue(key); + BOOST_CHECK(!result.has_value()); + BOOST_CHECK(!cacheClient->exists(key)); + } + std::cout << "check the key-expiration success, time:" << (utcSteadyTime() - startT) + << std::endl; + std::cout << "#### end cache test ####" << std::endl; + } + catch (std::exception const& e) + { + std::cout << "access cache error: \n" << boost::diagnostic_information(e) << std::endl; + } +} \ No newline at end of file diff --git a/cpp/ppc-storage/src/CMakeLists.txt b/cpp/ppc-storage/src/CMakeLists.txt new file mode 100644 index 00000000..e51b2278 --- /dev/null +++ b/cpp/ppc-storage/src/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${STORAGE_TARGET} ${SRCS}) + +find_package(redis++ REQUIRED) +find_package(unofficial-mysql-connector-cpp REQUIRED) + +target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB}) diff --git a/cpp/ppc-storage/src/CacheStorageFactoryImpl.h b/cpp/ppc-storage/src/CacheStorageFactoryImpl.h new file mode 100644 index 00000000..c7fe6b22 --- /dev/null +++ b/cpp/ppc-storage/src/CacheStorageFactoryImpl.h @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CacheStorageFactoryImpl.h + * @author: shawnhe + * @date 2023-03-13 + */ +#pragma once + +#include "Common.h" +#include "ppc-framework/storage/CacheStorage.h" +#include "redis/RedisStorage.h" + +namespace ppc::storage +{ +class CacheStorageFactoryImpl : public ppc::storage::CacheStorageFactory +{ +public: + using Ptr = std::shared_ptr; + +public: + CacheStorageFactoryImpl() = default; + ~CacheStorageFactoryImpl() = default; + +public: + CacheStorage::Ptr createCacheStorage(CacheStorageConfig const& _config) + { + switch (_config.type) + { + case ppc::protocol::CacheType::Redis: + { + STORAGE_LOG(INFO) << LOG_DESC("buildRedisCache"); + sw::redis::ConnectionOptions connection_options; + connection_options.host = _config.host; + connection_options.port = _config.port; + connection_options.password = _config.password; + connection_options.db = _config.database; + + // *0ms* by default. + connection_options.connect_timeout = + std::chrono::milliseconds(_config.connectionTimeout); + // Optional. Timeout before we successfully send request to or receive response from + // redis. By default, the timeout is 0ms, i.e. never timeout and block until we send or + // receive successfuly. NOTE: if any command is timed out, we throw a TimeoutError + // exception. 0ms by default + connection_options.socket_timeout = std::chrono::milliseconds(_config.socketTimeout); + connection_options.keep_alive = true; + + sw::redis::ConnectionPoolOptions pool_options; + pool_options.size = _config.pool; + + return std::make_shared(connection_options, pool_options); + } + default: + { + BOOST_THROW_EXCEPTION( + UnsupportedCacheStorage() << bcos::errinfo_comment( + "unsupported cache type: " + std::to_string((int)_config.type))); + } + } + } +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/Common.h b/cpp/ppc-storage/src/Common.h new file mode 100644 index 00000000..9041397b --- /dev/null +++ b/cpp/ppc-storage/src/Common.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +#define STORAGE_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("STORAGE") + +namespace ppc::storage +{ +DERIVE_PPC_EXCEPTION(UnsupportedCacheStorage); +DERIVE_PPC_EXCEPTION(UnsupportedSQLStorage); +DERIVE_PPC_EXCEPTION(UnsupportedRemoteStorage); +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/FileStorageFactoryImpl.h b/cpp/ppc-storage/src/FileStorageFactoryImpl.h new file mode 100644 index 00000000..5fcd1074 --- /dev/null +++ b/cpp/ppc-storage/src/FileStorageFactoryImpl.h @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileStorageFactoryImpl.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "hdfs/HDFSStorage.h" + +namespace ppc::storage +{ +class FileStorageFactoryImpl : public FileStorageFactory +{ +public: + using Ptr = std::shared_ptr; + FileStorageFactoryImpl() = default; + ~FileStorageFactoryImpl() override = default; + + FileStorage::Ptr createFileStorage(ppc::protocol::DataResourceType _type, + ppc::protocol::FileStorageConnectionOption::Ptr const& _option) override + { + switch (_type) + { + case ppc::protocol::DataResourceType::HDFS: + { + return std::make_shared(_option); + } + default: + { + BOOST_THROW_EXCEPTION( + UnsupportedFileStorage() << bcos::errinfo_comment("Only support HDFS now")); + } + } + } +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/SQLStorageFactoryImpl.h b/cpp/ppc-storage/src/SQLStorageFactoryImpl.h new file mode 100644 index 00000000..40d26cb5 --- /dev/null +++ b/cpp/ppc-storage/src/SQLStorageFactoryImpl.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SQLStorageFactoryImpl.h + * @author: yujiechen + * @date 2022-11-7 + */ +#pragma once +#include "mysql/MySQLStorage.h" +#include "ppc-framework/storage/FileStorage.h" +#include "ppc-framework/storage/SQLStorage.h" + +namespace ppc::storage +{ +class SQLStorageFactoryImpl : public SQLStorageFactory +{ +public: + using Ptr = std::shared_ptr; + + SQLStorageFactoryImpl() = default; + ~SQLStorageFactoryImpl() override = default; + + SQLStorage::Ptr createSQLStorage(ppc::protocol::DataResourceType _type, + ppc::protocol::SQLConnectionOption::Ptr const& _option) override + { + switch (_type) + { + case ppc::protocol::DataResourceType::MySQL: + { + return std::make_shared(_option); + } + default: + { + BOOST_THROW_EXCEPTION( + UnsupportedSQLStorage() << bcos::errinfo_comment("Only support MySQL now")); + } + } + } +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/hdfs/Common.h b/cpp/ppc-storage/src/hdfs/Common.h new file mode 100644 index 00000000..ba02d6e1 --- /dev/null +++ b/cpp/ppc-storage/src/hdfs/Common.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "../Common.h" +#define HDFS_STORAGE_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("HDFS_STORAGE") + +namespace ppc::storage +{ +DERIVE_PPC_EXCEPTION(ConnectToHDFSFailed); +DERIVE_PPC_EXCEPTION(CreateDirectoryFailed); +DERIVE_PPC_EXCEPTION(UnsupportedFileStorage); +DERIVE_PPC_EXCEPTION(DeleteHDFSFileFailed); +DERIVE_PPC_EXCEPTION(HDFSConnectionOptionNotSet); +DERIVE_PPC_EXCEPTION(RenameHDFSFileFailed); +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/hdfs/HDFSStorage.cpp b/cpp/ppc-storage/src/hdfs/HDFSStorage.cpp new file mode 100644 index 00000000..8eaa5455 --- /dev/null +++ b/cpp/ppc-storage/src/hdfs/HDFSStorage.cpp @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSStorage.cpp + * @author: yujiechen + * @date 2022-11-30 + */ +#include "HDFSStorage.h" +#include +#include +#include +#include + +using namespace ppc::storage; +using namespace ppc::protocol; +using namespace bcos; + +HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option) + : m_option(_option), m_builder(hdfsNewBuilder()) +{ + if (!m_option) + { + BOOST_THROW_EXCEPTION(HDFSConnectionOptionNotSet() + << errinfo_comment("Must set the hdfs-connection-option!")); + } + // Note: the libhdfs3 will try to connect to the hdfs with connect-timeout and retry + // 'rpc.client.connect.retry' times + // the default connect timeout is 600s + // the default 'rpc.client.connect.retry' is 10 + auto connectTimeout = + std::to_string(_option->connectionTimeout); // set 1s as the connectTimeout + hdfsBuilderConfSetStr(m_builder.get(), "rpc.client.connect.timeout", connectTimeout.c_str()); + + // default disable output.replace-datanode-on-failure, set to false to resolve append data error + if (_option->replaceDataNodeOnFailure) + { + hdfsBuilderConfSetStr(m_builder.get(), "output.replace-datanode-on-failure", "true"); + } + else + { + hdfsBuilderConfSetStr(m_builder.get(), "output.replace-datanode-on-failure", "false"); + } + + HDFS_STORAGE_LOG(INFO) << LOG_DESC("create HDFSStorage") << _option->desc() + << LOG_KV("connectTimeout", connectTimeout); + // set name node + hdfsBuilderSetNameNode(m_builder.get(), _option->nameNode.c_str()); + hdfsBuilderSetNameNodePort(m_builder.get(), _option->nameNodePort); + if (!_option->userName.empty()) + { + hdfsBuilderSetUserName(m_builder.get(), _option->userName.c_str()); + } + if (!_option->token.empty()) + { + hdfsBuilderSetToken(m_builder.get(), _option->token.c_str()); + } + // connect to the hdfs, Note: the m_fs is a pointer + m_fs = std::shared_ptr( + hdfsBuilderConnect(m_builder.get()), HDFSFSDeleter()); + if (!m_fs) + { + BOOST_THROW_EXCEPTION( + ConnectToHDFSFailed() << errinfo_comment( + "Connect to hdfs failed! error: " + std::string(hdfsGetLastError()))); + } +} + + +void HDFSStorage::tryToCreateHomeDirectory() const +{ + auto homeDir = getHomeDirectory(); + if (hdfsExists(m_fs.get(), homeDir.c_str()) != 0) + { + createDirectoryImpl(homeDir); + } +} + +void HDFSStorage::createDirectory(std::string const& _dirPath) const +{ + tryToCreateHomeDirectory(); + createDirectoryImpl(_dirPath); +} + +void HDFSStorage::createDirectoryImpl(std::string const& _dirPath) const +{ + HDFS_STORAGE_LOG(INFO) << LOG_DESC("createDirectory: ") << _dirPath; + if (hdfsCreateDirectory(m_fs.get(), _dirPath.c_str()) != 0) + { + BOOST_THROW_EXCEPTION(CreateDirectoryFailed() << bcos::errinfo_comment( + "createDirectory " + _dirPath + + " failed, error: " + std::string(hdfsGetLastError()))); + } + HDFS_STORAGE_LOG(INFO) << LOG_DESC("createDirectory success") << LOG_KV("path", _dirPath); +} + +FileHandler::Ptr HDFSStorage::openFile(std::string const& _path, bool _createIfNotExists) const +{ + // check the existence of the file + if (hdfsExists(m_fs.get(), _path.c_str()) != 0) + { + if (!_createIfNotExists) + { + BOOST_THROW_EXCEPTION(OpenFileFailed() << bcos::errinfo_comment( + "OpenFileFailed: The file " + _path + " not found!")); + } + else + { + HDFS_STORAGE_LOG(DEBUG) + << LOG_DESC("openFile: create the non-exists file") << LOG_KV("path", _path); + boost::filesystem::path filePath(_path); + auto parentPath = filePath.parent_path().string(); + if (parentPath.empty()) + { + parentPath = getHomeDirectory(); + } + // create the parent path + if (hdfsExists(m_fs.get(), parentPath.c_str()) != 0) + { + HDFS_STORAGE_LOG(DEBUG) << LOG_DESC("create parent path: ") << parentPath; + createDirectoryImpl(parentPath); + } + auto fd = hdfsOpenFile(m_fs.get(), _path.c_str(), O_CREAT, 0, 0, 0); + if (!fd) + { + HDFS_STORAGE_LOG(WARNING) + << LOG_DESC("openFileFailed: create the non-exists file failed") + << LOG_KV("path", _path) + << LOG_KV("exception", std::string(hdfsGetLastError())); + BOOST_THROW_EXCEPTION( + OpenFileFailed() << bcos::errinfo_comment( + "OpenFileFailed: create the non-exists file failed, path: " + _path + + ", error: " + std::string(hdfsGetLastError()))); + } + // release the fd + hdfsCloseFile(m_fs.get(), fd); + } + } + auto fileInfo = hdfsGetPathInfo(m_fs.get(), _path.c_str()); + HDFS_STORAGE_LOG(DEBUG) << LOG_DESC("HDFSStorage: openFile success") << LOG_KV("path", _path) + << LOG_KV("blockSize", fileInfo->mBlockSize) + << LOG_KV("blockNum", fileInfo->mSize); + + return std::make_shared(_path, m_fs, fileInfo); +} + +// delete the file +void HDFSStorage::deleteFile(std::string const& _path) const +{ + HDFS_STORAGE_LOG(INFO) << LOG_DESC("deleteFile") << LOG_KV("path", _path); + auto ret = hdfsDelete(m_fs.get(), _path.c_str(), 1); + // delete success + if (!ret) + { + return; + } + // delete failed + BOOST_THROW_EXCEPTION(DeleteHDFSFileFailed() + << bcos::errinfo_comment("DeleteHDFSFileFailed, path: " + _path + + ", error: " + std::string(hdfsGetLastError()))); +} + +void HDFSStorage::renameFile(std::string const& _old_path, std::string const& _new_path) const +{ + HDFS_STORAGE_LOG(INFO) << LOG_DESC("renameFile") << LOG_KV("old_path", _old_path) + << LOG_KV("new_path", _new_path); + auto ret = hdfsRename(m_fs.get(), _old_path.c_str(), _new_path.c_str()); + // delete success + if (!ret) + { + return; + } + // delete failed + BOOST_THROW_EXCEPTION(RenameHDFSFileFailed() + << bcos::errinfo_comment("RenameHDFSFileFailed, path: " + _old_path + + ", error: " + std::string(hdfsGetLastError()))); +} + +bool HDFSStorage::fileExists(std::string const& _path) const +{ + if (hdfsExists(m_fs.get(), _path.c_str()) != 0) + { + return false; + } + return true; +} \ No newline at end of file diff --git a/cpp/ppc-storage/src/hdfs/HDFSStorage.h b/cpp/ppc-storage/src/hdfs/HDFSStorage.h new file mode 100644 index 00000000..1bd62fa0 --- /dev/null +++ b/cpp/ppc-storage/src/hdfs/HDFSStorage.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HDFSStorage.h + * @author: yujiechen + * @date 2022-11-30 + */ +#pragma once +#include "Common.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/storage/FileStorage.h" +#include +#include +#include +#include + +namespace ppc::storage +{ +//// the hdfs handler +class HDFSHandler : public FileHandler +{ +public: + using Ptr = std::shared_ptr; + HDFSHandler(std::string const& _path, + std::shared_ptr _storageHandler, + hdfsFileInfo* _fileInfoHandler) + : m_path(_path), m_storageHandler(_storageHandler), m_fileInfoHandler(_fileInfoHandler) + {} + ~HDFSHandler() override { closeFile(); } + + void* fileInfoHandler() const override { return (void*)m_fileInfoHandler.get(); } + void* storageHandler() const override { return (void*)m_storageHandler.get(); } + std::string const& path() const override { return m_path; } + + void closeFile() { m_fileInfoHandler.reset(); } + +private: + std::string m_path; + std::shared_ptr m_storageHandler; + + struct HDFSFileInfoDeleter + { + void operator()(hdfsFileInfo* _fileInfo) const + { + if (!_fileInfo) + { + return; + } + hdfsFreeFileInfo(_fileInfo, 1); + } + }; + std::unique_ptr m_fileInfoHandler; +}; + +class HDFSStorage : public FileStorage +{ +public: + using Ptr = std::shared_ptr; + HDFSStorage(ppc::protocol::FileStorageConnectionOption::Ptr const& _connectOption); + ~HDFSStorage() {} + + void createDirectory(std::string const& _dirPath) const override; + FileHandler::Ptr openFile( + std::string const& _path, bool _createIfNotExists = false) const override; + // close the file + void closeFile(FileHandler::Ptr _handler) const override + { + auto hdfsHandler = std::dynamic_pointer_cast(_handler); + hdfsHandler->closeFile(); + } + + void renameFile(std::string const& _old_path, std::string const& _new_path) const override; + + void deleteFile(std::string const& _file) const override; + + bool fileExists(std::string const& _path) const override; + +private: + inline std::string getHomeDirectory() const + { + return std::string("/user/") + m_option->userName; + } + + void createDirectoryImpl(std::string const& _dirPath) const; + + void tryToCreateHomeDirectory() const; + +private: + ppc::protocol::FileStorageConnectionOption::Ptr m_option; + struct HDFSBuilderDeleter + { + void operator()(hdfsBuilder* _builder) const { hdfsFreeBuilder(_builder); } + }; + std::unique_ptr m_builder; + + struct HDFSFSDeleter + { + void operator()(HdfsFileSystemInternalWrapper* _fs) const { hdfsDisconnect(_fs); } + }; + std::shared_ptr m_fs; +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/mysql/Common.h b/cpp/ppc-storage/src/mysql/Common.h new file mode 100644 index 00000000..1d18ee71 --- /dev/null +++ b/cpp/ppc-storage/src/mysql/Common.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "../Common.h" +#define MYSQL_STORAGE_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("MYSQL_STORAGE") + +namespace ppc::storage +{ +DERIVE_PPC_EXCEPTION(ConnectMySQLError); +DERIVE_PPC_EXCEPTION(ExecMySQLError); +DERIVE_PPC_EXCEPTION(ParseMySQLResultError); +DERIVE_PPC_EXCEPTION(UnSupportedDataSchema); +DERIVE_PPC_EXCEPTION(NotSetMySQLConnectionOption); +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/mysql/MySQLStorage.cpp b/cpp/ppc-storage/src/mysql/MySQLStorage.cpp new file mode 100644 index 00000000..c8899196 --- /dev/null +++ b/cpp/ppc-storage/src/mysql/MySQLStorage.cpp @@ -0,0 +1,403 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MySQLStorage.cpp + * @author: yujiechen + * @date 2022-10-25 + */ +#include "MySQLStorage.h" +#include +#include + +using namespace ppc::storage; +using namespace ppc::io; +using namespace ppc::protocol; +using namespace bcos; + +MySQLStorage::MySQLStorage( + ppc::protocol::SQLConnectionOption::Ptr const& _opt, uint16_t _sessionPoolSize) + : m_opt(_opt), + m_sessionPoolSize(_sessionPoolSize), + m_timer(std::make_shared(1000 * 60 * 1, "mysqlTimer")) +{ + if (!m_opt) + { + BOOST_THROW_EXCEPTION(NotSetMySQLConnectionOption() + << errinfo_comment("Must set the mysql-connection-option!")); + } + if (m_opt->database.empty()) + { + useDataBase(nullptr); + } + else + { + useDataBase(m_opt->database.c_str()); + } + + if (m_timer) + { + m_timer->registerTimeoutHandler([this] { refreshConnection(); }); + m_timer->start(); + } +} + +void MySQLStorage::refreshConnection() +{ + MYSQL_STORAGE_LOG(INFO) << LOG_DESC("refreshConnection"); + + if (m_opt->database.empty()) + { + useDataBase(nullptr); + } + else + { + useDataBase(m_opt->database.c_str()); + } + + if (m_timer) + { + m_timer->restart(); + } +} + +// use the database +void MySQLStorage::useDataBase(const char* _database) +{ + try + { + m_opt->database = _database ? _database : ""; + // destory the current session + destroyAllSession(); + insertSession(createSession()); + MYSQL_STORAGE_LOG(INFO) << LOG_DESC("use mysql success") + << LOG_KV("connectOpt", m_opt->desc()); + } + catch (std::exception const& e) + { + BOOST_THROW_EXCEPTION(ConnectMySQLError() << errinfo_comment( + "connect to mysql error: " + boost::diagnostic_information(e))); + } +} + +mysqlx_session_t* MySQLStorage::createSession() const +{ + const char* dataBase = m_opt->database.empty() ? nullptr : m_opt->database.c_str(); + try + { + return connectDataBase(dataBase, false); + } + catch (std::exception const& e) + { + return connectDataBase(dataBase, true); + } +} + +mysqlx_session_t* MySQLStorage::connectDataBase(const char* _database, bool _disableSSl) const +{ + // allocate new session according to _database + mysqlx_error_t* error = NULL; + std::stringstream conn; + conn << m_opt->user << ":" << m_opt->password << "@" << m_opt->host; + if (_database) + { + conn << "/" << _database; + } + if (_disableSSl) + { + conn << "?ssl-mode=disabled&mysql41"; + } + MYSQL_STORAGE_LOG(INFO) << LOG_DESC("MySQLStorage connect") << LOG_KV("conn", conn.str()); + auto session = mysqlx_get_session_from_url(conn.str().c_str(), &error); + if (!session) + { + auto errorMsg = std::string(mysqlx_error_message(error)); + MYSQL_STORAGE_LOG(ERROR) << LOG_DESC("connect to mysql error") << m_opt->desc() + << LOG_KV("code", mysqlx_error_num(error)) + << LOG_KV("msg", errorMsg); + // free the error + mysqlx_free(error); + BOOST_THROW_EXCEPTION( + ConnectMySQLError() << errinfo_comment("connect to mysql error: " + errorMsg)); + } + return session; +} + +// exec the sql-query-command +QueryResult::Ptr MySQLStorage::execQuery(bool _parseByColumn, const char* _command, ...) const +{ + // parse the args + va_list args; + va_start(args, _command); + auto result = execSQL(_command, args); + va_end(args); + // query no data + uint32_t columnSize = mysqlx_column_get_count((mysqlx_result_t*)result->result()); + if (!columnSize) + { + return nullptr; + } + MYSQL_STORAGE_LOG(TRACE) << LOG_DESC("execQuery success: ") << LOG_KV("command", _command) + << LOG_KV("columnSize", columnSize); + auto queryResult = std::make_shared(); + parseMetaData(queryResult, result, columnSize); + if (_parseByColumn) + { + parseResultByColumn(queryResult, result, columnSize); + } + parseResultByRow(queryResult, result, columnSize); + return queryResult; +} + +void MySQLStorage::parseMetaData( + QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, uint32_t _columnSize) const +{ + for (uint32_t col = 0; col < _columnSize; col++) + { + _queryResult->mutableMetaData().emplace_back( + std::string(mysqlx_column_get_name((mysqlx_result_t*)_result->result(), col))); + } +} + +// exec the sql-write-command: insert/update/delete +void MySQLStorage::execCommit(const char* _command, ...) const +{ + // parse the args + va_list args; + va_start(args, _command); + auto result = execSQL(_command, args); + va_end(args); + MYSQL_STORAGE_LOG(TRACE) << LOG_DESC("execCommit success: ") << _command; +} + +Statement::Ptr MySQLStorage::generateStatement(const char* _command, ...) const +{ + // parse the args + va_list args; + va_start(args, _command); + auto result = generateSQLStatement(_command, args); + va_end(args); + return result; +} + +SQLResultContext::Ptr MySQLStorage::execStatement(Statement::Ptr _statement) const +{ + auto stmt = (mysqlx_stmt_t*)(_statement->statementContext()); + // execute the sql command + auto res = mysqlx_execute(stmt); + if (!res) + { + auto errorMsg = mysqlx_error_message((void*)stmt); + MYSQL_STORAGE_LOG(ERROR) << LOG_DESC("execStatement error") + << LOG_KV("code", mysqlx_error((void*)stmt)) + << LOG_KV("msg", errorMsg ? errorMsg : "None"); + BOOST_THROW_EXCEPTION(ExecMySQLError() << bcos::errinfo_comment( + "exec sql error: " + std::string(errorMsg ? errorMsg : "None"))); + } + MYSQL_STORAGE_LOG(TRACE) << LOG_DESC("execStatement success"); + + auto result = std::make_shared((mysqlx_result_t*)res); + // takeSession from the statement + result->setSession(_statement->takeSession()); + result->setReturnSessionCallback( + boost::bind(&MySQLStorage::returnSession, this, boost::placeholders::_1)); + return result; +} + +void MySQLStorage::appendStatement(Statement::Ptr _statement, ...) const +{ + va_list args; + va_start(args, _statement); + appendStatement(_statement, args); + va_end(args); +} + +void MySQLStorage::appendStatement(Statement::Ptr _stmt, va_list& args) const +{ + auto stmt = (mysqlx_stmt_t*)(_stmt->statementContext()); + // get the data type + int type = va_arg(args, int); + while (type != (int)FieldDataType::TERMINATE) + { + // get the value + switch (type) + { + case FieldDataType::SINT: + { + bindStmt(stmt, PARAM_SINT(va_arg(args, int64_t)), PARAM_END); + break; + } + case FieldDataType::UINT: + { + bindStmt(stmt, PARAM_UINT(va_arg(args, uint64_t)), PARAM_END); + break; + } + case FieldDataType::DOUBLE: + case FieldDataType::FLOAT: + { + bindStmt(stmt, PARAM_FLOAT(va_arg(args, double)), PARAM_END); + break; + } + case FieldDataType::STRING: + { + bindStmt(stmt, PARAM_STRING(va_arg(args, char*)), PARAM_END); + break; + } + case FieldDataType::BYTES: + { + bcos::byte* data = va_arg(args, bcos::byte*); + size_t len = va_arg(args, size_t); + bindStmt(stmt, PARAM_BYTES(data, len), PARAM_END); + break; + } + default: + { + BOOST_THROW_EXCEPTION( + ExecMySQLError() << bcos::errinfo_comment( + "unsupported data type for statement: " + std::to_string(type))); + } + } + type = va_arg(args, int); + } +} + +Statement::Ptr MySQLStorage::generateSQLStatement(const char* _command, va_list& args) const +{ + auto session = allocateSession(); + // Note: the session will be return back when statement destroied + auto statement = std::make_shared( + mysqlx_sql_new(session, _command, MYSQLX_NULL_TERMINATED), session); + // for return session + statement->setReturnSessionCallback( + boost::bind(&MySQLStorage::returnSession, this, boost::placeholders::_1)); + MYSQL_STORAGE_LOG(TRACE) << LOG_DESC("generateSQLStatement") << LOG_KV("command", _command); + if (!statement->statementContext()) + { + auto errorMsgPtr = mysqlx_error_message(mysqlx_error((void*)session)); + std::string errorMsg = std::string(errorMsgPtr ? errorMsgPtr : "None"); + MYSQL_STORAGE_LOG(ERROR) << LOG_DESC("generateSQLStatement error") + << LOG_KV("command", _command) + << LOG_KV("code", mysqlx_error((void*)session)) + << LOG_KV("msg", errorMsg); + BOOST_THROW_EXCEPTION( + ExecMySQLError() << bcos::errinfo_comment("generate sql statement error: " + errorMsg)); + } + appendStatement(statement, args); + return statement; +} + +void MySQLStorage::parseResult(ppc::io::DataBatch::Ptr _result, + SQLResultContext::Ptr const& _mysqlResult, mysqlx_row_t* _rowData, uint32_t _col) const +{ + auto colType = mysqlx_column_get_type((mysqlx_result_t*)_mysqlResult->result(), _col); + switch (colType) + { + case mysqlx_data_type_enum::MYSQLX_TYPE_BOOL: + case mysqlx_data_type_enum::MYSQLX_TYPE_SINT: + { + int64_t element; + auto ret = mysqlx_get_sint(_rowData, _col, &element); + appendResult(_rowData, _result, ret, std::move(element)); + _result->setDataSchema(DataSchema::Sint); + break; + } + case mysqlx_data_type_enum::MYSQLX_TYPE_UINT: + { + uint64_t element; + auto ret = mysqlx_get_uint(_rowData, _col, &element); + appendResult(_rowData, _result, ret, std::move(element)); + _result->setDataSchema(DataSchema::Uint); + break; + } + case mysqlx_data_type_enum::MYSQLX_TYPE_DOUBLE: + { + double element; + auto ret = mysqlx_get_double(_rowData, _col, &element); + appendResult(_rowData, _result, ret, std::move(element)); + _result->setDataSchema(DataSchema::Double); + break; + } + case mysqlx_data_type_enum::MYSQLX_TYPE_FLOAT: + { + float element; + auto ret = mysqlx_get_float(_rowData, _col, &element); + appendResult(_rowData, _result, ret, std::move(element)); + _result->setDataSchema(DataSchema::Float); + break; + } + case mysqlx_data_type_enum::MYSQLX_TYPE_BYTES: + case mysqlx_data_type_enum::MYSQLX_TYPE_STRING: + { + bool readFinished = false; + uint64_t offset = 0; + _result->append(bcos::bytes()); + // Note: the huge data larger then c_maxElementLen will not been readed-finished at-one-time + while (!readFinished) + { + bcos::bytes element; + element.resize(c_maxElementLen); + // Note: the elementLen should not be zero, otherwise, will trigger "The output buffer + // cannot have zero length" error + size_t elementLen = element.size(); + auto ret = mysqlx_get_bytes(_rowData, _col, offset, (void*)element.data(), &elementLen); + offset += elementLen; + readFinished = appendResult(_rowData, _result, ret, bcos::bytes(), false); + // append the element to line + _result->appendToLine(std::move(element)); + } + // Note: the last one 0x00-bytes means terminate, should been ignored + _result->resizeElement(_result->size() - 1, offset - 1); + _result->setDataSchema(DataSchema::Bytes); + break; + } + default: + { + BOOST_THROW_EXCEPTION(ParseMySQLResultError() << bcos::errinfo_comment( + "unsupported data type: " + std::to_string(colType))); + } + } +} + +void MySQLStorage::parseResultByColumn( + QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, uint32_t _columnSize) const +{ + _queryResult->mutableData().resize(_columnSize); + for (uint64_t i = 0; i < _columnSize; i++) + { + auto batchData = std::make_shared(); + _queryResult->mutableData()[i] = (std::move(batchData)); + } + + while (auto rowData = mysqlx_row_fetch_one((mysqlx_result_t*)_result->result())) + { + for (uint32_t col = 0; col < _columnSize; col++) + { + parseResult((_queryResult->mutableData())[col], _result, rowData, col); + } + } +} + +void MySQLStorage::parseResultByRow( + QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, uint32_t _columnSize) const +{ + while (auto rowData = mysqlx_row_fetch_one((mysqlx_result_t*)_result->result())) + { + auto batchData = std::make_shared(); + for (uint32_t col = 0; col < _columnSize; col++) + { + parseResult(batchData, _result, rowData, col); + } + _queryResult->mutableData().emplace_back(std::move(batchData)); + } +} diff --git a/cpp/ppc-storage/src/mysql/MySQLStorage.h b/cpp/ppc-storage/src/mysql/MySQLStorage.h new file mode 100644 index 00000000..ad3feef7 --- /dev/null +++ b/cpp/ppc-storage/src/mysql/MySQLStorage.h @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MySQLStorage.h + * @author: yujiechen + * @date 2022-10-25 + */ +#pragma once +#include "Common.h" +#include "ppc-framework/storage/SQLStorage.h" +#include +#include +#include + +namespace ppc::storage +{ +class MySQLStatement : public Statement +{ +public: + using Ptr = std::shared_ptr; + MySQLStatement(mysqlx_stmt_t* _statement, mysqlx_session_t* _session) + : m_statement(_statement), m_session(_session) + {} + + ~MySQLStatement() override + { + // return the session + if (m_callback) + { + m_callback(m_session); + } + } + + void* statementContext() override { return (void*)m_statement; } + void* takeSession() override + { + auto session = m_session; + m_session = nullptr; + return (void*)session; + } + + void setReturnSessionCallback(std::function const& _callback) + { + m_callback = _callback; + } + +private: + mysqlx_stmt_t* m_statement; + mysqlx_session_t* m_session; + std::function m_callback; +}; + +class MySQLResultContext : public SQLResultContext +{ +public: + using Ptr = std::shared_ptr; + MySQLResultContext(mysqlx_result_t* _result) : m_result(_result) {} + ~MySQLResultContext() override + { + if (m_result) + { + mysqlx_free(m_result); + } + if (m_callback) + { + m_callback(m_session); + } + } + + // get the sql-result + void* result() override { return (void*)m_result; } + virtual void setSession(void* _session) { m_session = (mysqlx_session_t*)_session; } + + void setReturnSessionCallback(std::function const& _callback) + { + m_callback = _callback; + } + +private: + mysqlx_result_t* m_result; + mysqlx_session_t* m_session; + std::function m_callback; +}; + +class MySQLStorage : public SQLStorage +{ +public: + // TODO: configure the session-pool-size + using Ptr = std::shared_ptr; + MySQLStorage( + ppc::protocol::SQLConnectionOption::Ptr const& _opt, uint16_t _sessionPoolSize = 16); + ~MySQLStorage() override + { + if (m_timer) + { + m_timer->stop(); + } + destroyAllSession(); + } + + // use the database + void useDataBase(const char* _database) override; + + // exec the sql-query-command, throws exception when query-error + QueryResult::Ptr execQuery(bool _parseByColumn, const char* _command, ...) const override; + // exec the sql-write-command: insert/update/delete, throws exception when exec-error + void execCommit(const char* _command, ...) const override; + + std::string const& databaseName() const override { return m_opt->database; } + + Statement::Ptr generateStatement(const char* _command, ...) const override; + SQLResultContext::Ptr execStatement(Statement::Ptr _statement) const override; + void appendStatement(Statement::Ptr _statement, ...) const override; + +protected: + virtual void refreshConnection(); + virtual mysqlx_session_t* connectDataBase(const char* _database, bool _disableSSl) const; + mysqlx_session_t* createSession() const; + virtual void destroySession(mysqlx_session_t* _session) const + { + if (_session) + { + // close the session + mysqlx_session_close(_session); + } + } + // destroy the current session + virtual void destroyAllSession() + { + std::vector sessionPool; + { + bcos::WriteGuard l(x_sessionPool); + sessionPool.swap(m_sessionPool); + } + for (auto const& it : sessionPool) + { + destroySession(it); + } + } + + void parseMetaData(QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, + uint32_t _columnSize) const; + + // parse the queried-result by row + void parseResultByRow(QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, + uint32_t _columnSize) const; + // parse the queried result by column + void parseResultByColumn(QueryResult::Ptr _queryResult, SQLResultContext::Ptr const& _result, + uint32_t _columnSize) const; + // parse the result according to the type + void parseResult(ppc::io::DataBatch::Ptr _result, SQLResultContext::Ptr const& _mysqlResult, + mysqlx_row_t* _rowData, uint32_t _col) const; + + template + bool inline appendResult(mysqlx_row_t* _rowData, ppc::io::DataBatch::Ptr _result, int _retCode, + T&& _element, bool _append = true) const + { + bool readFinished = true; + if (RESULT_ERROR == _retCode) + { + std::string errorMsg = mysqlx_error_message(_rowData); + BOOST_THROW_EXCEPTION(ParseMySQLResultError() << bcos::errinfo_comment( + "parse sql-result error, code:" + std::to_string(_retCode) + + ", error: " + errorMsg)); + } + // the data has not been read-finished yet + if (RESULT_MORE_DATA == _retCode) + { + readFinished = false; + } + if (_append) + { + _result->append(std::move(_element)); + } + return readFinished; + } + + template + void bindStmt(mysqlx_stmt_t* stmt, Args... args) const + { + auto result = mysqlx_stmt_bind(stmt, args...); + if (result != RESULT_OK) + { + MYSQL_STORAGE_LOG(ERROR) << LOG_DESC("bindStmt error") << LOG_KV("code", result) + << LOG_KV("msg", mysqlx_error_message(stmt)); + BOOST_THROW_EXCEPTION( + ExecMySQLError() << bcos::errinfo_comment( + "bindStmt error: " + std::string(mysqlx_error_message(stmt)))); + } + } + + Statement::Ptr generateSQLStatement(const char* _command, va_list& _args) const; + void appendStatement(Statement::Ptr stmt, va_list& args) const; + + SQLResultContext::Ptr execSQL(const char* _command, va_list& _args) const + { + auto statement = generateSQLStatement(_command, _args); + return execStatement(statement); + } + + + mysqlx_session_t* allocateSession() const + { + // Note: can't new multiple mysql-session at the same times + bcos::WriteGuard l(x_sessionPool); + // all session has been used-up, allocate a new session + if (m_sessionPool.empty()) + { + MYSQL_STORAGE_LOG(DEBUG) << LOG_DESC("allocateSession for the session pool is empty"); + return createSession(); + } + MYSQL_STORAGE_LOG(DEBUG) << LOG_DESC("allocateSession from the session pool") + << LOG_KV("poolSize", m_sessionPool.size()); + // obtain session from the sessionPool + auto session = m_sessionPool.back(); + m_sessionPool.pop_back(); + return session; + } + + // Note: the session can be only returned after the result obtained + void returnSession(mysqlx_session_t* _session) const + { + if (!_session) + { + return; + } + bcos::UpgradableGuard l(x_sessionPool); + if (m_sessionPool.size() >= m_sessionPoolSize) + { + MYSQL_STORAGE_LOG(DEBUG) + << LOG_DESC("returnSession: destroy the session for the pool-size over limit") + << LOG_KV("poolSize", m_sessionPool.size()); + destroySession(_session); + return; + } + bcos::UpgradeGuard ul(l); + m_sessionPool.emplace_back(_session); + MYSQL_STORAGE_LOG(DEBUG) << LOG_DESC("returnSession") + << LOG_KV("poolSize", m_sessionPool.size()); + } + + void insertSession(mysqlx_session_t* _session) + { + bcos::WriteGuard l(x_sessionPool); + m_sessionPool.emplace_back(_session); + } + +private: + ppc::protocol::SQLConnectionOption::Ptr m_opt; + // TODO: support session-pool + std::vector mutable m_sessionPool; + bcos::SharedMutex mutable x_sessionPool; + + uint16_t m_sessionPoolSize = 16; + std::shared_ptr m_timer; + + int const c_maxElementLen = 65536; +}; +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-storage/src/redis/RedisStorage.cpp b/cpp/ppc-storage/src/redis/RedisStorage.cpp new file mode 100644 index 00000000..2a057110 --- /dev/null +++ b/cpp/ppc-storage/src/redis/RedisStorage.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RedisStorage.cpp + * @author: yujiechen + * @date 2022-10-24 + */ +#include "RedisStorage.h" +#include + +using namespace ppc::storage; +using namespace sw::redis; + +bool RedisStorage::exists(const std::string& _key) +{ + // return 1 if key exists; return 0 if key doesn't exist + return (m_redis->exists(_key) == 0 ? false : true); +} + +void RedisStorage::setValue( + const std::string& _key, const std::string& _value, int32_t _expirationSeconds) +{ + m_redis->set(_key, _value); + // Note: when _expirationSeconds is set to -1, never trigger expiration + if (_expirationSeconds) + { + expireKey(_key, _expirationSeconds); + } +} + +// std::nullopt if the key doesn't exist +std::optional RedisStorage::getValue(const std::string& _key) +{ + OptionalString result = m_redis->get(_key); + if (!result) + { + return std::nullopt; + } + return std::string(result.value()); +} + +bool RedisStorage::expireKey(const std::string& _key, uint32_t _expirationSeconds) +{ + return m_redis->expire(_key, _expirationSeconds); +} + +uint64_t RedisStorage::deleteKey(const std::string& _key) +{ + return m_redis->del(_key); +} diff --git a/cpp/ppc-storage/src/redis/RedisStorage.h b/cpp/ppc-storage/src/redis/RedisStorage.h new file mode 100644 index 00000000..374c2030 --- /dev/null +++ b/cpp/ppc-storage/src/redis/RedisStorage.h @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RedisStorage.h + * @author: yujiechen + * @date 2022-10-24 + */ +#pragma once + +#include "ppc-framework/storage/CacheStorage.h" +#include + +namespace ppc::storage +{ + +class RedisStorage : public CacheStorage +{ +public: + using Ptr = std::shared_ptr; + RedisStorage(const sw::redis::ConnectionOptions& _connectionOpts, + const sw::redis::ConnectionPoolOptions& _poolOptions) + { + m_redis = std::make_shared(_connectionOpts, _poolOptions); + } + ~RedisStorage() override {} + +public: + /// Note: all these interfaces throws exception when error happened + /** + * @brief: check whether the key exists + * @param _key: key + * @return whether the key exists + */ + bool exists(const std::string& _key) override; + + /** + * @brief: set key value + * @param _expirationTime: timeout of key, seconds + */ + void setValue(const std::string& _key, const std::string& _value, + int32_t _expirationSeconds = -1) override; + + /** + * @brief: get value by key + * @param _key: key + * @return value + */ + std::optional getValue(const std::string& _key) override; + + /** + * @brief: set a timeout on key + * @param _expirationTime: timeout of key, ms + * @return whether setting is successful + */ + bool expireKey(const std::string& _key, uint32_t _expirationTime) override; + + /** + * @brief: delete key + * @param _key: key + * @return the number of key deleted + */ + uint64_t deleteKey(const std::string& _key) override; + +private: + std::shared_ptr m_redis; +}; + +} // namespace ppc::storage \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/CMakeLists.txt b/cpp/ppc-tars-protocol/CMakeLists.txt new file mode 100644 index 00000000..7c684a1a --- /dev/null +++ b/cpp/ppc-tars-protocol/CMakeLists.txt @@ -0,0 +1,53 @@ +cmake_minimum_required(VERSION 3.14) + +include(Version) +project(ppc-tars-protocol VERSION ${VERSION}) + +# for tars generator +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-tars-protocol/tars) +find_program(TARS_TARS2CPP tars2cpp REQUIRED) + +file(GLOB_RECURSE TARS_INPUT "*.tars") + +# generate tars +if (TARS_INPUT) + foreach (TARS_FILE ${TARS_INPUT}) + get_filename_component(TARS_NAME ${TARS_FILE} NAME_WE) + get_filename_component(TARS_PATH ${TARS_FILE} PATH) + add_custom_command( + OUTPUT ${TARS_HEADER_DIR}/${TARS_NAME}.h + WORKING_DIRECTORY ${TARS_PATH} + COMMAND ${TARS_TARS2CPP} ${TARS_FILE} --unjson --without-trace --dir=${TARS_HEADER_DIR} + COMMENT "generating ${TARS_FILE} to ${TARS_HEADER_DIR}" + VERBATIM + ) + + list(APPEND OUT_TARS_H_LIST ${TARS_HEADER_DIR}/${TARS_NAME}.h) + endforeach () +endif () + +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") + +file(GLOB_RECURSE SRC_LIST "ppc-tars-protocol/*.cpp" "ppc-tars-protocol/*.h") + +find_package(tarscpp REQUIRED) +add_library(${TARS_PROTOCOL_TARGET} ${SRC_LIST} ${OUT_TARS_H_LIST}) +target_include_directories(${TARS_PROTOCOL_TARGET} PUBLIC + $ + $ + $ + $) +target_link_libraries(${TARS_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} tarscpp::tarsservant tarscpp::tarsutil) + +# ut +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(test) +endif () + + +include(GNUInstallDirs) +#install(TARGETS ${TARS_PROTOCOL_TARGET} EXPORT ppcTargets ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") +install(DIRECTORY "ppc-tars-protocol" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" FILES_MATCHING PATTERN "*.h") +install(DIRECTORY "${CMAKE_BINARY_DIR}/generated/ppc-tars-protocol" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" FILES_MATCHING PATTERN "*.h") \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h new file mode 100644 index 00000000..576e3ec6 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h @@ -0,0 +1,396 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "Error.h" +#include "TarsServantProxyCallback.h" +#include "TaskInfo.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-framework/protocol/Task.h" +#include "ppc-tools/src/config/ParamChecker.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ppctars +{ +namespace serialize +{ +template +class BufferWriter +{ +protected: + using ByteType = typename Container::value_type; + using SizeType = typename Container::size_type; + + mutable Container _buffer; + ByteType* _buf; + SizeType _len; + SizeType _buf_len; + std::function _reserve; + +private: + BufferWriter(const BufferWriter&); + BufferWriter& operator=(const BufferWriter& buf); + +public: + BufferWriter() : _buf(NULL), _len(0), _buf_len(0), _reserve({}) + { +#ifndef GEN_PYTHON_MASK + _reserve = [](BufferWriter& os, size_t len) { + os._buffer.resize(len); + return os._buffer.data(); + }; +#endif + } + + ~BufferWriter() {} + + void reset() { _len = 0; } + + void writeBuf(const ByteType* buf, size_t len) + { + TarsReserveBuf(*this, _len + len); + memcpy(_buf + _len, buf, len); + _len += len; + } + + const Container& getByteBuffer() const + { + _buffer.resize(_len); + return _buffer; + } + Container& getByteBuffer() + { + _buffer.resize(_len); + return _buffer; + } + const ByteType* getBuffer() const { return _buf; } + size_t getLength() const { return _len; } + void swap(std::vector& v) + { + _buffer.resize(_len); + v.swap(_buffer); + _buf = NULL; + _buf_len = 0; + _len = 0; + } + void swap(BufferWriter& buf) + { + buf._buffer.swap(_buffer); + std::swap(_buf, buf._buf); + std::swap(_buf_len, buf._buf_len); + std::swap(_len, buf._len); + } +}; + +using BufferWriterByteVector = BufferWriter>; +using BufferWriterStdByteVector = BufferWriter>; +using BufferWriterString = BufferWriter; +} // namespace serialize + + +template +bool checkConnection(std::string const& _module, std::string const& _func, const T& prx, + std::function _errorCallback, bool _callsErrorCallback = true) +{ + auto cb = prx->tars_get_push_callback(); + assert(cb); + auto* tarsServantProxyCallback = (TarsServantProxyCallback*)cb.get(); + + if (tarsServantProxyCallback->available()) + { + return true; + } + + if (_errorCallback && _callsErrorCallback) + { + std::string errorMessage = + _module + " calls interface " + _func + " failed for empty connection"; + _errorCallback(std::make_shared(-1, errorMessage)); + } + return false; +} + + +inline ppctars::TaskInfo toTarsTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr _taskInfo) +{ + ppctars::TaskInfo tarsTaskInfo; + if (!_taskInfo) + { + return tarsTaskInfo; + } + + tarsTaskInfo.taskID = _taskInfo->taskID; + tarsTaskInfo.serviceEndpoint = _taskInfo->serviceEndpoint; + + return tarsTaskInfo; +} + +inline ppc::protocol::GatewayTaskInfo::Ptr toGatewayTaskInfo(ppctars::TaskInfo _taskInfo) +{ + auto gatewayTaskInfo = std::make_shared(); + + gatewayTaskInfo->taskID = _taskInfo.taskID; + gatewayTaskInfo->serviceEndpoint = _taskInfo.serviceEndpoint; + + return gatewayTaskInfo; +} + +inline ppctars::Error toTarsError(const bcos::Error& error) +{ + ppctars::Error tarsError; + tarsError.errorCode = error.errorCode(); + tarsError.errorMessage = error.errorMessage(); + + return tarsError; +} + +template +inline ppctars::Error toTarsError(const T& error) +{ + ppctars::Error tarsError; + + if (error) + { + tarsError.errorCode = error->errorCode(); + tarsError.errorMessage = error->errorMessage(); + } + + return tarsError; +} + +inline bcos::Error::Ptr toBcosError(const ppctars::Error& error) +{ + if (error.errorCode == 0) + { + return nullptr; + } + + return std::make_shared(error.errorCode, error.errorMessage); +} + +inline bcos::Error::Ptr toBcosError(tars::Int32 ret) +{ + if (ret == 0) + { + return nullptr; + } + + return std::make_shared(ret, "TARS error!"); +} + +inline std::string getProxyDesc(std::string const& _servantName) +{ + std::string desc = + tars::ServerConfig::Application + "." + tars::ServerConfig::ServerName + "." + _servantName; + return desc; +} + +inline std::string getLogPath() +{ + return tars::ServerConfig::LogPath + "/" + tars::ServerConfig::Application + "/" + + tars::ServerConfig::ServerName; +} + +inline tars::TC_Endpoint string2TarsEndPoint(const std::string& _strEndPoint) +{ + std::vector temp; + boost::split(temp, _strEndPoint, boost::is_any_of(":"), boost::token_compress_on); + + if (temp.size() != 2) + { + BOOST_THROW_EXCEPTION(bcos::InvalidParameter() << bcos::errinfo_comment( + "incorrect string endpoint, it should be in IP:Port format")); + } + + tars::TC_Endpoint ep(temp[0], boost::lexical_cast(temp[1]), 60000); + return ep; +} + +inline std::vector toTarsEndPoints(std::vector const& _endPoints) +{ + std::vector tarsEndPoints; + for (auto const& it : _endPoints) + { + tarsEndPoints.emplace_back(string2TarsEndPoint(it)); + } + return tarsEndPoints; +} + +inline std::string endPointToString(std::string const& _serviceName, const std::string& _endpoint) +{ + std::vector url; + boost::split(url, _endpoint, boost::is_any_of(":"), boost::token_compress_on); + return _serviceName + "@tcp -h " + url[0] + " -p " + url[1]; +} + +inline std::string endPointToString( + std::string const& _serviceName, const std::string& _host, uint16_t _port) +{ + return _serviceName + "@tcp -h " + _host + " -p " + boost::lexical_cast(_port); +} + +inline std::string endPointToString( + std::string const& _serviceName, tars::TC_Endpoint const& _endPoint) +{ + return endPointToString(_serviceName, _endPoint.getHost(), _endPoint.getPort()); +} + +inline std::string endPointToString( + std::string const& _serviceName, const std::vector& _eps) +{ + if (_eps.empty()) + { + BOOST_THROW_EXCEPTION( + bcos::InvalidParameter() << bcos::errinfo_comment( + "vector should not be empty in endPointToString")); + } + + bool first = true; + std::string endPointStr = _serviceName; + for (const auto& _ep : _eps) + { + endPointStr += (first ? "@" : ":"); + endPointStr += + ("tcp -h " + _ep.getHost() + " -p " + boost::lexical_cast(_ep.getPort())); + + first = false; + } + return endPointStr; +} + +inline std::pair getEndPointDescByAdapter(tars::Application* _application, + std::string const& _servantName, std::string const& _endpoint = "") +{ + auto adapters = _application->getEpollServer()->getBindAdapters(); + if (adapters.size() == 0) + { + return std::make_pair(false, ""); + } + auto prxDesc = getProxyDesc(_servantName); + if (!_endpoint.empty()) + { + if (!ppc::tools::checkEndpoint(_endpoint)) + { + BOOST_THROW_EXCEPTION(bcos::InvalidParameter() << bcos::errinfo_comment( + "incorrect string endpoint, it should be in IP:Port format")); + } + return std::make_pair(true, endPointToString(prxDesc, _endpoint)); + } + + auto adapterName = prxDesc + "Adapter"; + for (auto const& adapter : adapters) + { + if (adapter->getName() == adapterName) + { + return std::make_pair(true, endPointToString(prxDesc, adapter->getEndpoint())); + } + } + return std::make_pair(false, ""); +} + +template +S createServantProxy(tars::Communicator* communicator, std::string const& _endPointStr, + TarsServantProxyOnConnectHandler _connectHandler = TarsServantProxyOnConnectHandler(), + TarsServantProxyOnCloseHandler _closeHandler = TarsServantProxyOnCloseHandler()) +{ + auto prx = communicator->stringToProxy(_endPointStr); + + BCOS_LOG(DEBUG) << LOG_DESC("createServantProxy ") << _endPointStr; + + if (!prx->tars_get_push_callback()) + { + auto proxyCallback = new ppctars::TarsServantProxyCallback(_endPointStr, *prx); + + if (_connectHandler) + { + proxyCallback->setOnConnectHandler(_connectHandler); + } + + if (_closeHandler) + { + proxyCallback->setOnCloseHandler(_closeHandler); + } + + prx->tars_set_push_callback(proxyCallback); + proxyCallback->startTimer(); + } + return prx; +} + +template +S createServantProxy(std::string const& _endPointStr) +{ + return createServantProxy(tars::Application::getCommunicator().get(), _endPointStr, + TarsServantProxyOnConnectHandler(), TarsServantProxyOnCloseHandler()); +} + +template +S createServantProxy(std::string const& _serviceName, const std::string& _host, uint16_t _port) +{ + auto endPointStr = endPointToString(_serviceName, _host, _port); + return createServantProxy(endPointStr); +} + +template +S createServantProxy(std::string const& _serviceName, const tars::TC_Endpoint& _ep) +{ + auto endPointStr = endPointToString(_serviceName, _ep); + return createServantProxy(endPointStr); +} + +template +S createServantProxy(std::string const& _serviceName, const std::vector& _eps) +{ + std::string endPointStr = endPointToString(_serviceName, _eps); + + return createServantProxy(endPointStr); +} + +template +S createServantProxy(bool _withEndPoints, std::string const& _serviceName, + const std::vector& _eps = std::vector{}) +{ + std::string serviceParams; + if (_withEndPoints) + { + serviceParams = endPointToString(_serviceName, _eps); + } + else + { + serviceParams = _serviceName; + } + + return createServantProxy(serviceParams); +} +} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h new file mode 100644 index 00000000..b15ea3f4 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TarsSerializable.h + * @author: shawnhe + * @date 2022-11-4 + */ + +#pragma once + +#include "Common.h" +#include "TarsStruct.h" + +namespace ppctars::serialize +{ + +void encode(TarsStruct auto const& object, bcos::bytes& out) +{ + tars::TarsOutputStream output; + object.writeTo(output); + output.getByteBuffer().swap(out); +} + +void decode(const bcos::bytes& in, TarsStruct auto& out) +{ + tars::TarsInputStream input; + input.setBuffer((const char*)in.data(), in.size()); + out.readFrom(input); +} + +} // namespace ppctars diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h new file mode 100644 index 00000000..16521ee4 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h @@ -0,0 +1,242 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief tars implementation for tars ServantProxyCallback + * @file TarsServantProxyCallback.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "bcos-utilities/BoostLog.h" +#include "bcos-utilities/Error.h" +#include "bcos-utilities/Timer.h" +#include "servant/ServantProxy.h" +#include +#include +#include +#include +#include +#include +#include + +#define TARS_PING_PERIOD (3000) + +namespace ppctars +{ +struct TC_EndpointCompare +{ + bool operator()(const tars::TC_Endpoint& lhs, const tars::TC_Endpoint& rhs) const + { + return lhs.toString() < rhs.toString(); + } +}; + +using EndpointSet = std::set; + +using TarsServantProxyOnConnectHandler = std::function; +using TarsServantProxyOnCloseHandler = std::function; + +class TarsServantProxyCallback : public tars::ServantProxyCallback + +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + using UniquePtr = std::unique_ptr; + +public: + TarsServantProxyCallback(const std::string& _serviceName, tars::ServantProxy& _servantProxy) + : m_serviceName(_serviceName), m_servantProxy(_servantProxy) + { + m_timer = std::make_shared(TARS_PING_PERIOD, "tars_ping"); + + BCOS_LOG(INFO) << LOG_BADGE("[NEWOBJ][TarsServantProxyCallback]") + << LOG_KV("_serviceName", _serviceName) << LOG_KV("this", this); + } + + TarsServantProxyCallback(TarsServantProxyCallback&&) = delete; + TarsServantProxyCallback(const TarsServantProxyCallback&) = delete; + const TarsServantProxyCallback& operator=(const TarsServantProxyCallback&) = delete; + TarsServantProxyCallback& operator=(TarsServantProxyCallback&&) = delete; + + ~TarsServantProxyCallback() override + { + if (m_timer) + { + m_timer->stop(); + } + } + +public: + int onDispatch(tars::ReqMessagePtr) override { return 0; } + + void onClose(const tars::TC_Endpoint& ep) override + { + try + { + auto p = addInactiveEndpoint(ep); + BCOS_LOG(INFO) << LOG_DESC("ServantProxyCallback::onClose:") << m_serviceName + << LOG_KV("endpoint", ep.toString()) + << LOG_KV("inActiveEndPointSize", p.second); + if (p.first && m_onCloseHandler) + { + m_onCloseHandler(ep); + } + } + catch (const std::exception& e) + { + std::cout << "[TarsServantProxyCallback::onClose] " + << "exception: " << boost::diagnostic_information(e) << std::endl; + } + } + + void onConnect(const tars::TC_Endpoint& ep) override + { + auto p = addActiveEndpoint(ep); + BCOS_LOG(INFO) << LOG_BADGE("ServantProxyCallback::onConnect") << LOG_KV("this", this) + << LOG_KV("serviceName", m_serviceName) << LOG_KV("endpoint", ep.toString()) + << LOG_KV("result", p.first) << LOG_KV("activeEndpoints size", p.second); + + if (p.first && m_onConnectHandler) + { + m_onConnectHandler(ep); + } + } + +public: + void startTimer() + { + if (!m_timer) + { + return; + } + + m_timer->registerTimeoutHandler([this]() { + ping(); + restart(); + }); + + m_timer->start(); + } + + const std::string& serviceName() const { return m_serviceName; } + + auto activeEndpoints() + { + std::shared_lock l(x_endpoints); + return m_activeEndpoints; + } + + EndpointSet inactiveEndpoints() + { + std::shared_lock l(x_endpoints); + return m_inactiveEndpoints; + } + + std::pair addActiveEndpoint(const tars::TC_Endpoint& ep) + { + std::unique_lock l(x_endpoints); + auto result = m_activeEndpoints.insert(ep); + m_inactiveEndpoints.erase(ep); + return std::make_pair(result.second, m_activeEndpoints.size()); + } + + std::pair addInactiveEndpoint(const tars::TC_Endpoint& ep) + { + { + std::shared_lock l(x_endpoints); + if (m_inactiveEndpoints.find(ep) != m_inactiveEndpoints.end()) + { + return std::make_pair(false, m_inactiveEndpoints.size()); + } + } + + { + std::unique_lock l(x_endpoints); + auto result = m_inactiveEndpoints.insert(ep); + m_activeEndpoints.erase(ep); + + BCOS_LOG(INFO) << LOG_BADGE("ServantProxyCallback::addInactiveEndpoint") + << LOG_KV("this", this) << LOG_KV("result", result.second) + << LOG_KV("endpoint", ep.toString()); + + return std::make_pair(result.second, m_inactiveEndpoints.size()); + } + } + + TarsServantProxyOnConnectHandler onConnectHandler() const { return m_onConnectHandler; } + TarsServantProxyOnCloseHandler onCloseHandler() const { return m_onCloseHandler; } + + void setOnConnectHandler(TarsServantProxyOnConnectHandler _handler) + { + m_onConnectHandler = _handler; + } + + void setOnCloseHandler(TarsServantProxyOnCloseHandler _handler) { m_onCloseHandler = _handler; } + + bool available() { return !activeEndpoints().empty(); } + + void ping() + { + try + { + m_servantProxy.tars_async_ping(); + } + catch (const std::exception& _e) + { + BCOS_LOG(ERROR) << LOG_BADGE("ServantProxyCallback::tars_async_ping") + << LOG_KV("serviceName", m_serviceName) + << LOG_KV("e", boost::diagnostic_information(_e)); + } + } + + void restart() + { + if (m_timer) + { + m_timer->restart(); + } + } + +private: + std::string m_serviceName; + + // tars ServantProxy which the TarsServantProxyCallback belongs to + tars::ServantProxy& m_servantProxy; + + // timer for ping + std::shared_ptr m_timer; + + // lock for m_activeEndpoints and m_inactiveEndpoints + mutable std::shared_mutex x_endpoints; + // all active tars endpoints + EndpointSet m_activeEndpoints; + // all inactive tars endpoints + EndpointSet m_inactiveEndpoints; + + std::function m_onConnectHandler; + std::function m_onCloseHandler; +}; + +template +auto tarsProxyActiveEndPoints(const T& prx) +{ + auto cb = prx->tars_get_push_callback(); + assert(cb); + return ((TarsServantProxyCallback*)cb.get())->activeEndpoints(); +} +} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h new file mode 100644 index 00000000..7d775a45 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TarsStruct.h + * @author: shawnhe + * @date 2022-11-4 + */ + +#pragma once +#include +#include + +namespace ppctars::serialize +{ + +template +concept TarsStruct = requires(TarsStructType tarsStruct) +{ + { + tarsStruct.className() + } -> std::same_as; + { + tarsStruct.MD5() + } -> std::same_as; + tarsStruct.resetDefautlt(); +}; +} diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h new file mode 100644 index 00000000..2209f62f --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h @@ -0,0 +1,139 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief client for front service + * @file FrontServiceClient.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-parameter" + +#include "FrontService.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" +#include +#include + +#include + +#define FRONTCLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("FrontServiceClient") + +namespace ppctars +{ +class FrontServiceClient : public ppc::front::FrontInterface +{ +public: + void start() override {} + void stop() override {} + + explicit FrontServiceClient(const ppctars::FrontServicePrx& proxy) : m_proxy(proxy) {} + + void onReceiveMessage( + ppc::front::PPCMessageFace::Ptr _message, ppc::front::ErrorCallbackFunc _callback) override + { + class Callback : public FrontServicePrxCallback + { + public: + explicit Callback(ppc::front::ErrorCallbackFunc callback) + : m_callback(std::move(callback)) + {} + + void callback_onReceiveMessage(const ppctars::Error& ret) override + { + if (!m_callback) + { + return; + } + m_callback(toBcosError(ret)); + } + + void callback_onReceiveMessage_exception(tars::Int32 ret) override + { + if (!m_callback) + { + return; + } + m_callback(toBcosError(ret)); + } + + private: + ppc::front::ErrorCallbackFunc m_callback; + }; + + auto startT = bcos::utcSteadyTime(); + // encode message to bytes + bcos::bytes buffer; + _message->encode(buffer); + + FRONTCLIENT_LOG(TRACE) << LOG_DESC("after decode") + << LOG_KV("taskType", unsigned(_message->taskType())) + << LOG_KV("algorithmType", unsigned(_message->algorithmType())) + << LOG_KV("messageType", unsigned(_message->messageType())) + << LOG_KV("seq", _message->seq()) + << LOG_KV("taskID", _message->taskID()); + + m_proxy->tars_set_timeout(c_networkTimeout) + ->async_onReceiveMessage( + new Callback(_callback), std::vector(buffer.begin(), buffer.end())); + BCOS_LOG(TRACE) << LOG_DESC("call front onReceiveMessage") + << LOG_KV("msgSize", buffer.size()) + << LOG_KV("timecost", bcos::utcSteadyTime() - startT); + } + + // Note: since ppc-front is integrated with the node, no-need to implement this method + bcos::Error::Ptr notifyTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr) override + { + throw std::runtime_error("notifyTaskInfo: unimplemented interface!"); + } + + // Note: since ppc-front is integrated with the node, no-need to implement this method + // erase the task-info when task finished + bcos::Error::Ptr eraseTaskInfo(std::string const&) override + { + throw std::runtime_error("eraseTaskInfo: unimplemented interface!"); + } + + // Note: since ppc-front is integrated with the node, no-need to implement this method + void asyncSendMessage(const std::string&, ppc::front::PPCMessageFace::Ptr, uint32_t _timeout, + ppc::front::ErrorCallbackFunc _callback, ppc::front::CallbackFunc _respCallback) override + { + throw std::runtime_error("asyncSendMessage: unimplemented interface!"); + } + + // Note: since ppc-front is integrated with the node, no-need to implement this method + // send response when receiving message from given agencyID + void asyncSendResponse(const std::string&, std::string const&, ppc::front::PPCMessageFace::Ptr, + ppc::front::ErrorCallbackFunc) override + { + throw std::runtime_error("asyncSendResponse: unimplemented interface!"); + } + + // Note: since ppc-front is integrated with the node, no-need to implement this method + void asyncGetAgencyList(ppc::front::GetAgencyListCallback) override + { + throw std::runtime_error("asyncGetAgencyList: unimplemented interface!"); + } + +private: + // 1800s + const int c_networkTimeout = 1800000; + + ppctars::FrontServicePrx m_proxy; +}; +} // namespace ppctars diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp new file mode 100644 index 00000000..534a993f --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp @@ -0,0 +1,24 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServiceClient.cpp + * @author: shawnhe + * @date 2022-10-20 + */ + +#include "GatewayServiceClient.h" + +std::atomic ppctars::GatewayServiceClient::s_tarsTimeoutCount = {0}; +const int64_t ppctars::GatewayServiceClient::c_maxTarsTimeoutCount = 500; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h new file mode 100644 index 00000000..52db1dcc --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h @@ -0,0 +1,298 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief client for gateway service + * @file GatewayServiceClient.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-parameter" + +#include "GatewayService.h" +#include "ppc-framework/Common.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-tars-protocol/Common.h" +#include "ppc-tars-protocol/TarsServantProxyCallback.h" +#include +#include +#include + +#include + +#define GATEWAYCLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("GatewayServiceClient") + +namespace ppctars +{ +class GatewayServiceClient : public ppc::gateway::GatewayInterface +{ +public: + void start() override {} + void stop() override {} + + explicit GatewayServiceClient(std::string const& _gatewayServiceName, + const ppctars::GatewayServicePrx& _prx, int _holdingMessageMinutes) + : m_gatewayServiceName(_gatewayServiceName), + m_prx(_prx), + m_networkTimeout(_holdingMessageMinutes * 60 * 1000) // convert to ms + { + BCOS_LOG(INFO) << LOG_DESC("GatewayServiceClient") + << LOG_KV("networkTimeout", m_networkTimeout); + } + + + void asyncSendMessage(const std::string& _agencyID, ppc::front::PPCMessageFace::Ptr _message, + ppc::gateway::ErrorCallbackFunc _callback) override + { + class Callback : public GatewayServicePrxCallback + { + public: + explicit Callback(ppc::gateway::ErrorCallbackFunc callback) + : m_callback(std::move(callback)) + {} + + void callback_asyncSendMessage(const ppctars::Error& ret) override + { + s_tarsTimeoutCount.store(0); + if (!m_callback) + { + return; + } + m_callback(toBcosError(ret)); + } + + void callback_asyncSendMessage_exception(tars::Int32 ret) override + { + s_tarsTimeoutCount++; + if (!m_callback) + { + return; + } + m_callback(toBcosError(ret)); + } + + private: + ppc::gateway::ErrorCallbackFunc m_callback; + }; + + // encode message to bytes + bcos::bytes buffer; + _message->encode(buffer); + + GATEWAYCLIENT_LOG(TRACE) << LOG_DESC("send message to gateway by client") + << LOG_KV("taskType", unsigned(_message->taskType())) + << LOG_KV("algorithmType", unsigned(_message->algorithmType())) + << LOG_KV("messageType", unsigned(_message->messageType())) + << LOG_KV("seq", _message->seq()) + << LOG_KV("taskID", _message->taskID()) + << LOG_KV("receiver", _agencyID); + + m_prx->tars_set_timeout(m_networkTimeout) + ->async_asyncSendMessage(new Callback(_callback), _agencyID, + std::vector(buffer.begin(), buffer.end())); + } + + bcos::Error::Ptr notifyTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr _taskInfo) override + { + auto tarsTaskInfo = toTarsTaskInfo(_taskInfo); + auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); + // broadcast to all gateways + uint errorCount = 0; + std::string lastErrorMsg; + for (auto& endPoint : activeEndPoints) + { + auto prx = + ppctars::createServantProxy(m_gatewayServiceName, endPoint); + auto error = prx->tars_set_timeout(m_networkTimeout)->notifyTaskInfo(tarsTaskInfo); + if (error.errorCode) + { + ++errorCount; + lastErrorMsg = error.errorMessage; + } + } + if (errorCount) + { + return std::make_shared( + ppc::protocol::PPCRetCode::NOTIFY_TASK_ERROR, lastErrorMsg); + } + + return nullptr; + } + + bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override + { + auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); + // broadcast to all gateways + uint errorCount = 0; + for (auto& endPoint : activeEndPoints) + { + try + { + auto prx = + ppctars::createServantProxy(m_gatewayServiceName, endPoint); + auto error = prx->tars_set_timeout(m_networkTimeout)->eraseTaskInfo(_taskID); + if (error.errorCode) + { + ++errorCount; + } + } + catch (std::exception const& e) + { + ++errorCount; + BCOS_LOG(INFO) << LOG_DESC("eraseTaskInfo exception") + << LOG_KV("exception", boost::diagnostic_information(e)); + } + } + if (errorCount) + { + return std::make_shared( + -1, "eraseTaskInfo: error count: " + std::to_string(errorCount)); + } + + return nullptr; + } + + bcos::Error::Ptr registerGateway( + const std::vector& _gatewayList) override + { + std::vector gatewayList; + for (const auto& gateway : _gatewayList) + { + ppctars::GatewayInfo tarsGate; + tarsGate.agencyID = gateway.agencyID; + tarsGate.endpoint = gateway.endpoint; + gatewayList.push_back(tarsGate); + } + auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); + // broadcast to all gateways + uint errorCount = 0; + std::string lastErrorMsg; + for (auto& endPoint : activeEndPoints) + { + auto prx = + ppctars::createServantProxy(m_gatewayServiceName, endPoint); + auto error = prx->tars_set_timeout(m_networkTimeout)->registerGateway(gatewayList); + if (error.errorCode) + { + ++errorCount; + lastErrorMsg = error.errorMessage; + } + } + if (errorCount) + { + return std::make_shared( + ppc::protocol::PPCRetCode::REGISTER_GATEWAY_URL_ERROR, lastErrorMsg); + } + + return nullptr; + } + + + void registerFront(std::string const& _endPoint, ppc::front::FrontInterface::Ptr) override + { + // Error registerFront(string endPoint); + class Callback : public GatewayServicePrxCallback + { + public: + explicit Callback(std::function _callback) + : GatewayServicePrxCallback(), m_callback(_callback) + {} + ~Callback() override {} + + void callback_asyncRegisterFront(const ppctars::Error& ret) override + { + m_callback(toBcosError(ret)); + } + void callback_asyncRegisterFront_exception(tars::Int32 ret) override + { + m_callback(toBcosError(ret)); + } + + private: + std::function m_callback; + }; + auto startT = bcos::utcSteadyTime(); + auto callback = [_endPoint, startT](bcos::Error::Ptr _error) { + if (!_error || _error->errorCode() == 0) + { + GATEWAYCLIENT_LOG(TRACE) + << LOG_DESC("registerFront success") << LOG_KV("endPoint", _endPoint) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + return; + } + GATEWAYCLIENT_LOG(INFO) + << LOG_DESC("registerFront failed") << LOG_KV("code", _error->errorCode()) + << LOG_KV("endPoint", _endPoint) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + }; + m_prx->tars_set_timeout(m_networkTimeout) + ->async_asyncRegisterFront(new Callback(callback), _endPoint); + } + + void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override + { + class Callback : public GatewayServicePrxCallback + { + public: + explicit Callback(ppc::front::GetAgencyListCallback _callback) + : GatewayServicePrxCallback(), m_callback(_callback) + {} + ~Callback() override {} + + void callback_asyncGetAgencyList( + const ppctars::Error& ret, std::vector const& _agencyList) override + { + auto tmpAgencyList = _agencyList; + m_callback(toBcosError(ret), std::move(tmpAgencyList)); + } + void callback_asyncGetAgencyList_exception(tars::Int32 ret) override + { + std::vector emptyAgencyList; + m_callback(toBcosError(ret), std::move(emptyAgencyList)); + } + + private: + ppc::front::GetAgencyListCallback m_callback; + }; + m_prx->async_asyncGetAgencyList(new Callback(_callback)); + } + + // Note: unregisterFront is the function of the front-inner, no need to implement the client + void unregisterFront(std::string const&) override + { + throw std::runtime_error("unregisterFront: unimplemented interface!"); + } + + +protected: + static bool shouldStopCall() { return (s_tarsTimeoutCount >= c_maxTarsTimeoutCount); } + + +private: + std::string m_gatewayServiceName; + ppctars::GatewayServicePrx m_prx; + + // 1800s + int m_networkTimeout = 1800000; + std::string const c_moduleName = "GatewayServiceClient"; + + static std::atomic s_tarsTimeoutCount; + static const int64_t c_maxTarsTimeoutCount; +}; +} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars new file mode 100644 index 00000000..5fa26bd7 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars @@ -0,0 +1,6 @@ +module ppctars { + struct Error { + 1 optional int errorCode; + 2 optional string errorMessage; + }; +}; \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars new file mode 100644 index 00000000..763bf91f --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars @@ -0,0 +1,7 @@ +#include "Error.tars" +module ppctars { + interface FrontService { + // avoid using unsigned char + Error onReceiveMessage(vector message); + }; +}; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars new file mode 100644 index 00000000..017ca607 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars @@ -0,0 +1,19 @@ +#include "TaskInfo.tars" +#include "Error.tars" +module ppctars +{ + struct GatewayInfo { + 1 optional string agencyID; + 2 optional string endpoint; + }; + interface GatewayService + { + Error asyncSendMessage(string agencyID, vector message); + + Error notifyTaskInfo(TaskInfo taskInfo); + Error eraseTaskInfo(string taskID); + Error registerGateway(vector gatewayList); + Error asyncRegisterFront(string endPoint); + Error asyncGetAgencyList(out vector _agencyList); + }; +}; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars new file mode 100644 index 00000000..1d6af6f4 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars @@ -0,0 +1,7 @@ +module ppctars { +struct TaskInfo +{ + 1 require string taskID; + 2 require string serviceEndpoint; +}; +}; diff --git a/cpp/ppc-tars-protocol/test/CMakeLists.txt b/cpp/ppc-tars-protocol/test/CMakeLists.txt new file mode 100644 index 00000000..21935b94 --- /dev/null +++ b/cpp/ppc-tars-protocol/test/CMakeLists.txt @@ -0,0 +1,13 @@ +file(GLOB_RECURSE SOURCES "*.cpp") + +# cmake settings +set(TARS_PROTOCOL_TEST_BINARY_NAME test-ppc-tars-protocol) + +add_executable(${TARS_PROTOCOL_TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TARS_PROTOCOL_TEST_BINARY_NAME} PRIVATE .) + +target_compile_options(${TARS_PROTOCOL_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unused-variable) + +target_link_libraries(${TARS_PROTOCOL_TEST_BINARY_NAME} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) + +add_test(NAME test-tars-protocol WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TARS_PROTOCOL_TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/test/SerializeTest.cpp b/cpp/ppc-tars-protocol/test/SerializeTest.cpp new file mode 100644 index 00000000..b6f33f9f --- /dev/null +++ b/cpp/ppc-tars-protocol/test/SerializeTest.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayTest.cpp + * @author: shawnhe + * @date 2022-10-28 + */ + +#include "TaskInfo.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include +#include +#include +#include + +using namespace ppc; +using namespace ppctars; +using namespace ppctars::serialize; +using namespace ppc::protocol; +using namespace ppc::front; + +using namespace bcos; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(TarsProtocolTest, TestPromptFixture) + + +BOOST_AUTO_TEST_CASE(test_labeledPSIDataStructure) +{ + TaskInfo taskInfo; + taskInfo.taskID = "123fdsa456"; + taskInfo.serviceEndpoint = "234f567dfaj"; + + auto encodedData = std::make_shared(); + serialize::encode(taskInfo, *encodedData); + + TaskInfo taskInfoD; + serialize::decode(*encodedData, taskInfoD); + BOOST_CHECK(taskInfoD == taskInfo); + + auto messageFactory = std::make_shared(); + auto queryMessage = messageFactory->buildPPCMessage( + uint8_t(1), uint8_t(2), "345", std::make_shared()); + queryMessage->setMessageType(uint8_t(6)); + ppctars::serialize::encode(taskInfo, *queryMessage->data()); + auto buffer = std::make_shared(); + queryMessage->encode(*buffer); + + auto newMessage = messageFactory->buildPPCMessage(); + newMessage->decode(buffer); + + TaskInfo taskInfoN; + serialize::decode(*newMessage->data(), taskInfoN); + BOOST_CHECK(taskInfoN == taskInfo); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/cpp/ppc-tars-protocol/test/main.cpp b/cpp/ppc-tars-protocol/test/main.cpp new file mode 100644 index 00000000..8034bf67 --- /dev/null +++ b/cpp/ppc-tars-protocol/test/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include diff --git a/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp b/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp new file mode 100644 index 00000000..7ca25f65 --- /dev/null +++ b/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontServiceServer.cpp + * @author: shawnhe + * @date 2022-10-20 + */ + +#include "FrontServiceServer.h" +#include "ppc-framework/Common.h" +#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" + +#define FRONTSERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("FrontServer") + +using namespace ppctars; + +ppctars::Error FrontServiceServer::onReceiveMessage( + const vector& _message, tars::TarsCurrentPtr _current) +{ + FRONTSERVER_LOG(TRACE) << LOG_DESC("receive message from gateway"); + + auto startT = bcos::utcSteadyTime(); + _current->setResponse(false); + + auto ppcMessage = m_frontInitializer->messageFactory()->buildPPCMessage( + bcos::bytesConstRef((bcos::byte const*)_message.data(), _message.size())); + if (!ppcMessage) + { + auto error = ppctars::Error(); + error.errorCode = ppc::protocol::PPCRetCode::DECODE_PPC_MESSAGE_ERROR; + error.errorMessage = "decode ppc message error"; + return error; + } + auto msgSize = _message.size(); + m_frontInitializer->front()->onReceiveMessage( + ppcMessage, [_current, startT, msgSize](bcos::Error::Ptr error) { + BCOS_LOG(TRACE) << LOG_DESC("onReceiveMessage") << LOG_KV("msgSize", msgSize) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + async_response_onReceiveMessage(_current, toTarsError(error)); + }); + + return {}; +} \ No newline at end of file diff --git a/cpp/ppc-tars-service/FrontService/FrontServiceServer.h b/cpp/ppc-tars-service/FrontService/FrontServiceServer.h new file mode 100644 index 00000000..c0065f21 --- /dev/null +++ b/cpp/ppc-tars-service/FrontService/FrontServiceServer.h @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontServiceServer.h + * @author: shawnhe + * @date 2022-10-20 + */ + +#pragma once + +#include "FrontService.h" +#include "libinitializer/FrontInitializer.h" +#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" + +namespace ppctars +{ +struct FrontServiceParam +{ + ppc::initializer::FrontInitializer::Ptr frontInitializer; +}; + +class FrontServiceServer : public FrontService +{ +public: + FrontServiceServer(FrontServiceParam const& _param) + : m_frontInitializer(_param.frontInitializer) + {} + + ~FrontServiceServer() override {} + void initialize() override {} + void destroy() override {} + +public: + ppctars::Error onReceiveMessage( + const vector& _message, tars::TarsCurrentPtr _current) override; + +private: + ppc::initializer::FrontInitializer::Ptr m_frontInitializer; +}; +} // namespace ppctars diff --git a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp new file mode 100644 index 00000000..75ed9814 --- /dev/null +++ b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayInitializer.cpp + * @author: shawnhe + * @date 2022-10-21 + */ + +#include "GatewayInitializer.h" + +using namespace ppctars; +using namespace ppc; +using namespace ppc::front; +using namespace ppc::gateway; +using namespace ppc::storage; +using namespace ppc::protocol; + +using namespace bcos; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; + +void GatewayInitializer::start() +{ + if (m_gateway) + { + m_gateway->start(); + } +} + +void GatewayInitializer::stop() +{ + if (m_gateway) + { + m_gateway->stop(); + } +} + +void GatewayInitializer::init(ppc::protocol::NodeArch _arch) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("init gateway") << LOG_KV("arch", _arch); + m_gateway = m_gatewayFactory->buildGateway( + _arch, m_gatewayConfig, m_cache, m_messageFactory, m_threadPool); + GATEWAY_LOG(INFO) << LOG_BADGE("init gateway success"); +} \ No newline at end of file diff --git a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h new file mode 100644 index 00000000..00285359 --- /dev/null +++ b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayInitializer.h + * @author: shawnhe + * @date 2022-10-21 + */ + +#pragma once + +#include +#include + +namespace ppctars +{ +class GatewayInitializer +{ +public: + using Ptr = std::shared_ptr; + + explicit GatewayInitializer(ppc::protocol::NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, + ppc::storage::CacheStorage::Ptr _cache, ppc::front::PPCMessageFactory::Ptr _messageFactory, + std::shared_ptr _threadPool) + : m_gatewayConfig(std::move(_config)), + m_cache(std::move(_cache)), + m_messageFactory(std::move(_messageFactory)), + m_threadPool(std::move(_threadPool)) + + { + init(_arch); + } + + virtual ~GatewayInitializer() { stop(); } + + void start(); + void stop(); + +public: + ppc::storage::CacheStorage::Ptr cache() { return m_cache; } + + ppc::front::PPCMessageFactory::Ptr messageFactory() { return m_messageFactory; } + std::shared_ptr threadPool() { return m_threadPool; } + + ppc::gateway::GatewayInterface::Ptr gateway() { return m_gateway; } + ppc::gateway::GatewayFactory::Ptr gatewayFactory() { return m_gatewayFactory; } + +protected: + void init(ppc::protocol::NodeArch _arch); + +private: + ppc::tools::PPCConfig::Ptr m_gatewayConfig; + ppc::storage::CacheStorage::Ptr m_cache; + + ppc::front::PPCMessageFactory::Ptr m_messageFactory; + + std::shared_ptr m_threadPool; + + ppc::gateway::GatewayInterface::Ptr m_gateway; + ppc::gateway::GatewayFactory::Ptr m_gatewayFactory; + + std::atomic_bool m_running = {false}; +}; +} // namespace ppctars diff --git a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp new file mode 100644 index 00000000..ed7b217b --- /dev/null +++ b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp @@ -0,0 +1,141 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServiceServer.cpp + * @author: shawnhe + * @date 2022-10-21 + */ +#include "GatewayServiceServer.h" +#include "FrontService.h" +#include "ppc-tars-protocol/client/FrontServiceClient.h" +#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" + +using namespace ppctars; +using namespace ppc::gateway; + +ppctars::Error GatewayServiceServer::asyncSendMessage( + const std::string& _agencyID, const vector& _message, tars::TarsCurrentPtr _current) + +{ + GATEWAYSERVER_LOG(TRACE) << LOG_DESC("receive message from front"); + + _current->setResponse(false); + + auto ppcMessage = m_ppcMsgFactory->buildPPCMessage( + bcos::bytesConstRef((bcos::byte const*)_message.data(), _message.size())); + if (!ppcMessage) + { + auto error = ppctars::Error(); + error.errorCode = ppc::protocol::PPCRetCode::DECODE_PPC_MESSAGE_ERROR; + error.errorMessage = "decode ppc message error"; + async_response_asyncSendMessage(_current, error); + return error; + } + + m_gateway->asyncSendMessage(_agencyID, ppcMessage, [_current](const bcos::Error::Ptr& error) { + async_response_asyncSendMessage(_current, toTarsError(error)); + }); + + return ppctars::Error(); +} + +ppctars::Error GatewayServiceServer::notifyTaskInfo( + const ppctars::TaskInfo& _taskInfo, tars::TarsCurrentPtr _current) +{ + _current->setResponse(true); + + auto error = m_gateway->notifyTaskInfo(toGatewayTaskInfo(_taskInfo)); + + return toTarsError(error); +} + +ppctars::Error GatewayServiceServer::eraseTaskInfo( + const std::string& _taskID, tars::TarsCurrentPtr _current) +{ + _current->setResponse(true); + + auto error = m_gateway->eraseTaskInfo(_taskID); + + return toTarsError(error); +} + +ppctars::Error GatewayServiceServer::registerGateway( + const std::vector& _gatewayList, tars::TarsCurrentPtr _current) +{ + _current->setResponse(true); + + std::vector gatewayList; + for (const auto& tarsGate : _gatewayList) + { + ppc::protocol::GatewayInfo gateway; + gateway.agencyID = tarsGate.agencyID; + gateway.endpoint = tarsGate.endpoint; + gatewayList.push_back(gateway); + } + + auto error = m_gateway->registerGateway(gatewayList); + + return toTarsError(error); +} + +ppctars::Error GatewayServiceServer::asyncRegisterFront( + std::string const& _endPoint, tars::TarsCurrentPtr _current) +{ + _current->setResponse(false); + try + { + // obtain the frontServiceClient specified by _endPoint + auto self = weak_from_this(); + auto prx = createServantProxy(tars::Application::getCommunicator().get(), + _endPoint, [self, _endPoint](const tars::TC_Endpoint& ep) { + auto gatewayServer = self.lock(); + if (!gatewayServer) + { + return; + } + GATEWAYSERVER_LOG(INFO) + << LOG_DESC("UnRegisterFront for disconnect") << LOG_KV("endPoint", _endPoint); + // unregister when disconnected + gatewayServer->m_gateway->unregisterFront(_endPoint); + }); + m_gateway->registerFront(_endPoint, std::make_shared(prx)); + // register success + async_response_asyncRegisterFront( + _current, ppctars::toTarsError(nullptr)); + } + catch (std::exception const& e) + { + GATEWAYSERVER_LOG(WARNING) + << LOG_DESC("asyncRegisterFront exception") << LOG_KV("endPoint", _endPoint) + << LOG_KV("exception", boost::diagnostic_information(e)); + auto bcosError = std::make_shared(-1, + "asyncRegisterFront exception for " + std::string(boost::diagnostic_information(e))); + async_response_asyncRegisterFront(_current, ppctars::toTarsError(bcosError)); + } + return ppctars::Error(); +} + +ppctars::Error GatewayServiceServer::asyncGetAgencyList( + std::vector& _agencyList, tars::TarsCurrentPtr _current) +{ + _current->setResponse(false); + m_gateway->asyncGetAgencyList( + [_current](bcos::Error::Ptr _error, std::vector&& _agencyList) { + // response the agency-list + async_response_asyncGetAgencyList( + _current, ppctars::toTarsError(_error), std::move(_agencyList)); + }); + return ppctars::Error(); +} diff --git a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h new file mode 100644 index 00000000..ecc1e6a4 --- /dev/null +++ b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServiceServer.h + * @author: shawnhe + * @date 2022-10-21 + */ +#pragma once + +#include "GatewayInitializer.h" +#include "GatewayService.h" +#include "ppc-framework/Common.h" +#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-framework/protocol/PPCMessageFace.h" +#include + +#define GATEWAYSERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("GatewayServiceServer") + +namespace ppctars +{ +struct GatewayServiceParam +{ + ppc::gateway::GatewayInterface::Ptr gateway; + ppc::front::PPCMessageFaceFactory::Ptr ppcMsgFactory; +}; + +class GatewayServiceServer : public ppctars::GatewayService, + public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + + explicit GatewayServiceServer(GatewayServiceParam const& _param) + : m_gateway(_param.gateway), m_ppcMsgFactory(_param.ppcMsgFactory) + {} + void initialize() override {} + void destroy() override {} + +public: + ppctars::Error asyncSendMessage(const std::string& _agencyID, + const vector& _message, tars::TarsCurrentPtr _current) override; + + ppctars::Error notifyTaskInfo( + const ppctars::TaskInfo& _taskInfo, tars::TarsCurrentPtr _current) override; + + ppctars::Error eraseTaskInfo( + const std::string& _taskID, tars::TarsCurrentPtr _current) override; + + ppctars::Error registerGateway(const std::vector& _gatewayList, + tars::TarsCurrentPtr _current) override; + + ppctars::Error asyncRegisterFront( + std::string const& _endPoint, tars::TarsCurrentPtr _current) override; + + ppctars::Error asyncGetAgencyList( + std::vector& _agencyList, tars::TarsCurrentPtr _current) override; + +private: + ppc::gateway::GatewayInterface::Ptr m_gateway; + ppc::front::PPCMessageFaceFactory::Ptr m_ppcMsgFactory; +}; +} // namespace ppctars diff --git a/cpp/ppc-tools/CMakeLists.txt b/cpp/ppc-tools/CMakeLists.txt new file mode 100644 index 00000000..aa4ae9b6 --- /dev/null +++ b/cpp/ppc-tools/CMakeLists.txt @@ -0,0 +1,14 @@ +project(ppc-tools VERSION ${VERSION}) + +aux_source_directory(src/cuckoo SRCS) +aux_source_directory(src/config SRCS) + +add_library(${TOOLS_TARGET} ${SRCS}) + +target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB}) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif () diff --git a/cpp/ppc-tools/src/codec/CodecUtility.h b/cpp/ppc-tools/src/codec/CodecUtility.h new file mode 100644 index 00000000..5d37ddf0 --- /dev/null +++ b/cpp/ppc-tools/src/codec/CodecUtility.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CodecUtility.h + * @author: yujiechen + * @date 2023-8-11 + */ +#pragma once +#include "openssl/bn.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include +#include +#include + +#define CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, length) \ + do \ + { \ + if (offset > length) \ + { \ + throw std::out_of_range("Out of range error, offset:" + std::to_string(offset) + \ + " ,length: " + std::to_string(length) + \ + " ,file: " + __FILE__ + " ,func: " + __func__ + \ + " ,line: " + std::to_string(__LINE__)); \ + } \ + } while (0); +namespace ppc +{ +inline bcos::byte* encodeBuffer( + bcos::byte* _pbuffer, bcos::byte* _pend, bcos::bytesConstRef const& _data) +{ + if (_pend - _pbuffer < (int64_t)sizeof(uint16_t)) + { + std::out_of_range("Unenough allocated buffer"); + } + // encode the length of the _data + uint16_t dataNumBigEndianLen = boost::endian::native_to_big((uint16_t)_data.size()); + std::memcpy((void*)_pbuffer, (const void*)&dataNumBigEndianLen, sizeof(uint16_t)); + _pbuffer += sizeof(uint16_t); + if (_pend - _pbuffer <= (int64_t)_data.size()) + { + std::out_of_range("Unenough allocated buffer"); + } + std::memcpy((void*)_pbuffer, _data.data(), _data.size()); + _pbuffer += _data.size(); + return _pbuffer; +} +/// encode bigNum to the given buffer +// Note: _pbuffer will be updated +inline bcos::byte* encodeBigNum( + bcos::byte* _pbuffer, bcos::byte* _pend, ppc::crypto::BigNum const& _value) +{ + // convert the BigNum into bytes + bcos::bytes bigNumData; + _value.toBytes(bigNumData, true); + return encodeBuffer(_pbuffer, _pend, bcos::ref(bigNumData)); +} + +// encode the integer value into buffer +// Note: _pbuffer will be updated +template +inline bcos::byte* encodeInteger(bcos::byte* _pbuffer, bcos::byte* _pend, T const _value) +{ + if (_pend - _pbuffer < (int64_t)sizeof(T)) + { + std::out_of_range("Unenough allocated buffer"); + } + auto nativeValue = boost::endian::native_to_big(_value); + std::memcpy((void*)_pbuffer, (const void*)&nativeValue, sizeof(T)); + _pbuffer += sizeof(T); + return _pbuffer; +} + +// decode the bigNum from _buffer[_offset, _offset + sizeof(T)] +inline uint64_t decodeBuffer(bcos::bytes& _result, bcos::byte const* _buffer, + unsigned int _bufferLen, uint64_t const _offset) +{ + uint64_t offset = _offset; + CHECK_OFFSET_WITH_THROW_EXCEPTION(offset + 2, _bufferLen); + // decode the nativeDataLen + uint16_t nativeDataLen = boost::endian::big_to_native(*((uint16_t*)(_buffer + offset))); + offset += 2; + CHECK_OFFSET_WITH_THROW_EXCEPTION(offset + nativeDataLen, _bufferLen); + // decode the buffer + _result.insert(_result.begin(), (bcos::byte*)_buffer + offset, + (bcos::byte*)_buffer + offset + nativeDataLen); + offset += nativeDataLen; + return offset; +} + +// decode the bigNum from _buffer[_offset, _offset + sizeof(T)] +inline uint64_t decodeBigNum(ppc::crypto::BigNum& _result, bcos::byte const* _buffer, + unsigned int _bufferLen, uint64_t const _offset) +{ + bcos::bytes decodedBuffer; + auto offset = decodeBuffer(decodedBuffer, _buffer, _bufferLen, _offset); + // convert result into BigNum + _result.fromBytes(bcos::ref(decodedBuffer), true); + return offset; +} + +// decode integer from _buffer + _offset, return the new offset +template +inline uint64_t decodeInteger( + T& _result, bcos::byte const* _buffer, unsigned int _bufferLen, uint64_t _offset) +{ + CHECK_OFFSET_WITH_THROW_EXCEPTION(_offset + sizeof(T), _bufferLen); + _result = boost::endian::big_to_native(*((T*)(_buffer + _offset))); + return (_offset + sizeof(T)); +} +} // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-tools/src/common/ConcurrentPool.h b/cpp/ppc-tools/src/common/ConcurrentPool.h new file mode 100644 index 00000000..3b8d2b90 --- /dev/null +++ b/cpp/ppc-tools/src/common/ConcurrentPool.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ConcurrentPool.h + * @author: zachma + * @date 2023-8-30 + */ + +#pragma once +#include +#include +#include + +namespace ppc::tools +{ +template > +class ConcurrentPool +{ +public: + void insert(_K _key, _V _elem) + { + { + std::lock_guard guard{x_mutex}; + m_pool.insert(std::pair<_K, _V>(_key, _elem)); + } + m_cv.notify_one(); + } + + bool empty() + { + boost::unique_lock lock{x_mutex}; + return m_pool.empty(); + } + + _V pop(_K _key) + { + boost::unique_lock lock{x_mutex}; + m_cv.wait(lock, [this, _key] { return exist(_key); }); + auto v = m_pool[_key]; + m_pool.erase(_key); + return v; + } + + std::pair tryPop(_K _key, int milliseconds) + { + boost::unique_lock lock{x_mutex}; + // in consideration that when the system time has been changed, + // the process maybe stucked in 'wait_for' + auto ret = m_cv.wait_for( + lock, boost::chrono::milliseconds(milliseconds), [this, _key] { return exist(_key); }); + if (!ret) + { + return std::make_pair(false, _V()); + } + auto item = m_pool[_key]; + m_pool.erase(_key); + return std::make_pair(ret, item); + } + + bool exist(_K _key) { return m_pool.find(_key) != m_pool.end(); } + +private: + PoolKV m_pool; + boost::mutex x_mutex; + boost::condition_variable m_cv; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/src/common/MemInfo.h b/cpp/ppc-tools/src/common/MemInfo.h new file mode 100644 index 00000000..9c3e5108 --- /dev/null +++ b/cpp/ppc-tools/src/common/MemInfo.h @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2024 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MemInfo.h + * @author: shawnhe + * @date 2024-03-04 + */ + +#pragma once + +#ifdef __linux__ +#include +#include +#include +#include + +namespace ppc::tools +{ +bool hasAvailableMem(uint32_t _minNeedMemoryGB) +{ + std::ifstream meminfo("/proc/meminfo"); + if (!meminfo.is_open()) + { + std::cerr << "Failed to open /proc/meminfo!" << std::endl; + return -1; + } + + std::string line; + long long availableMem = 0; + + while (std::getline(meminfo, line)) + { + if (line.find("MemAvailable:") != std::string::npos) + { + sscanf(line.c_str(), "MemAvailable: %lld kB", &availableMem); + break; + } + } + + meminfo.close(); + + auto availableMemGB = availableMem / (1024 * 1024); // Convert to GB + return availableMemGB >= _minNeedMemoryGB; +} +} // namespace ppc::tools +#endif diff --git a/cpp/ppc-tools/src/common/Progress.h b/cpp/ppc-tools/src/common/Progress.h new file mode 100644 index 00000000..37a12ca1 --- /dev/null +++ b/cpp/ppc-tools/src/common/Progress.h @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Progress.h + * @author: shawnhe + * @date 2023-03-10 + */ + +#pragma once + +#include +#include +#include +#include + +#define TOOLS_PROGRESS_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PPCTools") + +namespace ppc::tools +{ +using FlagType = std::variant; + + +class Progress +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + Progress(bcos::ThreadPool::Ptr _threadPool) : m_threadPool(std::move(_threadPool)){}; + ~Progress() + { + std::unordered_set().swap(m_flags); + MallocExtension::instance()->ReleaseFreeMemory(); + } + + void reset(int64_t _target, std::function&& _finalizeHandler) + { + TOOLS_PROGRESS_LOG(INFO) << LOG_BADGE("reset") << LOG_KV("target", _target); + bcos::WriteGuard l(x_flags); + m_flags.clear(); + m_target = _target; + m_finalizeHandler = std::move(_finalizeHandler); + } + + template + int64_t mark(T&& _flag) + { + TOOLS_PROGRESS_LOG(TRACE) << LOG_BADGE("mark") << LOG_KV("flag", _flag); + int64_t fSize; + std::function callback = nullptr; + { + bcos::WriteGuard l(x_flags); + m_flags.insert(std::forward(_flag)); + fSize = (int64_t)m_flags.size(); + if (fSize == m_target) + { + callback = std::move(m_finalizeHandler); + } + } + + if (callback) + { + TOOLS_PROGRESS_LOG(INFO) + << LOG_BADGE("handle finalizeHandler") << LOG_KV("target", m_target); + + if (m_threadPool) + { + m_threadPool->enqueue([callback] { callback(); }); + } + else + { + callback(); + } + } + + return fSize; + } + + int64_t current() + { + bcos::ReadGuard l(x_flags); + return (int64_t)m_flags.size(); + } + + +private: + bcos::ThreadPool::Ptr m_threadPool; + mutable bcos::SharedMutex x_flags; + std::unordered_set m_flags; + int64_t m_target{-1}; + std::function m_finalizeHandler; +}; + +} // namespace ppc::tools diff --git a/cpp/ppc-tools/src/common/TransTools.h b/cpp/ppc-tools/src/common/TransTools.h new file mode 100644 index 00000000..75a72f5e --- /dev/null +++ b/cpp/ppc-tools/src/common/TransTools.h @@ -0,0 +1,191 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TransTools.h + * @author: shawnhe + * @date 2022-11-07 + */ + +#pragma once + +#include +#include + +namespace ppc::tools +{ +template +inline void encodeUnsignedNum(bcos::bytesPointer _buffer, UnsignedT _number) +{ + UnsignedT temp; + if (sizeof(_number) == 2) + { + temp = boost::asio::detail::socket_ops::host_to_network_short(_number); + } + else if (sizeof(_number) == 4) + { + temp = boost::asio::detail::socket_ops::host_to_network_long(_number); + } + else + { + temp = _number; + } + + _buffer->clear(); + _buffer->assign((bcos::byte*)&temp, (bcos::byte*)&temp + sizeof(_number)); +} + +template +inline void encodeUnsignedNum(bcos::byte* _buffer, UnsignedT _number) +{ + UnsignedT temp; + if (sizeof(_number) == 2) + { + temp = boost::asio::detail::socket_ops::host_to_network_short(_number); + } + else if (sizeof(_number) == 4) + { + temp = boost::asio::detail::socket_ops::host_to_network_long(_number); + } + else + { + temp = _number; + } + + memcpy(_buffer, (bcos::byte*)&temp, sizeof(_number)); +} + +template +inline void encodeUnsignedNum(std::string& _res, UnsignedT _number) +{ + _res = ""; + UnsignedT temp; + auto _buffer = std::make_shared(); + if (sizeof(_number) == 2) + { + temp = boost::asio::detail::socket_ops::host_to_network_short(_number); + } + else if (sizeof(_number) == 4) + { + temp = boost::asio::detail::socket_ops::host_to_network_long(_number); + } + else + { + temp = _number; + } + + _buffer->clear(); + _buffer->assign((bcos::byte*)&temp, (bcos::byte*)&temp + sizeof(_number)); + _res.assign(_buffer->begin(), _buffer->end()); +} + +template +inline void decodeUnsignedNum(UnsignedT& _number, const std::string& _restr) +{ + auto _buffer = std::make_shared(bcos::bytes(_restr.begin(), _restr.end())); + if (sizeof(_number) == 1) + { + _number = *((uint8_t*)_buffer->data()); + } + else if (sizeof(_number) == 2) + { + _number = + boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)_buffer->data())); + } + else if (sizeof(_number) == 4) + { + _number = + boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)_buffer->data())); + } + else + { + _number = UnsignedT(*((UnsignedT*)_buffer->data())); + } +} + +template +inline void decodeUnsignedNum(UnsignedT& _number, bcos::bytesPointer _buffer) +{ + if (sizeof(_number) == 1) + { + _number = *((uint8_t*)_buffer->data()); + } + else if (sizeof(_number) == 2) + { + _number = + boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)_buffer->data())); + } + else if (sizeof(_number) == 4) + { + _number = + boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)_buffer->data())); + } + else + { + _number = UnsignedT(*((UnsignedT*)_buffer->data())); + } +} + +template +inline void decodeUnsignedNum(UnsignedT& _number, bcos::byte* _buffer) +{ + if (sizeof(_number) == 1) + { + _number = *((uint8_t*)_buffer); + } + else if (sizeof(_number) == 2) + { + _number = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)_buffer)); + } + else if (sizeof(_number) == 4) + { + _number = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)_buffer)); + } + else + { + _number = UnsignedT(*((UnsignedT*)_buffer)); + } +} + +template +T toBigEndian(T value) +{ + T result = 0; + uint8_t* resultBytes = reinterpret_cast(&result); + uint8_t* valueBytes = reinterpret_cast(&value); + + for (size_t i = 0; i < sizeof(T); ++i) + { + resultBytes[i] = valueBytes[sizeof(T) - 1 - i]; + } + + return result; +} + +template +T fromBigEndian(T value) +{ + T result = 0; + uint8_t* resultBytes = reinterpret_cast(&result); + uint8_t* valueBytes = reinterpret_cast(&value); + + for (size_t i = 0; i < sizeof(T); ++i) + { + resultBytes[i] = valueBytes[sizeof(T) - 1 - i]; + } + + return result; +} + +} // namespace ppc::tools diff --git a/cpp/ppc-tools/src/config/Common.h b/cpp/ppc-tools/src/config/Common.h new file mode 100644 index 00000000..3a242820 --- /dev/null +++ b/cpp/ppc-tools/src/config/Common.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +#define MAXPORT 65535 +#define PPCConfig_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PPCConfig") + +namespace ppc::tools +{ +DERIVE_PPC_EXCEPTION(InvalidConfig); +} \ No newline at end of file diff --git a/cpp/ppc-tools/src/config/NetworkConfig.h b/cpp/ppc-tools/src/config/NetworkConfig.h new file mode 100644 index 00000000..a070cc12 --- /dev/null +++ b/cpp/ppc-tools/src/config/NetworkConfig.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file NetworkConfig.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once + +namespace ppc::tools +{ +struct NetworkConfig +{ + constexpr static int DefaultRpcListenPort = 10200; + + // the configuration for rpc + std::string listenIp; + int listenPort; + uint32_t threadPoolSize; + std::string token; + bool disableSsl; + uint32_t minNeededMemoryGB; + + bool enableSM; + // the gateway protocol + int protocol; // websocket or http + std::string url; + // the rpc cert path + std::string certPath; + // the cert info for non-sm + std::string caCertPath; + std::string sslKeyPath; + std::string sslCertPath; + // the cert info for sm + std::string smCaCertPath; + std::string smSslKeyPath; + std::string smSslCertPath; + std::string smEnSslKeyPath; + std::string smEnSslCertPath; + + constexpr static std::string_view CA_CERT_NAME = "ca.crt"; + constexpr static std::string_view SM_CA_CERT_NAME = "sm_ca.crt"; + + constexpr static std::string_view SSL_CERT_NAME = "ssl.crt"; + constexpr static std::string_view SM_SSL_CERT_NAME = "sm_ssl.crt"; + + constexpr static std::string_view SSL_KEY_NAME = "ssl.key"; + constexpr static std::string_view SM_SSL_KEY_NAME = "sm_ssl.key"; + + constexpr static std::string_view SM_SSL_EN_CERT_NAME = "sm_enssl.crt"; + constexpr static std::string_view SM_SSL_EN_KEY_NAME = "sm_enssl.key"; + + constexpr static int PROTOCOL_WEBSOCKET = 0; + constexpr static int PROTOCOL_HTTP = 1; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/src/config/PPCConfig.cpp b/cpp/ppc-tools/src/config/PPCConfig.cpp new file mode 100644 index 00000000..07a773c6 --- /dev/null +++ b/cpp/ppc-tools/src/config/PPCConfig.cpp @@ -0,0 +1,664 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCConfig.cpp + * @author: yujiechen + * @date 2022-11-4 + */ +#include "PPCConfig.h" +#include +#include +#include +#include +#include + +using namespace ppc::tools; +using namespace ppc::protocol; +using namespace ppc::storage; +using namespace bcos; + +void PPCConfig::loadGatewayConfig( + NodeArch _arch, const char* _certPath, boost::property_tree::ptree const& _pt) +{ + // load the network config + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config"); + // gateway default enable-ssl + loadNetworkConfig(m_gatewayConfig.networkConfig, _certPath, _pt, "gateway", + NetworkConfig::DefaultRpcListenPort, false); + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config success"); + + m_gatewayConfig.disableCache = _pt.get("gateway.disable_cache", false); + + // load the redis config + if (_arch == NodeArch::PRO && !m_gatewayConfig.disableCache) + { + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the redis config"); + initRedisConfigForGateway(m_gatewayConfig.cacheStorageConfig, _pt); + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the redis config success"); + } + + // load the agencies config + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the agency config"); + auto agencyInfo = parseAgencyConfig(_pt, "agency", "agency"); + m_gatewayConfig.agencies = std::move(agencyInfo); + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the agency config sucess") + << LOG_KV("agencySize", m_gatewayConfig.agencies.size()); + + m_gatewayConfig.reconnectTime = _pt.get("gateway.reconnect_time", 10000); + // load the maxAllowedMsgSize, in MBytes + m_gatewayConfig.maxAllowedMsgSize = + _pt.get("gateway.max_allow_msg_size", GatewayConfig::DefaultMaxAllowedMsgSize) * + 1024 * 1024; + if (m_gatewayConfig.maxAllowedMsgSize < GatewayConfig::MinMsgSize || + m_gatewayConfig.maxAllowedMsgSize >= GatewayConfig::MaxMsgSize) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment( + "the gateway.max_allow_msg_size must be smaller than " + + std::to_string(GatewayConfig::MinMsgSize) + "Bytes and no larger than" + + std::to_string(GatewayConfig::MaxMsgSize) + "Bytes, recommend to 100MBytes")); + } + // load the holdingMessageMinutes, in minutes + m_holdingMessageMinutes = loadHoldingMessageMinutes(_pt, "gateway.holding_msg_minutes"); + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") + << LOG_KV("maxAllowedMsgSize", m_gatewayConfig.maxAllowedMsgSize) + << LOG_KV("reconnectTime", m_gatewayConfig.reconnectTime) + << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); +} + + +int PPCConfig::loadHoldingMessageMinutes( + const boost::property_tree::ptree& _pt, std::string const& _section) +{ + auto holdingMessageMinutes = _pt.get(_section, 30); + if (holdingMessageMinutes < 0) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("The value of " + _section + " must positive")); + } + return holdingMessageMinutes; +} + +void PPCConfig::initRedisConfigForGateway( + CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt) +{ + _redisConfig.type = ppc::protocol::CacheType(_pt.get("cache.type", 0)); + if (_redisConfig.type > ppc::protocol::CacheType::Redis) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << bcos::errinfo_comment("unsupported cache type")); + } + + _redisConfig.proxy = _pt.get("cache.proxy", ""); + _redisConfig.obServer = _pt.get("cache.obServer", ""); + _redisConfig.cluster = _pt.get("cache.cluster", ""); + _redisConfig.user = _pt.get("cache.user", ""); + _redisConfig.host = _pt.get("cache.host", "127.0.0.1"); + _redisConfig.port = _pt.get("cache.port", 6379); + _redisConfig.password = _pt.get("cache.password", ""); + _redisConfig.database = _pt.get("cache.database", 1); + _redisConfig.pool = _pt.get("cache.pool_size", 16); + + // the connection_timeout + _redisConfig.connectionTimeout = _pt.get("cache.connection_timeout", 500); + // the socket_timeout + _redisConfig.socketTimeout = _pt.get("cache.socket_timeout", 500); + PPCConfig_LOG(INFO) << LOG_DESC("initRedisConfig") << LOG_KV("type", (int)_redisConfig.type) + << LOG_KV("proxy", _redisConfig.proxy) + << LOG_KV("obServer", _redisConfig.obServer) + << LOG_KV("cluster", _redisConfig.cluster) + << LOG_KV("user", _redisConfig.user) << LOG_KV("host", _redisConfig.host) + << LOG_KV("port", _redisConfig.port) + << LOG_KV("password", _redisConfig.password) + << LOG_KV("database", _redisConfig.database) + << LOG_KV("pool", _redisConfig.pool) + << LOG_KV("connectionTimeout", _redisConfig.connectionTimeout) + << LOG_KV("socketTimeout", _redisConfig.socketTimeout); +} + +std::map> PPCConfig::parseAgencyConfig( + const boost::property_tree::ptree& _pt, std::string const& _sectionName, + std::string const& _subSectionName) +{ + PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") << LOG_KV("section", _sectionName) + << LOG_KV("subSection", _subSectionName); + std::map> agencyInfo; + // without the config + if (!_pt.get_child_optional(_sectionName)) + { + PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig return for empty config") + << LOG_KV("section", _sectionName) + << LOG_KV("subSection", _subSectionName); + return agencyInfo; + } + // tranverse the child_section to parse the agencyInfo + for (auto const& it : _pt.get_child(_sectionName)) + { + if (it.first.find(_subSectionName) != 0) + { + continue; + } + // find and parse the agencyInfo + auto key = it.first.data(); + std::vector agencyIDInfo; + boost::split(agencyIDInfo, key, boost::is_any_of(".")); + // invalid agencyID + if (agencyIDInfo.size() < 2) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << bcos::errinfo_comment("Invalid agency key " + it.first + + ", the key must be in format of " + + _sectionName + ".${agencyID}")); + } + std::vector endPointInfos; + auto value = it.second.data(); + boost::split(endPointInfos, value, boost::is_any_of(",")); + for (auto& endpoint : endPointInfos) + { + if (!checkEndpoint(endpoint)) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << bcos::errinfo_comment("Invalid agency endpoint" + endpoint)); + } + } + auto& currentEndPoints = agencyInfo[agencyIDInfo.at(1)]; + currentEndPoints.reserve(currentEndPoints.size() + endPointInfos.size()); + std::move(std::begin(endPointInfos), std::end(endPointInfos), + std::back_inserter(currentEndPoints)); + PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") + << LOG_KV("agencyID", agencyIDInfo.at(1)) + << LOG_KV("endPointSize", currentEndPoints.size()); + } + PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") << LOG_KV("agencySize", agencyInfo.size()); + return agencyInfo; +} + +void PPCConfig::loadNetworkConfig(NetworkConfig& _config, const char* _certPath, + boost::property_tree::ptree const& _pt, std::string const& _sectionName, int _defaultListenPort, + bool _defaultDisableSSl) +{ + // the rpcListenIp + _config.listenIp = _pt.get(_sectionName + ".listen_ip", "0.0.0.0"); + // the rpcListenPort + _config.listenPort = _pt.get(_sectionName + ".listen_port", _defaultListenPort); + checkPort("rpc.listen_port", _config.listenPort); + // the rpcThreadPoolSize + _config.threadPoolSize = _pt.get(_sectionName + ".thread_count", + static_cast(std::thread::hardware_concurrency() * 0.75)); + _config.token = _pt.get(_sectionName + ".token", "ppcs_psi_apikey"); + // disable rpc-ssl or not + _config.disableSsl = _pt.get(_sectionName + ".disable_ssl", _defaultDisableSSl); + _config.protocol = _pt.get(_sectionName + ".protocol", NetworkConfig::PROTOCOL_WEBSOCKET); + if (_config.protocol == NetworkConfig::PROTOCOL_HTTP) + { + _config.url = _pt.get(_sectionName + ".url", "/api/v1/interconn/invoke"); + } + _config.minNeededMemoryGB = _pt.get(_sectionName + ".min_needed_memory", 5); + PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_KV("section", _sectionName) + << LOG_KV("listenIp", _config.listenIp) + << LOG_KV("listenPort", _config.listenPort) + << LOG_KV("threadCount", _config.threadPoolSize) + << LOG_KV("disableSsl", _config.disableSsl) + << LOG_KV("protocol", _config.protocol) << LOG_KV("url", _config.url) + << LOG_KV("minNeededMemory", _config.minNeededMemoryGB); + // no need to load the certificate when disable-ssl + if (_config.disableSsl) + { + return; + } + // enable sm-rpc or not + _config.enableSM = _pt.get(_sectionName + ".sm_ssl", false); + + // the rpc cert-path + if (_certPath == nullptr) + { + _config.certPath = _pt.get("cert.cert_path", "conf"); + } + else + { + _config.certPath = _certPath; + } + PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_KV("section", _sectionName) + << LOG_KV("certPath", _config.certPath); + + checkFileExists(_config.certPath, true); + // load the sm-cert + if (_config.enableSM) + { + _config.smCaCertPath = _config.certPath + "/" + std::string(NetworkConfig::SM_CA_CERT_NAME); + checkFileExists(_config.smCaCertPath, false); + + _config.smSslCertPath = + _config.certPath + "/" + std::string(NetworkConfig::SM_SSL_CERT_NAME); + checkFileExists(_config.smSslCertPath, false); + + _config.smEnSslCertPath = + _config.certPath + "/" + std::string(NetworkConfig::SM_SSL_EN_CERT_NAME); + checkFileExists(_config.smEnSslCertPath, false); + + _config.smSslKeyPath = _config.certPath + "/" + std::string(NetworkConfig::SM_SSL_KEY_NAME); + checkFileExists(_config.smSslKeyPath, false); + + _config.smEnSslKeyPath = + _config.certPath + "/" + std::string(NetworkConfig::SM_SSL_EN_KEY_NAME); + checkFileExists(_config.smEnSslKeyPath, false); + PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_DESC("load sm cert") + << LOG_KV("section", _sectionName) + << LOG_KV("caCert", _config.smCaCertPath) + << LOG_KV("sslCert", _config.smSslCertPath) + << LOG_KV("enSslCert", _config.smEnSslCertPath); + return; + } + // load the non-sm-cert + _config.caCertPath = _config.certPath + "/" + std::string(NetworkConfig::CA_CERT_NAME); + checkFileExists(_config.caCertPath, false); + + _config.sslCertPath = _config.certPath + "/" + std::string(NetworkConfig::SSL_CERT_NAME); + checkFileExists(_config.sslCertPath, false); + + _config.sslKeyPath = _config.certPath + "/" + std::string(NetworkConfig::SSL_KEY_NAME); + checkFileExists(_config.sslKeyPath, false); + + PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_DESC("load non-sm cert") + << LOG_KV("section", _sectionName) << LOG_KV("caCert", _config.caCertPath) + << LOG_KV("sslCert", _config.sslCertPath); +} + +void PPCConfig::checkPort(std::string const& _sectionName, int _port) +{ + if (_port <= 0 || _port > MAXPORT) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "The " + _sectionName + " must positive and no larger than" + + std::to_string(MAXPORT))); + } +} + +void PPCConfig::checkFileExists(std::string const& _filePath, bool _dir) +{ + // check for the dir + if (_dir) + { + if (!boost::filesystem::is_directory(boost::filesystem::path(_filePath))) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "Please check the existence of directory " + _filePath)); + } + return; + } + // check for the file + if (!boost::filesystem::exists(boost::filesystem::path(_filePath))) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Please check the existence of file " + _filePath)); + } +} + +void PPCConfig::loadRA2018Config(boost::property_tree::ptree const& _pt) +{ + m_disableRA2018 = _pt.get("agency.disable_ra2018", false); + if (m_disableRA2018) + { + PPCConfig_LOG(INFO) << LOG_DESC("The ra2018-psi has been disabled!"); + return; + } + // load the database + m_ra2018PSIConfig.dbName = _pt.get("ra2018psi.database", "ra2018"); + ///// load the cuckoo-filter-option + auto cuckooFilterOption = std::make_shared(); + // capacity, unit with MBytes + auto capacity = _pt.get( + "ra2018psi.cuckoofilter_capacity", RA2018Config::DefaultCuckooFilterCapacity); + if (capacity == 0) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "the ra2018psi.cuckoofilter_capacity must be larger than 0")); + } + cuckooFilterOption->capacity = capacity * 1024 * 1024; + // tagBits + cuckooFilterOption->tagBits = _pt.get( + "ra2018psi.cuckoofilter_tagBits", RA2018Config::DefaultCuckooFilterTagSize); + if (cuckooFilterOption->tagBits == 0 || cuckooFilterOption->tagBits % 8 != 0 || + cuckooFilterOption->tagBits > 64) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "the ra2018psi.cuckoofilter_tagBits must be 8/16/32/64")); + } + // bucket-size + cuckooFilterOption->tagNumPerBucket = + _pt.get("ra2018psi.cuckoofilter_buckets_num", RA2018Config::DefaultBucketSize); + if (cuckooFilterOption->tagNumPerBucket == 0) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment( + "the ra2018psi.cuckoofilter_buckets_num must be larger than 0, recommend to " + + std::to_string(RA2018Config::DefaultBucketSize))); + } + // maxKickOutCount + cuckooFilterOption->maxKickOutCount = _pt.get( + "ra2018psi.cuckoofilter_max_kick_out_count", RA2018Config::DefaultMaxKickoutCount); + if (cuckooFilterOption->maxKickOutCount == 0) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "the ra2018psi.cuckoofilter_max_kick_out_count must be larger " + "than 0, recommend to " + + std::to_string(RA2018Config::DefaultMaxKickoutCount))); + } + cuckooFilterOption->trashBucketSize = + _pt.get("ra2018psi.trash_bucket_size", RA2018Config::DefaultTrashBucketSize); + if (cuckooFilterOption->trashBucketSize == 0) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment( + "the ra2018psi.trash_bucket_size must be larger than 0, recommend to " + + boost::lexical_cast(RA2018Config::DefaultTrashBucketSize))); + } + m_ra2018PSIConfig.cuckooFilterOption = std::move(cuckooFilterOption); + ///// load the cuckoo-filter-cache-size + auto cacheSize = _pt.get( + "ra2018psi.cuckoofilter_cache_size", RA2018Config::DefaultCuckooFilterCacheSize); + if (cacheSize == 0) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "the ra2018psi.cuckoofilter_cache_size must be larger than 0" + + std::to_string(RA2018Config::DefaultCuckooFilterCacheSize))); + } + m_ra2018PSIConfig.cuckooFilterCacheSize = cacheSize * 1024 * 1024; + + ///// load the psi-cache-size + cacheSize = _pt.get("ra2018psi.psi_cache_size", RA2018Config::DefaultCacheSize); + if (cacheSize == 0) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment( + "the ra2018psi.psi_cache_size must be larger than 0, recommend to " + + std::to_string(RA2018Config::DefaultCacheSize))); + } + m_ra2018PSIConfig.cacheSize = cacheSize * 1024 * 1024; + ///// load the psi-data-batch-size + auto dataBatchSize = _pt.get("ra2018psi.data_batch_size", DefaultDataBatchSize); + m_ra2018PSIConfig.dataBatchSize = getDataBatchSize("ra2018psi.data_batch_size", dataBatchSize); + + m_ra2018PSIConfig.useHDFS = _pt.get("ra2018psi.use_hdfs", false); + // Note: the cuckooFilterOption has been moved, should not be accessed here + PPCConfig_LOG(INFO) << LOG_DESC("loadRA2018Config") + << LOG_KV("database", m_ra2018PSIConfig.dbName) + << printCuckooFilterOption(m_ra2018PSIConfig.cuckooFilterOption) + << LOG_KV("cuckooFilterCacheSize", m_ra2018PSIConfig.cuckooFilterCacheSize) + << LOG_KV("psiCacheSize", m_ra2018PSIConfig.cacheSize) + << LOG_KV("dataBatchSize", m_ra2018PSIConfig.dataBatchSize) + << LOG_KV("useHDFS", m_ra2018PSIConfig.useHDFS); +} + +// load the ecdh-psi config +void PPCConfig::loadEcdhPSIConfig(boost::property_tree::ptree const& _pt) +{ + auto dataBatchSize = _pt.get("ecdh-psi.data_batch_size", DefaultDataBatchSize); + m_ecdhPSIConfig.dataBatchSize = getDataBatchSize("ecdh-psi.data_batch_size", dataBatchSize); + PPCConfig_LOG(INFO) << LOG_DESC("loadEcdhPSIConfig") + << LOG_KV("dataBatchSize", m_ecdhPSIConfig.dataBatchSize); +} + +void PPCConfig::loadCM2020PSIConfig(boost::property_tree::ptree const& _pt) +{ + m_cm2020Config.parallelism = _pt.get("cm2020-psi.parallelism", DefaultParallelism); + PPCConfig_LOG(INFO) << LOG_DESC("loadCM2020PSIConfig") + << LOG_KV("parallelism", m_cm2020Config.parallelism); +} + +void PPCConfig::loadEcdhMultiPSIConfig(boost::property_tree::ptree const& _pt) +{ + auto dataBatchSize = _pt.get("ecdh-multi-psi.data_batch_size", DefaultDataBatchSize); + m_ecdhMultiPSIConfig.dataBatchSize = + getDataBatchSize("ecdh-multi-psi.data_batch_size", dataBatchSize); + PPCConfig_LOG(INFO) << LOG_DESC("loadEcdhMultiPSIConfig") + << LOG_KV("dataBatchSize", m_ecdhMultiPSIConfig.dataBatchSize); +} + +void PPCConfig::loadEcdhConnPSIConfig(boost::property_tree::ptree const& _pt) +{ + m_ecdhConnPSIConfig.rank = _pt.get("ecdh-conn-psi.rank", 0); + m_ecdhConnPSIConfig.algos = _pt.get("ecdh-conn-psi.algos", 1); + m_ecdhConnPSIConfig.protocol_families = _pt.get("ecdh-conn-psi.protocol_families", 1); + m_ecdhConnPSIConfig.curve = _pt.get("ecdh-conn-psi.curve", 2); + m_ecdhConnPSIConfig.hashtype = _pt.get("ecdh-conn-psi.hashtype", 11); + m_ecdhConnPSIConfig.hash2curve = _pt.get("ecdh-conn-psi.hash2curve", 2); + auto dataBatchSize = _pt.get("ecdh-conn-psi.data_batch_size", DefaultDataBatchSize); + m_ecdhConnPSIConfig.dataBatchSize = + getDataBatchSize("ecdh-conn-psi.data_batch_size", dataBatchSize); + PPCConfig_LOG(INFO) << LOG_DESC("loadEcdhConnPSIConfig Success"); +} + +int64_t PPCConfig::getDataBatchSize(std::string const& _section, int64_t _dataBatchSize) +{ + int64_t dataBatchSize = _dataBatchSize; + if (dataBatchSize != -1 && dataBatchSize <= 0) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("the " + _section + + " must be larger than 0 or equal to -1(load all " + "data into memory at-once) recommend to " + + std::to_string(DefaultDataBatchSize))); + } + // in-case of the data over the max-allowed-message size + int64_t maxNetworkItems = (GatewayConfig::MaxMsgSize - 1024 * 1024) / 32; + if (dataBatchSize == -1 || dataBatchSize > maxNetworkItems) + { + dataBatchSize = std::min((uint64_t)maxNetworkItems, (uint64_t)INT_MAX); + } + PPCConfig_LOG(INFO) << LOG_DESC("getDataBatchSize") << LOG_KV("section", _section) + << LOG_KV("maxNetworkItems", maxNetworkItems) + << LOG_KV("dataBatchSize", dataBatchSize); + return dataBatchSize; +} + +void PPCConfig::loadCommonConfig(boost::property_tree::ptree const& _pt) +{ + m_agencyID = _pt.get("agency.id", ""); + if (m_agencyID.empty()) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment("Must set agency.id")); + } + if (m_privateKeyPath.empty()) + { + m_privateKeyPath = + _pt.get("agency.private_key_path", "conf/" + std::string(NODE_PEM_NAME)); + checkFileExists(m_privateKeyPath, false); + } + // load the crypto config + m_smCrypto = _pt.get("crypto.sm_crypto", false); + + m_dataLocation = _pt.get("agency.data_location", "data"); + + m_taskTimeoutMinutes = _pt.get("agency.task_timeout_minutes", 180); + + m_threadPoolSize = _pt.get( + "agency.thread_count", static_cast(std::thread::hardware_concurrency() * 0.75)); + + PPCConfig_LOG(INFO) << LOG_DESC("loadCommonConfig success") << LOG_KV("agencyID", m_agencyID) + << LOG_KV("dataLocation", m_dataLocation) << LOG_KV("smCrypto", m_smCrypto) + << LOG_KV("taskTimeoutMinutes", m_taskTimeoutMinutes) + << LOG_KV("threadPoolSize", m_threadPoolSize); +} + +void PPCConfig::loadHDFSConfig(boost::property_tree::ptree const& _pt) +{ + SQLConnectionOption opt; + opt.host = _pt.get("storage.host", "127.0.0.1"); + // checkNonEmptyField("storage.host", opt.host); + + opt.port = _pt.get("storage.port", 3306); + + opt.user = _pt.get("storage.user", "ppcs"); + // checkNonEmptyField("storage.user", opt.user); + + opt.password = _pt.get("storage.password", "ppcs"); + // checkNonEmptyField("storage.password", opt.password); + + opt.database = _pt.get("storage.database", ""); + // checkNonEmptyField("storage.database", opt.database); + + m_storageConfig.sqlConnectionOpt = std::make_shared(opt); + PPCConfig_LOG(INFO) << LOG_DESC("loadStorageConfig: load sql connection option success") + << m_storageConfig.sqlConnectionOpt->desc(); + + auto option = std::make_shared(); + // the name node + option->nameNode = _pt.get("hdfs_storage.name_node", ""); + // checkNonEmptyField("hdfs_storage.name_node", option->nameNode); + // the name node port + option->nameNodePort = _pt.get("hdfs_storage.name_node_port", 8020); + // the user + option->userName = _pt.get("hdfs_storage.user", "root"); + // checkNonEmptyField("hdfs_storage.user", option->userName); + // the token + option->token = _pt.get("hdfs_storage.token", ""); + // replace-datanode-on-failure + option->replaceDataNodeOnFailure = + _pt.get("hdfs_storage.replace-datanode-on-failure", false); + // connection-timeout + option->connectionTimeout = _pt.get("hdfs_storage.connection-timeout", 1000); + m_storageConfig.fileStorageConnectionOpt = option; + PPCConfig_LOG(INFO) << LOG_DESC("loadStorageConfig: load hdfs connection option success") + << option->desc(); +} + +void PPCConfig::loadSQLConfig(boost::property_tree::ptree const& _pt) +{ + SQLConnectionOption opt; + opt.host = _pt.get("storage.host", "127.0.0.1"); + // checkNonEmptyField("storage.host", opt.host); + + opt.port = _pt.get("storage.port", 3306); + + opt.user = _pt.get("storage.user", "wedpr"); + // checkNonEmptyField("storage.user", opt.user); + + opt.password = _pt.get("storage.password", ""); + // checkNonEmptyField("storage.password", opt.password); + + opt.database = _pt.get("storage.database", ""); + // checkNonEmptyField("storage.database", opt.database); + + m_storageConfig.sqlConnectionOpt = std::make_shared(opt); + PPCConfig_LOG(INFO) << LOG_DESC("loadStorageConfig: load sql connection option success") + << m_storageConfig.sqlConnectionOpt->desc(); +} + + +void PPCConfig::loadStorageConfig(boost::property_tree::ptree const& _pt) +{ + loadSQLConfig(_pt); + loadHDFSConfig(_pt); +} + +void PPCConfig::loadCEMConfig(boost::property_tree::ptree const& _pt) +{ + m_cemConfig.datasetFilePath = _pt.get("cem.dataset_file_path", "./"); + m_cemConfig.datasetHDFSPath = _pt.get("cem.dataset_hdfs_path", "./"); + m_cemConfig.ciphertextSuffix = _pt.get("cem.ciphertext_suffix", "-encrypted"); + m_cemConfig.readPerBatchLines = _pt.get("cem.read_per_batch_lines", 100000); + loadHDFSConfig(_pt); +} + +void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt) +{ + m_mpcConfig.datasetHDFSPath = _pt.get("mpc.dataset_hdfs_path", "./"); + m_mpcConfig.jobPath = _pt.get("mpc.job_path", "./"); + m_mpcConfig.mpcRootPath = _pt.get("mpc.mpc_root_path", "./"); + m_mpcConfig.mpcRootPathNoGateway = _pt.get("mpc.mpc_root_path_no_gateway", "./"); + m_mpcConfig.readPerBatchLines = _pt.get("mpc.read_per_batch_lines", 100000); + loadHDFSConfig(_pt); +} + +//////// the tars config +void PPCConfig::loadTarsConfig(boost::property_tree::ptree const& _pt) +{ + PPCConfig_LOG(INFO) << LOG_DESC("loadTarsConfig"); + // load the service name + std::string gatewayServiceConfigSection = "tars_gateway"; + m_gatewayServiceName = + getServiceName(_pt, gatewayServiceConfigSection + ".name", GATEWAY_SERVANT_NAME); + // load the holding_msg_minutes + m_holdingMessageMinutes = + loadHoldingMessageMinutes(_pt, gatewayServiceConfigSection + ".holding_msg_minutes"); + // config the gateway information + loadServiceTarsConfig(_pt, m_gatewayServiceName, gatewayServiceConfigSection); + + PPCConfig_LOG(INFO) << LOG_DESC("loadTarsConfig success") + << LOG_KV("gatewayService", m_gatewayServiceName) + << LOG_KV("endPoints", m_serviceToEndPoints.at(m_gatewayServiceName).size()) + << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); +} + +void PPCConfig::loadServiceTarsConfig(boost::property_tree::ptree const& _pt, + std::string const& _serviceName, std::string const& _sectionName) +{ + PPCConfig_LOG(INFO) << LOG_DESC("loadServiceTarsConfig for ") << _sectionName + << LOG_KV("service", _serviceName); + if (!_pt.get_child_optional(_sectionName)) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment("Must config " + _serviceName)); + } + for (auto const& it : _pt.get_child(_sectionName)) + { + if (it.first.find("proxy.") != 0) + { + continue; + } + std::string endpoint = it.second.data(); + PPCConfig_LOG(INFO) << LOG_BADGE("loadServiceTarsConfig") + << LOG_KV("serviceName", _serviceName) << LOG_KV("endpoint", endpoint); + if (!checkEndpoint(endpoint)) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Invalid endpoint: " + endpoint)); + } + m_serviceToEndPoints[_serviceName].emplace_back(std::move(endpoint)); + } +} + +std::string PPCConfig::getServiceName(boost::property_tree::ptree const& _pt, + std::string const& _configSection, std::string const& _objName) +{ + auto serviceName = _pt.get(_configSection, ""); + checkService(_configSection, serviceName); + return getPrxDesc(serviceName, _objName); +} + +void PPCConfig::checkService(std::string const& _serviceType, std::string const& _serviceName) +{ + if (_serviceName.empty()) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Must set service name for " + _serviceType + "!")); + } + std::vector serviceNameList; + boost::split(serviceNameList, _serviceName, boost::is_any_of(".")); + std::string errorMsg = + "Must set service name in format of application_name.server_name with only include letters " + "and numbers for " + + _serviceType + ", invalid config now is:" + _serviceName; + if (serviceNameList.size() != 2) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment(errorMsg)); + } + for (const auto& serviceName : serviceNameList) + { + if (!isalNumStr(serviceName)) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment(errorMsg)); + } + } +} diff --git a/cpp/ppc-tools/src/config/PPCConfig.h b/cpp/ppc-tools/src/config/PPCConfig.h new file mode 100644 index 00000000..6fd10a14 --- /dev/null +++ b/cpp/ppc-tools/src/config/PPCConfig.h @@ -0,0 +1,366 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PPCConfig.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include "../cuckoo/Common.h" +#include "Common.h" +#include "NetworkConfig.h" +#include "ParamChecker.h" +#include "ppc-framework/storage/CacheStorage.h" +#include +#include +#include +#include +#include + +namespace ppc::tools +{ +// lines +constexpr static uint64_t DefaultDataBatchSize = 1000000; +constexpr static uint16_t DefaultParallelism = 3; +constexpr static std::string_view NODE_PEM_NAME = "node.pem"; +struct RA2018Config +{ + // MBytes + constexpr static uint64_t DefaultCuckooFilterCacheSize = 256; + // MBytes + constexpr static uint64_t DefaultCacheSize = 1024; + // the default cuckoo-filter-capacity: 1MBytes + constexpr static int DefaultCuckooFilterCapacity = 1; + constexpr static uint8_t DefaultCuckooFilterTagSize = 32; + constexpr static uint8_t DefaultBucketSize = 4; + constexpr static uint16_t DefaultMaxKickoutCount = 20; + constexpr static uint64_t DefaultTrashBucketSize = 10000; + + std::string dbName; // the db used to store cuckoo-filter + ppc::tools::CuckoofilterOption::Ptr cuckooFilterOption; + uint64_t cuckooFilterCacheSize = DefaultCuckooFilterCacheSize * 1024 * 1024; + uint64_t cacheSize = DefaultCacheSize * 1024 * 1024; + // default load 100w data per-time + int64_t dataBatchSize = DefaultDataBatchSize; + // use hdfs to store cuckoo-filter or not + bool useHDFS = false; +}; + +struct StorageConfig +{ + ppc::protocol::SQLConnectionOption::Ptr sqlConnectionOpt; + ppc::protocol::FileStorageConnectionOption::Ptr fileStorageConnectionOpt; + ppc::protocol::RemoteStorageConnectionOption::Ptr remoteConnectionOpt; +}; + +struct CEMConfig +{ + std::string datasetFilePath; + std::string datasetHDFSPath; + std::string ciphertextSuffix; + uint64_t readPerBatchLines; +}; + +struct MPCConfig +{ + std::string datasetHDFSPath; + std::string jobPath; + std::string mpcRootPath; + std::string mpcRootPathNoGateway; + uint64_t readPerBatchLines; +}; + +// struct AYSConfig +// { +// std::string datasetPath; +// }; + +struct GatewayConfig +{ + // the max allowed message size, default is 100MBytes + // the boostssl limit is 32MBytes + constexpr static uint64_t DefaultMaxAllowedMsgSize = 100; + constexpr static uint64_t MinMsgSize = 10 * 1024 * 1024; + constexpr static uint64_t MaxMsgSize = 1024 * 1024 * 1024; + + bool disableCache; + NetworkConfig networkConfig; + ppc::storage::CacheStorageConfig cacheStorageConfig; + // agencyID => endPointList + std::map> agencies; + uint64_t maxAllowedMsgSize = DefaultMaxAllowedMsgSize; + int reconnectTime = 10000; +}; + +// the ecdh-psi config +struct EcdhPSIParam +{ + int64_t dataBatchSize = DefaultDataBatchSize; +}; + +struct EcdhMultiPSIParam +{ + int64_t dataBatchSize = DefaultDataBatchSize; +}; + +struct EcdhConnPSIParam +{ + int64_t dataBatchSize = DefaultDataBatchSize; + int64_t rank; + int64_t algos; + int64_t protocol_families; + int64_t curve; + int64_t hashtype; + int64_t hash2curve; +}; + +struct CM2020PSIParam +{ + uint16_t parallelism = DefaultParallelism; +}; + +struct OtPIRParam +{ + uint16_t parallelism = DefaultParallelism; +}; + +class PPCConfig +{ +public: + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + PPCConfig() = default; + virtual ~PPCConfig() = default; + void loadConfig(std::string const& _configPath) + { + PPCConfig_LOG(INFO) << LOG_DESC("loadConfig") << LOG_KV("path", _configPath); + boost::property_tree::ptree iniConfig; + boost::property_tree::read_ini(_configPath, iniConfig); + // Note: must load common-config firstly since some ra-configs depends on the common-config + loadCommonConfig(iniConfig); + loadRA2018Config(iniConfig); + loadStorageConfig(iniConfig); + loadEcdhPSIConfig(iniConfig); + loadCM2020PSIConfig(iniConfig); + loadEcdhMultiPSIConfig(iniConfig); + loadEcdhConnPSIConfig(iniConfig); + } + + void loadRpcConfig(const char* _certPath, std::string const& _configPath) + { + PPCConfig_LOG(INFO) << LOG_DESC("loadRpcConfig") << LOG_KV("path", _configPath); + boost::property_tree::ptree iniConfig; + boost::property_tree::read_ini(_configPath, iniConfig); + loadRpcConfig(_certPath, iniConfig); + } + + void loadGatewayConfig( + ppc::protocol::NodeArch _arch, const char* _certPath, std::string const& _configPath) + { + PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("path", _configPath); + boost::property_tree::ptree iniConfig; + boost::property_tree::read_ini(_configPath, iniConfig); + loadGatewayConfig(_arch, _certPath, iniConfig); + } + + virtual void loadTarsConfig(std::string const& _configPath) + { + boost::property_tree::ptree iniConfig; + boost::property_tree::read_ini(_configPath, iniConfig); + loadTarsConfig(iniConfig); + } + + virtual void loadRpcConfig(const char* _certPath, boost::property_tree::ptree const& _pt) + { + // rpc default disable-ssl + loadNetworkConfig( + m_rpcConfig, _certPath, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true); + } + + virtual void loadSelfTarsEndpoint(boost::property_tree::ptree const& _pt) + { + m_endpoint = _pt.get("agency.endpoint", ""); + } + + virtual void loadGatewayConfig(ppc::protocol::NodeArch _arch, const char* _certPath, + boost::property_tree::ptree const& _pt); + + virtual void loadHDFSConfig(boost::property_tree::ptree const& _pt); + + virtual void loadSQLConfig(boost::property_tree::ptree const& _pt); + + virtual void loadCEMConfig(boost::property_tree::ptree const& _pt); + + virtual void loadMPCConfig(boost::property_tree::ptree const& _pt); + + virtual void loadTarsConfig(boost::property_tree::ptree const& _pt); + + NetworkConfig const& rpcConfig() const { return m_rpcConfig; } + // the gateway-config + GatewayConfig const& gatewayConfig() const { return m_gatewayConfig; } + + RA2018Config const& ra2018PSIConfig() const { return m_ra2018PSIConfig; } + RA2018Config& mutableRA2018PSIConfig() { return m_ra2018PSIConfig; } + + StorageConfig const& storageConfig() const { return m_storageConfig; } + CEMConfig const& cemConfig() const { return m_cemConfig; } + MPCConfig const& mpcConfig() const { return m_mpcConfig; } + std::string const& agencyID() const { return m_agencyID; } + void setAgencyID(std::string const& _agencyID) { m_agencyID = _agencyID; } + bool smCrypto() const { return m_smCrypto; } + std::string const& endpoint() const { return m_endpoint; } + + std::string const& dataLocation() const { return m_dataLocation; } + uint32_t const& taskTimeoutMinutes() const { return m_taskTimeoutMinutes; } + uint32_t const& threadPoolSize() const { return m_threadPoolSize; } + + std::string const& gatewayServiceName() const { return m_gatewayServiceName; } + std::vector getServiceEndPointsByName(std::string const& _service) + { + // Note: not ensure thread-safe, since only use when init + std::vector endPoints; + auto it = m_serviceToEndPoints.find(_service); + if (it != m_serviceToEndPoints.end()) + { + return it->second; + } + return endPoints; + } + + EcdhPSIParam const& ecdhPSIConfig() const { return m_ecdhPSIConfig; } + EcdhPSIParam& mutableEcdhPSIConfig() { return m_ecdhPSIConfig; } + + EcdhConnPSIParam const& ecdhConnPSIConfig() const { return m_ecdhConnPSIConfig; } + EcdhConnPSIParam& mutableEcdhConnPSIConfig() { return m_ecdhConnPSIConfig; } + + EcdhMultiPSIParam const& ecdhMultiPSIConfig() const { return m_ecdhMultiPSIConfig; } + EcdhMultiPSIParam& mutableEcdhMultiPSIConfig() { return m_ecdhMultiPSIConfig; } + + CM2020PSIParam const& cm2020PSIConfig() const { return m_cm2020Config; } + CM2020PSIParam& mutableCM2020PSIConfig() { return m_cm2020Config; } + + OtPIRParam const& otPIRParam() const { return m_otPIRConfig; } + OtPIRParam& mutableOtPIRParam() { return m_otPIRConfig; } + + bcos::bytes const& privateKey() const { return m_privateKey; } + void setPrivateKey(bcos::bytes const& _privateKey) { m_privateKey = _privateKey; } + + std::string const& privateKeyPath() const { return m_privateKeyPath; } + // for pro-mode + void setPrivateKeyPath(std::string const& _privateKeyPath) + { + m_privateKeyPath = _privateKeyPath; + } + + bool disableRA2018() const { return m_disableRA2018; } + + int holdingMessageMinutes() const { return m_holdingMessageMinutes; } + +private: + virtual void loadRA2018Config(boost::property_tree::ptree const& _pt); + virtual void loadEcdhPSIConfig(boost::property_tree::ptree const& _pt); + virtual void loadCM2020PSIConfig(boost::property_tree::ptree const& _pt); + virtual void loadEcdhMultiPSIConfig(boost::property_tree::ptree const& _pt); + virtual void loadEcdhConnPSIConfig(boost::property_tree::ptree const& _pt); + virtual void loadCommonConfig(boost::property_tree::ptree const& _pt); + virtual void loadStorageConfig(boost::property_tree::ptree const& _pt); + // Note: the gateway/rpc can share the loadNetworkConfig + void loadNetworkConfig(NetworkConfig& _config, const char* _certPath, + boost::property_tree::ptree const& _pt, std::string const& _sectionName, + int _defaultListenPort, bool _defaultDisableSSl); + void checkPort(std::string const& _sectionName, int _port); + void checkFileExists(std::string const& _filePath, bool _dir); + + inline void checkNonEmptyField(std::string const& _section, std::string const& _value) + { + if (_value.empty()) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << bcos::errinfo_comment("Must set " + _section)); + } + } + + void initRedisConfigForGateway( + ppc::storage::CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt); + + std::map> parseAgencyConfig( + const boost::property_tree::ptree& _pt, std::string const& _sectionName, + std::string const& _subSectionName); + + // load the tars-config for the given service, e.g: + /* + [tars_gateway] + name = agencyAGateway + proxy.0 = "192.168.0.1:3000" + proxy.0 = "192.168.0.2:3002" + proxy.0 = "192.168.0.3:3003" + */ + virtual void loadServiceTarsConfig(boost::property_tree::ptree const& _pt, + std::string const& _serviceName, std::string const& _sectionName); + + std::string getServiceName(boost::property_tree::ptree const& _pt, + std::string const& _configSection, std::string const& _objName); + void checkService(std::string const& _serviceType, std::string const& _serviceName); + + int64_t getDataBatchSize(std::string const& _section, int64_t _dataBatchSize); + + int loadHoldingMessageMinutes( + const boost::property_tree::ptree& _pt, std::string const& _section); + +private: + // the rpc-config + NetworkConfig m_rpcConfig; + // the gateway-config + GatewayConfig m_gatewayConfig; + // the gateway holding message time, in minutes, default 30min + int m_holdingMessageMinutes = 30; + + // the ra2018-psi config + RA2018Config m_ra2018PSIConfig; + // the storage config + StorageConfig m_storageConfig; + // the cem config + CEMConfig m_cemConfig; + // the mpc config + MPCConfig m_mpcConfig; + + // the ecdh config + EcdhPSIParam m_ecdhPSIConfig; + + EcdhMultiPSIParam m_ecdhMultiPSIConfig; + + EcdhConnPSIParam m_ecdhConnPSIConfig; + + CM2020PSIParam m_cm2020Config; + + OtPIRParam m_otPIRConfig; + + // the agencyID/partyID + std::string m_agencyID; + bool m_smCrypto = false; + bcos::bytes m_privateKey; + std::string m_privateKeyPath; + + // the tars config + std::map> m_serviceToEndPoints; + std::string m_gatewayServiceName; + + bool m_disableRA2018 = false; + + std::string m_endpoint; + std::string m_dataLocation; + uint32_t m_taskTimeoutMinutes; + uint32_t m_threadPoolSize; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/src/config/ParamChecker.h b/cpp/ppc-tools/src/config/ParamChecker.h new file mode 100644 index 00000000..ca8f7f22 --- /dev/null +++ b/cpp/ppc-tools/src/config/ParamChecker.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file UrlChecker.h + * @author: shawnhe + * @date 2023-04-21 + */ +#pragma once +#include "ppc-framework/Common.h" +#include + +namespace ppc::tools +{ +// ip:port or domain:port +inline bool checkEndpoint(const std::string& _endpoint) +{ + try + { + if (_endpoint.empty()) + { + return false; + } + std::vector url; + boost::split(url, _endpoint, boost::is_any_of(":"), boost::token_compress_on); + if (url.size() != 2) + { + return false; + } + + int port = std::stoi(url[1]); + if (0 > port || 65535 < port) + { + return false; + } + } + catch (std::exception& e) + { + return false; + } + return true; +} +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/src/cuckoo/Common.h b/cpp/ppc-tools/src/cuckoo/Common.h new file mode 100644 index 00000000..3cdc6494 --- /dev/null +++ b/cpp/ppc-tools/src/cuckoo/Common.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-26 + */ +#pragma once +#include "ppc-framework/Common.h" +#include +#include + +#define CUCKOO_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("CUCKOO") +namespace ppc::tools +{ +DERIVE_PPC_EXCEPTION(InvalidCuckooFilterOption); +inline uint64_t upperpower2(uint64_t x) +{ + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x |= x >> 32; + x++; + return x; +} +struct CuckoofilterOption +{ + using Ptr = std::shared_ptr; + using ConstPtr = std::shared_ptr; + // the capacity + uint64_t capacity = 0; + // the tag-size, default is 64bits + uint8_t tagBits = 64; + // the tagNumPerBucket, default is 4 + uint8_t tagNumPerBucket = 4; + // The maximum number of times an element has been kicked + uint16_t maxKickOutCount = 20; + // the trash bucket size + uint64_t trashBucketSize = 10000; + + // calculate the bucketCapacity + void calculateBucketCapacity() + { + m_bucketCapacity = upperpower2(std::max(1, capacity / tagNumPerBucket)); + } + uint64_t bucketCapacity() const { return m_bucketCapacity; } + +private: + uint64_t m_bucketCapacity = 0; +}; + +enum CuckooFilterResult : int +{ + OverLoaded, + OverMaxKickOff, + Success, +}; + +inline std::string printCuckooFilterOption(CuckoofilterOption::ConstPtr _option) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("capacity", _option->capacity) + << LOG_KV("tagBits", (int)_option->tagBits) + << LOG_KV("buckets", (int)_option->tagNumPerBucket) + << LOG_KV("maxKickOutCount", _option->maxKickOutCount) + << LOG_KV("trashBucketSize", _option->trashBucketSize); + return stringstream.str(); +} + +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/src/cuckoo/Cuckoofilter.h b/cpp/ppc-tools/src/cuckoo/Cuckoofilter.h new file mode 100644 index 00000000..7d78de40 --- /dev/null +++ b/cpp/ppc-tools/src/cuckoo/Cuckoofilter.h @@ -0,0 +1,339 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Cuckoofilter.h + * @author: yujiechen + * @date 2022-10-26 + */ +#pragma once +#include "Common.h" +#include "HashTable.h" +#include "ppc-crypto-core/src/hash/BitMixMurmurHash.h" +#include +#include +#include + +namespace ppc::tools +{ +template +class Cuckoofilter +{ +public: + using Ptr = std::shared_ptr; + Cuckoofilter(CuckoofilterOption::Ptr _opt) + : m_option(_opt), m_hash(std::make_shared()) + { + initCuckooFilter(_opt); + m_hashTable = std::make_shared(_opt); + } + + Cuckoofilter(bcos::bytesConstRef _data) + : m_hash(std::make_shared()), m_hashTable(std::make_shared(_data)) + { + m_option = m_hashTable->option(); + initCuckooFilter(m_option); + } + + virtual ~Cuckoofilter() = default; + + // serialize the cuckoo-filter(mainly serialize the hashTable) + virtual std::string serialize() { return m_hashTable->serialize(); } + + // insert _key into the cuckoofilter + template + CuckooFilterResult insert(T const& _key) + { + bcos::UpgradableGuard l(m_mutex); + // the key already exists + if (containsWithoutLock(_key)) + { + return CuckooFilterResult::Success; + } + bcos::UpgradeGuard ul(l); + return insertWithoutLock(_key); + } + + // return the successfully inserted key-count + template + bool batchInsert(T& _mutableKeyBatch) + { + auto startT = bcos::utcSteadyTime(); + auto orgKeySize = _mutableKeyBatch.size(); + bcos::WriteGuard l(m_mutex); + for (auto it = _mutableKeyBatch.begin(); it != _mutableKeyBatch.end();) + { + // the key already exist + if (containsWithoutLock(*it)) + { + it = _mutableKeyBatch.erase(it); + continue; + } + auto ret = insertWithoutLock(*it); + if (ret == CuckooFilterResult::Success) + { + // erase the inserted element from the _keyBatch + it = _mutableKeyBatch.erase(it); + continue; + } + // not overload, but conflict, just ignore this element + if (ret == CuckooFilterResult::OverMaxKickOff) + { + it++; + continue; + } + // overload: break + break; + } + CUCKOO_LOG(INFO) << LOG_DESC("batchInsert") << LOG_KV("inputSize", orgKeySize) + << LOG_KV("successSize", (orgKeySize - _mutableKeyBatch.size())) + << LOG_KV("left", _mutableKeyBatch.size()) << LOG_KV("load", loadFactor()) + << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); + return _mutableKeyBatch.empty(); + } + + // check the _key exists in the cuckoofilter or not + template + bool contains(T const& _key) const + { + bcos::ReadGuard l(m_mutex); + return containsWithoutLock(_key); + } + + template + std::vector batchContain(std::vector const& _batchKey) const + { + bcos::ReadGuard l(m_mutex); + std::vector result(_batchKey.size()); + for (uint64_t i = 0; i < _batchKey.size(); i++) + { + result[i] = containsWithoutLock(_batchKey.at(i)); + } + return result; + } + + // return the intersection element into _result + template + void batchIntersection(std::vector& _result, std::vector const& _batchKey) const + { + bcos::ReadGuard l(m_mutex); + for (uint64_t i = 0; i < _batchKey.size(); i++) + { + if (containsWithoutLock(_batchKey.at(i))) + { + _result.emplace_back(_batchKey.at(i)); + } + } + } + + template + std::vector batchIntersectionAndGetPos(std::vector const& _batchKey) const + { + bcos::ReadGuard l(m_mutex); + std::vector pos; + for (uint64_t i = 0; i < _batchKey.size(); i++) + { + if (containsWithoutLock(_batchKey.at(i))) + { + pos.emplace_back(i); + } + } + return pos; + } + + // delete the _key from the cuckoofilter + template + bool deleteKey(T const& _key) + { + bcos::WriteGuard l(m_mutex); + return deleteKeyWithoutLock(_key); + } + + template + std::vector batchDeleteKey(std::vector const& _keyBatch) + { + bcos::WriteGuard l(m_mutex); + std::vector result; + for (auto const& key : _keyBatch) + { + result.emplace_back(deleteKeyWithoutLock(key)); + } + return result; + } + + // delete the key from the cuckoo-filter, return the delete-failed key into _keyBatch + template + void batchDeleteAndGetRemainKeys(std::vector& _keyBatch) + { + bcos::WriteGuard l(m_mutex); + for (auto it = _keyBatch.begin(); it != _keyBatch.end();) + { + if (!deleteKeyWithoutLock(*it)) + { + it++; + continue; + } + it = _keyBatch.erase(it); + } + } + + double loadFactor() { return 1.0 * m_hashTable->occupiedCount() / m_totalTagSize; } + int64_t occupiedCount() const { return m_hashTable->occupiedCount(); } + + uint64_t capacity() const { return m_hashTable->capacity(); } + bool full() const { return m_hashTable->full(); } + +protected: + // insert the _key into cuckoo-filter + template + CuckooFilterResult insertWithoutLock(T const& _key) + { + // the cuckoo-filter is over the load-threshold + if (m_hashTable->full()) + { + return CuckooFilterResult::OverLoaded; + } + auto fingerPrintInfo = calculateIndexAndFingerPrint(_key); + uint64_t oldFingerPrint = 0; + // insert the _key-finger-print into the i1 success + if (m_hashTable->insert( + std::get<0>(fingerPrintInfo), std::get<2>(fingerPrintInfo), oldFingerPrint, false)) + { + return CuckooFilterResult::Success; + } + // insert the _key-finger-print into the i2 success + if (m_hashTable->insert( + std::get<1>(fingerPrintInfo), std::get<2>(fingerPrintInfo), oldFingerPrint, false)) + { + return CuckooFilterResult::Success; + } + // insert the occupied index + uint64_t index = std::get<0>(fingerPrintInfo); + uint64_t fingerPrint = std::get<2>(fingerPrintInfo); + for (uint16_t i = 0; i < m_option->maxKickOutCount; i++) + { + // replace the i2 fingerprint + m_hashTable->insert(index, fingerPrint, oldFingerPrint, true); + // find-out a empty position for the old-finger-print + if (oldFingerPrint != 0) + { + // find the place to hold oldFingerPrint + index = index ^ (m_hash->hash(oldFingerPrint, m_hashBitsLen) & + (m_option->bucketCapacity() - 1)); + fingerPrint = oldFingerPrint; + // reset oldFingerPrint + oldFingerPrint = 0; + continue; + } + return CuckooFilterResult::Success; + } + // insert into the trash bucket + m_hashTable->insertIntoTrash(fingerPrint); + return CuckooFilterResult::OverMaxKickOff; + } + + // check the cuckoo-filter contains _key or not + template + bool containsWithoutLock(T const& _key) const + { + auto fingerPrintInfo = calculateIndexAndFingerPrint(_key); + return m_hashTable->contains(std::get<0>(fingerPrintInfo), std::get<1>(fingerPrintInfo), + std::get<2>(fingerPrintInfo)); + } + + // delete the given _key from the cuckoo-filter + template + bool deleteKeyWithoutLock(T const& _key) + { + auto fingerprintInfo = calculateIndexAndFingerPrint(_key); + if (m_hashTable->deleteFingerPrint( + std::get<0>(fingerprintInfo), std::get<2>(fingerprintInfo))) + { + return true; + } + if (m_hashTable->deleteFingerPrint( + std::get<1>(fingerprintInfo), std::get<2>(fingerprintInfo))) + { + return true; + } + // delete from the trashBucket + if (m_hashTable->deleteFromTrashBucket(std::get<2>(fingerprintInfo))) + { + return true; + } + return false; + } + +private: + void initCuckooFilter(CuckoofilterOption::Ptr _opt) + { + m_option = _opt; + if (m_option->tagBits % 8 != 0) + { + BOOST_THROW_EXCEPTION(InvalidCuckooFilterOption() + << bcos::errinfo_comment("the tags size must be multiple of 8")); + } + if (m_option->capacity == 0) + { + BOOST_THROW_EXCEPTION(InvalidCuckooFilterOption() + << bcos::errinfo_comment("the capacity must be larger than 0")); + } + if (m_option->tagBits > 64) + { + BOOST_THROW_EXCEPTION(InvalidCuckooFilterOption() << bcos::errinfo_comment( + "the tagBits must be no larger than 64")); + } + m_hashBitsLen = (m_option->tagBits <= 32 ? 32 : 64); + m_tagBytes = m_option->tagBits >> 3; + // update the capacity to 2^n(the cuckoo-filter requires the capacity to be 2^n) + m_option->calculateBucketCapacity(); + m_totalTagSize = (uint64_t)m_option->bucketCapacity() * (uint64_t)m_option->tagNumPerBucket; + CUCKOO_LOG(INFO) << printCuckooFilterOption(_opt); + } + + uint64_t calculateFingerPrint(uint64_t _hash) const + { + auto fingerPrint = (uint64_t)(_hash >> (m_hashBitsLen - m_option->tagBits)); + // the case finger is 0 + if (0 == fingerPrint) + { + return 1; + } + return fingerPrint; + } + + template + std::tuple calculateIndexAndFingerPrint(T const _key) const + { + uint64_t hashResult = m_hash->hash(_key, m_hashBitsLen); + // calculate i1 + uint64_t i1 = hashResult & (m_option->bucketCapacity() - 1); + // calculate fingerprint + uint64_t fingerPrint = calculateFingerPrint(hashResult); + // calculate i2 + uint64_t i2 = + i1 ^ (m_hash->hash(fingerPrint, m_hashBitsLen) & (m_option->bucketCapacity() - 1)); + return std::make_tuple(i1, i2, fingerPrint); + } + +private: + CuckoofilterOption::Ptr m_option; + std::shared_ptr m_hash; + std::shared_ptr m_hashTable; + uint8_t m_tagBytes; + uint8_t m_hashBitsLen; + uint64_t m_totalTagSize = 0; + mutable bcos::SharedMutex m_mutex; +}; +} // namespace ppc::tools diff --git a/cpp/ppc-tools/src/cuckoo/HashTable.cpp b/cpp/ppc-tools/src/cuckoo/HashTable.cpp new file mode 100644 index 00000000..916d61cc --- /dev/null +++ b/cpp/ppc-tools/src/cuckoo/HashTable.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HashTable.cpp + * @author: yujiechen + * @date 2022-10-26 + */ +#include "HashTable.h" + +using namespace ppc::tools; + +std::string HashTable::serialize() +{ + std::string encodedData; + boost::iostreams::stream> outputStream( + encodedData); + boost::archive::binary_oarchive archive(outputStream, + boost::archive::no_header | boost::archive::no_codecvt | boost::archive::no_tracking); + archive << m_option->capacity; + archive << m_option->tagBits; + archive << m_option->tagNumPerBucket; + archive << m_option->maxKickOutCount; + archive << m_buckets; + // serialize the trash-bucket + archive << m_trashBucket; + outputStream.flush(); + return encodedData; +} + +void HashTable::deserialize(bcos::bytesConstRef _data) +{ + boost::iostreams::stream inputStream( + (const char*)_data.data(), _data.size()); + boost::archive::binary_iarchive archive(inputStream, + boost::archive::no_header | boost::archive::no_codecvt | boost::archive::no_tracking); + archive >> m_option->capacity; + archive >> m_option->tagBits; + archive >> m_option->tagNumPerBucket; + archive >> m_option->maxKickOutCount; + archive >> m_buckets; + // deserialize the trash-bucket + archive >> m_trashBucket; +} + +uint64_t HashTable::getFingerPrint(const uint64_t _entryIndex, const uint32_t _bucketIndex) const +{ + // empty finger-print + auto it = m_buckets.find(_entryIndex); + if (it == m_buckets.end()) + { + return 0; + } + auto const& bucket = it->second; + // empty bucket + if (bucket.size() == 0) + { + return 0; + } + // Note: use little-endian here + // get the bytes-data + int64_t fingerPrint = 0; + auto offset = _bucketIndex * m_bytesPerTag; + for (uint32_t i = offset; i < (offset + m_bytesPerTag); i++) + { + fingerPrint <<= 8; + fingerPrint |= bucket.at(i); + } + return fingerPrint; +} + +bool HashTable::insert( + const uint64_t _pos, const uint64_t _fingerPrint, uint64_t& _oldFingerPrint, bool _kickOut) +{ + // find the un-occupied bucket + for (uint32_t bucketIndex = 0; bucketIndex < m_option->tagNumPerBucket; bucketIndex++) + { + if (getFingerPrint(_pos, bucketIndex) == 0) + { + writeFingerPrint(_pos, bucketIndex, _fingerPrint); + m_occupiedCount++; + return true; + } + } + // random choose one fingerprint from the bucket to kickOut + if (_kickOut) + { + srand(bcos::utcSteadyTime()); + auto selectedBucket = rand() % (m_option->tagNumPerBucket); + _oldFingerPrint = getFingerPrint(_pos, selectedBucket); + writeFingerPrint(_pos, selectedBucket, _fingerPrint); + return true; + } + return false; +} + +bool HashTable::deleteFingerPrint(const uint64_t _pos, const uint64_t _fingerPrint) +{ + for (uint32_t bucketIndex = 0; bucketIndex < m_option->tagNumPerBucket; bucketIndex++) + { + if (getFingerPrint(_pos, bucketIndex) == _fingerPrint) + { + writeFingerPrint(_pos, bucketIndex, 0); + m_occupiedCount.store(std::max(0, m_occupiedCount.load() - 1)); + return true; + } + } + return false; +} + +void HashTable::writeFingerPrint( + const uint64_t _entryIndex, const uint32_t _bucketIndex, uint64_t const _fingerPrint) +{ + auto it = m_buckets.find(_entryIndex); + if (it == m_buckets.end()) + { + m_buckets[_entryIndex] = Bucket(); + } + auto& bucket = m_buckets[_entryIndex]; + if (bucket.size() <= m_bytesPerBucket) + { + bucket.resize(m_bytesPerBucket); + m_capacity += m_bytesPerBucket; + } + // write the fingerprint into the bucket + auto offset = _bucketIndex * m_bytesPerTag; + auto endPos = offset + (m_bytesPerTag - 1); + auto fingerPrintData = _fingerPrint; + for (int64_t i = endPos; i >= offset; i--) + { + bucket[i] = fingerPrintData & 0xff; + fingerPrintData >>= 8; + } +} + +bool HashTable::contains(const uint64_t _i1, const uint64_t _i2, const uint64_t _fingerPrint) const +{ + for (uint32_t bucketIndex = 0; bucketIndex < m_option->tagNumPerBucket; bucketIndex++) + { + if ((getFingerPrint(_i1, bucketIndex) == _fingerPrint) || + (getFingerPrint(_i2, bucketIndex) == _fingerPrint)) + { + return true; + } + } + // check the trash-bucket: hit in the trash bucket + if (m_trashBucket.count(_fingerPrint)) + { + return true; + } + return false; +} \ No newline at end of file diff --git a/cpp/ppc-tools/src/cuckoo/HashTable.h b/cpp/ppc-tools/src/cuckoo/HashTable.h new file mode 100644 index 00000000..a73ce8a4 --- /dev/null +++ b/cpp/ppc-tools/src/cuckoo/HashTable.h @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HashTable.h + * @author: yujiechen + * @date 2022-10-26 + */ +#pragma once +#include "Common.h" +#include +#include +#include +#include +#include +#include +#include +#include +namespace ppc::tools +{ +class HashTable +{ +public: + using Ptr = std::shared_ptr; + HashTable(CuckoofilterOption::Ptr _option) : m_option(_option) { initHashTableParams(); } + + HashTable(bcos::bytesConstRef _data) : m_option(std::make_shared()) + { + m_capacity = _data.size(); + deserialize(_data); + initHashTableParams(); + } + + virtual ~HashTable() = default; + /** + * @brief check the hashTable contains given finger-print in pos _i1 or _i2 + * + * @param _i1 the pos i1 + * @param _i2 the pos i2 + * @param _fingerPrint the finger-print + * @return true find the finger-print + * @return false the finger-print doesn't exist in the HashTable + */ + bool contains(const uint64_t _i1, const uint64_t _i2, const uint64_t _fingerPrint) const; + + /** + * @brief try to insert finger-print into the given pos + * + * @param _pos the pos that required to insert the finger-print + * @param _fingerPrint the finger-print to be inserted + * @param _oldFingerPrint the kickOuted finger-print + * @return true insert success + * @return false insert failed + */ + // Note: use lazy allocate policy here + bool insert(const uint64_t _pos, const uint64_t _fingerPrint, uint64_t& _oldFingerPrint, + bool _kickOut = true); + + // delete the given finger-print front the hashTable + bool deleteFingerPrint(const uint64_t _pos, const uint64_t _fingerPrint); + virtual std::string serialize(); + virtual void deserialize(bcos::bytesConstRef _data); + CuckoofilterOption::Ptr option() const { return m_option; } + int64_t occupiedCount() const { return m_occupiedCount.load(); } + + uint64_t capacity() const { return m_capacity; } + + // the hashTable is full or not + bool full() const { return m_trashBucket.size() >= m_option->trashBucketSize; } + + bool insertIntoTrash(uint64_t _fingerPrint) + { + if (m_trashBucket.count(_fingerPrint)) + { + return true; + } + if (m_trashBucket.size() >= m_option->trashBucketSize) + { + return false; + } + m_trashBucket.insert(_fingerPrint); + return true; + } + + bool deleteFromTrashBucket(uint64_t _fingerPrint) + { + auto it = m_trashBucket.find(_fingerPrint); + if (it != m_trashBucket.end()) + { + m_trashBucket.erase(it); + return true; + } + return false; + } + +protected: + void initHashTableParams() + { + m_bytesPerBucket = (uint32_t)(m_option->tagBits * m_option->tagNumPerBucket) >> 3; + // Note: the tagBits of Cuckoofilter should be multiple of 8 + m_bytesPerTag = (uint32_t)(m_option->tagBits >> 3); + // the bucket-capacity + CUCKOO_LOG(INFO) << LOG_DESC("create HashTable") + << LOG_KV("tagBits", std::to_string(m_option->tagBits)) + << LOG_KV("tagNumPerBucket", std::to_string(m_option->tagNumPerBucket)) + << LOG_KV("bucketCapacity", m_option->bucketCapacity()) + << LOG_KV("bytesPerBucket", m_bytesPerBucket) + << LOG_KV("capacity", m_option->capacity); + } + +private: + uint64_t getFingerPrint(const uint64_t _entryIndex, const uint32_t _bucketIndex) const; + // lazy-allocate + void writeFingerPrint( + const uint64_t _entryIndex, const uint32_t _bucketIndex, uint64_t const _fingerPrint); + +private: + // the option + CuckoofilterOption::Ptr m_option; + // bytes-per-bucket(determined by the tagBits and the tagNumPerBucket) + uint32_t m_bytesPerBucket; + // bytes per fingerprint + uint32_t m_bytesPerTag; + // Note: the unordered_map can't ensure all the node with the same cuckoo-filter encoded into + // the same data, so we use map here + using Bucket = bcos::bytes; + std::map m_buckets; + // trash bucket, storing elements kicked out + // Note: the trash bucket is neccessary to accept the element over max-kick-out-count + std::set m_trashBucket; + std::atomic m_capacity = {0}; + + // the occupied-count + std::atomic m_occupiedCount = {0}; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/ppc-tools/tests/CMakeLists.txt b/cpp/ppc-tools/tests/CMakeLists.txt new file mode 100644 index 00000000..9ab026cc --- /dev/null +++ b/cpp/ppc-tools/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-tools) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${TOOLS_TARGET} ${CRYPTO_CORE_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-tools WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-tools/tests/TestCuckoofilter.cpp b/cpp/ppc-tools/tests/TestCuckoofilter.cpp new file mode 100644 index 00000000..e38719a0 --- /dev/null +++ b/cpp/ppc-tools/tests/TestCuckoofilter.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestCuckoofilter.cpp + * @author: yujiechen + * @date 2022-10-26 + */ +#include "ppc-crypto-core/src/hash/SM3Hash.h" +#include "ppc-crypto-core/src/hash/Sha512Hash.h" +#include "ppc-tools/src/cuckoo/Cuckoofilter.h" +#include +#include +#include + +using namespace ppc::tools; +using namespace ppc::crypto; +using namespace bcos; + +using namespace bcos::test; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(CuckoofilterTest, TestPromptFixture) + +void testCuckoofilterFunc( + std::shared_ptr> filter, + uint64_t keySize) +{ + // check empty key + for (uint64_t i = 1; i <= keySize; i++) + { + BOOST_CHECK(filter->contains(i) == false); + } + // insert keys + for (uint64_t i = 1; i <= keySize; i++) + { + filter->insert(i); + BOOST_CHECK(filter->contains(i) == true); + } + // check the occupied-count + BOOST_CHECK((uint64_t)filter->occupiedCount() == keySize); + BOOST_CHECK(filter->loadFactor() < 1); + + // test encode/decode + auto encodedData = filter->serialize(); + auto decodedFilter = std::make_shared>( + bcos::bytesConstRef((const unsigned char*)encodedData.data(), encodedData.size())); + // check the keys + for (uint64_t i = 1; i <= keySize; i++) + { + BOOST_CHECK(decodedFilter->contains(i) == true); + } + BOOST_CHECK(decodedFilter->contains(keySize + 1) == false); +} + +void randomNumberTest(CuckoofilterOption::Ptr option, uint64_t keySize) +{ + uint64_t maxValue = 1000000000; + auto filter = std::make_shared>(option); + + std::set randomValuesSet; + std::mt19937_64 rng(bcos::utcSteadyTime()); + std::uniform_int_distribution dis(1, maxValue); + for (uint64_t i = 0; i < keySize; i++) + { + auto value = dis(rng); + randomValuesSet.insert(value); + } + std::vector randomValues(randomValuesSet.begin(), randomValuesSet.end()); + // insert and check randomValues + std::set tmpRandomValues(randomValues.begin(), randomValues.end()); + auto result = filter->batchInsert(tmpRandomValues); + BOOST_CHECK(result == true); + BOOST_CHECK(tmpRandomValues.empty()); + for (uint64_t i = 0; i < randomValues.size(); i++) + { + BOOST_CHECK(filter->contains(randomValues.at(i)) == true); + } + // batchContain + auto batchContainResult = filter->batchContain(randomValues); + BOOST_CHECK(batchContainResult.size() == randomValues.size()); + for (uint64_t i = 0; i < randomValues.size(); i++) + { + BOOST_CHECK(batchContainResult.at(i) == true); + } + // test encode/decode + auto encodedData = filter->serialize(); + auto decodedFilter = std::make_shared>( + bcos::bytesConstRef((const unsigned char*)encodedData.data(), encodedData.size())); + // check the keys + for (uint64_t i = 0; i < randomValues.size(); i++) + { + BOOST_CHECK(decodedFilter->contains(randomValues.at(i)) == true); + } + // delete and check + for (uint64_t i = 0; i < randomValues.size() / 2; i++) + { + BOOST_CHECK(filter->deleteKey(randomValues.at(i)) == true); + } + uint64_t falsePositiveCount = 0; + for (uint64_t i = 0; i < randomValues.size() / 2; i++) + { + if (filter->contains(randomValues.at(i)) == true) + { + falsePositiveCount++; + } + } + std::cout << "falsePositiveCount: " << falsePositiveCount << std::endl; + // batchDelete + filter->batchDeleteKey(randomValues); + for (uint64_t i = 0; i < randomValues.size(); i++) + { + BOOST_CHECK(filter->contains(randomValues.at(i)) == false); + } +} + +void bytesKeyTest(Hash::Ptr hashImpl, CuckoofilterOption::Ptr option, uint64_t keySize) +{ + auto filter = std::make_shared>(option); + std::vector hashList; + for (uint64_t i = 0; i < keySize; i++) + { + std::string input = std::to_string(i); + bcos::bytes inputData(input.begin(), input.end()); + hashList.emplace_back(hashImpl->hash(ref(inputData))); + } + // insert the hashList and check + std::set tmpHashList(hashList.begin(), hashList.end()); + auto result = filter->batchInsert(tmpHashList); + BOOST_CHECK(tmpHashList.empty() == true); + BOOST_CHECK(result == true); + // insert with std::string_view and std::string(insert multiple-times) + for (uint64_t i = 0; i < hashList.size(); i++) + { + std::string dataStr = std::string(hashList.at(i).begin(), hashList.at(i).end()); + filter->insert(dataStr); + filter->insert(std::string_view(dataStr)); + } + for (uint64_t i = 0; i < hashList.size(); i++) + { + BOOST_CHECK(filter->contains(hashList.at(i)) == true); + } + // batchContain + auto batchContainResult = filter->batchContain(hashList); + BOOST_CHECK(batchContainResult.size() == hashList.size()); + for (uint64_t i = 0; i < hashList.size(); i++) + { + BOOST_CHECK(batchContainResult.at(i) == true); + } + // test encode/decode + auto encodedData = filter->serialize(); + auto decodedFilter = std::make_shared>( + bcos::bytesConstRef((const unsigned char*)encodedData.data(), encodedData.size())); + // check the keys + for (uint64_t i = 0; i < hashList.size(); i++) + { + BOOST_CHECK(decodedFilter->contains(hashList.at(i)) == true); + } + // delete and check + for (uint64_t i = 0; i < hashList.size() / 2; i++) + { + BOOST_CHECK(filter->deleteKey(hashList.at(i)) == true); + } + uint64_t falsePositiveCount = 0; + for (uint64_t i = 0; i < hashList.size() / 2; i++) + { + if (filter->contains(hashList.at(i)) == true) + { + falsePositiveCount++; + } + } + std::cout << "bytesKeyTest falsePositiveCount: " << falsePositiveCount << std::endl; + // batchDelete + filter->batchDeleteKey(hashList); + for (uint64_t i = 0; i < hashList.size(); i++) + { + BOOST_CHECK(filter->contains(hashList.at(i)) == false); + } +} + +BOOST_AUTO_TEST_CASE(testCuckoofilter) +{ + auto sm3Hash = std::make_shared(); + auto sha512Hash = std::make_shared(); + // option with tagBits: 16, tagNumPerBucket: 4, maxKickOutCount: 20 + auto option = std::make_shared(); + option->capacity = 10000; + auto filter = std::make_shared>(option); + uint64_t keySize = 5000; + testCuckoofilterFunc(filter, keySize); + randomNumberTest(option, keySize); + bytesKeyTest(sm3Hash, option, keySize); + bytesKeyTest(sha512Hash, option, keySize); + + // option with tagBits: 32, tagNumPerBucket: 4, maxKickOutCount: 10 + option = std::make_shared(); + option->tagBits = 32; + option->maxKickOutCount = 10; + option->capacity = 20000; + filter = std::make_shared>(option); + keySize = 10000; + testCuckoofilterFunc(filter, keySize); + randomNumberTest(option, keySize); + bytesKeyTest(sm3Hash, option, keySize); + bytesKeyTest(sha512Hash, option, keySize); +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-tools/tests/TestTransTools.cpp b/cpp/ppc-tools/tests/TestTransTools.cpp new file mode 100644 index 00000000..073275ef --- /dev/null +++ b/cpp/ppc-tools/tests/TestTransTools.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TestTransTools.h + * @author: shawnhe + * @date 2022-11-07 + */ + +#include "ppc-tools/src/common/TransTools.h" +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc::tools; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(TransToolsTest, TestPromptFixture) + +template +void testEncodeAndDecodeFunc(T _number) +{ + auto buffer = std::make_shared(); + encodeUnsignedNum(buffer, _number); + + T num; + decodeUnsignedNum(num, buffer); + + BOOST_CHECK(num == _number); +} + +template +void testEncodeAndDecodeStrFunc(T _number) +{ + std::string buffer = ""; + encodeUnsignedNum(buffer, _number); + + T num; + decodeUnsignedNum(num, buffer); + + BOOST_CHECK(num == _number); +} + +BOOST_AUTO_TEST_CASE(testEncodeAndDecode) +{ + testEncodeAndDecodeFunc(uint8_t(17)); + testEncodeAndDecodeFunc(uint16_t(257)); + testEncodeAndDecodeFunc(uint32_t(65537)); + + testEncodeAndDecodeStrFunc(uint8_t(17)); + testEncodeAndDecodeStrFunc(uint16_t(257)); + testEncodeAndDecodeStrFunc(uint32_t(65537)); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-tools/tests/main.cpp b/cpp/ppc-tools/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-tools/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/ppc-udf/CMakeLists.txt b/cpp/ppc-udf/CMakeLists.txt new file mode 100644 index 00000000..e6fe5b4f --- /dev/null +++ b/cpp/ppc-udf/CMakeLists.txt @@ -0,0 +1,35 @@ +project(ppc-udf VERSION ${VERSION}) + +# export windows dll symbol +if(WIN32) + message(STATUS "Compile on Windows") + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") +endif() + +file(GLOB_RECURSE SRCS mysql/*.cpp mysql/*.c) + +if(EXISTS ${MYSQL_INCLUDE_PATH}) + include_directories(${MYSQL_INCLUDE_PATH}) +else() + find_package(MySQL REQUIRED) + if(MYSQL_FOUND) + include_directories(${MYSQL_INCLUDE_DIRS}) + else() + message(FATAL "MySQL not found! Must install the mysql or specify the include path using -DMYSQL_INCLUDE_PATH") + endif() +endif() + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +# generate the shared lib +add_definitions("-DHAVE_DLOPEN") +add_library(${PPC_UDF_TARGET} MODULE ${SRCS}) +target_link_libraries(${PPC_UDF_TARGET} PUBLIC ${PPC_CRYPTO_C_SDK_STATIC_TARGET}) + +add_library(${PPC_UDF_STATIC_TARGET} ${SRCS}) +target_link_libraries(${PPC_UDF_STATIC_TARGET} PUBLIC ${PPC_CRYPTO_C_SDK_STATIC_TARGET}) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() diff --git a/cpp/ppc-udf/hive/build.gradle b/cpp/ppc-udf/hive/build.gradle new file mode 100644 index 00000000..b866064c --- /dev/null +++ b/cpp/ppc-udf/hive/build.gradle @@ -0,0 +1,159 @@ +plugins { + id 'java' + id 'idea' + id 'org.ajoberstar.grgit' version '4.1.1' + id 'java-library' + id 'maven-publish' + id "de.undercouch.download" version "4.1.2" + id 'com.github.sherter.google-java-format' version '0.9' +} + +println("Notice: current gradle version is " + gradle.gradleVersion) +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +[compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8' + +ext { + if (!project.hasProperty("ossrhUsername")) { + ossrhUsername = "xxx" + } + + if (!project.hasProperty("ossrhPassword")) { + ossrhPassword = "xxx" + } + toml4jVersion = "0.7.2" + hiveExecVersion="2.3.9" +} + +repositories { + maven { url "https://maven.aliyun.com/nexus/content/groups/public/" } + maven { url "https://oss.sonatype.org/service/local/staging/deploy/maven2"} + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } + maven {url "https://plugins.gradle.org/m2/"} + mavenCentral() + mavenLocal() +} +googleJavaFormat { + toolVersion = '1.7' + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' + exclude '**/*Test.java' + exclude '**/Test*.java' + exclude '**/Mock*.java' +} + +sourceSets { + main { + java { + srcDir 'src/main/java' + } + resources { + srcDir 'src/main/resources' + } + } +} +dependencies { + api("org.apache.hive:hive-exec:${hiveExecVersion}") + api("com.moandjiezana.toml:toml4j:${toml4jVersion}") + implementation fileTree(dir:'libs',include:['*.jar']) + testImplementation ('junit:junit:4.13.2') +} + +archivesBaseName = 'ppc-hive-udf' +group = 'com.webank.wedpr' +version = '1.0.0' + +test { + dependsOn cleanTest + testLogging.showStandardStreams = true +} + +jar { + exclude '**/*.xml' + exclude '**/*.properties' + doLast { + copy { + from destinationDirectory + into 'dist/app' + } + + copy { + from configurations.runtimeClasspath + into 'dist/lib' + } + } +} + +javadoc { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +task sourcesJar(type: Jar) { + from sourceSets.main.allJava + archiveClassifier = 'sources' + duplicatesStrategy 'exclude' +} +tasks.withType(Copy).all { + duplicatesStrategy 'exclude' +} + + +task javadocJar(type: Jar) { + from javadoc + archiveClassifier = 'javadoc' +} +publishing { + publications { + mavenJava(MavenPublication) { + artifactId project.name + groupId project.group + version project.version + + from components.java + artifact sourcesJar + artifact javadocJar + + pom { + name = 'wedpr' + description = 'wedpr ppc-hive-udf' + url = 'http://www.fisco-bcos.org' + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + scm { + connection = 'http://github.com/WeBankBlockchain/wedpr-components.git' + url = 'http://github.com/WeBankBlockchain/wedpr-components.git' + } + + developers { + developer { + id = 'zhangsan' + name = 'zhangsan' + email = 'zhangsan@example.com' + } + } + } + } + } + repositories { + maven { + def releasesRepoURL = "https://oss.sonatype.org/service/local/staging/deploy/maven2" + def snapshotsRepoURL = "https://oss.sonatype.org/content/repositories/snapshots" + allowInsecureProtocol = true + url = !version.endsWith("SNAPSHOT") ? releasesRepoURL : snapshotsRepoURL + + credentials { + username 'deployment' + password 'deployment123' + } + } + } +} diff --git a/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar b/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q

Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties b/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..be792139 --- /dev/null +++ b/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +#distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip +distributionUrl=https\://mirrors.cloud.tencent.com/gradle/gradle-7.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/cpp/ppc-udf/hive/gradlew b/cpp/ppc-udf/hive/gradlew new file mode 100644 index 00000000..4f906e0c --- /dev/null +++ b/cpp/ppc-udf/hive/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/cpp/ppc-udf/hive/gradlew.bat b/cpp/ppc-udf/hive/gradlew.bat new file mode 100644 index 00000000..ac1b06f9 --- /dev/null +++ b/cpp/ppc-udf/hive/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/cpp/ppc-udf/hive/settings.gradle b/cpp/ppc-udf/hive/settings.gradle new file mode 100644 index 00000000..f8eea762 --- /dev/null +++ b/cpp/ppc-udf/hive/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'ppc-hive-udf' + diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java new file mode 100644 index 00000000..52d6cdae --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java @@ -0,0 +1,55 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.config; + +import com.webank.wedpr.hive.udf.config.model.ConfigProperty; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import java.util.Objects; +import org.apache.log4j.Logger; + +public class Config { + private static final Logger logger = Logger.getLogger(Config.class.getName()); + + private KeyConfig keyConfig; + + public Config(ConfigProperty configProperty) { + this.keyConfig = new KeyConfig(configProperty); + } + + public KeyConfig getKeyConfig() { + return keyConfig; + } + + public void setKeyConfig(KeyConfig keyConfig) { + this.keyConfig = keyConfig; + } + + @Override + public String toString() { + return "Config{" + "keyConfig=" + keyConfig + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Config that = (Config) o; + return Objects.equals(keyConfig, that.keyConfig); + } + + @Override + public int hashCode() { + return Objects.hash(keyConfig); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java new file mode 100644 index 00000000..d212e5c2 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java @@ -0,0 +1,46 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.config; + +import com.moandjiezana.toml.Toml; +import com.webank.wedpr.hive.udf.config.model.ConfigProperty; +import com.webank.wedpr.hive.udf.exceptions.ConfigException; +import java.io.InputStream; +import org.apache.log4j.Logger; + +public class ConfigLoader { + private static final Logger logger = Logger.getLogger(ConfigLoader.class.getName()); + private static final String DEFAULT_CONFIG_FILE = "config.toml"; + public static Config config; + + static { + InputStream configStream = + ConfigLoader.class.getClassLoader().getResourceAsStream(DEFAULT_CONFIG_FILE); + if (configStream == null) { + throw new ConfigException( + "The config file " + DEFAULT_CONFIG_FILE + " doesn't exists!"); + } + config = load(configStream); + } + + private static Config load(InputStream configStream) throws ConfigException { + try { + ConfigProperty property = new Toml().read(configStream).to(ConfigProperty.class); + return new Config(property); + } catch (Exception e) { + logger.error("load config error, e: " + e.getMessage()); + throw new ConfigException("load config error!", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java new file mode 100644 index 00000000..8345abcf --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java @@ -0,0 +1,59 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.config.model; + +import java.util.Map; +import java.util.Objects; +import org.apache.log4j.Logger; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfigProperty { + private static final Logger logger = Logger.getLogger(ConfigProperty.class.getName()); + + private Map keys; + + public Map getKeys() { + return keys; + } + + public void setKeys(Map keys) { + this.keys = keys; + } + + public static String getValue(Map config, String key, String defaultValue) { + if (config == null || config.get(key) == null) { + return defaultValue; + } + return (String) config.get(key); + } + + @Override + public String toString() { + return "ConfigProperty{" + "keys=" + keys + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConfigProperty that = (ConfigProperty) o; + return Objects.equals(keys, that.keys); + } + + @Override + public int hashCode() { + return Objects.hash(keys); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java new file mode 100644 index 00000000..a3308d29 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java @@ -0,0 +1,131 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.config.model; + +import com.webank.wedpr.hive.udf.exceptions.ConfigException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import java.util.Map; +import org.apache.log4j.Logger; + +public class KeyConfig { + private static final Logger logger = Logger.getLogger(KeyConfig.class.getName()); + + // AES key + private static final String AES_PRIVATE_KEY_PROPERTY = "aes_sk"; + private String aesPrivateKey; + // AES IV + private static final String AES_IV_PROPERTY = "aes_iv"; + private String aesIv; + // ore key + private static final String ORE_PRIVATE_KEY_PROPERTY = "ore_sk"; + private String orePrivateKey; + + // ihc key + private static final String IHC_PRIVATE_KEY_PROPERTY = "ihc_sk"; + private String ihcPrivateKey; + + // paillier key + private static final String PAILLIER_PRIVATE_KEY_PROPERTY = "paillier_sk"; + private static final String PAILLIER_PUBLIC_KEY_PROPERTY = "paillier_pk"; + private String paillierSk; + private String paillierPk; + + private final int ihcMode = NativeFloatingIhc.mode.IHC_128.ordinal(); + + public KeyConfig(ConfigProperty configProperty) throws ConfigException { + Map keyConfig = configProperty.getKeys(); + // TODO: use dpm key files + /// load the aes private key + aesPrivateKey = ConfigProperty.getValue(keyConfig, AES_PRIVATE_KEY_PROPERTY, null); + if (aesPrivateKey == null) { + throw new ConfigException("Must set the aes private key!"); + } + /// load the aesIv + aesIv = ConfigProperty.getValue(keyConfig, AES_IV_PROPERTY, null); + if (aesIv == null) { + throw new ConfigException("Must set the aes iv!"); + } + /// load the orePrivateKey + orePrivateKey = ConfigProperty.getValue(keyConfig, ORE_PRIVATE_KEY_PROPERTY, null); + if (orePrivateKey == null) { + throw new ConfigException("Must set the ore private key!"); + } + /// load the ihcPrivateKey + ihcPrivateKey = ConfigProperty.getValue(keyConfig, IHC_PRIVATE_KEY_PROPERTY, null); + if (ihcPrivateKey == null) { + throw new ConfigException("Must set the ihc private key"); + } + /// load the paillier keys + paillierPk = ConfigProperty.getValue(keyConfig, PAILLIER_PUBLIC_KEY_PROPERTY, null); + if (paillierPk == null) { + throw new ConfigException("Must set the paillier public key!"); + } + paillierSk = ConfigProperty.getValue(keyConfig, PAILLIER_PRIVATE_KEY_PROPERTY, null); + if (paillierSk == null) { + throw new ConfigException("Must set the paillier private key"); + } + } + + public String getAesPrivateKey() { + return aesPrivateKey; + } + + public void setAesPrivateKey(String aesPrivateKey) { + this.aesPrivateKey = aesPrivateKey; + } + + public String getOrePrivateKey() { + return orePrivateKey; + } + + public void setOrePrivateKey(String orePrivateKey) { + this.orePrivateKey = orePrivateKey; + } + + public String getIhcPrivateKey() { + return ihcPrivateKey; + } + + public void setIhcPrivateKey(String ihcPrivateKey) { + this.ihcPrivateKey = ihcPrivateKey; + } + + public String getAesIv() { + return aesIv; + } + + public void setAesIv(String aesIv) { + this.aesIv = aesIv; + } + + public String getPaillierSk() { + return paillierSk; + } + + public void setPaillierSk(String paillierSk) { + this.paillierSk = paillierSk; + } + + public String getPaillierPk() { + return paillierPk; + } + + public void setPaillierPk(String paillierPk) { + this.paillierPk = paillierPk; + } + + public int getIhcMode() { + return this.ihcMode; + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java new file mode 100644 index 00000000..f804b518 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java @@ -0,0 +1,61 @@ +package com.webank.wedpr.hive.udf.demo; + +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import java.io.File; +import java.io.FileWriter; +import java.math.BigDecimal; +import java.util.Random; +import javax.xml.bind.DatatypeConverter; + +public class GeneratePaillierMain { + + public static void main(String[] args) throws Exception { + String hexPublicKey = + "00000800010100E74CF4A6DD95032412678A2702C6B1C158F0DC94F72191311DC8FA3A2608AAC2F287D3BE7D2583EE7F042B1DBC42C8FE149669D4A36A77A49956D9DFC83DCBD38AB03F7324CB5071EEED8C7FAE108BA4C79639CAFEB4FF96A87EFAAB4580F8CDFCD9FC55BAD63AF2B27BF84B568348228ED7CE8A16A1220CB323301D7F4475A7BAB747477D9056CF9D1FDDE6EBDF4D94814B4A530FB84CC34E36AD0E8DCA220F7E354B4C471B699B155A3463C8A74DE5E824897F519A4BBF01E11FD3F62A349B0604003FB6BC00C1C25173AF4FA610B760114CC55BB92A4173B90A52DE2009C5B18F86725A27B6DA19ED60BD208C5F1875E151CD8DA8D689543C50347A23ED5D0201FF0D6877B3378C46D2839EEF71C78042F99218C2C3E6A3EEB741BAD372E2B5B7FFA222BEBA5C3C18A1B3E4AC890F5AFBAE460607A558743738D11E53EDFA00A54D209A0F52410576ACDD2F28D448B3E48F054A066BEEFA61207E29BB0BDC1EA1EA1B674A94D63D72CC57C55F8250E3EB01F1A4AD42FA14F9EAF92D880CF279DFF50A2AD6FD062B026886FF23BBE310EA1E324BF83B6860A864BD9E28B5DBE5D01173E090827262A65FA7EB0729860808DE9DEB9B2D17B0EABCB3699635EB5B4676AA7E7C533F361FDBE578DA114C6771EEE97894291796A03F1519C77ADBF8697741E48E1DC8E0B15667B617631155E88D9D6781B0FC2D26BD9A121620B2A65ACD099FE494BD887021759E7A16AAA570C881964DF67152E7D578A1751E09B11461A1ACF5FDAEFEAC5410BC5970BE924E56551F7763FD293669F24FAE322931DACDB27A6A865E0F5875A5C26082A329CEDEA47A5F2776DFD2144389AB5EC5358CFAB44B481851ED03292AB79B3E59EF0B0CB8DCE7CEE102E5E0A7597150B297FFEFFF80ED684C1CC5A0DE369EC2B636F1F9B41F4372B4AF662C71F89F0AD456595423CE4B3A38E5998CBB5EA096C9AFAABD9B48DE8BC5819F95C47E7576CAC92CADC4E00175252D6BFD1C5D585442AD5C4285A6878DE9A65D425AF3B61359D9B39C1427923CEB5B82B1CFBF4941DA0454746CD79E30912998A6407F34C29713C210"; + + long startTime1 = System.currentTimeMillis(); + long publicKeyPointer = + NativePaillier.loadPublicKey(DatatypeConverter.parseHexBinary(hexPublicKey)); + long endTime = System.currentTimeMillis(); + System.out.println( + "TestPaillierDemo loadPublicKey cost: " + (endTime - startTime1) + " ms"); + + final NumberCodec codec = new NumberCodecImpl(); + // 创建一个文件 + File file = new File("paillier.csv"); + // 创建一个文件写入器 + FileWriter writer = new FileWriter(file); + // 创建一个随机数生成器 + Random random = new Random(); + int count = Integer.parseInt(args[0]); + // 循环生成100万个随机数 + for (int i = 1; i <= count; i++) { + // 生成一个随机数 + double number = random.nextDouble() * 1000.99999 + 1.99999; + // System.out.println("TestPaillierDemo double number: " + number); + BigDecimal originalValue = new BigDecimal(number); + FloatingPointNumber plainValueFp = codec.encode(originalValue); + byte[] cipher = + NativeFloatingPointPaillier.encrypt( + plainValueFp.getSignificantBytes(), + plainValueFp.getExponent(), + publicKeyPointer); + String hexCipher = DatatypeConverter.printHexBinary(cipher); + // System.out.println("TestPaillierDemo BigDecimal encryptedValue: " + + // hexCipher); + // 将自增id和密文字符串写入文件 + writer.write(i + "," + hexCipher + "\n"); + } + // 关闭文件写入器 + writer.close(); + long startTime2 = System.nanoTime(); + NativePaillier.freePublicKey(publicKeyPointer); + long endTime2 = System.nanoTime(); + System.out.println( + "TestPaillierDemo freePublicKey cost: " + (endTime2 - startTime2) + " ns"); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java new file mode 100644 index 00000000..8c7f9cb5 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java @@ -0,0 +1,11 @@ +package com.webank.wedpr.hive.udf.exceptions; + +public class ConfigException extends RuntimeException { + public ConfigException(String message) { + super(message); + } + + public ConfigException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java new file mode 100644 index 00000000..0a6ca5e7 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java @@ -0,0 +1,11 @@ +package com.webank.wedpr.hive.udf.exceptions; + +public class DecryptionException extends RuntimeException { + public DecryptionException(String message) { + super(message); + } + + public DecryptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java new file mode 100644 index 00000000..b79484a9 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java @@ -0,0 +1,24 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.exceptions; + +public class EncryptionException extends RuntimeException { + public EncryptionException(String message) { + super(message); + } + + public EncryptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java new file mode 100644 index 00000000..aa43c71e --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java @@ -0,0 +1,11 @@ +package com.webank.wedpr.hive.udf.exceptions; + +public class KeyException extends RuntimeException { + public KeyException(String message) { + super(message); + } + + public KeyException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java new file mode 100644 index 00000000..e9d0522e --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java @@ -0,0 +1,74 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ihc; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * @date 2023/11/28 + */ +@Description( + name = "ihc_dec", + value = "迭代希尔密码解密", + extended = + "用法:ihc_dec(columnName, hexSk, iterRound, keyBits) \n" + + "示例:select ihc_dec(salary, '303132...', 10, 64) from t_user") +public class IhcDecryptionUDF extends UDF { + private static final Logger logger = LoggerFactory.getLogger(IhcDecryptionUDF.class); + private final NumberCodec codec = new NumberCodecImpl(); + + public String evaluate(final String cipherValue) { + KeyConfig config = ConfigLoader.config.getKeyConfig(); + return evaluate(config.getIhcMode(), config.getIhcPrivateKey(), cipherValue); + } + /** + * ihc_enc(columnName, hexSk, iterRound, keyBits) + * + * @param cipherValue + * @param hexSk + * @return + */ + public String evaluate(int mode, final String hexSk, final String cipherValue) { + try { + if (cipherValue == null) { + return null; + } + FloatingPointNumber cipherValueFp = + NativeFloatingIhc.decrypt( + mode, + DatatypeConverter.parseHexBinary(hexSk), + DatatypeConverter.parseHexBinary(cipherValue)); + int scale = (cipherValueFp.getExponent() <= 0 ? -cipherValueFp.getExponent() : 0); + return codec.decodeBigDecimal(cipherValueFp) + .setScale(scale, BigDecimal.ROUND_HALF_UP) + .toString(); + } catch (Exception e) { + logger.error("ihc解密失败 ", e); + throw new DecryptionException("ihc解密失败", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java new file mode 100644 index 00000000..90ef0e7f --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java @@ -0,0 +1,74 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ihc; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * @date 2023/11/28 + */ +@Description( + name = "ihc_enc", + value = "迭代希尔密码加密", + extended = + "用法:ihc_enc(columnName, hexSk, iterRound, keyBits) \n" + + "示例:select ihc_enc(salary, '303132...', 10, 64) from t_user") +public class IhcEncryptionUDF extends UDF { + private static final Logger logger = LoggerFactory.getLogger(IhcEncryptionUDF.class); + + private final NumberCodec codec = new NumberCodecImpl(); + + public String evaluate(final String plainValue) { + KeyConfig config = ConfigLoader.config.getKeyConfig(); + return evaluate(config.getIhcMode(), config.getIhcPrivateKey(), plainValue); + } + /** + * ihc_enc(columnName, hexSk, iterRound, keyBits) + * + * @param plainValue + * @param hexSk + * @return + */ + public String evaluate(int mode, final String hexSk, final String plainValue) { + try { + if (plainValue == null) { + return null; + } + FloatingPointNumber plainValueFp = codec.encode(new BigDecimal(plainValue)); + byte[] cipher = + NativeFloatingIhc.encrypt( + mode, + DatatypeConverter.parseHexBinary(hexSk), + plainValueFp.getSignificantBytes(), + plainValueFp.getExponent()); + return DatatypeConverter.printHexBinary(cipher); + } catch (Exception e) { + logger.error("ihc加密失败 ", e); + throw new DecryptionException("ihc加密失败", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java new file mode 100644 index 00000000..aad83e31 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java @@ -0,0 +1,211 @@ +package com.webank.wedpr.hive.udf.impl.ihc; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import java.util.ArrayList; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.serde2.objectinspector.*; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * @date 2023/11/9 + */ +@Description( + name = "ihc_sum", + value = "Returns the sum of a set of ihc ciphertexts", + extended = + "Usage:ihc_sum(columnName) \n" + + "Example:select ihc_sum(encrypted_salary, 64) from t_user\n") +public class IhcSumUDAF extends AbstractGenericUDAFResolver { + static final Logger logger = LoggerFactory.getLogger(IhcSumUDAF.class); + + @Override + public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) + throws SemanticException { + return getEvaluator(info.getParameters()); + } + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException { + int paramsCount = 1; + if (parameters.length != paramsCount) { + throw new UDFArgumentTypeException( + parameters.length - 1, "Exactly one arguments are expected."); + } + + if (((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory() + != PrimitiveObjectInspector.PrimitiveCategory.STRING) { + throw new UDFArgumentTypeException(0, "Only (string, int) type arguments are accepted"); + } + return new IhcSumUDAFEvaluator(); + } + + /** IhcSumUDAFEvaluator. */ + public static class IhcSumUDAFEvaluator extends GenericUDAFEvaluator { + + // Iterate 输入 + private PrimitiveObjectInspector inputOICipherText; + // Merge 输入 + private StructObjectInspector structOI; + + private BinaryObjectInspector sumFieldOI; + + private StructField sumField; + + // TerminatePartial 输出 + private Object[] partialResult; + + // Terminate 输出 + private Text result; + + // the keyConfig + private KeyConfig keyConfig; + + // 中间结果 + static class IhcSumBuffer implements AggregationBuffer { + byte[] sum; + } + + @Override + public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException { + super.init(mode, parameters); + // load keys + keyConfig = ConfigLoader.config.getKeyConfig(); + // 初始化输入参数类型 + if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { + inputOICipherText = (PrimitiveObjectInspector) parameters[0]; + } else { + // Struct + structOI = (StructObjectInspector) parameters[0]; + sumField = structOI.getStructFieldRef("sum"); + sumFieldOI = (BinaryObjectInspector) sumField.getFieldObjectInspector(); + } + // 初始化输出类型 + if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { + // 输出类型 + if (partialResult == null) { + partialResult = new Object[1]; + partialResult[0] = new BytesWritable(); + } + // 输出OI + // 字段类型 + ArrayList objectInspectors = new ArrayList(); + objectInspectors.add(PrimitiveObjectInspectorFactory.writableBinaryObjectInspector); + // 字段名称 + ArrayList fieldNames = new ArrayList(); + fieldNames.add("sum"); + return ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, objectInspectors); + + } else { + result = new Text(); + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + IhcSumBuffer buffer = new IhcSumBuffer(); + reset(buffer); + return buffer; + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + IhcSumBuffer buffer = (IhcSumBuffer) agg; + buffer.sum = null; + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { + try { + if (parameters[0] != null) { + IhcSumBuffer buffer = (IhcSumBuffer) agg; + String cipherText = + PrimitiveObjectInspectorUtils.getString( + parameters[0], inputOICipherText); + byte[] cipherTextBytes = DatatypeConverter.parseHexBinary(cipherText); + if (buffer.sum == null) { + buffer.sum = cipherTextBytes; + } else { + buffer.sum = + NativeFloatingIhc.add( + keyConfig.getIhcMode(), buffer.sum, cipherTextBytes); + } + } + } catch (JniException e) { + logger.error("ihc迭代过程失败", e); + throw new HiveException(e); + } catch (Exception e) { + logger.error("ihc迭代过程失败", e); + throw new HiveException(e); + } + } + + // 输出部分聚合结果 + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException { + IhcSumBuffer buffer = (IhcSumBuffer) agg; + if (buffer.sum == null) { + return null; + } + ((BytesWritable) partialResult[0]).set(new BytesWritable(buffer.sum)); + return partialResult; + } + + // 合并 + @Override + public void merge(AggregationBuffer agg, Object partialResult) throws HiveException { + IhcSumBuffer buffer = (IhcSumBuffer) agg; + if (partialResult == null) { + return; + } + // 通过StandardStructObjectInspector实例，分解出 partialResult 数组元素值 + Object partialSum = structOI.getStructFieldData(partialResult, sumField); + try { + // 通过基本数据类型的OI 实例解析 Object 的值 + byte[] sum = sumFieldOI.getPrimitiveJavaObject(partialSum); + if (buffer.sum == null) { + buffer.sum = sum; + } else { + buffer.sum = NativeFloatingIhc.add(keyConfig.getIhcMode(), buffer.sum, sum); + } + } catch (JniException e) { + logger.error("ihc聚合过程失败", e); + throw new HiveException(e); + } + } + + // 输出最终聚合结果 + @Override + public Object terminate(AggregationBuffer agg) throws HiveException { + IhcSumBuffer buffer = (IhcSumBuffer) agg; + if (buffer.sum == null) { + return null; + } + if (result == null) { + result = new Text(); + } + result.set(DatatypeConverter.printHexBinary(buffer.sum)); + return result; + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java new file mode 100644 index 00000000..a4d81ae6 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java @@ -0,0 +1,61 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ore; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +/** @author caryliao 2023/11/2 */ +@Description( + name = "ore_dec", + value = "decrypt ore cipher to plain value", + extended = + "Usage:ore_dec(columnName, hexSk) \n" + + "Example:select ore_dec(encrypted_name, '303132...') as user_name from t_user") +public class OreDecryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(OreDecryptionUDF.class.getName()); + + public String evaluate(final String hexCipher) { + if (hexCipher == null) { + return null; + } + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + return evaluate(hexCipher, keyConfig.getOrePrivateKey()); + } + + public String evaluate(final String hexCipher, String hexSk) { + try { + if (hexCipher == null) { + return null; + } + return new String( + FastOre.decrypt4String( + DatatypeConverter.parseHexBinary(hexSk), + hexCipher.getBytes(Charsets.UTF_8), + true), + Charsets.UTF_8) + .trim(); + } catch (Exception e) { + logger.error("OreDecryptionUDF jni fail ", e); + throw new DecryptionException("ore Decrypt jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java new file mode 100644 index 00000000..6e4bd854 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java @@ -0,0 +1,57 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ore; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.EncryptionException; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +@Description( + name = "ore_enc", + value = "encrypt ore plain-text to cipher", + extended = + "Usage:ore_enc(columnName, hexSk) \n" + + "Example:select ore_enc(encrypted_name, '303132...') as user_name from t_user") +public class OreEncryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(OreEncryptionUDF.class.getName()); + + public String evaluate(final String plainData) { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + return evaluate(plainData, keyConfig.getOrePrivateKey()); + } + + public String evaluate(final String plainData, String hexSk) { + try { + if (plainData == null) { + return null; + } + return new String( + FastOre.encrypt4String( + DatatypeConverter.parseHexBinary(hexSk), + plainData.getBytes(Charsets.UTF_8), + true), + Charsets.UTF_8) + .trim(); + } catch (Exception e) { + logger.warn("ORE Encryption exception", e); + throw new EncryptionException("ORE Encryption exception", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java new file mode 100644 index 00000000..e5509f47 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java @@ -0,0 +1,48 @@ +/** + * Copyright 2024 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ore; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +public class OreNumberDecryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(OreNumberDecryptionUDF.class.getName()); + private static final NumberCodec codec = new NumberCodecImpl(); + + public String evaluate(String cipher) { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + try { + if (cipher == null) { + return null; + } + return codec.toBigDecimal( + FastOre.decrypt4Float( + DatatypeConverter.parseHexBinary(keyConfig.getOrePrivateKey()), + cipher.getBytes(Charsets.UTF_8), + true)) + .toPlainString(); + } catch (Exception e) { + logger.error("OreNumberDecryptionUDF jni fail ", e); + throw new DecryptionException("ore Decrypt jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java new file mode 100644 index 00000000..b6f8a6a4 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java @@ -0,0 +1,72 @@ +/** + * Copyright 2024 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.ore; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.crypto.FastOre; +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +public class OreNumberEncryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(OreNumberEncryptionUDF.class.getName()); + private static final NumberCodec codec = new NumberCodecImpl(); + + public String evaluate(final BigDecimal value) { + if (value == null) { + return null; + } + return evaluate(codec.toBytes(value)); + } + + public String evaluate(final Double value) { + if (value == null) { + return null; + } + return evaluate(codec.toBytes(value)); + } + + public String evaluate(final String value) { + if (value == null) { + return null; + } + return evaluate(value.getBytes(Charsets.UTF_8)); + } + + public String evaluate(final byte[] plainBytes) { + + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + try { + if (plainBytes == null) { + return null; + } + return new String( + FastOre.encrypt4Float( + DatatypeConverter.parseHexBinary(keyConfig.getOrePrivateKey()), + plainBytes, + true), + Charsets.UTF_8) + .trim(); + } catch (Exception e) { + logger.error("OreNumberEncryptionUDFNumnerNumner jni fail ", e); + throw new DecryptionException("ore Decrypt jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java new file mode 100644 index 00000000..798fb21f --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java @@ -0,0 +1,83 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.paillier; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.hive.udf.exceptions.KeyException; +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * @date 2023/10/31 + */ +@Description( + name = "paillier_dec", + value = "decrypt paillier cipher to plain value", + extended = + "Usage:paillier_dec(columnName) \n" + + "Example:select paillier_dec(encrypted_salary)") +public class PaillierDecryptionUDF extends UDF { + private static final Logger logger = LoggerFactory.getLogger(PaillierDecryptionUDF.class); + + private static long keyPairPointer = 0L; + private static final NumberCodec codec = new NumberCodecImpl(); + + static { + try { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + keyPairPointer = + NativePaillier.loadKeyPair( + DatatypeConverter.parseHexBinary(keyConfig.getPaillierSk()), + DatatypeConverter.parseHexBinary(keyConfig.getPaillierPk())); + } catch (Exception e) { + throw new KeyException("Load key pair for paillier error", e); + } + } + /** + * paillier_dec(columnName) + * + * @param hexCipher + * @return + */ + public String evaluate(final Text hexCipher) { + if (hexCipher == null) { + return null; + } + try { + FloatingPointNumber cipherValueFp = + NativeFloatingPointPaillier.decrypt( + DatatypeConverter.parseHexBinary(hexCipher.toString()), keyPairPointer); + int scale = (cipherValueFp.getExponent() <= 0 ? -cipherValueFp.getExponent() : 0); + return codec.decodeBigDecimal(cipherValueFp) + .setScale(scale, BigDecimal.ROUND_HALF_UP) + .toString(); + } catch (Exception e) { + logger.error("Paillier Decrypt jni fail ", e); + throw new DecryptionException("Paillier Decrypt jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java new file mode 100644 index 00000000..259a205b --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java @@ -0,0 +1,86 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.paillier; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.EncryptionException; +import com.webank.wedpr.hive.udf.exceptions.KeyException; +import com.webank.wedpr.sdk.jni.codec.FloatingPointNumber; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecException; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * @date 2023/12/2 + */ +@Description( + name = "paillier_enc", + value = "encrypt plain value to paillier cipher", + extended = + "Usage:paillier_enc(columnName, hexPk) \n" + + "Example:select paillier_enc(encrypted_salary,'0000080001...')") +public class PaillierEncryptionUDF extends UDF { + private static final Logger logger = LoggerFactory.getLogger(PaillierEncryptionUDF.class); + private static long publicKeyPointer = 0L; + private static final NumberCodec codec = new NumberCodecImpl(); + + static { + try { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + publicKeyPointer = + NativePaillier.loadPublicKey( + DatatypeConverter.parseHexBinary(keyConfig.getPaillierPk())); + } catch (Exception e) { + throw new KeyException( + "Load public key for paillier exception, error: " + e.getMessage(), e); + } + } + + /** + * paillier_enc(columnName, hexPk) + * + * @param plainValue + * @return + */ + public Text evaluate(final Text plainValue) { + if (plainValue == null) { + return null; + } + try { + BigDecimal value = new BigDecimal(plainValue.toString()); + FloatingPointNumber plainValueFp = codec.encode(value); + byte[] cipherValue = + NativeFloatingPointPaillier.encrypt( + plainValueFp.getSignificantBytes(), + plainValueFp.getExponent(), + publicKeyPointer); + return new Text(DatatypeConverter.printHexBinary(cipherValue)); + } catch (NumberCodecException | JniException e) { + logger.error("Paillier Encrypt BigDecimal jni fail ", e); + throw new EncryptionException("WeFec Paillier Encrypt BigDecimal jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java new file mode 100644 index 00000000..f1c15846 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java @@ -0,0 +1,219 @@ +package com.webank.wedpr.hive.udf.impl.paillier; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.KeyException; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import com.webank.wedpr.sdk.jni.homo.NativePaillier; +import java.util.ArrayList; +import javax.xml.bind.DatatypeConverter; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.serde2.objectinspector.*; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** @author caryliao 2023/11/9 */ +@Description( + name = "paillier_sum", + value = "Returns the sum of a set of paillier ciphertexts", + extended = + "Usage:paillier_sum(columnName, hexPk) \n" + + "Example:select paillier_sum(encrypted_salary,'0000080001...')") +public class PaillierSumUDAF extends AbstractGenericUDAFResolver { + static final Logger logger = LoggerFactory.getLogger(PaillierSumUDAF.class); + + private static long publicKeyPointer = 0L; + + static { + try { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + publicKeyPointer = + NativePaillier.loadPublicKey( + DatatypeConverter.parseHexBinary(keyConfig.getPaillierPk())); + } catch (Exception e) { + throw new KeyException( + "Load public key for paillier exception, error: " + e.getMessage(), e); + } + } + + @Override + public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) + throws SemanticException { + return getEvaluator(info.getParameters()); + } + + @Override + public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException { + int paramsCount = 1; + if (parameters.length != paramsCount) { + throw new UDFArgumentTypeException( + parameters.length - 1, "Exactly one arguments are expected."); + } + + if (((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory() + != PrimitiveObjectInspector.PrimitiveCategory.STRING) { + throw new UDFArgumentTypeException(0, "Only string type arguments are accepted"); + } + return new PaillierSumUDAFEvaluator(); + } + + /** PaillierSumUDAFEvaluator. */ + public static class PaillierSumUDAFEvaluator extends GenericUDAFEvaluator { + + // Iterate 输入 + private PrimitiveObjectInspector inputOICipherText; + // Merge 输入 + private StructObjectInspector structOI; + + private BinaryObjectInspector sumFieldOI; + + private StructField sumField; + + // TerminatePartial 输出 + private Object[] partialResult; + + // Terminate 输出 + private Text result; + + // 中间结果 + static class PaillierSumBuffer implements AggregationBuffer { + byte[] sum; + } + + @Override + public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException { + super.init(mode, parameters); + // 初始化输入参数类型 + if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { + inputOICipherText = (PrimitiveObjectInspector) parameters[0]; + } else { + // Struct + structOI = (StructObjectInspector) parameters[0]; + sumField = structOI.getStructFieldRef("sum"); + sumFieldOI = (BinaryObjectInspector) sumField.getFieldObjectInspector(); + } + // 初始化输出类型 + if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) { + // 输出类型 + if (partialResult == null) { + partialResult = new Object[1]; + partialResult[0] = new BytesWritable(); + } + // 输出OI + // 字段类型 + ArrayList objectInspectors = new ArrayList(); + objectInspectors.add(PrimitiveObjectInspectorFactory.writableBinaryObjectInspector); + // 字段名称 + ArrayList fieldNames = new ArrayList(); + fieldNames.add("sum"); + return ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, objectInspectors); + + } else { + result = new Text(); + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + } + } + + @Override + public AggregationBuffer getNewAggregationBuffer() throws HiveException { + PaillierSumBuffer buffer = new PaillierSumBuffer(); + reset(buffer); + return buffer; + } + + @Override + public void reset(AggregationBuffer agg) throws HiveException { + PaillierSumBuffer buffer = (PaillierSumBuffer) agg; + buffer.sum = null; + } + + @Override + public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { + try { + if (parameters[0] != null) { + PaillierSumBuffer buffer = (PaillierSumBuffer) agg; + String cipherText = + PrimitiveObjectInspectorUtils.getString( + parameters[0], inputOICipherText); + byte[] cipherTextBytes = DatatypeConverter.parseHexBinary(cipherText); + if (buffer.sum == null) { + buffer.sum = cipherTextBytes; + } else { + buffer.sum = + NativeFloatingPointPaillier.add( + buffer.sum, cipherTextBytes, publicKeyPointer); + } + } + } catch (JniException e) { + logger.error("PaillierSumUDAFEvaluator iterate error", e); + throw new HiveException(e); + } catch (Exception e) { + logger.error("PaillierSumUDAFEvaluator iterate error", e); + throw new HiveException(e); + } + } + + // 输出部分聚合结果 + @Override + public Object terminatePartial(AggregationBuffer agg) throws HiveException { + PaillierSumBuffer buffer = (PaillierSumBuffer) agg; + if (buffer.sum == null) { + return null; + } + ((BytesWritable) partialResult[0]).set(new BytesWritable(buffer.sum)); + return partialResult; + } + + // 合并 + @Override + public void merge(AggregationBuffer agg, Object partialResult) throws HiveException { + PaillierSumBuffer buffer = (PaillierSumBuffer) agg; + if (partialResult == null) { + return; + } + // 通过StandardStructObjectInspector实例，分解出 partialResult 数组元素值 + Object partialSum = structOI.getStructFieldData(partialResult, sumField); + try { + // 通过基本数据类型的OI 实例解析 Object 的值 + byte[] sum = sumFieldOI.getPrimitiveJavaObject(partialSum); + if (buffer.sum == null) { + buffer.sum = sum; + } else { + buffer.sum = NativeFloatingPointPaillier.add(buffer.sum, sum, publicKeyPointer); + } + } catch (JniException e) { + logger.error("PaillierSumUDAFEvaluator merge error", e); + throw new HiveException(e); + } + } + + // 输出最终聚合结果 + @Override + public Object terminate(AggregationBuffer agg) throws HiveException { + PaillierSumBuffer buffer = (PaillierSumBuffer) agg; + if (buffer.sum == null) { + return null; + } + if (result == null) { + result = new Text(); + } + result.set(DatatypeConverter.printHexBinary(buffer.sum)); + return result; + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java new file mode 100644 index 00000000..fa08563b --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java @@ -0,0 +1,73 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.symmetric; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +/** @author caryliao 2023/11/1 */ +@Description( + name = "sym_dec", + value = "decrypt symmetric cipher to plain value", + extended = + "Usage:sym_dec(columnName, algorithmType, mode, hexIv, hexSk) \n" + + "algorithmType(0:AES_128, 1:AES_192, 2:AES_256, 3:TrippleDES, 4:SM4), mode(0:ECB, 1:CBC, 2:CFB, 3:OFB, 4:CTR)\n" + + "Example:select sym_dec(encrypted_name, 0, 0,'303132...', 'ce73b0...') as user_name from t_user") +public class SymDecryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(SymDecryptionUDF.class.getName()); + + public String evaluate(final String hexCipher) { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + return evaluate( + hexCipher, + SymmetricEncryption.AlgorithmType.AES_256.ordinal(), + SymmetricEncryption.OperationMode.CBC.ordinal(), + keyConfig.getAesIv(), + keyConfig.getAesPrivateKey()); + } + /* @param hexCipher + * @param algorithmType + * @param mode + * @param hexIv + * @param hexSk + * @return + */ + public String evaluate( + final String hexCipher, int algorithmType, int mode, String hexIv, String hexSk) { + try { + if (hexCipher == null) { + return null; + } + return new String( + SymmetricEncryption.decrypt( + algorithmType, + mode, + DatatypeConverter.parseHexBinary(hexSk), + DatatypeConverter.parseHexBinary(hexIv), + DatatypeConverter.parseHexBinary(hexCipher)), + Charsets.UTF_8) + .trim(); + } catch (Exception e) { + logger.error("Sym Decrypt fail ", e); + throw new DecryptionException("Sym Decrypt jni fail ", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java new file mode 100644 index 00000000..a6e67656 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java @@ -0,0 +1,83 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.hive.udf.impl.symmetric; + +import com.webank.wedpr.hive.udf.config.ConfigLoader; +import com.webank.wedpr.hive.udf.config.model.KeyConfig; +import com.webank.wedpr.hive.udf.exceptions.EncryptionException; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; +import javax.xml.bind.DatatypeConverter; +import org.apache.commons.codec.Charsets; +import org.apache.commons.codec.binary.Hex; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDF; +import org.apache.log4j.Logger; + +@Description( + name = "sym_enc", + value = "encrypt the plain-text into hex-cipher-data", + extended = + "Usage:sym_enc(columnName, algorithmType, mode, hexIv, hexSk) \n" + + "algorithmType(0:AES_128, 1:AES_192, 2:AES_256, 3:TrippleDES, 4:SM4), mode(0:ECB, 1:CBC, 2:CFB, 3:OFB, 4:CTR)\n" + + "Example: " + + "from t_user1 t1" + + "insert into t_user partition(dt=\"20231011\") select sym_enc(t1.id, 0, 0,'303132...', 'ce73b0...'),... ") +public class SymEncryptionUDF extends UDF { + private static final Logger logger = Logger.getLogger(SymEncryptionUDF.class.getName()); + + public String evaluate(final String plainData) { + KeyConfig keyConfig = ConfigLoader.config.getKeyConfig(); + return evaluate( + plainData, + SymmetricEncryption.AlgorithmType.AES_256.ordinal(), + SymmetricEncryption.OperationMode.CBC.ordinal(), + keyConfig.getAesIv(), + keyConfig.getAesPrivateKey()); + } + + public String evaluate( + final String plainData, + int algorithmType, + int mode, + String hexIv, + String hexPrivateKey) { + if (plainData == null) { + return null; + } + if (hexPrivateKey == null) { + logger.warn("Invalid empty private key"); + return plainData; + } + try { + return Hex.encodeHexString( + SymmetricEncryption.encrypt( + algorithmType, + mode, + DatatypeConverter.parseHexBinary(hexPrivateKey), + DatatypeConverter.parseHexBinary(hexIv), + plainData.getBytes(Charsets.UTF_8))) + .trim(); + } catch (JniException e) { + logger.warn( + "Symmetric Encryption exception, algorithm: " + + algorithmType + + " , mode: " + + mode + + ", exception: " + + e); + throw new EncryptionException("Symmetric Encryption exception", e); + } + } +} diff --git a/cpp/ppc-udf/hive/src/main/resources/config.toml b/cpp/ppc-udf/hive/src/main/resources/config.toml new file mode 100644 index 00000000..39e1b909 --- /dev/null +++ b/cpp/ppc-udf/hive/src/main/resources/config.toml @@ -0,0 +1,7 @@ +[keys] +aes_sk = "AEAC6BCB142F8945E57935D1506E3A9D" +aes_iv = "0123456789" +ore_sk = "A9F524FC2BC9CDCCC6C76EC26238D5E4" +ihc_sk = "436BBF943B43449B16BE19D90ED558F5" +paillier_sk = "000008000101006EFDA25C7DF1F497DBEEA2F08ADFD030066A4BC0ABD9905FA128BDFC81780D2936EAC40299B93F222675C75B0C57107EA034744C352C2B3EC07D2943A974A1F602A0AAA7FC382E6F0AFFFC4B878F007D80FDF03028502416D8FCABBED033EBFDF0A3EE552D41F167E52F1CFEE4990CDD003FF582CF18D095CB71A0212B80DB9D73A8451F5491F7F34CA523222CCBF5CB9FC2753D45C5CE327F3D9DDCC5E79C57BC956B4E84214CC37B048AD631F1DFA9EC0CCFACF16FD5B5EC804CCBBC46449CFFBE7D8DCA9577E643814389EFFEE62D8F4320D9E246CFB3B20464B74014A2FFFBBA0223514CBE479620A0F8D941DECB19B46497EA0B8CBBBDB42C0C0AE75432008100E8219B3E57789D4EC59D133446132F5B1697D5134F0B24967E5C99A63740F6904F159ED8AE4C59421C053E0225EBDB1929CD7A9D2B43FB86D02C8DADEE8F5570AFA2D5A2D0842832A1CB233CF0CE3F699FE8F29CE1326DA91AD02156E7E076A5B6CB2EB6F4B9828F4AA37D23F9D2772FD41634CDCC43DE94EC2401E3D31C5DFF008100F4CE7D67EC0992CAC78169EC32180EFC8E99C0DED88C1332E29B2895EC4F59B0339D335694F93A8262F2A48A9AFC1F8F7E9142F834761E3AA9FAE910C298CAF4B5D4A972BFB44A5CFF66814EE4785231BAE80DDCB65F7FF6B807D31CB12FE3901274019DDBD17BD9E257F5496D81F2B602BDB17FF1055B460B3D2946CD757DCF" +paillier_pk = "00000800010100DDFB44B8FBE3E92FB7DD45E115BFA0600CD4978157B320BF42517BF902F01A526DD5880533727E444CEB8EB618AE20FD4068E8986A58567D80FA528752E943EC0541554FF8705CDE15FFF8970F1E00FB01FBE06050A0482DB1F9577DA067D7FBE147DCAA5A83E2CFCA5E39FDC93219BA007FEB059E31A12B96E340425701B73CC440A2E4ECA620002668C364D1C329EEE4B6806CB322D42E5F72FDF5AF5F88EFFBDDA8CC4B882D4B7500F83924CBB9FC80785CEF4299C52D5328105629B4A99F64F47A312563625C28342B9FB5445DF67957422D5C1F8D0736E0BDE21939A035C0B3349B73247AF8593CB45F19D8277C0A3CAF7D9160535272C98342B66084310201FF0E06E2102360C675C143D2E0543D7A35FE8A9169368ADDE79082AD93E6EB62495A751FB46CAFE23C5990FE1BD8EDF894AB049A1970B6BD6E2B48BA3DD68ADE3FB916BDACB9948CDC03168E30EB9819A8A341AB4B5FFD9276388303F5771271AFFEEC2C3D9BED3354DF51971AC5E0EDB62A5F0796E29FB48C8DDB47B184F183CAD9751A22303D4D1C7415390B9F4E4954C226DAD2935801C64F605E3DE12607469408FEAEA3BC6AA277E2811EFBA39C50931B8D665CC35AF4D2E8A4DC62DE6B5087991AAB5C64ED389C66CF2096C4870F8BF893E73891F0E263F2181DB2B89C5F10A4D8C16FDE25FAFFF744B3517A3F13D3E3E0C9DD96F59775236E4986C19677813FB3914338F51620E8679C7CA64BAFF1C7C6CD0216874FF82EC403AE7A47FBD542724E49F2897A7CCA5F7D2340C946534BD45BB483EE56244BE5E61FBF669DB7024151E33D60AB50E44BBA9DD91584E67409F7240EAC4234D8AD6D2FD2CF41C81E3BEB894D593177B8DC162B77694FEBDDA2248E6603BF11DCF1C963F870F1A35767179E3112DED335E9D187B0A197667C47628B4718A326D754F6415B4A6F1DC4F99EFCFEA9A5F99DF56C9BB911825B7AF7308FF0BF595EB1B6E60FA864B73F3C8E82247567867470737BC1431FF5808B00B52C2A025D741E72A94594285BEBE9986C1A0FB13606C00F7CC6061EE141BEEE3E2DA9E740C4C23EC1FB778440" \ No newline at end of file diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java new file mode 100644 index 00000000..84df8b3e --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java @@ -0,0 +1,60 @@ +package com.webank.wedpr.hive.udf.impl; + +import java.math.BigDecimal; +import javax.xml.bind.DatatypeConverter; + +import com.webank.wedpr.hive.udf.impl.ihc.IhcDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcEncryptionUDF; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.webank.wedpr.hive.udf.impl.ihc.IhcDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcEncryptionUDF; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; + +/** + * @author caryliao + * @date 2023/11/29 + */ + +public class IhcEncAndDecUDFTest { + private static final Logger logger = LoggerFactory.getLogger(IhcEncAndDecUDFTest.class); + + public void testEncAndDecEvaluateImpl(int mode) throws JniException { + IhcEncryptionUDF ihcEncUDF = new IhcEncryptionUDF();; + int keyBits = 64; + String plainValue = "10000"; + String hexSk = "1F272ECA7DED4105"; + int iterRound = 16; + String cipherValue = ihcEncUDF.evaluate(mode, hexSk, plainValue); + System.out.println("cipherValue:" + cipherValue); + + IhcDecryptionUDF ihcDecUDF = new IhcDecryptionUDF(); + String decryptValueStr = ihcDecUDF.evaluate(mode, hexSk, cipherValue); + System.out.println("decryptValueStr:" + decryptValueStr); + BigDecimal decryptValue = new BigDecimal(decryptValueStr); + System.out.println("decryptValue:" + decryptValue); + byte[] cipherValueBytes = DatatypeConverter.parseHexBinary(cipherValue); + byte[] cipherValueSum = cipherValueBytes; + int addRound = 10; + long startTime = System.currentTimeMillis(); + for (int i = 0; i < addRound - 1; i++) { + cipherValueSum = NativeFloatingIhc.add(mode, cipherValueSum, cipherValueBytes); + } + long endTime = System.currentTimeMillis(); + System.out.println("NativeIhc.add cost time(ms):" + (endTime - startTime)); + String decryptValueSumStr = ihcDecUDF.evaluate(mode, hexSk, DatatypeConverter.printHexBinary(cipherValueSum)); + System.out.println("decryptValueSumStr:" + decryptValueSumStr); + Assert.assertTrue(new BigDecimal(plainValue).compareTo(decryptValue) == 0); + } + + @Test + public void testEncAndDecEvaluate() throws JniException { + testEncAndDecEvaluateImpl(NativeFloatingIhc.mode.IHC_256.ordinal()); + testEncAndDecEvaluateImpl(NativeFloatingIhc.mode.IHC_128.ordinal()); + } +} diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java new file mode 100644 index 00000000..06dbe2c1 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java @@ -0,0 +1,78 @@ +package com.webank.wedpr.hive.udf.impl; + +import java.math.BigDecimal; + +import com.webank.wedpr.hive.udf.impl.ihc.IhcDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcEncryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcSumUDAF; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingIhc; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaConstantStringObjectInspector; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.webank.wedpr.hive.udf.impl.ihc.IhcDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcEncryptionUDF; +import com.webank.wedpr.hive.udf.impl.ihc.IhcSumUDAF; + +import javax.xml.bind.DatatypeConverter; + +/** + * @author caryliao + * @date 2023/11/30 + */ + +public class IhcSumUDAFTest { + private static final Logger logger = LoggerFactory.getLogger(IhcSumUDAFTest.class); + + @Test + public void testIhcSumUDAFEvaluate() throws HiveException, JniException + { + //testIhcSumUDAFEvaluateImpl(NativeFloatingIhc.mode.IHC_256.ordinal()); + testIhcSumUDAFEvaluateImpl(NativeFloatingIhc.mode.IHC_128.ordinal()); + } + + public void testIhcSumUDAFEvaluateImpl(int mode) throws HiveException, JniException { + IhcEncryptionUDF ihcEncUDF = new IhcEncryptionUDF(); + String plainValue1 = "111.11"; + String plainValue2 = "222.22"; + String hexSk = DatatypeConverter.printHexBinary(NativeFloatingIhc.generateKey(mode)); + String cipherValue1 = ihcEncUDF.evaluate(mode, hexSk, plainValue1); + String cipherValue2 = ihcEncUDF.evaluate(mode, hexSk, plainValue2); + System.out.println("cipherValue1:" + cipherValue1); + System.out.println("cipherValue2:" + cipherValue2); + + IhcSumUDAF ihcSumUDAF = new IhcSumUDAF(); + JavaConstantStringObjectInspector stringObjectInspector = new JavaConstantStringObjectInspector(cipherValue1); + ObjectInspector[] params = {stringObjectInspector}; + GenericUDAFParameterInfo parameterInfo = new SimpleGenericUDAFParameterInfo(params, false, false, false); + GenericUDAFEvaluator evaluator = ihcSumUDAF.getEvaluator(parameterInfo); + evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, params); + GenericUDAFEvaluator.AggregationBuffer agg = evaluator.getNewAggregationBuffer(); + Object[] iterateParams1 = {cipherValue1}; + evaluator.iterate(agg, iterateParams1); + evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, params); + Object[] iterateParams2 = {cipherValue2}; + evaluator.iterate(agg, iterateParams2); + Text result = (Text) evaluator.terminate(agg); + System.out.println("result:" + result.toString()); + + long startTime = System.nanoTime(); + IhcDecryptionUDF ihcDecUDF = new IhcDecryptionUDF(); + String sumValueStr = ihcDecUDF.evaluate(mode, hexSk, result.toString()); + BigDecimal sumValue = new BigDecimal(sumValueStr); + long endTime = System.nanoTime(); + System.out.println("paillierSumUDF cost time(ns):" + (endTime - startTime)); + System.out.println("sumValue:" + sumValue); + Assert.assertTrue(new BigDecimal("333.33").compareTo(sumValue) == 0);; + } + +} + diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java new file mode 100644 index 00000000..e09d5f1e --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java @@ -0,0 +1,138 @@ +package com.webank.wedpr.hive.udf.impl; +import java.nio.charset.StandardCharsets; + +import javax.xml.bind.DatatypeConverter; + +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.hive.udf.impl.ore.OreDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ore.OreEncryptionUDF; +import com.webank.wedpr.hive.udf.impl.ore.OreNumberDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.ore.OreNumberEncryptionUDF; +import com.webank.wedpr.sdk.jni.codec.NumberCodec; +import com.webank.wedpr.sdk.jni.codec.NumberCodecImpl; +import org.apache.commons.codec.Charsets; +import org.apache.commons.codec.binary.Hex; +import org.junit.Assert; +import org.junit.Test; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.crypto.FastOre; + +/** + * @author caryliao + */ + +public class OreDecUDFTest { + + @Test + public void testOREDecUDFEvaluate() throws JniException { + String plainValue = "19014527865"; + String hexSk = "267C74CB4AFF39E0D59CB177CA5E495A"; + byte[] bytesSk = DatatypeConverter.parseHexBinary(hexSk); + byte[] bytesCipher = FastOre.encrypt4String(bytesSk, plainValue.getBytes(StandardCharsets.UTF_8), true); + System.out.println("hexCipher: " + new String(bytesCipher)); + + OreDecryptionUDF oreDecUDF = new OreDecryptionUDF(); + long startTime = System.nanoTime(); + String decryptValueStr = oreDecUDF.evaluate(new String(bytesCipher), hexSk); + long endTime = System.nanoTime(); + System.out.println("OREDecUDF cost time(ns):" + (endTime - startTime)); + System.out.println("decryptValueStr:" + decryptValueStr); + Assert.assertEquals(plainValue,decryptValueStr); + } + + public void testOREDecUDFEvaluateImpl2(String plainValue, String hexSk, byte[] cipher){ + OreDecryptionUDF oreDecUDF = new OreDecryptionUDF(); + String decryptValueStr = oreDecUDF.evaluate(new String(cipher), hexSk); + System.out.println("decryptValueStr:" + decryptValueStr); + System.out.println("##### decryptValueStr: " + decryptValueStr); + System.out.println("##### plainValue:" + plainValue); + Assert.assertEquals(plainValue, decryptValueStr); + } + + public void checkInvalidCase( String hexSk, byte[] cipher) + { + OreDecryptionUDF oreDecUDF = new OreDecryptionUDF(); + Assert.assertThrows(DecryptionException.class, ()-> oreDecUDF.evaluate(new String(cipher), hexSk)); + } + + @Test + public void testOREDecUDFEvaluate3() throws JniException{ + String plainValue = "456_world"; + String hexSk = "267C74CB4AFF39E0D59CB177CA5E495A"; + testOreEncDecImpl(plainValue, hexSk); + + plainValue = "202.103.123.99"; + String cipher = "af067bca0937b7420a60e45de9831bd5b70ae0f950b6d748c470da10"; + testOREDecUDFEvaluateImpl2(plainValue, hexSk, cipher.getBytes(Charsets.UTF_8)); + plainValue = "192.168.19.12"; + cipher = "af05856a300f0bee890821b3a560af45b537a7e561b606252eed"; + testOREDecUDFEvaluateImpl2(plainValue, hexSk, cipher.getBytes(Charsets.UTF_8)); + + checkInvalidCase("abc", cipher.getBytes(Charsets.UTF_8)); + cipher = "abcwsd"; + checkInvalidCase(hexSk, cipher.getBytes(Charsets.UTF_8)); + } + + public void testOreEncDecImpl(String plainData, String hexSk) { + OreEncryptionUDF oreEncUDF = new OreEncryptionUDF(); + OreDecryptionUDF oreDecUDF = new OreDecryptionUDF(); + + String tmpCipher = ""; + for (int i = 0; i < 10; i++) + { + String cipher = oreEncUDF.evaluate(plainData, hexSk); + if(i >=1) + { + Assert.assertEquals(cipher, tmpCipher); + } + tmpCipher = cipher; + String plain = oreDecUDF.evaluate(cipher, hexSk); + Assert.assertEquals(plain, plainData); + } + } + + @Test + public void testOreEncDecEvaluate() throws JniException + { + byte[] sk = FastOre.generateKey(); + String hexSk = Hex.encodeHexString(sk); + testOreEncDecImpl("abc", hexSk); + testOreEncDecImpl("中国werw34#$%546!!@3中国中文", hexSk); + testOreEncDecImpl("192.168.19.12", hexSk); + testOreEncDecImpl("202.103.123.99", hexSk); + } + + + public void testOreNumberEncDecImpl(String plainData) { + OreNumberEncryptionUDF oreEncUDF = new OreNumberEncryptionUDF(); + OreNumberDecryptionUDF oreDecUDF = new OreNumberDecryptionUDF(); + + String tmpCipher = ""; + for (int i = 0; i < 10; i++) + { + String cipher = oreEncUDF.evaluate(plainData); + if(i >=1) + { + Assert.assertEquals(cipher, tmpCipher); + } + tmpCipher = cipher; + String plain = oreDecUDF.evaluate(cipher); + System.out.println("#### plain: " + plain); + System.out.println("#### plainData: " + plainData); + Assert.assertEquals(plain, plainData); + } + } + + @Test + public void testOreNumberEncDecEvaluate() throws JniException + { + testOreNumberEncDecImpl("12332432423.23432444444444"); + testOreNumberEncDecImpl("-12332432423.23432444444444"); + testOreNumberEncDecImpl("12332432423"); + testOreNumberEncDecImpl("-12332432423"); + testOreNumberEncDecImpl("0"); + testOreNumberEncDecImpl("0.000000000000000000000000000000001"); + testOreNumberEncDecImpl("-0.000000000000000000000000000000001"); + } +} diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java new file mode 100644 index 00000000..84773780 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java @@ -0,0 +1,41 @@ +package com.webank.wedpr.hive.udf.impl; + +import java.math.BigDecimal; + +import com.webank.wedpr.hive.udf.impl.paillier.PaillierDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.paillier.PaillierEncryptionUDF; +import com.webank.wedpr.sdk.jni.homo.NativeFloatingPointPaillier; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; + +/** + * @author caryliao + */ + +public class PaillierDecUDFTest { + @Test + public void testPaillierDecryptionUDFEvaluate() { + String plain = "1600.7568000000001120497472584247589111328125"; + PaillierEncryptionUDF paillierEncryptionUDF = new PaillierEncryptionUDF(); + Text hexCipher = paillierEncryptionUDF.evaluate(new Text(plain)); + + PaillierDecryptionUDF paillierDecUDF = new PaillierDecryptionUDF(); + long startTime = System.nanoTime(); + String decryptValueStr = + paillierDecUDF.evaluate(hexCipher); + long endTime = System.nanoTime(); + System.out.println("paillierDecUDF cost time(ns):" + (endTime - startTime)); + String decryptValueStr2 = + paillierDecUDF.evaluate(hexCipher); + System.out.println("decryptValueStr:" + decryptValueStr); + System.out.println("decryptValueStr2:" + decryptValueStr2); + BigDecimal decryptValue = new BigDecimal(decryptValueStr); + BigDecimal decryptValue2 = new BigDecimal(decryptValueStr2); + System.out.println("##### decryptValue: " + decryptValueStr); + System.out.println("##### decryptValue2: " + decryptValueStr2); + Assert.assertTrue(new BigDecimal(plain).compareTo(decryptValue) == 0); + Assert.assertTrue(new BigDecimal(plain).compareTo(decryptValue2) == 0); + } + +} diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java new file mode 100644 index 00000000..a35fa3b4 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java @@ -0,0 +1,46 @@ +package com.webank.wedpr.hive.udf; + +import java.math.BigDecimal; + +import com.webank.wedpr.hive.udf.impl.paillier.PaillierDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.paillier.PaillierEncryptionUDF; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.webank.wedpr.hive.udf.impl.paillier.PaillierDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.paillier.PaillierEncryptionUDF; + +/** + * @author caryliao + * @date 2023/12/02 + */ + +public class PaillierEncUDFTest { + private static final Logger logger = LoggerFactory.getLogger(PaillierEncUDFTest.class); + + @Test + public void testPaillierDecryptionUDFEvaluate() { + String plainValue = "1000.123"; + PaillierEncryptionUDF paillierEncUDF = new PaillierEncryptionUDF(); + PaillierDecryptionUDF paillierDecUDF = new PaillierDecryptionUDF(); + String hexPublicKey = + "00000800010100E74CF4A6DD95032412678A2702C6B1C158F0DC94F72191311DC8FA3A2608AAC2F287D3BE7D2583EE7F042B1DBC42C8FE149669D4A36A77A49956D9DFC83DCBD38AB03F7324CB5071EEED8C7FAE108BA4C79639CAFEB4FF96A87EFAAB4580F8CDFCD9FC55BAD63AF2B27BF84B568348228ED7CE8A16A1220CB323301D7F4475A7BAB747477D9056CF9D1FDDE6EBDF4D94814B4A530FB84CC34E36AD0E8DCA220F7E354B4C471B699B155A3463C8A74DE5E824897F519A4BBF01E11FD3F62A349B0604003FB6BC00C1C25173AF4FA610B760114CC55BB92A4173B90A52DE2009C5B18F86725A27B6DA19ED60BD208C5F1875E151CD8DA8D689543C50347A23ED5D0201FF0D6877B3378C46D2839EEF71C78042F99218C2C3E6A3EEB741BAD372E2B5B7FFA222BEBA5C3C18A1B3E4AC890F5AFBAE460607A558743738D11E53EDFA00A54D209A0F52410576ACDD2F28D448B3E48F054A066BEEFA61207E29BB0BDC1EA1EA1B674A94D63D72CC57C55F8250E3EB01F1A4AD42FA14F9EAF92D880CF279DFF50A2AD6FD062B026886FF23BBE310EA1E324BF83B6860A864BD9E28B5DBE5D01173E090827262A65FA7EB0729860808DE9DEB9B2D17B0EABCB3699635EB5B4676AA7E7C533F361FDBE578DA114C6771EEE97894291796A03F1519C77ADBF8697741E48E1DC8E0B15667B617631155E88D9D6781B0FC2D26BD9A121620B2A65ACD099FE494BD887021759E7A16AAA570C881964DF67152E7D578A1751E09B11461A1ACF5FDAEFEAC5410BC5970BE924E56551F7763FD293669F24FAE322931DACDB27A6A865E0F5875A5C26082A329CEDEA47A5F2776DFD2144389AB5EC5358CFAB44B481851ED03292AB79B3E59EF0B0CB8DCE7CEE102E5E0A7597150B297FFEFFF80ED684C1CC5A0DE369EC2B636F1F9B41F4372B4AF662C71F89F0AD456595423CE4B3A38E5998CBB5EA096C9AFAABD9B48DE8BC5819F95C47E7576CAC92CADC4E00175252D6BFD1C5D585442AD5C4285A6878DE9A65D425AF3B61359D9B39C1427923CEB5B82B1CFBF4941DA0454746CD79E30912998A6407F34C29713C210"; + String hexPrivateKey = + "0000080001010073A67A536ECA81920933C513816358E0AC786E4A7B90C8988EE47D1D130455617943E9DF3E92C1F73F82158EDE21647F0A4B34EA51B53BD24CAB6CEFE41EE5E9C5581FB99265A838F776C63FD70845D263CB1CE57F5A7FCB543F7D55A2C07C66FE6CFE2ADD6B1D79593DFC25AB41A411476BE7450B5091065991980EBFA23AD2E9B9754B13E1E9AE0BC19C51CAEC71C1C9CD71F0232D3FED74C35F3D8A370CF85AED749E4E37E306B9CF5823A1F4257A536AF45983E85B171D2BB8E9A60ECABC53AA9C90D370DC05D348144C44E519FC397465EDA930FC4EA84D942E8B80D5C766E817790C6B1BBDECBEECFA30A966E9013302F8C9E28FE30292857F38951F8E008100FFA5C3FBECCA8C8EE0336C384A7CDFB8996480E432AF757624A8B24509DB56529C86A64B933E79649C54E8AE4059BF7EFE675B3678924497A866CEBAB68AC8331FE95A72FE494D5B99CD5BE8343C187DC00F788DF1D6263FEE4C09D99DA58AF004CBAA2EA523B92AB0C2935D92DB203ABB4BE46389A7CF0A55E8F56B0B0188CB008100E79E98B56901F6E4A569390B0B898A58544BE58E96AE577240073C4E6F80B1CC2BD3BBC4176D2A2905669B6E4465437242E74595D13750F91F22DF45F381D6EF3EC56CAB1190FB5A81F3EF2E919FC4412D19085C17810B6434D1D81C2978D346DEF3AD519C2DC6338FACF36B2C5E710BB82F6778703BE7B8F92E4FCAFDF82577"; + long startTime1 = System.nanoTime(); + Text cipherValue = paillierEncUDF.evaluate(new Text(plainValue)); + System.out.println("cipherValue:" + cipherValue.toString()); + long endTime1 = System.nanoTime(); + System.out.println("paillierEncUDF cost time(ns):" + (endTime1 - startTime1)); + long startTime2 = System.nanoTime(); + String decryptValueStr = paillierDecUDF.evaluate(cipherValue).toString(); + long endTime2 = System.nanoTime(); + System.out.println("paillierDecUDF cost time(ns):" + (endTime2 - startTime2)); + System.out.println("decryptValueStr:" + decryptValueStr); + Assert.assertTrue(new BigDecimal(plainValue).compareTo(new BigDecimal(decryptValueStr)) == 0);; + } + +} diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java new file mode 100644 index 00000000..8f6e6123 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java @@ -0,0 +1,61 @@ +package com.webank.wedpr.hive.udf.impl; + +import java.math.BigDecimal; + +import com.webank.wedpr.hive.udf.impl.paillier.PaillierDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.paillier.PaillierEncryptionUDF; +import com.webank.wedpr.hive.udf.impl.paillier.PaillierSumUDAF; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaConstantStringObjectInspector; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author caryliao + * 2023/11/9 + */ + +public class PaillierSumUDAFTest { + private static final Logger logger = LoggerFactory.getLogger(PaillierSumUDAFTest.class); + + @Test + public void testPaillierSumUDAFEvaluate() throws HiveException { + PaillierDecryptionUDF paillierDecUDF = new PaillierDecryptionUDF(); + PaillierEncryptionUDF encryptionUDF = new PaillierEncryptionUDF(); + String plain = "1232324234.34353543"; + Text cipher = encryptionUDF.evaluate(new Text(plain)); + + PaillierSumUDAF paillierSumUDAF = new PaillierSumUDAF(); + JavaConstantStringObjectInspector stringObjectInspector1 = new JavaConstantStringObjectInspector(cipher.toString()); + ObjectInspector[] params1 = {stringObjectInspector1}; + ObjectInspector[] params2 = {stringObjectInspector1}; + GenericUDAFParameterInfo parameterInfo = new SimpleGenericUDAFParameterInfo(params1, false, false, false); + GenericUDAFEvaluator evaluator = paillierSumUDAF.getEvaluator(parameterInfo); + evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, params1); + + GenericUDAFEvaluator.AggregationBuffer agg = evaluator.getNewAggregationBuffer(); + String[] iterateParams1 = {cipher.toString()}; + evaluator.iterate(agg, iterateParams1); + evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, params2); + String[] iterateParams2 = {cipher.toString()}; + evaluator.iterate(agg, iterateParams2); + Text result = (Text) evaluator.terminate(agg); + System.out.println("result:" + result.toString()); + + String decryptValueStr = paillierDecUDF.evaluate(new Text(result)).toString(); + System.out.println("###### decryptValueStr: " + decryptValueStr); + BigDecimal decryptValue = new BigDecimal(decryptValueStr); + BigDecimal m = new BigDecimal(plain); + BigDecimal expectedV = m.add(m); + Assert.assertTrue(decryptValue.compareTo(expectedV) == 0); + } + +} + diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java new file mode 100644 index 00000000..7dc8f5a6 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java @@ -0,0 +1,68 @@ +package com.webank.wedpr.hive.udf.impl; + +import java.nio.charset.StandardCharsets; + +import javax.xml.bind.DatatypeConverter; + +import com.webank.wedpr.hive.udf.exceptions.DecryptionException; +import com.webank.wedpr.hive.udf.impl.symmetric.SymDecryptionUDF; +import org.junit.Assert; +import org.junit.Test; + +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; + +/** + * @author caryliao + * @ 2023/11/1 + */ + +public class SymDecUDFTest { + @Test + public void testSymDecUDFEvaluate() throws JniException { + doTest(SymmetricEncryption.AlgorithmType.AES_128, SymmetricEncryption.OperationMode.ECB); + long startTime = System.nanoTime(); + doTest(SymmetricEncryption.AlgorithmType.AES_128, SymmetricEncryption.OperationMode.CTR); + long endTime = System.nanoTime(); + System.out.println("AES_128 CTR cost time(ns):" + (endTime - startTime)); + doTest(SymmetricEncryption.AlgorithmType.AES_192, SymmetricEncryption.OperationMode.CBC); + doTest(SymmetricEncryption.AlgorithmType.AES_256, SymmetricEncryption.OperationMode.OFB); + doTest(SymmetricEncryption.AlgorithmType.AES_256, SymmetricEncryption.OperationMode.CFB); + doTest(SymmetricEncryption.AlgorithmType.AES_256, SymmetricEncryption.OperationMode.CTR); + + doTest(SymmetricEncryption.AlgorithmType.TrippleDES, SymmetricEncryption.OperationMode.ECB); + doTest(SymmetricEncryption.AlgorithmType.TrippleDES, SymmetricEncryption.OperationMode.CBC); + doTest(SymmetricEncryption.AlgorithmType.TrippleDES, SymmetricEncryption.OperationMode.OFB); + doTest(SymmetricEncryption.AlgorithmType.TrippleDES, SymmetricEncryption.OperationMode.CFB); + + doTest(SymmetricEncryption.AlgorithmType.SM4, SymmetricEncryption.OperationMode.ECB); + doTest(SymmetricEncryption.AlgorithmType.SM4, SymmetricEncryption.OperationMode.CBC); + doTest(SymmetricEncryption.AlgorithmType.SM4, SymmetricEncryption.OperationMode.OFB); + doTest(SymmetricEncryption.AlgorithmType.SM4, SymmetricEncryption.OperationMode.CFB); + doTest(SymmetricEncryption.AlgorithmType.SM4, SymmetricEncryption.OperationMode.CTR); + } + + private void doTest(SymmetricEncryption.AlgorithmType algorithmType, + SymmetricEncryption.OperationMode operationMode) throws JniException { + String hexIv = "30313233343536373839616263646566"; + String hexSk = "8ff6bbbd8b5f79091cbf8e4676b7ff81"; + byte[] bytesIv = DatatypeConverter.parseHexBinary(hexIv); + byte[] bytesSk = DatatypeConverter.parseHexBinary(hexSk); + String plainValue = "caryliao"; + byte[] bytesCipher = SymmetricEncryption.encrypt(algorithmType.ordinal(), operationMode.ordinal(), bytesSk, + bytesIv, plainValue.getBytes(StandardCharsets.UTF_8)); + String hexCipher = DatatypeConverter.printHexBinary(bytesCipher); + System.out.println("hexCipher:" + hexCipher); + + SymDecryptionUDF symDecUDF = new SymDecryptionUDF(); + String decryptValueStr = + symDecUDF.evaluate(hexCipher, algorithmType.ordinal(), operationMode.ordinal(), hexIv, hexSk); + System.out.println("decryptValueStr:" + decryptValueStr); + Assert.assertTrue(plainValue.equals(decryptValueStr)); + + // evaluate with invalid cipher + Assert.assertThrows(DecryptionException.class, + () -> symDecUDF.evaluate("abc1wer", algorithmType.ordinal(), operationMode.ordinal(), hexIv, hexSk)); + } + +} diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java new file mode 100644 index 00000000..f57de858 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java @@ -0,0 +1,172 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedpr.hive.udf.impl; + +import com.webank.wedpr.hive.udf.impl.symmetric.SymDecryptionUDF; +import com.webank.wedpr.hive.udf.impl.symmetric.SymEncryptionUDF; +import com.webank.wedpr.sdk.jni.common.JniException; +import com.webank.wedpr.sdk.jni.crypto.SymmetricEncryption; +import org.apache.commons.codec.binary.Hex; +import org.junit.Assert; +import org.junit.Test; + +public class SymEncryptionUDFTest +{ +public String testSymmetricEncryptionUDFEvaluateImpl(String plainValue, int algorithmType, int mode, String hexIv, String hexPrivateKey) +{ + SymEncryptionUDF symEncryptionUDF = new SymEncryptionUDF(); + String cipher = symEncryptionUDF.evaluate(plainValue, algorithmType, mode, hexIv, hexPrivateKey); + // decrypt + SymDecryptionUDF symDecUDF = new SymDecryptionUDF(); + String decodedPlain = symDecUDF.evaluate(cipher, algorithmType, mode, hexIv, hexPrivateKey); + Assert.assertTrue(decodedPlain.equals(plainValue)); + return cipher; +} + +public void checkDeterministic(String plainValue, int algorithmType, int mode, String hexPrivateKey, String hexIv) +{ + String cipher = ""; + for(int i = 0; i < 20; i++) { + String currentCipher = testSymmetricEncryptionUDFEvaluateImpl(plainValue, algorithmType, mode, hexIv, hexPrivateKey); + if(i >= 1) + { + Assert.assertTrue(currentCipher.equals(cipher)); + } + cipher = currentCipher; + } +} + +@Test +public void testSymmetricEncryptionUDFEvaluate() throws JniException { + String hexIv = "AABBCC"; + // sm4-ctr + byte[] privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal()); + String hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("abc", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), hexPrivateKey, hexIv); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), hexPrivateKey, hexIv); + // sm4-cbc + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + // check deterministic for cbc mode + checkDeterministic("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), hexPrivateKey, hexIv); + // sm4-cfb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), hexPrivateKey, hexIv); + //sm4-ecb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), hexPrivateKey, hexIv); + // sm4-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.SM4.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), hexPrivateKey, hexIv); + + // AES-128-ctr + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), hexPrivateKey, hexIv); + + // AES-128-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), hexPrivateKey, hexIv); + + // AES-128-cfb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), hexPrivateKey, hexIv); + + // AES-128-ecb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), hexPrivateKey, hexIv); + + // AES-128-cbc + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + checkDeterministic("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_128.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), hexPrivateKey, hexIv); + + // AES-192-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), hexPrivateKey, hexIv); + + // AES-192-ctr + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), hexPrivateKey, hexIv); + + // AES-192-cfb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), hexPrivateKey, hexIv); + + // AES-192-ecb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), hexPrivateKey, hexIv); + + // AES-192-cbc + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + checkDeterministic("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_192.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), hexPrivateKey, hexIv); + + // AES-256-ctr + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CTR.ordinal(), hexPrivateKey, hexIv); + + // AES-256-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), hexPrivateKey, hexIv); + + // AES-256-cfb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), hexPrivateKey, hexIv); + + // AES-256-ecb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), hexPrivateKey, hexIv); + + // AES-256-cbc + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + checkDeterministic("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.AES_256.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), hexPrivateKey, hexIv); + + // 3des-cfb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CFB.ordinal(), hexPrivateKey, hexIv); + + // 3des-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.OFB.ordinal(), hexPrivateKey, hexIv); + + // 3des-ofb + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + testSymmetricEncryptionUDFEvaluateImpl("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.ECB.ordinal(), hexPrivateKey, hexIv); + + // 3des-cbc + privateKey = SymmetricEncryption.generateKey(SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal()); + hexPrivateKey = Hex.encodeHexString(privateKey); + checkDeterministic("中文abc@#@$中文@Sf中国", SymmetricEncryption.AlgorithmType.TrippleDES.ordinal(), SymmetricEncryption.OperationMode.CBC.ordinal(), hexPrivateKey, hexIv); +} +} \ No newline at end of file diff --git a/cpp/ppc-udf/hive/src/test/resources/config.toml b/cpp/ppc-udf/hive/src/test/resources/config.toml new file mode 100644 index 00000000..39e1b909 --- /dev/null +++ b/cpp/ppc-udf/hive/src/test/resources/config.toml @@ -0,0 +1,7 @@ +[keys] +aes_sk = "AEAC6BCB142F8945E57935D1506E3A9D" +aes_iv = "0123456789" +ore_sk = "A9F524FC2BC9CDCCC6C76EC26238D5E4" +ihc_sk = "436BBF943B43449B16BE19D90ED558F5" +paillier_sk = "000008000101006EFDA25C7DF1F497DBEEA2F08ADFD030066A4BC0ABD9905FA128BDFC81780D2936EAC40299B93F222675C75B0C57107EA034744C352C2B3EC07D2943A974A1F602A0AAA7FC382E6F0AFFFC4B878F007D80FDF03028502416D8FCABBED033EBFDF0A3EE552D41F167E52F1CFEE4990CDD003FF582CF18D095CB71A0212B80DB9D73A8451F5491F7F34CA523222CCBF5CB9FC2753D45C5CE327F3D9DDCC5E79C57BC956B4E84214CC37B048AD631F1DFA9EC0CCFACF16FD5B5EC804CCBBC46449CFFBE7D8DCA9577E643814389EFFEE62D8F4320D9E246CFB3B20464B74014A2FFFBBA0223514CBE479620A0F8D941DECB19B46497EA0B8CBBBDB42C0C0AE75432008100E8219B3E57789D4EC59D133446132F5B1697D5134F0B24967E5C99A63740F6904F159ED8AE4C59421C053E0225EBDB1929CD7A9D2B43FB86D02C8DADEE8F5570AFA2D5A2D0842832A1CB233CF0CE3F699FE8F29CE1326DA91AD02156E7E076A5B6CB2EB6F4B9828F4AA37D23F9D2772FD41634CDCC43DE94EC2401E3D31C5DFF008100F4CE7D67EC0992CAC78169EC32180EFC8E99C0DED88C1332E29B2895EC4F59B0339D335694F93A8262F2A48A9AFC1F8F7E9142F834761E3AA9FAE910C298CAF4B5D4A972BFB44A5CFF66814EE4785231BAE80DDCB65F7FF6B807D31CB12FE3901274019DDBD17BD9E257F5496D81F2B602BDB17FF1055B460B3D2946CD757DCF" +paillier_pk = "00000800010100DDFB44B8FBE3E92FB7DD45E115BFA0600CD4978157B320BF42517BF902F01A526DD5880533727E444CEB8EB618AE20FD4068E8986A58567D80FA528752E943EC0541554FF8705CDE15FFF8970F1E00FB01FBE06050A0482DB1F9577DA067D7FBE147DCAA5A83E2CFCA5E39FDC93219BA007FEB059E31A12B96E340425701B73CC440A2E4ECA620002668C364D1C329EEE4B6806CB322D42E5F72FDF5AF5F88EFFBDDA8CC4B882D4B7500F83924CBB9FC80785CEF4299C52D5328105629B4A99F64F47A312563625C28342B9FB5445DF67957422D5C1F8D0736E0BDE21939A035C0B3349B73247AF8593CB45F19D8277C0A3CAF7D9160535272C98342B66084310201FF0E06E2102360C675C143D2E0543D7A35FE8A9169368ADDE79082AD93E6EB62495A751FB46CAFE23C5990FE1BD8EDF894AB049A1970B6BD6E2B48BA3DD68ADE3FB916BDACB9948CDC03168E30EB9819A8A341AB4B5FFD9276388303F5771271AFFEEC2C3D9BED3354DF51971AC5E0EDB62A5F0796E29FB48C8DDB47B184F183CAD9751A22303D4D1C7415390B9F4E4954C226DAD2935801C64F605E3DE12607469408FEAEA3BC6AA277E2811EFBA39C50931B8D665CC35AF4D2E8A4DC62DE6B5087991AAB5C64ED389C66CF2096C4870F8BF893E73891F0E263F2181DB2B89C5F10A4D8C16FDE25FAFFF744B3517A3F13D3E3E0C9DD96F59775236E4986C19677813FB3914338F51620E8679C7CA64BAFF1C7C6CD0216874FF82EC403AE7A47FBD542724E49F2897A7CCA5F7D2340C946534BD45BB483EE56244BE5E61FBF669DB7024151E33D60AB50E44BBA9DD91584E67409F7240EAC4234D8AD6D2FD2CF41C81E3BEB894D593177B8DC162B77694FEBDDA2248E6603BF11DCF1C963F870F1A35767179E3112DED335E9D187B0A197667C47628B4718A326D754F6415B4A6F1DC4F99EFCFEA9A5F99DF56C9BB911825B7AF7308FF0BF595EB1B6E60FA864B73F3C8E82247567867470737BC1431FF5808B00B52C2A025D741E72A94594285BEBE9986C1A0FB13606C00F7CC6061EE141BEEE3E2DA9E740C4C23EC1FB778440" \ No newline at end of file diff --git a/cpp/ppc-udf/mysql/ore_dec.cpp b/cpp/ppc-udf/mysql/ore_dec.cpp new file mode 100644 index 00000000..a97eb109 --- /dev/null +++ b/cpp/ppc-udf/mysql/ore_dec.cpp @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ore_dec.cpp + * @author: caryliao + * @date 2023-10-26 + */ +#include "ore_dec.h" +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include +#include + +using namespace std; + +bool isHexInDec = true; +struct OreDecData { + InputBuffer oreSk; + char* oreDecryptResult = NULL; +}; + +// eg:ore_dec(columnName, sk) +my_bool ore_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + if (args->arg_count != 2) + { + const char* errorMsg = + "\nUsage:ore_dec(columnName, hexSk)" + "\nExample:select ore_dec(encrypted_name, '303132...') as user_name from t_user"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + snprintf(message, max_msg_size, "%s", "ore_dec() requires (string, string) as parameter types"); + return 1; + } + uint64_t skBytesLen = args->lengths[1] / 2; + unsigned char* skBytesData = (unsigned char*)malloc(skBytesLen); + if (!skBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for sk failed"); + return 1; + } + OutputBuffer skBytesBuffer = {skBytesData, skBytesLen}; + InputBuffer skHexBuffer = {(const unsigned char*)args->args[1], args->lengths[1]}; + from_hex(&skBytesBuffer, &skHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "invalid hexSk"); + return 1; + } + OreDecData* oreDataPtr = new OreDecData; + initid->ptr = (char*)oreDataPtr; + oreDataPtr->oreSk.data = skBytesBuffer.data; + oreDataPtr->oreSk.len = skBytesBuffer.len; + oreDataPtr->oreDecryptResult = (char*)malloc(fast_ore_get_plain_size(args->lengths[0] / 2, isHexInDec)); + if (!oreDataPtr->oreDecryptResult) + { + snprintf(message, max_msg_size, "%s", "malloc memory for oreDecryptResult failed!"); + return 1; + } + + return 0; +} + +void ore_dec_deinit(UDF_INIT* initid) +{ + if (initid->ptr) + { + OreDecData* oreDataPtr = (OreDecData*)initid->ptr; + if (oreDataPtr->oreSk.data) + { + free((void*)oreDataPtr->oreSk.data); + oreDataPtr->oreSk.data = NULL; + } + if (oreDataPtr->oreDecryptResult) + { + free(oreDataPtr->oreDecryptResult); + oreDataPtr->oreDecryptResult = NULL; + } + free(initid->ptr); + initid->ptr = NULL; + } +} + +char* ore_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, + char* error) +{ + InputBuffer ciphertext = {(const unsigned char*)args->args[0], args->lengths[0]}; + OreDecData* oreDataPtr = (OreDecData*)initid->ptr; + // ore_decrypt + OutputBuffer plainBuffer = { + (unsigned char*)oreDataPtr->oreDecryptResult, fast_ore_get_plain_size(ciphertext.len, isHexInDec)}; + string_fast_ore_decrypt(&plainBuffer, &oreDataPtr->oreSk, &ciphertext, isHexInDec); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s", "decrypt ore cipher failed"); + return NULL; + }; + *length = plainBuffer.len; + return oreDataPtr->oreDecryptResult; +} diff --git a/cpp/ppc-udf/mysql/ore_dec.h b/cpp/ppc-udf/mysql/ore_dec.h new file mode 100644 index 00000000..11719e46 --- /dev/null +++ b/cpp/ppc-udf/mysql/ore_dec.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ore_dec.h + * @author: caryliao + * @date 2023-10-26 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +my_bool ore_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void ore_dec_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +char* ore_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/ore_enc.cpp b/cpp/ppc-udf/mysql/ore_enc.cpp new file mode 100644 index 00000000..789eaed7 --- /dev/null +++ b/cpp/ppc-udf/mysql/ore_enc.cpp @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ore_enc.cpp + * @author: caryliao + * @date 2023-11-30 + */ +#include "ore_enc.h" +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include +#include + +using namespace std; + +bool isHexInEnc = true; +struct OreEncData { + InputBuffer oreSk; + char* oreEncryptResult = NULL; +}; + +// eg:ore_enc(columnName, sk) +my_bool ore_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + if (args->arg_count != 2) + { + const char* errorMsg = + "\nUsage:ore_enc(columnName, hexSk)" + "\nExample:select ore_enc(name, '303132...') as user_name from t_user"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + snprintf(message, max_msg_size, "%s", "ore_enc() requires (string, string) as parameter types"); + return 1; + } + uint64_t skBytesLen = args->lengths[1] / 2; + unsigned char* skBytesData = (unsigned char*)malloc(skBytesLen); + if (!skBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for sk failed"); + return 1; + } + OutputBuffer skBytesBuffer = {skBytesData, skBytesLen}; + InputBuffer skHexBuffer = {(const unsigned char*)args->args[1], args->lengths[1]}; + from_hex(&skBytesBuffer, &skHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "invalid hexSk"); + return 1; + } + OreEncData* oreDataPtr = new OreEncData; + initid->ptr = (char*)oreDataPtr; + oreDataPtr->oreSk.data = skBytesBuffer.data; + oreDataPtr->oreSk.len = skBytesBuffer.len; + oreDataPtr->oreEncryptResult = (char*)malloc(fast_ore_get_cipher_size(args->lengths[0], isHexInEnc)); + if (!oreDataPtr->oreEncryptResult) + { + snprintf(message, max_msg_size, "%s", "malloc memory for oreEncryptResult failed!"); + return 1; + } + + return 0; +} + +void ore_enc_deinit(UDF_INIT* initid) +{ + if (initid->ptr) + { + OreEncData* oreDataPtr = (OreEncData*)initid->ptr; + if (oreDataPtr->oreSk.data) + { + free((void*)oreDataPtr->oreSk.data); + oreDataPtr->oreSk.data = NULL; + } + if (oreDataPtr->oreEncryptResult) + { + free(oreDataPtr->oreEncryptResult); + oreDataPtr->oreEncryptResult = NULL; + } + free(initid->ptr); + initid->ptr = NULL; + } +} + +char* ore_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, + char* error) +{ + InputBuffer plaintext = {(const unsigned char*)args->args[0], args->lengths[0]}; + OreEncData* oreDataPtr = (OreEncData*)initid->ptr; + // ore_encrypt + OutputBuffer cipherBuffer = { + (unsigned char*)oreDataPtr->oreEncryptResult, fast_ore_get_cipher_size(plaintext.len, isHexInEnc)}; + string_fast_ore_encrypt(&cipherBuffer, &oreDataPtr->oreSk, &plaintext, isHexInEnc); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s", "encrypt ore cipher failed"); + return NULL; + }; + *length = cipherBuffer.len; + return oreDataPtr->oreEncryptResult; +} diff --git a/cpp/ppc-udf/mysql/ore_enc.h b/cpp/ppc-udf/mysql/ore_enc.h new file mode 100644 index 00000000..2b99afd5 --- /dev/null +++ b/cpp/ppc-udf/mysql/ore_enc.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ore_enc.h + * @author: caryliao + * @date 2023-11-30 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +my_bool ore_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void ore_enc_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +char* ore_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/paillier_dec.cpp b/cpp/ppc-udf/mysql/paillier_dec.cpp new file mode 100644 index 00000000..a3e7635c --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_dec.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_dec.cpp + * @author: caryliao + * @date 2023-10-24 + */ +#include "paillier_dec.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include +#include +#include +#include + +using namespace std; +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; + +struct PaillierDecData +{ + void* keyPair = NULL; +}; + +// eg:paillier_dec(columnName, hexPk, hexSk, scale) +my_bool paillier_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + if (args->arg_count != 4) + { + const char* errorMsg = + "\nUsage:paillier_dec(columnName, hexPk, hexSk, scale)" + "\nExample:select paillier_dec(encrypted_salary,'0000080001...', '0000821BA...', 2)"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT || + args->arg_type[2] != STRING_RESULT || args->arg_type[3] != INT_RESULT) + { + snprintf(message, max_msg_size, "%s", + "paillier_dec() requires as (string, string, string, int) as parameter types"); + return 1; + } + uint64_t pkBytesLen = args->lengths[1] / 2; + uint64_t skBytesLen = args->lengths[2] / 2; + unsigned char* pkBytesData = (unsigned char*)malloc(pkBytesLen); + unsigned char* skBytesData = (unsigned char*)malloc(skBytesLen); + if (!pkBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for public-key failed"); + return 1; + } + if (!skBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for private-key failed"); + return 1; + } + OutputBuffer pkBytesBuffer = {pkBytesData, pkBytesLen}; + OutputBuffer skBytesBuffer = {skBytesData, skBytesLen}; + InputBuffer pkHexBuffer = {(const unsigned char*)args->args[1], args->lengths[1]}; + InputBuffer skHexBuffer = {(const unsigned char*)args->args[2], args->lengths[2]}; + from_hex(&pkBytesBuffer, &pkHexBuffer); + if (get_last_error() != 0) + { + snprintf( + message, max_msg_size, "%s%s", "Invalid public key! error: ", get_last_error_msg()); + return 1; + } + from_hex(&skBytesBuffer, &skHexBuffer); + if (get_last_error() != 0) + { + snprintf( + message, max_msg_size, "%s%s", "Invalid private key! error: ", get_last_error_msg()); + return 1; + } + PaillierDecData* paillierDecDataPtr = new PaillierDecData; + initid->ptr = (char*)paillierDecDataPtr; + // load the public key + InputBuffer pkBuffer = {pkBytesBuffer.data, pkBytesBuffer.len}; + // load the private key + InputBuffer skBuffer = {skBytesBuffer.data, skBytesBuffer.len}; + paillierDecDataPtr->keyPair = paillier_load_keypair(&skBuffer, &pkBuffer); + free(pkBytesData); + free(skBytesData); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s%s", "Invalid key pair! error: ", get_last_error_msg()); + return 1; + } + return 0; +} + +void paillier_dec_deinit(UDF_INIT* initid) +{ + if (initid->ptr) + { + PaillierDecData* paillierDecDataPtr = (PaillierDecData*)initid->ptr; + if (paillierDecDataPtr->keyPair) + { + paillier_free_key_pair(paillierDecDataPtr->keyPair); + paillierDecDataPtr->keyPair = NULL; + } + } + free(initid->ptr); + initid->ptr = NULL; +} + +double paillier_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) +{ + // convert the args0 to bytes + InputBuffer hexArgs = {(const unsigned char*)args->args[0], args->lengths[0]}; + OutputBuffer bytesArgsBuffer = {(unsigned char*)args->args[0], args->lengths[0] / 2}; + from_hex(&bytesArgsBuffer, &hexArgs); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s%s", "Invalid hex cipher! error: ", get_last_error_msg()); + return 1; + } + // paillier_decrypt + InputBuffer ciphertext = {(const unsigned char*)bytesArgsBuffer.data, bytesArgsBuffer.len}; + BIGNUM* decrypted_significant = BN_new(); + int16_t exponent = 0; + PaillierDecData* paillierDecDataPtr = (PaillierDecData*)initid->ptr; + floating_point_paillier_decrypt( + decrypted_significant, &exponent, &ciphertext, paillierDecDataPtr->keyPair); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s%s", + "decrypt paillier cipher failed! error: ", get_last_error_msg()); + return 0; + } + int scale = *(int*)(args->args[3]); + BigNum v; + v.swap(decrypted_significant); + FloatingPointNumber floatingPointNumber(std::move(v), exponent); + auto floatingPointCodec = std::make_shared(); + float50 decryptResult = floatingPointCodec->toFloat50(floatingPointNumber); + double multiplier = pow(10.0, scale); + double finalResult = round((double)decryptResult * multiplier) / multiplier; + + return finalResult; +} diff --git a/cpp/ppc-udf/mysql/paillier_dec.h b/cpp/ppc-udf/mysql/paillier_dec.h new file mode 100644 index 00000000..60a0c43e --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_dec.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_dec.h + * @author: caryliao + * @date 2023-10-24 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +my_bool paillier_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void paillier_dec_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +double paillier_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/paillier_enc.cpp b/cpp/ppc-udf/mysql/paillier_enc.cpp new file mode 100644 index 00000000..3acf1f00 --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_enc.cpp @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_enc.cpp + * @author: caryliao + * @date 2023-10-24 + */ +#include "paillier_enc.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/libwrapper/FloatingPointNumber.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include +#include +#include +#include + +using namespace std; +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; + +struct PaillierEncData +{ + void* publicKey = NULL; + char* cipher_result; +}; + +// eg:paillier_enc(columnName, hexPk) +my_bool paillier_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + if (args->arg_count != 2) + { + const char* errorMsg = + "\nUsage:paillier_enc(columnName, hexPk)" + "\nExample:select paillier_enc(salary,'0000080001...')"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + // TODO 使用double类型，目前没有获取到参数值，一直是0.00000 + // double plainValue = *(double*)args->args[0]; + double plainValue = strtod(args->args[0], NULL); + // std::cout << "double plainValue:" << plainValue << std::endl; + // snprintf(message, max_msg_size, "%f", plainValue); + // return 1; + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != STRING_RESULT) + { + snprintf(message, max_msg_size, "%s", + "paillier_enc() requires as (string, string) as parameter types"); + return 1; + } + uint64_t pkBytesLen = args->lengths[1] / 2; + unsigned char* pkBytesData = (unsigned char*)malloc(pkBytesLen); + if (!pkBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for public-key failed"); + return 1; + } + OutputBuffer pkBytesBuffer = {pkBytesData, pkBytesLen}; + InputBuffer pkHexBuffer = {(const unsigned char*)args->args[1], args->lengths[1]}; + from_hex(&pkBytesBuffer, &pkHexBuffer); + if (get_last_error() != 0) + { + snprintf( + message, max_msg_size, "%s%s", "Invalid public key! error: ", get_last_error_msg()); + return 1; + } + PaillierEncData* paillierEncDataPtr = new PaillierEncData; + initid->ptr = (char*)paillierEncDataPtr; + // load the public key + InputBuffer pkBuffer = {pkBytesBuffer.data, pkBytesBuffer.len}; + paillierEncDataPtr->publicKey = paillier_load_public_key(&pkBuffer); + free(pkBytesData); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s%s", "Invalid key pair! error: ", get_last_error_msg()); + return 1; + } + auto keyBits = paillier_key_bits_from_public_key(paillierEncDataPtr->publicKey); + auto cipherBytes = floating_point_paillier_cipher_bytes(keyBits); + initid->max_length = cipherBytes; + paillierEncDataPtr->cipher_result = (char*)malloc(initid->max_length); + if (!paillierEncDataPtr->cipher_result) + { + snprintf(message, max_msg_size, "%s", "Malloc memory for cipher_result failed!"); + return 1; + } + return 0; +} + +void paillier_enc_deinit(UDF_INIT* initid) +{ + std::cout << "#### paillier_enc_deinit" << std::endl; + if (initid->ptr) + { + PaillierEncData* paillierEncDataPtr = (PaillierEncData*)initid->ptr; + if (paillierEncDataPtr->publicKey) + { + paillier_free_public_key(paillierEncDataPtr->publicKey); + paillierEncDataPtr->publicKey = NULL; + } + if (paillierEncDataPtr->cipher_result) + { + free(paillierEncDataPtr->cipher_result); + paillierEncDataPtr->cipher_result = NULL; + } + free(initid->ptr); + } + initid->ptr = NULL; +} + +char* paillier_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) +{ + PaillierEncData* paillierEncDataPtr = (PaillierEncData*)initid->ptr; + float50 plainValue(args->args[0]); + auto floatingPointCodec = std::make_shared(); + auto fpn = floatingPointCodec->toFloatingPoint(std::string(args->args[0])); + int16_t exponent = fpn.exponent; + OutputBuffer cipherBuffer{ + (unsigned char*)paillierEncDataPtr->cipher_result, initid->max_length}; + floating_point_paillier_encrypt( + &cipherBuffer, fpn.value.bn().get(), exponent, paillierEncDataPtr->publicKey); + if (get_last_error() != 0) + { + const char* errorMsg = get_last_error_msg(); + snprintf(error, max_msg_size, "%s%s", "encrypt failed! error: ", errorMsg); + return NULL; + } + // OutputBuffer resultBuffer = {(unsigned char*)paillierEncDataPtr->cipher_result, + // cipherBuffer.len * 2}; InputBuffer bytesBuffer = {(unsigned char*)cipherBuffer.data, + // cipherBuffer.len}; to_hex(&resultBuffer, &bytesBuffer); + *length = initid->max_length; + return paillierEncDataPtr->cipher_result; +} diff --git a/cpp/ppc-udf/mysql/paillier_enc.h b/cpp/ppc-udf/mysql/paillier_enc.h new file mode 100644 index 00000000..e0021348 --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_enc.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_enc.h + * @author: caryliao + * @date 2023-12-4 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif +my_bool paillier_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void paillier_enc_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +char* paillier_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/paillier_sum.cpp b/cpp/ppc-udf/mysql/paillier_sum.cpp new file mode 100644 index 00000000..c8fc3f05 --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_sum.cpp @@ -0,0 +1,184 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_sum.cpp + * @author: yujiechen + * @date 2023-08-18 + */ +#include "paillier_sum.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include +#include + +using namespace std; + +struct SumResult +{ + uint64_t max_cipher_bytes = 0; + void* public_key = NULL; + char* sum_result = NULL; + int64_t add_count = 0; +}; + +my_bool paillier_sum_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + if (args->arg_count != 2) + { + const char* errorMsg = + "invalid argument: paillier_sum requires the columnName and the publicKey, " + "e.g. paillier_sum(id, publicKey)"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + /// TODO: check the parameters + // convert the hexed public key to bytes + // TODO: decrease the overhead + uint64_t pkBytesLen = args->lengths[1] / 2; + unsigned char* pkBytesData = (unsigned char*)malloc(pkBytesLen); + if (!pkBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for public-key failed"); + return 1; + } + OutputBuffer pkBytesBuffer = {pkBytesData, pkBytesLen}; + InputBuffer pkHexBuffer = {(const unsigned char*)args->args[1], args->lengths[1]}; + from_hex(&pkBytesBuffer, &pkHexBuffer); + if (get_last_error() != 0) + { + snprintf( + message, max_msg_size, "%s%s", "Invalid public key! error: ", get_last_error_msg()); + return 1; + } + SumResult* resultBuffer = new SumResult; + // load the public key + InputBuffer pkBuffer = {pkBytesBuffer.data, pkBytesBuffer.len}; + resultBuffer->public_key = (void*)paillier_load_public_key(&pkBuffer); + // release the allocated memory + free(pkBytesData); + + if (!resultBuffer->public_key) + { + snprintf(message, max_msg_size, "%s%s", "Invalid public key:", get_last_error_msg()); + return 1; + } + + // allocate memory for the result + resultBuffer->max_cipher_bytes = floating_point_paillier_cipher_bytes( + paillier_key_bits_from_public_key(resultBuffer->public_key)); + initid->maybe_null = 1; + // will convert the result to hex when calling sum + initid->max_length = 2 * resultBuffer->max_cipher_bytes; + // Note: The MySQL UDF engine allocates 255 bytes as the buffer for a string return value. If + // the result fits within this limit, simply place the result into this buffer and return the + // pointer. If the result is bigger, you must use a string buffer that you have allocated. + // refer to: https://docstore.mik.ua/orelly/weblinux2/mysql/ch14_01.htm + resultBuffer->sum_result = (char*)malloc(initid->max_length); + if (!resultBuffer->sum_result) + { + snprintf(message, max_msg_size, "%s", "Malloc memory for sum_result failed!"); + return 1; + } + initid->ptr = (char*)resultBuffer; + if (!initid->ptr) + { + snprintf(message, max_msg_size, "%s", "Malloc memory for initid->ptr failed!"); + } + return 0; +} + +void paillier_sum_deinit(UDF_INIT* initid) +{ + paillier_sum_clear(initid, NULL, NULL); + if (initid->ptr) + { + delete initid->ptr; + } + initid->ptr = NULL; +} + +void paillier_sum_clear(UDF_INIT* initid, char* is_null, char* error) +{ + SumResult* result = (SumResult*)(initid->ptr); + result->add_count = 0; + result->max_cipher_bytes = 0; + if (result->public_key) + { + paillier_free_key_pair(result->public_key); + result->public_key = NULL; + } + if (result->sum_result) + { + free(result->sum_result); + result->sum_result = NULL; + } +} + +void paillier_sum_reset(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) +{ + paillier_sum_clear(initid, is_null, error); + paillier_sum_add(initid, args, is_null, error); +} + +void paillier_sum_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error) +{ + if (args->args[0] == NULL) + { + return; + } + SumResult* resultBuffer = (SumResult*)initid->ptr; + // convert the args0 to bytes + InputBuffer hexArgs = {(const unsigned char*)args->args[0], args->lengths[0]}; + OutputBuffer bytesArgsBuffer = {(unsigned char*)args->args[0], args->lengths[0] / 2}; + from_hex(&bytesArgsBuffer, &hexArgs); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s", "Invalid hex cipher!"); + return; + } + // the first add + if (0 == resultBuffer->add_count) + { + memcpy((void*)resultBuffer->sum_result, (const void*)bytesArgsBuffer.data, + bytesArgsBuffer.len); + resultBuffer->add_count++; + return; + } + // paillier_add + InputBuffer c1 = { + (const unsigned char*)resultBuffer->sum_result, resultBuffer->max_cipher_bytes}; + InputBuffer c2 = {(const unsigned char*)bytesArgsBuffer.data, bytesArgsBuffer.len}; + OutputBuffer result = { + (unsigned char*)resultBuffer->sum_result, resultBuffer->max_cipher_bytes}; + floating_point_paillier_add(&result, &c1, &c2, resultBuffer->public_key); + resultBuffer->add_count++; +} + +char* paillier_sum(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error) +{ + SumResult* sum_result = (SumResult*)initid->ptr; + // convert to hex + OutputBuffer resultBuffer = {(unsigned char*)sum_result->sum_result, initid->max_length}; + InputBuffer bytesBuffer = { + (unsigned char*)sum_result->sum_result, sum_result->max_cipher_bytes}; + to_hex(&resultBuffer, &bytesBuffer); + *length = resultBuffer.len; + return sum_result->sum_result; +} diff --git a/cpp/ppc-udf/mysql/paillier_sum.h b/cpp/ppc-udf/mysql/paillier_sum.h new file mode 100644 index 00000000..c4c67eac --- /dev/null +++ b/cpp/ppc-udf/mysql/paillier_sum.h @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file paillier_sum.h + * @author: yujiechen + * @date 2023-08-18 + */ + +#ifndef __PAILLIER_SUM_H__ +#define __PAILLIER_SUM_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif +my_bool paillier_sum_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void paillier_sum_deinit(UDF_INIT* initid); +void paillier_sum_clear(UDF_INIT* initid, char* is_null, char* error); +void paillier_sum_reset(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error); + +// Note: every row data will trigger this function +void paillier_sum_add(UDF_INIT* initid, UDF_ARGS* args, char* is_null, char* error); + +char* paillier_sum(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/sym_dec.cpp b/cpp/ppc-udf/mysql/sym_dec.cpp new file mode 100644 index 00000000..7bdec184 --- /dev/null +++ b/cpp/ppc-udf/mysql/sym_dec.cpp @@ -0,0 +1,178 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file sym_dec.cpp + * @author: caryliao + * @date 2023-10-25 + */ +#include "sym_dec.h" +#include "ppc-crypto-c-sdk/symmetric_encryption.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include +#include + +using namespace std; + +struct SymDecData { + int algorithmType = 0; + int mode = 0; + InputBuffer iv; + InputBuffer symSk; + char* symDecryptResult = NULL; +}; + + +// eg:sym_dec(columnName, algorithmType, mode, hexIv, hexSk) +my_bool sym_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + SymDecData* data = new SymDecData; + initid->ptr = (char*)data; + + if (args->arg_count != 5) + { + const char* errorMsg = + "\nUsage:sym_dec(columnName, algorithmType, mode, hexIv, hexSk)" + "\nalgorithmType(0:AES_128, 1:AES_192, 2:AES_256, 3:TrippleDES, 4:SM4), mode(0:ECB, " + "1:CBC, 2:CFB, 3:OFB, 4:CTR)"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != INT_RESULT || + args->arg_type[2] != INT_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + { + snprintf(message, max_msg_size, "%s", "sym_dec() requires (string, int, int, string, string) as parameters"); + return 1; + } + int algorithmType = *(int*)(args->args[1]); + int mode = *(int*)(args->args[2]); + // check algorithmType and mode in [0, 4] + if (algorithmType < 0 || algorithmType > 4) + { + snprintf(message, max_msg_size, "%s", "invalid algorithmType, algorithmType must be 0, 1, 2, 3 or 4"); + return 1; + } + if (mode < 0 || mode > 4) + { + snprintf(message, max_msg_size, "%s", "invalid mode, mode must be 0, 1, 2, 3 or 4"); + return 1; + } + // algorithmType is 3 and mode is 4 is not match + if (algorithmType == 3 && mode == 4) + { + snprintf(message, max_msg_size, "%s", "TrippleDES does not support CTR mode"); + return 1; + } + data->algorithmType = algorithmType; + data->mode = mode; + unsigned char* ivBytesData = (unsigned char*)malloc(args->lengths[3] / 2); + unsigned char* skBytesData = (unsigned char*)malloc(args->lengths[4] / 2); + if (!ivBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for iv failed"); + return 1; + } + if (!skBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for sk failed"); + return 1; + } + OutputBuffer ivBytesBuffer = {ivBytesData, args->lengths[3] / 2}; + OutputBuffer skBytesBuffer = {skBytesData, args->lengths[4] / 2}; + InputBuffer ivHexBuffer = {(const unsigned char*)args->args[3], args->lengths[3]}; + InputBuffer skHexBuffer = {(const unsigned char*)args->args[4], args->lengths[4]}; + + from_hex(&ivBytesBuffer, &ivHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "Invalid hex iv!"); + return 1; + } + from_hex(&skBytesBuffer, &skHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "Invalid hex sk!"); + return 1; + } + data->iv.data = ivBytesBuffer.data; + data->iv.len = ivBytesBuffer.len; + + data->symSk.data = skBytesBuffer.data; + data->symSk.len = skBytesBuffer.len; + + data->symDecryptResult = (char*)malloc(symmetric_block_size(data->algorithmType) + args->lengths[0] / 2); + if (!data->symDecryptResult) + { + snprintf(message, max_msg_size, "%s", "Malloc memory for symDecryptResult failed!"); + return 1; + } + + return 0; +} + +void sym_dec_deinit(UDF_INIT* initid) +{ + if (initid->ptr) + { + SymDecData* data = (SymDecData*)initid->ptr; + if(data->symDecryptResult) + { + free(data->symDecryptResult); + data->symDecryptResult = NULL; + } + if(data->iv.data) + { + free((void*)data->iv.data); + data->iv.data = NULL; + } + if(data->symSk.data) + { + free((void*)data->symSk.data); + data->symSk.data = NULL; + } + free(initid->ptr); + initid->ptr = NULL; + } +} + +char* sym_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, + char* error) +{ + InputBuffer hexArgs = {(const unsigned char*)args->args[0], args->lengths[0]}; + OutputBuffer bytesArgsBuffer = {(unsigned char*)args->args[0], args->lengths[0] / 2}; + from_hex(&bytesArgsBuffer, &hexArgs); + if (get_last_error() != 0) + { + snprintf(error, max_msg_size, "%s", "Invalid hex cipher!"); + return NULL; + } + // sym_decrypt + SymDecData* data = (SymDecData*)initid->ptr; + InputBuffer ciphertext = {(const unsigned char*)bytesArgsBuffer.data, bytesArgsBuffer.len}; + OutputBuffer plainBuffer = {(unsigned char*)(data->symDecryptResult), + symmetric_block_size(data->algorithmType) + args->lengths[0] / 2}; + symmetric_decrypt(&plainBuffer, data->algorithmType, data->mode, &data->symSk, &data->iv, &ciphertext); + if (get_last_error() != 0) + { + const char* errorMsg = get_last_error_msg(); + snprintf(error, max_msg_size, "%s", errorMsg); + return NULL; + }; + *length = plainBuffer.len; + return data->symDecryptResult; +} diff --git a/cpp/ppc-udf/mysql/sym_dec.h b/cpp/ppc-udf/mysql/sym_dec.h new file mode 100644 index 00000000..f5b90ca4 --- /dev/null +++ b/cpp/ppc-udf/mysql/sym_dec.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file sym_dec.h + * @author: caryliao + * @date 2023-10-25 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +my_bool sym_dec_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void sym_dec_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +char* sym_dec(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/mysql/sym_enc.cpp b/cpp/ppc-udf/mysql/sym_enc.cpp new file mode 100644 index 00000000..9ff3153c --- /dev/null +++ b/cpp/ppc-udf/mysql/sym_enc.cpp @@ -0,0 +1,171 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file sym_enc.cpp + * @author: caryliao + * @date 2023-11-29 + */ +#include "sym_enc.h" +#include "ppc-crypto-c-sdk/symmetric_encryption.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-crypto-c-sdk/utils/utilities.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include +#include +#include +#include + +using namespace std; +using namespace bcos; + +struct SymEncData { + int algorithmType = 0; + int mode = 0; + InputBuffer iv; + InputBuffer symSk; + char* symEncryptResult = NULL; +}; + + +// eg:sym_enc(columnName, algorithmType, mode, hexIv, hexSk) +my_bool sym_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message) +{ + SymEncData* data = new SymEncData; + initid->ptr = (char*)data; + + if (args->arg_count != 5) + { + const char* errorMsg = + "\nUsage:sym_enc(columnName, algorithmType, mode, hexIv, hexSk)" + "\nalgorithmType(0:AES_128, 1:AES_192, 2:AES_256, 3:TrippleDES, 4:SM4), mode(0:ECB, " + "1:CBC, 2:CFB, 3:OFB, 4:CTR)"; + snprintf(message, max_msg_size, "%s", errorMsg); + return 1; + } + if (args->arg_type[0] != STRING_RESULT || args->arg_type[1] != INT_RESULT || + args->arg_type[2] != INT_RESULT || args->arg_type[3] != STRING_RESULT || + args->arg_type[4] != STRING_RESULT) + { + snprintf(message, max_msg_size, "%s", "sym_enc() requires (string, int, int, string, string) as parameters"); + return 1; + } + int algorithmType = *(int*)(args->args[1]); + int mode = *(int*)(args->args[2]); + // check algorithmType and mode in [0, 4] + if (algorithmType < 0 || algorithmType > 4) + { + snprintf(message, max_msg_size, "%s", "invalid algorithmType, algorithmType must be 0, 1, 2, 3 or 4"); + return 1; + } + if (mode < 0 || mode > 4) + { + snprintf(message, max_msg_size, "%s", "invalid mode, mode must be 0, 1, 2, 3 or 4"); + return 1; + } + // algorithmType is 3 and mode is 4 is not match + if (algorithmType == 3 && mode == 4) + { + snprintf(message, max_msg_size, "%s", "TrippleDES does not support CTR mode"); + return 1; + } + data->algorithmType = algorithmType; + data->mode = mode; + unsigned char* ivBytesData = (unsigned char*)malloc(args->lengths[3] / 2); + unsigned char* skBytesData = (unsigned char*)malloc(args->lengths[4] / 2); + if (!ivBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for iv failed"); + return 1; + } + if (!skBytesData) + { + snprintf(message, max_msg_size, "%s", "allocate memory for sk failed"); + return 1; + } + OutputBuffer ivBytesBuffer = {ivBytesData, args->lengths[3] / 2}; + OutputBuffer skBytesBuffer = {skBytesData, args->lengths[4] / 2}; + InputBuffer ivHexBuffer = {(const unsigned char*)args->args[3], args->lengths[3]}; + InputBuffer skHexBuffer = {(const unsigned char*)args->args[4], args->lengths[4]}; + + from_hex(&ivBytesBuffer, &ivHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "Invalid hex iv!"); + return 1; + } + from_hex(&skBytesBuffer, &skHexBuffer); + if (get_last_error() != 0) + { + snprintf(message, max_msg_size, "%s", "Invalid hex sk!"); + return 1; + } + data->iv.data = ivBytesBuffer.data; + data->iv.len = ivBytesBuffer.len; + + data->symSk.data = skBytesBuffer.data; + data->symSk.len = skBytesBuffer.len; + + data->symEncryptResult = (char*)malloc(symmetric_block_size(data->algorithmType) + args->lengths[0]); + if (!data->symEncryptResult) + { + snprintf(message, max_msg_size, "%s", "Malloc memory for symEncryptResult failed!"); + return 1; + } + + return 0; +} + +void sym_enc_deinit(UDF_INIT* initid) +{ + if (initid->ptr) + { + SymEncData* data = (SymEncData*)initid->ptr; + if(data->symEncryptResult) + { + free(data->symEncryptResult); + data->symEncryptResult = NULL; + } + if(data->iv.data) + { + free((void*)data->iv.data); + data->iv.data = NULL; + } + if(data->symSk.data) + { + free((void*)data->symSk.data); + data->symSk.data = NULL; + } + free(initid->ptr); + initid->ptr = NULL; + } +} + +char* sym_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, char* is_null, + char* error) +{ + InputBuffer plainText = {(const unsigned char*)args->args[0], args->lengths[0]}; + SymEncData* data = (SymEncData*)initid->ptr; + OutputBuffer cipherBuffer = {(unsigned char*)(data->symEncryptResult), + symmetric_block_size(data->algorithmType) + args->lengths[0]}; + symmetric_encrypt(&cipherBuffer, data->algorithmType, data->mode, &data->symSk, &data->iv, &plainText); + if (get_last_error() != 0) + { + const char* errorMsg = get_last_error_msg(); + snprintf(error, max_msg_size, "%s", errorMsg); + return NULL; + }; + *length = cipherBuffer.len; + return data->symEncryptResult; +} diff --git a/cpp/ppc-udf/mysql/sym_enc.h b/cpp/ppc-udf/mysql/sym_enc.h new file mode 100644 index 00000000..aad19a22 --- /dev/null +++ b/cpp/ppc-udf/mysql/sym_enc.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file sym_enc.h + * @author: caryliao + * @date 2023-11-25 + */ + +#ifndef __PAILLIER_DECRYPT_H__ +#define __PAILLIER_DECRYPT_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +my_bool sym_enc_init(UDF_INIT* initid, UDF_ARGS* args, char* message); + +void sym_enc_deinit(UDF_INIT* initid); + +// Note: every row data will trigger this function +char* sym_enc(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, + char* is_null, char* error); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/cpp/ppc-udf/tests/CMakeLists.txt b/cpp/ppc-udf/tests/CMakeLists.txt new file mode 100644 index 00000000..aab162b0 --- /dev/null +++ b/cpp/ppc-udf/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-ppc-udf) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${PPC_UDF_STATIC_TARGET} ${BOOST_UNIT_TEST}) +add_test(NAME test-ppc-udf WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) diff --git a/cpp/ppc-udf/tests/OreDecUDFTest.cpp b/cpp/ppc-udf/tests/OreDecUDFTest.cpp new file mode 100644 index 00000000..bf605775 --- /dev/null +++ b/cpp/ppc-udf/tests/OreDecUDFTest.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OreDecUDFTest.cpp + * @author: caryliao + * @date 2023-10-25 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-udf/mysql/ore_dec.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(oreDecUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters() +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + int argCount = 2; + args->arg_count = argCount; + args->args = new char*[argCount]; + args->lengths = new unsigned long[argCount]; + args->arg_type = new Item_result[argCount]; + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = STRING_RESULT; + + return std::make_pair(initid, args); +} + +void testOreDec(InputBuffer const* plainTextBytesBuffer, InputBuffer const* skBytesBuffer) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + auto fakedArgs = fakeMySQLParameters(); + std::cout << "#### testOreDec" << std::endl; + bool isHex = true; + bcos::bytes cipherData(fast_ore_get_cipher_size(plainTextBytesBuffer->len, isHex)); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + string_fast_ore_encrypt(&cipherBuffer, skBytesBuffer, plainTextBytesBuffer, isHex); + // InputBuffer inputBufferCipher{cipherBuffer.data, cipherBuffer.len}; + + // convert cipherBytes to hex + // auto hexCipher = *(bcos::toHexString(cipherData.data(), cipherData.data() + cipherBuffer.len)); + // std::cout << "#### hexCipher: " << hexCipher << std::endl; + // fakedArgs.second->args[0] = (char*)hexCipher.data(); + // fakedArgs.second->lengths[0] = hexCipher.size(); + fakedArgs.second->args[0] = (char*)cipherData.data(); + fakedArgs.second->lengths[0] = cipherBuffer.len; + + // std::cout << "#### decrypt" << std::endl; + // bcos::bytes plain(plainTextBytesBuffer->len); + // OutputBuffer plainBuffer{plain.data(), plain.size()}; + // string_fast_ore_decrypt(&plainBuffer, skBytesBuffer, &inputBufferCipher, isHex); + // std::cout << "#### decrypt end: " << plainBuffer.data << "# len: " << + // plainBuffer.len << ", size: " << plain.size()<data, skBytesBuffer->data + skBytesBuffer->len)); + InputBuffer inputBufferHexSk{(const unsigned char*)hexSk.data(), hexSk.size()}; + + fakedArgs.second->args[1] = (char*)hexSk.data(); + fakedArgs.second->lengths[1] = hexSk.size(); + + std::cout << "args[0]" << fakedArgs.second->args[0] << std::endl; + std::cout << "lengths[0]" << fakedArgs.second->lengths[0] << std::endl; + std::cout << "args[1]" << fakedArgs.second->args[1] << std::endl; + std::cout << "lengths[1]" << fakedArgs.second->lengths[1] << std::endl; + + std::cout << "### before init" << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + auto ret = ore_dec_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + return; + } + std::cout << "#### init success" << std::endl; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto decryptV = ore_dec(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " ore execution time: " << duration << " microseconds" << std::endl; + auto plainText = plainTextBytesBuffer->data; + std::cout << "#### ore pV: " << plainText << ", len:" << plainTextBytesBuffer->len << std::endl; + std::cout << "#### ore dV: " << decryptV << ", len:" << strlen(decryptV) << std::endl; + BOOST_CHECK(*plainText == *decryptV); + ore_dec_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testOreDecUDFImpl() +{ + srand(bcos::utcSteadyTime()); + std::string plainText = "19014527865"; + bcos::bytes plainTextBytes(plainText.begin(), plainText.end()); + InputBuffer plainTextBytesBuffer{ + (const unsigned char*)plainTextBytes.data(), plainTextBytes.size()}; + + bcos::bytes skBytes(fast_ore_key_bytes(), 0); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + fast_ore_generate_key(&skBuffer); + + + InputBuffer skBytesBuffer{(const unsigned char*)skBytes.data(), skBytes.size()}; + testOreDec(&plainTextBytesBuffer, &skBytesBuffer); +} + +BOOST_AUTO_TEST_CASE(testOreDecUDF) +{ + testOreDecUDFImpl(); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/OreEncUDFTest.cpp b/cpp/ppc-udf/tests/OreEncUDFTest.cpp new file mode 100644 index 00000000..e789af1e --- /dev/null +++ b/cpp/ppc-udf/tests/OreEncUDFTest.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file OreEncUDFTest.cpp + * @author: caryliao + * @date 2023-11-30 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/fast_ore.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-udf/mysql/ore_enc.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(oreEncUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters() +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + int argCount = 2; + args->arg_count = argCount; + args->args = new char*[argCount]; + args->lengths = new unsigned long[argCount]; + args->arg_type = new Item_result[argCount]; + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = STRING_RESULT; + + return std::make_pair(initid, args); +} + +void testOreEnc( + InputBuffer const* plainTextBytesBuffer, InputBuffer const* skBytesBuffer) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + auto fakedArgs = fakeMySQLParameters(); + std::cout << "#### testOreEnc" << std::endl; + bool isHex = true; + bcos::bytes cipherData(fast_ore_get_cipher_size(plainTextBytesBuffer->len, isHex)); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + string_fast_ore_encrypt(&cipherBuffer, skBytesBuffer, plainTextBytesBuffer, isHex); + // InputBuffer inputBufferCipher{cipherBuffer.data, cipherBuffer.len}; + + // convert cipherBytes to hex + // auto hexCipher = *(bcos::toHexString(cipherData.data(), cipherData.data() + cipherBuffer.len)); + // std::cout << "#### hexCipher: " << hexCipher << std::endl; + // fakedArgs.second->args[0] = (char*)hexCipher.data(); + // fakedArgs.second->lengths[0] = hexCipher.size(); + fakedArgs.second->args[0] = (char*)plainTextBytesBuffer->data; + fakedArgs.second->lengths[0] = plainTextBytesBuffer->len; + + // std::cout << "#### encrypt" << std::endl; + // bcos::bytes plain(plainTextBytesBuffer->len); + // OutputBuffer plainBuffer{plain.data(), plain.size()}; + // string_fast_ore_encrypt(&plainBuffer, skBytesBuffer, &inputBufferCipher, isHex); + // std::cout << "#### encrypt end: " << plainBuffer.data << "# len: " << + // plainBuffer.len << ", size: " << plain.size()<data, skBytesBuffer->data + skBytesBuffer->len)); + InputBuffer inputBufferHexSk{(const unsigned char*)hexSk.data(), hexSk.size()}; + + fakedArgs.second->args[1] = (char*)hexSk.data(); + fakedArgs.second->lengths[1] = hexSk.size(); + + std::cout << "args[0]" << fakedArgs.second->args[0] << std::endl; + std::cout << "lengths[0]" << fakedArgs.second->lengths[0] << std::endl; + std::cout << "args[1]" << fakedArgs.second->args[1] << std::endl; + std::cout << "lengths[1]" << fakedArgs.second->lengths[1] << std::endl; + + std::cout << "### before init" << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + auto ret = ore_enc_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + return; + } + std::cout << "#### init success" << std::endl; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto encryptV = ore_enc(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " ore execution time: " << duration << " microseconds" << std::endl; + auto plainText = plainTextBytesBuffer->data; + std::cout << "#### ore pV: " << plainText << ", len:" << plainTextBytesBuffer->len << std::endl; + std::cout << "#### ore dV: " << encryptV << ", len:" << strlen(encryptV) << std::endl; + BOOST_CHECK(*cipherBuffer.data == *encryptV); + ore_enc_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testOreEncUDFImpl() +{ + srand(bcos::utcSteadyTime()); + std::string plainText = "19014527865"; + bcos::bytes plainTextBytes(plainText.begin(), plainText.end()); + InputBuffer plainTextBytesBuffer{ + (const unsigned char*)plainTextBytes.data(), plainTextBytes.size()}; + + bcos::bytes skBytes(fast_ore_key_bytes(), 0); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + fast_ore_generate_key(&skBuffer); + + + InputBuffer skBytesBuffer{(const unsigned char*)skBytes.data(), skBytes.size()}; + testOreEnc(&plainTextBytesBuffer, &skBytesBuffer); +} + +BOOST_AUTO_TEST_CASE(testOreEncUDF) +{ + testOreEncUDFImpl(); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/PaillierDecUDFTest.cpp b/cpp/ppc-udf/tests/PaillierDecUDFTest.cpp new file mode 100644 index 00000000..7b1154d2 --- /dev/null +++ b/cpp/ppc-udf/tests/PaillierDecUDFTest.cpp @@ -0,0 +1,299 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PaillierDecryptionUDFTest.cpp + * @author: caryliao + * @date 2023-10-24 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include "ppc-udf/mysql/paillier_dec.h" +#include +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; +using namespace ppc::homo; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(PaillierDecryptionUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters( + InputBuffer const* cipherBytes, InputBuffer const* pkBytes, InputBuffer const* skBytes) +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + args->arg_count = 4; + args->args = new char*[4]; + args->lengths = new unsigned long[4]; + args->arg_type = new Item_result[4]; + if (cipherBytes) + { + args->args[0] = (char*)cipherBytes->data; + args->lengths[0] = cipherBytes->len; + } + args->args[1] = (char*)pkBytes->data; + args->lengths[1] = pkBytes->len; + args->args[2] = (char*)skBytes->data; + args->lengths[2] = skBytes->len; + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = STRING_RESULT; + args->arg_type[2] = STRING_RESULT; + args->arg_type[3] = INT_RESULT; + return std::make_pair(initid, args); +} + +void testPaillierDecInt(std::vector const& values, InputBuffer const* pkBytes, + InputBuffer const* skBytes, int keyBits, void* keypair) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + auto fakedArgs = fakeMySQLParameters(nullptr, pkBytes, skBytes); + std::cout << "#### testPaillierDec" << std::endl; + int i = 0; + for (auto it : values) + { + BigNum v(it); + bcos::bytes cipherData(floating_point_paillier_cipher_bytes(keyBits), 0); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + int16_t exponent = 0; + auto _keyPair = (OpenSSLPaillierKeyPair*)keypair; + auto pk = _keyPair->pk(); + floating_point_paillier_encrypt(&cipherBuffer, v.bn().get(), exponent, pk); + if (get_last_error() != 0) + { + const char* errorMsg = get_last_error_msg(); + std::cout << "#### floating_point_paillier_encrypt error:" << errorMsg << std::endl; + return; + } + InputBuffer cipherBytes{cipherData.data(), cipherData.size()}; + // convert cipherBytes to hex + auto hexCipher = *(bcos::toHexString(cipherData)); + std::cout << "#### hexCipher: " << hexCipher << std::endl; + fakedArgs.second->args[0] = (char*)hexCipher.data(); + fakedArgs.second->lengths[0] = hexCipher.size(); + // fakedArgs.second->args[3] = (char*)"0"; + // fakedArgs.second->lengths[3] = 1; + int scale = 0; + fakedArgs.second->args[3] = (char*)&scale; + fakedArgs.second->lengths[3] = sizeof(int*); + // init for the fist element + if (i == 0) + { + std::cout << "#### before init" << std::endl; + auto ret = paillier_dec_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + break; + } + std::cout << "#### init success" << std::endl; + } + i++; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto decryptV = + paillier_dec(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + std::cout << "#### int pV: " << it << std::endl; + std::cout << "#### int dV: " << decryptV << std::endl; + BOOST_CHECK(decryptV == it); + } + paillier_dec_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testPaillierDoubleDec(std::vector const& values, InputBuffer const* pkBytes, + InputBuffer const* skBytes, int keyBits, void* keypair) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + auto fakedArgs = fakeMySQLParameters(nullptr, pkBytes, skBytes); + std::cout << "#### testPaillierDoubleDec" << std::endl; + int i = 0; + auto codec = std::make_shared(); + for (auto strValue : values) + { + auto it = float50(strValue); + auto fpn = codec->toFloatingPoint(strValue); + bcos::bytes cipherData(floating_point_paillier_cipher_bytes(keyBits), 0); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + int16_t exponent = fpn.exponent; + auto _keyPair = (OpenSSLPaillierKeyPair*)keypair; + auto pk = _keyPair->pk(); + floating_point_paillier_encrypt(&cipherBuffer, fpn.value.bn().get(), exponent, pk); + if (get_last_error() != 0) + { + const char* errorMsg = get_last_error_msg(); + std::cout << "#### floating_point_paillier_encrypt error:" << errorMsg << std::endl; + return; + } + InputBuffer cipherBytes{cipherData.data(), cipherData.size()}; + // convert cipherBytes to hex + auto hexCipher = *(bcos::toHexString(cipherData)); + std::cout << "#### hexCipher: " << hexCipher << std::endl; + fakedArgs.second->args[0] = (char*)hexCipher.data(); + fakedArgs.second->lengths[0] = hexCipher.size(); + int scale = 2; + fakedArgs.second->args[3] = (char*)&scale; + fakedArgs.second->lengths[3] = sizeof(int*); + // init for the fist element + auto start = std::chrono::high_resolution_clock::now(); + if (i == 0) + { + // std::cout << "#### before init" << std::endl; + auto ret = paillier_dec_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + break; + } + std::cout << "#### init success" << std::endl; + } + i++; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto decryptV = + paillier_dec(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " paillier execution time: " << duration << " microseconds" << std::endl; + + double multiplier = pow(10.0, scale); + double roundPv = round((double)it * multiplier) / multiplier; + std::cout << "#### double pV: " << roundPv << std::endl; + std::cout << "#### double dV: " << decryptV << std::endl; + BOOST_CHECK(decryptV == roundPv); + } + paillier_dec_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testPaillierDecryptionUDFIntImpl(int iter) +{ + srand(bcos::utcSteadyTime()); + int64_t m = 0; + std::vector values(iter); + for (int64_t i = 0; i < iter; i++) + { + m = 12323434 + rand(); + if (i % 3 == 0) + { + m = -1234324 - rand(); + } + values[i] = m; + } + int keyBits = 2048; + auto keypair = paillier_generate_keypair(keyBits); + + bcos::bytes pkBytes(paillier_max_public_key_bytes(keyBits), 0); + bcos::bytes skBytes(paillier_max_private_key_bytes(keyBits), 0); + OutputBuffer pkBuffer{pkBytes.data(), pkBytes.size()}; + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + paillier_set_public_key_bytes_from_keypair(&pkBuffer, keypair); + paillier_set_private_key_bytes_from_keypair(&skBuffer, keypair); + + // convert to hexedPk + auto hexPk = *(bcos::toHexString(pkBytes)); + auto hexSk = *(bcos::toHexString(skBytes)); + InputBuffer pkBytesBuffer{(const unsigned char*)hexPk.data(), hexPk.size()}; + InputBuffer skBytesBuffer{(const unsigned char*)hexSk.data(), hexSk.size()}; + std::cout << "#### hexPk:" << pkBytesBuffer.data << std::endl; + std::cout << "#### hexSk:" << skBytesBuffer.data << std::endl; + testPaillierDecInt(values, &pkBytesBuffer, &skBytesBuffer, keyBits, keypair); +} + +void testPaillierDecryptionUDFDoubleImpl(int iter) +{ + srand(bcos::utcSteadyTime()); + std::vector values; + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_real_distribution distribution(-109900234.23432434, 10990023423432434); + for (int i = 0; i < iter; ++i) + { + float50 randomValue = float50(distribution(generator)); + values.push_back(randomValue.str(10)); + } + for (const auto& value : values) + { + std::cout << value << " "; + } + int keyBits = 2048; + auto keypair = paillier_generate_keypair(keyBits); + + bcos::bytes pkBytes(paillier_max_public_key_bytes(keyBits), 0); + bcos::bytes skBytes(paillier_max_private_key_bytes(keyBits), 0); + OutputBuffer pkBuffer{pkBytes.data(), pkBytes.size()}; + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + paillier_set_public_key_bytes_from_keypair(&pkBuffer, keypair); + paillier_set_private_key_bytes_from_keypair(&skBuffer, keypair); + + // convert to hexedPk + auto hexPk = *(bcos::toHexString(pkBytes)); + auto hexSk = *(bcos::toHexString(skBytes)); + InputBuffer pkBytesBuffer{(const unsigned char*)hexPk.data(), hexPk.size()}; + InputBuffer skBytesBuffer{(const unsigned char*)hexSk.data(), hexSk.size()}; + std::cout << "#### hexPk:" << pkBytesBuffer.data << std::endl; + std::cout << "#### hexSk:" << skBytesBuffer.data << std::endl; + testPaillierDoubleDec(values, &pkBytesBuffer, &skBytesBuffer, keyBits, keypair); +} + +BOOST_AUTO_TEST_CASE(testPaillierDecryptionUDF) +{ + testPaillierDecryptionUDFIntImpl(3); + testPaillierDecryptionUDFDoubleImpl(3); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/PaillierEncUDFTest.cpp b/cpp/ppc-udf/tests/PaillierEncUDFTest.cpp new file mode 100644 index 00000000..5b1c7f93 --- /dev/null +++ b/cpp/ppc-udf/tests/PaillierEncUDFTest.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PaillierEncryptionUDFTest.cpp + * @author: caryliao + * @date 2023-10-24 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-udf/mysql/paillier_enc.h" +#include +#include +#include +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-homo/paillier/OpenSSLPaillier.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; +using namespace ppc::homo; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(PaillierEncryptionUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters( + InputBuffer const* cipherBytes, InputBuffer const* pkBytes, InputBuffer const* skBytes) +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + args->arg_count = 2; + args->args = new char*[args->arg_count]; + args->lengths = new unsigned long[args->arg_count]; + args->arg_type = new Item_result[args->arg_count]; + args->args[1] = (char*)pkBytes->data; + args->lengths[1] = pkBytes->len; + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = STRING_RESULT; + return std::make_pair(initid, args); +} + +void testPaillierDoubleEnc(std::vector const& values, InputBuffer const* pkBytes, + InputBuffer const* skBytes, int keyBits, void* keypair) +{ + char* message = new char[1000]; + char* error = new char[1000]; + char* result = new char[10000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + memset(result, 0, 10000); + auto fakedArgs = fakeMySQLParameters(nullptr, pkBytes, skBytes); + std::cout << "#### testPaillierDoubleEnc" << std::endl; + int i = 0; + auto codec = std::make_shared(); + for (auto it : values) + { + // fakedArgs.second->args[0] = (char*)⁢ + const char* strValue = it.c_str(); + fakedArgs.second->args[0] = (char*)strValue; + fakedArgs.second->lengths[0] = strlen(strValue); + // init for the fist element + auto start = std::chrono::high_resolution_clock::now(); + if (i == 0) + { + // std::cout << "#### before init" << std::endl; + auto ret = paillier_enc_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + break; + } + std::cout << "#### init success" << std::endl; + } + i++; + // obtain the result + unsigned long length = 0; + auto cipherValue = + paillier_enc(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " paillier enc execution time: " << duration << " microseconds" << std::endl; + std::cout << "#### cipherValue: " << cipherValue << std::endl; + + InputBuffer cipherBuffer{(unsigned char*)cipherValue, length}; + BigNum decryptV; + int16_t exponent; + floating_point_paillier_decrypt(decryptV.bn().get(), &exponent, &cipherBuffer, keypair); + if (get_last_error()) + { + std::cout << "#### floating_point_paillier_decrypt error: " << get_last_error_msg() + << std::endl; + } + BOOST_CHECK(get_last_error() == 0); + FloatingPointNumber floatingPointNumber(std::move(decryptV), exponent); + BN_print_fp(stdout, floatingPointNumber.value.bn().get()); + auto decryptedResult = codec->toFloat50(floatingPointNumber); + std::cout << "#### decryptedResult: " << decryptedResult << std::endl; + std::cout << "#### float50(it): " << float50(it) << std::endl; + // BOOST_CHECK(decryptedResult == float50(it)); + } + paillier_enc_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testPaillierEncryptionUDFDoubleImpl(int iter) +{ + srand(bcos::utcSteadyTime()); + std::vector values; + std::random_device rd; + std::mt19937 generator(rd()); + std::uniform_real_distribution distribution(0.0, 1.0); + // for (int i = 0; i < iter; ++i) { + // double randomValue = distribution(generator); + // values.push_back(randomValue*10000); + // } + values.push_back("123.56"); + values.push_back("1000.89"); + values.push_back("2000.76"); + for (const auto& value : values) { + std::cout << value << " "; + } + int keyBits = 2048; + auto keypair = paillier_generate_keypair(keyBits); + + bcos::bytes pkBytes(paillier_max_public_key_bytes(keyBits), 0); + bcos::bytes skBytes(paillier_max_private_key_bytes(keyBits), 0); + OutputBuffer pkBuffer{pkBytes.data(), pkBytes.size()}; + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + paillier_set_public_key_bytes_from_keypair(&pkBuffer, keypair); + paillier_set_private_key_bytes_from_keypair(&skBuffer, keypair); + + // convert to hexedPk + auto hexPk = *(bcos::toHexString(pkBytes)); + auto hexSk = *(bcos::toHexString(skBytes)); + InputBuffer pkBytesBuffer{(const unsigned char*)hexPk.data(), hexPk.size()}; + InputBuffer skBytesBuffer{(const unsigned char*)hexSk.data(), hexSk.size()}; + std::cout << "#### hexPk:" << pkBytesBuffer.data << std::endl; + std::cout << "#### hexSk:" << skBytesBuffer.data << std::endl; + testPaillierDoubleEnc(values, &pkBytesBuffer, &skBytesBuffer, keyBits, keypair); +} + +BOOST_AUTO_TEST_CASE(testPaillierEncryptionUDF) +{ + testPaillierEncryptionUDFDoubleImpl(3); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/PaillierSumUDFTest.cpp b/cpp/ppc-udf/tests/PaillierSumUDFTest.cpp new file mode 100644 index 00000000..7b93fcc6 --- /dev/null +++ b/cpp/ppc-udf/tests/PaillierSumUDFTest.cpp @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PaillierSumUDFTest.cpp + * @author: yujiechen + * @date 2023-08-22 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/floating_point_paillier.h" +#include "ppc-crypto-c-sdk/homo_paillier.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-homo/codec/FloatingPointCodec.h" +#include "ppc-udf/mysql/paillier_sum.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::homo; +using namespace ppc::crypto; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(paillierSumUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters( + InputBuffer const* cipherBytes, InputBuffer const* pkBytes) +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + args->arg_count = 2; + args->args = new char*[2]; + if (cipherBytes) + { + args->args[0] = (char*)cipherBytes->data; + } + args->args[1] = (char*)pkBytes->data; + args->lengths = new unsigned long[2]; + if (cipherBytes) + { + args->lengths[0] = cipherBytes->len; + } + args->lengths[1] = pkBytes->len; + return std::make_pair(initid, args); +} + +void testPaillierSum( + std::vector const& values, InputBuffer const* pkBytes, int keyBits, void* keypair) +{ + int i = 0; + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + float50 expectedSum = 0; + auto fakedArgs = fakeMySQLParameters(nullptr, pkBytes); + auto codec = std::make_shared(); + for (auto it : values) + { + expectedSum += it; + bcos::bytes cipherData(floating_point_paillier_cipher_bytes(keyBits), 0); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + + auto ffpNumber = codec->toFloatingPoint(it); + BN_print_fp(stdout, ffpNumber.value.bn().get()); + floating_point_paillier_encrypt_fast( + &cipherBuffer, ffpNumber.value.bn().get(), ffpNumber.exponent, keypair); + + InputBuffer cipherBytes{cipherData.data(), cipherBuffer.len}; + // convert cipherBytes to hex + auto hexCipher = *(bcos::toHexString(cipherData)); + fakedArgs.second->args[0] = (char*)hexCipher.data(); + fakedArgs.second->lengths[0] = hexCipher.size(); + // init for the fist element + if (i == 0) + { + auto ret = paillier_sum_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + break; + } + } + // add for every element + paillier_sum_add(fakedArgs.first, fakedArgs.second, nullptr, error); + i++; + } + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto retResult = + paillier_sum(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + // check the result + std::string hexCipherSum = std::string(retResult, retResult + length); + + bcos::bytes bytesCipherSum = *(bcos::fromHexString(hexCipherSum)); + InputBuffer sumCipherBuffer{bytesCipherSum.data(), bytesCipherSum.size()}; + BigNum decryptV; + int16_t exponent; + floating_point_paillier_decrypt(decryptV.bn().get(), &exponent, &sumCipherBuffer, keypair); + if (get_last_error()) + { + std::cout << "#### floating_point_paillier_decrypt error: " << get_last_error_msg() + << std::endl; + } + BOOST_CHECK(get_last_error() == 0); + FloatingPointNumber floatingPointNumber(std::move(decryptV), exponent); + BN_print_fp(stdout, floatingPointNumber.value.bn().get()); + auto decryptedResult = codec->toFloat50(floatingPointNumber); + BOOST_CHECK(decryptedResult == expectedSum); + + paillier_sum_clear(fakedArgs.first, NULL, NULL); + paillier_sum_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testPaillierSumUDFImpl(int iter) +{ + srand(bcos::utcSteadyTime()); + int64_t m = 0; + std::vector values(iter); + for (int64_t i = 0; i < iter; i++) + { + m = 12323434 + rand(); + if (i % 3 == 0) + { + m = -1234324 - rand(); + } + values[i] = m; + } + int keyBits = 2048; + auto keypair = paillier_generate_keypair(keyBits); + + bcos::bytes pkBytes(paillier_max_public_key_bytes(keyBits), 0); + OutputBuffer pkBuffer{pkBytes.data(), pkBytes.size()}; + paillier_set_public_key_bytes_from_keypair(&pkBuffer, keypair); + + // convert to hexedPk + auto hexPk = *(bcos::toHexString(pkBytes)); + InputBuffer pkBytesBuffer{(const unsigned char*)hexPk.data(), hexPk.size()}; + testPaillierSum(values, &pkBytesBuffer, keyBits, keypair); +} + +BOOST_AUTO_TEST_CASE(testPaillierSumUDF) +{ + testPaillierSumUDFImpl(1); + testPaillierSumUDFImpl(2); + testPaillierSumUDFImpl(3); + testPaillierSumUDFImpl(10); +} + +BOOST_AUTO_TEST_CASE(testInvalidPaillierSum) +{ + std::string pk = "234324"; + std::string cipher = "234234"; + InputBuffer cipherBuffer{(const unsigned char*)cipher.data(), cipher.size()}; + InputBuffer pkBuffer{(const unsigned char*)pk.data(), pk.size()}; + auto ret = fakeMySQLParameters(&cipherBuffer, &pkBuffer); + char* message = new char[1000]; + paillier_sum_init(ret.first, ret.second, message); + if (ret.first) + { + delete ret.first; + } + if (ret.second) + { + delete ret.second; + } + if (message) + { + delete[] message; + } +} +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-udf/tests/SymDecUDFTest.cpp b/cpp/ppc-udf/tests/SymDecUDFTest.cpp new file mode 100644 index 00000000..0151ee74 --- /dev/null +++ b/cpp/ppc-udf/tests/SymDecUDFTest.cpp @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SymDecUDFTest.cpp + * @author: caryliao + * @date 2023-10-25 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/symmetric_encryption.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/crypto/SymCrypto.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-udf/mysql/sym_dec.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(symDecUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters(InputBuffer const* cipherBytes, + int* algorithmType, int* mode, InputBuffer const* ivBytesBuffer, + InputBuffer const* skBytesBuffer) +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + args->arg_count = 5; + args->args = new char*[5]; + args->lengths = new unsigned long[5]; + args->arg_type = new Item_result[5]; + if (cipherBytes) + { + args->args[0] = (char*)cipherBytes->data; + args->lengths[0] = cipherBytes->len; + } + args->args[1] = (char*)algorithmType; + args->lengths[1] = sizeof(int); + args->args[2] = (char*)mode; + args->lengths[2] = sizeof(int); + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = INT_RESULT; + args->arg_type[2] = INT_RESULT; + args->arg_type[3] = STRING_RESULT; + args->arg_type[4] = STRING_RESULT; + return std::make_pair(initid, args); +} + +void testSymDec(InputBuffer const* plainTextBytesBuffer, int algorithmType, int mode, + InputBuffer const* ivBytesBuffer, InputBuffer const* skBytesBuffer) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + BigNum expectedSum(0); + auto fakedArgs = + fakeMySQLParameters(nullptr, &algorithmType, &mode, ivBytesBuffer, skBytesBuffer); + std::cout << "#### testSymDec" << std::endl; + bcos::bytes cipherData(symmetric_block_size(algorithmType) + plainTextBytesBuffer->len); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + symmetric_encrypt( + &cipherBuffer, algorithmType, mode, skBytesBuffer, ivBytesBuffer, plainTextBytesBuffer); + + // encrypt and decrypt test + // std::cout << "### before dec, plain: " + // << std::string(plainTextBytesBuffer->data, + // plainTextBytesBuffer->data + plainTextBytesBuffer->len) + // << std::endl; + // std::cout << "#### decrypt, cipher len: " << cipherBuffer.len << std::endl; + // bcos::bytes plain(cipherBuffer.len + symmetric_block_size(algorithmType)); + // OutputBuffer plainBuffer{plain.data(), plain.size()}; + // InputBuffer cipherBuffer2{cipherBuffer.data, cipherBuffer.len}; + // symmetric_decrypt( + // &plainBuffer, algorithmType, mode, skBytesBuffer, ivBytesBuffer, &cipherBuffer2); + // std::cout << "#### error: " << get_last_error_msg() << std::endl; + // std::cout << "#### decrypt end: " << std::string(plain.begin(), plain.end()) + // << "# len: " << plainBuffer.len << ", size: " << plain.size() << std::endl; + + InputBuffer cipherBytes{cipherData.data(), cipherBuffer.len}; + // convert cipherBytes to hex + auto hexCipher = *(bcos::toHexString(cipherData.data(), cipherData.data() + cipherBuffer.len)); + std::cout << "#### hexCipher: " << hexCipher << std::endl; + fakedArgs.second->args[0] = (char*)hexCipher.data(); + fakedArgs.second->lengths[0] = hexCipher.size(); + + auto hexIv = + *(bcos::toHexString(ivBytesBuffer->data, ivBytesBuffer->data + ivBytesBuffer->len)); + // InputBuffer ivHexBuffer{(const unsigned char*)hexIv.data(), hexIv.size()}; + fakedArgs.second->args[3] = (char*)hexIv.data(); + fakedArgs.second->lengths[3] = hexIv.size(); + std::cout << "#### fakeMySQLParameters iv:" << fakedArgs.second->args[3] << std::endl; + std::cout << "#### fakeMySQLParameters iv len:" << fakedArgs.second->lengths[3] << std::endl; + + auto hexSk = + *(bcos::toHexString(skBytesBuffer->data, skBytesBuffer->data + skBytesBuffer->len)); + // InputBuffer skHexBuffer{(const unsigned char*)hexSk.data(), hexSk.size()}; + fakedArgs.second->args[4] = (char*)hexSk.data(); + fakedArgs.second->lengths[4] = hexSk.size(); + std::cout << "fakeMySQLParameters sk:" << fakedArgs.second->args[4] << std::endl; + std::cout << "fakeMySQLParameters sk len:" << fakedArgs.second->lengths[4] << std::endl; + + std::cout << "args[0]" << fakedArgs.second->args[0] << std::endl; + std::cout << "lengths[0]" << fakedArgs.second->lengths[0] << std::endl; + std::cout << "args[1]" << fakedArgs.second->args[1] << std::endl; + std::cout << "lengths[1]" << fakedArgs.second->lengths[1] << std::endl; + std::cout << "args[2]" << fakedArgs.second->args[2] << std::endl; + std::cout << "lengths[2]" << fakedArgs.second->lengths[2] << std::endl; + std::cout << "args[3]" << fakedArgs.second->args[3] << std::endl; + std::cout << "lengths[3]" << fakedArgs.second->lengths[3] << std::endl; + std::cout << "args[4]" << fakedArgs.second->args[4] << std::endl; + std::cout << "lengths[4]" << fakedArgs.second->lengths[4] << std::endl; + + std::cout << "#### before init" << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + auto ret = sym_dec_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + return; + } + std::cout << "#### init success" << std::endl; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto decryptV = sym_dec(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " sym execution time: " << duration << " microseconds" << std::endl; + auto plainText = plainTextBytesBuffer->data; + std::cout << "#### sym pV: " << plainText << std::endl; + std::cout << "#### sym pV len: " << plainTextBytesBuffer->len << std::endl; + std::cout << "#### sym dV: " << decryptV << std::endl; + std::cout << "#### sym dV len : " << strlen(decryptV) << std::endl; + BOOST_CHECK(*plainText == *decryptV); + sym_dec_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testSymDecUDFImpl(int algorithmType, int mode) +{ + srand(bcos::utcSteadyTime()); + // std::string plainText = "Hello, World, Hello Tomorrow."; + std::string plainText = "caryliao"; + bcos::bytes plainTextBytes(plainText.begin(), plainText.end()); + // auto hexPlain = *(bcos::toHexString(plainTextBytes)); + // InputBuffer plainTextBytesBuffer{(const unsigned char*)hexPlain.data(), hexPlain.size()}; + InputBuffer plainTextBytesBuffer{ + (const unsigned char*)plainTextBytes.data(), plainTextBytes.size()}; + std::cout << "plain:" << plainTextBytesBuffer.data << std::endl; + + std::string iv = "0123456789abcdef"; + bcos::bytes ivBytes(iv.begin(), iv.end()); + InputBuffer ivBytesBuffer{(const unsigned char*)ivBytes.data(), ivBytes.size()}; + std::cout << "#### symmetric_key_bytes(algorithmType, mode):" + << symmetric_key_bytes(algorithmType, mode) << std::endl; + if (get_last_error() != 0) + { + std::cout << "error: " << get_last_error_msg() << std::endl; + } + bcos::bytes skBytes(symmetric_key_bytes(algorithmType, mode), 0); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + symmetric_generate_key(&skBuffer, algorithmType, mode); + + // convert to hexedSk + InputBuffer skBytesBuffer{(const unsigned char*)skBytes.data(), skBytes.size()}; + testSymDec(&plainTextBytesBuffer, algorithmType, mode, &ivBytesBuffer, &skBytesBuffer); +} + +BOOST_AUTO_TEST_CASE(testSymDecUDF) +{ + std::cout << "run AES_128 ECB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run AES_128 CBC" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run AES_128 CFB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run AES_128 CTR" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CTR)); + std::cout << "run AES_192 OFB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_192), static_cast(SymCrypto::OperationMode::OFB)); + std::cout << "run AES_256 CTR" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::AES_256), static_cast(SymCrypto::OperationMode::CTR)); + + std::cout << "run TrippleDES ECB" << std::endl; + testSymDecUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run TrippleDES CBC" << std::endl; + testSymDecUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run TrippleDES CFB" << std::endl; + testSymDecUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run TrippleDES OFB" << std::endl; + testSymDecUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::OFB)); + + std::cout << "run SM4 ECB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run SM4 CBC" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run SM4 CFB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run SM4 OFB" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::OFB)); + std::cout << "run SM4 CTR" << std::endl; + testSymDecUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CTR)); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/SymEncUDFTest.cpp b/cpp/ppc-udf/tests/SymEncUDFTest.cpp new file mode 100644 index 00000000..b3197438 --- /dev/null +++ b/cpp/ppc-udf/tests/SymEncUDFTest.cpp @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SymEncUDFTest.cpp + * @author: caryliao + * @date 2023-10-25 + */ +#include "openssl/bn.h" +#include "ppc-crypto-c-sdk/symmetric_encryption.h" +#include "ppc-crypto-c-sdk/utils/error.h" +#include "ppc-framework/crypto/SymCrypto.h" +#include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-udf/mysql/sym_enc.h" +#include +#include +#include + +using namespace bcos; +using namespace bcos::test; +using namespace ppc; +using namespace ppc::crypto; + +namespace ppc::test +{ +BOOST_FIXTURE_TEST_SUITE(symEncUDFTest, TestPromptFixture) + +std::pair fakeMySQLParameters(InputBuffer const* plainBytes, + int* algorithmType, int* mode, InputBuffer const* ivBytesBuffer, InputBuffer const* skBytesBuffer) +{ + UDF_INIT* initid = new UDF_INIT(); + UDF_ARGS* args = new UDF_ARGS(); + args->arg_count = 5; + args->args = new char*[5]; + args->lengths = new unsigned long[5]; + args->arg_type = new Item_result[5]; + if (plainBytes) + { + args->args[0] = (char*)plainBytes->data; + args->lengths[0] = plainBytes->len; + } + args->args[1] = (char*)algorithmType; + args->lengths[1] = sizeof(int); + args->args[2] = (char*)mode; + args->lengths[2] = sizeof(int); + + args->arg_type[0] = STRING_RESULT; + args->arg_type[1] = INT_RESULT; + args->arg_type[2] = INT_RESULT; + args->arg_type[3] = STRING_RESULT; + args->arg_type[4] = STRING_RESULT; + return std::make_pair(initid, args); +} + +void testSymEnc(InputBuffer const* plainTextBytesBuffer, int algorithmType, int mode, + InputBuffer const* ivBytesBuffer, InputBuffer const* skBytesBuffer) +{ + char* message = new char[1000]; + char* error = new char[1000]; + memset(message, 0, 1000); + memset(error, 0, 1000); + BigNum expectedSum(0); + auto fakedArgs = + fakeMySQLParameters(nullptr, &algorithmType, &mode, ivBytesBuffer, skBytesBuffer); + std::cout << "#### testSymEnc" << std::endl; + fakedArgs.second->args[0] = (char*)plainTextBytesBuffer->data; + fakedArgs.second->lengths[0] = plainTextBytesBuffer->len; + + auto hexIv = + *(bcos::toHexString(ivBytesBuffer->data, ivBytesBuffer->data + ivBytesBuffer->len)); + // InputBuffer ivHexBuffer{(const unsigned char*)hexIv.data(), hexIv.size()}; + fakedArgs.second->args[3] = (char*)hexIv.data(); + fakedArgs.second->lengths[3] = hexIv.size(); + std::cout << "#### fakeMySQLParameters iv:" << fakedArgs.second->args[3] << std::endl; + std::cout << "#### fakeMySQLParameters iv len:" << fakedArgs.second->lengths[3] << std::endl; + + auto hexSk = + *(bcos::toHexString(skBytesBuffer->data, skBytesBuffer->data + skBytesBuffer->len)); + // InputBuffer skHexBuffer{(const unsigned char*)hexSk.data(), hexSk.size()}; + fakedArgs.second->args[4] = (char*)hexSk.data(); + fakedArgs.second->lengths[4] = hexSk.size(); + std::cout << "fakeMySQLParameters sk:" << fakedArgs.second->args[4] << std::endl; + std::cout << "fakeMySQLParameters sk len:" << fakedArgs.second->lengths[4] << std::endl; + + std::cout << "args[0]" << fakedArgs.second->args[0] << std::endl; + std::cout << "lengths[0]" << fakedArgs.second->lengths[0] << std::endl; + std::cout << "args[1]" << fakedArgs.second->args[1] << std::endl; + std::cout << "lengths[1]" << fakedArgs.second->lengths[1] << std::endl; + std::cout << "args[2]" << fakedArgs.second->args[2] << std::endl; + std::cout << "lengths[2]" << fakedArgs.second->lengths[2] << std::endl; + std::cout << "args[3]" << fakedArgs.second->args[3] << std::endl; + std::cout << "lengths[3]" << fakedArgs.second->lengths[3] << std::endl; + std::cout << "args[4]" << fakedArgs.second->args[4] << std::endl; + std::cout << "lengths[4]" << fakedArgs.second->lengths[4] << std::endl; + + std::cout << "#### before init" << std::endl; + auto start = std::chrono::high_resolution_clock::now(); + auto ret = sym_enc_init(fakedArgs.first, fakedArgs.second, message); + // init failed + if (ret) + { + std::cout << "#### init fail" << std::endl; + return; + } + std::cout << "#### init success" << std::endl; + // obtain the result + char* result = new char[fakedArgs.first->max_length]; + unsigned long length; + auto cipherValue = sym_enc(fakedArgs.first, fakedArgs.second, result, &length, nullptr, error); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start).count(); + // Print the execution time + std::cout << " sym execution time: " << duration << " microseconds" << std::endl; + std::cout << "#### sym cipher: " << cipherValue << std::endl; + std::cout << "#### sym cipher len : " << strlen(cipherValue) << std::endl; + + bcos::bytes cipherData(symmetric_block_size(algorithmType) + plainTextBytesBuffer->len); + OutputBuffer cipherBuffer{cipherData.data(), cipherData.size()}; + symmetric_encrypt( + &cipherBuffer, algorithmType, mode, skBytesBuffer, ivBytesBuffer, plainTextBytesBuffer); + + BOOST_CHECK(strlen(cipherValue) != 0); + BOOST_CHECK(*cipherData.data() = *cipherValue); + sym_enc_deinit(fakedArgs.first); + delete fakedArgs.first; + delete fakedArgs.second; + if (message) + { + delete[] message; + } + if (error) + { + delete[] error; + } +} + +void testSymEncUDFImpl(int algorithmType, int mode) +{ + srand(bcos::utcSteadyTime()); + // std::string plainText = "Hello, World, Hello Tomorrow."; + std::string plainText = "caryliao"; + bcos::bytes plainTextBytes(plainText.begin(), plainText.end()); + // auto hexPlain = *(bcos::toHexString(plainTextBytes)); + // InputBuffer plainTextBytesBuffer{(const unsigned char*)hexPlain.data(), hexPlain.size()}; + InputBuffer plainTextBytesBuffer{ + (const unsigned char*)plainTextBytes.data(), plainTextBytes.size()}; + std::cout << "plain:" << plainTextBytesBuffer.data << std::endl; + + std::string iv = "0123456789abcdef"; + bcos::bytes ivBytes(iv.begin(), iv.end()); + InputBuffer ivBytesBuffer{(const unsigned char*)ivBytes.data(), ivBytes.size()}; + std::cout << "#### symmetric_key_bytes(algorithmType, mode):" + << symmetric_key_bytes(algorithmType, mode) << std::endl; + std::cout << "error: " << get_last_error_msg() << std::endl; + bcos::bytes skBytes(symmetric_key_bytes(algorithmType, mode), 0); + OutputBuffer skBuffer{skBytes.data(), skBytes.size()}; + symmetric_generate_key(&skBuffer, algorithmType, mode); + + // convert to hexedSk + InputBuffer skBytesBuffer{(const unsigned char*)skBytes.data(), skBytes.size()}; + testSymEnc(&plainTextBytesBuffer, algorithmType, mode, &ivBytesBuffer, &skBytesBuffer); +} + +BOOST_AUTO_TEST_CASE(testSymEncUDF) +{ + std::cout << "run AES_128 ECB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run AES_128 CBC" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run AES_128 CFB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run AES_128 CTR" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_128), static_cast(SymCrypto::OperationMode::CTR)); + std::cout << "run AES_192 OFB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_192), static_cast(SymCrypto::OperationMode::OFB)); + std::cout << "run AES_256 CTR" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::AES_256), static_cast(SymCrypto::OperationMode::CTR)); + + std::cout << "run TrippleDES ECB" << std::endl; + testSymEncUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run TrippleDES CBC" << std::endl; + testSymEncUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run TrippleDES CFB" << std::endl; + testSymEncUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run TrippleDES OFB" << std::endl; + testSymEncUDFImpl(static_cast(AlgorithmType::TrippleDES), + static_cast(SymCrypto::OperationMode::OFB)); + + std::cout << "run SM4 ECB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::ECB)); + std::cout << "run SM4 CBC" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CBC)); + std::cout << "run SM4 CFB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CFB)); + std::cout << "run SM4 OFB" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::OFB)); + std::cout << "run SM4 CTR" << std::endl; + testSymEncUDFImpl( + static_cast(AlgorithmType::SM4), static_cast(SymCrypto::OperationMode::CTR)); +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace ppc::test diff --git a/cpp/ppc-udf/tests/main.cpp b/cpp/ppc-udf/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/ppc-udf/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h new file mode 100644 index 00000000..c5f36374 --- /dev/null +++ b/cpp/test-utils/FakeFront.h @@ -0,0 +1,240 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief fake ppc network message interface + * @file FakeFront.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/protocol/Protocol.h" +#include "ppc-framework/task/TaskFrameworkInterface.h" + +using namespace ppc::protocol; +using namespace ppc::front; +// using namespace ppc::psi; +using namespace ppc::task; + +namespace ppc::test +{ +class FakeFront : public FrontInterface +{ +public: + using Ptr = std::shared_ptr; + FakeFront() = default; + ~FakeFront() override = default; + + void start() override {} + void stop() override {} + + void registerRA2018(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) + { + std::cout << "### registerRA2018: " << _agencyID << std::endl; + m_agencyToRA2018[_agencyID] = _psi; + } + + void registerEcdhPSI(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) + { + std::cout << "### registerEcdhPSI: " << _agencyID << std::endl; + m_agencyToEcdhPSI[_agencyID] = _psi; + } + + void registerEcdhMultiPSI(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) + { + std::cout << "### registerEcdhMultiPSI: " << _agencyID << std::endl; + m_agencyToEcdhMutliPSI[_agencyID] = _psi; + } + + void registerLabeledPSI(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) + { + m_agencyToLabeledPSI[_agencyID] = _psi; + } + + void registerCM2020(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) + { + m_agencyToCM2020[_agencyID] = _psi; + } + + void registerOTPIR(std::string const& _agencyID, TaskFrameworkInterface::Ptr _pir) + { + m_agencyToOTPIR[_agencyID] = _pir; + } + + void onReceiveMessage(front::PPCMessageFace::Ptr, ErrorCallbackFunc) override + { + throw std::runtime_error("FakeFront: unimplemented interface onReceiveMessage!"); + } + + void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, + uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _responseCallback) override + { + m_uuid++; + auto id = std::to_string(m_uuid); + if (_responseCallback) + { + insertCallback(id, _responseCallback); + _message->setUuid(id); + } + + + uint8_t type = _message->algorithmType(); + switch (type) + { + case uint8_t(PSIAlgorithmType::RA_PSI_2PC): + { + if (!m_agencyToRA2018.count(_agencyID)) + { + _callback(std::make_shared( + -1, "RA_PSI_2PC asyncSendMessage error! The gateway '" + _agencyID + + "' is not reachable!")); + return; + } + auto psi = m_agencyToRA2018.at(_agencyID); + psi->onReceiveMessage(_message); + _callback(nullptr); + break; + } + case uint8_t(PSIAlgorithmType::LABELED_PSI_2PC): + { + if (!m_agencyToLabeledPSI.count(_agencyID)) + { + _callback(std::make_shared( + -1, "LABELED_PSI_2PC asyncSendMessage error! The gateway " + _agencyID + + " is not reachable!")); + return; + } + auto psi = m_agencyToLabeledPSI.at(_agencyID); + psi->onReceiveMessage(_message); + _callback(nullptr); + break; + } + case uint8_t(PSIAlgorithmType::CM_PSI_2PC): + { + if (!m_agencyToCM2020.count(_agencyID)) + { + _callback(std::make_shared( + -1, "CM_PSI_2PC asyncSendMessage error! The gateway " + _agencyID + + " is not reachable!")); + return; + } + auto psi = m_agencyToCM2020.at(_agencyID); + psi->onReceiveMessage(_message); + _callback(nullptr); + break; + } + case uint8_t(PSIAlgorithmType::ECDH_PSI_2PC): + { + if (!m_agencyToEcdhPSI.count(_agencyID)) + { + _callback(std::make_shared( + -1, "ECDH_PSI_2PC asyncSendMessage error! The gateway " + _agencyID + + " is not reachable!")); + return; + } + auto psi = m_agencyToEcdhPSI.at(_agencyID); + psi->onReceiveMessage(_message); + _callback(nullptr); + break; + } + case uint8_t(PSIAlgorithmType::ECDH_PSI_MULTI): + { + if (!m_agencyToEcdhMutliPSI.count(_agencyID)) + { + _callback(std::make_shared( + -1, "ECDH_PSI_MULTI asyncSendMessage error! The gateway " + _agencyID + + " is not reachable!")); + return; + } + auto psi = m_agencyToEcdhMutliPSI.at(_agencyID); + psi->onReceiveMessage(_message); + _callback(nullptr); + break; + } + case uint8_t(PSIAlgorithmType::OT_PIR_2PC): + { + if (!m_agencyToOTPIR.count(_agencyID)) + { + _callback(std::make_shared( + -1, "OT_PIR_2PC asyncSendMessage error! The gateway " + _agencyID + + " is not reachable!")); + return; + } + auto pir = m_agencyToOTPIR.at(_agencyID); + pir->onReceiveMessage(_message); + _callback(nullptr); + break; + } + default: // unreachable-branch + break; + } + } + + bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr) override { return nullptr; } + + // erase the task-info when task finished + bcos::Error::Ptr eraseTaskInfo(std::string const&) override { return nullptr; } + + // send response when receiving message from given agencyID + void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, + front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override + { + if (m_uuidToCallback.count(_uuid)) + { + auto callback = m_uuidToCallback[_uuid]; + removeCallback(_uuid); + if (callback) + { + callback(nullptr, _agencyID, _message, nullptr); + } + } + } + + void asyncGetAgencyList(GetAgencyListCallback) override {} + +private: + // the uuid to _callback + void insertCallback(std::string const& _id, CallbackFunc _callback) + { + bcos::Guard l(m_mutex); + m_uuidToCallback[_id] = _callback; + } + + void removeCallback(std::string const& _id) + { + bcos::Guard l(m_mutex); + m_uuidToCallback.erase(_id); + } + +private: + // Note: here assume that one agency has one psi + std::map m_agencyToRA2018; + + std::map m_agencyToEcdhPSI; + + std::map m_agencyToEcdhMutliPSI; + + std::map m_agencyToLabeledPSI; + + std::map m_agencyToCM2020; + + std::map m_agencyToOTPIR; + + // uuid to callback + std::map m_uuidToCallback; + bcos::Mutex m_mutex; + std::atomic m_uuid = 0; +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/test-utils/FakePPCMessage.h b/cpp/test-utils/FakePPCMessage.h new file mode 100644 index 00000000..279cb1cf --- /dev/null +++ b/cpp/test-utils/FakePPCMessage.h @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @brief fake ppc network message interface + * @file FakePPCMessage.h + * @author: yujiechen + * @date 2022-11-16 + */ +#pragma once +#include "ppc-framework/protocol/PPCMessageFace.h" + +using namespace ppc::front; +namespace ppc::test +{ +class FakePPCMessage : public PPCMessageFace +{ +public: + using Ptr = std::shared_ptr; + FakePPCMessage() = default; + ~FakePPCMessage() override = default; + + uint8_t version() const override { return m_version; } + void setVersion(uint8_t _version) override { m_version = _version; } + uint8_t taskType() const override { return m_taskType; } + void setTaskType(uint8_t _taskType) override { m_taskType = _taskType; } + uint8_t algorithmType() const override { return m_algorithmType; } + void setAlgorithmType(uint8_t _algorithmType) override { m_algorithmType = _algorithmType; } + uint8_t messageType() const override { return m_messageType; } + void setMessageType(uint8_t _messageType) override { m_messageType = _messageType; } + uint32_t seq() const override { return m_seq; } + void setSeq(uint32_t _seq) override { m_seq = _seq; } + std::string const& taskID() const override { return m_taskID; } + void setTaskID(std::string const& _taskID) override { m_taskID = _taskID; } + std::map header() override { return m_header; } + void setHeader(std::map _header) override { m_header = _header; } + + std::string const& sender() const override { return m_sender; } + void setSender(std::string const& _sender) override { m_sender = _sender; } + + uint16_t ext() const override { return m_ext; } + void setExt(uint16_t _ext) override { m_ext = _ext; } + std::shared_ptr data() const override { return m_data; } + void setData(std::shared_ptr _data) override { m_data = _data; } + // Note: we don't fake the encode-decode here + void encode(bcos::bytes&) override + { + throw std::runtime_error("FakePPCMessage: unimplemented interface!"); + } + int64_t decode(bcos::bytesPointer) override + { + throw std::runtime_error("FakePPCMessage: unimplemented interface!"); + } + int64_t decode(bcos::bytesConstRef) override + { + throw std::runtime_error("FakePPCMessage: unimplemented interface!"); + } + uint32_t length() const override { return 0; } + + std::string uuid() const override { return m_uuid; } + void setUuid(std::string const& _uuid) override { m_uuid = _uuid; } + + // determine the message is response or not + bool response() const override { return m_response; } + // set the message to be response + void setResponse() override { m_response = true; } + +private: + uint8_t m_version; + uint8_t m_taskType; + uint8_t m_algorithmType; + uint8_t m_messageType; + uint32_t m_seq; + std::string m_taskID; + std::string m_sender; + uint16_t m_ext; + std::shared_ptr m_data; + std::string m_uuid; + std::map m_header; + bool m_response = false; +}; + +class FakePPCMessageFactory : public PPCMessageFaceFactory +{ +public: + using Ptr = std::shared_ptr; + FakePPCMessageFactory() = default; + ~FakePPCMessageFactory() override = default; + PPCMessageFace::Ptr buildPPCMessage() override { return std::make_shared(); } + PPCMessageFace::Ptr buildPPCMessage(bcos::bytesConstRef) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } + + PPCMessageFace::Ptr buildPPCMessage(bcos::bytesPointer) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } + + PPCMessageFace::Ptr buildPPCMessage(uint8_t _taskType, uint8_t _algorithmType, + std::string const& _taskID, bcos::bytesPointer _data) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } +}; +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/test-utils/FileTool.h b/cpp/test-utils/FileTool.h new file mode 100644 index 00000000..fee10152 --- /dev/null +++ b/cpp/test-utils/FileTool.h @@ -0,0 +1,134 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicabl law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FileTool.cpp + * @author: shawnhe + * @date 2022-12-19 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ppc::test +{ +inline void writeByMmap( + const std::string& _path, const std::shared_ptr>& _buffer) +{ + int fd = open(_path.c_str(), O_RDWR | O_CREAT, (mode_t)0600); + if (fd == -1) + { + BOOST_THROW_EXCEPTION(std::runtime_error("open file error")); + } + + uint64_t textSize = _buffer->size() * (64); + + // update the size of file + ftruncate(fd, textSize); + + char* map = (char*)mmap(nullptr, textSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) + { + close(fd); + BOOST_THROW_EXCEPTION(std::runtime_error("mmap file to memory error")); + } + + uint64_t count = 0; + + for (std::string line : *_buffer) + { + for (char c : line) + { + map[count++] = c; + } + map[count++] = '\n'; + } + + ftruncate(fd, count); + + // write it now to disk + if (msync(map, count, MS_SYNC) == -1) + { + BOOST_THROW_EXCEPTION(std::runtime_error("mmap file to disk error")); + } + + munmap(map, count); + + close(fd); +} + +inline void prepareInputs(const std::string& _path1, uint32_t _size1, const std::string& _path2, + uint32_t _size2, uint32_t _common) +{ + assert(_size1 >= _common && _size2 >= _common); + + uint32_t start1 = 100000, end1 = 100000 + _size1; + auto input1 = std::make_shared>(); + input1->reserve(_size1); + for (uint i = start1; i < end1; i++) + { + input1->push_back(std::to_string(i) + "\r"); + } + writeByMmap(_path1, input1); + + uint32_t start2 = end1 - _common, end2 = end1 - _common + _size2; + auto input2 = std::make_shared>(); + input2->reserve(_size2); + for (uint i = start2; i < end2; i++) + { + input2->push_back(std::to_string(i)); + } + writeByMmap(_path2, input2); +} + +inline void prepareInputs(const std::string& _path, uint32_t _size) +{ + uint32_t start = 100000, end = 100000 + _size; + auto input = std::make_shared>(); + input->reserve(_size); + for (uint i = start; i < end; i++) + { + input->push_back(std::to_string(i)); + } + writeByMmap(_path, input); +} + +inline void prepareItemsAndLabels(const std::string& _path, uint32_t _size) +{ + uint32_t start = 100000, end = 100000 + _size; + auto input = std::make_shared>(); + input->reserve(_size); + for (uint i = start; i < end; i++) + { + input->push_back( + std::to_string(i) + "," + std::to_string(i) + std::to_string(i) + std::to_string(i)); + } + writeByMmap(_path, input); +} + +} // namespace ppc::test diff --git a/cpp/test-utils/TaskMock.h b/cpp/test-utils/TaskMock.h new file mode 100644 index 00000000..1956946a --- /dev/null +++ b/cpp/test-utils/TaskMock.h @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TaskMock.h + * @author: yujiechen + * @date 2022-10-19 + */ +#pragma once +#include "ppc-framework/protocol/Task.h" +#include "ppc-io/src/FileLineReader.h" +#include + +using namespace ppc::io; +using namespace ppc::protocol; +namespace ppc::test +{ +inline PartyResource::Ptr mockParty(uint16_t _partyIndex, std::string const& _id, + std::string const& _desc, std::string const& _dataId, ppc::protocol::DataResourceType _dataType, + std::string const& _path, const std::vector>& _rowData = {}) +{ + auto party = std::make_shared(_id, (uint16_t)_partyIndex); + party->setDesc(_desc); + auto dataResource = std::make_shared(_dataId); + auto dataDesc = std::make_shared(); + dataDesc->setType((uint16_t)_dataType); + dataDesc->setPath(_path); + dataResource->setDesc(dataDesc); + party->setDataResource(dataResource); + + if (!_rowData.empty()) + { + dataResource->setRawData(_rowData); + } + return party; +} + +inline void checkParty(PartyResource::ConstPtr _party1, PartyResource::ConstPtr _party2) +{ + BOOST_CHECK(_party1->partyIndex() == _party2->partyIndex()); + BOOST_CHECK(_party1->desc() == _party2->desc()); + BOOST_CHECK(_party1->id() == _party2->id()); + BOOST_CHECK(_party1->partyIndex() == _party2->partyIndex()); + if (!_party1->dataResource()) + { + return; + } + // check the dataResource + BOOST_CHECK(_party1->dataResource()->resourceID() == _party2->dataResource()->resourceID()); + if (!_party1->dataResource()->desc()) + { + return; + } + BOOST_CHECK(_party1->dataResource()->desc()->type() == _party2->dataResource()->desc()->type()); + // BOOST_CHECK(_party1->dataResource()->desc()->path() == + // _party2->dataResource()->desc()->path()); + BOOST_CHECK(_party1->dataResource()->desc()->accessCommand() == + _party2->dataResource()->desc()->accessCommand()); + + BOOST_CHECK( + _party1->dataResource()->rawData().size() == _party2->dataResource()->rawData().size()); +} + +inline void checkTask(Task::Ptr _task1, Task::Ptr _task2) +{ + BOOST_CHECK(_task1->id() == _task2->id()); + BOOST_CHECK(_task1->type() == _task2->type()); + BOOST_CHECK(_task1->param() == _task2->param()); + BOOST_CHECK(_task1->syncResultToPeer() == _task2->syncResultToPeer()); + // check self-party + if (_task1->selfParty()) + { + checkParty(_task1->selfParty(), _task2->selfParty()); + } + // check peersParty + auto const& peersParty = _task1->getAllPeerParties(); + BOOST_CHECK(peersParty.size() == _task2->getAllPeerParties().size()); + for (auto const& it : peersParty) + { + checkParty(it.second, _task2->getPartyByID(it.first)); + } +} +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh new file mode 100644 index 00000000..23c1fb99 --- /dev/null +++ b/cpp/tools/build_ppc.sh @@ -0,0 +1,926 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +binary_name="ppc-air-node" +binary_path= +output_dir="ppc-nodes" + +listen_ip="0.0.0.0" + +cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS" +OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b" + +port_start=(40300 10200) +gateway_listen_port=port_start[0] +rpc_listen_port=port_start[1] + +use_ip_param= +ip_param= + +ip_array= +ip_file= + +ca_dir="" +cert_conf="${output_dir}/cert.cnf" +sm_cert_conf='sm_cert.cnf' +sm2_params="sm_sm2.param" +sm_mode="false" +days=36500 +rsa_key_length=2048 + +default_version="v1.1.0" +compatibility_version=${default_version} +command="deploy" + +disable_ra2018="false" + +LOG_WARN() { + local content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + local content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +LOG_FATAL() { + local content=${1} + echo -e "\033[31m[FATAL] ${content}\033[0m" + exit 1 +} + +dir_must_exists() { + if [ ! -d "$1" ]; then + LOG_FATAL "$1 DIR does not exist, please check!" + fi +} + +get_value() { + local var_name=${1} + var_name=var_${var_name//./} + local res=$(eval echo '$'"${var_name}") + echo ${res} +} + +set_value() { + local var_name=${1} + var_name=var_${var_name//./} + local var_value=${2} + eval "${var_name}=${var_value}" +} + +dir_must_not_exists() { + if [ -d "$1" ]; then + LOG_FATAL "$1 DIR already exist, please check!" + fi +} + +file_must_not_exists() { + if [ -f "$1" ]; then + LOG_FATAL "$1 file already exist, please check!" + fi +} + +file_must_exists() { + if [ ! -f "$1" ]; then + LOG_FATAL "$1 file does not exist, please check!" + fi +} + +check_env() { + if [ "$(uname)" == "Darwin" ];then + macOS="macOS" + fi + if [ "$(uname -m)" != "x86_64" ];then + x86_64_arch="false" + fi +} +check_name() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[a-zA-Z0-9._-]+$ ]] || { + LOG_FATAL "$name name [$value] invalid, it should match regex: ^[a-zA-Z0-9._-]+\$" + } +} +parse_ip_config() { + local config=$1 + n=0 + while read line; do + ip_array[n]=$(echo ${line} | awk '{print $1}') + if [ -z "${ip_array[$n]}" ]; then + exit_with_clean "Please check ${config}, make sure there is no empty line!" + fi + ((++n)) + done <${config} +} +check_and_install_tassl(){ + if [ -f "${OPENSSL_CMD}" ];then + return + fi + # https://en.wikipedia.org/wiki/Uname#Examples + local x86_64_name="x86_64" + local arm_name="aarch64" + local tassl_mid_name="linux" + if [[ -n "${macOS}" ]];then + x86_64_name="x86_64" + arm_name="arm64" + tassl_mid_name="macOS" + fi + + local tassl_post_fix="x86_64" + local platform="$(uname -m)" + if [[ "${platform}" == "${arm_name}" ]];then + tassl_post_fix="aarch64" + elif [[ "${platform}" == "${x86_64_name}" ]];then + tassl_post_fix="x86_64" + else + LOG_FATAL "Unsupported platform ${platform} for ${tassl_mid_name}" + exit 1 + fi + local tassl_package_name="tassl-1.1.1b-${tassl_mid_name}-${tassl_post_fix}" + local tassl_tgz_name="${tassl_package_name}.tar.gz" + local tassl_link_prefix="${cdn_link_header}/FISCO-BCOS/tools/tassl-1.1.1b/${tassl_tgz_name}" + LOG_INFO "Downloading tassl binary from ${tassl_link_prefix}..." + wget --no-check-certificate "${tassl_link_prefix}" + tar zxvf ${tassl_tgz_name} && rm ${tassl_tgz_name} + chmod u+x ${tassl_package_name} + mkdir -p "${HOME}"/.fisco + mv ${tassl_package_name} "${HOME}"/.fisco/tassl-1.1.1b +} + + +generate_sm_sm2_param() { + local output=$1 + cat << EOF > ${output} +-----BEGIN EC PARAMETERS----- +BggqgRzPVQGCLQ== +-----END EC PARAMETERS----- + +EOF +} + +generate_sm_cert_conf() { + local output=$1 + cat <"${output}" +oid_section = new_oids + +[ new_oids ] +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem # The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extensions to add to the cert + +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +default_days = 36500 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +policy = policy_match + +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_md = sm3 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +x509_extensions = v3_ca # The extensions to add to the self signed cert + +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = fisco +commonName = Organizational commonName (eg, fisco) +commonName_default = fisco +commonName_max = 64 + +[ usr_cert ] +basicConstraints=CA:FALSE +nsComment = "OpenSSL Generated Certificate" + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature + +[ v3enc_req ] + +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = keyAgreement, keyEncipherment, dataEncipherment + +[ v3_agency_root ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +[ v3_ca ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +EOF +} + +generate_cert_conf() { + local output=$1 + cat <"${output}" +[ca] +default_ca=default_ca + +[default_ca] +default_days = 36500 +default_md = sha256 + +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = WeDPR-PPC +commonName = Organizational commonName (eg, WeDPR-PPC) +commonName_default = WeDPR-PPC +commonName_max = 64 + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v4_req ] +basicConstraints = CA:FALSE + +EOF +} +print_result() { + echo "==============================================================" + LOG_INFO "${binary_name} path : ${binary_path}" + LOG_INFO "Start port : ${port_start[*]}" + LOG_INFO "Server IP : ${ip_array[*]}" + LOG_INFO "SM model : ${sm_mode}" + LOG_INFO "Output dir : ${output_dir}" + LOG_INFO "All completed. Files in ${output_dir}" +} + +help() { + echo $1 + cat < [Optional] the command, support 'deploy' and 'expand' now, default is deploy + -v [Optional] Default is the latest ${default_version} + -l [Required] "ip1:nodeNum1,ip2:nodeNum2" e.g:"192.168.0.1:2,192.168.0.2:3" + -e [Optional] ppc-air-node binary exec + -o [Optional] output directory, default ./nodes + -p [Optional] Default 40300,10200 means p2p_port start from 40300, rpc_port from 10200 + -s [Optional] SM SSL connection or not, default is false + -d [Optional] Disable ra2018 psi or not, default is false + -h Help + +deploy nodes e.g + bash $0 -p 40300,10200 -l 127.0.0.1:4 -o ppc-nodes -e ./ppc-air-node + bash $0 -p 40300,10200 -l 127.0.0.1:4 -o ppc-nodes -e ./ppc-air-node -s +EOF + exit 0 +} + +# generate the config.ini +generate_config_ini() { + local output="${1}" + local gateway_listen_ip="${2}" + local gateway_listen_port="${3}" + + local rpc_listen_ip="${4}" + local rpc_listen_port="${5}" + local agency_info="${6}" + local agency_id="${7}" + local index="${8}" + + cat <"${output}" +[agency] + ; the agency-id of self-party + id = ${agency_id} + ; the agency info + ${agency_info} + ; the private-key for the psi-server + private_key_path = conf/node.pem + ; disable the ra2018 or not, default enable ra2018 + disable_ra2018 = ${disable_ra2018} + ; the path that allows programs to access + ; data_location = data + ; task_timeout_minutes = 180 + ; thread_count = 8 + +[crypto] + sm_crypto = ${sm_mode} + +[gateway] + listen_ip=${gateway_listen_ip} + listen_port=${gateway_listen_port} + ; thread_count = 4 + ; ssl or sm ssl + sm_ssl=${sm_mode} + ; the max allowed message size in MBytes, default is 100MBytes + max_allow_msg_size = 100 + ;ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes + holding_msg_minutes = 30 + ;disable_cache = false + ;reconnect_time = 10000 + +[rpc] + listen_ip=${rpc_listen_ip} + listen_port=${rpc_listen_port} + ; token = ppcs_psi_apikey + thread_count = 4 + ; ssl or sm ssl + sm_ssl=${sm_mode} + ; ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + ;disable_db = true + ; GB, reject task if available memory is less than min_needed_memory + ; min_needed_memory = 5 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[cache] + ; the cache type, only support redis now + type = 0 + proxy = 127.0.0.1:20002 + obServer = 127.0.0.1:10904 + cluster = REDIS_CLUSTER + user = 1194 + host = 127.0.0.1 + port = 6379 + password = + database = 1 + pool_size = 16 + ; the redis connection timeout, in ms, default is 500ms + connection_timeout = 500 + ; the redis socket timeout, in ms, default is 500ms + socket_timeout = 500 + +[storage] + host = 127.0.0.1 + ; the mysqlx_port + port = 3306 + user = app + password = + database = + +[hdfs_storage] + ; the hdfs configuration + user = app + name_node = 127.0.0.1 + name_node_port = 9000 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 1000 + + +[ra2018psi] + ; The database used to store cuckoo-filter + database=ra2018_${index} + ; The capacity of the cuckoo filter in MB, the default is 1MB + cuckoofilter_capacity = 1 + ; The length of each element of the cuckoo filter (bits), default 64bits + cuckoofilter_tagBits = 64 + ; The number of buckets for the cuckoo filter, the default is 4 + cuckoofilter_buckets_num = 4 + ; When the cuckoo filter replaces an element, the maximum number of element replacements allowed + cuckoofilter_max_kick_out_count = 20 + ; The trash-bucket-size for the cuckoo-filter + trash_bucket_size = 10000 + ; The cuckoo filter capacity that can be cached in memory in MB + cuckoofilter_cache_size = 256 + ; Cache size in MB + psi_cache_size = 1024 + ; When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + ; data_batch_size = 1000000 + ; use hdfs to store the cuckoo-filter or not + use_hdfs = false + +[ecdh-psi] + ; When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + ; data_batch_size = 1000000 + +[cm2020-psi] + ; Number of tasks that can be executed concurrently + parallelism = 3 + +[log] + enable=true + log_path=./log + ; info debug trace + level=debug + ; MB + max_log_file_size=200 + ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message + format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + enable_rotate_by_hour=false + log_name_pattern=ppcs-psi4ef.log + ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log + rotate_name_pattern=log_%Y%m%d.%H%M.log + ; if archive_path is empty, the archive function will be disabled + ; archive_path=./log/ + compress_archive_file=true + ; ; 0: no limit, in MB + ; max_archive_files=10 + ; ; 0: no limit, in MB + ; max_archive_size=0 + ; min_free_space=0 +EOF +} + +generate_script_template() +{ + local filepath=$1 + mkdir -p $(dirname $filepath) + cat << EOF > "${filepath}" +#!/bin/bash +SHELL_FOLDER=\$(cd \$(dirname \$0);pwd) + +LOG_ERROR() { + content=\${1} + echo -e "\033[31m[ERROR] \${content}\033[0m" +} + +LOG_INFO() { + content=\${1} + echo -e "\033[32m[INFO] \${content}\033[0m" +} + +EOF + chmod +x ${filepath} +} + +# TODO: support docker-mode +generate_node_scripts() { + local output=${1} + local ps_cmd="\$(ps aux|grep \${ppc_psi}|grep -v grep|awk '{print \$2}')" + local start_cmd="nohup \${ppc_psi} -c config.ini >>nohup.out 2>&1 &" + local stop_cmd="kill \${node_pid}" + local pid="pid" + local log_cmd="tail -n20 nohup.out" + local check_success="\$(${log_cmd} | grep running)" + generate_script_template "$output/start.sh" + cat <> "${output}/start.sh" +ppc_psi=\${SHELL_FOLDER}/../${binary_name} +cd \${SHELL_FOLDER} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +if [ ! -z \${node_pid} ];then + echo " \${node} is running, ${pid} is \$node_pid." + exit 0 +else + ${start_cmd} + sleep 1.5 +fi +try_times=4 +i=0 +while [ \$i -lt \${try_times} ] +do + node_pid=${ps_cmd} + success_flag=${check_success} + if [[ ! -z \${node_pid} && ! -z "\${success_flag}" ]];then + echo -e "\033[32m \${node} start successfully pid=\${node_pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start \${node} \033[0m" +${log_cmd} +EOF + chmod u+x "${output}/start.sh" + generate_script_template "$output/stop.sh" + cat <> "${output}/stop.sh" +ppc_psi=\${SHELL_FOLDER}/../${binary_name} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +try_times=10 +i=0 +if [ -z \${node_pid} ];then + echo " \${node} isn't running." + exit 0 +fi +[ ! -z \${node_pid} ] && ${stop_cmd} > /dev/null +while [ \$i -lt \${try_times} ] +do + sleep 1 + node_pid=${ps_cmd} + if [ -z \${node_pid} ];then + echo -e "\033[32m stop \${node} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop \${node}" +exit 1 +EOF + chmod u+x "${output}/stop.sh" +} + +generate_all_node_scripts() { + local output=${1} + mkdir -p ${output} + + cat <"${output}/start_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/start.sh" ]];then + echo "try to start \${dir}" + bash \${dirpath}/\${dir}/start.sh & + fi +done +wait +EOF + chmod u+x "${output}/start_all.sh" + + cat <"${output}/stop_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/stop.sh" ]];then + echo "try to stop \${dir}" + bash \${dirpath}/\${dir}/stop.sh + fi +done +wait +EOF + chmod u+x "${output}/stop_all.sh" +} + +gen_non_sm_ca_cert() { + if [ ! -f "${cert_conf}" ]; then + generate_cert_conf "${cert_conf}" + fi + local ca_cert_dir="${1}" + file_must_not_exists "${ca_cert_dir}"/ca.key + file_must_not_exists "${ca_cert_dir}"/ca.crt + file_must_exists "${cert_conf}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + ${OPENSSL_CMD} genrsa -out "${ca_cert_dir}"/ca.key "${rsa_key_length}" + ${OPENSSL_CMD} req -new -x509 -days "${days}" -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=chain" -key "${ca_cert_dir}"/ca.key -config "${cert_conf}" -out "${ca_cert_dir}"/ca.crt 2>/dev/null + if [ ! -f "${ca_cert_dir}/cert.cnf" ];then + mv "${cert_conf}" "${ca_cert_dir}" + fi + LOG_INFO "Generate ca cert successfully!" +} + +gen_sm_ca_cert() { + local ca_cert_dir="${1}" + name=$(basename "$ca_cert_dir") + check_name chain "$name" + + if [ ! -f "${sm_cert_conf}" ]; then + generate_sm_cert_conf ${sm_cert_conf} + fi + + generate_sm_sm2_param "${sm2_params}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + "$OPENSSL_CMD" genpkey -paramfile "${sm2_params}" -out "$ca_cert_dir/sm_ca.key" 2>/dev/null + "$OPENSSL_CMD" req -config sm_cert.cnf -x509 -days "${days}" -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=ca" -key "$ca_cert_dir/sm_ca.key" -extensions v3_ca -out "$ca_cert_dir/sm_ca.crt" 2>/dev/null + if [ ! -f "${ca_cert_dir}/${sm_cert_conf}" ];then + cp "${sm_cert_conf}" "${ca_cert_dir}" + fi + if [ ! -f "${ca_cert_dir}/${sm2_params}" ];then + cp "${sm2_params}" "${ca_cert_dir}" + fi +} + + +gen_rsa_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/ca.key" + file_must_exists "$capath/ca.crt" + # check_name node "$node" + + file_must_not_exists "$ndpath"/"${type}".key + file_must_not_exists "$ndpath"/"${type}".crt + + mkdir -p "${ndpath}" + dir_must_exists "${ndpath}" + + ${OPENSSL_CMD} genrsa -out "${ndpath}"/"${type}".key "${rsa_key_length}" 2>/dev/null + ${OPENSSL_CMD} req -new -sha256 -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=agency" -key "$ndpath"/"${type}".key -config "$capath"/cert.cnf -out "$ndpath"/"${type}".csr + ${OPENSSL_CMD} x509 -req -days "${days}" -sha256 -CA "${capath}"/ca.crt -CAkey "$capath"/ca.key -CAcreateserial \ + -in "$ndpath"/"${type}".csr -out "$ndpath"/"${type}".crt -extensions v4_req -extfile "$capath"/cert.cnf 2>/dev/null + + ${OPENSSL_CMD} pkcs8 -topk8 -in "$ndpath"/"${type}".key -out "$ndpath"/pkcs8_node.key -nocrypt + cp "$capath"/ca.crt "$capath"/cert.cnf "$ndpath"/ + + rm -f "$ndpath"/"$type".csr + rm -f "$ndpath"/"$type".key + + mv "$ndpath"/pkcs8_node.key "$ndpath"/"$type".key + + LOG_INFO "Generate ${ndpath} cert successful!" +} + +gen_sm_node_cert_with_ext() { + local capath="$1" + local certpath="$2" + local type="$3" + local extensions="$4" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + file_must_not_exists "$ndpath/sm_${type}.crt" + file_must_not_exists "$ndpath/sm_${type}.key" + + "$OPENSSL_CMD" genpkey -paramfile "$capath/${sm2_params}" -out "$certpath/sm_${type}.key" 2> /dev/null + "$OPENSSL_CMD" req -new -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=${type}" -key "$certpath/sm_${type}.key" -config "$capath/sm_cert.cnf" -out "$certpath/sm_${type}.csr" 2> /dev/null + + "$OPENSSL_CMD" x509 -sm3 -req -CA "$capath/sm_ca.crt" -CAkey "$capath/sm_ca.key" -days "${days}" -CAcreateserial -in "$certpath/sm_${type}.csr" -out "$certpath/sm_${type}.crt" -extfile "$capath/sm_cert.cnf" -extensions "$extensions" 2> /dev/null + + rm -f "$certpath/sm_${type}.csr" +} + +gen_sm_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + mkdir -p "$ndpath" + dir_must_exists "$ndpath" + local node=$(basename "$ndpath") + check_name node "$node" + + gen_sm_node_cert_with_ext "$capath" "$ndpath" "${type}" v3_req + gen_sm_node_cert_with_ext "$capath" "$ndpath" "en${type}" v3enc_req + #nodeid is pubkey + $OPENSSL_CMD ec -in "$ndpath/sm_${type}.key" -text -noout 2> /dev/null | sed -n '7,11p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | awk '{print substr($0,3);}' | cat > "${ndpath}/sm_${type}.nodeid" + cp "$capath/sm_ca.crt" "$ndpath" +} + +generate_ca_cert() { + local sm_mode="$1" + local ca_cert_path="$2" + LOG_INFO "generate ca cert, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" + mkdir -p "${ca_cert_path}" + if [[ "${sm_mode}" == "false" ]]; then + gen_non_sm_ca_cert "${ca_cert_path}" + else + gen_sm_ca_cert "${ca_cert_path}" + fi + LOG_INFO "generate ca cert successfully, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" +} + +generate_node_cert() { + local sm_mode="$1" + local ca_cert_path="${2}" + local node_cert_path="${3}" + + mkdir -p ${node_cert_path} + if [[ "${sm_mode}" == "false" ]]; then + gen_rsa_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + else + gen_sm_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + fi +} + +# we use sm_param to generate the private key +generate_private_key() { + local output_path="${1}" + if [ ! -d "${output_path}" ]; then + mkdir -p ${output_path} + fi + if [ ! -f ${sm2_params} ]; then + generate_sm_sm2_param ${sm2_params} + fi + ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null + $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" + private_key=$(cat $output_path/node.privateKey) + echo ${private_key} +} + +deploy_nodes() +{ + dir_must_not_exists "${output_dir}" + mkdir -p "$output_dir" + dir_must_exists "${output_dir}" + gateway_listen_port=port_start[0] + rpc_listen_port=port_start[1] + + # check the binary + if [[ ! -f "$binary_path" ]]; then + LOG_FATAL "wedpr-component binary exec ${binary_path} not exist, Must copy binary file ${binary_name} to ${binary_path}" + fi + + # parse the ip + [ -z $use_ip_param ] && help 'ERROR: Please set -l or -f option.' + if [ "${use_ip_param}" == "true" ]; then + ip_array=(${ip_param//,/ }) + elif [ "${use_ip_param}" == "false" ]; then + if ! parse_ip_config $ip_file; then + exit_with_clean "Parse $ip_file error!" + fi + else + help + fi + + local agency_index=0 + local agency_info="" + # generate the ca-cert + ca_dir="${output_dir}"/ca + generate_ca_cert "${sm_mode}" "${ca_dir}" + for line in ${ip_array[*]}; do + ip=${line%:*} + num=${line#*:} + if [ -z $(echo $ip | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$") ]; then + LOG_WARN "Please check IP address: ${ip}, if you use domain name please ignore this." + fi + # echo $num + [ "$num" == "$ip" ] || [ -z "${num}" ] && num=${node_num} + echo "Processing IP:${ip} Total:${num}" + [ -z "$(get_value ${ip//./}_count)" ] && set_value ${ip//./}_count 0 + + nodes_dir="${output_dir}/${ip}" + # start_all.sh and stop_all.sh + generate_all_node_scripts "${nodes_dir}" + cp "${binary_path}" "${nodes_dir}" + # generate cert for the node + ca_cert_dir="${nodes_dir}"/ca + mkdir -p ${ca_cert_dir} + cp -r ${ca_dir}/* ${ca_cert_dir} + + # generate the node config + for ((i = 0; i < num; ++i)); do + local node_count=$(get_value ${ip//./}_count) + node_dir="${output_dir}/${ip}/node${node_count}" + mkdir -p "${node_dir}" + # generate the node-cert + generate_node_cert "${sm_mode}" "${ca_dir}" "${node_dir}/conf" + # generate the node-script + generate_node_scripts "${node_dir}" + local port=$((gateway_listen_port + node_count)) + # generate the agency_info + agency_info="${agency_info}agency.agency${agency_index}=${ip}:${port} + " + ((agency_index += 1)) + set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) + ((++count)) + ((++node_count)) + done + done + # generate the config.ini + local i=0 + local count=0 + for line in ${ip_array[*]}; do + ip=${line%:*} + num=${line#*:} + if [ -z $(echo $ip | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$") ]; then + LOG_WARN "Please check IP address: ${ip}, if you use domain name please ignore this." + fi + [ "$num" == "$ip" ] || [ -z "${num}" ] && num=${node_num} + set_value ${ip//./}_count 0 + for ((i = 0; i < num; ++i)); do + local node_count=$(get_value ${ip//./}_count) + node_dir="${output_dir}/${ip}/node${node_count}" + local gateway_port=$((gateway_listen_port + node_count)) + local rpc_port=$((rpc_listen_port + node_count)) + local agency_id="agency${count}" + private_key=$(generate_private_key "${node_dir}/conf") + generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${count}" + set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) + ((++count)) + done + done + print_result +} + +parse_params() { + while getopts "l:C:o:e:p:v:sdh" option; do + case $option in + l) + ip_param=$OPTARG + use_ip_param="true" + ;; + o) + output_dir="$OPTARG" + ;; + e) + binary_path="$OPTARG" + file_must_exists "${binary_path}" + ;; + C) command="${OPTARG}" + ;; + p) + port_start=(${OPTARG//,/ }) + if [ ${#port_start[@]} -ne 2 ]; then LOG_WARN "p2p start port error. e.g: 30300" && exit 1; fi + ;; + s) sm_mode="true" ;; + d) disable_ra2018="true" ;; + v) compatibility_version="${OPTARG}";; + h) help ;; + *) help ;; + esac + done +} + +main() { + check_env + check_and_install_tassl + parse_params "$@" + cert_conf="${output_dir}/cert.cnf" + # TODO: support node-expand + if [[ "${command}" == "deploy" ]]; then + deploy_nodes + else + LOG_FATAL "Unsupported command ${command}, only support \'deploy\' and \'expand\' now!" + fi +} +main "$@" diff --git a/cpp/tools/build_wedpr_cem.sh b/cpp/tools/build_wedpr_cem.sh new file mode 100644 index 00000000..5cb54e9d --- /dev/null +++ b/cpp/tools/build_wedpr_cem.sh @@ -0,0 +1,729 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +binary_name="wedpr-cem" +binary_path= +output_dir="wedpr-cem-node" + +listen_ip="0.0.0.0" + +cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS" +OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b" + +ca_dir="" +cert_conf="" +sm_cert_conf='sm_cert.cnf' +sm2_params="sm_sm2.param" +sm_mode="false" +days=36500 +rsa_key_length=2048 + +default_version="v1.1.0" +compatibility_version=${default_version} +command="deploy" + + +LOG_WARN() { + local content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + local content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +LOG_FATAL() { + local content=${1} + echo -e "\033[31m[FATAL] ${content}\033[0m" + exit 1 +} + +dir_must_exists() { + if [ ! -d "$1" ]; then + LOG_FATAL "$1 DIR does not exist, please check!" + fi +} + +get_value() { + local var_name=${1} + var_name=var_${var_name//./} + local res=$(eval echo '$'"${var_name}") + echo ${res} +} + +set_value() { + local var_name=${1} + var_name=var_${var_name//./} + local var_value=${2} + eval "${var_name}=${var_value}" +} + +dir_must_not_exists() { + if [ -d "$1" ]; then + LOG_FATAL "$1 DIR already exist, please check!" + fi +} + +file_must_not_exists() { + if [ -f "$1" ]; then + LOG_FATAL "$1 file already exist, please check!" + fi +} + +file_must_exists() { + if [ ! -f "$1" ]; then + LOG_FATAL "$1 file does not exist, please check!" + fi +} + +check_env() { + if [ "$(uname)" == "Darwin" ];then + macOS="macOS" + fi + if [ "$(uname -m)" != "x86_64" ];then + x86_64_arch="false" + fi +} +check_name() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[a-zA-Z0-9._-]+$ ]] || { + LOG_FATAL "$name name [$value] invalid, it should match regex: ^[a-zA-Z0-9._-]+\$" + } +} +check_and_install_tassl(){ + if [ -f "${OPENSSL_CMD}" ];then + return + fi + # https://en.wikipedia.org/wiki/Uname#Examples + local x86_64_name="x86_64" + local arm_name="aarch64" + local tassl_mid_name="linux" + if [[ -n "${macOS}" ]];then + x86_64_name="x86_64" + arm_name="arm64" + tassl_mid_name="macOS" + fi + + local tassl_post_fix="x86_64" + local platform="$(uname -m)" + if [[ "${platform}" == "${arm_name}" ]];then + tassl_post_fix="aarch64" + elif [[ "${platform}" == "${x86_64_name}" ]];then + tassl_post_fix="x86_64" + else + LOG_FATAL "Unsupported platform ${platform} for ${tassl_mid_name}" + exit 1 + fi + local tassl_package_name="tassl-1.1.1b-${tassl_mid_name}-${tassl_post_fix}" + local tassl_tgz_name="${tassl_package_name}.tar.gz" + local tassl_link_prefix="${cdn_link_header}/FISCO-BCOS/tools/tassl-1.1.1b/${tassl_tgz_name}" + LOG_INFO "Downloading tassl binary from ${tassl_link_prefix}..." + wget --no-check-certificate "${tassl_link_prefix}" + tar zxvf ${tassl_tgz_name} && rm ${tassl_tgz_name} + chmod u+x ${tassl_package_name} + mkdir -p "${HOME}"/.fisco + mv ${tassl_package_name} "${HOME}"/.fisco/tassl-1.1.1b +} + + +generate_sm_sm2_param() { + local output=$1 + cat << EOF > ${output} +-----BEGIN EC PARAMETERS----- +BggqgRzPVQGCLQ== +-----END EC PARAMETERS----- + +EOF +} + +generate_sm_cert_conf() { + local output=$1 + cat <"${output}" +oid_section = new_oids + +[ new_oids ] +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem # The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extensions to add to the cert + +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +default_days = 36500 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +policy = policy_match + +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_md = sm3 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +x509_extensions = v3_ca # The extensions to add to the self signed cert + +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = fisco +commonName = Organizational commonName (eg, fisco) +commonName_default = fisco +commonName_max = 64 + +[ usr_cert ] +basicConstraints=CA:FALSE +nsComment = "OpenSSL Generated Certificate" + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature + +[ v3enc_req ] + +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = keyAgreement, keyEncipherment, dataEncipherment + +[ v3_agency_root ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +[ v3_ca ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +EOF +} + +generate_cert_conf() { + local output=$1 + cat <"${output}" +[ca] +default_ca=default_ca +[default_ca] +default_days = 36500 +default_md = sha256 + +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req +[req_distinguished_name] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = FISCO-BCOS +commonName = Organizational commonName (eg, FISCO-BCOS) +commonName_default = FISCO-BCOS +commonName_max = 64 + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v4_req ] +basicConstraints = CA:TRUE + +EOF +} +print_result() { + echo "==============================================================" + LOG_INFO "${binary_name} path : ${binary_path}" + LOG_INFO "SM model : ${sm_mode}" + LOG_INFO "Output dir : ${output_dir}" + LOG_INFO "All completed. Files in ${output_dir}" +} + +help() { + echo $1 + cat < [Optional] wedpr-cem binary exec + -o [Optional] output directory, default ./nodes + -s [Optional] SM SSL connection or not, default is false + -h Help + +deploy nodes e.g + bash $0 -o wedpr-cem-node -e ./wedpr-cem + bash $0 -o wedpr-cem-node -e ./wedpr-cem -s +EOF + exit 0 +} + +# generate the config.ini +generate_config_ini() { + local output="${1}" + + local rpc_listen_ip="${2}" + local rpc_listen_port="${3}" + local agency_info="${4}" + local agency_id="${5}" + + cat <"${output}" +[agency] + ; the agency-id of self-party + id = ${agency_id} + ; the agency info + +[cem] + dataset_hdfs_path = /user/ppc/ + dataset_file_path = ../../../ppc-cem/tests/data + ciphertext_suffix = -encrypted + read_per_batch_lines = 100000 + +[crypto] + sm_crypto = ${sm_mode} + +[rpc] + listen_ip=${rpc_listen_ip} + listen_port=${rpc_listen_port} + token = ppcs_psi_apikey + thread_count=4 + ; ssl or sm ssl + sm_ssl=${sm_mode} + ; ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + ; GB, reject task if available memory is less than min_needed_memory + ; min_needed_memory = 5 + +[hdfs_storage] + ; the hdfs configuration + user = ppc + name_node = 127.0.0.1 + name_node_port = 9900 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 2000 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[log] + enable=true + log_path=./log + ; info debug trace + level=debug + ; MB + max_log_file_size=200 + ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message + format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + enable_rotate_by_hour=false + log_name_pattern=ppcs-psi4ef.log + ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log + rotate_name_pattern=log_%Y%m%d.%H%M.log + ; if archive_path is empty, the archive function will be disabled + ; archive_path=./log/ + compress_archive_file=true + ; ; 0: no limit, in MB + ; max_archive_files=10 + ; ; 0: no limit, in MB + ; max_archive_size=0 + ; min_free_space=0 +EOF +} + +generate_script_template() +{ + local filepath=$1 + mkdir -p $(dirname $filepath) + cat << EOF > "${filepath}" +#!/bin/bash +SHELL_FOLDER=\$(cd \$(dirname \$0);pwd) + +LOG_ERROR() { + content=\${1} + echo -e "\033[31m[ERROR] \${content}\033[0m" +} + +LOG_INFO() { + content=\${1} + echo -e "\033[32m[INFO] \${content}\033[0m" +} + +EOF + chmod +x ${filepath} +} + +# TODO: support docker-mode +generate_node_scripts() { + local output=${1} + local ps_cmd="\$(ps aux|grep \${ppc_cem}|grep -v grep|awk '{print \$2}')" + local start_cmd="nohup \${ppc_cem} -c config.ini >>nohup.out 2>&1 &" + local stop_cmd="kill \${node_pid}" + local pid="pid" + local log_cmd="tail -n20 nohup.out" + local check_success="\$(${log_cmd} | grep running)" + generate_script_template "$output/start.sh" + cat <> "${output}/start.sh" +ppc_cem=\${SHELL_FOLDER}/${binary_name} +cd \${SHELL_FOLDER} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +if [ ! -z \${node_pid} ];then + echo " \${node} is running, ${pid} is \$node_pid." + exit 0 +else + ${start_cmd} + sleep 1.5 +fi +try_times=4 +i=0 +while [ \$i -lt \${try_times} ] +do + node_pid=${ps_cmd} + success_flag=${check_success} + if [[ ! -z \${node_pid} && ! -z "\${success_flag}" ]];then + echo -e "\033[32m \${node} start successfully pid=\${node_pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start \${node} \033[0m" +${log_cmd} +EOF + chmod u+x "${output}/start.sh" + generate_script_template "$output/stop.sh" + cat <> "${output}/stop.sh" +ppc_cem=\${SHELL_FOLDER}/${binary_name} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +try_times=10 +i=0 +if [ -z \${node_pid} ];then + echo " \${node} isn't running." + exit 0 +fi +[ ! -z \${node_pid} ] && ${stop_cmd} > /dev/null +while [ \$i -lt \${try_times} ] +do + sleep 1 + node_pid=${ps_cmd} + if [ -z \${node_pid} ];then + echo -e "\033[32m stop \${node} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop \${node}" +exit 1 +EOF + chmod u+x "${output}/stop.sh" +} + +generate_all_node_scripts() { + local output=${1} + mkdir -p ${output} + + cat <"${output}/start_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/start.sh" ]];then + echo "try to start \${dir}" + bash \${dirpath}/\${dir}/start.sh & + fi +done +wait +EOF + chmod u+x "${output}/start_all.sh" + + cat <"${output}/stop_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/stop.sh" ]];then + echo "try to stop \${dir}" + bash \${dirpath}/\${dir}/stop.sh + fi +done +wait +EOF + chmod u+x "${output}/stop_all.sh" +} + +gen_non_sm_ca_cert() { + if [ ! -f "${cert_conf}" ]; then + generate_cert_conf "${cert_conf}" + fi + local ca_cert_dir="${1}" + file_must_not_exists "${ca_cert_dir}"/ca.key + file_must_not_exists "${ca_cert_dir}"/ca.crt + file_must_exists "${cert_conf}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + ${OPENSSL_CMD} genrsa -out "${ca_cert_dir}"/ca.key "${rsa_key_length}" + ${OPENSSL_CMD} req -new -x509 -days "${days}" -subj "/CN=FISCO-BCOS/O=FISCO-BCOS/OU=chain" -key "${ca_cert_dir}"/ca.key -config "${cert_conf}" -out "${ca_cert_dir}"/ca.crt 2>/dev/null + if [ ! -f "${ca_cert_dir}/cert.cnf" ];then + mv "${cert_conf}" "${ca_cert_dir}" + fi + LOG_INFO "Generate ca cert successfully!" +} + +gen_sm_ca_cert() { + local ca_cert_dir="${1}" + name=$(basename "$ca_cert_dir") + check_name chain "$name" + + if [ ! -f "${sm_cert_conf}" ]; then + generate_sm_cert_conf ${sm_cert_conf} + fi + + generate_sm_sm2_param "${sm2_params}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + "$OPENSSL_CMD" genpkey -paramfile "${sm2_params}" -out "$ca_cert_dir/sm_ca.key" 2>/dev/null + "$OPENSSL_CMD" req -config sm_cert.cnf -x509 -days "${days}" -subj "/CN=wedpr/O=wedpr/OU=ca" -key "$ca_cert_dir/sm_ca.key" -extensions v3_ca -out "$ca_cert_dir/sm_ca.crt" 2>/dev/null + if [ ! -f "${ca_cert_dir}/${sm_cert_conf}" ];then + cp "${sm_cert_conf}" "${ca_cert_dir}" + fi + if [ ! -f "${ca_cert_dir}/${sm2_params}" ];then + cp "${sm2_params}" "${ca_cert_dir}" + fi +} + + +gen_rsa_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/ca.key" + file_must_exists "$capath/ca.crt" + # check_name node "$node" + + file_must_not_exists "$ndpath"/"${type}".key + file_must_not_exists "$ndpath"/"${type}".crt + + mkdir -p "${ndpath}" + dir_must_exists "${ndpath}" + + ${OPENSSL_CMD} genrsa -out "${ndpath}"/"${type}".key "${rsa_key_length}" 2>/dev/null + ${OPENSSL_CMD} req -new -sha256 -subj "/CN=FISCO-BCOS/O=fisco-bcos/OU=agency" -key "$ndpath"/"${type}".key -config "$capath"/cert.cnf -out "$ndpath"/"${type}".csr + ${OPENSSL_CMD} x509 -req -days "${days}" -sha256 -CA "${capath}"/ca.crt -CAkey "$capath"/ca.key -CAcreateserial \ + -in "$ndpath"/"${type}".csr -out "$ndpath"/"${type}".crt -extensions v4_req -extfile "$capath"/cert.cnf 2>/dev/null + + ${OPENSSL_CMD} pkcs8 -topk8 -in "$ndpath"/"${type}".key -out "$ndpath"/pkcs8_node.key -nocrypt + cp "$capath"/ca.crt "$capath"/cert.cnf "$ndpath"/ + + rm -f "$ndpath"/"$type".csr + rm -f "$ndpath"/"$type".key + + mv "$ndpath"/pkcs8_node.key "$ndpath"/"$type".key + + LOG_INFO "Generate ${ndpath} cert successful!" +} + +gen_sm_node_cert_with_ext() { + local capath="$1" + local certpath="$2" + local type="$3" + local extensions="$4" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + file_must_not_exists "$ndpath/sm_${type}.crt" + file_must_not_exists "$ndpath/sm_${type}.key" + + "$OPENSSL_CMD" genpkey -paramfile "$capath/${sm2_params}" -out "$certpath/sm_${type}.key" 2> /dev/null + "$OPENSSL_CMD" req -new -subj "/CN=FISCO-BCOS/O=fisco-bcos/OU=${type}" -key "$certpath/sm_${type}.key" -config "$capath/sm_cert.cnf" -out "$certpath/sm_${type}.csr" 2> /dev/null + + "$OPENSSL_CMD" x509 -sm3 -req -CA "$capath/sm_ca.crt" -CAkey "$capath/sm_ca.key" -days "${days}" -CAcreateserial -in "$certpath/sm_${type}.csr" -out "$certpath/sm_${type}.crt" -extfile "$capath/sm_cert.cnf" -extensions "$extensions" 2> /dev/null + + rm -f "$certpath/sm_${type}.csr" +} + +gen_sm_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + mkdir -p "$ndpath" + dir_must_exists "$ndpath" + local node=$(basename "$ndpath") + check_name node "$node" + + gen_sm_node_cert_with_ext "$capath" "$ndpath" "${type}" v3_req + gen_sm_node_cert_with_ext "$capath" "$ndpath" "en${type}" v3enc_req + #nodeid is pubkey + $OPENSSL_CMD ec -in "$ndpath/sm_${type}.key" -text -noout 2> /dev/null | sed -n '7,11p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | awk '{print substr($0,3);}' | cat > "${ndpath}/sm_${type}.nodeid" + cp "$capath/sm_ca.crt" "$ndpath" +} + +generate_ca_cert() { + local sm_mode="$1" + local ca_cert_path="$2" + LOG_INFO "generate ca cert, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" + mkdir -p "${ca_cert_path}" + if [[ "${sm_mode}" == "false" ]]; then + gen_non_sm_ca_cert "${ca_cert_path}" + else + gen_sm_ca_cert "${ca_cert_path}" + fi + LOG_INFO "generate ca cert successfully, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" +} + +generate_node_cert() { + local sm_mode="$1" + local ca_cert_path="${2}" + local node_cert_path="${3}" + + mkdir -p ${node_cert_path} + if [[ "${sm_mode}" == "false" ]]; then + gen_rsa_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + else + gen_sm_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + fi +} + + +deploy_nodes() +{ + echo "output_dir:${output_dir}" + dir_must_not_exists "${output_dir}" + mkdir -p "$output_dir" + dir_must_exists "${output_dir}" + + # check the binary + if [[ ! -f "$binary_path" ]]; then + LOG_FATAL "wedpr-component binary exec ${binary_path} not exist, Must copy binary file ${binary_name} to ${binary_path}" + fi + + local agency_index=0 + local agency_info="" + # generate the ca-cert + ca_dir="${output_dir}"/ca + cert_conf="${output_dir}/cert.cnf" + generate_ca_cert "${sm_mode}" "${ca_dir}" + # start_all.sh and stop_all.sh + # generate_all_node_scripts "${output_dir}" + cp "${binary_path}" "${output_dir}" + # generate cert for the node + ca_cert_dir="${output_dir}"/ca + mkdir -p ${ca_cert_dir} + # cp -r ${ca_dir}/* ${ca_cert_dir} + + # generate the node config + mkdir -p "${output_dir}" + # generate the node-cert + generate_node_cert "${sm_mode}" "${ca_dir}" "${output_dir}/conf" + # generate the node-script + generate_node_scripts "${output_dir}" + # generate the config.ini + local rpc_port=5895 + local agency_id="agency${count}" + generate_config_ini "${output_dir}/config.ini" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} + print_result +} + +parse_params() { + while getopts "o:e:sh" option; do + case $option in + o) + output_dir="$OPTARG" + ;; + e) + binary_path="$OPTARG" + file_must_exists "${binary_path}" + ;; + s) sm_mode="true" ;; + h) help ;; + *) help ;; + esac + done +} + +main() { + check_env + check_and_install_tassl + parse_params "$@" + deploy_nodes +} +main "$@" \ No newline at end of file diff --git a/cpp/tools/build_wedpr_mpc.sh b/cpp/tools/build_wedpr_mpc.sh new file mode 100644 index 00000000..2195fe7c --- /dev/null +++ b/cpp/tools/build_wedpr_mpc.sh @@ -0,0 +1,716 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +binary_name="wedpr-mpc" +binary_path= +output_dir="wedpr-mpc-node" + +listen_ip="0.0.0.0" + +cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS" +OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b" + +ca_dir="" +cert_conf="" +sm_cert_conf='sm_cert.cnf' +sm2_params="sm_sm2.param" +sm_mode="false" +days=36500 +rsa_key_length=2048 + +default_version="v1.1.0" +compatibility_version=${default_version} +command="deploy" + + +LOG_WARN() { + local content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + local content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +LOG_FATAL() { + local content=${1} + echo -e "\033[31m[FATAL] ${content}\033[0m" + exit 1 +} + +dir_must_exists() { + if [ ! -d "$1" ]; then + LOG_FATAL "$1 DIR does not exist, please check!" + fi +} + +get_value() { + local var_name=${1} + var_name=var_${var_name//./} + local res=$(eval echo '$'"${var_name}") + echo ${res} +} + +set_value() { + local var_name=${1} + var_name=var_${var_name//./} + local var_value=${2} + eval "${var_name}=${var_value}" +} + +dir_must_not_exists() { + if [ -d "$1" ]; then + LOG_FATAL "$1 DIR already exist, please check!" + fi +} + +file_must_not_exists() { + if [ -f "$1" ]; then + LOG_FATAL "$1 file already exist, please check!" + fi +} + +file_must_exists() { + if [ ! -f "$1" ]; then + LOG_FATAL "$1 file does not exist, please check!" + fi +} + +check_env() { + if [ "$(uname)" == "Darwin" ];then + macOS="macOS" + fi + if [ "$(uname -m)" != "x86_64" ];then + x86_64_arch="false" + fi +} +check_name() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[a-zA-Z0-9._-]+$ ]] || { + LOG_FATAL "$name name [$value] invalid, it should match regex: ^[a-zA-Z0-9._-]+\$" + } +} +check_and_install_tassl(){ + if [ -f "${OPENSSL_CMD}" ];then + return + fi + # https://en.wikipedia.org/wiki/Uname#Examples + local x86_64_name="x86_64" + local arm_name="aarch64" + local tassl_mid_name="linux" + if [[ -n "${macOS}" ]];then + x86_64_name="x86_64" + arm_name="arm64" + tassl_mid_name="macOS" + fi + + local tassl_post_fix="x86_64" + local platform="$(uname -m)" + if [[ "${platform}" == "${arm_name}" ]];then + tassl_post_fix="aarch64" + elif [[ "${platform}" == "${x86_64_name}" ]];then + tassl_post_fix="x86_64" + else + LOG_FATAL "Unsupported platform ${platform} for ${tassl_mid_name}" + exit 1 + fi + local tassl_package_name="tassl-1.1.1b-${tassl_mid_name}-${tassl_post_fix}" + local tassl_tgz_name="${tassl_package_name}.tar.gz" + local tassl_link_prefix="${cdn_link_header}/FISCO-BCOS/tools/tassl-1.1.1b/${tassl_tgz_name}" + LOG_INFO "Downloading tassl binary from ${tassl_link_prefix}..." + wget --no-check-certificate "${tassl_link_prefix}" + tar zxvf ${tassl_tgz_name} && rm ${tassl_tgz_name} + chmod u+x ${tassl_package_name} + mkdir -p "${HOME}"/.fisco + mv ${tassl_package_name} "${HOME}"/.fisco/tassl-1.1.1b +} + + +generate_sm_sm2_param() { + local output=$1 + cat << EOF > ${output} +-----BEGIN EC PARAMETERS----- +BggqgRzPVQGCLQ== +-----END EC PARAMETERS----- + +EOF +} + +generate_sm_cert_conf() { + local output=$1 + cat <"${output}" +oid_section = new_oids + +[ new_oids ] +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem # The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extensions to add to the cert + +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +default_days = 36500 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +policy = policy_match + +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_md = sm3 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +x509_extensions = v3_ca # The extensions to add to the self signed cert + +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = fisco +commonName = Organizational commonName (eg, fisco) +commonName_default = fisco +commonName_max = 64 + +[ usr_cert ] +basicConstraints=CA:FALSE +nsComment = "OpenSSL Generated Certificate" + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature + +[ v3enc_req ] + +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = keyAgreement, keyEncipherment, dataEncipherment + +[ v3_agency_root ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +[ v3_ca ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +EOF +} + +generate_cert_conf() { + local output=$1 + cat <"${output}" +[ca] +default_ca=default_ca +[default_ca] +default_days = 36500 +default_md = sha256 + +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req +[req_distinguished_name] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = FISCO-BCOS +commonName = Organizational commonName (eg, FISCO-BCOS) +commonName_default = FISCO-BCOS +commonName_max = 64 + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v4_req ] +basicConstraints = CA:TRUE + +EOF +} +print_result() { + echo "==============================================================" + LOG_INFO "${binary_name} path : ${binary_path}" + LOG_INFO "SM model : ${sm_mode}" + LOG_INFO "Output dir : ${output_dir}" + LOG_INFO "All completed. Files in ${output_dir}" +} + +help() { + echo $1 + cat < [Optional] wedpr-mpc binary exec + -o [Optional] output directory, default ./nodes + -s [Optional] SM SSL connection or not, default is false + -h Help + +deploy nodes e.g + bash $0 -o wedpr-mpc-node -e ./wedpr-mpc + bash $0 -o wedpr-mpc-node -e ./wedpr-mpc -s +EOF + exit 0 +} + +# generate the config.ini +generate_config_ini() { + local output="${1}" + + local rpc_listen_ip="${2}" + local rpc_listen_port="${3}" + local agency_info="${4}" + local agency_id="${5}" + + cat <"${output}" +[agency] + ; the agency-id of self-party + id = ${agency_id} + ; the agency info + +[mpc] + dataset_hdfs_path = /user/ppc/ + job_path = /data/app/ppc/mpc-job/ + mpc_root_path = /ppc/scripts/ppc-mpc/ + mpc_root_path_no_gateway = /ppc/scripts/ppc-mpc-no-gateway/ + read_per_batch_lines = 100000 + +[crypto] + sm_crypto = ${sm_mode} + +[rpc] + listen_ip=${rpc_listen_ip} + listen_port=${rpc_listen_port} + token = ppcs_psi_apikey + thread_count=4 + ; ssl or sm ssl + sm_ssl=${sm_mode} + ; ssl connection switch, if disable the ssl connection, default: false + ;disable_ssl = true + +[hdfs_storage] + ; the hdfs configuration + user = ppc + name_node = 127.0.0.1 + name_node_port = 9900 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 2000 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[log] + enable=true + ; print the log to std::cout or not, default print to the log files + enable_console_output = false + log_path=./log + ; info debug trace + level=info + ; MB + max_log_file_size=200 +EOF +} + +generate_script_template() +{ + local filepath=$1 + mkdir -p $(dirname $filepath) + cat << EOF > "${filepath}" +#!/bin/bash +SHELL_FOLDER=\$(cd \$(dirname \$0);pwd) + +LOG_ERROR() { + content=\${1} + echo -e "\033[31m[ERROR] \${content}\033[0m" +} + +LOG_INFO() { + content=\${1} + echo -e "\033[32m[INFO] \${content}\033[0m" +} + +EOF + chmod +x ${filepath} +} + +# TODO: support docker-mode +generate_node_scripts() { + local output=${1} + local ps_cmd="\$(ps aux|grep \${ppc_mpc}|grep -v grep|awk '{print \$2}')" + local start_cmd="nohup \${ppc_mpc} -c config.ini >>nohup.out 2>&1 &" + local stop_cmd="kill \${node_pid}" + local pid="pid" + local log_cmd="tail -n20 nohup.out" + local check_success="\$(${log_cmd} | grep running)" + generate_script_template "$output/start.sh" + cat <> "${output}/start.sh" +ppc_mpc=\${SHELL_FOLDER}/${binary_name} +cd \${SHELL_FOLDER} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +if [ ! -z \${node_pid} ];then + echo " \${node} is running, ${pid} is \$node_pid." + exit 0 +else + ${start_cmd} + sleep 1.5 +fi +try_times=4 +i=0 +while [ \$i -lt \${try_times} ] +do + node_pid=${ps_cmd} + success_flag=${check_success} + if [[ ! -z \${node_pid} && ! -z "\${success_flag}" ]];then + echo -e "\033[32m \${node} start successfully pid=\${node_pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start \${node} \033[0m" +${log_cmd} +EOF + chmod u+x "${output}/start.sh" + generate_script_template "$output/stop.sh" + cat <> "${output}/stop.sh" +ppc_mpc=\${SHELL_FOLDER}/${binary_name} +node=\$(basename \${SHELL_FOLDER}) +node_pid=${ps_cmd} +try_times=10 +i=0 +if [ -z \${node_pid} ];then + echo " \${node} isn't running." + exit 0 +fi +[ ! -z \${node_pid} ] && ${stop_cmd} > /dev/null +while [ \$i -lt \${try_times} ] +do + sleep 1 + node_pid=${ps_cmd} + if [ -z \${node_pid} ];then + echo -e "\033[32m stop \${node} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop \${node}" +exit 1 +EOF + chmod u+x "${output}/stop.sh" +} + +generate_all_node_scripts() { + local output=${1} + mkdir -p ${output} + + cat <"${output}/start_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/start.sh" ]];then + echo "try to start \${dir}" + bash \${dirpath}/\${dir}/start.sh & + fi +done +wait +EOF + chmod u+x "${output}/start_all.sh" + + cat <"${output}/stop_all.sh" +#!/bin/bash +dirpath="\$(cd "\$(dirname "\$0")" && pwd)" +cd "\${dirpath}" + +dirs=(\$(ls -l \${dirpath} | awk '/^d/ {print \$NF}')) +for dir in \${dirs[*]} +do + if [[ -f "\${dirpath}/\${dir}/config.ini" && -f "\${dirpath}/\${dir}/stop.sh" ]];then + echo "try to stop \${dir}" + bash \${dirpath}/\${dir}/stop.sh + fi +done +wait +EOF + chmod u+x "${output}/stop_all.sh" +} + +gen_non_sm_ca_cert() { + if [ ! -f "${cert_conf}" ]; then + generate_cert_conf "${cert_conf}" + fi + local ca_cert_dir="${1}" + file_must_not_exists "${ca_cert_dir}"/ca.key + file_must_not_exists "${ca_cert_dir}"/ca.crt + file_must_exists "${cert_conf}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + ${OPENSSL_CMD} genrsa -out "${ca_cert_dir}"/ca.key "${rsa_key_length}" + ${OPENSSL_CMD} req -new -x509 -days "${days}" -subj "/CN=FISCO-BCOS/O=FISCO-BCOS/OU=chain" -key "${ca_cert_dir}"/ca.key -config "${cert_conf}" -out "${ca_cert_dir}"/ca.crt 2>/dev/null + if [ ! -f "${ca_cert_dir}/cert.cnf" ];then + mv "${cert_conf}" "${ca_cert_dir}" + fi + LOG_INFO "Generate ca cert successfully!" +} + +gen_sm_ca_cert() { + local ca_cert_dir="${1}" + name=$(basename "$ca_cert_dir") + check_name chain "$name" + + if [ ! -f "${sm_cert_conf}" ]; then + generate_sm_cert_conf ${sm_cert_conf} + fi + + generate_sm_sm2_param "${sm2_params}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + "$OPENSSL_CMD" genpkey -paramfile "${sm2_params}" -out "$ca_cert_dir/sm_ca.key" 2>/dev/null + "$OPENSSL_CMD" req -config sm_cert.cnf -x509 -days "${days}" -subj "/CN=wedpr/O=wedpr/OU=ca" -key "$ca_cert_dir/sm_ca.key" -extensions v3_ca -out "$ca_cert_dir/sm_ca.crt" 2>/dev/null + if [ ! -f "${ca_cert_dir}/${sm_cert_conf}" ];then + cp "${sm_cert_conf}" "${ca_cert_dir}" + fi + if [ ! -f "${ca_cert_dir}/${sm2_params}" ];then + cp "${sm2_params}" "${ca_cert_dir}" + fi +} + + +gen_rsa_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/ca.key" + file_must_exists "$capath/ca.crt" + # check_name node "$node" + + file_must_not_exists "$ndpath"/"${type}".key + file_must_not_exists "$ndpath"/"${type}".crt + + mkdir -p "${ndpath}" + dir_must_exists "${ndpath}" + + ${OPENSSL_CMD} genrsa -out "${ndpath}"/"${type}".key "${rsa_key_length}" 2>/dev/null + ${OPENSSL_CMD} req -new -sha256 -subj "/CN=FISCO-BCOS/O=fisco-bcos/OU=agency" -key "$ndpath"/"${type}".key -config "$capath"/cert.cnf -out "$ndpath"/"${type}".csr + ${OPENSSL_CMD} x509 -req -days "${days}" -sha256 -CA "${capath}"/ca.crt -CAkey "$capath"/ca.key -CAcreateserial \ + -in "$ndpath"/"${type}".csr -out "$ndpath"/"${type}".crt -extensions v4_req -extfile "$capath"/cert.cnf 2>/dev/null + + ${OPENSSL_CMD} pkcs8 -topk8 -in "$ndpath"/"${type}".key -out "$ndpath"/pkcs8_node.key -nocrypt + cp "$capath"/ca.crt "$capath"/cert.cnf "$ndpath"/ + + rm -f "$ndpath"/"$type".csr + rm -f "$ndpath"/"$type".key + + mv "$ndpath"/pkcs8_node.key "$ndpath"/"$type".key + + LOG_INFO "Generate ${ndpath} cert successful!" +} + +gen_sm_node_cert_with_ext() { + local capath="$1" + local certpath="$2" + local type="$3" + local extensions="$4" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + file_must_not_exists "$ndpath/sm_${type}.crt" + file_must_not_exists "$ndpath/sm_${type}.key" + + "$OPENSSL_CMD" genpkey -paramfile "$capath/${sm2_params}" -out "$certpath/sm_${type}.key" 2> /dev/null + "$OPENSSL_CMD" req -new -subj "/CN=FISCO-BCOS/O=fisco-bcos/OU=${type}" -key "$certpath/sm_${type}.key" -config "$capath/sm_cert.cnf" -out "$certpath/sm_${type}.csr" 2> /dev/null + + "$OPENSSL_CMD" x509 -sm3 -req -CA "$capath/sm_ca.crt" -CAkey "$capath/sm_ca.key" -days "${days}" -CAcreateserial -in "$certpath/sm_${type}.csr" -out "$certpath/sm_${type}.crt" -extfile "$capath/sm_cert.cnf" -extensions "$extensions" 2> /dev/null + + rm -f "$certpath/sm_${type}.csr" +} + +gen_sm_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + mkdir -p "$ndpath" + dir_must_exists "$ndpath" + local node=$(basename "$ndpath") + check_name node "$node" + + gen_sm_node_cert_with_ext "$capath" "$ndpath" "${type}" v3_req + gen_sm_node_cert_with_ext "$capath" "$ndpath" "en${type}" v3enc_req + #nodeid is pubkey + $OPENSSL_CMD ec -in "$ndpath/sm_${type}.key" -text -noout 2> /dev/null | sed -n '7,11p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | awk '{print substr($0,3);}' | cat > "${ndpath}/sm_${type}.nodeid" + cp "$capath/sm_ca.crt" "$ndpath" +} + +generate_ca_cert() { + local sm_mode="$1" + local ca_cert_path="$2" + LOG_INFO "generate ca cert, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" + mkdir -p "${ca_cert_path}" + if [[ "${sm_mode}" == "false" ]]; then + gen_non_sm_ca_cert "${ca_cert_path}" + else + gen_sm_ca_cert "${ca_cert_path}" + fi + LOG_INFO "generate ca cert successfully, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" +} + +generate_node_cert() { + local sm_mode="$1" + local ca_cert_path="${2}" + local node_cert_path="${3}" + + mkdir -p ${node_cert_path} + if [[ "${sm_mode}" == "false" ]]; then + gen_rsa_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + else + gen_sm_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + fi +} + + +deploy_nodes() +{ + echo "output_dir:${output_dir}" + dir_must_not_exists "${output_dir}" + mkdir -p "$output_dir" + dir_must_exists "${output_dir}" + + # check the binary + if [[ ! -f "$binary_path" ]]; then + LOG_FATAL "wedpr-component binary exec ${binary_path} not exist, Must copy binary file ${binary_name} to ${binary_path}" + fi + + local agency_index=0 + local agency_info="" + # generate the ca-cert + ca_dir="${output_dir}"/ca + cert_conf="${output_dir}/cert.cnf" + generate_ca_cert "${sm_mode}" "${ca_dir}" + # start_all.sh and stop_all.sh + # generate_all_node_scripts "${output_dir}" + cp "${binary_path}" "${output_dir}" + # generate cert for the node + ca_cert_dir="${output_dir}"/ca + mkdir -p ${ca_cert_dir} + # cp -r ${ca_dir}/* ${ca_cert_dir} + + # generate the node config + mkdir -p "${output_dir}" + # generate the node-cert + generate_node_cert "${sm_mode}" "${ca_dir}" "${output_dir}/conf" + # generate the node-script + generate_node_scripts "${output_dir}" + # generate the config.ini + local rpc_port=5894 + local agency_id="agency${count}" + generate_config_ini "${output_dir}/config.ini" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} + print_result +} + +parse_params() { + while getopts "o:e:sh" option; do + case $option in + o) + output_dir="$OPTARG" + ;; + e) + binary_path="$OPTARG" + file_must_exists "${binary_path}" + ;; + s) sm_mode="true" ;; + h) help ;; + *) help ;; + esac + done +} + +main() { + check_env + check_and_install_tassl + parse_params "$@" + deploy_nodes +} +main "$@" \ No newline at end of file diff --git a/cpp/tools/generate_psi.py b/cpp/tools/generate_psi.py new file mode 100644 index 00000000..06263db7 --- /dev/null +++ b/cpp/tools/generate_psi.py @@ -0,0 +1,75 @@ +# Copyright 2022 Ant Group Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from random import randint +from random import sample +import csv +import sys + + +def random_with_N_digits(n): + range_start = 10 ** (n - 1) + range_end = (10**n) - 1 + return randint(range_start, range_end) + + +row_list = [] +len1 = 10**2 +len2 = 10 +len3 = 10 +len4 = 10 + +if len(sys.argv) > 1: + len1 = int(sys.argv[1]) + len2 = int(len1 / 2) + +if len(sys.argv) > 2: + len3 = int(sys.argv[2]) + +len4 = int(len3 / 2) +print(len1, len2) + + +for i in range(len1): + data_list = [random_with_N_digits(18)] + row_list.append(data_list) + +row_list2 = sample(row_list, len2) +for i in range(len2, len1): + data_list = [random_with_N_digits(18)] + row_list2.append(data_list) + +row_list3 = sample(row_list, len4) +for i in range(len4, len3): + data_list = [random_with_N_digits(18)] + row_list3.append(data_list) + +print(len(row_list2)) +print(len(row_list3)) + +with open('psi_1.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["id"]) + writer.writerows(row_list) + +with open('psi_2.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["id"]) + writer.writerows(row_list2) + +with open('psi_3.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["id"]) + writer.writerows(row_list3) diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh new file mode 100644 index 00000000..f254e7cf --- /dev/null +++ b/cpp/tools/install_depends.sh @@ -0,0 +1,153 @@ +#!/bin/bash +os="" +LOG_WARN() { + local content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + local content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +LOG_FATAL() { + local content=${1} + echo -e "\033[31m[FATAL] ${content}\033[0m" + exit 1 +} + +install_gsasl_depend() +{ + os_type="${1}" + LOG_INFO "check gsasl..." + gsasl=$(whereis libgsasl|grep "libgsasl.a") + if [[ ! -z "${gsasl}" ]];then + LOG_INFO "the libgasal.a has already exists!" + return + fi + LOG_INFO "download and install gsasl..." + wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz && tar -xvf libgsasl-1.8.0.tar.gz + + # centos + if [[ "${os_type}" == "centos" ]];then + cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install + fi + # ubuntu + if [[ "${os_type}" == "ubuntu" ]];then + cd libgsasl-1.8.0 && ./configure --with-pic --disable-ntlm --disable-gs2 --disable-gssapi --without-stringprep && make -j4 && make install + fi + LOG_INFO "download and install gsasl success..." +} + +install_nasm_depend() +{ + LOG_INFO "check nasm..." + nasm=$(nasm --version | awk -F' ' '{print $3}') + + echo "### current nasm: ${nasm}" + # require the nasm >= 2.15 + if [[ "${nasm}" == "2.15" ]] || [[ "${nasm}" > "2.15" ]];then + LOG_INFO "the nasm >= 2.15 has already exists!" + return + fi + LOG_INFO "download and install nasm 2.15..." + wget --no-check-certificate https://www.nasm.us/pub/nasm/releasebuilds/2.15/nasm-2.15.tar.gz && tar -xvf nasm-2.15.tar.gz + cd nasm-2.15 && ./configure && make -j4 && make install + LOG_INFO "download and install nasm success..." +} + +install_binutils_depend() +{ + LOG_INFO "download and install binutils 2.35..." + wget --no-check-certificate http://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.gz && tar -zxf binutils-2.35.tar.gz + cd binutils-2.35 && ./configure && make -j4 && make install +} + +install_centos_depends() +{ + LOG_INFO "install depends for centos ..." + # install the basic package + sudo yum install -y bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf + LOG_INFO "install basic-package-depends for centos success..." + + LOG_INFO "install gsasl ... " + install_gsasl_depend "centos" + LOG_INFO "install gsasl success!" + + LOG_INFO "install nasm ... " + install_nasm_depend + LOG_INFO "install nasm success!" + + LOG_INFO "install depends for centos success ..." +} + +install_ubuntu_depends() +{ + LOG_INFO "install depends for ubuntu ..." + # install the basic package + sudo apt install -y bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf + LOG_INFO "install basic-package-depends for ubuntu success..." + + LOG_INFO "install gsasl ... " + install_gsasl_depend "ubuntu" + LOG_INFO "install gsasl success!" + + LOG_INFO "install nasm ... " + install_nasm_depend + LOG_INFO "install nasm success!" + + LOG_INFO "install binutils ... " + install_binutils_depend + LOG_INFO "install binutils success!" + + LOG_INFO "install depends for ubuntu success ..." +} + +install_macos_depends() +{ + LOG_INFO "install depends for macos ..." + brew install clang cmake autoconf gsasl nasm + LOG_INFO "install depends for macos success ..." +} + +help() { + echo $1 + cat < [Optional] the os to install depends, support 'centos', 'ubuntu' and 'macos' now, default is empty + -h Help + +e.g: + bash $0 -o centos + bash $0 -o ubuntu + bash $0 -o macos +EOF + exit 0 +} + +parse_params() { + while getopts "o:h" option; do + case $option in + o) os="${OPTARG}" + ;; + h) help ;; + *) help ;; + esac + done +} +main() { + parse_params "$@" + if [[ -z \${os} ]];then + help + fi + if [ "${os}" == "centos" ];then + install_centos_depends + fi + if [ "${os}" == "ubuntu" ];then + install_ubuntu_depends + fi + if [ "${os}" == "macos" ];then + install_macos_depends + fi +} +main "$@" diff --git a/cpp/tools/ppc-builder/build_ppc.py b/cpp/tools/ppc-builder/build_ppc.py new file mode 100644 index 00000000..e609a1c5 --- /dev/null +++ b/cpp/tools/ppc-builder/build_ppc.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +# Note: here can't be refactored by autopep +import sys +sys.path.append("src/") + +from controller import commandline_helper +from common import utilities + + +def main(): + try: + args = commandline_helper.parse_command() + commandline_helper.execute_command(args) + except Exception as error: + utilities.log_error("%s" % error) + + +if __name__ == "__main__": + main() diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml new file mode 100644 index 00000000..167afe85 --- /dev/null +++ b/cpp/tools/ppc-builder/conf/config-example.toml @@ -0,0 +1,208 @@ +[tars] +tars_pkg_dir = "bin/" +app_name = "ppc1" + +[crypto] +# the gateway-service enable ssl or not, default enable ssl +gateway_disable_ssl = false +# the gateway-service enable sm-ssl or not, default disable sm-ssl +gateway_sm_ssl = false +# the rpc use sm-ssl or not +rpc_sm_ssl = false +# the rpc disable ssl or not +rpc_disable_ssl = true +# use sm-crypto or not +sm_crypto = false + +[[agency]] +name = "agency0" +#the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes +holding_msg_minutes = 30 + + [agency.gateway] + deploy_ip=["127.0.0.1:2"] + # gateway listen ip + listen_ip="0.0.0.0" + # gateway listen start port + listen_port=40300 + # the thread count + thread_count = 4 + # the cache config for the gateway + cache_type = 0 + cache_proxy = "" + cache_obServer = "" + cache_cluster = "" + cache_host = "127.0.0.1" + cache_port = 6379 + cache_password = "" + cache_database = 1 + cache_pool_size = 16 + # the redis connection timeout, in ms, default is 500ms + cache_connection_timeout = 500 + # the redis socket timeout, in ms, default is 500ms + cache_socket_timeout = 500 + # the tars config + # gateway tars server listen ip + tars_listen_ip="0.0.0.0" + # gateway tars server listen start port + tars_listen_port=40600 + # gateway connected peers, should be all of the gateway peers info + [[agency.gateway.peers]] + agency = "agency0" + endpoints = ["127.0.0.1:40300", "127.0.0.1:40301"] + [[agency.gateway.peers]] + agency = "agency1" + endpoints = ["127.0.0.1:40320", "127.0.0.1:40321"] + + # configuration for the ppc-node + [[agency.node]] + # disable the ra2018 psi or not, default enable ra2018 + disable_ra2018 = false + deploy_ip=["127.0.0.1:2"] + # node name, Notice: node_name in the same agency and group must be unique + node_name = "node0" + # node tars server listen ip + tars_listen_ip="0.0.0.0" + # node tars server listen port + tars_listen_port=40402 + # the rpc config for the node + [agency.node.rpc] + listen_ip = "0.0.0.0" + listen_port = 10200 + thread_count = 4 + # the ra2018 config for the node + [agency.node.ra2018psi] + # The database used to store cuckoo-filter + database = "ra2018_0" + # The capacity of the cuckoo filter in MB, the default is 1MB + cuckoofilter_capacity = 1 + # The length of each element of the cuckoo filter (bits), default 64bits + cuckoofilter_tagBits = 64 + # The number of buckets for the cuckoo filter, the default is 4 + cuckoofilter_buckets_num = 4 + # When the cuckoo filter replaces an element, the maximum number of element replacements allowed + cuckoofilter_max_kick_out_count = 20 + # The trash-bucket-size for the cuckoo-filter + trash_bucket_size = 10000 + # The cuckoo filter capacity that can be cached in memory in MB + cuckoofilter_cache_size = 256 + # Cache size in MB + psi_cache_size = 1024 + # When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + data_batch_size = 1000000 + # use hdfs to store the cuckoo-filter or not + use_hdfs = false + # the storage config + [agency.node.storage] + host = "127.0.0.1" + # the mysqlx_port + port = 33060 + user = "wedpr" + password = "" + database = "" + # the hdfs storage config + [agency.node.hdfs_storage] + user = "" + name_node = "127.0.0.1" + name_node_port = 9000 + token = "" + # the gateway config + [agency.node.gateway] + tars_endpoints = ["127.0.0.1:40600", "127.0.0.1:40601"] + + +[[agency]] +name = "agency1" +#the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes +holding_msg_minutes = 30 + + [agency.gateway] + deploy_ip=["127.0.0.1:2"] + # gateway listen ip + listen_ip="0.0.0.0" + # gateway listen start port + listen_port=40320 + # the thread count + thread_count = 4 + # the cache config for the gateway + cache_type = 0 + cache_proxy = "" + cache_obServer = "" + cache_cluster = "" + cache_host = "127.0.0.1" + cache_port = 6379 + cache_password = "" + cache_database = 2 + cache_pool_size = 16 + # the redis connection timeout, in ms, default is 500ms + cache_connection_timeout = 500 + # the redis socket timeout, in ms, default is 500ms + cache_socket_timeout = 500 + # the tars config + # gateway tars server listen ip + tars_listen_ip="0.0.0.0" + # gateway tars server listen start port + tars_listen_port=40620 + # gateway connected peers, should be all of the gateway peers info + [[agency.gateway.peers]] + agency = "agency0" + endpoints = ["127.0.0.1:40300", "127.0.0.1:40301"] + [[agency.gateway.peers]] + agency = "agency1" + endpoints = ["127.0.0.1:40320", "127.0.0.1:40321"] + + # configuration for the ppc-node + [[agency.node]] + # disable the ra2018 psi or not, default enable ra2018 + disable_ra2018 = false + deploy_ip=["127.0.0.1:2"] + # node name, Notice: node_name in the same agency and group must be unique + node_name = "node0" + # node tars server listen ip + tars_listen_ip="0.0.0.0" + # node tars server listen port + tars_listen_port=40422 + # the rpc config for the node + [agency.node.rpc] + listen_ip = "0.0.0.0" + listen_port = 10220 + thread_count = 4 + # the ra2018 config for the node + [agency.node.ra2018psi] + # The database used to store cuckoo-filter + database = "ra2018_1" + # The capacity of the cuckoo filter in MB, the default is 1MB + cuckoofilter_capacity = 1 + # The length of each element of the cuckoo filter (bits), default 64bits + cuckoofilter_tagBits = 64 + # The number of buckets for the cuckoo filter, the default is 4 + cuckoofilter_buckets_num = 4 + # When the cuckoo filter replaces an element, the maximum number of element replacements allowed + cuckoofilter_max_kick_out_count = 20 + # The trash-bucket-size for the cuckoo-filter + trash_bucket_size = 10000 + # The cuckoo filter capacity that can be cached in memory in MB + cuckoofilter_cache_size = 256 + # Cache size in MB + psi_cache_size = 1024 + # When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + data_batch_size = 1000000 + # use hdfs to store the cuckoo-filter or not + use_hdfs = false + # the storage config + [agency.node.storage] + host = "127.0.0.1" + # the mysqlx_port + port = 33060 + user = "wedpr" + password = "" + database = "" + # the hdfs storage config + [agency.node.hdfs_storage] + user = "" + name_node = "127.0.0.1" + name_node_port = 9000 + token = "" + # the gateway config + [agency.node.gateway] + tars_endpoints = ["127.0.0.1:40620", "127.0.0.1:40621"] diff --git a/cpp/tools/ppc-builder/requirements.txt b/cpp/tools/ppc-builder/requirements.txt new file mode 100644 index 00000000..aee2aa42 --- /dev/null +++ b/cpp/tools/ppc-builder/requirements.txt @@ -0,0 +1,4 @@ +configparser +requests +toml +uuid diff --git a/cpp/tools/ppc-builder/src/common/utilities.py b/cpp/tools/ppc-builder/src/common/utilities.py new file mode 100644 index 00000000..34679544 --- /dev/null +++ b/cpp/tools/ppc-builder/src/common/utilities.py @@ -0,0 +1,189 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import sys +import re +import os +import subprocess +import logging +import configparser + +logging.basicConfig(format='%(message)s', + level=logging.INFO) + + +class ServiceInfo: + ssl_file_list = ["ca.crt", "ssl.key", "ssl.crt"] + sm_ssl_file_list = ["sm_ca.crt", "sm_ssl.key", + "sm_ssl.crt", "sm_enssl.key", "sm_enssl.crt"] + + node_service_postfix = "NodeService" + gateway_servant = "GatewayService" + gateway_servant_obj = ["GatewayServiceObj"] + + node_servant = ["FrontService"] + node_servant_object = ["FrontServiceObj"] + + cert_generation_script_path = "src/scripts/gen_cert.sh" + node_service_type = "node" + gateway_service_type = "gateway" + supported_service_type = [node_service_type, gateway_service_type] + + +class ConfigInfo: + config_ini_file = "config.ini" + tars_config_file = "tars.conf" + + tpl_abs_path = "src/tpl/" + pwd_path = os.getcwd() + node_config_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "config.ini.node") + gateway_config_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "config.ini.gateway") + tars_config_tpl_path = os.path.join(pwd_path, tpl_abs_path, "tars.conf") + + tars_start_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "tars_start.sh") + tars_stop_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "tars_stop.sh") + + tars_start_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "tars_start_all.sh") + tars_stop_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "tars_stop_all.sh") + + ppc_gateway_binary_name = "ppc-gateway-service" + ppc_node_binary_name = "ppc-pro-node" + + +class CommandInfo: + generate_config = "genconfig" + extend_config = "extend" + supported_command_list = [generate_config, extend_config] + + +def log_error(error_msg): + logging.error("\033[31m%s \033[0m" % error_msg) + + +def log_info(error_msg): + logging.info("\033[32m%s \033[0m" % error_msg) + + +def format_info(info): + return ("\033[32m%s \033[0m" % info) + + +def log_debug(error_msg): + logging.debug("%s" % error_msg) + + +def get_item_value(config, key, default_value, must_exist, desc): + if key in config: + return config[key] + if must_exist: + raise Exception("the value for %s.%s must be set" % (desc, key)) + return default_value + + +def get_value(config, section, key, default_value, must_exist): + if section in config and key in config[section]: + return config[section][key] + if must_exist: + raise Exception("the value for %s must be set" % key) + return default_value + + +def execute_command_and_getoutput(command): + status, output = subprocess.getstatusoutput(command) + if status != 0: + log_error( + "execute command %s failed, error message: %s" % (command, output)) + return (False, output) + return (True, output) + + +def execute_command(command): + (ret, result) = execute_command_and_getoutput(command) + return ret + + +def mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + + +def removeDir(path): + if os.path.exists(path): + os.removedirs(path) + + +def mkfiledir(filepath): + parent_dir = os.path.abspath(os.path.join(filepath, "..")) + if os.path.exists(parent_dir) is False: + mkdir(parent_dir) + + +def generate_service_name(prefix, service_name): + return prefix + service_name + + +def convert_bool_to_str(value): + if value is True: + return "true" + return "false" + + +def print_split_info(): + log_info("=========================================================") + + +def print_badge(badge): + log_info("----------- %s -----------" % badge) + + +def file_must_exist(file_path): + if not os.path.exists(file_path): + log_error("%s does not exist, please check" % file_path) + sys.exit(-1) + + +def store_config(config_content, config_type, config_path, desc): + """ + store the generated genesis config content for given node + """ + if os.path.exists(config_path): + log_error("* store %s config for %s failed for the config %s already exists." % + (config_type, desc, config_path)) + return False + log_info("* store %s config for %s\n\t path: %s" % + (config_type, desc, config_path)) + + if os.path.exists(os.path.dirname(config_path)) is False: + mkdir(os.path.dirname(config_path)) + + with open(config_path, 'w') as configFile: + config_content.write(configFile) + log_info("* store %s config for %s success" % + (config_type, desc)) + return True + + +def load_config(file_path): + # load the config from tpl_config_path + config_content = configparser.ConfigParser( + comment_prefixes='/', allow_no_value=True) + # to distinguish uppercase/lowercase letters + config_content.optionxform = str + config_content.read(file_path) + return config_content + + +def check_service_name(tag, service_name): + """ + Note: tars service name can only contain letters and numbers + """ + service_name_len = len(service_name) + ret = re.search(r'^[A-Za-z0-9]+', service_name).span() + if ret is None or (ret[0] != 0 or ret[1] != service_name_len): + raise Exception( + "the %s must be letters|numbers, invalid value: %s" % (tag, service_name)) diff --git a/cpp/tools/ppc-builder/src/config/binary_generator.py b/cpp/tools/ppc-builder/src/config/binary_generator.py new file mode 100644 index 00000000..d4bc175b --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/binary_generator.py @@ -0,0 +1,26 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import os +from common import utilities + + +class BinaryGenerator: + """ + generate the binary + """ + def generate_binary(binary_path, dst_path): + if os.path.exists(binary_path) is False: + utilities.log_error( + "The specified binary %s not exists!" % binary_path) + return False + # the binary has already been generated + if os.path.exists(dst_path) is True: + return True + utilities.mkfiledir(dst_path) + command = "cp %s %s" % (binary_path, dst_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error("copy binary from %s to %s failed, error: %s") % ( + binary_path, dst_path, output) + return False + return True diff --git a/cpp/tools/ppc-builder/src/config/cert_generator.py b/cpp/tools/ppc-builder/src/config/cert_generator.py new file mode 100644 index 00000000..77ae0fb9 --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/cert_generator.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import os +from common import utilities + + +class CertGenerator: + """ + the cert generator + """ + def generate_ca_cert(sm_mode, ca_cert_path): + """ + generate the ca cert + """ + use_sm = "false" + if sm_mode is True: + use_sm = "true" + command = "bash %s generate_ca_cert %s %s" % ( + utilities.ServiceInfo.cert_generation_script_path, use_sm, ca_cert_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error( + "* generate ca cert error, sm_mode: %d, ca cert path: %s, error: %s" % (sm_mode, ca_cert_path, output)) + return False + utilities.log_info( + "* generate ca cert success, sm_mode: %d, ca cert path: %s" % (sm_mode, ca_cert_path)) + return True + + def generate_node_cert(sm_mode, ca_cert_path, node_cert_path): + """ + generate the node cert + """ + use_sm = "false" + if sm_mode is True: + use_sm = "true" + command = "bash %s generate_node_cert %s %s %s" % ( + utilities.ServiceInfo.cert_generation_script_path, use_sm, ca_cert_path, node_cert_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error("* generate node cert error, sm_mode: %d, ca cert path: %s, node cert path: %s" % + (sm_mode, ca_cert_path, node_cert_path)) + return False + utilities.log_info("* generate the node cert success, sm_mode: %d, ca cert path: %s, node cert path: %s" % + (sm_mode, ca_cert_path, node_cert_path)) + return True + + def generate_private_key(output_path): + command = "bash %s generate_private_key_for_psi_server %s" % ( + utilities.ServiceInfo.cert_generation_script_path, output_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error( + "* generate private key error, output_path: %s" % output_path) + return (False, output) + utilities.log_info( + "* generate private_key success, path: %s, private_key: %s" % (output_path, output)) + return (True, output) diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py new file mode 100644 index 00000000..285baf34 --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py @@ -0,0 +1,370 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from common import utilities + + +class TarsConfig: + """ + the tars config + """ + + def __init__(self, config): + self.config = config + self.app_name = utilities.get_value( + self.config, "tars", "app_name", None, True) + self.binary_path = utilities.get_value( + self.config, "tars", "tars_pkg_dir", None, True) + + +class PeerInfo: + def __init__(self, agency, endpoints): + self.agency = agency + self.endpoints = endpoints + + +class GatewayConfig: + """ + the gateway config + """ + + def __init__(self, app_name, agency_name, holding_msg_minutes, config, config_section, must_exist): + self.config = config + self.config_section = config_section + self.app_name = app_name + self.holding_msg_minutes = holding_msg_minutes + self.agency_name = agency_name + self.service_name = "%s%s" % ( + self.agency_name, utilities.ServiceInfo.gateway_servant) + self.servant_list = [utilities.ServiceInfo.gateway_servant] + self.servant_object_list = utilities.ServiceInfo.gateway_servant_obj + + # the deploy_ip + self.deploy_ip = utilities.get_item_value( + self.config, "deploy_ip", None, must_exist, config_section) + # the listen_ip + self.listen_ip = utilities.get_item_value( + self.config, "listen_ip", "0.0.0.0", False, config_section) + # the listen_port + self.listen_port = utilities.get_item_value( + self.config, "listen_port", None, must_exist, config_section) + # the thread count + self.thread_count = utilities.get_item_value( + self.config, "thread_count", 4, False, config_section) + # the peers + self.peers = [] + peers = utilities.get_item_value( + self.config, "peers", None, must_exist, config_section) + for peer in peers: + agency = utilities.get_item_value( + peer, "agency", None, must_exist, "[[peers]]") + endpoints = utilities.get_item_value( + peer, "endpoints", None, must_exist, "[[peers]]") + self.peers.append(PeerInfo(agency, endpoints)) + + self.cache_type = utilities.get_item_value( + self.config, "cache_type", 0, must_exist, config_section) + self.cache_proxy = utilities.get_item_value( + self.config, "cache_proxy", None, must_exist, config_section) + self.cache_obServer = utilities.get_item_value( + self.config, "cache_obServer", None, must_exist, config_section) + self.cache_cluster = utilities.get_item_value( + self.config, "cache_cluster", None, must_exist, config_section) + self.cache_host = utilities.get_item_value( + self.config, "cache_host", None, must_exist, config_section) + self.cache_port = utilities.get_item_value( + self.config, "cache_port", None, must_exist, config_section) + self.cache_password = utilities.get_item_value( + self.config, "cache_password", "", must_exist, config_section) + self.cache_database = utilities.get_item_value( + self.config, "cache_database", None, must_exist, config_section) + self.cache_pool_size = utilities.get_item_value( + self.config, "cache_pool_size", 16, False, config_section) + self.cache_connection_timeout = utilities.get_item_value( + self.config, "cache_connection_timeout", 500, False, config_section) + self.cache_socket_timeout = utilities.get_item_value( + self.config, "cache_socket_timeout", 500, False, config_section) + # the tars_listen_ip + self.tars_listen_ip = utilities.get_item_value( + self.config, "tars_listen_ip", "0.0.0.0", False, config_section) + # the tars_listen_port + self.tars_listen_port = utilities.get_item_value( + self.config, "tars_listen_port", None, must_exist, config_section) + + +class RpcConfig: + """ + the rpc config + """ + + def __init__(self, config, config_section, must_exist): + self.config = config + self.config_section = config_section + self.listen_ip = utilities.get_item_value( + self.config, "listen_ip", "0.0.0.0", False, config_section) + self.listen_port = utilities.get_item_value( + self.config, "listen_port", None, must_exist, config_section) + self.thread_count = utilities.get_item_value( + self.config, "thread_count", 4, False, config_section) + + +class StorageConfig: + """ + the sql storage config + """ + + def __init__(self, config, config_section, must_exist): + self.config = config + self.config_section = config_section + # the mysql configuration + self.host = utilities.get_item_value( + self.config, "host", None, must_exist, config_section) + self.port = utilities.get_item_value( + self.config, "port", None, must_exist, config_section) + self.user = utilities.get_item_value( + self.config, "user", None, must_exist, config_section) + self.password = utilities.get_item_value( + self.config, "password", None, must_exist, config_section) + self.database = utilities.get_item_value( + self.config, "database", None, must_exist, config_section) + + +class HDFSStorageConfig: + """ + the hdfs storage config + """ + + def __init__(self, config, config_section, must_exist): + self.config = config + self.config_section = config_section + # the hdfs configuration + self.user = utilities.get_item_value( + self.config, "user", None, must_exist, config_section) + self.name_node = utilities.get_item_value( + self.config, "name_node", None, must_exist, config_section) + self.name_node_port = utilities.get_item_value( + self.config, "name_node_port", None, must_exist, config_section) + self.token = utilities.get_item_value( + self.config, "token", "", False, config_section) + + +class RA2018PSIConfig: + """ + the ra2018-psi config + """ + + def __init__(self, config, config_section, must_exist): + self.config = config + self.config_section = config_section + self.database = utilities.get_item_value( + self.config, "database", None, must_exist, config_section) + self.cuckoofilter_capacity = utilities.get_item_value( + self.config, "cuckoofilter_capacity", 1, False, config_section) + self.cuckoofilter_tagBits = utilities.get_item_value( + self.config, "cuckoofilter_tagBits", 32, False, config_section) + self.cuckoofilter_buckets_num = utilities.get_item_value( + self.config, "cuckoofilter_buckets_num", 4, False, config_section) + self.cuckoofilter_max_kick_out_count = utilities.get_item_value( + self.config, "cuckoofilter_max_kick_out_count", 20, False, config_section) + self.trash_bucket_size = utilities.get_item_value( + self.config, "trash_bucket_size", 10000, False, config_section) + self.cuckoofilter_cache_size = utilities.get_item_value( + self.config, "cuckoofilter_cache_size", 256, False, config_section) + self.psi_cache_size = utilities.get_item_value( + self.config, "psi_cache_size", 1024, False, config_section) + self.data_batch_size = utilities.get_item_value( + self.config, "data_batch_size", -1, False, config_section) + self.use_hdfs = utilities.get_item_value( + self.config, "use_hdfs", False, False, config_section) + + +class NodeGatewayConfig: + """ + the gateway config for the node + """ + + def __init__(self, app_name, agency_name, config, node_must_exists): + self.config = config + self.app_name = app_name + self.agency_name = agency_name + self.desc = "[agency.node]" + self.endpoints = utilities.get_item_value( + self.config, "tars_endpoints", None, node_must_exists, self.desc) + # obtain the gateway name + self.service_name = "%s.%s%s" % ( + self.app_name, self.agency_name, utilities.ServiceInfo.gateway_servant) + + +class NodeConfig: + """ + the ppc-node config + """ + + def __init__(self, app_name, agency_name, holding_msg_minutes, config, must_exist): + self.config = config + self.section_name = "[[agency.node]]." + self.holding_msg_minutes = holding_msg_minutes + self.app_name = app_name + # set the agency_name + self.agency_name = agency_name + # disable ra2018 or not, default enable the ra2018 + self.disable_ra2018 = utilities.get_item_value( + self.config, "disable_ra2018", False, False, self.section_name) + # the deploy_ip + self.deploy_ip = utilities.get_item_value( + self.config, "deploy_ip", None, must_exist, self.section_name) + # the node_name + self.node_name = utilities.get_item_value( + self.config, "node_name", None, must_exist, self.section_name) + # the tars_listen_ip + self.tars_listen_ip = utilities.get_item_value( + self.config, "tars_listen_ip", "0.0.0.0", False, self.section_name) + # the tars_listen_port + self.tars_listen_port = utilities.get_item_value( + self.config, "tars_listen_port", None, must_exist, self.section_name) + utilities.log_debug("load the node config success") + + # parse the rpc config + utilities.log_debug("load the rpc config") + rpc_config_section_name = "[[agency.node.rpc]]" + rpc_config_object = utilities.get_item_value( + self.config, "rpc", None, must_exist, rpc_config_section_name) + self.rpc_config = None + if rpc_config_object is not None: + self.rpc_config = RpcConfig( + rpc_config_object, rpc_config_section_name, must_exist) + utilities.log_debug("load the rpc config success") + + # parse the ra2018-psi config + utilities.log_debug("load the ra2018psi config") + ra2018psi_config_section = "[[agency.node.ra2018psi]]" + ra2018psi_config_object = utilities.get_item_value( + self.config, "ra2018psi", None, must_exist, ra2018psi_config_section) + self.ra2018psi_config = None + if ra2018psi_config_object is not None: + self.ra2018psi_config = RA2018PSIConfig( + ra2018psi_config_object, ra2018psi_config_section, must_exist) + utilities.log_debug("load the ra2018psi config success") + # parse the storage config + utilities.log_debug("load the sql storage config") + storage_config_section = "[[agency.node.storage]]" + storage_config_object = utilities.get_item_value( + self.config, "storage", None, must_exist, storage_config_section) + self.storage_config = None + if storage_config_object is not None: + self.storage_config = StorageConfig( + storage_config_object, storage_config_section, must_exist) + utilities.log_debug("load the sql storage success") + # parse the hdfs storage config + hdfs_storage_must_configured = False + if self.ra2018psi_config is not None: + hdfs_storage_must_configured = self.ra2018psi_config.use_hdfs + utilities.log_debug("load the hdfs storage config") + storage_config_section = "[[agency.node.hdfs_storage]]" + hdfs_storage_config_object = utilities.get_item_value( + self.config, "hdfs_storage", None, hdfs_storage_must_configured, storage_config_section) + self.hdfs_storage_config = None + if hdfs_storage_config_object is not None: + self.hdfs_storage_config = HDFSStorageConfig( + hdfs_storage_config_object, storage_config_section, hdfs_storage_must_configured) + utilities.log_debug("load the hdfs storage success") + # parse the gateway-inforamtion + utilities.log_debug("load the gateway config") + gateway_config_section = "[[agency.node.gateway]]" + gateway_config_object = utilities.get_item_value( + self.config, "gateway", None, must_exist, gateway_config_section) + self.gateway_config = None + if gateway_config_object is not None: + self.gateway_config = NodeGatewayConfig( + self.app_name, self.agency_name, gateway_config_object, must_exist) + utilities.log_debug("load the gateway success") + + # set the server name + self.service_name = "%s%s%s" % ( + self.agency_name, self.node_name, utilities.ServiceInfo.node_service_postfix) + # set the servant name + self.servant_list = utilities.ServiceInfo.node_servant + self.servant_object_list = utilities.ServiceInfo.node_servant_object + + +class AgencyConfig: + """ + the agency config + """ + + def __init__(self, app_name, config, gateway_must_exists, node_must_exists): + self.app_name = app_name + self.config = config + self.section_name = "[[agency]]" + # the agency-name + self.agency_name = utilities.get_item_value( + self.config, "name", None, True, self.section_name) + # the holding_msg_minutes + self.holding_msg_minutes = utilities.get_item_value( + self.config, "holding_msg_minutes", 30, False, self.section_name) + # parse the gateway config + utilities.log_debug("load the gateway config") + gateway_config_section_name = "[agency.gateway]" + gateway_config_object = utilities.get_item_value( + self.config, "gateway", None, gateway_must_exists, gateway_config_section_name) + self.gateway_config = None + if gateway_config_object is not None: + self.gateway_config = GatewayConfig( + self.app_name, self.agency_name, self.holding_msg_minutes, gateway_config_object, + gateway_config_section_name, gateway_must_exists) + utilities.log_debug("load the gateway config success") + + # parse the node config + utilities.log_debug("load the node config") + node_config_section_name = "[[agency.node]]" + node_config_list = utilities.get_item_value( + self.config, "node", None, node_must_exists, node_config_section_name) + self.node_list = {} + # TODO: check the node-name + for node_object in node_config_list: + node_config = NodeConfig( + self.app_name, self.agency_name, self.holding_msg_minutes, node_object, node_must_exists) + self.node_list[node_config.node_name] = node_config + utilities.log_debug( + "load node config for %s success" % node_config.node_name) + utilities.log_debug("load the node config success") + + +class PPCDeployConfig: + """ + load all config from config.toml + """ + + def __init__(self, config, gateway_must_exists, node_must_exists): + self.config = config + # load the crypto config + utilities.log_debug("load the crypto config") + crypto_section = "crypto" + self.gateway_disable_ssl = utilities.get_value( + self.config, crypto_section, "gateway_disable_ssl", False, False) + self.gateway_sm_ssl = utilities.get_value( + self.config, crypto_section, "gateway_sm_ssl", False, False) + # the rpc disable ssl or not + self.rpc_disable_ssl = utilities.get_value( + self.config, crypto_section, "rpc_disable_ssl", False, False) + # the rpc use sm-ssl or not + self.rpc_sm_ssl = utilities.get_value( + self.config, crypto_section, "rpc_sm_ssl", False, False) + self.sm_crypto = utilities.get_value( + self.config, crypto_section, "sm_crypto", False, False) + utilities.log_debug("load the crypto config success") + # load the tars config + self.tars_config = TarsConfig(self.config) + # load the agency config + # TODO: check duplicated case + utilities.log_debug("load the agency config") + self.agency_list = {} + agency_list_object = utilities.get_item_value( + self.config, "agency", None, False, "[[agency]]") + for agency_object in agency_list_object: + agency_config = AgencyConfig( + self.tars_config.app_name, agency_object, gateway_must_exists, node_must_exists) + self.agency_list[agency_config.agency_name] = agency_config + utilities.log_debug( + "load the agency config for %s success" % agency_config.agency_name) + utilities.log_debug("load the agency config success") diff --git a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py new file mode 100644 index 00000000..b75f7987 --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py @@ -0,0 +1,207 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import os +from common import utilities +from config.binary_generator import BinaryGenerator +from config.cert_generator import CertGenerator +from config.tars_config_generator import TarsConfigGenerator + + +class PPCGatewayConfigGenerator: + """ + the ppc-gateway-config-generator + """ + + def __init__(self, config, output_dir): + self.config = config + self.output_dir = output_dir + self.with_tars = False + self.binary_name = utilities.ConfigInfo.ppc_gateway_binary_name + self.service_type = utilities.ServiceInfo.gateway_service_type + + def generate_gateway_config(self): + utilities.print_badge("* generate gateway config, app: %s" % + self.config.tars_config.app_name) + # generate the ca + ret = CertGenerator.generate_ca_cert( + self.config.gateway_sm_ssl, self.__generate_ca_cert_path__()) + if ret is False: + utilities.log_error( + "* generate ca-cert config for %s failed" % self.config.tars_config.app_name) + return False + for agency_config in self.config.agency_list.values(): + gateway_config = agency_config.gateway_config + ret = self.__generate_single_gateway_config__(gateway_config) + if ret is False: + return False + utilities.print_badge( + "* generate gateway config success, app: %s" % self.config.tars_config.app_name) + return True + + def __generate_single_gateway_config__(self, gateway_config): + # load the config from tpl_config_path + utilities.log_info("* generate config for ppc-gateway %s.%s" % + (gateway_config.app_name, gateway_config.service_name)) + # store the + # store the config.ini + for ip_str in gateway_config.deploy_ip: + ip_array = ip_str.split(":") + ip = ip_array[0] + node_count = 1 + if len(ip_array) >= 2: + node_count = int(ip_array[1]) + for node_index in range(node_count): + node_name = "node" + str(node_index) + utilities.print_badge("* generate config for ppc-gateway %s.%s.%s" % + (gateway_config.app_name, gateway_config.service_name, node_name)) + config_content = utilities.load_config( + utilities.ConfigInfo.gateway_config_tpl_path) + # load the common config + self.__generate_common_config__(gateway_config, config_content) + # load the gateway config + listen_port = gateway_config.listen_port + node_index + self.__generate_gateway_config_content__( + gateway_config, config_content, listen_port) + # load the cache config + self.__generate_cache_config__(gateway_config, config_content) + # generate the shell scripts for the given ip + ret = TarsConfigGenerator.generate_ip_shell_scripts( + self.__generate_ip_shell_scripts_output_path__(ip), "start_all.sh", "stop_all.sh") + if ret is False: + return False + # generate the start.sh/stop.sh for given service + ret = TarsConfigGenerator.generate_ip_shell_scripts( + self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), "start.sh", + "stop.sh") + if ret is False: + return False + # generate the binary + binary_path = os.path.join( + self.config.tars_config.binary_path, self.binary_name) + dst_binary_path = os.path.join( + self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), + self.binary_name) + ret = BinaryGenerator.generate_binary( + binary_path, dst_binary_path) + if ret is False: + return False + # generate the shell scripts for the node + service_name = gateway_config.service_name + if self.with_tars is False: + service_name = utilities.ConfigInfo.ppc_gateway_binary_name + ret = TarsConfigGenerator.generate_node_shell_scripts( + self.__generate_node_path__(gateway_config, ip, node_name), service_name) + if ret is False: + return False + # generate the ini config + config_output_path = self.__generate_conf_output_path__( + gateway_config, ip, node_name) + ini_config_output_path = os.path.join( + config_output_path, utilities.ConfigInfo.config_ini_file) + ret = utilities.store_config( + config_content, "ini", ini_config_output_path, gateway_config.service_name) + if ret is False: + utilities.log_error("* generate config for ppc-gateway %s.%s, ip: %s failed" % + (gateway_config.app_name, gateway_config.service_name, ip)) + return False + # generate tars config + tars_config_path = os.path.join( + config_output_path, utilities.ConfigInfo.tars_config_file) + tars_listen_port = gateway_config.tars_listen_port + node_index + ret = TarsConfigGenerator.generate_and_store_tars_conf(utilities.ConfigInfo.tars_config_tpl_path, + tars_config_path, + gateway_config.app_name, + gateway_config.service_name, + gateway_config.servant_object_list, ip, + tars_listen_port) + if ret is False: + return False + # generate the node config + ret = CertGenerator.generate_node_cert(self.config.gateway_sm_ssl, self.__generate_ca_cert_path__( + ), self.__generate_conf_output_path__(gateway_config, ip, node_name)) + if ret is False: + utilities.log_error( + "* generate config for ppc-gateway %s.%s failed for generate the node config failed" % + (gateway_config.app_name, gateway_config.service_name)) + return False + utilities.print_badge("* generate config for ppc-gateway %s.%s.%s success" % + (gateway_config.app_name, gateway_config.service_name, node_name)) + utilities.log_info("* generate config for ppc-gateway %s.%s success" % + (gateway_config.app_name, gateway_config.service_name)) + return True + + def __generate_ca_cert_path__(self): + return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + + def __generate_node_path__(self, config, ip, node_name): + return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, + config.service_name, node_name) + + def __generate_conf_output_path__(self, config, ip, node_name): + node_path = self.__generate_node_path__(config, ip, node_name) + return os.path.join(node_path, "conf") + + def __generate_ip_shell_scripts_output_path__(self, ip): + return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) + + def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): + ip_path = self.__generate_ip_shell_scripts_output_path__(ip) + return os.path.join(ip_path, service_name) + + def __generate_common_config__(self, config, config_content): + """ + generate the common config + """ + section = "agency" + # the agency + config_content[section]["id"] = config.agency_name + config_content["gateway"]["holding_msg_minutes"] = str( + config.holding_msg_minutes) + # the peers info + for peer in config.peers: + key = "agency." + peer.agency + config_content[section][key] = str(','.join(peer.endpoints)) + + def __generate_gateway_config_content__(self, config, config_content, listen_port): + """ + generate the gateway config + """ + section = "gateway" + # the listen_ip + config_content[section]["listen_ip"] = config.listen_ip + # the listen port + config_content[section]["listen_port"] = str(listen_port) + # the thread count + config_content[section]["thread_count"] = str(config.thread_count) + # sm ssl + config_content[section]["sm_ssl"] = utilities.convert_bool_to_str( + self.config.gateway_sm_ssl) + # disable ssl + config_content[section]["disable_ssl"] = utilities.convert_bool_to_str( + self.config.gateway_disable_ssl) + + def __generate_cache_config__(self, config, config_content): + """ + generate the cache config + """ + section = "cache" + config_content[section]["type"] = config.cache_type + config_content[section]["proxy"] = str(config.cache_proxy) + config_content[section]["obServer"] = str(config.cache_obServer) + config_content[section]["cluster"] = str(config.cache_cluster) + # the host + config_content[section]["host"] = config.cache_host + # the port + config_content[section]["port"] = str(config.cache_port) + # the password + config_content[section]["password"] = str(config.cache_password) + # the database + config_content[section]["database"] = str(config.cache_database) + # the pool_size + config_content[section]["pool_size"] = str(config.cache_pool_size) + # the connection_timeout + config_content[section]["connection_timeout"] = str( + config.cache_connection_timeout) + # the socket_timeout + config_content[section]["socket_timeout"] = str( + config.cache_socket_timeout) diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py new file mode 100644 index 00000000..b3962b4c --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py @@ -0,0 +1,259 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import os +from common import utilities +from config.binary_generator import BinaryGenerator +from config.cert_generator import CertGenerator +from config.tars_config_generator import TarsConfigGenerator + + +class PPCNodeConfigGenerator: + """ + the ppc-node-config-generator + """ + + def __init__(self, config, output_dir): + self.config = config + self.output_dir = output_dir + self.with_tars = False + self.binary_name = utilities.ConfigInfo.ppc_node_binary_name + self.service_type = utilities.ServiceInfo.node_service_type + + def generate_node_config(self): + utilities.print_badge("* generate_node_config") + # generate the ca cert for rpc + ret = CertGenerator.generate_ca_cert( + self.config.rpc_sm_ssl, self.__generate_ca_cert_path__()) + if ret is False: + utilities.log_error( + "* generate_node_config failed for generate ca error, app: %s" % (self.config.tars_config.app_name)) + return False + for agency_config in self.config.agency_list.values(): + for node_config in agency_config.node_list.values(): + for ip_str in node_config.deploy_ip: + ip_array = ip_str.split(":") + ip = ip_array[0] + node_count = 1 + if len(ip_array) >= 2: + node_count = int(ip_array[1]) + for node_index in range(node_count): + node_name = "node" + str(node_index) + if self.__generate_single_node_config__(node_config, ip, node_name, node_index) is False: + return False + utilities.print_badge("* generate_node_config success") + return True + + def __generate_single_node_config__(self, node_config, ip, node_name, node_index): + utilities.print_badge("* generate node config for %s.%s.%s, ip: %s" % + (node_config.app_name, node_config.service_name, node_name, ip)) + # generate the shell scripts for the given ip + ret = TarsConfigGenerator.generate_ip_shell_scripts( + self.__generate_ip_output_path__(ip), "start_all.sh", "stop_all.sh") + if ret is False: + return False + # generate the start.sh/stop.sh for given service + ret = TarsConfigGenerator.generate_ip_shell_scripts( + self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), "start.sh", "stop.sh") + if ret is False: + return False + # copy the binary + binary_path = os.path.join( + self.config.tars_config.binary_path, self.binary_name) + dst_binary_path = os.path.join( + self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), self.binary_name) + ret = BinaryGenerator.generate_binary(binary_path, dst_binary_path) + if ret is False: + return False + # generate the node config + node_path = self.__generate_node_conf_path__( + node_config, ip, node_name) + private_key_path = self.__generate_node_conf_path__( + node_config, ip, node_name) + tars_output_path = private_key_path + if self.__generate_single_node_inner_config__(utilities.ConfigInfo.node_config_tpl_path, + utilities.ConfigInfo.tars_config_tpl_path, node_path, + private_key_path, tars_output_path, node_config, ip, + node_index) is False: + utilities.log_error("* generate node config for %s.%s, ip: %s failed" % + (node_config.app_name, node_config.service_name, ip)) + return False + # generate the node shell_scipts + service_name = node_config.service_name + if self.with_tars is False: + service_name = utilities.ConfigInfo.ppc_node_binary_name + ret = TarsConfigGenerator.generate_node_shell_scripts( + self.__generate_node_path__(node_config, ip, node_name), service_name) + if ret is False: + return False + # generate the node cert(for rpc) + ret = CertGenerator.generate_node_cert(self.config.rpc_sm_ssl, self.__generate_ca_cert_path__( + ), self.__generate_node_conf_path__(node_config, ip, node_name)) + if ret is False: + utilities.log_error("* generate node config for %s.%s, ip: %s failed for generate rpc cert failed" % + (node_config.app_name, node_config.service_name, ip)) + return False + utilities.print_badge("* generate node config for %s.%s.%s, ip: %s success" % + (node_config.app_name, node_config.service_name, node_name, ip)) + return True + + def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl_path, node_path, private_key_path, + tars_output_path, node_config, ip, node_index): + config_content = utilities.load_config(tpl_config_path) + utilities.log_debug( + "__generate_single_node_config__, load config.ini from %s" % tpl_config_path) + # generate the private key + (ret, private_key) = CertGenerator.generate_private_key(private_key_path) + if ret is False: + return False + # load the common config + self.__generate_common_config__( + config_content, node_config) + # load the rpc config + self.__generate_rpc_config__( + config_content, node_config.rpc_config, node_index) + # TODO: check the configuration + # load the storage config + self.__generate_storage_config__( + config_content, node_config.storage_config) + # load the hdfs_storage_config + self.__generate_hdfs_storage_config__( + config_content, node_config.hdfs_storage_config) + # load the ra2018psi config + self.__generate_ra2018psi_config__( + config_content, node_config.ra2018psi_config) + # load the gateway tars config + self.__generate_tars_gateway_config__( + config_content, node_config.gateway_config) + # store the config + ini_config_output_path = os.path.join( + node_path, utilities.ConfigInfo.config_ini_file) + ret = utilities.store_config( + config_content, "ini", ini_config_output_path, node_config.service_name) + if ret is False: + return False + # load the tars config + self.__generate_tars_config__( + tars_config_tpl_path, tars_output_path, node_config, ip, node_index) + return True + + def __generate_ca_cert_path__(self): + return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + + def __generate_node_path__(self, node_config, ip, node_name): + return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, + node_config.service_name, node_name) + + def __generate_node_conf_path__(self, node_config, ip, node_name): + node_path = self.__generate_node_path__(node_config, ip, node_name) + return os.path.join(node_path, "conf") + + def __generate_ip_output_path__(self, ip): + return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) + + def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): + ip_path = self.__generate_ip_output_path__(ip) + return os.path.join(ip_path, service_name) + + def __generate_common_config__(self, config_content, node_config): + """ + generate the common config + """ + # the agency config + config_content["agency"]["id"] = node_config.agency_name + # the holding_msg_minutes + config_content["tars_gateway"]["holding_msg_minutes"] = str( + node_config.holding_msg_minutes) + # disable ra2018 or not + config_content["agency"]["disable_ra2018"] = utilities.convert_bool_to_str( + node_config.disable_ra2018) + # the crypto config + config_content["crypto"]["sm_crypto"] = utilities.convert_bool_to_str( + self.config.sm_crypto) + + def __generate_rpc_config__(self, config_content, rpc_config, node_index): + """ + generate the rpc config + """ + section_name = "rpc" + # listen_ip + config_content[section_name]["listen_ip"] = rpc_config.listen_ip + # listen_port + rpc_listen_port = rpc_config.listen_port + node_index + config_content[section_name]["listen_port"] = str(rpc_listen_port) + # sm_ssl + config_content[section_name]["sm_ssl"] = utilities.convert_bool_to_str( + self.config.rpc_sm_ssl) + # disable_ssl + config_content[section_name]["disable_ssl"] = utilities.convert_bool_to_str( + self.config.rpc_disable_ssl) + + def __generate_storage_config__(self, config_content, storage_config): + """ + generate the storage config + """ + if storage_config is None: + utilities.log_error("Must set the mysql-storage-config!") + sys.exit(-1) + + section_name = "storage" + config_content[section_name]["host"] = storage_config.host + config_content[section_name]["port"] = str(storage_config.port) + config_content[section_name]["user"] = storage_config.user + config_content[section_name]["password"] = storage_config.password + config_content[section_name]["database"] = storage_config.database + + def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): + if hdfs_storage_config is None: + return + section_name = "hdfs_storage" + config_content[section_name]["user"] = hdfs_storage_config.user + config_content[section_name]["name_node"] = hdfs_storage_config.name_node + config_content[section_name]["name_node_port"] = str( + hdfs_storage_config.name_node_port) + config_content[section_name]["token"] = hdfs_storage_config.token + + def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): + """ + generate the ra2018psi config + """ + section_name = "ra2018psi" + config_content[section_name]["database"] = ra2018psi_config.database + config_content[section_name]["cuckoofilter_capacity"] = str( + ra2018psi_config.cuckoofilter_capacity) + config_content[section_name]["cuckoofilter_tagBits"] = str( + ra2018psi_config.cuckoofilter_tagBits) + config_content[section_name]["cuckoofilter_buckets_num"] = str( + ra2018psi_config.cuckoofilter_buckets_num) + config_content[section_name]["cuckoofilter_max_kick_out_count"] = str( + ra2018psi_config.cuckoofilter_max_kick_out_count) + config_content[section_name]["trash_bucket_size"] = str( + ra2018psi_config.trash_bucket_size) + config_content[section_name]["cuckoofilter_cache_size"] = str( + ra2018psi_config.cuckoofilter_cache_size) + config_content[section_name]["psi_cache_size"] = str( + ra2018psi_config.psi_cache_size) + config_content[section_name]["data_batch_size"] = str( + ra2018psi_config.data_batch_size) + config_content[section_name]["use_hdfs"] = utilities.convert_bool_to_str( + ra2018psi_config.use_hdfs) + + def __generate_tars_gateway_config__(self, config_content, tars_gateway_config): + section_name = "tars_gateway" + config_content[section_name]["name"] = tars_gateway_config.service_name + i = 0 + for endpoint in tars_gateway_config.endpoints: + key = "proxy.%d" % i + config_content[section_name][key] = endpoint + i = i + 1 + + def __generate_tars_config__(self, tars_config_tpl_path, tars_config_dir, node_config, ip, node_index): + """ + generate the tars config + """ + tars_config_path = os.path.join( + tars_config_dir, utilities.ConfigInfo.tars_config_file) + # generate the tars config of the gateway-service + tars_listen_port = node_config.tars_listen_port + node_index + TarsConfigGenerator.generate_and_store_tars_conf(tars_config_tpl_path, tars_config_path, + node_config.app_name, node_config.service_name, + node_config.servant_object_list, ip, tars_listen_port) diff --git a/cpp/tools/ppc-builder/src/config/tars_config_generator.py b/cpp/tools/ppc-builder/src/config/tars_config_generator.py new file mode 100644 index 00000000..1540f486 --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/tars_config_generator.py @@ -0,0 +1,123 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +from common import utilities +import os + + +class TarsConfigGenerator: + """ + generate the tars-config + """ + def __generate_adapter_info__(app_name, server_name, servant_object, tars_listen_ip, tars_listen_port): + adapter_name = "%s.%s.%sAdapter" % ( + app_name, server_name, servant_object) + servant_name = "%s.%s.%s" % (app_name, server_name, servant_object) + result = "<%s>\n" % adapter_name + result = "%s\t\tallow\n" % result + result = "%s\t\tendpoint=tcp -h %s -p %d -t 60000\n" % ( + result, tars_listen_ip, tars_listen_port) + result = "%s\t\tmaxconns=100000\n\t\tprotocol=tars\n\t\tqueuecap=50000\n\t\tqueuetimeout=20000\n" % result + result = "%s\t\tservant=%s\n" % (result, servant_name) + result = "%s\t\tthreads=8\n\t\t\n" % (result, adapter_name) + return result + + def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name): + tars_start_all_path = os.path.join( + script_output_dir, start_shell_script_name) + utilities.mkdir(script_output_dir) + if os.path.exists(tars_start_all_path) is False: + utilities.log_debug( + "* generate shell script, dst: %s" % tars_start_all_path) + # tars_start_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.tars_start_all_tpl_path, tars_start_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (tars_start_all_path, output)) + return False + tars_stop_all_path = os.path.join( + script_output_dir, stop_shell_script_name) + if os.path.exists(tars_stop_all_path) is False: + # tars stop_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.tars_stop_all_tpl_path, tars_stop_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (tars_stop_all_path, output)) + return False + utilities.log_debug( + "* generate_ip_shell_scripts success, output: %s" % script_output_dir) + return True + + def __update_service_name__(file_path, output_path, service_name): + command = "cp %s %s" % (file_path, output_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + return False + updated_config = "" + with open(output_path, 'r', encoding='UTF-8') as file: + updated_config = file.read() + updated_config = updated_config.replace( + "@SERVICE_NAME@", "../" + service_name) + with open(output_path, 'w', encoding='UTF-8') as file: + file.write(updated_config) + return True + + def generate_node_shell_scripts(script_output_dir, service_name): + utilities.log_debug("* generate shell scripts for %s, dst: %s" % + (service_name, script_output_dir)) + utilities.mkdir(script_output_dir) + # the start.sh + output_path = os.path.join(script_output_dir, "start.sh") + ret = TarsConfigGenerator.__update_service_name__( + utilities.ConfigInfo.tars_start_tpl_path, output_path, service_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + # the stop.sh + output_path = os.path.join(script_output_dir, "stop.sh") + ret = TarsConfigGenerator.__update_service_name__( + utilities.ConfigInfo.tars_stop_tpl_path, output_path, service_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + return True + + def generate_and_store_tars_conf(conf_tpl_path, output_path, app_name, server_name, servant_object_list, tars_listen_ip, tars_listen_port): + utilities.log_debug("generate_and_store_tars_conf: %s" % conf_tpl_path) + utilities.mkfiledir(output_path) + # copy tpl to output_path + if os.path.exists(output_path): + utilities.log_error("the tars file %s already exists!") + return False + command = "cp %s %s" % (conf_tpl_path, output_path) + (ret, output) = utilities.execute_command_and_getoutput(command) + if ret is False: + utilities.log_error("copy tpl tars config %s to %s error: %s" % ( + conf_tpl_path, output_path, output)) + return False + # config + module_name = "%s.%s" % (app_name, server_name) + + tars_adapters = "" + for servant_obj in servant_object_list: + adapter_info = TarsConfigGenerator.__generate_adapter_info__( + app_name, server_name, servant_obj, tars_listen_ip, tars_listen_port) + tars_adapters = "%s%s" % (tars_adapters, adapter_info) + utilities.log_debug("tars_adapters: %s" % tars_adapters) + updated_config = "" + with open(output_path, 'r', encoding='UTF-8') as file: + updated_config = file.read() + updated_config = updated_config.replace("@TARS_APP@", app_name) + updated_config = updated_config.replace( + "@TARS_SERVER@", server_name) + updated_config = updated_config.replace( + "@MODULE_NAME@", module_name) + updated_config = updated_config.replace( + "@TARS_ADAPTERS@", tars_adapters) + with open(output_path, 'w', encoding='UTF-8') as file: + file.write(updated_config) diff --git a/cpp/tools/ppc-builder/src/controller/commandline_helper.py b/cpp/tools/ppc-builder/src/controller/commandline_helper.py new file mode 100644 index 00000000..7873925c --- /dev/null +++ b/cpp/tools/ppc-builder/src/controller/commandline_helper.py @@ -0,0 +1,88 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +from common import utilities +from config.ppc_gateway_config_generator import PPCGatewayConfigGenerator +from config.ppc_node_config_generator import PPCNodeConfigGenerator +from config.ppc_deploy_config import PPCDeployConfig +from argparse import RawTextHelpFormatter +import sys +import toml +import os +import argparse + + +def parse_command(): + help_info = "examples:\n * generate node config:\t python3 build_ppc.py -t node\n * generate gateway config:\t python3 build_ppc.py -t gateway\n * generate gateway config:\t python3 build_ppc.py -o genconfig -c config.toml -t gateway -d ppc-generated\n * generate node config:\t python3 build_ppc.py -o genconfig -c config.toml -t node -d ppc-generated" + parser = argparse.ArgumentParser( + prog=sys.argv[0], description=help_info, formatter_class=RawTextHelpFormatter, add_help=True) + + # the command option, now support genconfig/extend + help_info = "[Optional] specify the command: \n* supported command list: %s\n" % ( + ''.join(utilities.CommandInfo.supported_command_list)) + parser.add_argument("-o", '--operation', help=help_info, + required=False, default=utilities.CommandInfo.generate_config) + + # config option + help_info = "[Optional] the config file, default is config.toml\n" + parser.add_argument( + "-c", "--config", help=help_info, default="config.toml") + # the output option + help_info = "[Optional] the output path, default is pp-generated\n" + parser.add_argument( + "-d", "--output", help=help_info, default="ppc-generated") + + # the type option + supported_service_type_str = ', '.join( + utilities.ServiceInfo.supported_service_type) + help_info = "[Required] the service type:\n* now support: %s \n" % ( + supported_service_type_str) + parser.add_argument("-t", "--type", help=help_info, default="") + args = parser.parse_args() + return args + + +def generate_node_config(args, toml_config): + """ + generate the node config + """ + service_type = args.type + # check the type + if service_type not in utilities.ServiceInfo.supported_service_type: + utilities.log_error("The service type must be " + + ', '.join(utilities.ServiceInfo.supported_service_type)) + sys.exit(-1) + if service_type == utilities.ServiceInfo.node_service_type: + utilities.log_debug("generate config for the ppc-node") + config = PPCDeployConfig(toml_config, False, True) + node_generator = PPCNodeConfigGenerator(config, args.output) + ret = node_generator.generate_node_config() + if ret is False: + sys.exit(-1) + if service_type == utilities.ServiceInfo.gateway_service_type: + utilities.log_debug("generate config for the ppc-success") + config = PPCDeployConfig(toml_config, True, False) + gateway_generator = PPCGatewayConfigGenerator(config, args.output) + ret = gateway_generator.generate_gateway_config() + if ret is False: + sys.exit(-1) + + +def execute_command(args): + # check the config path + if os.path.exists(args.config) is False: + utilities.log_error("The config file %s not found!" % args.config) + sys.exit(-1) + # load the toml config + toml_config = toml.load(args.config) + # check the command + command = args.operation + if command not in utilities.CommandInfo.supported_command_list: + utilities.log_error("The command must be " + + ', '.join(utilities.CommandInfo.supported_command_list)) + sys.exit(-1) + if command == utilities.CommandInfo.generate_config: + generate_node_config(args, toml_config) + return + # TODO: implement extend + if command == utilities.CommandInfo.extend: + utilities.log_error("unimplemented command %s" % command) diff --git a/cpp/tools/ppc-builder/src/scripts/gen_cert.sh b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh new file mode 100644 index 00000000..2ee333b6 --- /dev/null +++ b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh @@ -0,0 +1,465 @@ + +cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS" +OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b" + +cert_conf="cert.cnf" +sm_cert_conf='sm_cert.cnf' +sm2_params="sm_sm2.param" +days=36500 +rsa_key_length=2048 + +LOG_WARN() { + local content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + local content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +LOG_FATAL() { + local content=${1} + echo -e "\033[31m[FATAL] ${content}\033[0m" + exit 1 +} + +dir_must_exists() { + if [ ! -d "$1" ]; then + LOG_FATAL "$1 DIR does not exist, please check!" + fi +} + +get_value() { + local var_name=${1} + var_name=var_${var_name//./} + local res=$(eval echo '$'"${var_name}") + echo ${res} +} + +set_value() { + local var_name=${1} + var_name=var_${var_name//./} + local var_value=${2} + eval "${var_name}=${var_value}" +} + +dir_must_not_exists() { + if [ -d "$1" ]; then + LOG_FATAL "$1 DIR already exist, please check!" + fi +} + +file_must_not_exists() { + if [ -f "$1" ]; then + LOG_FATAL "$1 file already exist, please check!" + fi +} + +file_must_exists() { + if [ ! -f "$1" ]; then + LOG_FATAL "$1 file does not exist, please check!" + fi +} + +check_env() { + if [ "$(uname)" == "Darwin" ];then + macOS="macOS" + fi + if [ "$(uname -m)" != "x86_64" ];then + x86_64_arch="false" + fi +} +check_name() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[a-zA-Z0-9._-]+$ ]] || { + LOG_FATAL "$name name [$value] invalid, it should match regex: ^[a-zA-Z0-9._-]+\$" + } +} + +check_and_install_tassl(){ + if [ -f "${OPENSSL_CMD}" ];then + return + fi + # https://en.wikipedia.org/wiki/Uname#Examples + local x86_64_name="x86_64" + local arm_name="aarch64" + local tassl_mid_name="linux" + if [[ -n "${macOS}" ]];then + x86_64_name="x86_64" + arm_name="arm64" + tassl_mid_name="macOS" + fi + + local tassl_post_fix="x86_64" + local platform="$(uname -m)" + if [[ "${platform}" == "${arm_name}" ]];then + tassl_post_fix="aarch64" + elif [[ "${platform}" == "${x86_64_name}" ]];then + tassl_post_fix="x86_64" + else + LOG_FATAL "Unsupported platform ${platform} for ${tassl_mid_name}" + exit 1 + fi + local tassl_package_name="tassl-1.1.1b-${tassl_mid_name}-${tassl_post_fix}" + local tassl_tgz_name="${tassl_package_name}.tar.gz" + local tassl_link_prefix="${cdn_link_header}/FISCO-BCOS/tools/tassl-1.1.1b/${tassl_tgz_name}" + LOG_INFO "Downloading tassl binary from ${tassl_link_prefix}..." + wget --no-check-certificate "${tassl_link_prefix}" + tar zxvf ${tassl_tgz_name} && rm ${tassl_tgz_name} + chmod u+x ${tassl_package_name} + mkdir -p "${HOME}"/.fisco + mv ${tassl_package_name} "${HOME}"/.fisco/tassl-1.1.1b +} + +generate_sm_sm2_param() { + local output=$1 + cat << EOF > ${output} +-----BEGIN EC PARAMETERS----- +BggqgRzPVQGCLQ== +-----END EC PARAMETERS----- + +EOF +} + +generate_sm_cert_conf() { + local output=$1 + cat <"${output}" +oid_section = new_oids + +[ new_oids ] +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = ./demoCA # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem # The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extensions to add to the cert + +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +default_days = 36500 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +policy = policy_match + +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_md = sm3 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +x509_extensions = v3_ca # The extensions to add to the self signed cert + +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = fisco +commonName = Organizational commonName (eg, fisco) +commonName_default = fisco +commonName_max = 64 + +[ usr_cert ] +basicConstraints=CA:FALSE +nsComment = "OpenSSL Generated Certificate" + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature + +[ v3enc_req ] + +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = keyAgreement, keyEncipherment, dataEncipherment + +[ v3_agency_root ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +[ v3_ca ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign + +EOF +} + +generate_cert_conf() { + local output=$1 + cat <"${output}" +[ca] +default_ca=default_ca + +[default_ca] +default_days = 36500 +default_md = sha256 + +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = CN +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default =GuangDong +localityName = Locality Name (eg, city) +localityName_default = ShenZhen +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = WeDPR-PPC +commonName = Organizational commonName (eg, WeDPR-PPC) +commonName_default = WeDPR-PPC +commonName_max = 64 + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v4_req ] +basicConstraints = CA:FALSE + +EOF +} + +gen_non_sm_ca_cert() { + if [ ! -f "${cert_conf}" ]; then + generate_cert_conf "${cert_conf}" + fi + local ca_cert_dir="${1}" + file_must_not_exists "${ca_cert_dir}"/ca.key + file_must_not_exists "${ca_cert_dir}"/ca.crt + file_must_exists "${cert_conf}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + ${OPENSSL_CMD} genrsa -out "${ca_cert_dir}"/ca.key "${rsa_key_length}" + ${OPENSSL_CMD} req -new -x509 -days "${days}" -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=chain" -key "${ca_cert_dir}"/ca.key -config "${cert_conf}" -out "${ca_cert_dir}"/ca.crt 2>/dev/null + if [ ! -f "${ca_cert_dir}/cert.cnf" ];then + mv "${cert_conf}" "${ca_cert_dir}" + fi + LOG_INFO "Generate ca cert successfully!" +} + +gen_sm_ca_cert() { + local ca_cert_dir="${1}" + name=$(basename "$ca_cert_dir") + check_name chain "$name" + + if [ ! -f "${sm_cert_conf}" ]; then + generate_sm_cert_conf ${sm_cert_conf} + fi + + generate_sm_sm2_param "${sm2_params}" + + mkdir -p "$ca_cert_dir" + dir_must_exists "$ca_cert_dir" + + "$OPENSSL_CMD" genpkey -paramfile "${sm2_params}" -out "$ca_cert_dir/sm_ca.key" 2>/dev/null + "$OPENSSL_CMD" req -config sm_cert.cnf -x509 -days "${days}" -subj "/CN=wedpr/O=wedpr/OU=ca" -key "$ca_cert_dir/sm_ca.key" -extensions v3_ca -out "$ca_cert_dir/sm_ca.crt" 2>/dev/null + if [ ! -f "${ca_cert_dir}/${sm_cert_conf}" ];then + cp "${sm_cert_conf}" "${ca_cert_dir}" + fi + if [ ! -f "${ca_cert_dir}/${sm2_params}" ];then + cp "${sm2_params}" "${ca_cert_dir}" + fi +} + + +gen_rsa_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/ca.key" + file_must_exists "$capath/ca.crt" + # check_name node "$node" + + file_must_not_exists "$ndpath"/"${type}".key + file_must_not_exists "$ndpath"/"${type}".crt + + mkdir -p "${ndpath}" + dir_must_exists "${ndpath}" + + ${OPENSSL_CMD} genrsa -out "${ndpath}"/"${type}".key "${rsa_key_length}" 2>/dev/null + ${OPENSSL_CMD} req -new -sha256 -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=agency" -key "$ndpath"/"${type}".key -config "$capath"/cert.cnf -out "$ndpath"/"${type}".csr + ${OPENSSL_CMD} x509 -req -days "${days}" -sha256 -CA "${capath}"/ca.crt -CAkey "$capath"/ca.key -CAcreateserial \ + -in "$ndpath"/"${type}".csr -out "$ndpath"/"${type}".crt -extensions v4_req -extfile "$capath"/cert.cnf 2>/dev/null + + ${OPENSSL_CMD} pkcs8 -topk8 -in "$ndpath"/"${type}".key -out "$ndpath"/pkcs8_node.key -nocrypt + cp "$capath"/ca.crt "$capath"/cert.cnf "$ndpath"/ + + rm -f "$ndpath"/"$type".csr + rm -f "$ndpath"/"$type".key + + mv "$ndpath"/pkcs8_node.key "$ndpath"/"$type".key + + LOG_INFO "Generate ${ndpath} cert successful!" +} + +gen_sm_node_cert_with_ext() { + local capath="$1" + local certpath="$2" + local type="$3" + local extensions="$4" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + file_must_not_exists "$ndpath/sm_${type}.crt" + file_must_not_exists "$ndpath/sm_${type}.key" + + "$OPENSSL_CMD" genpkey -paramfile "$capath/${sm2_params}" -out "$certpath/sm_${type}.key" 2> /dev/null + "$OPENSSL_CMD" req -new -subj "/CN=WeDPR-PPC/O=WeDPR-PPC/OU=${type}" -key "$certpath/sm_${type}.key" -config "$capath/sm_cert.cnf" -out "$certpath/sm_${type}.csr" 2> /dev/null + + "$OPENSSL_CMD" x509 -sm3 -req -CA "$capath/sm_ca.crt" -CAkey "$capath/sm_ca.key" -days "${days}" -CAcreateserial -in "$certpath/sm_${type}.csr" -out "$certpath/sm_${type}.crt" -extfile "$capath/sm_cert.cnf" -extensions "$extensions" 2> /dev/null + + rm -f "$certpath/sm_${type}.csr" +} + +gen_sm_node_cert() { + local capath="${1}" + local ndpath="${2}" + local type="${3}" + + file_must_exists "$capath/sm_ca.key" + file_must_exists "$capath/sm_ca.crt" + + mkdir -p "$ndpath" + dir_must_exists "$ndpath" + local node=$(basename "$ndpath") + check_name node "$node" + + gen_sm_node_cert_with_ext "$capath" "$ndpath" "${type}" v3_req + gen_sm_node_cert_with_ext "$capath" "$ndpath" "en${type}" v3enc_req + #nodeid is pubkey + $OPENSSL_CMD ec -in "$ndpath/sm_${type}.key" -text -noout 2> /dev/null | sed -n '7,11p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | awk '{print substr($0,3);}' | cat > "${ndpath}/sm_${type}.nodeid" + cp "$capath/sm_ca.crt" "$ndpath" +} + +generate_ca_cert() { + local sm_mode="$1" + local ca_cert_path="$2" + LOG_INFO "generate ca cert, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" + mkdir -p "${ca_cert_path}" + if [[ "${sm_mode}" == "false" ]]; then + gen_non_sm_ca_cert "${ca_cert_path}" + else + gen_sm_ca_cert "${ca_cert_path}" + fi + LOG_INFO "generate ca cert successfully, sm: ${sm_mode}, ca_cert_path: ${ca_cert_path}" +} + +generate_node_cert() { + local sm_mode="$1" + local ca_cert_path="${2}" + local node_cert_path="${3}" + + mkdir -p ${node_cert_path} + if [[ "${sm_mode}" == "false" ]]; then + gen_rsa_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + else + gen_sm_node_cert "${ca_cert_path}" "${node_cert_path}" "ssl" 2>&1 + fi +} + +# we use sm_param to generate the private key +generate_private_key_for_psi_server() { + local output_path="${1}" + if [ ! -d "${output_path}" ]; then + mkdir -p ${output_path} + fi + if [ ! -f ${sm2_params} ]; then + generate_sm_sm2_param ${sm2_params} + fi + ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null + $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" + private_key=$(cat $output_path/node.privateKey) + echo ${private_key} +} + +check_env() { + if [ "$(uname)" == "Darwin" ];then + macOS="macOS" + fi + if [ "$(uname -m)" != "x86_64" ];then + x86_64_arch="false" + fi +} + +main() { + check_env + check_and_install_tassl + command="${1}" + if [ "${command}" == "generate_node_cert" ];then + generate_node_cert "${2}" "${3}" "${4}" + fi + if [ "${command}" == "generate_ca_cert" ];then + generate_ca_cert "${2}" "${3}" + fi + if [ "${command}" == "generate_private_key_for_psi_server" ];then + generate_private_key_for_psi_server "${2}" + fi +} +main "$@" \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway new file mode 100644 index 00000000..a7319da2 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway @@ -0,0 +1,58 @@ +[agency] + ; the agency-id of self-party + id = agency0 + ; the agency info + +[cert] + ; directory the certificates located in + cert_path=./conf + +[gateway] + listen_ip=0.0.0.0 + listen_port=40300 + ;thread_count = 4 + ; ssl or sm ssl + sm_ssl=false + ;ssl connection switch, if disable the ssl connection, default: false + disable_ssl = false + ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes + holding_msg_minutes = 30 + ; disable_cache = false +[cache] + ; the cache type, only support redis now + type = 0 + proxy = 127.0.0.1:20002 + obServer = 127.0.0.1:10904 + cluster = REDIS_CLUSTER + user = 1194 + host = 127.0.0.1 + port = 6379 + password = + database = 1 + pool_size = 16 + ; the redis connection timeout, in ms, default is 500ms + connection_timeout = 500 + ; the redis socket timeout, in ms, default is 500ms + socket_timeout = 500 + +[log] + enable=true + log_path=./log + ; info debug trace + level=debug + ; MB + max_log_file_size=200 + ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message + format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + enable_rotate_by_hour=false + log_name_pattern=ppcs-gateway.log + ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log + rotate_name_pattern=log_%Y%m%d.%H%M.log + ; if archive_path is empty, the archive function will be disabled + ; archive_path=./log/ + compress_archive_file=true + ; ; 0: no limit, in MB + ; max_archive_files=10 + ; ; 0: no limit, in MB + ; max_archive_size=0 + ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node new file mode 100644 index 00000000..4989f1d1 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node @@ -0,0 +1,110 @@ +[agency] + ; the agency-id of self-party + id = agency0 + ; the private key path for the psi-server + private_key_path = conf/node.pem + ; disable the ra2018 or not, default enable ra2018 + disable_ra2018 = false + ; the path that allows programs to access + ; data_location = data + ; task_timeout_minutes = 180 + ; thread_count = 8 + +[crypto] + sm_crypto = false + +[rpc] + listen_ip=0.0.0.0 + listen_port=10200 + ; token = ppcs_psi_apikey + thread_count=4 + ; ssl or sm ssl + sm_ssl=false + ; ssl connection switch, if disable the ssl connection, default: true + disable_ssl = true + ; disable_db = true + ; GB, reject task if available memory is less than min_needed_memory + ; min_needed_memory = 5 + +[cert] + ; directory the certificates located in + cert_path=./conf + +[tars_gateway] + ; the gateway service name + name = + ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes + holding_msg_minutes = 30 + ; the gateway service endpoints + +[storage] + host = 127.0.0.1 + ; the mysqlx_port + port = 3306 + user = app + password = + database = + +[hdfs_storage] + ; the hdfs configuration + user = app + name_node = 127.0.0.1 + name_node_port = 9000 + token = + ; enable replace-datanode-on-failure or not + replace-datanode-on-failure = false + ; the connection-timeout, in ms, default is 1000ms + connection-timeout = 1000 + + +[ra2018psi] + ; The database used to store cuckoo-filter + database=ra2018_0 + ; The capacity of the cuckoo filter in MB, the default is 256MB + cuckoofilter_capacity = 1 + ; The length of each element of the cuckoo filter (bits), default 64bits + cuckoofilter_tagBits = 64 + ; The number of buckets for the cuckoo filter, the default is 4 + cuckoofilter_buckets_num = 4 + ; When the cuckoo filter replaces an element, the maximum number of element replacements allowed + cuckoofilter_max_kick_out_count = 20 + ; The trash-bucket-size for the cuckoo-filter + trash_bucket_size = 10000 + ; The cuckoo filter capacity that can be cached in memory in MB + cuckoofilter_cache_size = 256 + ; Cache size in MB + psi_cache_size = 1024 + ; When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + ; data_batch_size = 1000000 + ; use hdfs to store the cuckoo-filter or not + use_hdfs = false + +[ecdh-psi] + ; When reading large files, the line size loaded into memory each time, default is 100w, -1 means load the max-size-of data + ; data_batch_size = 1000000 + +[cm2020-psi] + ; Number of tasks that can be executed concurrently + parallelism = 3 + +[log] + enable=true + log_path=./log + ; info debug trace + level=debug + ; MB + max_log_file_size=200 + ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message + format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + enable_rotate_by_hour=false + log_name_pattern=ppcs-psi4ef.log + ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log + rotate_name_pattern=log_%Y%m%d.%H%M.log + ; if archive_path is empty, the archive function will be disabled + ; archive_path=./log/ + compress_archive_file=true + ; ; 0: no limit, in MB + ; max_archive_files=10 + ; ; 0: no limit, in MB + ; max_archive_size=0 + ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/tars.conf b/cpp/tools/ppc-builder/src/tpl/tars.conf new file mode 100644 index 00000000..f2157319 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/tars.conf @@ -0,0 +1,31 @@ + + + enableset=n + setdivision=NULL + + app=@TARS_APP@ + server=@TARS_SERVER@ + localip=127.0.0.1 + basepath=./conf/ + datapath=./.data/ + logpath=./log/ + logsize=100M + lognum=5 + logLevel=INFO + deactivating-timeout=3000 + activating-timeout=10000 + opencoroutine=0 + coroutinememsize=1G + coroutinestack=128K + closecout=0 + netthread=4 + @TARS_ADAPTERS@ + + + sync-invoke-timeout=3000000 + async-invoke-timeout=5000000 + asyncthread=8 + modulename=@MODULE_NAME@ + + + diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start.sh b/cpp/tools/ppc-builder/src/tpl/tars_start.sh new file mode 100644 index 00000000..a03b6fb4 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/tars_start.sh @@ -0,0 +1,33 @@ +#!/bin/bash +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd ${dirpath} + +service_name='@SERVICE_NAME@' +service=${dirpath}/${service_name} + +pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') +name=$(basename ${dirpath}) + +if [ ! -z ${pid} ];then + echo " ${service}/${name} is running, pid is ${pid}." + exit 0 +else + nohup ${service} --config=conf/tars.conf >>nohup.out 2>&1 & + sleep 1.5 +fi + +try_times=4 +i=0 +while [ $i -lt ${try_times} ] +do + pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') + if [[ ! -z ${pid} ]];then + echo -e "\033[32m ${service}/${name} start successfully pid=${pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start ${service}/${name} \033[0m" +tail -n20 nohup.out + diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh b/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh new file mode 100644 index 00000000..006f53b4 --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh @@ -0,0 +1,14 @@ +#!/bin/bash +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd "${dirpath}" + +dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) +for dir in ${dirs[*]} +do + if [[ -f "${dirpath}/${dir}/start.sh" ]];then + echo "try to start ${dirpath}/${dir}" + bash ${dirpath}/${dir}/start.sh & + fi +done +wait + diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop.sh b/cpp/tools/ppc-builder/src/tpl/tars_stop.sh new file mode 100644 index 00000000..3e8931ce --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/tars_stop.sh @@ -0,0 +1,31 @@ +#!/bin/bash +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd ${dirpath} + +service_name='@SERVICE_NAME@' +service=${dirpath}/${service_name} +name=$(basename ${dirpath}) + +pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') + +if [ -z ${pid} ];then + echo " ${service}/${name} isn't running." + exit 0 +fi + +kill ${pid} > /dev/null + +i=0 +try_times=10 +while [ $i -lt ${try_times} ] +do + sleep 1 + pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') + if [ -z ${pid} ];then + echo -e "\033[32m stop ${service}/${name} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop ${service}/${name}" +exit 1 diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh b/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh new file mode 100644 index 00000000..7a71beca --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh @@ -0,0 +1,14 @@ +#!/bin/bash +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd "${dirpath}" + +dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) +for dir in ${dirs[*]} +do + if [[ -f "${dirpath}/${dir}/stop.sh" ]];then + echo "try to stop ${dirpath}/${dir}" + bash ${dirpath}/${dir}/stop.sh + fi +done +wait + diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json new file mode 100644 index 00000000..71d2a9ae --- /dev/null +++ b/cpp/vcpkg-configuration.json @@ -0,0 +1,16 @@ +{ + "registries": [ + { + "kind": "git", + "repository": "https://github.com/FISCO-BCOS/registry", + "baseline": "a3508ded2bb7f83d95dd3f7406b05b2500a1fdbe", + "packages": [ + "openssl", + "bcos-utilities", + "bcos-boostssl", + "tarscpp", + "libhdfs3" + ] + } + ] +} \ No newline at end of file diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json new file mode 100644 index 00000000..c85989f4 --- /dev/null +++ b/cpp/vcpkg.json @@ -0,0 +1,79 @@ +{ + "name": "fiscobcos", + "version-string": "3.7.3", + "homepage": "https://github.com/WeBankBlockchain/WeDPR-Component", + "description": "FISCO BCOS", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + { + "name": "openssl", + "version>=": "1.1.1-tassl" + }, + "tarscpp", + "libsodium", + { + "name": "bcos-utilities", + "version>=": "1.0.0" + } + , + { + "name": "bcos-boostssl", + "version>=": "3.2.3" + }, + { + "name": "seal", + "version>=": "4.0.0", + "features": ["no-throw-tran"] + }, + { + "name": "kuku", + "version>=": "2.1" + }, + { + "name": "redis-plus-plus", + "version>=": "1.3.6" + }, + { + "name": "mysql-connector-cpp", + "version>=": "8.0.32" + }, + { + "name": "cpu-features", + "version>=": "0.9.0" + }, + "libhdfs3", + "tarscpp", + "tbb", + "libxml2" + ], + "builtin-baseline": "51b14cd4e1230dd51c11ffeff6f7d53c61cc5297", + "overrides": [ + { + "name": "seal", + "version": "4.0.0" + }, + { + "name": "kuku", + "version": "2.1#3" + }, + { + "name": "libhdfs3", + "version": "2024-04-27" + }, + { + "name": "tbb", + "version": "2021.8.0" + }, + { + "name": "tarscpp", + "version": "3.0.3-1#1" + } + ] +} \ No newline at end of file From d95e5f3e42ff9f597bc3300c397e375a41e4045e Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 21 Aug 2024 15:55:44 +0800 Subject: [PATCH 002/120] add ppml and ppc-model-gateway (#2) --- python/__init__.py | 0 python/ppc_common/__init__.py | 0 python/ppc_common/application-sample.yml | 6 + python/ppc_common/application.yml | 6 + python/ppc_common/config.py | 18 + python/ppc_common/db_models/__init__.py | 6 + .../ppc_common/db_models/file_object_meta.py | 12 + python/ppc_common/db_models/file_path.py | 10 + .../ppc_common/db_models/job_unit_record.py | 14 + python/ppc_common/deps_services/__init__.py | 0 .../ppc_common/deps_services/file_object.py | 85 ++ .../ppc_common/deps_services/hdfs_storage.py | 97 ++ .../ppc_common/deps_services/mysql_storage.py | 65 ++ .../deps_services/serialize_type.py | 7 + .../deps_services/sharding_file_object.py | 429 ++++++++ .../deps_services/sql_storage_api.py | 57 ++ .../ppc_common/deps_services/storage_api.py | 53 + .../deps_services/storage_loader.py | 14 + .../deps_services/tests/mysql_storage_test.py | 104 ++ .../tests/sharding_file_object_test.py | 79 ++ .../ppc_common/ppc_async_executor/__init__.py | 0 .../ppc_async_executor/async_executor.py | 35 + .../async_subprocess_executor.py | 60 ++ .../async_thread_executor.py | 75 ++ .../ppc_async_executor/test/__init__.py | 0 .../test/async_executor_unittest.py | 109 +++ .../thread_event_manager.py | 34 + python/ppc_common/ppc_config/__init__.py | 0 .../ppc_config/file_chunk_config.py | 27 + .../ppc_config/sql_storage_config_loader.py | 37 + python/ppc_common/ppc_crypto/__init__.py | 0 python/ppc_common/ppc_crypto/crypto_utils.py | 130 +++ python/ppc_common/ppc_crypto/ihc_cipher.py | 93 ++ python/ppc_common/ppc_crypto/ihc_codec.py | 29 + .../ppc_common/ppc_crypto/paillier_cipher.py | 45 + .../ppc_common/ppc_crypto/paillier_codec.py | 34 + python/ppc_common/ppc_crypto/phe_cipher.py | 25 + python/ppc_common/ppc_crypto/phe_factory.py | 25 + python/ppc_common/ppc_crypto/test/__init__.py | 0 .../ppc_crypto/test/phe_unittest.py | 105 ++ python/ppc_common/ppc_dataset/__init__.py | 0 .../ppc_common/ppc_dataset/dataset_helper.py | 60 ++ .../ppc_dataset/dataset_helper_factory.py | 61 ++ python/ppc_common/ppc_initialize/__init__.py | 0 .../dataset_handler_initialize.py | 30 + .../tests/dataset_initializer_test.py | 126 +++ python/ppc_common/ppc_ml/__init__.py | 0 python/ppc_common/ppc_ml/feature/__init__.py | 0 .../ppc_ml/feature/feature_importance.py | 246 +++++ .../feature/tests/feature_importance_test.py | 79 ++ .../ppc_common/ppc_ml/model/algorithm_info.py | 25 + python/ppc_common/ppc_mock/__init__.py | 0 python/ppc_common/ppc_mock/mock_objects.py | 59 ++ python/ppc_common/ppc_protos/__init__.py | 0 .../ppc_protos/generated/__init__.py | 0 .../ppc_protos/generated/ppc_model_pb2.py | 51 + .../generated/ppc_model_pb2_grpc.py | 67 ++ .../ppc_protos/generated/ppc_pb2.py | 64 ++ python/ppc_common/ppc_protos/ppc.proto | 160 +++ python/ppc_common/ppc_protos/ppc_model.proto | 83 ++ python/ppc_common/ppc_utils/__init__.py | 1 + .../ppc_common/ppc_utils/anonymous_search.py | 420 ++++++++ python/ppc_common/ppc_utils/audit_utils.py | 56 ++ python/ppc_common/ppc_utils/cem_utils.py | 160 +++ python/ppc_common/ppc_utils/common_func.py | 28 + python/ppc_common/ppc_utils/exception.py | 146 +++ python/ppc_common/ppc_utils/http_utils.py | 102 ++ python/ppc_common/ppc_utils/path.py | 51 + python/ppc_common/ppc_utils/permission.py | 79 ++ python/ppc_common/ppc_utils/plot_utils.py | 230 +++++ .../ppc_utils/ppc_model_config_parser.py | 474 +++++++++ .../ppc_model_config_parser_proxy.py | 491 ++++++++++ .../ppc_utils/tests/thread_safe_list_test.py | 48 + .../ppc_common/ppc_utils/tests/utils_test.py | 40 + .../ppc_common/ppc_utils/thread_safe_list.py | 56 ++ python/ppc_common/ppc_utils/utils.py | 914 ++++++++++++++++++ python/ppc_model/__init__.py | 0 python/ppc_model/common/__init__.py | 0 python/ppc_model/common/base_context.py | 83 ++ python/ppc_model/common/context.py | 14 + python/ppc_model/common/global_context.py | 13 + python/ppc_model/common/initializer.py | 108 +++ python/ppc_model/common/mock/__init__.py | 0 .../ppc_model/common/mock/rpc_client_mock.py | 31 + python/ppc_model/common/model_result.py | 212 ++++ python/ppc_model/common/model_setting.py | 90 ++ python/ppc_model/common/protocol.py | 88 ++ python/ppc_model/conf/application-sample.yml | 43 + python/ppc_model/conf/logging.conf | 40 + python/ppc_model/datasets/__init__.py | 0 .../datasets/data_reduction/__init__.py | 0 .../data_reduction/feature_selection.py | 30 + .../datasets/data_reduction/sampling.py | 86 ++ .../datasets/data_reduction/test/__init__.py | 0 .../test/test_data_reduction.py | 61 ++ python/ppc_model/datasets/dataset.py | 232 +++++ .../datasets/feature_binning/__init__.py | 0 .../feature_binning/feature_binning.py | 131 +++ .../datasets/feature_binning/test/__init__.py | 0 .../test/test_feature_binning.py | 102 ++ python/ppc_model/datasets/test/__init__.py | 0 .../ppc_model/datasets/test/test_dataset.py | 213 ++++ .../ppc_model/feature_engineering/__init__.py | 0 .../feature_engineering_context.py | 48 + .../feature_engineering_engine.py | 41 + .../feature_engineering/test/__init__.py | 0 .../test/feature_engineering_unittest.py | 153 +++ .../feature_engineering/vertical/__init__.py | 0 .../vertical/active_party.py | 202 ++++ .../vertical/passive_party.py | 177 ++++ .../feature_engineering/vertical/utils.py | 76 ++ python/ppc_model/interface/__init__.py | 0 python/ppc_model/interface/model_base.py | 33 + python/ppc_model/interface/rpc_client.py | 8 + python/ppc_model/interface/task_engine.py | 9 + python/ppc_model/metrics/__init__.py | 0 python/ppc_model/metrics/evaluation.py | 276 ++++++ python/ppc_model/metrics/loss.py | 32 + python/ppc_model/metrics/model_plot.py | 181 ++++ python/ppc_model/metrics/test/__init__.py | 0 python/ppc_model/metrics/test/test_metrics.py | 165 ++++ .../model_result/task_result_handler.py | 388 ++++++++ python/ppc_model/network/__init__.py | 0 python/ppc_model/network/grpc/__init__.py | 0 python/ppc_model/network/grpc/grpc_client.py | 82 ++ python/ppc_model/network/grpc/grpc_server.py | 17 + python/ppc_model/network/http/__init__.py | 0 python/ppc_model/network/http/body_schema.py | 19 + .../network/http/model_controller.py | 95 ++ python/ppc_model/network/http/restx.py | 35 + python/ppc_model/network/stub.py | 218 +++++ python/ppc_model/network/test/__init__.py | 0 .../ppc_model/network/test/stub_unittest.py | 83 ++ python/ppc_model/ppc_model_app.py | 112 +++ python/ppc_model/preprocessing/__init__.py | 0 .../local_processing_party.py | 100 ++ .../local_processing/preprocessing.py | 660 +++++++++++++ .../local_processing/psi_select.py | 95 ++ .../local_processing/standard_type_enum.py | 7 + .../preprocessing/preprocessing_engine.py | 19 + .../preprocessing/processing_context.py | 27 + .../preprocessing/tests/test_preprocessing.py | 672 +++++++++++++ python/ppc_model/secure_lgbm/__init__.py | 0 .../ppc_model/secure_lgbm/monitor/__init__.py | 0 .../ppc_model/secure_lgbm/monitor/callback.py | 83 ++ python/ppc_model/secure_lgbm/monitor/core.py | 144 +++ .../secure_lgbm/monitor/early_stopping.py | 122 +++ .../secure_lgbm/monitor/evaluation_monitor.py | 123 +++ .../secure_lgbm/monitor/feature/__init__.py | 0 .../feature/feature_evaluation_info.py | 90 ++ .../monitor/feature/test/__init__.py | 0 .../test/feature_evalution_info_test.py | 73 ++ .../monitor/train_callback_unittest.py | 105 ++ .../secure_lgbm/secure_lgbm_context.py | 254 +++++ .../secure_lgbm_prediction_engine.py | 38 + .../secure_lgbm_training_engine.py | 40 + python/ppc_model/secure_lgbm/test/__init__.py | 0 .../secure_lgbm/test/test_cipher_packing.py | 61 ++ .../secure_lgbm/test/test_pack_gh.py | 43 + .../secure_lgbm/test/test_save_load_model.py | 92 ++ .../test/test_secure_lgbm_context.py | 77 ++ .../test_secure_lgbm_performance_training.py | 172 ++++ .../test/test_secure_lgbm_training.py | 178 ++++ .../secure_lgbm/vertical/__init__.py | 4 + .../secure_lgbm/vertical/active_party.py | 461 +++++++++ .../ppc_model/secure_lgbm/vertical/booster.py | 345 +++++++ .../secure_lgbm/vertical/passive_party.py | 271 ++++++ python/ppc_model/task/__init__.py | 0 python/ppc_model/task/task_manager.py | 183 ++++ python/ppc_model/task/test/__init__.py | 0 .../task/test/task_manager_unittest.py | 155 +++ python/ppc_model/tools/start.sh | 39 + python/ppc_model/tools/stop.sh | 19 + python/ppc_model_gateway/__init__.py | 0 python/ppc_model_gateway/clients/__init__.py | 0 .../clients/client_manager.py | 57 ++ .../conf/application-sample.yml | 19 + python/ppc_model_gateway/conf/logging.conf | 40 + python/ppc_model_gateway/config.py | 60 ++ .../ppc_model_gateway/endpoints/__init__.py | 0 .../endpoints/node_to_partner.py | 36 + .../endpoints/partner_to_node.py | 34 + .../endpoints/response_builder.py | 8 + .../ppc_model_gateway_app.py | 90 ++ python/ppc_model_gateway/test/__init__.py | 0 python/ppc_model_gateway/test/client.py | 41 + python/ppc_model_gateway/test/server.py | 36 + python/ppc_model_gateway/tools/gen_cert.sh | 70 ++ python/ppc_model_gateway/tools/start.sh | 35 + python/ppc_model_gateway/tools/stop.sh | 11 + python/requirements.txt | 65 ++ python/tools/fake_id_data.py | 187 ++++ python/tools/fake_ml_train_data.py | 170 ++++ python/tools/requirements.txt | 1 + 194 files changed, 16071 insertions(+) create mode 100644 python/__init__.py create mode 100644 python/ppc_common/__init__.py create mode 100644 python/ppc_common/application-sample.yml create mode 100644 python/ppc_common/application.yml create mode 100644 python/ppc_common/config.py create mode 100644 python/ppc_common/db_models/__init__.py create mode 100644 python/ppc_common/db_models/file_object_meta.py create mode 100644 python/ppc_common/db_models/file_path.py create mode 100644 python/ppc_common/db_models/job_unit_record.py create mode 100644 python/ppc_common/deps_services/__init__.py create mode 100644 python/ppc_common/deps_services/file_object.py create mode 100644 python/ppc_common/deps_services/hdfs_storage.py create mode 100644 python/ppc_common/deps_services/mysql_storage.py create mode 100644 python/ppc_common/deps_services/serialize_type.py create mode 100644 python/ppc_common/deps_services/sharding_file_object.py create mode 100644 python/ppc_common/deps_services/sql_storage_api.py create mode 100644 python/ppc_common/deps_services/storage_api.py create mode 100644 python/ppc_common/deps_services/storage_loader.py create mode 100644 python/ppc_common/deps_services/tests/mysql_storage_test.py create mode 100644 python/ppc_common/deps_services/tests/sharding_file_object_test.py create mode 100644 python/ppc_common/ppc_async_executor/__init__.py create mode 100644 python/ppc_common/ppc_async_executor/async_executor.py create mode 100644 python/ppc_common/ppc_async_executor/async_subprocess_executor.py create mode 100644 python/ppc_common/ppc_async_executor/async_thread_executor.py create mode 100644 python/ppc_common/ppc_async_executor/test/__init__.py create mode 100644 python/ppc_common/ppc_async_executor/test/async_executor_unittest.py create mode 100644 python/ppc_common/ppc_async_executor/thread_event_manager.py create mode 100644 python/ppc_common/ppc_config/__init__.py create mode 100644 python/ppc_common/ppc_config/file_chunk_config.py create mode 100644 python/ppc_common/ppc_config/sql_storage_config_loader.py create mode 100644 python/ppc_common/ppc_crypto/__init__.py create mode 100644 python/ppc_common/ppc_crypto/crypto_utils.py create mode 100644 python/ppc_common/ppc_crypto/ihc_cipher.py create mode 100644 python/ppc_common/ppc_crypto/ihc_codec.py create mode 100644 python/ppc_common/ppc_crypto/paillier_cipher.py create mode 100644 python/ppc_common/ppc_crypto/paillier_codec.py create mode 100644 python/ppc_common/ppc_crypto/phe_cipher.py create mode 100644 python/ppc_common/ppc_crypto/phe_factory.py create mode 100644 python/ppc_common/ppc_crypto/test/__init__.py create mode 100644 python/ppc_common/ppc_crypto/test/phe_unittest.py create mode 100644 python/ppc_common/ppc_dataset/__init__.py create mode 100644 python/ppc_common/ppc_dataset/dataset_helper.py create mode 100644 python/ppc_common/ppc_dataset/dataset_helper_factory.py create mode 100644 python/ppc_common/ppc_initialize/__init__.py create mode 100644 python/ppc_common/ppc_initialize/dataset_handler_initialize.py create mode 100644 python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py create mode 100644 python/ppc_common/ppc_ml/__init__.py create mode 100644 python/ppc_common/ppc_ml/feature/__init__.py create mode 100644 python/ppc_common/ppc_ml/feature/feature_importance.py create mode 100644 python/ppc_common/ppc_ml/feature/tests/feature_importance_test.py create mode 100644 python/ppc_common/ppc_ml/model/algorithm_info.py create mode 100644 python/ppc_common/ppc_mock/__init__.py create mode 100644 python/ppc_common/ppc_mock/mock_objects.py create mode 100644 python/ppc_common/ppc_protos/__init__.py create mode 100644 python/ppc_common/ppc_protos/generated/__init__.py create mode 100644 python/ppc_common/ppc_protos/generated/ppc_model_pb2.py create mode 100644 python/ppc_common/ppc_protos/generated/ppc_model_pb2_grpc.py create mode 100644 python/ppc_common/ppc_protos/generated/ppc_pb2.py create mode 100644 python/ppc_common/ppc_protos/ppc.proto create mode 100644 python/ppc_common/ppc_protos/ppc_model.proto create mode 100644 python/ppc_common/ppc_utils/__init__.py create mode 100644 python/ppc_common/ppc_utils/anonymous_search.py create mode 100644 python/ppc_common/ppc_utils/audit_utils.py create mode 100644 python/ppc_common/ppc_utils/cem_utils.py create mode 100644 python/ppc_common/ppc_utils/common_func.py create mode 100644 python/ppc_common/ppc_utils/exception.py create mode 100644 python/ppc_common/ppc_utils/http_utils.py create mode 100644 python/ppc_common/ppc_utils/path.py create mode 100644 python/ppc_common/ppc_utils/permission.py create mode 100644 python/ppc_common/ppc_utils/plot_utils.py create mode 100644 python/ppc_common/ppc_utils/ppc_model_config_parser.py create mode 100644 python/ppc_common/ppc_utils/ppc_model_config_parser_proxy.py create mode 100644 python/ppc_common/ppc_utils/tests/thread_safe_list_test.py create mode 100644 python/ppc_common/ppc_utils/tests/utils_test.py create mode 100644 python/ppc_common/ppc_utils/thread_safe_list.py create mode 100644 python/ppc_common/ppc_utils/utils.py create mode 100644 python/ppc_model/__init__.py create mode 100644 python/ppc_model/common/__init__.py create mode 100644 python/ppc_model/common/base_context.py create mode 100644 python/ppc_model/common/context.py create mode 100644 python/ppc_model/common/global_context.py create mode 100644 python/ppc_model/common/initializer.py create mode 100644 python/ppc_model/common/mock/__init__.py create mode 100644 python/ppc_model/common/mock/rpc_client_mock.py create mode 100644 python/ppc_model/common/model_result.py create mode 100644 python/ppc_model/common/model_setting.py create mode 100644 python/ppc_model/common/protocol.py create mode 100644 python/ppc_model/conf/application-sample.yml create mode 100644 python/ppc_model/conf/logging.conf create mode 100644 python/ppc_model/datasets/__init__.py create mode 100644 python/ppc_model/datasets/data_reduction/__init__.py create mode 100644 python/ppc_model/datasets/data_reduction/feature_selection.py create mode 100644 python/ppc_model/datasets/data_reduction/sampling.py create mode 100644 python/ppc_model/datasets/data_reduction/test/__init__.py create mode 100644 python/ppc_model/datasets/data_reduction/test/test_data_reduction.py create mode 100644 python/ppc_model/datasets/dataset.py create mode 100644 python/ppc_model/datasets/feature_binning/__init__.py create mode 100644 python/ppc_model/datasets/feature_binning/feature_binning.py create mode 100644 python/ppc_model/datasets/feature_binning/test/__init__.py create mode 100644 python/ppc_model/datasets/feature_binning/test/test_feature_binning.py create mode 100644 python/ppc_model/datasets/test/__init__.py create mode 100644 python/ppc_model/datasets/test/test_dataset.py create mode 100644 python/ppc_model/feature_engineering/__init__.py create mode 100644 python/ppc_model/feature_engineering/feature_engineering_context.py create mode 100644 python/ppc_model/feature_engineering/feature_engineering_engine.py create mode 100644 python/ppc_model/feature_engineering/test/__init__.py create mode 100644 python/ppc_model/feature_engineering/test/feature_engineering_unittest.py create mode 100644 python/ppc_model/feature_engineering/vertical/__init__.py create mode 100644 python/ppc_model/feature_engineering/vertical/active_party.py create mode 100644 python/ppc_model/feature_engineering/vertical/passive_party.py create mode 100644 python/ppc_model/feature_engineering/vertical/utils.py create mode 100644 python/ppc_model/interface/__init__.py create mode 100644 python/ppc_model/interface/model_base.py create mode 100644 python/ppc_model/interface/rpc_client.py create mode 100644 python/ppc_model/interface/task_engine.py create mode 100644 python/ppc_model/metrics/__init__.py create mode 100644 python/ppc_model/metrics/evaluation.py create mode 100644 python/ppc_model/metrics/loss.py create mode 100644 python/ppc_model/metrics/model_plot.py create mode 100644 python/ppc_model/metrics/test/__init__.py create mode 100644 python/ppc_model/metrics/test/test_metrics.py create mode 100644 python/ppc_model/model_result/task_result_handler.py create mode 100644 python/ppc_model/network/__init__.py create mode 100644 python/ppc_model/network/grpc/__init__.py create mode 100644 python/ppc_model/network/grpc/grpc_client.py create mode 100644 python/ppc_model/network/grpc/grpc_server.py create mode 100644 python/ppc_model/network/http/__init__.py create mode 100644 python/ppc_model/network/http/body_schema.py create mode 100644 python/ppc_model/network/http/model_controller.py create mode 100644 python/ppc_model/network/http/restx.py create mode 100644 python/ppc_model/network/stub.py create mode 100644 python/ppc_model/network/test/__init__.py create mode 100644 python/ppc_model/network/test/stub_unittest.py create mode 100644 python/ppc_model/ppc_model_app.py create mode 100644 python/ppc_model/preprocessing/__init__.py create mode 100644 python/ppc_model/preprocessing/local_processing/local_processing_party.py create mode 100644 python/ppc_model/preprocessing/local_processing/preprocessing.py create mode 100644 python/ppc_model/preprocessing/local_processing/psi_select.py create mode 100644 python/ppc_model/preprocessing/local_processing/standard_type_enum.py create mode 100644 python/ppc_model/preprocessing/preprocessing_engine.py create mode 100644 python/ppc_model/preprocessing/processing_context.py create mode 100644 python/ppc_model/preprocessing/tests/test_preprocessing.py create mode 100644 python/ppc_model/secure_lgbm/__init__.py create mode 100644 python/ppc_model/secure_lgbm/monitor/__init__.py create mode 100644 python/ppc_model/secure_lgbm/monitor/callback.py create mode 100644 python/ppc_model/secure_lgbm/monitor/core.py create mode 100644 python/ppc_model/secure_lgbm/monitor/early_stopping.py create mode 100644 python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py create mode 100644 python/ppc_model/secure_lgbm/monitor/feature/__init__.py create mode 100644 python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py create mode 100644 python/ppc_model/secure_lgbm/monitor/feature/test/__init__.py create mode 100644 python/ppc_model/secure_lgbm/monitor/feature/test/feature_evalution_info_test.py create mode 100644 python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py create mode 100644 python/ppc_model/secure_lgbm/secure_lgbm_context.py create mode 100644 python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py create mode 100644 python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py create mode 100644 python/ppc_model/secure_lgbm/test/__init__.py create mode 100644 python/ppc_model/secure_lgbm/test/test_cipher_packing.py create mode 100644 python/ppc_model/secure_lgbm/test/test_pack_gh.py create mode 100644 python/ppc_model/secure_lgbm/test/test_save_load_model.py create mode 100644 python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py create mode 100644 python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py create mode 100644 python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py create mode 100644 python/ppc_model/secure_lgbm/vertical/__init__.py create mode 100644 python/ppc_model/secure_lgbm/vertical/active_party.py create mode 100644 python/ppc_model/secure_lgbm/vertical/booster.py create mode 100644 python/ppc_model/secure_lgbm/vertical/passive_party.py create mode 100644 python/ppc_model/task/__init__.py create mode 100644 python/ppc_model/task/task_manager.py create mode 100644 python/ppc_model/task/test/__init__.py create mode 100644 python/ppc_model/task/test/task_manager_unittest.py create mode 100644 python/ppc_model/tools/start.sh create mode 100644 python/ppc_model/tools/stop.sh create mode 100644 python/ppc_model_gateway/__init__.py create mode 100644 python/ppc_model_gateway/clients/__init__.py create mode 100644 python/ppc_model_gateway/clients/client_manager.py create mode 100644 python/ppc_model_gateway/conf/application-sample.yml create mode 100644 python/ppc_model_gateway/conf/logging.conf create mode 100644 python/ppc_model_gateway/config.py create mode 100644 python/ppc_model_gateway/endpoints/__init__.py create mode 100644 python/ppc_model_gateway/endpoints/node_to_partner.py create mode 100644 python/ppc_model_gateway/endpoints/partner_to_node.py create mode 100644 python/ppc_model_gateway/endpoints/response_builder.py create mode 100644 python/ppc_model_gateway/ppc_model_gateway_app.py create mode 100644 python/ppc_model_gateway/test/__init__.py create mode 100644 python/ppc_model_gateway/test/client.py create mode 100644 python/ppc_model_gateway/test/server.py create mode 100644 python/ppc_model_gateway/tools/gen_cert.sh create mode 100644 python/ppc_model_gateway/tools/start.sh create mode 100644 python/ppc_model_gateway/tools/stop.sh create mode 100644 python/requirements.txt create mode 100644 python/tools/fake_id_data.py create mode 100644 python/tools/fake_ml_train_data.py create mode 100644 python/tools/requirements.txt diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/__init__.py b/python/ppc_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/application-sample.yml b/python/ppc_common/application-sample.yml new file mode 100644 index 00000000..b30c0146 --- /dev/null +++ b/python/ppc_common/application-sample.yml @@ -0,0 +1,6 @@ +# mysql or dm +DB_TYPE: "mysql" +SQLALCHEMY_DATABASE_URI: "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8" +# SQLALCHEMY_DATABASE_URI: "dm+dmPython://ppcv16:ppc12345678@127.0.0.1:5236" + +MPC_BIT_LENGTH: [@IDC_PPCS_COMMON_MPC_BIT] diff --git a/python/ppc_common/application.yml b/python/ppc_common/application.yml new file mode 100644 index 00000000..0ada951d --- /dev/null +++ b/python/ppc_common/application.yml @@ -0,0 +1,6 @@ +# mysql or dm +DB_TYPE: "mysql" +SQLALCHEMY_DATABASE_URI: "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8" +# SQLALCHEMY_DATABASE_URI: "dm+dmPython://ppcv16:ppc12345678@127.0.0.1:5236" + +MPC_BIT_LENGTH: 1000 diff --git a/python/ppc_common/config.py b/python/ppc_common/config.py new file mode 100644 index 00000000..e8e565ae --- /dev/null +++ b/python/ppc_common/config.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import os +import yaml + + +dirName, _ = os.path.split(os.path.abspath(__file__)) +config_path = '{}/application.yml'.format(dirName) + +CONFIG_DATA = {} + + +def read_config(): + global CONFIG_DATA + with open(config_path, 'rb') as f: + CONFIG_DATA = yaml.safe_load(f.read()) + + +read_config() diff --git a/python/ppc_common/db_models/__init__.py b/python/ppc_common/db_models/__init__.py new file mode 100644 index 00000000..591b4182 --- /dev/null +++ b/python/ppc_common/db_models/__init__.py @@ -0,0 +1,6 @@ +from flask_sqlalchemy import SQLAlchemy + + +db = SQLAlchemy() + +# __all__ = ['computation_provider','data_provider','job_computation_queue','job_data_queue', 'job_result'] diff --git a/python/ppc_common/db_models/file_object_meta.py b/python/ppc_common/db_models/file_object_meta.py new file mode 100644 index 00000000..ef48ef9a --- /dev/null +++ b/python/ppc_common/db_models/file_object_meta.py @@ -0,0 +1,12 @@ +from ppc_common.db_models import db +from sqlalchemy import text + + +class FileObjectMeta(db.Model): + __tablename__ = 't_file_object' + file_path = db.Column(db.String(255), primary_key=True) + file_count = db.Column(db.Integer) + create_time = db.Column(db.TIMESTAMP( + True), nullable=False, server_default=text('NOW()')) + last_update_time = db.Column(db.TIMESTAMP(True), nullable=False, server_default=text( + 'CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP')) diff --git a/python/ppc_common/db_models/file_path.py b/python/ppc_common/db_models/file_path.py new file mode 100644 index 00000000..4c626750 --- /dev/null +++ b/python/ppc_common/db_models/file_path.py @@ -0,0 +1,10 @@ +from ppc_common.db_models import db + + +class FilePathRecord(db.Model): + __tablename__ = 't_file_path' + path = db.Column(db.String(255), primary_key=True) + storage_type = db.Column(db.String(255)) + file_id = db.Column(db.String(255)) + file_hash = db.Column(db.String(255)) + create_time = db.Column(db.BigInteger) diff --git a/python/ppc_common/db_models/job_unit_record.py b/python/ppc_common/db_models/job_unit_record.py new file mode 100644 index 00000000..f953ecbb --- /dev/null +++ b/python/ppc_common/db_models/job_unit_record.py @@ -0,0 +1,14 @@ + +from ppc_common.db_models import db + + +class JobUnitRecord(db.Model): + __tablename__ = 't_job_unit' + unit_id = db.Column(db.String(100), primary_key=True) + job_id = db.Column(db.String(255), index=True) + type = db.Column(db.String(255)) + status = db.Column(db.String(255), index=True) + upstream_units = db.Column(db.Text) + inputs_statement = db.Column(db.Text) + outputs = db.Column(db.Text) + update_time = db.Column(db.BigInteger) diff --git a/python/ppc_common/deps_services/__init__.py b/python/ppc_common/deps_services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/deps_services/file_object.py b/python/ppc_common/deps_services/file_object.py new file mode 100644 index 00000000..1e69fef0 --- /dev/null +++ b/python/ppc_common/deps_services/file_object.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from enum import Enum +from abc import ABC, abstractmethod + + +class SplitMode(Enum): + NONE = 0 # not split + SIZE = 1 # split by size + LINES = 2 # split by lines + + +class FileObject(ABC): + @abstractmethod + def split(self, split_mode, granularity): + """ + split large file into many small files with given granularity + """ + pass + + @abstractmethod + def download(self, enforce_flush=False): + """ + download the files + """ + pass + + @abstractmethod + def upload(self, split_mode, granularity): + """ + upload the files + """ + pass + + @abstractmethod + def delete(self): + """ + delete the files + """ + pass + + @abstractmethod + def existed(self) -> bool: + """ + check the file object exist or not + """ + pass + + @abstractmethod + def rename(self, storage_path: str): + """ + rename the file object + """ + pass + + @abstractmethod + def hit_local_cache(self): + """hit the local cache or not + """ + pass + + @abstractmethod + def get_local_path(self): + """get the local path + """ + pass + + @abstractmethod + def get_remote_path(self): + """get the remote path + """ + pass + + @abstractmethod + def get_data(self): + """get data + """ + pass + + @abstractmethod + def save_data(self, data, split_mode, granularity): + pass + + @abstractmethod + def update_data(self, updated_data, split_mode, granularity): + pass diff --git a/python/ppc_common/deps_services/hdfs_storage.py b/python/ppc_common/deps_services/hdfs_storage.py new file mode 100644 index 00000000..370ed2d3 --- /dev/null +++ b/python/ppc_common/deps_services/hdfs_storage.py @@ -0,0 +1,97 @@ +import os +from typing import AnyStr + +from hdfs.client import InsecureClient +from ppc_common.ppc_utils import common_func +from ppc_common.deps_services.storage_api import StorageApi, StorageType + +from ppc_common.ppc_utils import utils + + +class HdfsStorage(StorageApi): + + DEFAULT_HDFS_USER = "ppc" + DEFAULT_HDFS_USER_PATH = "/user/" + + def __init__(self, endpoint, hdfs_user, hdfs_home=None): + + # self.client = Client('http://127.0.0.1:9870') + self.endpoint = endpoint + self._user = common_func.get_config_value( + "HDFS_USER", HdfsStorage.DEFAULT_HDFS_USER, hdfs_user, False) + self._hdfs_storage_path = hdfs_home + if hdfs_home is None: + self._hdfs_storage_path = os.path.join( + HdfsStorage.DEFAULT_HDFS_USER_PATH, self._user) + + self.client = InsecureClient(endpoint, user=self._user) + # print(self.client.list('/')) + # print(self.client.list('/user/root/')) + + def get_home_path(self): + return self._hdfs_storage_path + + def storage_type(self): + return StorageType.HDFS + + def download_file(self, hdfs_path, local_file_path, enable_cache=False): + # hit the cache + if enable_cache is True and utils.file_exists(local_file_path): + return + if utils.file_exists(local_file_path): + utils.delete_file(local_file_path) + local_path = os.path.dirname(local_file_path) + if len(local_path) > 0 and not os.path.exists(local_path): + os.makedirs(local_path) + self.client.download(os.path.join(self._hdfs_storage_path, + hdfs_path), local_file_path) + return + + def upload_file(self, local_file_path, hdfs_path): + self.make_file_path(hdfs_path) + self.client.upload(os.path.join(self._hdfs_storage_path, hdfs_path), + local_file_path, overwrite=True) + return + + def make_file_path(self, hdfs_path): + hdfs_dir = os.path.dirname(hdfs_path) + if self.client.status(os.path.join(self._hdfs_storage_path, hdfs_dir), strict=False) is None: + self.client.makedirs(os.path.join( + self._hdfs_storage_path, hdfs_dir)) + return + + def delete_file(self, hdfs_path): + self.client.delete(os.path.join( + self._hdfs_storage_path, hdfs_path), recursive=True) + return + + def save_data(self, data: AnyStr, hdfs_path): + self.make_file_path(hdfs_path) + self.client.write(os.path.join(self._hdfs_storage_path, + hdfs_path), data, overwrite=True) + return + + def get_data(self, hdfs_path) -> AnyStr: + with self.client.read(os.path.join(self._hdfs_storage_path, hdfs_path)) as reader: + content = reader.read() + return content + + def mkdir(self, hdfs_dir): + self.client.makedirs(hdfs_dir) + + def file_existed(self, hdfs_path): + if self.client.status(os.path.join(self._hdfs_storage_path, hdfs_path), strict=False) is None: + return False + return True + + def file_rename(self, old_hdfs_path, hdfs_path): + old_path = os.path.join(self._hdfs_storage_path, old_hdfs_path) + new_path = os.path.join(self._hdfs_storage_path, hdfs_path) + # return for the file not exists + if not self.file_existed(old_path): + return + parent_path = os.path.dirname(new_path) + if len(parent_path) > 0 and not self.file_existed(parent_path): + self.mkdir(parent_path) + self.client.rename(old_path, new_path) + return diff --git a/python/ppc_common/deps_services/mysql_storage.py b/python/ppc_common/deps_services/mysql_storage.py new file mode 100644 index 00000000..ada66a53 --- /dev/null +++ b/python/ppc_common/deps_services/mysql_storage.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from ppc_common.deps_services.sql_storage_api import SQLStorageAPI +from sqlalchemy.orm import sessionmaker, scoped_session +from sqlalchemy import create_engine +from sqlalchemy import delete +from sqlalchemy import text +from contextlib import contextmanager + + +class MySQLStorage(SQLStorageAPI): + def __init__(self, storage_config): + self._engine_url = storage_config.engine_url + self._storage_config = storage_config + connect_args = {} + if storage_config.db_name is not None: + connect_args = {'schema': storage_config.db_name} + self._mysql_engine = create_engine(self._engine_url, pool_recycle=self._storage_config.pool_recycle, + pool_size=self._storage_config.pool_size, max_overflow=self._storage_config.max_overflow, + pool_timeout=self._storage_config.pool_timeout, connect_args=connect_args) + self._session_factory = sessionmaker(bind=self._mysql_engine) + # Note: scoped_session is threadLocal + self._session = scoped_session(self._session_factory) + + @contextmanager + def _get_session(self): + session = self._session() + try: + yield session + session.commit() + except Exception: + session.rollback() + self._session.remove() + raise + finally: + session.close() + + def query(self, object, condition): + """ + query according to the condition + """ + with self._get_session() as session: + return session.query(object).filter(condition) + + def merge(self, record): + """merge the given record to db + Args: + record (Any): the record should been inserted + """ + with self._get_session() as session: + session.merge(record) + + def execute(self, sql: str): + text_sql = text(sql) + with self._get_session() as session: + session.execute(text_sql) + + def delete(self, object, condition): + """delete according to condition + Args: + object (Any): the object + condition (Any): the condition + """ + stmt = delete(object).where(condition) + with self._get_session() as session: + session.execute(stmt) diff --git a/python/ppc_common/deps_services/serialize_type.py b/python/ppc_common/deps_services/serialize_type.py new file mode 100644 index 00000000..31b84f6f --- /dev/null +++ b/python/ppc_common/deps_services/serialize_type.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +from enum import Enum + + +class SerializeType(Enum): + CSV = 'csv' + JSON = 'gain' diff --git a/python/ppc_common/deps_services/sharding_file_object.py b/python/ppc_common/deps_services/sharding_file_object.py new file mode 100644 index 00000000..d89df5bd --- /dev/null +++ b/python/ppc_common/deps_services/sharding_file_object.py @@ -0,0 +1,429 @@ +# -*- coding: utf-8 -*- +from ppc_common.deps_services.file_object import FileObject, SplitMode +from ppc_common.db_models.file_object_meta import FileObjectMeta +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +import shutil +# from memory_profiler import profile +import os +import time +import uuid +import copy + + +class FileMeta: + SUB_FILE_OBJECT_DIR = "object" + + def __init__(self, local_path, remote_path, file_count, logger): + self.logger = logger + self._local_path = local_path + self._remote_path = remote_path + if self._local_path is None: + self._local_path = os.path.basename( + self.remote_path) + "." + str(uuid.uuid4()) + logger.info(f"generate random local path: {self._local_path}") + self._local_file_dir = os.path.dirname(self._local_path) + self._local_file_name = os.path.basename(self._local_path) + if self._remote_path is None: + self._remote_path = local_path + if file_count is not None: + self._file_count = file_count + + @property + def local_path(self): + return self._local_path + + @local_path.setter + def local_path(self, local_path): + self._local_path = local_path + + @property + def remote_path(self): + return self._remote_path + + @remote_path.setter + def remote_path(self, remote_path): + self._remote_path = remote_path + + @property + def file_count(self): + return self._file_count + + @file_count.setter + def file_count(self, file_count): + self._file_count = file_count + + def get_local_sub_file_name(self, file_index): + return os.path.join(self.get_local_sub_files_dir(), str(file_index)) + + def get_local_sub_files_dir(self): + return os.path.join( + self._local_file_dir, FileMeta.SUB_FILE_OBJECT_DIR, self._local_file_name) + + def mk_local_sub_files_dir(self): + local_sub_files_dir = self.get_local_sub_files_dir() + if os.path.exists(local_sub_files_dir): + return + os.makedirs(local_sub_files_dir) + + def remove_local_cache(self): + local_sub_files_dir = self.get_local_sub_files_dir() + if os.path.exists(local_sub_files_dir) is False: + return + shutil.rmtree(local_sub_files_dir) + + def get_remote_sub_file_name(self, file_index): + return os.path.join(self._remote_path, str(file_index)) + + +class ShardingFileObject(FileObject): + def __init__(self, local_path, remote_path, remote_storage_client, sql_storage, logger): + self.logger = logger + self._remote_storage_client = remote_storage_client + self._sql_storage = sql_storage + self._file_meta = FileMeta(local_path, remote_path, None, logger) + self.logger.info( + f"create ShardingFileObject, storage type: {self._remote_storage_client.storage_type()},local path: {self._file_meta.local_path}, remote path: {self._file_meta.remote_path}") + + @property + def file_meta(self): + return self._file_meta + + def split(self, split_mode, granularity): + """ + split large file many small files + """ + if split_mode is SplitMode.NONE or granularity is None: + return None + local_file_size = os.stat(self._file_meta.local_path).st_size + if local_file_size < granularity: + self.logger.info( + f"upload small files directly without split: {self._file_meta.local_path}") + return None + if split_mode is SplitMode.SIZE: + file_list = self._split_by_size(granularity) + if split_mode is SplitMode.LINES: + file_list = self._split_by_lines(granularity) + self._file_meta.file_count = len(file_list) + return file_list + + # @profile + def _split_by_size(self, granularity): + file_index = 0 + start_t = time.time() + file_list = [] + with open(self._file_meta.local_path, "rb") as fp: + while True: + start = time.time() + fp.seek(file_index * granularity) + data = fp.read(granularity) + if not data: + break + self._file_meta.mk_local_sub_files_dir() + sub_file_path = self._file_meta.get_local_sub_file_name( + file_index) + with open(sub_file_path, "wb") as wfp: + wfp.write(data) + file_list.append(sub_file_path) + self.logger.info( + f"split file by size, file: {self._file_meta.local_path}, sub_file: {sub_file_path}, time cost: {time.time() - start} seconds") + file_index += 1 + self.logger.info( + f"split file by size, file: {self._file_meta.local_path}, split granularity: {granularity}, sub file count: {file_index}, time cost: {time.time() - start_t} seconds") + return (file_list) + + # @profile + def _split_by_lines(self, granularity): + file_index = 0 + self._file_meta.mk_local_sub_files_dir() + start = time.time() + file_list = [] + with open(self._file_meta.local_path, "rb") as fp: + while True: + start_t = time.time() + lines = fp.readlines(granularity) + if not lines: + break + file_name = self._file_meta.get_local_sub_file_name(file_index) + with open(file_name, "wb") as wfp: + wfp.writelines(lines) + file_list.append(file_name) + self.logger.debug( + f"split file by lines, file: {self._file_meta.local_path}, sub file path: {file_name}, timecost: {time.time() - start_t} seconds") + file_index += 1 + self.logger.info( + f"split file by lines, file: {self._file_meta.local_path}, split granularity: {granularity}, sub file count: {file_index}, timecost: {time.time() - start} seconds") + return (file_list) + + def _local_cache_miss(self): + for file_index in range(0, self._file_meta.file_count): + sub_file_path = self._file_meta.get_local_sub_file_name(file_index) + if not os.path.exists(sub_file_path): + return True + return False + + def _check_uploaded_files(self): + if self._local_cache_miss(): + error_msg = f"check upload file failed, {self._file_meta.local_path} => {self._file_meta.remote_path}" + raise PpcException( + PpcErrorCode.FILE_OBJECT_UPLOAD_CHECK_FAILED.get_code(), error_msg) + + def upload(self, split_mode, granularity): + """split and upload the file + """ + if self.split(split_mode, granularity) is not None: + self._upload_chunks() + self.logger.info( + f"Upload success, remove local file cache: {self._file_meta.get_local_sub_files_dir()}") + self._file_meta.remove_local_cache() + return + # upload directly + start = time.time() + self.logger.info( + f"Upload: {self._file_meta.local_path}=>{self._file_meta.remote_path}") + self._remote_storage_client.upload_file( + self._file_meta.local_path, self._file_meta.remote_path) + self.logger.info( + f"Upload success: {self._file_meta.local_path}=>{self._file_meta.remote_path}, timecost: {time.time() - start}s") + + def _upload_chunks(self): + """ + upload the files + """ + start = time.time() + self._check_uploaded_files() + for file_index in range(0, self._file_meta.file_count): + start_t = time.time() + local_file_path = self._file_meta.get_local_sub_file_name( + file_index) + remote_file_path = self._file_meta.get_remote_sub_file_name( + file_index) + self.logger.info(f"upload: {local_file_path}=>{remote_file_path}") + self._remote_storage_client.upload_file( + local_file_path, remote_file_path) + self.logger.info( + f"upload: {local_file_path}=>{remote_file_path} success, timecost: {time.time() - start_t} seconds") + self.logger.info( + f"upload: {self._file_meta.local_path}=>{self._file_meta.remote_path} success, timecost: {time.time() - start} seconds, begin to store the meta information") + start = time.time() + record = FileObjectMeta( + file_path=self._file_meta.remote_path, file_count=self._file_meta.file_count) + self._sql_storage.merge(record) + self.logger.info( + f"store meta for {self._file_meta.remote_path} success, timecost: {time.time() - start} seconds") + + def _fetch_file_meta(self): + file_meta_info = self._sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path) + # the file not exists + if file_meta_info is None or file_meta_info.count() == 0: + return False + self.logger.info( + f"fetch file meta information: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path}, file count: {file_meta_info.first().file_count}") + self._file_meta.file_count = file_meta_info.first().file_count + return True + + def download(self, enforce_flush=False): + """ + download the files + """ + if enforce_flush is not True and self.hit_local_cache(): + return + ret = self._fetch_file_meta() + start = time.time() + # the remote file not exists + if ret is False: + # no sharding case + if self._remote_storage_client.file_existed(self._file_meta.remote_path): + self.logger.info( + f"Download file: {self._file_meta.remote_path}=>{self._file_meta.local_path}") + self._remote_storage_client.download_file( + self._file_meta.remote_path, self._file_meta.local_path) + self.logger.info( + f"Download file success: {self._file_meta.remote_path}=>{self._file_meta.local_path}, timecost: {time.time() - start}") + return + error_msg = f"Download file from {self._remote_storage_client.storage_type()}: {self._file_meta.remote_path} failed for the file not exists!" + self.logger.error(error_msg) + raise PpcException( + PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg) + # remove the local file + if os.path.exists(self._file_meta.local_path): + self.logger.info( + f"Download: remove the existed local file {self._file_meta.local_path}") + os.remove(self._file_meta.local_path) + # download from the remote storage client + start = time.time() + # merge the file + offset = 0 + try: + with open(self._file_meta.local_path, "wb") as fp: + for file_index in range(0, self._file_meta.file_count): + start_t = time.time() + fp.seek(offset) + remote_file_path = self._file_meta.get_remote_sub_file_name( + file_index) + local_file_path = self._file_meta.get_local_sub_file_name( + file_index) + self._remote_storage_client.download_file( + remote_file_path, local_file_path) + with open(local_file_path, "rb") as f: + fp.write(f.read()) + offset += os.stat(local_file_path).st_size + self.logger.info( + f"Download: {self._remote_storage_client.storage_type()}:{remote_file_path}=>{local_file_path}, timecost: {time.time() - start_t} seconds") + # remove the local cache + self._file_meta.remove_local_cache() + self.logger.info( + f"Download: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds") + except Exception as e: + self.logger.warn( + f"Download: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path} failed, {e}") + self._remove_local_file() + raise e + + def _remove_local_file(self): + if not os.path.exists(self._file_meta.local_path): + return + os.remove(self._file_meta.local_path) + + def delete(self): + """ + delete the files + """ + if self._fetch_file_meta() is False: + if self._remote_storage_client.file_existed(self._file_meta.remote_path): + self._remote_storage_client.delete_file( + self._file_meta.remote_path) + self.logger.info( + f"Delete file {self._file_meta.remote_path} success") + return + self.logger.info( + f"Delete nothing for file {self._file_meta.remote_path} not exists") + return + start = time.time() + for file_index in range(0, self._file_meta.file_count): + start_t = time.time() + remote_file_path = self._file_meta.get_remote_sub_file_name( + file_index) + self._remote_storage_client.delete_file(remote_file_path) + self.logger.info( + f"Delete: {self._remote_storage_client.storage_type()}:{remote_file_path}, timecost: {time.time() - start_t} seconds") + self._delete_remote_dir() + # delete the record + self._sql_storage.delete( + FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path) + self.logger.info( + f"Delete: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds") + + def _delete_remote_dir(self): + if self._remote_storage_client.file_existed(self._file_meta.remote_path): + self._remote_storage_client.delete_file( + self._file_meta.remote_path) + + def existed(self) -> bool: + """ + check the file object exist or not + """ + if self._fetch_file_meta() is True: + return True + return self._remote_storage_client.file_existed(self._file_meta.remote_path) + + def hit_local_cache(self): + """hit the local cache or not + """ + if not self.existed(): + error_msg = f"The file object: local:{self._file_meta.local_path}, remote: {self._file_meta.remote_path} not exists!" + self.logger.info(f"find local cache failed: {error_msg}") + raise PpcException( + PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg) + return os.path.exists(self._file_meta.local_path) + + def rename(self, storage_path: str): + """ + rename the file object + """ + ret = self._fetch_file_meta() + start = time.time() + # the remote file not exists + if ret is False: + # no sharding case + if self._remote_storage_client.file_existed(self._file_meta.remote_path): + self.logger.info( + f"Rename file: {self._file_meta.remote_path}=>{storage_path}") + self._remote_storage_client.file_rename( + self._file_meta.remote_path, storage_path) + self.logger.info( + f"Rename file success: {self._file_meta.remote_path}=>{storage_path}, timecost: {time.time() - start} seconds") + return + error_msg = f"Rename file {self._remote_storage_client.storage_type()} => {self._file_meta.remote_path} failed for the file not exists!" + self.logger.error(error_msg) + raise PpcException( + PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg) + # rename from the remote storage client + start = time.time() + new_file_meta = copy.deepcopy(self._file_meta) + new_file_meta.remote_path = storage_path + for file_index in range(0, self._file_meta.file_count): + start_t = time.time() + remote_file_path = self._file_meta.get_remote_sub_file_name( + file_index) + new_file_path = new_file_meta.get_remote_sub_file_name( + file_index) + self._remote_storage_client.file_rename( + remote_file_path, new_file_path) + self.logger.info( + f"Rename: {self._remote_storage_client.storage_type()}:{remote_file_path}=>{new_file_path}, timecost: {time.time() - start_t} seconds") + # delete the old record + self._delete_remote_dir() + self._sql_storage.delete( + FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path) + self._file_meta.remote_path = storage_path + # update the meta + self.logger.info( + f"Rename: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{storage_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds") + start = time.time() + record = FileObjectMeta(file_path=storage_path, + file_count=self._file_meta.file_count) + self._sql_storage.merge(record) + self.logger.info( + f"Rename: store meta for {self._file_meta.remote_path} => {storage_path} success, timecost: {time.time() - start} seconds") + + def get_data(self): + self.download() + with open(self._file_meta.local_path, 'r') as file: + file_content = file.read() + file_bytes = file_content.encode('utf-8') + os.remove(self._file_meta.local_path) + return file_bytes + + def save_data(self, data, split_mode, granularity): + with open(self._file_meta.local_path, 'wb') as file: + file.write(data) + self.upload(split_mode, granularity) + os.remove(self._file_meta.local_path) + return + + def update_data(self, updated_data, split_mode, granularity): + start = time.time() + # try to remove the remote file + if self.existed(): + self.logger.info( + f"UpdateData: remove existed remote file: {self._file_meta.remote_path}") + self.delete() + # update the data + self.logger.info( + f"UpdateData: update the remote file: {self._file_meta.remote_path}") + self.save_data(data=updated_data if type(updated_data) is bytes else bytes(updated_data, "utf-8"), + split_mode=split_mode, granularity=granularity) + self.logger.info( + f"UpdateData success: update the remote file: {self._file_meta.remote_path}, timecost: {time.time() - start}") + return len(updated_data) + + def get_local_path(self): + """get the local path + """ + return self._file_meta.local_path + + def get_remote_path(self): + """get the remote path + """ + return self._file_meta.remote_path diff --git a/python/ppc_common/deps_services/sql_storage_api.py b/python/ppc_common/deps_services/sql_storage_api.py new file mode 100644 index 00000000..7f240f05 --- /dev/null +++ b/python/ppc_common/deps_services/sql_storage_api.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +from abc import ABC, abstractmethod +from ppc_common.ppc_utils import common_func + + +class SQLStorageConfig(ABC): + """ + configuration for sql storage + """ + DEFAULT_RECYCLE = 7200 + DEFAULT_POOL_SIZE = 16 + DEFAULT_MAX_OVERFLOW = 8 + DEFAULT_POOL_TIMEOUT = 30 + + def __init__(self, url, pool_recycle=None, pool_size=None, max_overflow=None, pool_timeout=None, db_type="mysql", db_name=None): + self.pool_recycle = common_func.get_config_value("pool_recycle", + SQLStorageConfig.DEFAULT_RECYCLE, pool_recycle, False) + self.pool_size = common_func.get_config_value("pool_size", + SQLStorageConfig.DEFAULT_POOL_SIZE, pool_size, False) + self.max_overflow = common_func.get_config_value("max_overflow", + SQLStorageConfig.DEFAULT_MAX_OVERFLOW, max_overflow, False) + self.pool_timeout = common_func.get_config_value("pool_timeout", + SQLStorageConfig.DEFAULT_POOL_TIMEOUT, pool_timeout, False) + self.db_type = db_type + self.engine_url = url + self.db_name = db_name + + +class SQLStorageAPI: + @abstractmethod + def query(self, object, condition): + """ + query the result + """ + pass + + @abstractmethod + def merge(self, record): + """insert the record into db + + Args: + record (Any): the record need to be inserted + """ + pass + + @abstractmethod + def execute(self, sql: str): + pass + + @abstractmethod + def delete(self, object, condition): + """delete according to condition + Args: + object (Any): the object + condition (Any): the condition + """ + pass diff --git a/python/ppc_common/deps_services/storage_api.py b/python/ppc_common/deps_services/storage_api.py new file mode 100644 index 00000000..ffc622de --- /dev/null +++ b/python/ppc_common/deps_services/storage_api.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import AnyStr + + +class StorageType(Enum): + HDFS = 'HDFS' + + +class StorageApi(ABC): + @abstractmethod + def download_file(self, storage_path: str, local_file_path: str, enable_cache=False): + pass + + @abstractmethod + def upload_file(self, local_file_path: str, storage_path: str): + pass + + @abstractmethod + def make_file_path(self, storage_path: str): + pass + + @abstractmethod + def delete_file(self, storage_path: str): + pass + + @abstractmethod + def save_data(self, data: AnyStr, storage_path: str): + pass + + @abstractmethod + def get_data(self, storage_path: str) -> AnyStr: + pass + + @abstractmethod + def mkdir(self, storage_path: str): + pass + + @abstractmethod + def file_existed(self, storage_path: str) -> bool: + pass + + @abstractmethod + def file_rename(self, old_storage_path: str, storage_path: str): + pass + + @abstractmethod + def storage_type(self): + pass + + @abstractmethod + def get_home_path(self): + return "" diff --git a/python/ppc_common/deps_services/storage_loader.py b/python/ppc_common/deps_services/storage_loader.py new file mode 100644 index 00000000..ba64fdac --- /dev/null +++ b/python/ppc_common/deps_services/storage_loader.py @@ -0,0 +1,14 @@ +from ppc_common.deps_services.storage_api import StorageType +from ppc_common.deps_services.hdfs_storage import HdfsStorage +from ppc_common.ppc_utils import common_func + + +def load(config: dict, logger): + if config['STORAGE_TYPE'] == StorageType.HDFS.value: + hdfs_user = common_func.get_config_value( + 'HDFS_USER', None, config, False) + hdfs_home = common_func.get_config_value( + "HDFS_HOME", None, config, False) + return HdfsStorage(config['HDFS_ENDPOINT'], hdfs_user, hdfs_home) + else: + raise Exception('unsupported storage type') diff --git a/python/ppc_common/deps_services/tests/mysql_storage_test.py b/python/ppc_common/deps_services/tests/mysql_storage_test.py new file mode 100644 index 00000000..b0854d7a --- /dev/null +++ b/python/ppc_common/deps_services/tests/mysql_storage_test.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- + +import unittest +from ppc_common.deps_services.mysql_storage import MySQLStorage +from ppc_common.deps_services.sql_storage_api import SQLStorageConfig +from ppc_common.db_models.file_object_meta import FileObjectMeta +import threading + + +class MySQLStorageWrapper: + def __init__(self): + # use the default config + self.engine_url = "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8mb4" + self.sql_storage_config = SQLStorageConfig(url=self.engine_url) + self.sql_storage = MySQLStorage(self.sql_storage_config) + + def single_thread_test(self, thread_name, path, ut_obj): + print(f"# begin test for thread: {thread_name}, path: {path}") + record_num = 100 + # insert records + file_count_set = set() + for i in range(0, record_num): + tmp_path = path + "_" + str(i) + file_object_meta = FileObjectMeta(file_path=tmp_path, file_count=i) + self.sql_storage.merge(file_object_meta) + file_count_set.add(i) + # query records + result_list = self.sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path.startswith(path)) + ut_obj.assertEqual(result_list.count(), record_num) + # check the file_count + for item in result_list: + ut_obj.assertTrue(item.file_count in file_count_set) + # check the path + expected_path = path + "_" + str(item.file_count) + ut_obj.assertEqual(item.file_path, expected_path) + file_count_set.remove(item.file_count) + ut_obj.assertEqual(len(file_count_set), 0) + # update the file_count + delta = 100 + for i in range(0, record_num): + tmp_path = path + "_" + str(i) + file_count = i + delta + file_object_meta = FileObjectMeta( + file_path=tmp_path, file_count=file_count) + self.sql_storage.merge(file_object_meta) + file_count_set.add(file_count) + # query and check + result_list = self.sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path.startswith(path)) + ut_obj.assertEqual(result_list.count(), record_num) + for item in result_list: + ut_obj.assertTrue(item.file_count in file_count_set) + # check the path + expected_path = path + "_" + str(item.file_count - delta) + ut_obj.assertEqual(item.file_path, expected_path) + file_count_set.remove(item.file_count) + ut_obj.assertEqual(len(file_count_set), 0) + # delete test + tmp_path = path + "_0" + self.sql_storage.delete( + FileObjectMeta, FileObjectMeta.file_path == tmp_path) + result_list = self.sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path == tmp_path) + ut_obj.assertEqual(result_list.count(), 0) + result_list = self.sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path.startswith(path)) + ut_obj.assertEqual(result_list.count(), record_num - 1) + # delete all data throw raw sql + self.sql_storage.execute( + f"delete from t_file_object where file_path like '{path}%'") + result_list = self.sql_storage.query( + FileObjectMeta, FileObjectMeta.file_path.startswith(path)) + ut_obj.assertEqual(result_list.count(), 0) + print(f"# test for thread: {thread_name}, path: {path} success") + + +class TestMySQLStorage(unittest.TestCase): + def test_single_thread(self): + wrapper = MySQLStorageWrapper() + path = "a/b/c/test_single_thread.csv" + wrapper.single_thread_test(path, "single_thread", self) + + def test_multi_thread(self): + loops = 5 + for j in range(loops): + thread_list = [] + thread_num = 20 + path = "a/b/c/test_multi_thread.csv" + wrapper = MySQLStorageWrapper() + for i in range(thread_num): + thread_name = "job_" + str(i) + tmp_path = "thread_" + str(i) + "_" + path + t = threading.Thread(target=wrapper.single_thread_test, name=thread_name, args=( + thread_name, tmp_path, self,)) + thread_list.append(t) + for t in thread_list: + t.start() + for t in thread_list: + t.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/deps_services/tests/sharding_file_object_test.py b/python/ppc_common/deps_services/tests/sharding_file_object_test.py new file mode 100644 index 00000000..60094585 --- /dev/null +++ b/python/ppc_common/deps_services/tests/sharding_file_object_test.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +import unittest +from ppc_common.deps_services.sharding_file_object import ShardingFileObject +from ppc_common.deps_services.file_object import SplitMode +from ppc_common.deps_services.mysql_storage import MySQLStorage +from ppc_common.deps_services.sql_storage_api import SQLStorageConfig +from ppc_common.deps_services.hdfs_storage import HdfsStorage +import sys +import logging +import pandas as pd + + +class ShardingFileObjectTestWrapper: + def __init__(self, ut_obj, local_path, remote_path): + self.ut_obj = ut_obj + self.engine_url = "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8mb4" + self.sql_storage_config = SQLStorageConfig(url=self.engine_url) + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + self.logger = logging.getLogger(__name__) + self.sql_storage = MySQLStorage(self.sql_storage_config) + remote_storage_url = "http://127.0.0.1:9870" + remote_storage_client = HdfsStorage(remote_storage_url, "chenyujie") + self.sharding_file_object = ShardingFileObject( + local_path, remote_path, remote_storage_client, self.sql_storage, self.logger) + + def upload(self, split_mode, granularity, expected_lines): + file_list = self.sharding_file_object.upload(split_mode, granularity) + if split_mode is SplitMode.LINES: + self._check_lines(file_list, expected_lines) + + def _check_lines(self, file_list, expected_lines): + lines = 0 + # check the lines + i = 0 + columns_info = None + df = None + for file in file_list: + # can be parsed by read_csv + if i == 0: + df = pd.read_csv(file, header=0) + columns_info = df.columns + else: + df = pd.read_csv(file, header=None) + df.columns = columns_info + lines += len(df) + i += 1 + self.ut_obj.assertEqual(lines, expected_lines) + + +class ShardingFileObjectTest(unittest.TestCase): + + def test_split_by_lines(self): + local_path = "bak/train_test.csv" + remote_path = "train_test" + sharding_object_wrapper = ShardingFileObjectTestWrapper( + self, local_path, remote_path) + split_mode = SplitMode.LINES + # 100w + granularity = 20 * 1024 * 1024 + sharding_object_wrapper.upload(split_mode, granularity, 500000) + sharding_object_wrapper.sharding_file_object.download() + + +""" + def test_split_by_size(self): + local_path = "bak/train_test.csv" + remote_path = "train_test" + sharding_object_wrapper = ShardingFileObjectTestWrapper( + self, local_path, remote_path) + split_mode = SplitMode.SIZE + # 20M + granularity = 20 * 1024 * 1024 + sharding_object_wrapper.sharding_file_object.split( + split_mode, granularity) +""" + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/ppc_async_executor/__init__.py b/python/ppc_common/ppc_async_executor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_async_executor/async_executor.py b/python/ppc_common/ppc_async_executor/async_executor.py new file mode 100644 index 00000000..50b62b7c --- /dev/null +++ b/python/ppc_common/ppc_async_executor/async_executor.py @@ -0,0 +1,35 @@ +from abc import ABC +from typing import Callable + + +class AsyncExecutor(ABC): + def execute(self, + task_id: str, + target: Callable, + on_target_finish: Callable[[str, bool, Exception], None], + args=()): + """ + 启动一个新进程/线程执行指定的目标函数。 + :param task_id: 任务id + :param target: 目标函数 + :param on_target_finish: 目标函数函数退出后的回调 + :param args: 函数的参数元组 + """ + pass + + def kill(self, task_id: str) -> bool: + """ + 强制终止目标函数。 + :param task_id: 任务id + :return 是否成功 + """ + pass + + def kill_all(self): + """ + 强制终止所有目标函数。 + """ + pass + + def __del__(self): + self.kill_all() diff --git a/python/ppc_common/ppc_async_executor/async_subprocess_executor.py b/python/ppc_common/ppc_async_executor/async_subprocess_executor.py new file mode 100644 index 00000000..87083ac8 --- /dev/null +++ b/python/ppc_common/ppc_async_executor/async_subprocess_executor.py @@ -0,0 +1,60 @@ +import multiprocessing +import threading +import time +from typing import Callable + +from ppc_common.ppc_async_executor.async_executor import AsyncExecutor + + +class AsyncSubprocessExecutor(AsyncExecutor): + def __init__(self, logger): + self.logger = logger + self.processes = {} + self.lock = threading.Lock() + self._cleanup_thread = threading.Thread(target=self._loop_cleanup) + self._cleanup_thread.daemon = True + self._cleanup_thread.start() + + def execute(self, task_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], + args=()): + process = multiprocessing.Process(target=target, args=args) + process.start() + with self.lock: + self.processes[task_id] = process + + def kill(self, task_id: str): + with self.lock: + if task_id not in self.processes: + return False + else: + process = self.processes[task_id] + + process.terminate() + self.logger.info(f"Task {task_id} has been terminated!") + return True + + def kill_all(self): + with self.lock: + keys = self.processes.keys() + + for task_id in keys: + self.kill(task_id) + + def _loop_cleanup(self): + while True: + self._cleanup_finished_processes() + time.sleep(3) + + def _cleanup_finished_processes(self): + with self.lock: + finished_processes = [ + (task_id, proc) for task_id, proc in self.processes.items() if not proc.is_alive()] + + for task_id, process in finished_processes: + with self.lock: + process.join() # 确保进程资源释放 + del self.processes[task_id] + self.logger.info(f"Cleanup finished task process {task_id}") + + def __del__(self): + self.kill_all() diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py new file mode 100644 index 00000000..0bc65b91 --- /dev/null +++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py @@ -0,0 +1,75 @@ +import threading +import time +import traceback +from typing import Callable + +from ppc_common.ppc_async_executor.async_executor import AsyncExecutor +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager + + +class AsyncThreadExecutor(AsyncExecutor): + def __init__(self, event_manager: ThreadEventManager, logger): + self.event_manager = event_manager + self.logger = logger + self.threads = {} + self.lock = threading.Lock() + self._cleanup_thread = threading.Thread(target=self._loop_cleanup) + self._cleanup_thread.daemon = True + self._cleanup_thread.start() + + def execute(self, task_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], + args=()): + def thread_target(logger, on_finish, *args): + try: + target(*args) + on_finish(task_id, True) + except Exception as e: + logger.warn(traceback.format_exc()) + on_finish(task_id, False, e) + + thread = threading.Thread(target=thread_target, args=( + self.logger, on_target_finish,) + args) + thread.start() + + with self.lock: + self.threads[task_id] = thread + + stop_event = threading.Event() + self.event_manager.add_event(task_id, stop_event) + + def kill(self, task_id: str): + with self.lock: + if task_id not in self.threads: + return False + else: + thread = self.threads[task_id] + + self.event_manager.set_event(task_id) + thread.join() + self.logger.info(f"Task {task_id} has been stopped!") + return True + + def kill_all(self): + with self.lock: + keys = self.threads.keys() + + for task_id in keys: + self.kill(task_id) + + def _loop_cleanup(self): + while True: + self._cleanup_finished_threads() + time.sleep(3) + + def _cleanup_finished_threads(self): + with self.lock: + finished_threads = [ + task_id for task_id, thread in self.threads.items() if not thread.is_alive()] + + for task_id in finished_threads: + with self.lock: + del self.threads[task_id] + self.logger.info(f"Cleanup finished task thread {task_id}") + + def __del__(self): + self.kill_all() diff --git a/python/ppc_common/ppc_async_executor/test/__init__.py b/python/ppc_common/ppc_async_executor/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py b/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py new file mode 100644 index 00000000..f4a91e7b --- /dev/null +++ b/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py @@ -0,0 +1,109 @@ +import multiprocessing +import time +import unittest + +# 示例用法 +from ppc_common.ppc_async_executor.async_subprocess_executor import AsyncSubprocessExecutor +from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_mock.mock_objects import MockLogger + + +def task1(shared_status1, key, dur): + shared_status1[key] = 1 + time.sleep(dur) + shared_status1[key] = 2 + + +def task2(shared_status2, key, dur): + shared_status2[key] = 1 + time.sleep(dur) + shared_status2[key] = 2 + + +def on_target_finish(task_id, flag, e=None): + if flag: + print(f'{task_id}, success') + else: + print(f'{task_id}, failed, {e}') + + +class TestSubprocessExecutor(unittest.TestCase): + + def test_kill(self): + logger = MockLogger() + executor = AsyncSubprocessExecutor(logger) + + # 使用 Manager 来创建共享状态字典 + manager = multiprocessing.Manager() + shared_status1 = manager.dict() + shared_status2 = manager.dict() + + # 启动两个任务 + key = 'test_kill' + executor.execute('0xaa', task1, on_target_finish, + (shared_status1, key, 2)) + executor.execute('0xbb', task2, on_target_finish, + (shared_status2, key, 2)) + + time.sleep(1) + + # 终止一个任务 + if executor.kill('0xaa'): + print(f"Task {'0xaa'} has been terminated") + + # 等待一段时间 + time.sleep(3) + + self.assertEqual(shared_status1.get(key, None), 1) + self.assertEqual(shared_status2.get(key, None), 2) + + +thread_event_manager = ThreadEventManager() + + +def thread_task1(shared_status1, task_id, key, dur): + shared_status1[key] = 1 + while not thread_event_manager.event_status(task_id): + time.sleep(dur) + shared_status1[key] = 2 + + +def thread_task2(shared_status2, key, dur): + shared_status2[key] = 1 + time.sleep(dur) + shared_status2[key] = 2 + raise Exception("raise error") + + +class TestThreadExecutor(unittest.TestCase): + + def test_kill(self): + logger = MockLogger() + executor = AsyncThreadExecutor(thread_event_manager, logger) + + shared_status1 = {'test': 'test'} + shared_status2 = {'test': 'test'} + key = 'test_kill' + task_id_1 = '0xaa' + task_id_2 = '0xbb' + + executor.execute(task_id_1, thread_task1, on_target_finish, + (shared_status1, task_id_1, key, 1)) + executor.execute(task_id_2, thread_task2, + on_target_finish, (shared_status2, key, 1)) + + time.sleep(2) + + # 终止一个任务 + if executor.kill(task_id_1): + print(f"Task {task_id_1} has been terminated") + + # 等待一段时间 + time.sleep(1) + self.assertEqual(shared_status1.get(key, None), 2) + self.assertEqual(shared_status2.get(key, None), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/ppc_async_executor/thread_event_manager.py b/python/ppc_common/ppc_async_executor/thread_event_manager.py new file mode 100644 index 00000000..f4311b69 --- /dev/null +++ b/python/ppc_common/ppc_async_executor/thread_event_manager.py @@ -0,0 +1,34 @@ +import threading +from typing import Dict + +from readerwriterlock import rwlock + + +class ThreadEventManager: + def __init__(self): + # Event清理由TaskManager完成 + self.events: Dict[str, threading.Event] = {} + self.rw_lock = rwlock.RWLockWrite() + + def add_event(self, task_id: str, event: threading.Event) -> None: + with self.rw_lock.gen_wlock(): + self.events[task_id] = event + + def remove_event(self, task_id: str): + with self.rw_lock.gen_wlock(): + if task_id in self.events: + del self.events[task_id] + + def set_event(self, task_id: str): + with self.rw_lock.gen_wlock(): + if task_id in self.events: + self.events[task_id].set() + else: + raise KeyError(f"Task ID {task_id} not found") + + def event_status(self, task_id: str) -> bool: + with self.rw_lock.gen_rlock(): + if task_id in self.events: + return self.events[task_id].is_set() + else: + return False diff --git a/python/ppc_common/ppc_config/__init__.py b/python/ppc_common/ppc_config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_config/file_chunk_config.py b/python/ppc_common/ppc_config/file_chunk_config.py new file mode 100644 index 00000000..80049ce4 --- /dev/null +++ b/python/ppc_common/ppc_config/file_chunk_config.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from ppc_common.ppc_utils import common_func + + +class FileChunkConfig: + # default read chunk size is 512M + DEFAULT_READ_CHUNK_SIZE = 512 + # default file object chunk size is 2G + DEFAULT_FILE_OBJECT_CHUNK_SIZE = 2048 + + READ_CHUNK_SIZE_MB_KEY = "read_chunk_size_mb" + FILE_CHUNK_SIZE_MB_KEY = "file_chunk_size_mb" + ENABLE_ALL_CHUNCK_FILE_MGR_KEY = "enable_file_chunk_on_all_mode" + + def __init__(self, config): + read_chunk_size = common_func.get_config_value( + FileChunkConfig.READ_CHUNK_SIZE_MB_KEY, + FileChunkConfig.DEFAULT_READ_CHUNK_SIZE, + config, False) + self.read_chunk_size = int(read_chunk_size) * 1024 * 1024 + file_object_chunk_size = common_func.get_config_value( + FileChunkConfig.FILE_CHUNK_SIZE_MB_KEY, + FileChunkConfig.DEFAULT_FILE_OBJECT_CHUNK_SIZE, config, False) + self.file_object_chunk_size = int(file_object_chunk_size) * 1024 * 1024 + self.enable_file_chunk_on_all_mode = common_func.get_config_value( + FileChunkConfig.ENABLE_ALL_CHUNCK_FILE_MGR_KEY, + False, config, False) diff --git a/python/ppc_common/ppc_config/sql_storage_config_loader.py b/python/ppc_common/ppc_config/sql_storage_config_loader.py new file mode 100644 index 00000000..9ed1cf3b --- /dev/null +++ b/python/ppc_common/ppc_config/sql_storage_config_loader.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from ppc_common.deps_services.sql_storage_api import SQLStorageConfig +from ppc_common.ppc_utils import common_func + + +class SQLStorageConfigLoader: + POOL_RECYCLE_KEY = "SQL_POOL_RECYCLE_SECONDS" + POOL_SIZE_KEY = "SQL_POOL_SIZE" + MAX_OVERFLOW_KEY = "SQL_MAX_OVERFLOW_SIZE" + POOL_TIMEOUT_KEY = "SQL_POOL_TIMEOUT_SECONDS" + DATABASE_URL_KEY = "SQLALCHEMY_DATABASE_URI" + DATABASE_TYPE_KEY = "DB_TYPE" + DATABASE_NAME_KEY = "DB_NAME" + + @staticmethod + def load(config): + db_url = common_func.get_config_value( + SQLStorageConfigLoader.DATABASE_URL_KEY, None, config, True) + db_type = common_func.get_config_value( + SQLStorageConfigLoader.DATABASE_TYPE_KEY, None, config, True) + db_name = common_func.get_config_value( + SQLStorageConfigLoader.DATABASE_NAME_KEY, None, config, False) + pool_recycle = common_func.get_config_value( + SQLStorageConfigLoader.POOL_RECYCLE_KEY, None, config, False) + pool_size = common_func.get_config_value( + SQLStorageConfigLoader.POOL_SIZE_KEY, None, config, False) + max_overflow = common_func.get_config_value( + SQLStorageConfigLoader.MAX_OVERFLOW_KEY, None, config, False) + pool_timeout = common_func.get_config_value( + SQLStorageConfigLoader.POOL_TIMEOUT_KEY, None, config, False) + return SQLStorageConfig( + pool_recycle=pool_recycle, + pool_size=pool_size, + max_overflow=max_overflow, + pool_timeout=pool_timeout, + url=db_url, db_type=db_type, db_name=db_name) diff --git a/python/ppc_common/ppc_crypto/__init__.py b/python/ppc_common/ppc_crypto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_crypto/crypto_utils.py b/python/ppc_common/ppc_crypto/crypto_utils.py new file mode 100644 index 00000000..43a13aaa --- /dev/null +++ b/python/ppc_common/ppc_crypto/crypto_utils.py @@ -0,0 +1,130 @@ +import binascii + +import base64 +import random +from Crypto.Cipher import AES +from Crypto.Cipher import PKCS1_v1_5 as Cipher_PKCS1_v1_5 +from Crypto.PublicKey import RSA + +from ppc_common.config import CONFIG_DATA + +try: + import gmpy2 + + IS_GMP = True +except ImportError: + IS_GMP = False + +_USE_MOD_GMP_SIZE = (1 << (8 * 2)) +_USE_MULMOD_GMP_SIZE = (1 << 1000) + +RSA_PUBLIC_HEADER = "-----BEGIN PUBLIC KEY-----" +RSA_PUBLIC_END = "-----END PUBLIC KEY-----" + +RSA_PRIVATE_HEADER = "-----BEGIN RSA PRIVATE KEY-----" +RSA_PRIVATE_END = "-----END RSA PRIVATE KEY-----" + + +def powmod(a, b, c): + if a == 1: + return 1 + if not IS_GMP or max(a, b, c) < _USE_MOD_GMP_SIZE: + return pow(a, b, c) + else: + return int(gmpy2.powmod(a, b, c)) + + +def mulmod(a, b, c): + if not IS_GMP or max(a, b, c) < _USE_MULMOD_GMP_SIZE: + return a * b % c + else: + a, b, c = gmpy2.mpz(a), gmpy2.mpz(b), gmpy2.mpz(c) + return int(gmpy2.mod(gmpy2.mul(a, b), c)) + + +DEFAULT_KEYSIZE = 1024 +DEFAULT_G = 9020881489161854992071763483314773468341853433975756385639545080944698236944020124874820917267762049756743282301106459062535797137327360192691469027152272 +DEFAULT_N = 102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885212534669604964926915306738569799518792945024759514373214412797317972739022405456550476153212687312211184540248262330559143446510677062823907392904449451177 +DEFAULT_FI = 102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885192126363163670343672910761259882348623401714459980712242233796355982147797162316532450768783823909695360736554767341443201861573989081253763975895939627220 + +# OUTPUT_BIT_LENGTH = 128 +OUTPUT_BIT_LENGTH = CONFIG_DATA['MPC_BIT_LENGTH'] +DEFAULT_MPC_N = pow(2, OUTPUT_BIT_LENGTH) + + +# DEFAULT_MPC_N = DEFAULT_N + +def ot_base_pown(value): + return powmod(DEFAULT_G, value, DEFAULT_N) + + +def ot_pown(base, value): + return powmod(base, value, DEFAULT_N) + + +def ot_mul_fi(a_val, b_val): + return mulmod(a_val, b_val, DEFAULT_FI) + + +def ot_mul_n(a_val, b_val): + return mulmod(a_val, b_val, DEFAULT_N) + + +# def ot_add(a_val, b_val): +# return (a_val + b_val)%DEFAULT_N + + +def ot_str_to_int(input_str): + return int.from_bytes(input_str.encode('utf-8'), 'big'), len(input_str) + + +def ot_int_to_str(input_int, len_input_str): + # len_int = len(str(input_int))//2 + result = input_int.to_bytes(len_input_str, 'big').decode('utf-8') + # TODO: check all 0x00 valid + # if result[0].encode('ascii') == b'\x00': + # result = result[1:] + return result + + +def get_random_int(): + return random.SystemRandom().randrange(1, DEFAULT_N) + + +def make_rsa_decrypt(encrypted_hex, private_key): + if RSA_PRIVATE_HEADER not in private_key: + private_key = RSA_PRIVATE_HEADER + "\n" + private_key + if RSA_PRIVATE_END not in private_key: + private_key = private_key + "\n" + RSA_PRIVATE_END + rsa_private_key = RSA.importKey(private_key) + encrypted_text = binascii.unhexlify(encrypted_hex) + cipher = Cipher_PKCS1_v1_5.new(rsa_private_key) + decrypted_text = cipher.decrypt(encrypted_text, None) + return decrypted_text.decode('utf-8') + + +class AESCipher: + + def __init__(self, key): + # self.key = bytes(key, 'utf-8') + self.key = key + + def encrypt(self, raw): + raw = pad(raw) + iv = "encryptionIntVec".encode('utf-8') + cipher = AES.new(self.key, AES.MODE_CBC, iv) + return base64.b64encode(cipher.encrypt(raw)) + + def decrypt(self, enc): + iv = "encryptionIntVec".encode('utf-8') + enc = base64.b64decode(enc) + cipher = AES.new(self.key, AES.MODE_CBC, iv) + return unpad(cipher.decrypt(enc)).decode('utf8') + + +BS = 16 +def pad(s): return bytes(s + (BS - len(s) % + BS) * chr(BS - len(s) % BS), 'utf-8') + + +def unpad(s): return s[0:-ord(s[-1:])] diff --git a/python/ppc_common/ppc_crypto/ihc_cipher.py b/python/ppc_common/ppc_crypto/ihc_cipher.py new file mode 100644 index 00000000..7cc385ff --- /dev/null +++ b/python/ppc_common/ppc_crypto/ihc_cipher.py @@ -0,0 +1,93 @@ +# from abc import ABC +# import namedTuple +from dataclasses import dataclass + +import struct +from ppc_common.ppc_crypto.phe_cipher import PheCipher +import secrets + + +@dataclass +class IhcCiphertext(): + __slots__ = ['c_left', 'c_right'] + + def __init__(self, c_left: int, c_right: int) -> None: + self.c_left = c_left + self.c_right = c_right + + def __add__(self, other): + cipher_left = self.c_left + other.c_left + cipher_right = self.c_right + other.c_right + return IhcCiphertext(cipher_left, cipher_right) + + def __eq__(self, other): + return self.c_left == other.c_left and self.c_right == other.c_right + + def encode(self) -> bytes: + # 计算每个整数的字节长度 + len_c_left = (self.c_left.bit_length() + 7) // 8 + len_c_right = (self.c_right.bit_length() + 7) // 8 + + # 将整数转换为字节序列，使用大端字节序和带符号整数 + c_left_bytes = self.c_left.to_bytes(len_c_left, byteorder='big') + c_right_bytes = self.c_right.to_bytes(len_c_right, byteorder='big') + + # 编码整数的长度 + len_bytes = struct.pack('>II', len_c_left, len_c_right) + + # 返回所有数据 + return len_bytes + c_left_bytes + c_right_bytes + + @classmethod + def decode(cls, encoded_data: bytes): + # 解码整数的长度 + len_c_left, len_c_right = struct.unpack('>II', encoded_data[:8]) + + # 根据长度解码整数 + c_left = int.from_bytes(encoded_data[8:8 + len_c_left], byteorder='big') + c_right = int.from_bytes(encoded_data[8 + len_c_left:8 + len_c_left + len_c_right], byteorder='big') + return cls(c_left, c_right) + +class IhcCipher(PheCipher): + def __init__(self, key_length: int = 256, iter_round: int = 16) -> None: + super().__init__(key_length) + key = secrets.randbits(key_length) + self.public_key = key + self.private_key = key + self.iter_round = iter_round + self.key_length = key_length + + self.max_mod = 1 << key_length + + def encrypt(self, number: int) -> IhcCiphertext: + random_u = secrets.randbits(self.key_length) + x_this = number + x_last = random_u + for i in range(0, self.iter_round): + x_tmp = (self.private_key * x_this - x_last) % self.max_mod + x_last = x_this + x_this = x_tmp + # cipher = IhcCiphertext(x_this, x_last, self.max_mod) + cipher = IhcCiphertext(x_this, x_last) + return cipher + + def decrypt(self, cipher: IhcCiphertext) -> int: + x_this = cipher.c_right + x_last = cipher.c_left + for i in range(0, self.iter_round-1): + x_tmp = (self.private_key * x_this - x_last) % self.max_mod + x_last = x_this + x_this = x_tmp + return x_this + + def encrypt_batch(self, numbers) -> list: + return [self.encrypt(num) for num in numbers] + + def decrypt_batch(self, ciphers) -> list: + return [self.decrypt(cipher) for cipher in ciphers] + + def encrypt_batch_parallel(self, numbers: list) -> list: + return self.encrypt_batch(numbers) + + def decrypt_batch_parallel(self, ciphers: list) -> list: + return self.decrypt_batch(ciphers) diff --git a/python/ppc_common/ppc_crypto/ihc_codec.py b/python/ppc_common/ppc_crypto/ihc_codec.py new file mode 100644 index 00000000..9de0c858 --- /dev/null +++ b/python/ppc_common/ppc_crypto/ihc_codec.py @@ -0,0 +1,29 @@ +from typing import Tuple + +from ppc_common.ppc_crypto.ihc_cipher import IhcCiphertext + + +class IhcCodec: + @staticmethod + def _int_to_bytes(x): + return x.to_bytes((x.bit_length() + 7) // 8, 'big') + + @staticmethod + def _bytes_to_int(x): + return int.from_bytes(x, 'big') + + @staticmethod + def encode_enc_key(public_key) -> bytes: + return bytes() + + @staticmethod + def decode_enc_key(public_key_bytes) -> bytes: + return bytes() + + @staticmethod + def encode_cipher(cipher: IhcCiphertext, be_secure=True) -> Tuple[bytes, bytes]: + return cipher.encode(), bytes() + + @staticmethod + def decode_cipher(enc_key, ciphertext: bytes, exponent) -> IhcCiphertext: + return IhcCiphertext.decode(ciphertext) diff --git a/python/ppc_common/ppc_crypto/paillier_cipher.py b/python/ppc_common/ppc_crypto/paillier_cipher.py new file mode 100644 index 00000000..d2e0232a --- /dev/null +++ b/python/ppc_common/ppc_crypto/paillier_cipher.py @@ -0,0 +1,45 @@ +import math +import os +from concurrent.futures import ProcessPoolExecutor + +from phe import paillier, EncryptedNumber + +from ppc_common.ppc_crypto.phe_cipher import PheCipher + + +class PaillierCipher(PheCipher): + + def __init__(self, key_length: int = 2048) -> None: + super().__init__(key_length) + self.public_key, self.private_key = paillier.generate_paillier_keypair( + n_length=self.key_length) + + def encrypt(self, number) -> EncryptedNumber: + return self.public_key.encrypt(int(number)) + + def decrypt(self, cipher: EncryptedNumber) -> int: + return self.private_key.decrypt(cipher) + + def encrypt_batch(self, numbers) -> list: + return [self.encrypt(num) for num in numbers] + + def decrypt_batch(self, ciphers) -> list: + return [self.decrypt(cipher) for cipher in ciphers] + + def encrypt_batch_parallel(self, numbers) -> list: + num_cores = os.cpu_count() + batch_size = math.ceil(len(numbers) / num_cores) + batches = [numbers[i:i + batch_size] for i in range(0, len(numbers), batch_size)] + with ProcessPoolExecutor(max_workers=num_cores) as executor: + futures = [executor.submit(self.encrypt_batch, batch) for batch in batches] + result = [future.result() for future in futures] + return [item for sublist in result for item in sublist] + + def decrypt_batch_parallel(self, ciphers) -> list: + num_cores = os.cpu_count() + batch_size = math.ceil(len(ciphers) / num_cores) + batches = [ciphers[i:i + batch_size] for i in range(0, len(ciphers), batch_size)] + with ProcessPoolExecutor(max_workers=num_cores) as executor: + futures = [executor.submit(self.decrypt_batch, batch) for batch in batches] + result = [future.result() for future in futures] + return [item for sublist in result for item in sublist] diff --git a/python/ppc_common/ppc_crypto/paillier_codec.py b/python/ppc_common/ppc_crypto/paillier_codec.py new file mode 100644 index 00000000..fb5f3bf0 --- /dev/null +++ b/python/ppc_common/ppc_crypto/paillier_codec.py @@ -0,0 +1,34 @@ +from typing import Tuple + +from phe import PaillierPublicKey, paillier, EncryptedNumber + + +class PaillierCodec: + @staticmethod + def _int_to_bytes(x): + return x.to_bytes((x.bit_length() + 7) // 8, 'big') + + @staticmethod + def _bytes_to_int(x): + return int.from_bytes(x, 'big') + + @staticmethod + def encode_enc_key(public_key: PaillierPublicKey) -> bytes: + return PaillierCodec._int_to_bytes(public_key.n) + + @staticmethod + def decode_enc_key(public_key_bytes: bytes) -> PaillierPublicKey: + public_key_n = PaillierCodec._bytes_to_int(public_key_bytes) + return paillier.PaillierPublicKey(n=public_key_n) + + @staticmethod + def encode_cipher(cipher: EncryptedNumber, be_secure=True) -> Tuple[bytes, bytes]: + return PaillierCodec._int_to_bytes(cipher.ciphertext(be_secure=be_secure)), \ + PaillierCodec._int_to_bytes(cipher.exponent) + + @staticmethod + def decode_cipher(public_key: PaillierPublicKey, ciphertext: bytes, exponent: bytes) -> EncryptedNumber: + return paillier.EncryptedNumber(public_key, + PaillierCodec._bytes_to_int( + ciphertext), + PaillierCodec._bytes_to_int(exponent)) diff --git a/python/ppc_common/ppc_crypto/phe_cipher.py b/python/ppc_common/ppc_crypto/phe_cipher.py new file mode 100644 index 00000000..3581ac45 --- /dev/null +++ b/python/ppc_common/ppc_crypto/phe_cipher.py @@ -0,0 +1,25 @@ +from abc import ABC + + +class PheCipher(ABC): + def __init__(self, key_length: int) -> None: + self.key_length = key_length + pass + + def encrypt(self, number: int): + pass + + def decrypt(self, cipher) -> int: + pass + + def encrypt_batch(self, numbers: list) -> list: + pass + + def decrypt_batch(self, ciphers: list) -> list: + pass + + def encrypt_batch_parallel(self, numbers: list) -> list: + pass + + def decrypt_batch_parallel(self, ciphers: list) -> list: + pass diff --git a/python/ppc_common/ppc_crypto/phe_factory.py b/python/ppc_common/ppc_crypto/phe_factory.py new file mode 100644 index 00000000..5db01063 --- /dev/null +++ b/python/ppc_common/ppc_crypto/phe_factory.py @@ -0,0 +1,25 @@ +from ppc_common.ppc_crypto.ihc_cipher import IhcCipher +from ppc_common.ppc_crypto.ihc_codec import IhcCodec +from ppc_common.ppc_crypto.paillier_cipher import PaillierCipher +from ppc_common.ppc_crypto.paillier_codec import PaillierCodec + + +class PheCipherFactory(object): + + @staticmethod + def build_phe(homo_algorithm=0, key_length=2048): + if homo_algorithm == 0: + return IhcCipher() + if homo_algorithm == 1: + return PaillierCipher(key_length) + else: + raise ValueError("Unsupported homo algorithm") + + @staticmethod + def build_codec(homo_algorithm=0): + if homo_algorithm == 0: + return IhcCodec() + if homo_algorithm == 1: + return PaillierCodec() + else: + raise ValueError("Unsupported homo algorithm") diff --git a/python/ppc_common/ppc_crypto/test/__init__.py b/python/ppc_common/ppc_crypto/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_crypto/test/phe_unittest.py b/python/ppc_common/ppc_crypto/test/phe_unittest.py new file mode 100644 index 00000000..5d036b75 --- /dev/null +++ b/python/ppc_common/ppc_crypto/test/phe_unittest.py @@ -0,0 +1,105 @@ +import time +import unittest + +import numpy as np + +from ppc_common.ppc_crypto.ihc_cipher import IhcCipher, IhcCiphertext +from ppc_common.ppc_crypto.ihc_codec import IhcCodec +from ppc_common.ppc_crypto.paillier_cipher import PaillierCipher + + +class PaillierUtilsTest(unittest.TestCase): + + def test_enc_and_dec_parallel(self): + paillier = PaillierCipher(key_length=1024) + inputs = np.random.randint(1, 10001, size=10) + + # start_time = time.time() + # paillier.encrypt_batch(inputs) + # end_time = time.time() + # print("enc:", end_time - start_time, "seconds") + + start_time = time.time() + ciphers = paillier.encrypt_batch_parallel(inputs) + end_time = time.time() + print("enc_p:", end_time - start_time, "seconds") + + start_time = time.time() + outputs = paillier.decrypt_batch_parallel(ciphers) + end_time = time.time() + print("dec_p:", end_time - start_time, "seconds") + + self.assertListEqual(list(inputs), list(outputs)) + + def test_ihc_enc_and_dec_parallel(self): + ihc = IhcCipher(key_length=256) + try_size = 100000 + inputs = np.random.randint(1, 10001, size=try_size) + expected = np.sum(inputs) + + start_time = time.time() + ciphers = ihc.encrypt_batch_parallel(inputs) + end_time = time.time() + print( + f"size:{try_size}, enc_p: {end_time - start_time} seconds, " + f"average times: {(end_time - start_time) / try_size * 1000 * 1000} us") + + start_time = time.time() + cipher_start = ciphers[0] + for i in range(1, len(ciphers)): + cipher_left = (cipher_start.c_left + ciphers[i].c_left) + cipher_right = (cipher_start.c_right + ciphers[i].c_right ) + # IhcCiphertext(cipher_left, cipher_right, cipher_start.max_mod) + IhcCiphertext(cipher_left, cipher_right) + end_time = time.time() + print(f"size:{try_size}, add_p raw with class: {end_time - start_time} seconds, average times: {(end_time - start_time)/try_size * 1000 * 1000} us") + + start_time = time.time() + cipher_start = ciphers[0] + for i in range(1, len(ciphers)): + cipher_left = (cipher_start.c_left + ciphers[i].c_left) + cipher_right = (cipher_start.c_right + ciphers[i].c_right ) + # IhcCiphertext(cipher_left, cipher_right) + end_time = time.time() + print(f"size:{try_size}, add_p raw: {end_time - start_time} seconds, average times: {(end_time - start_time)/try_size * 1000 * 1000} us") + + start_time = time.time() + cipher_start = ciphers[0] + for i in range(1, len(ciphers)): + cipher_start = cipher_start + ciphers[i] + end_time = time.time() + print( + f"size:{try_size}, add_p: {end_time - start_time} seconds, " + f"average times: {(end_time - start_time) / try_size * 1000 * 1000} us") + + start_time = time.time() + outputs = ihc.decrypt_batch_parallel(ciphers) + end_time = time.time() + print( + f"size:{try_size}, dec_p: {end_time - start_time} seconds, " + f"average times: {(end_time - start_time) / try_size * 1000 * 1000} us") + + decrypted = ihc.decrypt(cipher_start) + self.assertListEqual(list(inputs), list(outputs)) + assert decrypted == expected + + def test_ihc_code(self): + ihc = IhcCipher(key_length=256) + try_size = 100000 + inputs = np.random.randint(1, 10001, size=try_size) + start_time = time.time() + ciphers = ihc.encrypt_batch_parallel(inputs) + end_time = time.time() + print( + f"size:{try_size}, enc_p: {end_time - start_time} seconds, " + f"average times: {(end_time - start_time) / try_size * 1000 * 1000} us") + for i in range(0, len(ciphers)): + cipher: IhcCiphertext = ciphers[i] + encoded, _ = IhcCodec.encode_cipher(cipher) + + decoded = IhcCodec.decode_cipher(None, encoded, None) + assert cipher == decoded + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/ppc_dataset/__init__.py b/python/ppc_common/ppc_dataset/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_dataset/dataset_helper.py b/python/ppc_common/ppc_dataset/dataset_helper.py new file mode 100644 index 00000000..718134a0 --- /dev/null +++ b/python/ppc_common/ppc_dataset/dataset_helper.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +from ppc_common.deps_services.sharding_file_object import ShardingFileObject +from ppc_common.deps_services.file_object import SplitMode +from ppc_common.deps_services.storage_api import StorageType +import os + + +class DataSetHelper: + # default split granularity is 1G + def __init__(self, dataset_user, dataset_id, dataset_local_path, + sql_storage, remote_storage, chunk_config, logger): + self._split_mode = SplitMode.NONE + self._chunk_config = chunk_config + self._chunk_size = self._chunk_config.file_object_chunk_size + self._dataset_user = dataset_user + remote_path = os.path.join(dataset_user, dataset_id) + self._file_object = ShardingFileObject( + dataset_local_path, remote_path, remote_storage, sql_storage, logger) + + @property + def file_object(self): + return self._file_object + + def download_dataset(self, enforce_flush=False): + """download the dataset + """ + self._file_object.download(enforce_flush=enforce_flush) + + def upload_dataset(self): + """upload the dataset + """ + self._file_object.upload(self._split_mode, self._chunk_size) + + def save_data(self, data): + """save the dataset + """ + self._file_object.save_data( + data=data, split_mode=self._split_mode, granularity=self._chunk_size) + + def update_data(self, data): + return self._file_object.update_data( + updated_data=data, split_mode=self._split_mode, granularity=self._chunk_size) + + def file_rename(self, new_storage_path, with_user_home=False): + if with_user_home: + self._file_object.rename(os.path.join( + self._dataset_user, new_storage_path)) + else: + self._file_object.rename(new_storage_path) + + def get_local_path(self): + """get the local path + """ + return self._file_object.get_local_path() + + def get_remote_path(self): + """get the remote path + """ + return self._file_object.get_remote_path() diff --git a/python/ppc_common/ppc_dataset/dataset_helper_factory.py b/python/ppc_common/ppc_dataset/dataset_helper_factory.py new file mode 100644 index 00000000..4fc0a2b0 --- /dev/null +++ b/python/ppc_common/ppc_dataset/dataset_helper_factory.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +from ppc_common.ppc_dataset.dataset_helper import DataSetHelper + + +class DataSetHelperFactory: + def __init__(self, sql_storage, remote_storage, chunk_config, logger): + self._sql_storage = sql_storage + self._remote_storage = remote_storage + self._chunk_config = chunk_config + self._logger = logger + + def create(self, dataset_user, dataset_id, dataset_local_path): + return DataSetHelper(dataset_user=dataset_user, dataset_id=dataset_id, + dataset_local_path=dataset_local_path, sql_storage=self._sql_storage, + chunk_config=self._chunk_config, logger=self._logger, + remote_storage=self._remote_storage) + + +def upload_dataset(dataset_helper_factory, dataset_user, dataset_id, dataset_local_path, logger, log_keyword): + dataset_helper = dataset_helper_factory.create( + dataset_user, dataset_id, dataset_local_path) + logger.info( + f"{log_keyword}: Upload dataset: {dataset_helper.get_local_path()} => {dataset_helper.get_remote_path()}") + dataset_helper.upload_dataset() + logger.info( + f"{log_keyword}: Upload dataset success: {dataset_helper.get_local_path()} => {dataset_helper.get_remote_path()}") + return dataset_helper + + +def delete_dataset(dataset_helper_factory, dataset_user, dataset_id, logger, log_keyword): + dataset_helper = dataset_helper_factory.create( + dataset_user=dataset_user, dataset_id=dataset_id, dataset_local_path=None) + logger.info( + f"{log_keyword}: Delete dataset: {dataset_helper.get_remote_path()}") + dataset_helper.file_object.delete() + logger.info( + f"{log_keyword}: Delete dataset success: {dataset_helper.get_remote_path()}") + return dataset_helper + + +def download_dataset(dataset_helper_factory, dataset_user, dataset_id, dataset_local_path, logger, log_keyword, enforce_flush=False): + dataset_helper = dataset_helper_factory.create( + dataset_user, dataset_id, dataset_local_path) + logger.info( + f"{log_keyword}: Download dataset: {dataset_helper.get_remote_path()}=>{dataset_helper.get_local_path()}") + dataset_helper.download_dataset(enforce_flush) + logger.info( + f"{log_keyword} dataset success: {dataset_helper.get_remote_path()}=>{dataset_helper.get_local_path()}") + return dataset_helper + + +def get_dataset(dataset_helper_factory, dataset_user, dataset_id, logger, log_keyword): + dataset_helper = dataset_helper_factory.create( + dataset_user, dataset_id, None) + logger.info( + f"{log_keyword}: Get dataset from: {dataset_helper.get_remote_path()}") + data = dataset_helper.file_object.get_data() + logger.info( + f"{log_keyword}: Get dataset success from: {dataset_helper.get_remote_path()}") + return data diff --git a/python/ppc_common/ppc_initialize/__init__.py b/python/ppc_common/ppc_initialize/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_initialize/dataset_handler_initialize.py b/python/ppc_common/ppc_initialize/dataset_handler_initialize.py new file mode 100644 index 00000000..4b2e64fe --- /dev/null +++ b/python/ppc_common/ppc_initialize/dataset_handler_initialize.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader +from ppc_common.ppc_config.file_chunk_config import FileChunkConfig +from ppc_common.ppc_dataset.dataset_helper_factory import DataSetHelperFactory +from ppc_common.deps_services.mysql_storage import MySQLStorage +from ppc_common.deps_services import storage_loader + + +class DataSetHandlerInitialize: + def __init__(self, config, logger): + self._config = config + self._logger = logger + self._init_sql_storage() + self._init_remote_storage() + self._init_dataset_factory() + + def _init_sql_storage(self): + self.sql_storage = MySQLStorage( + SQLStorageConfigLoader.load(self._config)) + + def _init_remote_storage(self): + self.storage_client = storage_loader.load(self._config, self._logger) + + def _init_dataset_factory(self): + self.file_chunk_config = FileChunkConfig(self._config) + self.dataset_helper_factory = DataSetHelperFactory( + sql_storage=self.sql_storage, + remote_storage=self.storage_client, + chunk_config=self.file_chunk_config, + logger=self._logger) diff --git a/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py b/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py new file mode 100644 index 00000000..af3018e2 --- /dev/null +++ b/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +import unittest +import logging +from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader +from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize +from ppc_common.ppc_config.file_chunk_config import FileChunkConfig +from ppc_common.ppc_dataset import dataset_helper_factory +import sys +from ppc_common.ppc_utils import utils + + +class DataSetInitializerWrapper: + def __init__(self, ut_obj, file_chunk_size): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + self.logger = logging.getLogger(__name__) + self.config = {} + self.config[SQLStorageConfigLoader.DATABASE_URL_KEY] = "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8mb4" + self.config[FileChunkConfig.ENABLE_ALL_CHUNCK_FILE_MGR_KEY] = True + self.config["STORAGE_TYPE"] = "HDFS" + self.config["HDFS_USER"] = "chenyujie" + self.config["HDFS_ENDPOINT"] = "http://127.0.0.1:9870" + self.config[FileChunkConfig.FILE_CHUNK_SIZE_MB_KEY] = file_chunk_size + self.dataset_handler_initializer = DataSetHandlerInitialize( + self.config, self.logger) + self.ut_obj = ut_obj + + def _download_and_check(self, dataset_user, dataset_id, dataset_local_path, expected_md5=None): + # download + download_local_path = dataset_local_path + ".download" + dataset_helper_factory.download_dataset( + dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, + dataset_id=dataset_id, dataset_user=dataset_user, logger=self.logger, + dataset_local_path=download_local_path, + log_keyword="testDownload", enforce_flush=True + ) + # check md5 + downloaded_file_md5 = utils.calculate_md5(download_local_path) + self.ut_obj.assertEquals(expected_md5, downloaded_file_md5) + # get the dataset + data = dataset_helper_factory.get_dataset( + dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, + dataset_id=dataset_id, dataset_user=dataset_user, logger=self.logger, + log_keyword="testGetDataSet" + ) + get_data_md5sum = utils.md5sum(data) + self.ut_obj.assertEquals(expected_md5, get_data_md5sum) + + def test_dataset_ops(self, dataset_user, dataset_id, dataset_local_path, updated_data): + # upload the dataset + dataset_helper_factory.upload_dataset( + dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, + dataset_id=dataset_id, dataset_user=dataset_user, + dataset_local_path=dataset_local_path, + logger=self.logger, log_keyword="testUpload") + origin_md5 = utils.calculate_md5(dataset_local_path) + self._download_and_check(dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=dataset_local_path, + expected_md5=origin_md5) + # update dataset + dataset_helper = self.dataset_handler_initializer.dataset_helper_factory.create( + dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=None) + dataset_helper.update_data(updated_data) + # check + self._download_and_check(dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=dataset_local_path, + expected_md5=utils.md5sum(updated_data)) + # rename + rename_path = dataset_id + ".rename" + dataset_helper.file_rename(rename_path, True) + # check + self._download_and_check(dataset_user=dataset_user, + dataset_id=rename_path, + dataset_local_path=dataset_local_path + ".rename", + expected_md5=utils.md5sum(updated_data)) + # delete + dataset_helper = dataset_helper_factory.delete_dataset( + dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, + dataset_id=rename_path, dataset_user=dataset_user, + logger=self.logger, log_keyword="testDelete") + self.ut_obj.assertEquals(dataset_helper.file_object.existed(), False) + + +class DatasetInitializerTest(unittest.TestCase): + def testNoFileSplit(self): + wrapper = DataSetInitializerWrapper(self, 100) + dataset_user = "yujiechen" + dataset_id = "test_file_no_split" + dataset_local_path = "tools/test" + uploaded_data = "abcdsfsdfsdfssdfsd" + wrapper.test_dataset_ops(dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=dataset_local_path, + updated_data=uploaded_data) + + def testFileSplit(self): + wrapper = DataSetInitializerWrapper(self, 1) + dataset_user = "yujiechen" + dataset_id = "test_file_split_file" + dataset_local_path = "tools/test" + uploaded_data = "abcdsfsdfsdfssdfsd" + wrapper.test_dataset_ops(dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=dataset_local_path, + updated_data=uploaded_data) + + def testFileSplitAndUpdateSplit(self): + wrapper = DataSetInitializerWrapper(self, 1) + dataset_user = "yujiechen" + dataset_id = "test_large_file_split_file" + dataset_local_path = "tools/test" + updated_file = "tools/test2" + uploaded_data = None + with open(updated_file, "rb") as fp: + uploaded_data = fp.read() + wrapper.test_dataset_ops(dataset_user=dataset_user, + dataset_id=dataset_id, + dataset_local_path=dataset_local_path, + updated_data=uploaded_data) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/ppc_ml/__init__.py b/python/ppc_common/ppc_ml/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_ml/feature/__init__.py b/python/ppc_common/ppc_ml/feature/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_ml/feature/feature_importance.py b/python/ppc_common/ppc_ml/feature/feature_importance.py new file mode 100644 index 00000000..2441a20e --- /dev/null +++ b/python/ppc_common/ppc_ml/feature/feature_importance.py @@ -0,0 +1,246 @@ +# -*- coding: utf-8 -*- +from enum import Enum +import pandas as pd +from ppc_common.deps_services.serialize_type import SerializeType + + +class FeatureGainInfo: + def __init__(self, feature_idx, feature_gain): + self.feature_idx = feature_idx + self.feature_gain = feature_gain + +# TODO: cover + + +class FeatureImportanceType(Enum): + WEIGHT = 'weight' # The number of times the feature is used in all trees + GAIN = 'gain' # The gain of features in predictions across all trees + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class FeatureImportance: + """the feature importance + """ + + def __init__(self, importance_type: FeatureImportanceType, importance=0): + self.importance_type = importance_type + self.importance = importance + + def inc(self, value): + self.importance += value + + def desc(self): + return f"importance type: {self._importance_type}, importance: {self.importance}" + + def __cmp__(self, other): + if self.importance > other.importance: + return 1 + elif self.importance < other.importance: + return -1 + return 0 + + def __eq__(self, other): + return self.importance == other.importance + + def __lt__(self, other): + return self.importance < other.importance + + def __add__(self, other): + new_importance = FeatureImportance( + self.importance_type, importance=self.importance + other.importance) + return new_importance + + +class ReadOnlyFeatureImportanceStore: + RANK_PROPERTY = "score_rank" + DEAULT_SCORE_THRESHOLD = 0.95 + DEFAULT_TOPK_PROPERTY = "topk" + DEFAULT_EMPTY_TOPK_FLAG = " " + DEFAULT_TOPK_FLAG = "True" + + def __init__(self, feature_importance_dict, logger): + # feature_importance_type ==> fid ==> value + self.feature_importance_dict = feature_importance_dict + self.logger = logger + + def get_feature_importance(self, fid, type): + if type not in self.feature_importance_dict: + return 0 + if fid not in self.feature_importance_dict[type]: + return 0 + return self.feature_importance_dict[type][fid].importance + + def _get_sorting_columns(self): + sorting_columns = [] + ascending_list = [] + if FeatureImportanceType.GAIN in self.feature_importance_type_list: + sorting_columns.append(FeatureImportanceType.GAIN.name) + ascending_list.append(False) + if FeatureImportanceType.WEIGHT in self.feature_importance_type_list: + sorting_columns.append(FeatureImportanceType.WEIGHT.name) + ascending_list.append(False) + return (sorting_columns, ascending_list) + + @staticmethod + def load(df, logger): + """load the feature importance + + Args: + df (DataFrame): the feature importance + """ + logger.debug(f"load feature_importance data: {df}") + feature_importance_type_list = [] + for column in df.columns: + if not FeatureImportanceType.has_value(column.lower()): + continue + enum_feature_importance_type = FeatureImportanceType( + column.lower()) + feature_importance_type_list.append(enum_feature_importance_type) + logger.debug( + f"load feature_importance, feature_importance_type_list: {feature_importance_type_list}") + # feature_importance_type ==> fid_key ==> value + feature_importance_dict = dict() + for row in df.itertuples(): + fid_key = getattr( + row, FeatureImportanceStore.DEFAULT_FEATURE_PROPERTY) + for importance_type in feature_importance_type_list: + value = getattr(row, importance_type.name) + if importance_type not in feature_importance_dict: + feature_importance_dict.update({importance_type: dict()}) + feature_importance_dict[importance_type].update( + {fid_key: FeatureImportance(importance_type, value)}) + return ReadOnlyFeatureImportanceStore(feature_importance_dict, logger) + + def to_dataframe(self, topk_threshold=0.95): + """convert the feature importance into pd + the format: + | feature | score | score_rank | topk| + | x16 | 0.08234 | 1 | | + | x11 | 0.08134 | 2 | | + | x1 | 0.08034 | 3 | | + """ + if self.feature_importance_dict is None or len(self.feature_importance_dict) < 1: + return None + df = pd.DataFrame() + # the feature column + df.insert(df.shape[1], FeatureImportanceStore.DEFAULT_FEATURE_PROPERTY, + self.feature_importance_dict[self.feature_importance_type_list[0]].keys()) + # the importance columns + for importance_type in self.feature_importance_dict.keys(): + feature_importance_values = [] + for feature_importance in self.feature_importance_dict[importance_type].values(): + feature_importance_values.append(feature_importance.importance) + # calculate weight-average for the score + if importance_type == FeatureImportanceType.GAIN: + feature_importance_sum = sum(feature_importance_values) + if feature_importance_sum != 0: + for i in range(len(feature_importance_values)): + feature_importance_values[i] = float( + feature_importance_values[i]) / feature_importance_sum + + df.insert(df.shape[1], importance_type.name, + feature_importance_values) + # sort by the importance + (sorting_columns, ascending_list) = self._get_sorting_columns() + df = df.sort_values(by=sorting_columns, ascending=ascending_list) + # rank + df.insert(df.shape[1], + ReadOnlyFeatureImportanceStore.RANK_PROPERTY, + range(1, len(df) + 1)) + # top-k + if FeatureImportanceType.GAIN not in self.feature_importance_type_list: + return df + score_sum = float(0) + topk_list = [ + ReadOnlyFeatureImportanceStore.DEFAULT_EMPTY_TOPK_FLAG for _ in range(len(df))] + i = 0 + for score in df[FeatureImportanceType.GAIN.name].T: + score_sum += score + if score_sum >= topk_threshold: + topk_list[i] = ReadOnlyFeatureImportanceStore.DEFAULT_TOPK_FLAG + break + i += 1 + df.insert( + df.shape[1], ReadOnlyFeatureImportanceStore.DEFAULT_TOPK_PROPERTY, topk_list) + return df + + +class FeatureImportanceStore(ReadOnlyFeatureImportanceStore): + """store all the feature importance + """ + DEFAULT_FEATURE_PROPERTY = "feature" + DEFAULT_IMPORTANCE_LIST = [ + FeatureImportanceType.GAIN, FeatureImportanceType.WEIGHT] + + def __init__(self, feature_importance_type_list, feature_list, logger): + # feature_importance_type ==> fid ==> value + super().__init__(dict(), logger) + self.feature_list = feature_list + if self.feature_list is not None: + self.logger.info( + f"create FeatureImportanceStore, all features: {self.feature_list}") + self.feature_importance_type_list = feature_importance_type_list + for importance_type in self.feature_importance_type_list: + self.feature_importance_dict.update({importance_type: dict()}) + self._init() + + def _init(self): + if self.feature_list is None: + return + for i in range(len(self.feature_list)): + gain_list = {FeatureImportanceType.GAIN: 0, + FeatureImportanceType.WEIGHT: 0} + self.update_feature_importance(i, gain_list) + + def set_init(self, feature_list): + self.feature_list = feature_list + self._init() + + def generate_fid_key(self, fid): + if fid >= len(self.feature_list): + return None + return f"{self.feature_list[fid]}" + + def update_feature_importance(self, fid, gain_list): + """update the feature importance + + Args: + fid (int): the idx of the best feature(maxk) + gain_list (dict): the gain list for every importance_type + + Raises: + Exception: invalid gain_list + """ + fid_key = self.generate_fid_key(fid) + if fid_key is None: + return + for importance_type in gain_list.keys(): + # unknown importance_type + if importance_type not in self.feature_importance_dict: + continue + if fid_key not in self.feature_importance_dict[importance_type]: + self.feature_importance_dict[importance_type][fid_key] = FeatureImportance( + importance_type, 0) + self.feature_importance_dict[importance_type][fid_key].inc( + gain_list[importance_type]) + + def store(self, serialize_type: SerializeType, local_file_path, remote_file_path, storage_client): + """store the feature importance into file, upload using storage_client + + Args: + storage_client: the client used to upload the result + """ + df = self.to_dataframe() + if serialize_type is SerializeType.CSV: + df.to_csv(local_file_path, index=False) + else: + df.to_json(orient='split', path_or_buf=local_file_path) + self.logger.info( + f"Store feature_importance to {local_file_path}, file type: {serialize_type}") + if storage_client is not None: + storage_client.upload_file(local_file_path, remote_file_path) + self.logger.info( + f"Upload feature_importance to {local_file_path} success, file type: {serialize_type}") diff --git a/python/ppc_common/ppc_ml/feature/tests/feature_importance_test.py b/python/ppc_common/ppc_ml/feature/tests/feature_importance_test.py new file mode 100644 index 00000000..3b0f5661 --- /dev/null +++ b/python/ppc_common/ppc_ml/feature/tests/feature_importance_test.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +import unittest +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceType +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceStore +from ppc_common.ppc_ml.feature.feature_importance import ReadOnlyFeatureImportanceStore +from ppc_common.deps_services.serialize_type import SerializeType +import logging +import random +import sys +import pandas as pd + + +class FeatureImportanceWrapper: + def __init__(self, ut_obj, feature_size): + self.ut_obj = ut_obj + self.feature_size = feature_size + self.feature_list = [] + self._fake_features() + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + self.logger = logging.getLogger(__name__) + self.feature_importance_type_list = [ + FeatureImportanceType.GAIN, FeatureImportanceType.WEIGHT] + self.feature_importance_store = FeatureImportanceStore( + self.feature_importance_type_list, self.feature_list, self.logger) + self.epsilon = 0.000000001 + + def _fake_features(self): + for i in range(self.feature_size): + self.feature_list.append("feature_" + str(i)) + + def update_feature_importance_and_check(self, rounds, local_file_path): + for i in range(rounds): + selected_feature = random.randint(0, self.feature_size - 1) + key = self.feature_importance_store.generate_fid_key( + selected_feature) + gain = random.uniform(0, 101) + pre_gain = self.feature_importance_store.get_feature_importance( + key, FeatureImportanceType.GAIN) + pre_weight = self.feature_importance_store.get_feature_importance( + key, FeatureImportanceType.WEIGHT) + self.feature_importance_store.update_feature_importance( + selected_feature, {FeatureImportanceType.GAIN: gain, FeatureImportanceType.WEIGHT: 1}) + self.ut_obj.assertEqual( + pre_gain + gain, self.feature_importance_store.get_feature_importance(key, FeatureImportanceType.GAIN)) + self.ut_obj.assertEqual( + pre_weight + 1, self.feature_importance_store.get_feature_importance(key, FeatureImportanceType.WEIGHT)) + # store + self.feature_importance_store.store( + SerializeType.CSV, local_file_path, None, None) + # load + df = pd.read_csv(local_file_path) + loaded_feature_importance_store = ReadOnlyFeatureImportanceStore.load( + df, self.logger) + # check the dict + for importance_type in loaded_feature_importance_store.feature_importance_dict: + self.ut_obj.assertTrue( + importance_type in self.feature_importance_store.feature_importance_dict) + feature_importances = self.feature_importance_store.feature_importance_dict[ + importance_type] + loaded_feature_importances = loaded_feature_importance_store.feature_importance_dict[ + importance_type] + sum_data = 0 + for fid in loaded_feature_importances.keys(): + sum_data += loaded_feature_importances[fid].importance + self.ut_obj.assertTrue(abs(sum_data - 1) < self.epsilon) + local_json_file = local_file_path + ".json" + self.feature_importance_store.store( + SerializeType.JSON, local_json_file, None, None) + + +class TestFeatureImportance(unittest.TestCase): + def test_update_gain_and_store(self): + feature_size = 100 + self.wrapper = FeatureImportanceWrapper( + self, feature_size=feature_size) + rounds = 100000 + local_file_path = "feature_importance_case1.csv" + self.wrapper.update_feature_importance_and_check( + rounds, local_file_path=local_file_path) diff --git a/python/ppc_common/ppc_ml/model/algorithm_info.py b/python/ppc_common/ppc_ml/model/algorithm_info.py new file mode 100644 index 00000000..a117887d --- /dev/null +++ b/python/ppc_common/ppc_ml/model/algorithm_info.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from enum import Enum + + +class ClassificationType(Enum): + TWO = 'two' + MULTI = 'multi' + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class EvaluationType(Enum): + ROC = "roc", + PR = "pr", + KS = "ks", + ACCURACY = "accuracy", + CONFUSION_MATRIX = "confusion_matrix" + + +class ModelRole(Enum): + ACTIVE = "active" + PASSIVE = "passive" diff --git a/python/ppc_common/ppc_mock/__init__.py b/python/ppc_common/ppc_mock/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_mock/mock_objects.py b/python/ppc_common/ppc_mock/mock_objects.py new file mode 100644 index 00000000..4ac711e0 --- /dev/null +++ b/python/ppc_common/ppc_mock/mock_objects.py @@ -0,0 +1,59 @@ +from datetime import datetime +from typing import AnyStr + +from ppc_common.deps_services.storage_api import StorageApi + + +class MockLogger: + @staticmethod + def debug(msg): + current_time = datetime.now() + print(f"{current_time}, Debug: {msg}") + + @staticmethod + def info(msg): + current_time = datetime.now() + print(f"{current_time}, Info: {msg}") + + @staticmethod + def warn(msg): + current_time = datetime.now() + print(f"{current_time}, Warn: {msg}") + + @staticmethod + def error(msg): + current_time = datetime.now() + print(f"{current_time}, Error: {msg}") + + +class MockStorageClient(StorageApi): + + def download_file(self, storage_path: str, local_file_path: str, enable_cache=False): + print(f'download_file: {storage_path} -> {local_file_path}') + + def upload_file(self, local_file_path: str, storage_path: str): + print(f'upload_file: {storage_path} -> {local_file_path}') + + def make_file_path(self, storage_path: str): + pass + + def delete_file(self, storage_path: str): + pass + + def save_data(self, data: AnyStr, storage_path: str): + pass + + def get_data(self, storage_path: str) -> AnyStr: + pass + + def mkdir(self, storage_path: str): + pass + + def file_existed(self, storage_path: str) -> bool: + pass + + def file_rename(self, old_storage_path: str, storage_path: str): + pass + + def storage_type(self): + pass diff --git a/python/ppc_common/ppc_protos/__init__.py b/python/ppc_common/ppc_protos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_protos/generated/__init__.py b/python/ppc_common/ppc_protos/generated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py b/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py new file mode 100644 index 00000000..57f7ca3a --- /dev/null +++ b/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ppc_model.proto +# Protobuf Python Version: 4.25.3 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fppc_model.proto\x12\tppc.model\"|\n\x0cModelRequest\x12\x0e\n\x06sender\x18\x01 \x01(\t\x12\x10\n\x08receiver\x18\x02 \x01(\t\x12\x0f\n\x07task_id\x18\x03 \x01(\t\x12\x0b\n\x03key\x18\x04 \x01(\t\x12\x0b\n\x03seq\x18\x05 \x01(\x03\x12\x11\n\tslice_num\x18\x06 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x07 \x01(\x0c\"3\n\x0c\x42\x61seResponse\x12\x12\n\nerror_code\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"M\n\rModelResponse\x12.\n\rbase_response\x18\x01 \x01(\x0b\x32\x17.ppc.model.BaseResponse\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"#\n\rPlainBoolList\x12\x12\n\nplain_list\x18\x01 \x03(\x08\"\xb1\x01\n\rBestSplitInfo\x12\x0f\n\x07tree_id\x18\x01 \x01(\x03\x12\x0f\n\x07leaf_id\x18\x02 \x01(\x03\x12\x0f\n\x07\x66\x65\x61ture\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x03\x12\x12\n\nagency_idx\x18\x05 \x01(\x03\x12\x16\n\x0e\x61gency_feature\x18\x06 \x01(\x03\x12\x11\n\tbest_gain\x18\x07 \x01(\x02\x12\x0e\n\x06w_left\x18\x08 \x01(\x02\x12\x0f\n\x07w_right\x18\t \x01(\x02\"3\n\x0bModelCipher\x12\x12\n\nciphertext\x18\x01 \x01(\x0c\x12\x10\n\x08\x65xponent\x18\x02 \x01(\x0c\"M\n\nCipherList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12+\n\x0b\x63ipher_list\x18\x02 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"=\n\x0e\x43ipher1DimList\x12+\n\x0b\x63ipher_list\x18\x01 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"W\n\x0e\x43ipher2DimList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\x31\n\x0e\x63ipher_1d_list\x18\x02 \x03(\x0b\x32\x19.ppc.model.Cipher1DimList\"_\n\rEncAggrLabels\x12\r\n\x05\x66ield\x18\x01 \x01(\t\x12\x12\n\ncount_list\x18\x02 \x03(\x03\x12+\n\x0b\x63ipher_list\x18\x03 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"_\n\x11\x45ncAggrLabelsList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\x36\n\x14\x65nc_aggr_labels_list\x18\x02 \x03(\x0b\x32\x18.ppc.model.EncAggrLabels\"/\n\x10IterationRequest\x12\r\n\x05\x65poch\x18\x01 \x01(\x03\x12\x0c\n\x04stop\x18\x02 \x01(\x08\x32Y\n\x0cModelService\x12I\n\x12MessageInteraction\x12\x17.ppc.model.ModelRequest\x1a\x18.ppc.model.ModelResponse\"\x00\x42\x08P\x01\xa2\x02\x03PPCb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ppc_model_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'P\001\242\002\003PPC' + _globals['_MODELREQUEST']._serialized_start=30 + _globals['_MODELREQUEST']._serialized_end=154 + _globals['_BASERESPONSE']._serialized_start=156 + _globals['_BASERESPONSE']._serialized_end=207 + _globals['_MODELRESPONSE']._serialized_start=209 + _globals['_MODELRESPONSE']._serialized_end=286 + _globals['_PLAINBOOLLIST']._serialized_start=288 + _globals['_PLAINBOOLLIST']._serialized_end=323 + _globals['_BESTSPLITINFO']._serialized_start=326 + _globals['_BESTSPLITINFO']._serialized_end=503 + _globals['_MODELCIPHER']._serialized_start=505 + _globals['_MODELCIPHER']._serialized_end=556 + _globals['_CIPHERLIST']._serialized_start=558 + _globals['_CIPHERLIST']._serialized_end=635 + _globals['_CIPHER1DIMLIST']._serialized_start=637 + _globals['_CIPHER1DIMLIST']._serialized_end=698 + _globals['_CIPHER2DIMLIST']._serialized_start=700 + _globals['_CIPHER2DIMLIST']._serialized_end=787 + _globals['_ENCAGGRLABELS']._serialized_start=789 + _globals['_ENCAGGRLABELS']._serialized_end=884 + _globals['_ENCAGGRLABELSLIST']._serialized_start=886 + _globals['_ENCAGGRLABELSLIST']._serialized_end=981 + _globals['_ITERATIONREQUEST']._serialized_start=983 + _globals['_ITERATIONREQUEST']._serialized_end=1030 + _globals['_MODELSERVICE']._serialized_start=1032 + _globals['_MODELSERVICE']._serialized_end=1121 +# @@protoc_insertion_point(module_scope) diff --git a/python/ppc_common/ppc_protos/generated/ppc_model_pb2_grpc.py b/python/ppc_common/ppc_protos/generated/ppc_model_pb2_grpc.py new file mode 100644 index 00000000..e56168ae --- /dev/null +++ b/python/ppc_common/ppc_protos/generated/ppc_model_pb2_grpc.py @@ -0,0 +1,67 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from ppc_common.ppc_protos.generated import ppc_model_pb2 as ppc__model__pb2 + + +class ModelServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.MessageInteraction = channel.unary_unary( + '/ppc.model.ModelService/MessageInteraction', + request_serializer=ppc__model__pb2.ModelRequest.SerializeToString, + response_deserializer=ppc__model__pb2.ModelResponse.FromString, + ) + + +class ModelServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def MessageInteraction(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ModelServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'MessageInteraction': grpc.unary_unary_rpc_method_handler( + servicer.MessageInteraction, + request_deserializer=ppc__model__pb2.ModelRequest.FromString, + response_serializer=ppc__model__pb2.ModelResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'ppc.model.ModelService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + # This class is part of an EXPERIMENTAL API. + + +class ModelService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def MessageInteraction(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/ppc.model.ModelService/MessageInteraction', + ppc__model__pb2.ModelRequest.SerializeToString, + ppc__model__pb2.ModelResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/ppc_common/ppc_protos/generated/ppc_pb2.py b/python/ppc_common/ppc_protos/generated/ppc_pb2.py new file mode 100644 index 00000000..5c8200ef --- /dev/null +++ b/python/ppc_common/ppc_protos/generated/ppc_pb2.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ppc.proto +# Protobuf Python Version: 4.25.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tppc.proto\x12\tppc.proto\"\xe5\x01\n\x0f\x41lgorithmDetail\x12\x1d\n\x15\x61lgorithm_description\x18\x01 \x01(\t\x12\x16\n\x0e\x61lgorithm_type\x18\x02 \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x03 \x01(\x03\x12\x13\n\x0bupdate_time\x18\x04 \x01(\x03\x12\x17\n\x0fowner_user_name\x18\x05 \x01(\t\x12\x17\n\x0fowner_agency_id\x18\x06 \x01(\t\x12\x19\n\x11owner_agency_name\x18\x07 \x01(\t\x12$\n\x08ppc_flow\x18\x08 \x01(\x0b\x32\x12.ppc.proto.PpcFlow\"\xc1\x01\n\x07PpcFlow\x12\x13\n\x0binput_count\x18\x01 \x01(\t\x12\x12\n\nsql_module\x18\x02 \x01(\t\x12\x12\n\nmpc_module\x18\x03 \x01(\t\x12\x18\n\x10mpc_model_module\x18\x04 \x01(\t\x12\x14\n\x0cmatch_module\x18\x05 \x01(\t\x12\x19\n\x11\x61lgorithm_subtype\x18\x06 \x01(\t\x12\x1a\n\x12participant_agency\x18\x07 \x01(\t\x12\x12\n\nmodel_type\x18\x08 \x01(\t\"\xee\x01\n\rDatasetDetail\x12\x15\n\rdataset_title\x18\x01 \x01(\t\x12\x1b\n\x13\x64\x61taset_description\x18\x02 \x01(\t\x12\x14\n\x0c\x64\x61taset_hash\x18\x03 \x01(\t\x12\x14\n\x0c\x63olumn_count\x18\x04 \x01(\x03\x12\x11\n\trow_count\x18\x05 \x01(\x03\x12\x13\n\x0b\x63reate_time\x18\x06 \x01(\x03\x12\x11\n\tuser_name\x18\x07 \x01(\t\x12\x17\n\x0f\x64\x61ta_field_list\x18\x08 \x01(\t\x12\x14\n\x0c\x64\x61taset_size\x18\t \x01(\x03\x12\x13\n\x0bupdate_time\x18\n \x01(\x03\"K\n\x0f\x44\x61tasetAuthData\x12\x0f\n\x07is_auth\x18\x01 \x01(\x08\x12\x13\n\x0b\x63reate_time\x18\x02 \x01(\x03\x12\x12\n\nvalid_time\x18\x03 \x01(\x03\"\xde\x01\n\x12jobDatasetProvider\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x13\n\x0b\x61gency_name\x18\x02 \x01(\t\x12\x12\n\ndataset_id\x18\x03 \x01(\t\x12\x15\n\rdataset_title\x18\x04 \x01(\t\x12\x15\n\rloading_index\x18\x05 \x01(\x03\x12\x1b\n\x13\x64\x61taset_description\x18\x06 \x01(\t\x12\x17\n\x0fowner_user_name\x18\x07 \x01(\t\x12\x14\n\x0c\x63olumns_size\x18\x08 \x01(\x03\x12\x12\n\nis_instant\x18\t \x01(\x03\"\x94\x01\n\x15\x64\x61tasourceInstantInfo\x12\r\n\x05\x64\x62_ip\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x62_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x62_password\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x62_port\x18\x04 \x01(\x03\x12\x0e\n\x06\x64\x62_sql\x18\x05 \x01(\t\x12\x14\n\x0c\x64\x62_user_name\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x62_type\x18\x07 \x01(\t\"I\n\x16jobDatasetProviderList\x12/\n\x08provider\x18\x01 \x03(\x0b\x32\x1d.ppc.proto.jobDatasetProvider\"m\n\x16jobComputationProvider\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x13\n\x0b\x61gency_name\x18\x02 \x01(\t\x12\x12\n\nagency_url\x18\x03 \x01(\t\x12\x17\n\x0f\x63omputing_index\x18\x04 \x01(\x03\"Q\n\x1ajobComputationProviderList\x12\x33\n\x08provider\x18\x01 \x03(\x0b\x32!.ppc.proto.jobComputationProvider\")\n\x15jobResultReceiverList\x12\x10\n\x08receiver\x18\x01 \x03(\t\"\x8f\x05\n\x08JobEvent\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tjob_title\x18\x02 \x01(\t\x12\x17\n\x0fjob_description\x18\x03 \x01(\t\x12\x14\n\x0cjob_priority\x18\x04 \x01(\x03\x12\x13\n\x0bjob_creator\x18\x05 \x01(\t\x12\x1b\n\x13initiator_agency_id\x18\x06 \x01(\t\x12\x1d\n\x15initiator_agency_name\x18\x07 \x01(\t\x12\x1b\n\x13initiator_signature\x18\x08 \x01(\t\x12\x18\n\x10job_algorithm_id\x18\t \x01(\t\x12\x1b\n\x13job_algorithm_title\x18\n \x01(\t\x12\x1d\n\x15job_algorithm_version\x18\x0b \x01(\t\x12\x1a\n\x12job_algorithm_type\x18\x0c \x01(\t\x12$\n\x1cjob_dataset_provider_list_pb\x18\r \x01(\t\x12#\n\x1bjob_result_receiver_list_pb\x18\x0e \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x0f \x01(\x03\x12\x13\n\x0bupdate_time\x18\x10 \x01(\x03\x12\x12\n\npsi_fields\x18\x11 \x01(\t\x12\x1d\n\x15job_algorithm_subtype\x18\x12 \x01(\t\x12\x14\n\x0cmatch_fields\x18\x13 \x01(\t\x12\x16\n\x0eis_cem_encrypt\x18\x14 \x01(\x08\x12\x17\n\x0f\x64\x61taset_id_list\x18\x15 \x03(\t\x12\x30\n\x0e\x64\x61taset_detail\x18\x16 \x01(\x0b\x32\x18.ppc.proto.DatasetDetail\x12\x1e\n\x16tag_provider_agency_id\x18\x17 \x01(\t\x12\x10\n\x08job_type\x18\x18 \x01(\t\"/\n\tAuditItem\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"5\n\tAuditData\x12(\n\naudit_item\x18\x01 \x03(\x0b\x32\x14.ppc.proto.AuditItem\"E\n\x10JobOutputPreview\x12\x0e\n\x06header\x18\x01 \x03(\t\x12!\n\x04line\x18\x02 \x03(\x0b\x32\x13.ppc.proto.DataLine\"\x19\n\x08\x44\x61taLine\x12\r\n\x05value\x18\x01 \x03(\t\"=\n\x0eInputStatement\x12\x15\n\rupstream_unit\x18\x01 \x01(\t\x12\x14\n\x0coutput_index\x18\x02 \x01(\x03\"M\n\x16JobUnitInputsStatement\x12\x33\n\x10inputs_statement\x18\x01 \x03(\x0b\x32\x19.ppc.proto.InputStatement\"!\n\x0eJobUnitOutputs\x12\x0f\n\x07outputs\x18\x01 \x03(\t\")\n\x0fJobUnitUpstream\x12\x16\n\x0eupstream_units\x18\x01 \x03(\t\"<\n\tAlgorithm\x12\x14\n\x0c\x61lgorithm_id\x18\x01 \x01(\t\x12\x19\n\x11\x61lgorithm_version\x18\x02 \x01(\t\"6\n\nAlgorithms\x12(\n\nalgorithms\x18\x01 \x03(\x0b\x32\x14.ppc.proto.Algorithmb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ppc_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_ALGORITHMDETAIL']._serialized_start = 25 + _globals['_ALGORITHMDETAIL']._serialized_end = 254 + _globals['_PPCFLOW']._serialized_start = 257 + _globals['_PPCFLOW']._serialized_end = 450 + _globals['_DATASETDETAIL']._serialized_start = 453 + _globals['_DATASETDETAIL']._serialized_end = 691 + _globals['_DATASETAUTHDATA']._serialized_start = 693 + _globals['_DATASETAUTHDATA']._serialized_end = 768 + _globals['_JOBDATASETPROVIDER']._serialized_start = 771 + _globals['_JOBDATASETPROVIDER']._serialized_end = 993 + _globals['_DATASOURCEINSTANTINFO']._serialized_start = 996 + _globals['_DATASOURCEINSTANTINFO']._serialized_end = 1144 + _globals['_JOBDATASETPROVIDERLIST']._serialized_start = 1146 + _globals['_JOBDATASETPROVIDERLIST']._serialized_end = 1219 + _globals['_JOBCOMPUTATIONPROVIDER']._serialized_start = 1221 + _globals['_JOBCOMPUTATIONPROVIDER']._serialized_end = 1330 + _globals['_JOBCOMPUTATIONPROVIDERLIST']._serialized_start = 1332 + _globals['_JOBCOMPUTATIONPROVIDERLIST']._serialized_end = 1413 + _globals['_JOBRESULTRECEIVERLIST']._serialized_start = 1415 + _globals['_JOBRESULTRECEIVERLIST']._serialized_end = 1456 + _globals['_JOBEVENT']._serialized_start = 1459 + _globals['_JOBEVENT']._serialized_end = 2114 + _globals['_AUDITITEM']._serialized_start = 2116 + _globals['_AUDITITEM']._serialized_end = 2163 + _globals['_AUDITDATA']._serialized_start = 2165 + _globals['_AUDITDATA']._serialized_end = 2218 + _globals['_JOBOUTPUTPREVIEW']._serialized_start = 2220 + _globals['_JOBOUTPUTPREVIEW']._serialized_end = 2289 + _globals['_DATALINE']._serialized_start = 2291 + _globals['_DATALINE']._serialized_end = 2316 + _globals['_INPUTSTATEMENT']._serialized_start = 2318 + _globals['_INPUTSTATEMENT']._serialized_end = 2379 + _globals['_JOBUNITINPUTSSTATEMENT']._serialized_start = 2381 + _globals['_JOBUNITINPUTSSTATEMENT']._serialized_end = 2458 + _globals['_JOBUNITOUTPUTS']._serialized_start = 2460 + _globals['_JOBUNITOUTPUTS']._serialized_end = 2493 + _globals['_JOBUNITUPSTREAM']._serialized_start = 2495 + _globals['_JOBUNITUPSTREAM']._serialized_end = 2536 + _globals['_ALGORITHM']._serialized_start = 2538 + _globals['_ALGORITHM']._serialized_end = 2598 + _globals['_ALGORITHMS']._serialized_start = 2600 + _globals['_ALGORITHMS']._serialized_end = 2654 +# @@protoc_insertion_point(module_scope) diff --git a/python/ppc_common/ppc_protos/ppc.proto b/python/ppc_common/ppc_protos/ppc.proto new file mode 100644 index 00000000..a9571c0f --- /dev/null +++ b/python/ppc_common/ppc_protos/ppc.proto @@ -0,0 +1,160 @@ +syntax = "proto3"; + +package ppc.proto; + +message AlgorithmDetail { + string algorithm_description = 1; + string algorithm_type = 2; + int64 create_time = 3; + int64 update_time = 4; + string owner_user_name = 5; + string owner_agency_id = 6; + string owner_agency_name = 7; + PpcFlow ppc_flow = 8; +} + +message PpcFlow { + // input_count need present '3+' + string input_count = 1; + string sql_module = 2; + string mpc_module = 3; + string mpc_model_module = 4; + string match_module = 5; + string algorithm_subtype = 6; + string participant_agency = 7; + string model_type = 8; +} + +message DatasetDetail { + string dataset_title = 1; + string dataset_description = 2; + string dataset_hash = 3; + int64 column_count = 4; + int64 row_count = 5; + int64 create_time = 6; + string user_name = 7; + string data_field_list = 8; + int64 dataset_size = 9; + int64 update_time = 10; +} + +message DatasetAuthData { + bool is_auth = 1; + int64 create_time = 2; + int64 valid_time = 3; +} + + +message jobDatasetProvider { + string agency_id = 1; + string agency_name = 2; + string dataset_id = 3; + string dataset_title = 4; + int64 loading_index = 5; + string dataset_description = 6; + string owner_user_name = 7; + int64 columns_size = 8; + int64 is_instant = 9; +} + +message datasourceInstantInfo { + string db_ip = 1; + string db_name = 2; + string db_password = 3; + int64 db_port = 4; + string db_sql = 5; + string db_user_name = 6; + string db_type = 7; +} + +message jobDatasetProviderList { + repeated jobDatasetProvider provider = 1; +} + +message jobComputationProvider { + string agency_id = 1; + string agency_name = 2; + string agency_url = 3; + int64 computing_index = 4; +} + +message jobComputationProviderList { + repeated jobComputationProvider provider = 1; +} + +message jobResultReceiverList { + repeated string receiver = 1; +} + + +message JobEvent { + string job_id = 1; + string job_title = 2; + string job_description = 3; + int64 job_priority = 4; + string job_creator = 5; + string initiator_agency_id = 6; + string initiator_agency_name = 7; + string initiator_signature = 8; + string job_algorithm_id = 9; + string job_algorithm_title = 10; + string job_algorithm_version = 11; + string job_algorithm_type = 12; + string job_dataset_provider_list_pb = 13; // jobDatasetProviderList + string job_result_receiver_list_pb = 14; // jobResultReceiverList + int64 create_time = 15; + int64 update_time = 16; + string psi_fields = 17; + string job_algorithm_subtype = 18; + string match_fields = 19; + bool is_cem_encrypt = 20; + repeated string dataset_id_list = 21; // cem encrypt dataset id without username prefix + DatasetDetail dataset_detail = 22; // cem encrypt dataset transfer + string tag_provider_agency_id = 23; + string job_type = 24; +} + +message AuditItem { + string description = 1; + string value = 2; +} + +message AuditData { + repeated AuditItem audit_item = 1; +} + +message JobOutputPreview { + repeated string header = 1; + repeated DataLine line = 2; +} + +message DataLine { + repeated string value = 1; +} + +message InputStatement { + string upstream_unit = 1; + int64 output_index = 2; +} + +message JobUnitInputsStatement { + repeated InputStatement inputs_statement = 1; +} + +message JobUnitOutputs { + repeated string outputs = 1; +} + +message JobUnitUpstream { + repeated string upstream_units = 1; +} + +message Algorithm { + string algorithm_id = 1; + string algorithm_version = 2; +} + +message Algorithms { + repeated Algorithm algorithms = 1; +} + diff --git a/python/ppc_common/ppc_protos/ppc_model.proto b/python/ppc_common/ppc_protos/ppc_model.proto new file mode 100644 index 00000000..ac1d72f1 --- /dev/null +++ b/python/ppc_common/ppc_protos/ppc_model.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +option java_multiple_files = true; +//option java_package = "unknown"; +//option java_outer_classname = "unknown"; +option objc_class_prefix = "PPC"; + +package ppc.model; + +service ModelService { + rpc MessageInteraction (ModelRequest) returns (ModelResponse) {} +} + +message ModelRequest { + string sender = 1; + string receiver = 2; + string task_id = 3; + string key = 4; + int64 seq = 5; + int64 slice_num = 6; + bytes data = 7; +} + +message BaseResponse { + int64 error_code = 1; + string message = 2; +} + +message ModelResponse { + BaseResponse base_response = 1; + bytes data = 2; +} + +message PlainBoolList{ + repeated bool plain_list = 1; +} + +message BestSplitInfo{ + int64 tree_id = 1; + int64 leaf_id = 2; + int64 feature = 3; + int64 value = 4; + int64 agency_idx = 5; + int64 agency_feature = 6; + float best_gain = 7; + float w_left = 8; + float w_right = 9; +} + +message ModelCipher { + bytes ciphertext = 1; + bytes exponent = 2; +} + +message CipherList { + bytes public_key = 1; + repeated ModelCipher cipher_list = 2; +} + +message Cipher1DimList { + repeated ModelCipher cipher_list = 1; +} + +message Cipher2DimList { + bytes public_key = 1; + repeated Cipher1DimList cipher_1d_list = 2; +} + +message EncAggrLabels { + string field = 1; + repeated int64 count_list = 2; + repeated ModelCipher cipher_list = 3; +} + +message EncAggrLabelsList { + bytes public_key = 1; + repeated EncAggrLabels enc_aggr_labels_list = 2; +} + +message IterationRequest { + int64 epoch = 1; + bool stop = 2; +} diff --git a/python/ppc_common/ppc_utils/__init__.py b/python/ppc_common/ppc_utils/__init__.py new file mode 100644 index 00000000..8d7ee87a --- /dev/null +++ b/python/ppc_common/ppc_utils/__init__.py @@ -0,0 +1 @@ +__all__ = ['exception', 'path', 'permission', 'utils', 'common_func'] diff --git a/python/ppc_common/ppc_utils/anonymous_search.py b/python/ppc_common/ppc_utils/anonymous_search.py new file mode 100644 index 00000000..80c3df61 --- /dev/null +++ b/python/ppc_common/ppc_utils/anonymous_search.py @@ -0,0 +1,420 @@ +import hashlib +import json +import logging +import os +import random +import string +import unittest +import uuid +from io import BytesIO + +import pandas as pd + +from ppc_common.ppc_utils import utils, http_utils +from ppc_common.ppc_crypto import crypto_utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + +log = logging.getLogger(__name__) + + +def make_hash(data): + m = hashlib.sha3_256() + m.update(data) + return m.hexdigest() + + +def requester_gen_ot_cipher(id_list, obfuscation_order): + blinding_a = crypto_utils.get_random_int() + blinding_b = crypto_utils.get_random_int() + x_value = crypto_utils.ot_base_pown(blinding_a) + y_value = crypto_utils.ot_base_pown(blinding_b) + c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) + id_index_list = [] + send_hash_vec = [] + z_value_list = [] + for id_hash in id_list: + obs_list = id_obfuscation(obfuscation_order, None) + id_index = random.randint(0, obfuscation_order) + obs_list[id_index] = id_hash + id_index_list.append(id_index) + send_hash_vec.append(obs_list) + z_value = crypto_utils.ot_base_pown(c_blinding - id_index) + z_value_list.append(str(z_value)) + return id_index_list, blinding_b, x_value, y_value, send_hash_vec, z_value_list + + +def provider_gen_ot_cipher(x_value, y_value, send_hash_vec, z_value_list, data_map, is_contain_result=True): + """ + data_map = hashmap[id1: "message1", + id2: "message2"] + """ + if isinstance(x_value, str): + x_value = int(x_value) + if isinstance(y_value, str): + y_value = int(y_value) + if len(send_hash_vec) != len(z_value_list): + raise PpcException(PpcErrorCode.AYS_LENGTH_ERROR.get_code(), + PpcErrorCode.AYS_LENGTH_ERROR.get_msg()) + message_cipher_vec = [] + for idx, z_value in enumerate(z_value_list): + if isinstance(z_value, str): + z_value = int(z_value) + message_cipher_list = [] + message_int_len = 0 + for send_hash in send_hash_vec[idx]: + blinding_r = crypto_utils.get_random_int() + blinding_s = crypto_utils.get_random_int() + w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), + crypto_utils.ot_base_pown(blinding_r)) + key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), + crypto_utils.ot_pown(y_value, blinding_r)) + aes_key_bytes = os.urandom(16) + aes_default = utils.AESCipher(aes_key_bytes) + aes_key_base64str = utils.bytes_to_base64str(aes_key_bytes) + message_int, message_int_len = crypto_utils.ot_str_to_int( + aes_key_base64str) + + if send_hash not in data_map: + letters = string.ascii_lowercase + # random string + ot_message_str = 'message not found' + cipher = aes_default.encrypt(ot_message_str) + cipher_str = utils.bytes_to_base64str(cipher) + else: + if is_contain_result: + ot_message_str = json.dumps(data_map[send_hash]) + cipher = aes_default.encrypt(ot_message_str) + cipher_str = utils.bytes_to_base64str(cipher) + else: + ot_message_str = 'True' + cipher = aes_default.encrypt(ot_message_str) + cipher_str = utils.bytes_to_base64str(cipher) + message_cipher = key_value ^ message_int + z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) + message_cipher_list.append({ + "w": str(w_value), + "e": str(message_cipher), + "len": message_int_len, + 'aesCipher': cipher_str, + }) + message_cipher_vec.append(message_cipher_list) + return message_cipher_vec + + +def requester_ot_recover_result(id_index_list, blinding_b, message_cipher_vec): + if len(id_index_list) != len(message_cipher_vec): + raise PpcException(PpcErrorCode.AYS_LENGTH_ERROR.get_code(), + PpcErrorCode.AYS_LENGTH_ERROR.get_msg()) + result_list = [] + for idx, id_index in enumerate(id_index_list): + w_value = message_cipher_vec[idx][id_index]['w'] + e_value = message_cipher_vec[idx][id_index]['e'] + message_len = message_cipher_vec[idx][id_index]['len'] + cipher_str = message_cipher_vec[idx][id_index]['aesCipher'] + + if isinstance(w_value, str): + w_value = int(w_value) + if isinstance(e_value, str): + e_value = int(e_value) + w_1 = crypto_utils.ot_pown(w_value, blinding_b) + message_recover = w_1 ^ e_value + try: + aes_key = crypto_utils.ot_int_to_str(message_recover, message_len) + key_recover = utils.base64str_to_bytes(aes_key) + aes_recover = utils.AESCipher(key_recover) + cipher_recover = utils.base64str_to_bytes(cipher_str) + message_result = aes_recover.decrypt(cipher_recover) + result_list.append(message_result) + except Exception as be: + result_list.append(None) + return result_list + + +def id_obfuscation(obfuscation_order, rule=None): + # use rule extend different id order, such as driver card or name: + if rule is not None: + print('obfuscation is work in progress') + obs_list = [] + for i in range(0, obfuscation_order+1): + obs_list.append(make_hash(bytes(str(uuid.uuid4()), 'utf8'))) + return obs_list + + +def prepare_dataset_with_matrix(search_id_matrix, tmp_file_path, prepare_dataset_tmp_file_path): + search_reg = 'ppc-normal-prefix' + for search_list in search_id_matrix: + for search_id in search_list: + search_reg = '{}|{}'.format(search_reg, search_id) + exec_command = 'head -n 1 {} >> {}'.format( + tmp_file_path, prepare_dataset_tmp_file_path) + (status, result) = utils.getstatusoutput(exec_command) + if status != 0: + log.error( + f'[OnError]prepare_dataset_with_matrix! status is {status}, output is {result}') + else: + log.info( + f'prepare_dataset_with_matrix success! status is {status}, output is {result}') + exec_command = 'grep -E \'{}\' {} >> {}'.format( + search_reg, tmp_file_path, prepare_dataset_tmp_file_path) + (status, result) = utils.getstatusoutput(exec_command) + if status != 0: + log.error( + f'[OnError]prepare_dataset_with_matrix! status is {status}, output is {result}') + else: + log.info( + f'prepare_dataset_with_matrix success! status is {status}, output is {result}') + + +class TestOtMethods(unittest.TestCase): + + def test_choice_all_flow(self): + file_path = '/Users/asher/Downloads/test_file.csv' + # data_pd = pd.read_csv(file_path, index_col=0, header=0) + data_pd = pd.read_csv(file_path) + data_map = data_pd.set_index(data_pd.columns[0]).T.to_dict('list') + + choice = ['bob', '小鸡', '美丽'] + obs_order = 10 + _id_index_list, _blinding_b, _x_value, _y_value, _send_hash_vec, _z_value_list = requester_gen_ot_cipher( + choice, obs_order) + _message_cipher_vec = provider_gen_ot_cipher( + _x_value, _y_value, _send_hash_vec, _z_value_list, data_map) + result = requester_ot_recover_result( + _id_index_list, _blinding_b, _message_cipher_vec) + for idx, id_num in enumerate(choice): + print(f"found {id_num} value is {result[idx]}") + + # def test_choice_ot(self): + # choice_list = [1, 4] + # blinding_a = crypto_utils.get_random_int() + # blinding_b = crypto_utils.get_random_int() + # x_value = crypto_utils.ot_base_pown(blinding_a) + # y_value = crypto_utils.ot_base_pown(blinding_b) + # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) + # # c_value = crypto_utils.ot_base_pown(c_blinding) + # z_value_list = [] + # for choice in choice_list: + # z_value = crypto_utils.ot_base_pown(c_blinding - choice) + # z_value_list.append(z_value) + # # send x_value, y_value, z_value + # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] + # message_list = [] + # for message_str in message_str_list: + # message_int, message_int_len = crypto_utils.ot_str_to_int(message_str) + # message_list.append(message_int) + # # message_list = [111111, 222222, 333333, 444444, 555555] + # cipher_vec = [] + # for z_value in z_value_list: + # cipher_list = [] + # for message in message_list: + # blinding_r = crypto_utils.get_random_int() + # blinding_s = crypto_utils.get_random_int() + # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), + # crypto_utils.ot_base_pown(blinding_r)) + # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), + # crypto_utils.ot_pown(y_value, blinding_r)) + # z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) + # e_cipher = key_value ^ message + # cipher_list.append({ + # "w": w_value, + # "e": e_cipher + # }) + # cipher_vec.append(cipher_list) + # + # for cipher_each in cipher_vec: + # for cipher in cipher_each: + # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) + # message_recover = w_1 ^ cipher['e'] + # print(message_recover) + # for idx, cipher_list in enumerate(cipher_vec): + # w_1 = crypto_utils.ot_pown(cipher_list[choice_list[idx]]['w'], blinding_b) + # message_recover = w_1 ^ cipher_list[choice_list[idx]]['e'] + # s = crypto_utils.ot_int_to_str(message_recover) + # print(s) + + # def test_id_ot(self): + # print("test_id_ot") + # choice_id_list = [crypto_utils.ot_str_to_int('小明'), crypto_utils.ot_str_to_int('张三')] + # blinding_a = crypto_utils.get_random_int() + # blinding_b = crypto_utils.get_random_int() + # x_value = crypto_utils.ot_base_pown(blinding_a) + # y_value = crypto_utils.ot_base_pown(blinding_b) + # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) + # # c_value = crypto_utils.ot_base_pown(c_blinding) + # z_value_list = [] + # for choice in choice_id_list: + # z_value = crypto_utils.ot_base_pown(c_blinding - choice) + # z_value_list.append(z_value) + # # z_value = crypto_utils.ot_base_pown(c_blinding - choice_id) + # # send x_value, y_value, z_value + # id_str_list = ['小往', '小明', 'asher', 'set', '张三'] + # id_list = [] + # for id_str in id_str_list: + # id_list.append(crypto_utils.ot_str_to_int(id_str)) + # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] + # message_list = [] + # for message_str in message_str_list: + # message_list.append(crypto_utils.ot_str_to_int(message_str)) + # # message_list = [111111, 222222, 333333, 444444, 555555] + # cipher_vec = [] + # for z_value in z_value_list: + # cipher_list = [] + # for idx, message in enumerate(message_list): + # blinding_r = crypto_utils.get_random_int() + # blinding_s = crypto_utils.get_random_int() + # z_value_use = crypto_utils.ot_mul_n(z_value, crypto_utils.ot_base_pown(id_list[idx])) + # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), + # crypto_utils.ot_base_pown(blinding_r)) + # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value_use, blinding_s), + # crypto_utils.ot_pown(y_value, blinding_r)) + # e_cipher = key_value ^ message + # cipher_list.append({ + # "w": w_value, + # "e": e_cipher + # }) + # cipher_vec.append(cipher_list) + # + # for idx, cipher_list in enumerate(cipher_vec): + # for now_idx, cipher in enumerate(cipher_list): + # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) + # message_recover = w_1 ^ cipher['e'] + # # print(message_recover) + # # print(idx) + # # print(now_idx) + # if (idx == 0 and now_idx == 1) or (idx == 1 and now_idx == 4): + # s = crypto_utils.ot_int_to_str(message_recover) + # print(s) + + def test_pd_with_multi_index(self): + print(True) + df = pd.DataFrame( + [[21, 'Amol', 72, 67], + [23, 'Lini', 78, 69], + [32, 'Kiku', 74, 56], + [52, 'Ajit', 54, 76], + [53, 'Ajit', 55, 78] + ], + columns=['rollno', 'name', 'physics', 'botony']) + + print('DataFrame with default index\n', df) + # set multiple columns as index + # df_map = df.set_index('name').T.to_dict('list') + # df_map = df.set_index('name').groupby(level=0).apply(lambda x: x.to_dict('r')).to_dict() + df_map = df.set_index('name').groupby(level=0).apply( + lambda x: x.to_dict('r')).to_dict() + print(df_map) + print(json.dumps(df_map['Ajit'])) + print(type(json.dumps(df_map['Kiku']))) + + def test_prepare_dataset_with_matrix(self): + tmp_file_path = "/Users/asher/Desktop/数据集2021/8_1_100w.csv" + prepare_dataset_tmp_file_path = "/Users/asher/Desktop/数据集2021/pre-test_100.csv" + search_id_matrix = [['645515750175253924', '779808531920530393'], [ + '399352968694137676', '399352968694137676222', '399352968694137']] + prepare_dataset_with_matrix( + search_id_matrix, tmp_file_path, prepare_dataset_tmp_file_path) + + # + # def test_choice_ot_multi(self): + # choice_list = [1, 2, 4] + # blinding_a = crypto_utils.get_random_int() + # blinding_b = crypto_utils.get_random_int() + # x_value = crypto_utils.ot_base_pown(blinding_a) + # y_value = crypto_utils.ot_base_pown(blinding_b) + # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) + # # c_value = crypto_utils.ot_base_pown(c_blinding) + # choice_final = 0 + # for choice in choice_list: + # choice_final = choice_final + choice + # z_value = crypto_utils.ot_base_pown(c_blinding - choice_final) + # # send x_value, y_value, z_value + # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] + # message_list = [] + # for message_str in message_str_list: + # message_list.append(crypto_utils.ot_str_to_int(message_str)) + # # message_list = [111111, 222222, 333333, 444444, 555555] + # cipher_list = [] + # for message in message_list: + # blinding_r = crypto_utils.get_random_int() + # blinding_s = crypto_utils.get_random_int() + # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), + # crypto_utils.ot_base_pown(blinding_r)) + # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), + # crypto_utils.ot_pown(y_value, blinding_r)) + # z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) + # e_cipher = key_value ^ message + # cipher_list.append({ + # "w": w_value, + # "e": e_cipher + # }) + # + # # for cipher in cipher_list: + # # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) + # # message_recover = w_1 ^ cipher['e'] + # # print(message_recover) + # + # for choice in choice_list: + # w_1 = crypto_utils.ot_pown(cipher_list[choice]['w'], blinding_b) + # for item in choice_list: + # if choice == item: + # continue + # else: + # base_g = crypto_utils.ot_base_pown(-item) + # w_1 = crypto_utils.ot_mul_n(w_1, base_g) + # message_recover = w_1 ^ cipher_list[choice]['e'] + # print(message_recover) + # # s = crypto_utils.ot_int_to_str(message_recover) + # # print(s) + # + + +# if __name__ == '__main__': + + # print(True) + # df = pd.DataFrame( + # [[21, 'Amol', 72, 67], + # [23, 'Lini', 78, 69], + # [32, 'Kiku', 74, 56], + # [52, 'Ajit', 54, 76]], + # columns=['rollno', 'name', 'physics', 'botony']) + # + # print('DataFrame with default index\n', df) + # # set multiple columns as index + # df = df.set_index(['rollno', 'name']) + # + # print('\nDataFrame with MultiIndex\n', df) + # point1 = point.base() + # print(point.base(scalar1).hex()) + + # json_response = "{\"id\": {\"0\": \"67b176705b46206614219f47a05aee7ae6a3edbe850bbbe214c536b989aea4d2\", \"1\": \"b1b1bd1ed240b1496c81ccf19ceccf2af6fd24fac10ae42023628abbe2687310\"}, \"x0\": {\"0\": 10, \"1\": 20}, \"x1\": {\"0\": 11, \"1\": 22}}" + # json_dict = json.loads(json_response) + # # json_pd = pd.json_normalize(json_dict) + # print(json_dict) + + # hex_str = utils.make_hash(bytes(str(796443), 'utf8'), CryptoType.ECDSA, HashType.HEXSTR) + # print(hex_str) + # csv_path1 = '/Users/asher/Downloads/UseCase120/usecase120_party1.csv' + # csv_path2 = '/Users/asher/Downloads/UseCase120/usecase120_party2.csv' + # data = pd.read_csv(csv_path1) + # if 'id' in data.columns.values: + # duplicated_list = data.duplicated('id', False).tolist() + # if True in duplicated_list: + # log.error(f"[OnError]id duplicated, check csv file") + # raise PpcException(PpcErrorCode.DATASET_CSV_ERROR.get_code(), + # PpcErrorCode.DATASET_CSV_ERROR.get_msg()) + # start = time.time() + # get_pd_file_with_hash_requester(csv_path1, f'{csv_path1}-pre', 2) + # start2 = time.time() + # print(f"requester prepare time is {start2 - start}s") + # + # get_pd_file_with_hash_data_provider(csv_path2, f'{csv_path2}-pre') + # start3 = time.time() + # print(f"provider prepare time is {start3 - start2}s") + # provider_output_path = '/Users/asher/Downloads/UseCase120/output.csv' + # get_anonymous_data(f'{csv_path1}-pre-requester', f'{csv_path2}-pre', provider_output_path) + # start4 = time.time() + # print(f"provider compute time is {start4 - start3}s") + # result_path = '/Users/asher/Downloads/UseCase120/result.csv' + # recover_result_data(f'{csv_path1}-pre', '/Users/asher/Downloads/UseCase120/output.csv', result_path) + # end = time.time() + # print(f"requester get result time is {end - start4}s") diff --git a/python/ppc_common/ppc_utils/audit_utils.py b/python/ppc_common/ppc_utils/audit_utils.py new file mode 100644 index 00000000..913a3934 --- /dev/null +++ b/python/ppc_common/ppc_utils/audit_utils.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +import getopt +import sys + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.utils import CryptoType + +AUDIT_KEYS = ["agency_name", "input_dataset_hash", + "psi_input_hash", "psi_output_hash", + "mpc_result_hash"] + + +def parse_parameter(argv): + file_path = 0 + data_hash_value = 0 + crypto_type = None + try: + opts, args = getopt.getopt( + argv, "hf:v:c:", ["file_path=", "data_hash_value="]) + except getopt.GetoptError: + usage() + sys.exit(2) + if len(opts) == 0: + usage() + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + usage() + sys.exit(0) + elif opt in ("-f", "--file_path"): + file_path = arg + elif opt in ("-v", "--data_hash_value"): + data_hash_value = arg + elif opt in ("-c", "--crypto_type"): + crypto_type = arg + else: + usage() + sys.exit(2) + return file_path, data_hash_value, crypto_type + + +def usage(): + print('audit.py -f -v -c crypto_type') + print('usage:') + print(' -f Notice:The file will be hashed to audit.') + print(' -v Notice:The dataset hash value will be compared to audit.') + print( + ' -c Notice:The crypto type[ECDSA or GM] will be used.') + + +if __name__ == '__main__': + file_path, data_hash_value, crypto_type = parse_parameter(sys.argv[1:]) + file_data_hash = utils.make_hash_from_file_path( + file_path, CryptoType[crypto_type]) + print(f'The file hash is:{file_data_hash}') + print(f'Audit result is:{file_data_hash == data_hash_value}') diff --git a/python/ppc_common/ppc_utils/cem_utils.py b/python/ppc_common/ppc_utils/cem_utils.py new file mode 100644 index 00000000..6e9d9d5f --- /dev/null +++ b/python/ppc_common/ppc_utils/cem_utils.py @@ -0,0 +1,160 @@ +import json +import unittest + +LEGAL_OPERATOR_LIST = ['>', '<', '>=', '<=', '==', '!='] +LEGAL_QUOTA_LIST = ['and', 'or'] + + +def get_rule_detail(match_module_dict): + ruleset = match_module_dict['ruleset'] + operator_set = set() + count_set = set() + quota_set = set() + for ruleset_item in ruleset: + sub_rule_set = ruleset_item['set'] + for sub_rule_item in sub_rule_set: + rule = sub_rule_item['rule'] + operator_set.add(rule['operator']) + count_set.add(rule['count']) + quota_set.add(rule['quota']) + return operator_set, count_set, quota_set + + +def check_dataset_id_has_duplicated(match_module_dict): + ruleset = match_module_dict['ruleset'] + for ruleset_item in ruleset: + sub_rule_set = ruleset_item['set'] + for sub_rule_item in sub_rule_set: + dataset_id_set = set() + dataset_id_list = sub_rule_item['dataset'] + for dataset_id in dataset_id_list: + dataset_id_set.add(dataset_id) + if len(dataset_id_set) != len(dataset_id_list): + return True + return False + + +def get_dataset_id_set(match_module_dict): + dataset_id_set = set() + ruleset = match_module_dict['ruleset'] + for ruleset_item in ruleset: + sub_rule_set = ruleset_item['set'] + for sub_rule_item in sub_rule_set: + dataset_id_list = sub_rule_item['dataset'] + for dataset_id in dataset_id_list: + dataset_id_set.add(dataset_id) + return dataset_id_set + + +# get field_dataset_map: {'x1':['d1', 'd2', 'd3'], 'x2':['d1', 'd2', 'd3']} +def parse_field_dataset_map(match_module_dict): + field_dataset_map = {} + ruleset = match_module_dict['ruleset'] + for ruleset_item in ruleset: + field = ruleset_item['field'] + if field in field_dataset_map: + field_dataset_id_set = field_dataset_map[field] + else: + field_dataset_id_set = set() + field_dataset_map[field] = field_dataset_id_set + sub_rule_set = ruleset_item['set'] + for sub_rule_item in sub_rule_set: + dataset_id_list = sub_rule_item['dataset'] + for dataset_id in dataset_id_list: + field_dataset_id_set.add(dataset_id) + return field_dataset_map + + +# get dataset_field_map: {'d1':['x1', 'x2'], 'd2':['x1', 'x2'], 'd3':['x1', 'x2']} +def parse_dataset_field_map(dataset_id_set, field_dataset_map): + dataset_field_map = {} + for dataset_id in dataset_id_set: + for field, field_dataset_id_set in field_dataset_map.items(): + if dataset_id in field_dataset_id_set: + if dataset_id in dataset_field_map: + dataset_field_set = dataset_field_map[dataset_id] + else: + dataset_field_set = set() + dataset_field_map[dataset_id] = dataset_field_set + dataset_field_set.add(field) + return dataset_field_map + + +def parse_match_param(match_fields, match_module_dict): + # step1 get field_dataset_map: {'x1':['d1', 'd2', 'd3'], 'x2':['d1', 'd2', 'd3']} + field_dataset_map = parse_field_dataset_map(match_module_dict) + # step2 get all dataset_id {'d1', 'd2', 'd3'} + dataset_id_set = set() + for field, field_dataset_id_set in field_dataset_map.items(): + dataset_id_set.update(field_dataset_id_set) + # step3 get dataset_field_map: {'d1':['x1', 'x2'], 'd2':['x1', 'x2'], 'd3':['x1', 'x2']} + dataset_field_map = parse_dataset_field_map( + dataset_id_set, field_dataset_map) + # step4 get match_param_list from dataset_field_map and match_field: + # [ + # {'dataset_id':'d1', 'match_field':{'x1':'xxx, 'x2':'xxx}, + # {'dataset_id':'d2', 'match_field':{'x1':'xxx, 'x2':'xxx}, + # {'dataset_id':'d3', 'match_field':{'x1':'xxx, 'x2':'xxx}, + # ] + match_param_list = parse_match_param_list(dataset_field_map, match_fields) + return dataset_id_set, match_param_list + + +# get match_param_list from dataset_field_map and match_field: +# [ +# {'dataset_id':'d1', 'match_field':{'x1':'xxx, 'x2':'xxx}, +# {'dataset_id':'d2', 'match_field':{'x1':'xxx, 'x2':'xxx}, +# {'dataset_id':'d3', 'match_field':{'x1':'xxx, 'x2':'xxx}, +# ] +def parse_match_param_list(dataset_field_map, match_fields): + match_param_list = [] + match_fields = match_fields.replace("'", '"') + match_fields_object = json.loads(match_fields) + for dataset_id, field_set in dataset_field_map.items(): + match_param = {'dataset_id': dataset_id} + field_value_map = {} + for field in field_set: + # allow some part field match + if field in match_fields_object.keys(): + field_value_map[field] = match_fields_object[field] + match_param['match_field'] = field_value_map + match_param_list.append(match_param) + return match_param_list + + +class TestCemUtils(unittest.TestCase): + def test_cem_match_algorithm_load(self): + match_module = '{"ruleset":[' \ + '{"field":"x1",' \ + '"set":[' \ + '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ + '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ + '{"field":"x2","set":[' \ + '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d2-encrypted","d3-encrypted"]}]}]}' + match_module_dict = json.loads(match_module) + # match_module_dict = utils.json_loads(match_module) + print(match_module_dict) + + def test_check_dataset_id_has_duplicated(self): + match_module = '{"ruleset":[' \ + '{"field":"x1",' \ + '"set":[' \ + '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ + '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ + '{"field":"x2","set":[' \ + '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d2-encrypted","d3-encrypted"]}]}]}' + match_module_dict = json.loads(match_module) + has_duplicated = check_dataset_id_has_duplicated(match_module_dict) + assert has_duplicated == False + + def test_check_dataset_id_has_duplicated(self): + match_module = '{"ruleset":[' \ + '{"field":"x1",' \ + '"set":[' \ + '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ + '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ + '{"field":"x2","set":[' \ + '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d1-encrypted","d3-encrypted"]}]}]}' + match_module_dict = json.loads(match_module) + has_duplicated = check_dataset_id_has_duplicated(match_module_dict) + assert has_duplicated == True diff --git a/python/ppc_common/ppc_utils/common_func.py b/python/ppc_common/ppc_utils/common_func.py new file mode 100644 index 00000000..e907d6bd --- /dev/null +++ b/python/ppc_common/ppc_utils/common_func.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from contextlib import contextmanager +import chardet + + +def get_config_value(key, default_value, config_value, required): + if required and config_value is None: + raise Exception(f"Invalid config for '{key}' for not set!") + value = config_value + if type(config_value) is dict: + if key in config_value: + value = config_value[key] + else: + value = default_value + if value is None: + return default_value + return value + + +def get_file_encoding(file_path): + encoding = None + with open(file_path, "rb") as fp: + header = fp.readline() + file_chardet = chardet.detect(header) + if file_chardet is None: + raise Exception(f"Unknown File Encoding, file: {file_path}") + encoding = file_chardet["encoding"] + return encoding diff --git a/python/ppc_common/ppc_utils/exception.py b/python/ppc_common/ppc_utils/exception.py new file mode 100644 index 00000000..212b3f71 --- /dev/null +++ b/python/ppc_common/ppc_utils/exception.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +from enum import Enum, unique + + +@unique +class PpcErrorCode(Enum): + SUCCESS = {0: 'success'} + INTERNAL_ERROR = {10000: "internal error"} + + NETWORK_ERROR = {10001: 'network error'} + JOB_STATUS_ERROR = {10002: 'job status check error'} + JOB_ROLE_ERROR = {10003: 'job role check error'} + DATABASE_ERROR = {10004: 'database related operation error'} + DATASET_CSV_ERROR = {10005: 'dataset csv format error'} + DATASET_PATH_ERROR = {10006: 'dataset path permission check error'} + PARAMETER_CHECK_ERROR = {10007: 'parameter check error'} + CALL_SYNCS_SERVICE_ERROR = {10008: 'call syncs service error'} + DATA_SET_ERROR = {10009: 'dataset operation error'} + + INSUFFICIENT_AUTHORITY = {10010: 'insufficient authority'} + UNDEFINED_TYPE = {10011: 'undefined type'} + UNDEFINED_STATUS = {10012: 'undefined status'} + QUERY_USERNAME_ERROR = {10013: 'query username error'} + DATASET_NOT_FOUND = {10014: 'dataset not found'} + ALGORITHM_NOT_FOUND = {10015: 'algorithm queried not found'} + JOB_NOT_FOUND = {10016: 'job not found'} + AUTH_INFO_FOUND = {10017: 'authorization not found'} + AGENCY_NOT_FOUND = {10018: 'agency not found'} + AGENCY_MANAGEMENT_NOT_FOUND = {10019: 'agency management not found'} + FIELD_NOT_FOUND = {10020: 'dataset field not found'} + AUTH_ALREADY_EXISTED = {10021: 'authorization already existed'} + DATABASE_TYPE_ERROR = {10022: 'database type error'} + DATABASE_IP_ERROR = {10023: 'database ip not in the white list'} + DATASET_FROM_DB_RETRY_OVER_LIMIT_ERROR = { + 10024: 'access the database has exceeded the allowed limit times'} + + DATASET_PARSE_ERROR = {10300: 'parse dataset failed'} + DATASET_EXIST_ERROR = {10301: 'dataset already existed!'} + DATASET_DELETE_ERROR = {10302: 'dataset already deleted!'} + DATASET_PERMISSION_ERROR = {10303: 'dataset permission check failed!'} + DATASET_UPLOAD_ERROR = {10304: 'dataset upload error'} + + ALGORITHM_PARSE_ERROR = {10400: 'parse algorithm failed'} + ALGORITHM_EXIST_ERROR = {10401: 'algorithm already existed!'} + ALGORITHM_DELETE_ERROR = {10402: 'algorithm already deleted!'} + ALGORITHM_TYPE_ERROR = {10403: 'algorithm type is not existed!'} + ALGORITHM_COMPILE_ERROR = {10410: 'compile mpc algorithm error'} + ALGORITHM_BAD_SQL = {10411: 'bad sql'} + ALGORITHM_PPC_CONFIG_ERROR = {10412: 'parse algorithm config error'} + ALGORITHM_PPC_MODEL_ALGORITHM_NAME_ERROR = { + 10413: 'algorithm subtype not found'} + ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR = {10414: 'output number error'} + ALGORITHM_PPC_MODEL_LAYERS_ERROR = { + 10415: 'layers attribute should not be set'} + ALGORITHM_MPC_SYNTAX_CHECK_ERROR = {10416: 'check ppc mpc syntax error'} + ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR_TEMP = { + 10417: 'output number should be set 1'} + ALGORITHM_PPC_MODEL_PARTICIPANTS_ERROR_TEMP = { + 10418: 'participants should be greater or equal to 2'} + ALGORITHM_PPC_MODEL_TEST_DATASET_PERCENTAGE_ERROR = { + 10419: 'test_dataset_percentage should be set in (0, 0.5]'} + ALGORITHM_PPC_MODEL_EPOCHS_ERROR = { + 10420: 'epochs should be set in [1, 5]'} + ALGORITHM_PPC_MODEL_BATCH_SIZE_ERROR = { + 10421: 'batch_size should be set [1, min(128, max_train_dataset_size)]'} + ALGORITHM_PPC_MODEL_THREADS_ERROR = { + 10422: 'threads should be set in [1,8]'} + ALGORITHM_PPC_MODEL_OPTIMIZER_ERROR = {10423: 'optimizer not found'} + ALGORITHM_PPC_MODEL_LEARNING_RATE_ERROR = { + 10424: 'learning rate should be set in (0, 1)'} + ALGORITHM_PPC_MODEL_LAYERS_ERROR2 = { + 10425: 'Conv2d layer should not be the first layer in HeteroNN'} + ALGORITHM_NOT_EXIST_ERROR = {10426: 'algorithm does not exist!'} + ALGORITHM_PPC_MODEL_TREES_ERROR = { + 10427: 'num_trees should be set in [1, 300]'} + ALGORITHM_PPC_MODEL_DEPTH_ERROR = { + 10428: 'max_depth should be set in [1, 10]'} + + JOB_CREATE_ERROR = {10500: 'job create failed'} + JOB_COMPUTATION_EXISTED_ERROR = {10501: 'job computation not existed'} + JOB_AYS_MODE_CHECK_ERROR = {10502: 'patch request need static token'} + JOB_MANAGEMENT_RUN_ERROR = {10503: 'job run failed'} + JOB_CEM_ERROR = {10504: 'at least one field need to be provided'} + JOB_IS_RUNNING_ERROR = {10505: 'job is running'} + NO_PARTICAPATING_IN_JOB_ERROR = {10506: 'not participating in the job'} + JOB_DOWNLOAD_RESULT_EMPTY = {10507: '任务结果为空，无法下载'} + + HDFS_STORAGE_ERROR = {10601: 'hdfs storage error'} + + AYS_LENGTH_ERROR = {10701: 'base ot message length check error'} + AYS_ORDER_ERROR = {10702: 'search obfuscation order must > 1'} + AYS_RESULT_LENGTH_ERROR = {10703: 'message result length check error'} + AYS_FIELD_ERROR = {10704: 'search filed not found in obfuscate file'} + CALL_SCS_ERROR = {10705: 'computation node call error'} + + MERGE_FILE_CHECK_ERROR = {10801: 'merge check files failed'} + MERGE_FILE_FORMAT_ERROR = {10802: 'file format is not csv'} + FILE_SIZE_ERROR = {10803: 'cannot get file size'} + FILE_SPLIT_ERROR = {10804: 'split file failed!'} + FILE_NOT_EXIST_ERROR = {10805: 'share file not existed!'} + DUPLICATED_MERGE_FILE_REQUEST = {10806: 'duplicated merge file request'} + + XGB_PREPROCESSING_ERROR = {10901: 'xgb preprocessing failed!'} + + FILE_OBJECT_UPLOAD_CHECK_FAILED = {20000: "upload file object failed!"} + FILE_OBJECT_NOT_EXISTS = {20001: "the file not exists!"} + + TASK_EXISTS = {11000: "the task already exists!"} + TASK_NOT_FOUND = {11001: "the task not found!"} + TASK_IS_KILLED = {11002: "the task is killed!"} + + ROLE_TYPE_ERROR = {12000: "role type is illegal."} + + def get_code(self): + return list(self.value.keys())[0] + + def get_error_code(self): + return list(self.value.keys())[0] + + def get_msg(self): + return list(self.value.values())[0] + + def get_message(self): + return list(self.value.values())[0] + + +class PpcException(Exception): + + def __init__(self, code, message): + Exception.__init__(self) + self.code = code + self.message = message + + def to_dict(self): + return {'code': self.code, 'message': self.message} + + def get_code(self): + return self.code + + def __str__(self): + return self.message + + @classmethod + def by_ppc_error_code(cls, ppc_error_code): + cls.code = ppc_error_code.get_code() + cls.message = ppc_error_code.get_msg() diff --git a/python/ppc_common/ppc_utils/http_utils.py b/python/ppc_common/ppc_utils/http_utils.py new file mode 100644 index 00000000..4489240d --- /dev/null +++ b/python/ppc_common/ppc_utils/http_utils.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +import json +import logging +from json.decoder import JSONDecodeError + +import requests +import urllib3 +from urllib3.exceptions import SecurityWarning + +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + +log = logging.getLogger(__name__) + + +def check_response(response): + if response.status_code != 200 and response.status_code != 201: + message = response.text + raise PpcException(PpcErrorCode.NETWORK_ERROR, + f"Call request failed, response message:{message}") + + +def send_get_request(endpoint, uri, params=None, headers=None): + urllib3.disable_warnings(SecurityWarning) + if not headers: + headers = {'content-type': 'application/json'} + if uri: + url = f"http://{endpoint}{uri}" + else: + url = f"http://{endpoint}" + log.debug(f"send a get request, url: {url}, params: {params}") + response = requests.get(url=url, params=params, headers=headers, timeout=30) + log.debug(f"response: {response.text}") + check_response(response) + response_data = json.loads(response.text) + + return response_data + + +def send_post_request(endpoint, uri, params=None, headers=None, data=None): + if not headers: + headers = {'content-type': 'application/json'} + if uri: + url = f"http://{endpoint}{uri}" + else: + url = f"http://{endpoint}" + log.debug(f"send a post request, url: {url}, params: {params}") + response = requests.post(url, data=data, json=params, headers=headers) + log.debug(f"response: {response.text}") + # check_response(response) + try: + response_data = json.loads(response.text) + except JSONDecodeError: + response_data = response.text + return response_data + + +def send_delete_request(endpoint, uri, params=None, headers=None): + if not headers: + headers = {'content-type': 'application/json'} + if uri: + url = f"http://{endpoint}{uri}" + else: + url = f"http://{endpoint}" + log.debug(f"send a delete request, url: {url}, params: {params}") + response = requests.delete(url, json=params, headers=headers) + check_response(response) + log.debug(f"response: {response.text}") + response_data = json.loads(response.text) + + return response_data + + +def send_patch_request(endpoint, uri, params=None, headers=None, data=None): + if not headers: + headers = {'content-type': 'application/json'} + url = f"http://{endpoint}{uri}" + log.debug(f"send a patch request, url: {url}, params: {params}") + response = requests.patch(url, data=data, json=params, headers=headers) + check_response(response) + log.debug(f"response: {response.text}") + response_data = json.loads(response.text) + + return response_data + + +def send_upload_request(endpoint, uri, params=None, headers=None, data=None): + if not headers: + headers = {'content-type': 'application/json'} + if uri: + url = f"http://{endpoint}{uri}" + else: + url = endpoint + log.debug(f"send a post request, url: {url}, params: {params}") + response = requests.post(url, data=data, json=params, headers=headers) + log.debug(f"response: {response.text}") + check_response(response) + try: + response_data = json.loads(response.text) + except JSONDecodeError: + response_data = response.text + return response_data + diff --git a/python/ppc_common/ppc_utils/path.py b/python/ppc_common/ppc_utils/path.py new file mode 100644 index 00000000..b04daab5 --- /dev/null +++ b/python/ppc_common/ppc_utils/path.py @@ -0,0 +1,51 @@ +# coding:utf-8 +"""[path.py] + +Returns: + [type] -- [description] +""" + + +class Path(object): + ''' + fisco generator path configuration + ''' + dir = '' + + def get_name(self): + """[get some name] + + maybe it will usedful not now + + Returns: + [string] -- [name] + """ + + def get_pylint(self): + """[get some name] + + maybe it will usedful not now + + Returns: + [string] -- [name] + """ + + +def set_path(_dir): + """[set path] + + Arguments: + dir {[PATH]} -- [path] + """ + + Path.dir = _dir + + +def get_path(): + """[get path] + + Returns: + [PATH] -- [path] + """ + + return Path.dir diff --git a/python/ppc_common/ppc_utils/permission.py b/python/ppc_common/ppc_utils/permission.py new file mode 100644 index 00000000..7eb0c619 --- /dev/null +++ b/python/ppc_common/ppc_utils/permission.py @@ -0,0 +1,79 @@ +from enum import Enum + +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_common.ppc_utils.utils import JobRole, JobStatus + + +def check_job_status(job_status): + if job_status in JobStatus.__members__: + return True + return False + + +def check_job_role(job_role): + if job_role in JobRole.__members__: + return True + return False + + +ADMIN_PERMISSIONS = 'ADMIN_PERMISSIONS' + + +class UserRole(Enum): + ADMIN = 1 + DATA_PROVIDER = 2 + ALGO_PROVIDER = 3 + DATA_CONSUMER = 4 + + +class PermissionGroup(Enum): + AGENCY_GROUP = 1 + DATA_GROUP = 2 + ALGO_GROUP = 3 + JOB_GROUP = 4 + AUDIT_GROUP = 5 + + +class AgencyGroup(Enum): + LIST_AGENCY = 1 + WRITE_AGENCY = 2 + + +class DataGroup(Enum): + LIST_DATA = 1 + READ_DATA_PUBLIC_INFO = 2 + READ_DATA_PRIVATE_INFO = 3 + WRITE_DATA = 4 + + +class AlgoGroup(Enum): + LIST_ALGO = 1 + READ_ALGO_PUBLIC_INFO = 2 + READ_ALGO_PRIVATE_INFO = 3 + WRITE_ALGO = 4 + + +class JobGroup(Enum): + LIST_JOB = 1 + READ_JOB_PUBLIC_INFO = 2 + READ_JOB_PRIVATE_INFO = 3 + WRITE_JOB = 4 + + +class AuditGroup(Enum): + READ_AUDIT = 1 + WRITE_AUDIT = 2 + + +# permissions formed as permission_a|permission_b|permission_a_group|... +def check_permission(permissions, needed_permission_group, *needed_permissions): + permission_list = permissions.split('|') + if ADMIN_PERMISSIONS in permission_list: + return 0 + if needed_permission_group in permission_list: + return 1 + for needed_permission in needed_permissions: + if needed_permission in permission_list: + return 1 + raise PpcException(PpcErrorCode.INSUFFICIENT_AUTHORITY.get_code( + ), PpcErrorCode.INSUFFICIENT_AUTHORITY.get_msg()) diff --git a/python/ppc_common/ppc_utils/plot_utils.py b/python/ppc_common/ppc_utils/plot_utils.py new file mode 100644 index 00000000..3420c164 --- /dev/null +++ b/python/ppc_common/ppc_utils/plot_utils.py @@ -0,0 +1,230 @@ +import gc + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +from sklearn.metrics import accuracy_score +from sklearn.metrics import confusion_matrix +from sklearn.metrics import precision_recall_curve +from sklearn.metrics import roc_curve, auc + + +def plot_two_class_graph(job_context, y_scores=None, y_true=None): + y_pred_probs = job_context.y_pred_probs + y_label_probs = job_context.y_label_probs + if y_scores: + y_pred_probs = y_scores + if y_true: + y_label_probs = y_true + plt.rcParams['figure.figsize'] = (12.0, 8.0) + + # plot ROC + fpr, tpr, thresholds = roc_curve(y_label_probs, y_pred_probs, pos_label=1) + auc_value = auc(fpr, tpr) + plt.figure(f'roc-{job_context.job_id}') + plt.title('ROC Curve') # give plot a title + plt.xlabel('False Positive Rate (1 - Specificity)') + plt.ylabel('True Positive Rate (Sensitivity)') + plt.plot([0, 1], [0, 1], 'k--', lw=2) + plt.plot(fpr, tpr, label='area = {0:0.5f}' + ''.format(auc_value)) + plt.legend(loc="lower right") + plt.savefig(job_context.mpc_metric_roc_path, dpi=1000) + plt.show() + + plt.close('all') + gc.collect() + + # plot KS + plt.figure(f'ks-{job_context.job_id}') + threshold_x = np.sort(thresholds) + threshold_x[-1] = 1 + ks_value = max(abs(fpr - tpr)) + plt.title('KS Curve') + plt.xlabel('Threshold') + plt.plot(threshold_x, tpr, label='True Positive Rate') + plt.plot(threshold_x, fpr, label='False Positive Rate') + # 标记最大ks值 + x_index = np.argwhere(abs(fpr - tpr) == ks_value)[0, 0] + plt.plot((threshold_x[x_index], threshold_x[x_index]), (fpr[x_index], tpr[x_index]), + label='ks = {:.3f}'.format(ks_value), color='r', marker='o', markerfacecolor='r', markersize=5) + plt.legend(loc="lower right") + plt.savefig(job_context.mpc_metric_ks_path, dpi=1000) + plt.show() + + plt.close('all') + gc.collect() + + # plot Precision Recall + plt.figure(f'pr-{job_context.job_id}') + plt.title('Precision/Recall Curve') + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.xlim(0.0, 1.0) + plt.ylim(0.0, 1.05) + precision, recall, thresholds = precision_recall_curve( + y_label_probs, y_pred_probs) + plt.plot(recall, precision) + plt.savefig(job_context.mpc_metric_pr_path, dpi=1000) + plt.show() + + plt.close('all') + gc.collect() + + # plot accuracy + plt.figure(f'accuracy-{job_context.job_id}') + thresholds = np.linspace(0, 1, num=100) # 在0~1之间生成100个阈值 + accuracies = [] + for threshold in thresholds: + predicted_labels = (y_pred_probs >= threshold).astype(int) + accuracy = accuracy_score(y_label_probs, predicted_labels) + accuracies.append(accuracy) + plt.title('Accuracy Curve') + plt.xlabel('Threshold') + plt.ylabel('Accuracy') + plt.xlim(0.0, 1.0) + plt.ylim(0.0, 1.05) + plt.plot(thresholds, accuracies) + plt.savefig(job_context.mpc_metric_accuracy_path, dpi=1000) + plt.show() + + plt.close('all') + gc.collect() + return (ks_value, auc_value) + + +def plot_multi_class_graph(job_context, n_class=None, y_label_value=None, y_pred_value=None): + if not n_class: + n_class = job_context.n_class + if not y_label_value: + y_label_value = job_context.y_label_value + if not y_pred_value: + y_pred_value = job_context.y_pred_value + y_label_probs = job_context.y_label_probs + y_pred_probs = job_context.y_pred_probs + + class_names = [x for x in range(n_class)] + plt.rcParams['figure.figsize'] = (12.0, 8.0) + plt.figure(f'roc-{job_context.job_id}') + multi_class_roc(job_context, plt, y_label_probs, y_pred_probs, class_names) + plt.figure(f'pr-{job_context.job_id}') + multi_class_precision_recall( + job_context, plt, y_label_probs, y_pred_probs, class_names) + plt.figure(f'cm-{job_context.job_id}') + multi_class_confusion_matrix( + job_context, plt, y_label_value, y_pred_value, class_names) + + +def sigmoid(x): + return 1 / (1 + np.exp(-x)) + + +def softmax(x): + x -= np.max(x, axis=1, keepdims=True) + return np.exp(x) / np.sum(np.exp(x), axis=1, keepdims=True) + + +# Converts [[0.3, 0.6, 0.1], [0.1, 0.2, 0.7], [0.8, 0.1, 0.1]] to [1, 2, 0] +def get_value_from_probs(probs): + return [np.argmax(prob) for prob in probs] + + +# Converts [1, 2, 0] to [[0, 1, 0], [0, 0, 1], [1, 0, 0]] +def get_probs_from_value(values, n_classes): + probs = np.zeros((len(values), n_classes), int) + for p, v in zip(probs, values): + p[v] = 1 + + return probs + + +# This need one-hot encoding +def multi_class_roc(job_context, plt, y_label, y_pred, class_names): + n_classes = len(class_names) + + fpr = dict() + tpr = dict() + roc_auc = dict() + for i in range(n_classes): + fpr[i], tpr[i], _ = roc_curve(y_label[:, i], y_pred[:, i]) + roc_auc[i] = auc(fpr[i], tpr[i]) + + # micro + fpr["micro"], tpr["micro"], _ = roc_curve(y_label.ravel(), y_pred.ravel()) + roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) + + # macro + # First aggregate all false positive rates + all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)])) + # Then interpolate all ROC curves at this points + mean_tpr = np.zeros_like(all_fpr) + for i in range(n_classes): + mean_tpr += np.interp(all_fpr, fpr[i], tpr[i]) + # Finally average it and compute AUC + mean_tpr /= n_classes + fpr["macro"] = all_fpr + tpr["macro"] = mean_tpr + roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) + + # Plot all ROC curves + lw = 2 + plt.plot(fpr["micro"], tpr["micro"], + label='Micro-averaging (area = {0:0.5f})' + ''.format(roc_auc["micro"]), + linestyle=':', linewidth=4) + + plt.plot(fpr["macro"], tpr["macro"], + label='Macro-averaging (area = {0:0.5f})' + ''.format(roc_auc["macro"]), + linestyle=':', linewidth=4) + + for i in range(n_classes): + plt.plot(fpr[i], tpr[i], lw=lw, + label='Class {0} (area = {1:0.5f})' + ''.format(class_names[i], roc_auc[i])) + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate (1 - Specificity)') + plt.ylabel('True Positive Rate (Sensitivity)') + plt.title('Multi-class ROC Curve') + plt.legend(loc="lower right") + plt.plot([0, 1], [0, 1], 'k--', lw=lw) + plt.savefig(job_context.mpc_metric_roc_path, dpi=1000) + + +# This need one-hot encoding +def multi_class_precision_recall(job_context, plt, y_label, y_pred, class_names): + n_classes = len(class_names) + precision = dict() + recall = dict() + for i in range(n_classes): + precision[i], recall[i], _ = precision_recall_curve( + y_label[:, i], y_pred[:, i]) + + # Plot all ROC curves + lw = 2 + for i in range(n_classes): + plt.plot(recall[i], precision[i], lw=lw, + label='Class {0}'.format(class_names[i])) + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.title('Multi-class Precision/Recall Curve') + plt.legend(loc="lower right") + plt.savefig(job_context.mpc_metric_pr_path, dpi=1000) + + +# This need value encoding +def multi_class_confusion_matrix(job_context, plt, y_label, y_pred, class_names): + cm = confusion_matrix(y_label, y_pred) + conf_matrix = pd.DataFrame(cm, index=class_names, columns=class_names) + + sns.heatmap(conf_matrix, annot=True, annot_kws={ + "size": 19}, cmap="Blues", fmt='d') + plt.ylabel('True label') + plt.xlabel('Predicted label') + plt.title('Confusion Matrix') + plt.savefig(job_context.mpc_metric_confusion_matrix_path, dpi=1000) diff --git a/python/ppc_common/ppc_utils/ppc_model_config_parser.py b/python/ppc_common/ppc_utils/ppc_model_config_parser.py new file mode 100644 index 00000000..ab2cce26 --- /dev/null +++ b/python/ppc_common/ppc_utils/ppc_model_config_parser.py @@ -0,0 +1,474 @@ +# coding:utf-8 +import json +import os +import unittest +from enum import unique, Enum + +from jsoncomment import JsonComment + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_common.ppc_utils.utils import check_ppc_model_algorithm_is_homo, parse_n_class + + +@unique +class ModelAlgorithmType(Enum): + HeteroLR = 1 + HomoLR = 2 + HeteroNN = 3 + HomoNN = 4 + HeteroXGB = 5 + + +@unique +class OptimizerType(Enum): + sgd = 1 + adam = 2 + + +algorithm_types = [ModelAlgorithmType.HeteroLR.name, ModelAlgorithmType.HomoLR.name, ModelAlgorithmType.HeteroNN.name, + ModelAlgorithmType.HomoNN.name, ModelAlgorithmType.HeteroXGB.name] + +optimizer_types = [OptimizerType.sgd.name, OptimizerType.adam.name] + +default_epochs = 10 +default_threads = 8 + + +FILE_PATH = os.path.abspath(__file__) +CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".") + + +def get_dir(): + ppc_model_template_dir = f'{CURRENT_PATH}{os.sep}..{os.sep}ppc_model_template{os.sep}' + return ppc_model_template_dir + + +def parse_read_hetero_dataset_loop(participants): + loop_start = [] + loop_end = [] + start = '' + end = '' + for i in range(participants): + if i == 0 or i == participants - 1: + if i == 0: + start = f'{start}source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + else: + start = f'{start} + source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + else: + start = f'{start} + source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + loop_start.append(start) + loop_end.append(end) + return participants - 1, loop_start[0:participants - 1], loop_end[0:participants - 1] + + +def parse_read_homo_dataset_loop(participants): + loop_start = [] + loop_end = [] + start = '' + end = '' + for i in range(participants): + if i == 0 or i == participants - 1: + if i == 0: + start = f'{start}source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + else: + start = f'{start} + source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + else: + start = f'{start} + source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + loop_start.append(start) + loop_end.append(end) + return participants - 1, loop_start[0:participants - 1], loop_end[0:participants - 1] + + +def insert_train_record_count(layer, index, record_type): + if 'Dense' in layer: + layer_arr = layer.split('(') + else: + layer_arr = layer.split('([') + if index == 0 and 'Dense' in layer: + new_layer = f'{layer_arr[0]}({record_type}, total_feature_count, {layer_arr[1]}' + else: + if 'Dense' in layer: + new_layer = f'{layer_arr[0]}({record_type}, {layer_arr[1]}' + else: + new_layer = f'{layer_arr[0]}([{record_type}, {layer_arr[1]}' + if 'Conv2d' in new_layer: + if ', [' in new_layer: + layer_arr2 = new_layer.split(', [') + new_layer = f'{layer_arr2[0]}, [{record_type}, {layer_arr2[1]}' + if ',[' in new_layer: + layer_arr2 = new_layer.split(',[') + new_layer = f'{layer_arr2[0]},[{record_type}, {layer_arr2[1]}' + return new_layer + + +def set_nn_layers(mpc_train_algorithm, layers, record_type): + mpc_train_algorithm = f"{mpc_train_algorithm}\n" + if layers: + layers_str = 'layers = [' + for i in range(len(layers)): + new_layer = insert_train_record_count(layers[i], i, record_type) + layers_str = f"{layers_str}{new_layer},\n" + n_class = parse_n_class(layers[-1]) + mpc_train_algorithm = f"{mpc_train_algorithm}{layers_str}" + + if n_class == 1: + mpc_train_algorithm = f"{mpc_train_algorithm}ml.Output({record_type}, approx=3)]\n\n" + mpc_train_algorithm = f"{mpc_train_algorithm}test_Y = pfix.Array(test_record_count)\n" + mpc_train_algorithm = f"{mpc_train_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + elif n_class > 1: + mpc_train_algorithm = f"{mpc_train_algorithm}ml.MultiOutput({record_type}, {n_class})]\n\n" + mpc_train_algorithm = f"{mpc_train_algorithm}total_class_count = {n_class}\n" + mpc_train_algorithm = f"{mpc_train_algorithm}test_Y = pint.Matrix(test_record_count, total_class_count)\n" + mpc_train_algorithm = f"{mpc_train_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + else: + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR.get_msg()) + else: + mpc_train_algorithm = f"{mpc_train_algorithm}test_Y = pfix.Array(test_record_count)\n" + mpc_train_algorithm = f"{mpc_train_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + mpc_train_algorithm = f"{mpc_train_algorithm}" \ + f"layers = [pDense({record_type}, total_feature_count, 128, " \ + f"activation='relu'), pDense({record_type}, 128, 1), " \ + f"ml.Output({record_type}, approx=3)]\n" + mpc_train_algorithm = f"{mpc_train_algorithm}\n" + return mpc_train_algorithm + + +def set_logreg_train_layers(mpc_algorithm): + mpc_algorithm = f"{mpc_algorithm}\n" + mpc_algorithm = f"{mpc_algorithm}layers = [pDense(train_record_count, total_feature_count, 1), ml.Output(train_record_count, approx=3)]\n\n" + mpc_algorithm = f"{mpc_algorithm}test_Y = pfix.Array(test_record_count)\n" + mpc_algorithm = f"{mpc_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + return mpc_algorithm + + +def generate_set_logreg_predict_layers(mpc_algorithm): + mpc_algorithm = f"{mpc_algorithm}\n" + mpc_algorithm = f"{mpc_algorithm}layers = [pDense(test_record_count, total_feature_count, 1), ml.Output(test_record_count, approx=3)]\n" + return mpc_algorithm + + +def generate_homo_predict_static_template(mpc_predict_algorithm): + homo_nn_static_template = utils.read_content_from_file( + f'{get_dir()}homo_predict_static_template.mpc') + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}{homo_nn_static_template}' + return mpc_predict_algorithm + + +def set_hetero_train_static_template(model_config_dict, mpc_train_algorithm): + optimizer = model_config_dict['optimizer'] + learning_rate = model_config_dict['learning_rate'] + if optimizer == OptimizerType.sgd.name: + hetero_train_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_train_sgd_static_template.mpc') + hetero_train_static_template = hetero_train_static_template.replace('gamma = MemValue(cfix(.1))', + f'gamma = MemValue(cfix({learning_rate}))') + elif optimizer == OptimizerType.adam.name: + hetero_train_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_train_adam_static_template.mpc') + hetero_train_static_template = hetero_train_static_template.replace('gamma = MemValue(cfix(.001))', + f'gamma = MemValue(cfix({learning_rate}))') + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}{hetero_train_static_template}' + return mpc_train_algorithm + + +def generate_hetero_predict_static_template(mpc_predict_algorithm): + hetero_logreg_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_predict_static_template.mpc') + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}{hetero_logreg_static_template}' + return mpc_predict_algorithm + + +def set_homo_train_static_template(model_config_dict, mpc_train_algorithm): + optimizer = model_config_dict['optimizer'] + learning_rate = model_config_dict['learning_rate'] + if optimizer == OptimizerType.sgd.name: + homo_train_static_template = utils.read_content_from_file( + f'{get_dir()}homo_train_sgd_static_template.mpc') + homo_train_static_template = homo_train_static_template.replace('gamma = MemValue(cfix(.1))', + f'gamma = MemValue(cfix({learning_rate}))') + elif optimizer == OptimizerType.adam.name: + homo_train_static_template = utils.read_content_from_file( + f'{get_dir()}homo_train_adam_static_template.mpc') + homo_train_static_template = homo_train_static_template.replace('gamma = MemValue(cfix(.001))', + f'gamma = MemValue(cfix({learning_rate}))') + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}{homo_train_static_template}' + return mpc_train_algorithm + + +def generate_set_common_code(is_psi, participants): + mpc_algorithm = '#This file is auto generated by ppc. DO NOT EDIT!\n\n' + if is_psi: + mpc_algorithm = f'{mpc_algorithm}#PSI_OPTION=True' + else: + mpc_algorithm = f'{mpc_algorithm}#PSI_OPTION=False' + mpc_algorithm = f'{mpc_algorithm}\n' + mpc_algorithm = f'{mpc_algorithm}from ppc import *\n' + mpc_algorithm = f'{mpc_algorithm}from Compiler import config\n' + mpc_algorithm = f'{mpc_algorithm}import sys\n' + mpc_algorithm = f'{mpc_algorithm}program.options_from_args()\n\n' + mpc_algorithm = f'{mpc_algorithm}program.use_trunc_pr = True\n' + mpc_algorithm = f'{mpc_algorithm}program.use_split(3)\n\n' + # if participants == 3: + # mpc_algorithm = f'{mpc_algorithm}program.use_trunc_pr = True\n' + # mpc_algorithm = f'{mpc_algorithm}program.use_split(3)\n\n' + for i in range(participants): + mpc_algorithm = f'{mpc_algorithm}SOURCE{i}={i}\n' + return mpc_algorithm + + +def set_hetero_feature_count(mpc_algorithm, participants): + total_feature_count_str = 'total_feature_count=' + for i in range(participants): + mpc_algorithm = f'{mpc_algorithm}source{i}_feature_count=$(source{i}_feature_count)\n' + if i == participants - 1: + total_feature_count_str = f'{total_feature_count_str}source{i}_feature_count' + else: + total_feature_count_str = f'{total_feature_count_str}source{i}_feature_count+' + mpc_algorithm = f'{mpc_algorithm}{total_feature_count_str}\n' + return mpc_algorithm + + +def set_homo_train_record_count(mpc_train_algorithm, participants): + total_record_count_str = 'total_record_count=' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_count=$(source{i}_record_count)\n' + if i == participants - 1: + total_record_count_str = f'{total_record_count_str}source{i}_record_count' + else: + total_record_count_str = f'{total_record_count_str}source{i}_record_count+' + mpc_train_algorithm = f'{mpc_train_algorithm}{total_record_count_str}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}train_record_count=$(train_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}test_record_count=$(test_record_count)\n' + return mpc_train_algorithm + + +def generate_set_homo_predict_record_count(mpc_predict_algorithm): + mpc_predict_algorithm = f'{mpc_predict_algorithm}total_feature_count=$(total_feature_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_record_count=$(test_record_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n' + return mpc_predict_algorithm + + +def set_hetero_train_record_count(mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}total_record_count=$(total_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}train_record_count=$(train_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}test_record_count=$(test_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + return mpc_train_algorithm + + +def generate_set_hetero_predict_record_count(mpc_predict_algorithm): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_record_count=$(test_record_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n' + return mpc_predict_algorithm + + +def set_homo_feature_count(mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}total_feature_count=$(total_feature_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + return mpc_train_algorithm + + +def read_hetero_train_dataset(mpc_train_algorithm, participants): + mpc_train_algorithm = f'{mpc_train_algorithm}def read_hetero_dataset():\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"train_record_count:{{train_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"test_record_count:{{test_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_hetero_y_part(SOURCE0)\n' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_hetero_x_part(SOURCE{i}, source{i}_feature_count)\n' + return mpc_train_algorithm + + +def generate_read_hetero_predict_dataset(mpc_predict_algorithm, participants): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}def read_hetero_test_dataset():\n' + for i in range(participants): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\tdo_read_hetero_x_part(SOURCE{i}, source{i}_feature_count)\n' + return mpc_predict_algorithm + + +def read_homo_dataset(mpc_train_algorithm, participants): + mpc_train_algorithm = f'{mpc_train_algorithm}def read_homo_dataset():\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"total_feature_count:{{total_feature_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"train_record_count:{{train_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"test_record_count:{{test_record_count}}")\n' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_homo_dataset(SOURCE{i}, source{i}_record_count)\n' + + return mpc_train_algorithm + + +def set_parameters(model_config_dict, mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + epochs = model_config_dict['epochs'] + batch_size = model_config_dict['batch_size'] + threads = model_config_dict['threads'] + if epochs <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}epochs={default_epochs}\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}epochs={epochs}\n' + if int(batch_size) <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}batch_size=train_record_count\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}user_batch_size={batch_size}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}batch_size=min(user_batch_size, min(128, train_record_count))\n' + if threads <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}threads={default_threads}\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}threads={threads}\n' + return mpc_train_algorithm + + +def generate_mpc_train_algorithm(model_config_dict, algorithm_name, is_psi): + participants = model_config_dict['participants'] + mpc_train_algorithm = generate_set_common_code(is_psi, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name or algorithm_name == ModelAlgorithmType.HeteroNN.name: + mpc_train_algorithm = set_hetero_feature_count( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_hetero_train_record_count( + mpc_train_algorithm) + mpc_train_algorithm = read_hetero_train_dataset( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_parameters( + model_config_dict, mpc_train_algorithm) + if algorithm_name == ModelAlgorithmType.HeteroLR.name: + if 'layers' in model_config_dict.keys(): + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_msg()) + mpc_train_algorithm = set_logreg_train_layers(mpc_train_algorithm) + mpc_train_algorithm = set_hetero_train_static_template( + model_config_dict, mpc_train_algorithm) + else: + layers = [] + if 'layers' in model_config_dict.keys(): + layers = model_config_dict['layers'] + if layers and 'Conv2d' in layers[0]: + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR2.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR2.get_msg()) + mpc_train_algorithm = set_nn_layers( + mpc_train_algorithm, layers, 'train_record_count') + mpc_train_algorithm = set_hetero_train_static_template( + model_config_dict, mpc_train_algorithm) + elif algorithm_name == ModelAlgorithmType.HomoLR.name or algorithm_name == ModelAlgorithmType.HomoNN.name: + mpc_train_algorithm = set_homo_train_record_count( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_parameters( + model_config_dict, mpc_train_algorithm) + mpc_train_algorithm = set_homo_feature_count(mpc_train_algorithm) + mpc_train_algorithm = read_homo_dataset( + mpc_train_algorithm, participants) + if algorithm_name == ModelAlgorithmType.HomoLR.name: + if 'layers' in model_config_dict.keys(): + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_msg()) + mpc_train_algorithm = set_logreg_train_layers(mpc_train_algorithm) + mpc_train_algorithm = set_homo_train_static_template( + model_config_dict, mpc_train_algorithm) + else: + layers = [] + if 'layers' in model_config_dict.keys(): + layers = model_config_dict['layers'] + mpc_train_algorithm = set_nn_layers( + mpc_train_algorithm, layers, 'train_record_count') + mpc_train_algorithm = set_homo_train_static_template( + model_config_dict, mpc_train_algorithm) + + return mpc_train_algorithm + + +def generate_mpc_predict_algorithm(algorithm_name, layers, participants, is_psi): + mpc_predict_algorithm = generate_set_common_code(is_psi, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name or algorithm_name == ModelAlgorithmType.HeteroNN.name: + mpc_predict_algorithm = set_hetero_feature_count( + mpc_predict_algorithm, participants) + mpc_predict_algorithm = generate_set_hetero_predict_record_count( + mpc_predict_algorithm) + mpc_predict_algorithm = generate_read_hetero_predict_dataset( + mpc_predict_algorithm, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name: + mpc_predict_algorithm = generate_set_logreg_predict_layers( + mpc_predict_algorithm) + else: + mpc_predict_algorithm = set_nn_layers( + mpc_predict_algorithm, layers, 'test_record_count') + mpc_predict_algorithm = generate_hetero_predict_static_template( + mpc_predict_algorithm) + + elif algorithm_name == ModelAlgorithmType.HomoLR.name or algorithm_name == ModelAlgorithmType.HomoNN.name: + mpc_predict_algorithm = generate_set_homo_predict_record_count( + mpc_predict_algorithm) + if algorithm_name == ModelAlgorithmType.HomoLR.name: + mpc_predict_algorithm = generate_set_logreg_predict_layers( + mpc_predict_algorithm) + else: + mpc_predict_algorithm = set_nn_layers( + mpc_predict_algorithm, layers, 'test_record_count') + mpc_predict_algorithm = generate_homo_predict_static_template( + mpc_predict_algorithm) + + return mpc_predict_algorithm + + # try: + # test_dataset_percentage = float(model_config_dict['test_dataset_percentage']) + # model_config_dict['test_dataset_percentage'] = test_dataset_percentage + # if not 0 < test_dataset_percentage <= 0.5: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_TEST_DATASET_PERCENTAGE_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_TEST_DATASET_PERCENTAGE_ERROR.get_msg()) + # except BaseException as e: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_TEST_DATASET_PERCENTAGE_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_TEST_DATASET_PERCENTAGE_ERROR.get_msg()) + + # try: + # learning_rate = float(model_config_dict['learning_rate']) + # model_config_dict['learning_rate'] = learning_rate + # if not 0 < learning_rate <= 1: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LEARNING_RATE_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_LEARNING_RATE_ERROR.get_msg()) + # except BaseException as e: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LEARNING_RATE_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_LEARNING_RATE_ERROR.get_msg()) + + # try: + # num_trees = int(model_config_dict['num_trees']) + # model_config_dict['num_trees'] = num_trees + # if not 0 < num_trees <= 300: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_TREES_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_TREES_ERROR.get_msg()) + # except BaseException as e: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_TREES_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_TREES_ERROR.get_msg()) + + # try: + # max_depth = int(model_config_dict['max_depth']) + # model_config_dict['max_depth'] = max_depth + # if not 0 < max_depth <= 10: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_DEPTH_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_DEPTH_ERROR.get_msg()) + # except BaseException as e: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_DEPTH_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_DEPTH_ERROR.get_msg()) + + # try: + # threads = int(model_config_dict['threads']) + # model_config_dict['threads'] = threads + # if not (0 < threads <= 8): + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_msg()) + # except BaseException as e: + # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_code(), + # PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_msg()) \ No newline at end of file diff --git a/python/ppc_common/ppc_utils/ppc_model_config_parser_proxy.py b/python/ppc_common/ppc_utils/ppc_model_config_parser_proxy.py new file mode 100644 index 00000000..b5aa49e9 --- /dev/null +++ b/python/ppc_common/ppc_utils/ppc_model_config_parser_proxy.py @@ -0,0 +1,491 @@ +# coding:utf-8 +import os +import unittest +from enum import unique, Enum + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_common.ppc_utils.utils import check_ppc_model_algorithm_is_homo, parse_n_class, PPCModleType + + +@unique +class ModelAlgorithmType(Enum): + HeteroLR = 1 + HomoLR = 2 + HeteroNN = 3 + HomoNN = 4 + + +@unique +class OptimizerType(Enum): + sgd = 1 + adam = 2 + + +algorithm_types = [ModelAlgorithmType.HeteroLR.name, ModelAlgorithmType.HomoLR.name, ModelAlgorithmType.HeteroNN.name, + ModelAlgorithmType.HomoNN.name] + +optimizer_types = [OptimizerType.sgd.name, OptimizerType.adam.name] + +default_epochs = 10 +default_threads = 8 + +FILE_PATH = os.path.abspath(__file__) +CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".") + + +def get_dir(): + ppc_model_template_dir = f'{CURRENT_PATH}{os.sep}..{os.sep}ppc_model_template{os.sep}' + return ppc_model_template_dir + + +def parse_read_hetero_dataset_loop(participants): + loop_start = [] + loop_end = [] + start = '' + end = '' + for i in range(participants): + if i == 0 or i == participants - 1: + if i == 0: + start = f'{start}source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + else: + start = f'{start} + source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + else: + start = f'{start} + source{i}_feature_count' + end = f'{start} + source{i + 1}_feature_count' + loop_start.append(start) + loop_end.append(end) + return participants - 1, loop_start[0:participants - 1], loop_end[0:participants - 1] + + +def parse_read_homo_dataset_loop(participants): + loop_start = [] + loop_end = [] + start = '' + end = '' + for i in range(participants): + if i == 0 or i == participants - 1: + if i == 0: + start = f'{start}source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + else: + start = f'{start} + source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + else: + start = f'{start} + source{i}_record_count' + end = f'{start} + source{i + 1}_record_count' + loop_start.append(start) + loop_end.append(end) + return participants - 1, loop_start[0:participants - 1], loop_end[0:participants - 1] + + +def insert_train_record_count(layer, index, record_type): + if 'Dense' in layer: + layer_arr = layer.split('(') + else: + layer_arr = layer.split('([') + if index == 0 and 'Dense' in layer: + new_layer = f'{layer_arr[0]}({record_type}, total_feature_count, {layer_arr[1]}' + else: + if 'Dense' in layer: + new_layer = f'{layer_arr[0]}({record_type}, {layer_arr[1]}' + else: + new_layer = f'{layer_arr[0]}([{record_type}, {layer_arr[1]}' + if 'Conv2d' in new_layer: + if ', [' in new_layer: + layer_arr2 = new_layer.split(', [') + new_layer = f'{layer_arr2[0]}, [{record_type}, {layer_arr2[1]}' + if ',[' in new_layer: + layer_arr2 = new_layer.split(',[') + new_layer = f'{layer_arr2[0]},[{record_type}, {layer_arr2[1]}' + return new_layer + + +def set_nn_layers(mpc_algorithm, layers, record_type, ppc_model_type=None, model_algorithm_type=None, participants=None): + mpc_algorithm = f"{mpc_algorithm}\n" + if layers: + layers_str = 'layers = [' + for i in range(len(layers)): + new_layer = insert_train_record_count(layers[i], i, record_type) + layers_str = f"{layers_str}{new_layer},\n" + n_class = parse_n_class(layers[-1]) + if ppc_model_type == PPCModleType.Train: + mpc_algorithm = set_nn_train_output(mpc_algorithm, n_class) + else: + mpc_algorithm = set_nn_predict_output(mpc_algorithm, n_class) + mpc_algorithm = f"{mpc_algorithm}{layers_str}" + if n_class == 1: + mpc_algorithm = f"{mpc_algorithm}ml.Output({record_type}, approx=3)]\n\n" + if ppc_model_type == PPCModleType.Train: + mpc_algorithm = f"{mpc_algorithm}train_Y = layers[-1].Y\n" + mpc_algorithm = f"{mpc_algorithm}train_X = layers[0].X\n" + mpc_algorithm = f"{mpc_algorithm}test_Y = pint.Array(test_record_count)\n" + mpc_algorithm = f"{mpc_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + elif n_class > 1: + mpc_algorithm = f"{mpc_algorithm}ml.MultiOutput({record_type}, {n_class})]\n\n" + if ppc_model_type == PPCModleType.Train: + mpc_algorithm = f"{mpc_algorithm}total_class_count = {n_class}\n" + mpc_algorithm = f"{mpc_algorithm}train_Y = layers[-1].Y\n" + mpc_algorithm = f"{mpc_algorithm}train_X = layers[0].X\n" + mpc_algorithm = f"{mpc_algorithm}test_Y = pint.Matrix(test_record_count, total_class_count)\n" + mpc_algorithm = f"{mpc_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + else: + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_OUTPUT_NUMBER_ERROR.get_msg()) + + if ppc_model_type == PPCModleType.Train and model_algorithm_type == ModelAlgorithmType.HeteroNN: + mpc_algorithm = read_hetero_train_dataset( + mpc_algorithm, participants, n_class) + if ppc_model_type == PPCModleType.Train and model_algorithm_type == ModelAlgorithmType.HomoNN: + mpc_algorithm = read_homo_train_dataset( + mpc_algorithm, participants, n_class) + else: + mpc_algorithm = set_nn_train_output(mpc_algorithm, 1) + mpc_algorithm = f"{mpc_algorithm}" \ + f"layers = [pDense({record_type}, total_feature_count, 128, " \ + f"activation='relu'), pDense({record_type}, 128, 1), " \ + f"ml.Output({record_type}, approx=3)]\n" + if ppc_model_type == PPCModleType.Train: + mpc_algorithm = f"{mpc_algorithm}train_Y = layers[-1].Y\n" + mpc_algorithm = f"{mpc_algorithm}train_X = layers[0].X\n" + mpc_algorithm = f"{mpc_algorithm}test_Y = pint.Array(test_record_count)\n" + mpc_algorithm = f"{mpc_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + mpc_algorithm = f"{mpc_algorithm}\n" + mpc_algorithm = read_homo_train_dataset(mpc_algorithm, participants) + + return mpc_algorithm + + +def set_logreg_train_layers(mpc_algorithm): + mpc_algorithm = f"{mpc_algorithm}\n" + mpc_algorithm = f"{mpc_algorithm}layers = [pDense(train_record_count, total_feature_count, 1), ml.Output(train_record_count, approx=3)]\n\n" + mpc_algorithm = f"{mpc_algorithm}train_Y = layers[-1].Y\n" + mpc_algorithm = f"{mpc_algorithm}train_X = layers[0].X\n" + mpc_algorithm = f"{mpc_algorithm}test_Y = pint.Array(test_record_count)\n" + mpc_algorithm = f"{mpc_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n" + return mpc_algorithm + + +def generate_set_logreg_predict_layers(mpc_algorithm): + mpc_algorithm = f"{mpc_algorithm}\n" + mpc_algorithm = f"{mpc_algorithm}layers = [pDense(test_record_count, total_feature_count, 1), ml.Output(test_record_count, approx=3)]\n\n" + mpc_algorithm = f"{mpc_algorithm}result_columns = 1\n" + mpc_algorithm = f"{mpc_algorithm}result_matrix = Matrix(test_record_count, result_columns, pfix)\n\n" + return mpc_algorithm + + +def generate_homo_predict_static_template(mpc_predict_algorithm): + homo_nn_static_template = utils.read_content_from_file( + f'{get_dir()}homo_predict_static_template_proxy.mpc') + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}{homo_nn_static_template}' + return mpc_predict_algorithm + + +def set_hetero_train_static_template(model_config_dict, mpc_train_algorithm): + optimizer = model_config_dict['optimizer'] + learning_rate = model_config_dict['learning_rate'] + if optimizer == OptimizerType.sgd.name: + hetero_train_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_train_sgd_static_template_proxy.mpc') + hetero_train_static_template = hetero_train_static_template.replace('gamma = MemValue(cfix(.1))', + f'gamma = MemValue(cfix({learning_rate}))') + elif optimizer == OptimizerType.adam.name: + hetero_train_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_train_adam_static_template_proxy.mpc') + hetero_train_static_template = hetero_train_static_template.replace('gamma = MemValue(cfix(.001))', + f'gamma = MemValue(cfix({learning_rate}))') + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}{hetero_train_static_template}' + return mpc_train_algorithm + + +def generate_hetero_predict_static_template(mpc_predict_algorithm): + hetero_logreg_static_template = utils.read_content_from_file( + f'{get_dir()}hetero_predict_static_template_proxy.mpc') + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}{hetero_logreg_static_template}' + return mpc_predict_algorithm + + +def set_homo_train_static_template(model_config_dict, mpc_train_algorithm): + optimizer = model_config_dict['optimizer'] + learning_rate = model_config_dict['learning_rate'] + if optimizer == OptimizerType.sgd.name: + homo_train_static_template = utils.read_content_from_file( + f'{get_dir()}homo_train_sgd_static_template_proxy.mpc') + homo_train_static_template = homo_train_static_template.replace('gamma = MemValue(cfix(.1))', + f'gamma = MemValue(cfix({learning_rate}))') + elif optimizer == OptimizerType.adam.name: + homo_train_static_template = utils.read_content_from_file( + f'{get_dir()}homo_train_adam_static_template_proxy.mpc') + homo_train_static_template = homo_train_static_template.replace('gamma = MemValue(cfix(.001))', + f'gamma = MemValue(cfix({learning_rate}))') + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}{homo_train_static_template}' + return mpc_train_algorithm + + +def generate_set_common_code(is_psi, participants): + mpc_algorithm = '#This file is auto generated by ppc. DO NOT EDIT!\n\n' + if is_psi: + mpc_algorithm = f'{mpc_algorithm}#PSI_OPTION=True' + else: + mpc_algorithm = f'{mpc_algorithm}#PSI_OPTION=False' + mpc_algorithm = f'{mpc_algorithm}\n' + mpc_algorithm = f'{mpc_algorithm}from ppc import *\n' + mpc_algorithm = f'{mpc_algorithm}from Compiler import config\n' + mpc_algorithm = f'{mpc_algorithm}import sys\n' + mpc_algorithm = f'{mpc_algorithm}program.options_from_args()\n\n' + # if participants == 3: + # mpc_algorithm = f'{mpc_algorithm}program.use_trunc_pr = True\n' + # mpc_algorithm = f'{mpc_algorithm}program.use_split(3)\n\n' + for i in range(participants): + mpc_algorithm = f'{mpc_algorithm}SOURCE{i}={i}\n' + return mpc_algorithm + + +def set_hetero_feature_count(mpc_algorithm, participants): + total_feature_count_str = 'total_feature_count=' + for i in range(participants): + mpc_algorithm = f'{mpc_algorithm}source{i}_feature_count=$(source{i}_feature_count)\n' + if i == participants - 1: + total_feature_count_str = f'{total_feature_count_str}source{i}_feature_count' + else: + total_feature_count_str = f'{total_feature_count_str}source{i}_feature_count+' + mpc_algorithm = f'{mpc_algorithm}{total_feature_count_str}\n' + return mpc_algorithm + + +def set_homo_train_record_count(mpc_train_algorithm, participants): + total_record_count_str = 'total_record_count=' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_count=$(source{i}_record_count)\n' + if i == participants - 1: + total_record_count_str = f'{total_record_count_str}source{i}_record_count' + else: + total_record_count_str = f'{total_record_count_str}source{i}_record_count+' + mpc_train_algorithm = f'{mpc_train_algorithm}{total_record_count_str}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}train_record_count=$(train_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}test_record_count=$(test_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}total_feature_count=$(total_feature_count)\n' + return mpc_train_algorithm + + +def generate_set_homo_predict_record_count(mpc_predict_algorithm): + mpc_predict_algorithm = f'{mpc_predict_algorithm}total_feature_count=$(total_feature_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_record_count=$(test_record_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}file_offset = 0\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}file_offset = test_X.read_from_file(file_offset)\n' + return mpc_predict_algorithm + + +def set_hetero_train_record_count(mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}total_record_count=$(total_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}train_record_count=$(train_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}test_record_count=$(test_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + return mpc_train_algorithm + + +def generate_set_hetero_predict_record_count(mpc_predict_algorithm): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_record_count=$(test_record_count)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}test_X = pfix.Matrix(test_record_count, total_feature_count)\n' + return mpc_predict_algorithm + + +def read_hetero_train_dataset(mpc_train_algorithm, participants, n_class=1): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = 0\n' + for i in range(participants): + if i == 0: + if n_class == 1: + mpc_train_algorithm = f'{mpc_train_algorithm}source0_record_y = Array(total_record_count, pint)\n' + elif n_class > 1: + mpc_train_algorithm = f'{mpc_train_algorithm}source0_record_y = Matrix(total_record_count, total_class_count, pint)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}source0_record_x = Matrix(total_record_count, source0_feature_count, pfix)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = source0_record_y.read_from_file(file_offset)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = source0_record_x.read_from_file(file_offset)\n\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_x = Matrix(total_record_count, source{i}_feature_count, pfix)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = source{i}_record_x.read_from_file(file_offset)\n\n' + + mpc_train_algorithm = f'{mpc_train_algorithm}def read_hetero_dataset():\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"train_record_count:{{train_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"test_record_count:{{test_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_hetero_y_part()\n' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_hetero_x_part(source{i}_record_x, source{i}_feature_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + return mpc_train_algorithm + + +def generate_read_hetero_predict_dataset(mpc_predict_algorithm, participants): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}file_offset = 0\n' + for i in range(participants): + mpc_predict_algorithm = f'{mpc_predict_algorithm}source{i}_record_x = Matrix(test_record_count, source{i}_feature_count, pfix)\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}file_offset = source{i}_record_x.read_from_file(file_offset)\n\n' + + mpc_predict_algorithm = f'{mpc_predict_algorithm}\n' + mpc_predict_algorithm = f'{mpc_predict_algorithm}def read_hetero_test_dataset():\n' + for i in range(participants): + mpc_predict_algorithm = f'{mpc_predict_algorithm}\tdo_read_hetero_x_part(source{i}_record_x, source{i}_feature_count)\n' + return mpc_predict_algorithm + + +def read_homo_train_dataset(mpc_train_algorithm, participants, n_class=1): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = 0\n' + for i in range(participants): + if n_class == 1: + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_y = Array(source{i}_record_count, pint)\n' + elif n_class > 1: + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_y = Matrix(source{i}_record_count, total_class_count, pint)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}source{i}_record_x = Matrix(source{i}_record_count, total_feature_count, pfix)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = source{i}_record_y.read_from_file(file_offset)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}file_offset = source{i}_record_x.read_from_file(file_offset)\n\n' + + mpc_train_algorithm = f'{mpc_train_algorithm}def read_homo_dataset():\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"total_feature_count:{{total_feature_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"train_record_count:{{train_record_count}}")\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\tprint(f"test_record_count:{{test_record_count}}")\n' + for i in range(participants): + mpc_train_algorithm = f'{mpc_train_algorithm}\tdo_read_homo_dataset(source{i}_record_y, source{i}_record_x, source{i}_record_count)\n' + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + return mpc_train_algorithm + + +def set_parameters(model_config_dict, mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}\n' + epochs = model_config_dict['epochs'] + batch_size = model_config_dict['batch_size'] + threads = model_config_dict['threads'] + if epochs <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}epochs={default_epochs}\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}epochs={epochs}\n' + if int(batch_size) <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}batch_size=train_record_count\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}user_batch_size={batch_size}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}batch_size=min(user_batch_size, min(128, train_record_count))\n' + if threads <= 0: + mpc_train_algorithm = f'{mpc_train_algorithm}threads={default_threads}\n' + else: + mpc_train_algorithm = f'{mpc_train_algorithm}threads={threads}\n' + return mpc_train_algorithm + + +def set_lr_train_output(mpc_train_algorithm): + mpc_train_algorithm = f'{mpc_train_algorithm}result_columns = 2\n' + mpc_train_algorithm = f'{mpc_train_algorithm}result_matrix = Matrix(test_record_count, result_columns, pfix)\n\n' + return mpc_train_algorithm + + +def set_nn_train_output(mpc_train_algorithm, n_class): + mpc_train_algorithm = f'{mpc_train_algorithm}result_columns = {2*n_class}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}result_matrix = Matrix(test_record_count, result_columns, pfix)\n\n' + return mpc_train_algorithm + + +def set_nn_predict_output(mpc_train_algorithm, n_class): + mpc_train_algorithm = f'{mpc_train_algorithm}result_columns = {n_class}\n' + mpc_train_algorithm = f'{mpc_train_algorithm}result_matrix = Matrix(test_record_count, result_columns, pfix)\n\n' + return mpc_train_algorithm + + +def generate_mpc_train_algorithm(model_config_dict, algorithm_name, is_psi): + participants = model_config_dict['participants'] + mpc_train_algorithm = generate_set_common_code(is_psi, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name or algorithm_name == ModelAlgorithmType.HeteroNN.name: + mpc_train_algorithm = set_hetero_feature_count( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_hetero_train_record_count( + mpc_train_algorithm) + mpc_train_algorithm = set_parameters( + model_config_dict, mpc_train_algorithm) + if algorithm_name == ModelAlgorithmType.HeteroLR.name: + if 'layers' in model_config_dict.keys(): + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_msg()) + mpc_train_algorithm = read_hetero_train_dataset( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_lr_train_output(mpc_train_algorithm) + mpc_train_algorithm = set_logreg_train_layers(mpc_train_algorithm) + mpc_train_algorithm = set_hetero_train_static_template( + model_config_dict, mpc_train_algorithm) + else: + layers = [] + if 'layers' in model_config_dict.keys(): + layers = model_config_dict['layers'] + if layers and 'Conv2d' in layers[0]: + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR2.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR2.get_msg()) + mpc_train_algorithm = set_nn_layers( + mpc_train_algorithm, layers, 'train_record_count', PPCModleType.Train, ModelAlgorithmType.HeteroNN, participants) + mpc_train_algorithm = set_hetero_train_static_template( + model_config_dict, mpc_train_algorithm) + elif algorithm_name == ModelAlgorithmType.HomoLR.name or algorithm_name == ModelAlgorithmType.HomoNN.name: + mpc_train_algorithm = set_homo_train_record_count( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_parameters( + model_config_dict, mpc_train_algorithm) + if algorithm_name == ModelAlgorithmType.HomoLR.name: + if 'layers' in model_config_dict.keys(): + raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_MODEL_LAYERS_ERROR.get_msg()) + mpc_train_algorithm = read_homo_train_dataset( + mpc_train_algorithm, participants) + mpc_train_algorithm = set_lr_train_output(mpc_train_algorithm) + mpc_train_algorithm = set_logreg_train_layers(mpc_train_algorithm) + mpc_train_algorithm = set_homo_train_static_template( + model_config_dict, mpc_train_algorithm) + else: + layers = [] + if 'layers' in model_config_dict.keys(): + layers = model_config_dict['layers'] + mpc_train_algorithm = set_nn_layers( + mpc_train_algorithm, layers, 'train_record_count', PPCModleType.Train, ModelAlgorithmType.HomoNN, participants) + mpc_train_algorithm = set_homo_train_static_template( + model_config_dict, mpc_train_algorithm) + + return mpc_train_algorithm + + +def generate_mpc_predict_algorithm(algorithm_name, layers, participants, is_psi): + mpc_predict_algorithm = generate_set_common_code(is_psi, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name or algorithm_name == ModelAlgorithmType.HeteroNN.name: + mpc_predict_algorithm = set_hetero_feature_count( + mpc_predict_algorithm, participants) + mpc_predict_algorithm = generate_set_hetero_predict_record_count( + mpc_predict_algorithm) + mpc_predict_algorithm = generate_read_hetero_predict_dataset( + mpc_predict_algorithm, participants) + if algorithm_name == ModelAlgorithmType.HeteroLR.name: + mpc_predict_algorithm = generate_set_logreg_predict_layers( + mpc_predict_algorithm) + else: + mpc_predict_algorithm = set_nn_layers( + mpc_predict_algorithm, layers, 'test_record_count') + mpc_predict_algorithm = generate_hetero_predict_static_template( + mpc_predict_algorithm) + + elif algorithm_name == ModelAlgorithmType.HomoLR.name or algorithm_name == ModelAlgorithmType.HomoNN.name: + mpc_predict_algorithm = generate_set_homo_predict_record_count( + mpc_predict_algorithm) + if algorithm_name == ModelAlgorithmType.HomoLR.name: + mpc_predict_algorithm = generate_set_logreg_predict_layers( + mpc_predict_algorithm) + else: + mpc_predict_algorithm = set_nn_layers(mpc_predict_algorithm, layers, 'test_record_count', + PPCModleType.Predict, participants) + mpc_predict_algorithm = generate_homo_predict_static_template( + mpc_predict_algorithm) + + return mpc_predict_algorithm diff --git a/python/ppc_common/ppc_utils/tests/thread_safe_list_test.py b/python/ppc_common/ppc_utils/tests/thread_safe_list_test.py new file mode 100644 index 00000000..74028266 --- /dev/null +++ b/python/ppc_common/ppc_utils/tests/thread_safe_list_test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import unittest +import threading +from ppc_common.ppc_utils.thread_safe_list import ThreadSafeList + + +if __name__ == '__main__': + unittest.main() + +thread_safe_list = ThreadSafeList() + + +def test(object, thread_name, ut_obj): + try: + print(f"### thread: {thread_name}") + # append + thread_safe_list.append(object) + # contains + ut_obj.assertTrue(thread_safe_list.contains(object)) + # remove + thread_safe_list.remove(object) + ut_obj.assertTrue(thread_safe_list.contains(object) is False) + print(f"### result: {thread_safe_list.get()}") + # get element + copied_list = thread_safe_list.get() + if len(copied_list) >= 1: + thread_safe_list.get_element(len(copied_list) - 1) + thread_safe_list.get_element(len(copied_list)) + except Exception as e: + print(f"### Exception: {e}") + + +class TestThreadSafeList(unittest.TestCase): + def test_multi_thread(self): + loops = 5 + for j in range(loops): + thread_list = [] + thread_num = 20 + for i in range(thread_num): + thread_name = "t" + str(i) + "_" + str(j) + object = "job_" + str(j) + "_" + str(i) + t = threading.Thread(target=test, name=thread_name, args=(object, + thread_name, self)) + thread_list.append(t) + for t in thread_list: + t.start() + for t in thread_list: + t.join() diff --git a/python/ppc_common/ppc_utils/tests/utils_test.py b/python/ppc_common/ppc_utils/tests/utils_test.py new file mode 100644 index 00000000..42c11b4b --- /dev/null +++ b/python/ppc_common/ppc_utils/tests/utils_test.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +import unittest +from ppc_common.ppc_utils import utils +import os +import time + + +class FileOperationWrapper: + def __init__(self, file_chunk_dir, merged_file_path): + self.file_chunk_dir = file_chunk_dir + self.merged_file_path = merged_file_path + chunk_list = os.listdir(self.file_chunk_dir) + self.file_chunk_list = [] + for chunk in chunk_list: + self.file_chunk_list.append(os.path.join(file_chunk_dir, chunk)) + + +class TestFileOperations(unittest.TestCase): + def test_merge_and_calculate_hash_for_files(self): + chunk_file_dir = "/data/app/files/ppcs-modeladm/dataset" + # chunk_file_dir = "bak/object/train_test.csv" + wrapper = FileOperationWrapper(chunk_file_dir, "dataset1") + start_t = time.time() + print( + f"#### begin merge file for {len(wrapper.file_chunk_list)} chunks") + utils.merge_files(file_list=wrapper.file_chunk_list, + output_file=wrapper.merged_file_path) + print( + f"#### success merge file for {len(wrapper.file_chunk_list)} chunks success, time cost: {time.time() - start_t}") + + print( + f"#### calculate hash for {wrapper.merged_file_path}, size: {os.stat(wrapper.merged_file_path).st_size}") + start_t = time.time() + utils.calculate_md5(wrapper.merged_file_path) + print( + f"#### calculate hash for {wrapper.merged_file_path} success, size: {os.stat(wrapper.merged_file_path).st_size}, timecost: {time.time() - start_t}") + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_common/ppc_utils/thread_safe_list.py b/python/ppc_common/ppc_utils/thread_safe_list.py new file mode 100644 index 00000000..d34437f4 --- /dev/null +++ b/python/ppc_common/ppc_utils/thread_safe_list.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import threading + + +class ThreadSafeList: + def __init__(self): + self.processing_list = [] + self.lock = threading.Lock() + + def append(self, object): + try: + self.lock.acquire() + self.processing_list.append(object) + except Exception as e: + raise e + finally: + self.lock.release() + + def remove(self, object): + try: + self.lock.acquire() + self.processing_list.remove(object) + except Exception as e: + raise e + finally: + self.lock.release() + + def contains(self, object): + try: + self.lock.acquire() + return object in self.processing_list + except Exception as e: + raise e + finally: + self.lock.release() + + def get(self): + try: + copyed_list = [] + self.lock.acquire() + copyed_list.append(self.processing_list) + return copyed_list + except Exception as e: + raise e + finally: + self.lock.release() + + def get_element(self, i): + try: + self.lock.acquire() + return self.processing_list[i] + except Exception as e: + raise e + finally: + self.lock.release() diff --git a/python/ppc_common/ppc_utils/utils.py b/python/ppc_common/ppc_utils/utils.py new file mode 100644 index 00000000..d2564cce --- /dev/null +++ b/python/ppc_common/ppc_utils/utils.py @@ -0,0 +1,914 @@ +import base64 +import datetime +import hashlib +import io +import json +import logging +import os +import random +import re +import shutil +import string +import subprocess +import time +import uuid +from concurrent.futures.thread import ThreadPoolExecutor +from enum import Enum, IntEnum, unique + +import jwt +import pandas as pd +from ecdsa import SigningKey, SECP256k1, VerifyingKey +from gmssl import func, sm2, sm3 +from google.protobuf.descriptor import FieldDescriptor +from jsoncomment import JsonComment +from pysmx.SM3 import SM3 + +from ppc_common.ppc_protos.generated.ppc_pb2 import DatasetDetail +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + +log = logging.getLogger(__name__) + +MAX_SUPPORTED_PARTIES = 5 + +SERVER_RUNNING_STATUS = 0 +DEFAULT_DATASET_RECORD_COUNT = 5 +DEFAULT_PAGE_OFFSET = 0 +DEFAULT_PAGE_SIZE = 5 +MIN_PARTICIPATE_NUMBER = 2 +TWO_PARTY_PSI_PARTICIPATE_NUMBER = 2 +MIN_MULTI_PARTY_PSI_PARTICIPATE_NUMBER = 3 +MPC_MAX_SOURCE_COUNT = 50 +MPC_MAX_FIELD_COUNT = 50 + +BASE_RESPONSE = {'errorCode': PpcErrorCode.SUCCESS.get_code( +), 'message': PpcErrorCode.SUCCESS.get_msg()} +LOG_NAME = 'ppcs-modeladm-scheduler.log' +LOG_CHARACTER_NUMBER = 100000 +CSV_SEP = ',' +BLANK_SEP = ' ' +NORMALIZED_NAMES = 'field{}' + +PPC_RESULT_FIELDS_FLAG = 'result_fields' +PPC_RESULT_VALUES_FLAG = 'result_values' + +ADMIN_USER = 'admin' + +MPC_RECORD_PLACE_HOLDER = '$(ppc_max_record_count)' +MPC_START_PLACE_HOLDER = '${ph_start}' +MPC_END_PLACE_HOLDER = '${ph_end}' + +HOMO_MODEL_ALGORITHM = 'homo' +MPC_TOTAL_RECORD_COUNT_PLACE_HOLDER = '$(total_record_count)' +MPC_TRAIN_RECORD_COUNT_PLACE_HOLDER = '$(train_record_count)' +MPC_TEST_RECORD_COUNT_PLACE_HOLDER = '$(test_record_count)' +MPC_TOTAL_FEATURE_COUNT_PLACE_HOLDER = '$(total_feature_count)' + +XGB_TREE_PERFIX = "xgb_tree" +MPC_TRAIN_METRIC_ROC_FILE = "mpc_metric_roc.svg" +MPC_TRAIN_METRIC_KS_FILE = "mpc_metric_ks.svg" +MPC_TRAIN_METRIC_PR_FILE = "mpc_metric_pr.svg" +MPC_TRAIN_METRIC_ACCURACY_FILE = "mpc_metric_accuracy.svg" +MPC_TRAIN_METRIC_KS_TABLE = "mpc_metric_ks.csv" +MPC_TRAIN_SET_METRIC_ROC_FILE = "mpc_train_metric_roc.svg" +MPC_TRAIN_SET_METRIC_KS_FILE = "mpc_train_metric_ks.svg" +MPC_TRAIN_SET_METRIC_PR_FILE = "mpc_train_metric_pr.svg" +MPC_TRAIN_SET_METRIC_ACCURACY_FILE = "mpc_train_metric_accuracy.svg" +MPC_TRAIN_SET_METRIC_KS_TABLE = "mpc_train_metric_ks.csv" +MPC_EVAL_METRIC_ROC_FILE = "mpc_eval_metric_roc.svg" +MPC_EVAL_METRIC_KS_FILE = "mpc_eval_metric_ks.svg" +MPC_EVAL_METRIC_PR_FILE = "mpc_eval_metric_pr.svg" +MPC_EVAL_METRIC_ACCURACY_FILE = "mpc_eval_metric_accuracy.svg" +MPC_EVAL_METRIC_KS_TABLE = "mpc_eval_metric_ks.csv" +MPC_TRAIN_METRIC_CONFUSION_MATRIX_FILE = "mpc_metric_confusion_matrix.svg" +METRICS_OVER_ITERATION_FILE = "metrics_over_iterations.svg" + +# the ks-auc table, e.g.: +# |总样本|正样本| KS | AUC | +# 训练集| 500 | 100 | 0.4161 | 0.7685 | +# 验证集| 154 | 37 | 0.2897 | 0.6376 | +MPC_XGB_EVALUATION_TABLE = "mpc_xgb_evaluation_table.csv" + +# the feature-importance table, e.g.: +# |特征 | score | weight | score_rank| topk | +# | x1 | 0.08 | 1000 | 1 | | +# | x2 | 0.07 | 900 | 2 | | +XGB_FEATURE_IMPORTANCE_TABLE = "xgb_result_feature_importance_table.csv" + +# png, jpeg, pdf etc. ref: https://graphviz.org/docs/outputs/ +WORKFLOW_VIEW_FORMAT = 'svg' + +WORKFLOW_VIEW_NAME = 'workflow_view' + +PROXY_PSI_CIPHER_SUITE = "HMAC_BASED_PRIVATE_SET_INTERSECTION" +PROXY_MPC_CIPHER_SUITE = "SHA256_WITH_REPLICATED_SECRET_SHARING" +PROXY_PSI_MPC_CIPHER_SUITE = PROXY_PSI_CIPHER_SUITE + "-" + PROXY_MPC_CIPHER_SUITE +PPC_ALL_AUTH_FLAG = "PPC_ALGO_ALL" + +CEM_CIPHER_LEN = 288 + + +@unique +class PPCModleType(Enum): + Train = 1 + Predict = 2 + + +@unique +class CryptoType(Enum): + ECDSA = 1 + GM = 2 + + +@unique +class HashType(Enum): + BYTES = 1 + HEXSTR = 2 + + +class DeployMode(IntEnum): + PRIVATE_MODE = 1 + SAAS_MODE = 2 + PROXY_MODE = 3 + + +class JobRole(Enum): + ALGORITHM_PROVIDER = 1 + COMPUTATION_PROVIDER = 2 + DATASET_PROVIDER = 3 + DATA_CONSUMER = 4 + + +class JobStatus(Enum): + SUCCEED = 1 + FAILED = 2 + RUNNING = 3 + WAITING = 4 + NONE = 5 + RETRY = 6 + + +class AlgorithmType(Enum): + Train = "Train", + Predict = "Predict" + + +class AlgorithmSubtype(Enum): + HeteroLR = 1 + HomoLR = 2 + HeteroNN = 3 + HomoNN = 4 + HeteroXGB = 5 + + +class DataAlgorithmType(Enum): + PIR = 'pir' + CEM = 'cem' + ALL = 'all' + + +class IdPrefixEnum(Enum): + DATASET = "d-" + ALGORITHM = "a-" + JOB = "j-" + + +class JobCemResultSuffixEnum(Enum): + JOB_ALGORITHM_AGENCY = "-1" + INITIAL_JOB_AGENCY = "-2" + + +class OriginAlgorithm(Enum): + PPC_AYS = ["a-1001", "匿踪查询", "1", "1.0"] + PPC_PSI = ["a-1002", "隐私求交", "2+", "1.0"] + + +executor = ThreadPoolExecutor(max_workers=10) + + +def async_fn(func): + def wrapper(*args, **kwargs): + executor.submit(func, *args, **kwargs) + + return wrapper + + +def json_loads(json_config): + try: + json_comment = JsonComment(json) + return json_comment.loads(json_config) + except BaseException: + raise PpcException(PpcErrorCode.ALGORITHM_PPC_CONFIG_ERROR.get_code(), + PpcErrorCode.ALGORITHM_PPC_CONFIG_ERROR.get_msg()) + + +def parse_n_class(layer_str): + if layer_str == '[]': + return 1 + else: + return int(re.findall(r"\d+\.?\d*", layer_str)[-1]) + + +def check_ppc_model_algorithm_is_homo(algorithm_name): + return algorithm_name[0:4].lower() == HOMO_MODEL_ALGORITHM + + +def get_log_file_path(app_dir): + return os.sep.join([app_dir, "logs", LOG_NAME]) + + +def get_log_temp_file_path(app_dir, job_id): + return os.sep.join([app_dir, "logs", f"{job_id}.log"]) + + +def df_to_dict(df, orient='split'): + data_json_str = df.to_json(orient=orient, force_ascii=False) + data_dict = json.loads(data_json_str) + del data_dict['index'] + return data_dict + + +def file_exists(_file): + if os.path.exists(_file) and os.path.isfile(_file): + return True + return False + + +def decode_jwt(token): + """ + decode jwt + :param token: + :return: + """ + result = {"data": None, "error": None} + try: + payload = jwt.decode(token.split(" ")[1], options={ + "verify_signature": False}) + result["data"] = payload + except (IndexError, jwt.DecodeError): + result["error"] = "JWT token is decoded fail" + return result + + +def make_timestamp(): + return int(round(time.time() * 1000)) + + +def encode(data_bytes): + return base64.b64encode(data_bytes) + + +def decode(data_str): + return base64.b64decode(data_str) + + +def pb_to_str(data_pb): + return encode(data_pb.SerializeToString()).decode("utf-8") + + +def str_to_pb(data_pb, data_str): + data_pb.ParseFromString(decode(data_str)) + return data_pb + + +def pb_to_bytes(data_pb): + return data_pb.SerializeToString() + + +def bytes_to_pb(data_pb, data_bytes): + return data_pb.ParseFromString(data_bytes) + + +def str_to_base64str(data_str): + message_bytes = data_str.encode('utf-8') + base64_bytes = base64.b64encode(message_bytes) + base64_message = base64_bytes.decode('utf-8') + return base64_message + + +def base64str_to_str(base64_str): + base64_bytes = base64_str.encode('utf-8') + message_bytes = base64.b64decode(base64_bytes) + message = message_bytes.decode('utf-8') + return message + + +def bytes_to_base64str(data_bytes): + base64_bytes = base64.b64encode(data_bytes) + base64_message = base64_bytes.decode('utf-8') + return base64_message + + +def make_response(code, message, data=None): + return {'errorCode': code, 'message': message, 'data': data} + + +def base64str_to_bytes(base64_str): + base64_bytes = base64_str.encode('utf-8') + message_bytes = base64.b64decode(base64_bytes) + return message_bytes + + +def make_hash_from_file_path(file_path, crypto_type): + file_data = read_content_from_file_by_binary(file_path) + return make_hash(file_data, crypto_type, HashType.HEXSTR) + + +def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): + while True: + chunk = bytes(file.read(size), 'utf-8') + if not chunk: + break + yield chunk + + +def make_hash_from_file_path_by_chunks(file_path, crypto_type, block_size=1 << 10): + m = SM3() + if crypto_type == CryptoType.ECDSA: + m = hashlib.sha3_256() + dataset_size = 0 + with open(file_path, 'r', encoding='utf-8') as f: + for block in read_chunks(f, size=block_size): + dataset_size += len(block) + m.update(block) + return m.hexdigest(), dataset_size + + +def read_content_from_file_by_binary(file_path): + with open(file_path, 'rb') as file: + content = file.read() + return content + + +def make_hash(data, crypto_type, hash_type=None): + if crypto_type == CryptoType.ECDSA: + m = hashlib.sha3_256() + m.update(data) + if hash_type == HashType.HEXSTR: + return m.hexdigest() + if hash_type == HashType.BYTES: + return m.digest() + if crypto_type == CryptoType.GM: + return sm3.sm3_hash(func.bytes_to_list(data)) + + +def pb2dict(obj): + """ + Takes a ProtoBuf Message obj and convertes it to a dict. + """ + adict = {} + if not obj.IsInitialized(): + return None + for field in obj.DESCRIPTOR.fields: + if not getattr(obj, field.name): + continue + if not field.label == FieldDescriptor.LABEL_REPEATED: + if not field.type == FieldDescriptor.TYPE_MESSAGE: + adict[field.name] = getattr(obj, field.name) + else: + value = pb2dict(getattr(obj, field.name)) + if value: + adict[field.name] = value + else: + if field.type == FieldDescriptor.TYPE_MESSAGE: + adict[field.name] = \ + [pb2dict(v) for v in getattr(obj, field.name)] + else: + adict[field.name] = [v for v in getattr(obj, field.name)] + return adict + + +def write_content_to_file(content, file_path): + with open(file_path, 'w', encoding="utf-8") as file: + file.write(content) + + +def write_content_to_file_by_append(content, file_path): + with open(file_path, 'a', encoding="utf-8") as file: + file.write(content) + + +def read_content_from_file(file_path): + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + return content + + +def make_job_event_message(job_id, job_priority, initiator_agency_id, receiver_agency_id, job_algorithm_id, + job_dataset): + message = '{}|{}|{}|{}|{}'.format(job_id, job_priority, initiator_agency_id, receiver_agency_id, job_algorithm_id, + job_dataset) + return message.encode('utf-8') + + +def sign_with_secp256k1(message, private_key_str): + if isinstance(private_key_str, str): + private_key = SigningKey.from_string( + bytes().fromhex(private_key_str), curve=SECP256k1) + else: + private_key = SigningKey.from_string(private_key_str, curve=SECP256k1) + signature_bytes = private_key.sign( + make_hash(message, CryptoType.ECDSA, HashType.BYTES)) + return str(encode(signature_bytes), 'utf-8') + + +def verify_with_secp256k1(message, signature_str, public_key_str): + if isinstance(public_key_str, str): + verify_key = VerifyingKey.from_string( + bytes().fromhex(public_key_str), curve=SECP256k1) + else: + verify_key = VerifyingKey.from_string( + decode(public_key_str), curve=SECP256k1) + return verify_key.verify(decode(signature_str), make_hash(message, CryptoType.ECDSA, HashType.BYTES)) + + +def sign_with_sm2(message, private_key_str): + sm2_crypt = sm2.CryptSM2(private_key_str, "") + random_hex_str = func.random_hex(sm2_crypt.para_len) + message_hash = make_hash(message, CryptoType.GM).encode(encoding='utf-8') + signature_str = sm2_crypt.sign(message_hash, random_hex_str) + return signature_str + + +def verify_with_sm2(message, signature_str, public_key_str): + sm2_crypt = sm2.CryptSM2("", public_key_str) + message_hash = make_hash(message, CryptoType.GM).encode(encoding='utf-8') + return sm2_crypt.verify(signature_str, message_hash) + + +def make_signature(message, private_key_str, crypto_type): + if crypto_type == CryptoType.ECDSA: + return sign_with_secp256k1(message, private_key_str) + + if crypto_type == CryptoType.GM: + return sign_with_sm2(message, private_key_str) + + +def verify_signature(message, signature, public_key_str, crypto_type): + if crypto_type == CryptoType.ECDSA: + return verify_with_secp256k1(message, signature, public_key_str) + + if crypto_type == CryptoType.GM: + return verify_with_sm2(message, signature, public_key_str) + + +def exec_bash_command(cmd): + """replace commands.get_status_output + + Arguments: + cmd {[string]} + """ + + get_cmd = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ret = get_cmd.communicate() + out = ret[0] + err = ret[1] + output = '' + if out is not None: + output = output + out.decode('utf-8') + if err is not None: + output = output + err.decode('utf-8') + + return get_cmd.returncode, output + + +def delete_file(path): + """[delete data_dir] + + Arguments: + path {[get_dir]} -- [description] + """ + + if os.path.isfile(path): + os.remove(path) + elif os.path.isdir(path): + shutil.rmtree(path) + else: + raise (Exception(' path not exisited ! path => %s', path)) + + +def make_dir(_dir): + if not os.path.exists(_dir): + os.mkdir(_dir) + + +def load_credential_from_file(filepath): + real_path = os.path.join(os.path.dirname(__file__), filepath) + with open(real_path, 'rb') as f: + return f.read() + + +def getstatusoutput(cmd): + """replace commands.getstatusoutput + + Arguments: + cmd {[string]} + """ + + get_cmd = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ret = get_cmd.communicate() + out = ret[0] + err = ret[1] + output = '' + if not out is None: + output = output + out.decode('utf-8') + if not err is None: + output = output + err.decode('utf-8') + + log.debug(' cmd is %s, status is %s, output is %s', + cmd, str(get_cmd.returncode), output) + + return (get_cmd.returncode, output) + + +def make_dataset_metadata(data_field_list, dataset_description, dataset_size, row_count, column_count, + version_hash, user_name): + dataset_metadata = DatasetDetail() + dataset_metadata.data_field_list = data_field_list + dataset_metadata.dataset_description = dataset_description + dataset_metadata.dataset_size = dataset_size + dataset_metadata.row_count = row_count + dataset_metadata.column_count = column_count + dataset_metadata.dataset_hash = version_hash + dataset_metadata.user_name = user_name + + date_time = make_timestamp() + dataset_metadata.create_time = date_time + dataset_metadata.update_time = date_time + return pb_to_str(dataset_metadata) + + +def default_model_module(subtype): + defaultModelModule = [ + { + 'label': 'use_psi', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否进行隐私求交,只能为0或1' + }, + { + 'label': 'fillna', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否进行缺失值填充,只能为0或1' + }, + { + 'label': 'na_select', + 'type': 'float', + 'value': 1, + 'min_value': 0, + 'max_value': 1, + 'description': '缺失值筛选阈值,取值范围为0~1之间(0表示只要有缺失值就移除,1表示移除全为缺失值的列)' + }, + { + 'label': 'filloutlier', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否进行异常值处理,只能为0或1' + }, + { + 'label': 'normalized', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否归一化,只能为0或1' + }, + { + 'label': 'standardized', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否标准化,只能为0或1' + }, + { + 'label': 'categorical', + 'type': 'string', + 'value': '', + 'description': '标记所有分类特征字段，格式："x1,x12" (空代表无分类特征)' + }, + { + 'label': 'psi_select_col', + 'type': 'string', + 'value': '', + 'description': 'PSI稳定性筛选时间列名(空代表不进行PSI筛选)' + }, + { + 'label': 'psi_select_base', + 'type': 'string', + 'value': '', + 'description': 'PSI稳定性筛选的基期(空代表不进行PSI筛选)' + }, + { + 'label': 'psi_select_thresh', + 'type': 'float', + 'value': 0.3, + 'min_value': 0, + 'max_value': 1, + 'description': 'PSI筛选阈值,取值范围为0~1之间' + }, + { + 'label': 'psi_select_bins', + 'type': 'int', + 'value': 4, + 'min_value': 3, + 'max_value': 100, + 'description': '计算PSI时分箱数, 取值范围为3~100之间' + }, + { + 'label': 'corr_select', + 'type': 'float', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '特征相关性筛选阈值,取值范围为0~1之间(值为0时不进行相关性筛选)' + }, + { + 'label': 'use_iv', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否使用iv进行特征筛选,只能为0或1' + }, + { + 'label': 'group_num', + 'type': 'int', + 'value': 4, + 'min_value': 3, + 'max_value': 100, + 'description': 'woe计算分箱数,取值范围为3~100之间的整数' + }, + { + 'label': 'iv_thresh', + 'type': 'float', + 'value': 0.1, + 'min_value': 0.01, + 'max_value': 1, + 'description': 'iv特征筛选的阈值,取值范围为0.01~1之间' + }] + + if subtype == 'HeteroXGB': + defaultModelModule.extend([ + { + 'label': 'use_goss', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '是否采用goss抽样加速训练, 只能为0或1' + }, + { + 'label': 'test_dataset_percentage', + 'type': 'float', + 'value': 0.3, + 'min_value': 0.1, + 'max_value': 0.5, + 'description': '测试集比例, 取值范围为0.1~0.5之间' + }, + { + 'label': 'learning_rate', + 'type': 'float', + 'value': 0.1, + 'min_value': 0.01, + 'max_value': 1, + 'description': '学习率, 取值范围为0.01~1之间' + }, + { + 'label': 'num_trees', + 'type': 'int', + 'value': 6, + 'min_value': 1, + 'max_value': 300, + 'description': 'XGBoost迭代树棵树, 取值范围为1~300之间的整数' + }, + { + 'label': 'max_depth', + 'type': 'int', + 'value': 3, + 'min_value': 1, + 'max_value': 6, + 'description': 'XGBoost树深度, 取值范围为1~6之间的整数' + }, + { + 'label': 'max_bin', + 'type': 'int', + 'value': 4, + 'min_value': 3, + 'max_value': 100, + 'description': '特征分箱数, 取值范围为3~100之间的整数' + }, + { + 'label': 'silent', + 'type': 'bool', + 'value': 0, + 'min_value': 0, + 'max_value': 1, + 'description': '值为1时不打印运行信息,只能为0或1' + }, + { + 'label': 'subsample', + 'type': 'float', + 'value': 1, + 'min_value': 0.1, + 'max_value': 1, + 'description': '训练每棵树使用的样本比例,取值范围为0.1~1之间' + }, + { + 'label': 'colsample_bytree', + 'type': 'float', + 'value': 1, + 'min_value': 0.1, + 'max_value': 1, + 'description': '训练每棵树使用的特征比例,取值范围为0.1~1之间' + }, + { + 'label': 'colsample_bylevel', + 'type': 'float', + 'value': 1, + 'min_value': 0.1, + 'max_value': 1, + 'description': '训练每一层使用的特征比例,取值范围为0.1~1之间' + }, + { + 'label': 'reg_alpha', + 'type': 'float', + 'value': 0, + 'min_value': 0, + 'description': 'L1正则化项，用于控制模型复杂度,取值范围为大于等于0的数值' + }, + { + 'label': 'reg_lambda', + 'type': 'float', + 'value': 1, + 'min_value': 0, + 'description': 'L2正则化项,用于控制模型复杂度,取值范围为大于等于0的数值' + }, + { + 'label': 'gamma', + 'type': 'float', + 'value': 0, + 'min_value': 0, + 'description': '最优分割点所需的最小损失函数下降值,取值范围为大于等于0的数值' + }, + { + 'label': 'min_child_weight', + 'type': 'float', + 'value': 0, + 'min_value': 0, + 'description': '最优分割点所需的最小叶子节点权重,取值范围为大于等于0的数值' + }, + { + 'label': 'min_child_samples', + 'type': 'int', + 'value': 10, + 'min_value': 1, + 'max_value': 1000, + 'description': '最优分割点所需的最小叶子节点样本数量,取值范围为1~1000之间的整数' + }, + { + 'label': 'seed', + 'type': 'int', + 'value': 2024, + 'min_value': 0, + 'max_value': 10000, + 'description': '分割训练集/测试集时随机数种子,取值范围为0~10000之间的整数' + }, + { + 'label': 'early_stopping_rounds', + 'type': 'int', + 'value': 5, + 'min_value': 0, + 'max_value': 100, + 'description': '指定迭代多少次没有提升则停止训练, 值为0时不执行, 取值范围为0~100之间的整数' + }, + { + 'label': 'eval_metric', + 'type': 'string', + 'value': 'auc', + 'description': '早停的评估指标,支持:auc, acc, recall, precision' + }, + { + 'label': 'verbose_eval', + 'type': 'int', + 'value': 1, + 'min_value': 0, + 'max_value': 100, + 'description': '按传入的间隔输出训练过程中的评估信息,0表示不打印' + }, + { + 'label': 'eval_set_column', + 'type': 'string', + 'value': '', + 'description': '指定训练集测试集标记字段名称' + }, + { + 'label': 'train_set_value', + 'type': 'string', + 'value': '', + 'description': '指定训练集标记值' + }, + { + 'label': 'eval_set_value', + 'type': 'string', + 'value': '', + 'description': '指定测试集标记值' + }, + { + 'label': 'train_features', + 'type': 'string', + 'value': '', + 'description': '指定入模特征' + } # , + # { + # 'label': 'threads', + # 'type': 'int', + # 'value': 8, + # 'min_value': 1, + # 'max_value': 8, + # 'description': '取值范围为1~8之间的整数' + # } + ]) + else: + defaultModelModule.extend([ + { + 'label': 'test_dataset_percentage', + 'type': 'float', + 'value': 0.3, + 'min_value': 0.1, + 'max_value': 0.5, + 'description': '取值范围为0.1~0.5之间' + }, + { + 'label': 'learning_rate', + 'type': 'float', + 'value': 0.3, + 'min_value': 0.001, + 'max_value': 0.999, + 'description': '取值范围为0.001~0.999之间' + }, + { + 'label': 'epochs', + 'type': 'int', + 'value': 3, + 'min_value': 1, + 'max_value': 5, + 'description': '取值范围为1~5之间的整数' + }, + { + 'label': 'batch_size', + 'type': 'int', + 'value': 16, + 'min_value': 1, + 'max_value': 100, + 'description': '取值范围为1~100之间的整数' + }, + { + 'label': 'threads', + 'type': 'int', + 'value': 8, + 'min_value': 1, + 'max_value': 8, + 'description': '取值范围为1~8之间的整数' + } + ]) + + return defaultModelModule + + +def merge_files(file_list, output_file): + try: + with open(output_file, 'wb') as outfile: + for file_name in file_list: + with open(file_name, 'rb') as f: + outfile.write(f.read()) + except Exception as pe: + log.info(f"merge files failed: {pe}") + raise PpcException(-1, f"merge files failed for: {pe}") + + +def md5sum(data_content): + md5_hash = hashlib.md5() + data = data_content if type( + data_content) is bytes else bytes(data_content, "utf-8") + md5_hash.update(data) + return md5_hash.hexdigest() + + +def calculate_md5(file_path, granularity=2 * 1024 * 1024): + md5_hash = hashlib.md5() + + with open(file_path, 'rb') as file: + # 逐块读取文件内容，以提高性能 + for chunk in iter(lambda: file.read(granularity), b''): + md5_hash.update(chunk) + + return md5_hash.hexdigest() diff --git a/python/ppc_model/__init__.py b/python/ppc_model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/common/__init__.py b/python/ppc_model/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py new file mode 100644 index 00000000..a6c1b582 --- /dev/null +++ b/python/ppc_model/common/base_context.py @@ -0,0 +1,83 @@ +import os +from ppc_common.ppc_utils import utils + + +class BaseContext: + PSI_RESULT_FILE = "psi_result.csv" + MODEL_PREPARE_FILE = "model_prepare.csv" + PREPROCESSING_RESULT_FILE = "preprocessing_result.csv" + EVAL_COLUMN_FILE = "model_eval_column.csv" + WOE_IV_FILE = 'woe_iv.csv' + IV_SELECTED_FILE = 'iv_selected.csv' + SELECTED_COL_FILE = "xgb_result_column_info_selected.csv" + + # TODO: rename xgb filename + FEATURE_BIN_FILE = "feature_bin.json" + # MODEL_DATA_FILE = "model_data.json" + MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' + TEST_MODEL_RESULT_FILE = "model_result.csv" + # TEST_MODEL_OUTPUT_FILE = "model_output.csv" + TEST_MODEL_OUTPUT_FILE = "xgb_output.csv" + TRAIN_MODEL_RESULT_FILE = "train_model_result.csv" + # TRAIN_MODEL_OUTPUT_FILE = "train_model_output.csv" + TRAIN_MODEL_OUTPUT_FILE = "xgb_train_output.csv" + + def __init__(self, job_id: str, job_temp_dir: str): + self.job_id = job_id + self.workspace = os.path.join(job_temp_dir, self.job_id) + if not os.path.exists(self.workspace): + os.makedirs(self.workspace) + self.psi_result_path = os.path.join(self.workspace, self.PSI_RESULT_FILE) + self.model_prepare_file = os.path.join(self.workspace, self.MODEL_PREPARE_FILE) + self.preprocessing_result_file = os.path.join(self.workspace, self.PREPROCESSING_RESULT_FILE) + self.eval_column_file = os.path.join(self.workspace, self.EVAL_COLUMN_FILE) + self.woe_iv_file = os.path.join(self.workspace, self.WOE_IV_FILE) + self.iv_selected_file = os.path.join(self.workspace, self.IV_SELECTED_FILE) + self.selected_col_file = os.path.join(self.workspace, self.SELECTED_COL_FILE) + self.remote_selected_col_file = os.path.join(self.job_id, self.SELECTED_COL_FILE) + + self.summary_evaluation_file = os.path.join(self.workspace, utils.MPC_XGB_EVALUATION_TABLE) + self.feature_importance_file = os.path.join(self.workspace, utils.XGB_FEATURE_IMPORTANCE_TABLE) + self.feature_bin_file = os.path.join(self.workspace, self.FEATURE_BIN_FILE) + self.model_data_file = os.path.join(self.workspace, self.MODEL_DATA_FILE) + self.test_model_result_file = os.path.join(self.workspace, self.TEST_MODEL_RESULT_FILE) + self.test_model_output_file = os.path.join(self.workspace, self.TEST_MODEL_OUTPUT_FILE) + self.train_model_result_file = os.path.join(self.workspace, self.TRAIN_MODEL_RESULT_FILE) + self.train_model_output_file = os.path.join(self.workspace, self.TRAIN_MODEL_OUTPUT_FILE) + + self.train_metric_roc_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) + self.train_metric_ks_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_FILE) + self.train_metric_pr_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_PR_FILE) + self.train_metric_acc_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) + self.test_metric_roc_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_ROC_FILE) + self.test_metric_ks_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_KS_FILE) + self.test_metric_pr_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_PR_FILE) + self.test_metric_acc_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) + self.train_metric_ks_table = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) + self.test_metric_ks_table = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_KS_TABLE) + self.model_tree_prefix = os.path.join(self.workspace, utils.XGB_TREE_PERFIX) + self.metrics_iteration_file = os.path.join(self.workspace, utils.METRICS_OVER_ITERATION_FILE) + + self.remote_summary_evaluation_file = os.path.join(self.job_id, utils.MPC_XGB_EVALUATION_TABLE) + self.remote_feature_importance_file = os.path.join(self.job_id, utils.XGB_FEATURE_IMPORTANCE_TABLE) + self.remote_feature_bin_file = os.path.join(self.job_id, self.FEATURE_BIN_FILE) + self.remote_model_data_file = os.path.join(self.job_id, self.MODEL_DATA_FILE) + self.remote_test_model_output_file = os.path.join(self.job_id, self.TEST_MODEL_OUTPUT_FILE) + self.remote_train_model_output_file = os.path.join(self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) + + self.remote_train_metric_roc_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) + self.remote_train_metric_ks_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_FILE) + self.remote_train_metric_pr_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_PR_FILE) + self.remote_train_metric_acc_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) + self.remote_test_metric_roc_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_ROC_FILE) + self.remote_test_metric_ks_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_KS_FILE) + self.remote_test_metric_pr_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_PR_FILE) + self.remote_test_metric_acc_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) + self.remote_train_metric_ks_table = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) + self.remote_test_metric_ks_table = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_KS_TABLE) + self.remote_model_tree_prefix = os.path.join(self.job_id, utils.XGB_TREE_PERFIX) + self.remote_metrics_iteration_file = os.path.join(self.job_id, utils.METRICS_OVER_ITERATION_FILE) + + @staticmethod + def feature_engineering_input_path(job_id: str, job_temp_dir: str): + return os.path.join(job_temp_dir, job_id, BaseContext.MODEL_PREPARE_FILE) diff --git a/python/ppc_model/common/context.py b/python/ppc_model/common/context.py new file mode 100644 index 00000000..02614e47 --- /dev/null +++ b/python/ppc_model/common/context.py @@ -0,0 +1,14 @@ +import unittest + +from ppc_model.common.base_context import BaseContext +from ppc_model.common.initializer import Initializer +from ppc_model.common.protocol import TaskRole + + +class Context(BaseContext): + + def __init__(self, job_id: str, task_id: str, components: Initializer, role: TaskRole = None): + super().__init__(job_id, components.config_data['JOB_TEMP_DIR']) + self.task_id = task_id + self.components = components + self.role = role diff --git a/python/ppc_model/common/global_context.py b/python/ppc_model/common/global_context.py new file mode 100644 index 00000000..13437d90 --- /dev/null +++ b/python/ppc_model/common/global_context.py @@ -0,0 +1,13 @@ +import os +import threading + +from ppc_model.common.initializer import Initializer + +dirName, _ = os.path.split(os.path.abspath(__file__)) +# config_path = '{}/../application.yml'.format(dirName) +config_path = "application.yml" + +components = Initializer(log_config_path='logging.conf', config_path=config_path) + +# matplotlib 线程不安全，并行任务绘图增加全局锁 +plot_lock = threading.Lock() diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py new file mode 100644 index 00000000..6dd6258f --- /dev/null +++ b/python/ppc_model/common/initializer.py @@ -0,0 +1,108 @@ +import logging +import logging.config +import os + +import yaml + +from ppc_common.deps_services import storage_loader +from ppc_common.deps_services.storage_api import StorageType +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_utils import common_func +from ppc_model.network.grpc.grpc_client import GrpcClient +from ppc_model.network.stub import ModelStub +from ppc_model.task.task_manager import TaskManager + + +class Initializer: + def __init__(self, log_config_path, config_path): + self.log_config_path = log_config_path + self.config_path = config_path + self.config_data = None + self.grpc_options = None + self.stub = None + self.task_manager = None + self.thread_event_manager = None + self.storage_client = None + # 只用于测试 + self.mock_logger = None + self.public_key_length = 2048 + self.homo_algorithm = 0 + + def init_all(self): + self.init_log() + self.init_config() + self.init_stub() + self.init_task_manager() + self.init_storage_client() + self.init_cache() + + def init_log(self): + logging.config.fileConfig(self.log_config_path) + + def init_cache(self): + self.job_cache_dir = common_func.get_config_value( + "JOB_TEMP_DIR", "/tmp", self.config_data, False) + + def init_config(self): + with open(self.config_path, 'rb') as f: + self.config_data = yaml.safe_load(f.read()) + self.public_key_length = self.config_data['PUBLIC_KEY_LENGTH'] + storage_type = common_func.get_config_value( + "STORAGE_TYPE", "HDFS", self.config_data, False) + if 'HOMO_ALGORITHM' in self.config_data: + self.homo_algorithm = self.config_data['HOMO_ALGORITHM'] + + def init_stub(self): + self.thread_event_manager = ThreadEventManager() + self.grpc_options = [ + ('grpc.ssl_target_name_override', 'PPCS MODEL GATEWAY'), + ('grpc.max_send_message_length', + self.config_data['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024), + ('grpc.max_receive_message_length', + self.config_data['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024), + ('grpc.keepalive_time_ms', 15000), # 每 15 秒发送一次心跳 + ('grpc.keepalive_timeout_ms', 5000), # 等待心跳回应的超时时间为 5 秒 + ('grpc.keepalive_permit_without_calls', True), # 即使没有调用也允许发送心跳 + ('grpc.http2.min_time_between_pings_ms', 15000), # 心跳之间最小时间间隔为 15 秒 + ('grpc.http2.max_pings_without_data', 0), # 在发送数据前不限制心跳次数 + # 在没有数据传输的情况下，确保心跳包之间至少有 20 秒的间隔 + ('grpc.http2.min_ping_interval_without_data_ms', 20000), + ("grpc.so_reuseport", 1), + ("grpc.use_local_subchannel_pool", 1), + ('grpc.enable_retries', 1), + ('grpc.service_config', + '{ "retryPolicy":{ "maxAttempts": 4, "initialBackoff": "0.1s", "maxBackoff": "1s", "backoffMutiplier": ' + '2, "retryableStatusCodes": [ "UNAVAILABLE" ] } }') + ] + rpc_client = GrpcClient( + logger=self.logger(), + endpoint=self.config_data['GATEWAY_ENDPOINT'], + grpc_options=self.grpc_options, + ssl_switch=self.config_data['SSL_SWITCH'], + ca_path=self.config_data['CA_CRT'], + ssl_key_path=self.config_data['SSL_KEY'], + ssl_crt_path=self.config_data['SSL_CRT'] + ) + self.stub = ModelStub( + agency_id=self.config_data['AGENCY_ID'], + thread_event_manager=self.thread_event_manager, + rpc_client=rpc_client + ) + + def init_task_manager(self): + self.task_manager = TaskManager( + logger=self.logger(), + thread_event_manager=self.thread_event_manager, + stub=self.stub, + task_timeout_h=self.config_data['TASK_TIMEOUT_H'] + ) + + def init_storage_client(self): + self.storage_client = storage_loader.load( + self.config_data, self.logger()) + + def logger(self, name=None): + if self.mock_logger is None: + return logging.getLogger(name) + else: + return self.mock_logger diff --git a/python/ppc_model/common/mock/__init__.py b/python/ppc_model/common/mock/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/common/mock/rpc_client_mock.py b/python/ppc_model/common/mock/rpc_client_mock.py new file mode 100644 index 00000000..c3094c4d --- /dev/null +++ b/python/ppc_model/common/mock/rpc_client_mock.py @@ -0,0 +1,31 @@ +import threading + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse +from ppc_model.interface.rpc_client import RpcClient + + +class RpcClientMock(RpcClient): + def __init__(self, need_failed=False): + self._need_failed = need_failed + self._bad_guy = 0 + self._lock = threading.Lock() + self._on_message_received = None + + def set_message_handler(self, on_message_received): + self._on_message_received = on_message_received + + def send(self, request: ModelRequest): + # print( + # f"send data to {request.receiver}, task_id: {request.task_id}, " + # f"key: {request.key}, seq: {request.seq}") + self._on_message_received(request) + response = ModelResponse() + if self._need_failed: + # 模拟网络断连 + with self._lock: + self._bad_guy += 1 + response.base_response.error_code = self._bad_guy % 2 + else: + response.base_response.error_code = 0 + response.base_response.message = "success" + return response diff --git a/python/ppc_model/common/model_result.py b/python/ppc_model/common/model_result.py new file mode 100644 index 00000000..5b156948 --- /dev/null +++ b/python/ppc_model/common/model_result.py @@ -0,0 +1,212 @@ +import os +import shutil +import pandas as pd +import time +from enum import Enum +import base64 +from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.common.context import Context +from ppc_model.common.protocol import TaskRole +from ppc_model.network.stub import PushRequest, PullRequest + + +class ResultFileHandling: + + def __init__(self, ctx: Context) -> None: + self.ctx = ctx + self.log = ctx.components.logger() + + if ctx.algorithm_type == AlgorithmType.Train.name: + self._process_fe_result() + + # remove job workspace + # self._remove_workspace() + + # Synchronization result file + if (len(ctx.result_receiver_id_list) == 1 and ctx.participant_id_list[0] != ctx.result_receiver_id_list[0]) \ + or len(ctx.result_receiver_id_list) > 1: + self._sync_result_files() + + def _process_fe_result(self): + if os.path.exists(self.ctx.preprocessing_result_file): + column_info_fm = pd.read_csv(self.ctx.preprocessing_result_file, index_col=0) + if os.path.exists(self.ctx.iv_selected_file): + column_info_iv_fm = pd.read_csv(self.ctx.iv_selected_file, index_col=0) + merged_df = self.union_column_info(column_info_fm, column_info_iv_fm) + else: + merged_df = column_info_fm + + merged_df.fillna("None", inplace=True) + merged_df.to_csv(self.ctx.selected_col_file, sep=utils.CSV_SEP, header=True, index_label='id') + # 存储column_info到hdfs给前端展示 + self._upload_file(self.ctx.components.storage_client, self.ctx.selected_col_file, self.ctx.remote_selected_col_file) + + @staticmethod + def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame): + """ + union the column_info1 with the column_info2. + + Args: + column_info1 (DataFrame): The column_info1 to be merged. + column_info2 (DataFrame): The column_info2 to be merged. + + Returns: + column_info_merge (DataFrame): The union column_info. + """ + # 将column_info1和column_info2按照left_index=True, right_index=True的方式进行合并 如果列有缺失则赋值为None 行的顺序按照column_info1 + column_info_conbine = column_info1.merge(column_info2, how='outer', left_index=True, right_index=True, sort=False) + col1_index_list = column_info1.index.to_list() + col2_index_list = column_info2.index.to_list() + merged_list = col1_index_list + [item for item in col2_index_list if item not in col1_index_list] + column_info_conbine = column_info_conbine.reindex(merged_list) + return column_info_conbine + + @staticmethod + def _upload_file(storage_client, local_file, remote_file): + if storage_client is not None: + storage_client.upload_file(local_file, remote_file) + + @staticmethod + def _download_file(storage_client, local_file, remote_file): + if storage_client is not None and not os.path.exists(local_file): + storage_client.download_file(remote_file, local_file) + @staticmethod + def make_graph_data(components, job_id, graph_file_name): + graph_format = 'svg+xml' + # download with cache + remote_file_path = os.path.join(job_id, graph_file_name) + local_file_path = os.path.join( + components.job_cache_dir, remote_file_path) + components.storage_client.download_file( + remote_file_path, local_file_path, True) + file_bytes = None + with open(local_file_path, 'r') as file: + file_content = file.read() + file_bytes = file_content.encode('utf-8') + encoded_data = "" + if file_bytes is not None: + encoded_data = base64.b64encode(file_bytes).decode('ascii') + time.sleep(0.1) + return f"data:image/{graph_format};base64,{encoded_data}" + + def get_remote_path(components, job_id, csv_file_name): + if components.storage_client.get_home_path() is None: + return os.path.join(job_id, csv_file_name) + return os.path.join(components.storage_client.get_home_path(), job_id, csv_file_name) + + @staticmethod + def make_csv_data(components, job_id, csv_file_name): + import pandas as pd + from io import StringIO + remote_file_path = os.path.join(job_id, csv_file_name) + local_file_path = os.path.join( + components.job_cache_dir, remote_file_path) + components.storage_client.download_file( + remote_file_path, local_file_path, True) + file_bytes = None + with open(local_file_path, 'r') as file: + file_content = file.read() + file_bytes = file_content.encode('utf-8') + # encoded_data = base64.b64encode(data).decode('ascii') + # csv_data = np.genfromtxt(data.decode(), delimiter=',') + # csv_data = np.genfromtxt(StringIO(data.decode()), delimiter=',') + csv_data = "" + if file_bytes is not None: + csv_data = pd.read_csv(StringIO(file_bytes.decode())).astype('str') + return csv_data + + def _remove_workspace(self): + if os.path.exists(self.ctx.workspace): + shutil.rmtree(self.ctx.workspace) + self.log.info(f'job {self.ctx.job_id}: {self.ctx.workspace} has been removed.') + else: + self.log.info(f'job {self.ctx.job_id}: {self.ctx.workspace} does not exist.') + + def _sync_result_files(self): + if self.ctx.algorithm_type == AlgorithmType.Train.name: + self.sync_result_file(self.ctx, self.ctx.metrics_iteration_file, + self.ctx.remote_metrics_iteration_file, 'f1') + self.sync_result_file(self.ctx, self.ctx.feature_importance_file, + self.ctx.remote_feature_importance_file, 'f2') + self.sync_result_file(self.ctx, self.ctx.summary_evaluation_file, + self.ctx.remote_summary_evaluation_file, 'f3') + self.sync_result_file(self.ctx, self.ctx.train_metric_ks_table, + self.ctx.remote_train_metric_ks_table, 'f4') + self.sync_result_file(self.ctx, self.ctx.train_metric_roc_file, + self.ctx.remote_train_metric_roc_file, 'f5') + self.sync_result_file(self.ctx, self.ctx.train_metric_ks_file, + self.ctx.remote_train_metric_ks_file, 'f6') + self.sync_result_file(self.ctx, self.ctx.train_metric_pr_file, + self.ctx.remote_train_metric_pr_file, 'f7') + self.sync_result_file(self.ctx, self.ctx.train_metric_acc_file, + self.ctx.remote_train_metric_acc_file, 'f8') + self.sync_result_file(self.ctx, self.ctx.test_metric_ks_table, + self.ctx.remote_test_metric_ks_table, 'f9') + self.sync_result_file(self.ctx, self.ctx.test_metric_roc_file, + self.ctx.remote_test_metric_roc_file, 'f10') + self.sync_result_file(self.ctx, self.ctx.test_metric_ks_file, + self.ctx.remote_test_metric_ks_file, 'f11') + self.sync_result_file(self.ctx, self.ctx.test_metric_pr_file, + self.ctx.remote_test_metric_pr_file, 'f12') + self.sync_result_file(self.ctx, self.ctx.test_metric_acc_file, + self.ctx.remote_test_metric_acc_file, 'f13') + + @staticmethod + def sync_result_file(ctx, local_file, remote_file, key_file): + if ctx.role == TaskRole.ACTIVE_PARTY: + with open(local_file, 'rb') as f: + byte_data = f.read() + for partner_index in range(1, len(ctx.participant_id_list)): + if ctx.participant_id_list[partner_index] in ctx.result_receiver_id_list: + SendMessage._send_byte_data(ctx.components.stub, ctx, f'{CommonMessage.SYNC_FILE.value}_{key_file}', + byte_data, partner_index) + else: + if ctx.components.config_data['AGENCY_ID'] in ctx.result_receiver_id_list: + byte_data = SendMessage._receive_byte_data(ctx.components.stub, ctx, + f'{CommonMessage.SYNC_FILE.value}_{key_file}', 0) + with open(local_file, 'wb') as f: + f.write(byte_data) + ResultFileHandling._upload_file(ctx.components.storage_client, local_file, remote_file) + + +class CommonMessage(Enum): + SYNC_FILE = "SYNC_FILE" + EVAL_SET_FILE = "EVAL_SET_FILE" + + +class SendMessage: + + @staticmethod + def _send_byte_data(stub, ctx, key_type, byte_data, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=byte_data + )) + + log.info( + f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + + @staticmethod + def _receive_byte_data(stub, ctx, key_type, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + byte_data = stub.pull(PullRequest( + sender=partner_id, + task_id=ctx.task_id, + key=key_type + )) + + log.info( + f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + return byte_data diff --git a/python/ppc_model/common/model_setting.py b/python/ppc_model/common/model_setting.py new file mode 100644 index 00000000..ab139ec0 --- /dev/null +++ b/python/ppc_model/common/model_setting.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +from ppc_common.ppc_utils import common_func + + +class ModelSetting: + def __init__(self, model_dict): + self.use_psi = common_func.get_config_value( + "use_psi", False, model_dict, False) + self.fillna = common_func.get_config_value( + "fillna", False, model_dict, False) + self.na_select = float(common_func.get_config_value( + "na_select", 1.0, model_dict, False)) + self.filloutlier = common_func.get_config_value( + "filloutlier", False, model_dict, False) + self.normalized = common_func.get_config_value( + "normalized", False, model_dict, False) + self.standardized = common_func.get_config_value( + "standardized", False, model_dict, False) + self.categorical = common_func.get_config_value( + "categorical", '', model_dict, False) + self.psi_select_col = common_func.get_config_value( + "psi_select_col", '', model_dict, False) + self.psi_select_base = common_func.get_config_value( + "psi_select_base", '', model_dict, False) + self.psi_select_base = float(common_func.get_config_value( + "psi_select_thresh", 0.3, model_dict, False)) + self.psi_select_bins = int(common_func.get_config_value( + "psi_select_bins", 4, model_dict, False)) + self.corr_select = float(common_func.get_config_value( + "corr_select", 0, model_dict, False)) + self.use_iv = common_func.get_config_value( + "use_iv", False, model_dict, False) + self.group_num = int(common_func.get_config_value( + "group_num", 4, model_dict, False)) + self.iv_thresh = float(common_func.get_config_value( + "iv_thresh", 0.1, model_dict, False)) + self.use_goss = common_func.get_config_value( + "use_goss", False, model_dict, False) + self.test_dataset_percentage = float(common_func.get_config_value( + "test_dataset_percentage", 0.3, model_dict, False)) + self.learning_rate = float(common_func.get_config_value( + "learning_rate", 0.1, model_dict, False)) + self.num_trees = int(common_func.get_config_value( + "num_trees", 6, model_dict, False)) + self.max_depth = int(common_func.get_config_value( + "max_depth", 3, model_dict, False)) + self.max_bin = int(common_func.get_config_value( + "max_bin", 4, model_dict, False)) + self.silent = common_func.get_config_value( + "silent", False, model_dict, False) + self.subsample = float(common_func.get_config_value( + "subsample", 1, model_dict, False)) + self.colsample_bytree = float(common_func.get_config_value( + "colsample_bytree", 1, model_dict, False)) + self.colsample_bylevel = float(common_func.get_config_value( + "colsample_bylevel", 1, model_dict, False)) + self.reg_alpha = float(common_func.get_config_value( + "reg_alpha", 0, model_dict, False)) + self.reg_lambda = float(common_func.get_config_value( + "reg_lambda", 1, model_dict, False)) + self.gamma = float(common_func.get_config_value( + "gamma", 0, model_dict, False)) + self.min_child_weight = float(common_func.get_config_value( + "min_child_weight", 0.0, model_dict, False)) + self.min_child_samples = int(common_func.get_config_value( + "min_child_samples", 10, model_dict, False)) + self.seed = int(common_func.get_config_value( + "seed", 2024, model_dict, False)) + self.early_stopping_rounds = int(common_func.get_config_value( + "early_stopping_rounds", 5, model_dict, False)) + self.eval_metric = common_func.get_config_value( + "eval_metric", "auc", model_dict, False) + self.verbose_eval = int(common_func.get_config_value( + "verbose_eval", 1, model_dict, False)) + self.eval_set_column = common_func.get_config_value( + "eval_set_column", "", model_dict, False) + self.train_set_value = common_func.get_config_value( + "train_set_value", "", model_dict, False) + self.eval_set_value = common_func.get_config_value( + "eval_set_value", "", model_dict, False) + self.train_features = common_func.get_config_value( + "train_features", "", model_dict, False) + self.epochs = int(common_func.get_config_value( + "epochs", 3, model_dict, False)) + self.batch_size = int(common_func.get_config_value( + "batch_size", 16, model_dict, False)) + self.threads = int(common_func.get_config_value( + "threads", 8, model_dict, False)) + self.one_hot = common_func.get_config_value( + "one_hot", 0, model_dict, False) diff --git a/python/ppc_model/common/protocol.py b/python/ppc_model/common/protocol.py new file mode 100644 index 00000000..c09a4da4 --- /dev/null +++ b/python/ppc_model/common/protocol.py @@ -0,0 +1,88 @@ +from enum import Enum + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import Cipher1DimList, Cipher2DimList +from ppc_common.ppc_protos.generated.ppc_model_pb2 import CipherList, ModelCipher +from ppc_common.ppc_utils import utils + + +class TaskRole(Enum): + ACTIVE_PARTY = "ACTIVE_PARTY" + PASSIVE_PARTY = "PASSIVE_PARTY" + + +class ModelTask(Enum): + PREPROCESSING = "PREPROCESSING" + FEATURE_ENGINEERING = "FEATURE_ENGINEERING" + XGB_TRAINING = "XGB_TRAINING" + XGB_PREDICTING = "XGB_PREDICTING" + + +class TaskStatus(Enum): + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + + +class RpcType(Enum): + HTTP = "HTTP" + GRPC = "GRPC" + + +class PheMessage: + + @staticmethod + def packing_data(codec, public_key, cipher_list): + enc_data_pb = CipherList() + enc_data_pb.public_key = codec.encode_enc_key(public_key) + + for cipher in cipher_list: + model_cipher = ModelCipher() + model_cipher.ciphertext, model_cipher.exponent = codec.encode_cipher(cipher) + enc_data_pb.cipher_list.append(model_cipher) + + return utils.pb_to_bytes(enc_data_pb) + + @staticmethod + def unpacking_data(codec, data): + enc_data_pb = CipherList() + utils.bytes_to_pb(enc_data_pb, data) + public_key = codec.decode_enc_key(enc_data_pb.public_key) + enc_data = [codec.decode_cipher(public_key, + cipher.ciphertext, + cipher.exponent + ) for cipher in enc_data_pb.cipher_list] + return public_key, enc_data + + @staticmethod + def packing_2dim_data(codec, public_key, cipher_2d_list): + enc_data_pb = Cipher2DimList() + enc_data_pb.public_key = codec.encode_enc_key(public_key) + + for cipher_list in cipher_2d_list: + enc_1d_pb = Cipher1DimList() + for cipher in cipher_list: + model_cipher = ModelCipher() + model_cipher.ciphertext, model_cipher.exponent = \ + codec.encode_cipher(cipher, be_secure=False) + enc_1d_pb.cipher_list.append(model_cipher) + enc_data_pb.cipher_1d_list.append(enc_1d_pb) + + return utils.pb_to_bytes(enc_data_pb) + + @staticmethod + def unpacking_2dim_data(codec, data): + enc_data_pb = Cipher2DimList() + utils.bytes_to_pb(enc_data_pb, data) + public_key = codec.decode_enc_key(enc_data_pb.public_key) + enc_data = [] + for enc_1d_pb in enc_data_pb.cipher_1d_list: + enc_1d_data = [codec.decode_cipher(public_key, + cipher.ciphertext, + cipher.exponent + ) for cipher in enc_1d_pb.cipher_list] + enc_data.append(enc_1d_data) + return public_key, enc_data + + +LOG_START_FLAG_FORMATTER = "$$$StartModelJob:{job_id}" +LOG_END_FLAG_FORMATTER = "$$$EndModelJob:{job_id}" diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml new file mode 100644 index 00000000..2fecc8eb --- /dev/null +++ b/python/ppc_model/conf/application-sample.yml @@ -0,0 +1,43 @@ +HOST: "0.0.0.0" +HTTP_PORT: 43471 +RPC_PORT: 43472 + +AGENCY_ID: 'WeBank' +GATEWAY_ENDPOINT: "127.0.0.1:43454" + +PUBLIC_KEY_LENGTH: 2048 + +MAX_MESSAGE_LENGTH_MB: 100 +TASK_TIMEOUT_H: 1800 + +SSL_SWITCH: 0 +CA_CRT: "./ca.crt" +SSL_CRT: "./ssl.crt" +SSL_KEY: "./ssl.key" + + +PEM_PATH: "/data/app/ppcs-model4ef/wedpr-model-node/ppc_model_service/server.pem" +SHARE_PATH: "/data/app/ppcs-model4ef/wedpr-model-node/ppc_model_service/dataset_share/" + +DB_TYPE: "mysql" +SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4" + +# interagency services +HDFS_ENDPOINT: "http://127.0.0.1:50070" +# HDFS, +STORAGE_TYPE: "HDFS" +HDFS_ENDPOINT: "http://127.0.0.1:9870" +HDFS_USER: "ppc" +HDFS_HOME: "/user/ppc/model/webank" + +# ECDSA or GM +CRYPTO_TYPE: "ECDSA" +private_key: "" +public_key: "" +gm_private_key: "" +gm_public_key: "" + +UPLOAD_FOLDER: "./upload_data_folder" +JOB_TEMP_DIR: ".cache/job" + +FE_TIMEOUT_S: 5400 diff --git a/python/ppc_model/conf/logging.conf b/python/ppc_model/conf/logging.conf new file mode 100644 index 00000000..31f30acb --- /dev/null +++ b/python/ppc_model/conf/logging.conf @@ -0,0 +1,40 @@ +[loggers] +keys=root,wsgi + +[logger_root] +level=INFO +handlers=consoleHandler,fileHandler + +[logger_wsgi] +level = INFO +handlers = accessHandler +qualname = wsgi +propagate = 0 + +[handlers] +keys=fileHandler,consoleHandler,accessHandler + +[handler_accessHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/logs/ppcs-model4ef/appmonitor.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_fileHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/logs/ppcs-model4ef/ppcs-model4ef-node.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_consoleHandler] +class=StreamHandler +args=(sys.stdout,) +level=ERROR +formatter=simpleFormatter + +[formatters] +keys=simpleFormatter + +[formatter_simpleFormatter] +format=[%(levelname)s][%(asctime)s %(msecs)03d][%(process)d][%(filename)s:%(lineno)d] %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/python/ppc_model/datasets/__init__.py b/python/ppc_model/datasets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/data_reduction/__init__.py b/python/ppc_model/datasets/data_reduction/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/data_reduction/feature_selection.py b/python/ppc_model/datasets/data_reduction/feature_selection.py new file mode 100644 index 00000000..cfeec89b --- /dev/null +++ b/python/ppc_model/datasets/data_reduction/feature_selection.py @@ -0,0 +1,30 @@ +import numpy as np + + +class FeatureSelection: + + @staticmethod + def feature_selecting(feature_name: list, train_feats: list, fr: float): + if train_feats is not None and len(train_feats) > 0: + feature_select = FeatureSelection._get_train_feature( + feature_name, train_feats) + elif fr > 0 and fr < 1: + feature_select = FeatureSelection._get_feature_rate( + feature_name, fr) + else: + feature_select = list(range(len(feature_name))) + return feature_select + + @staticmethod + def _get_train_feature(feature_name: list, train_feats: list): + feature_select = [] + for i, feature in enumerate(feature_name): + if feature in train_feats: + feature_select.append(i) + return feature_select + + @staticmethod + def _get_feature_rate(feature_name: list, fr: float): + feature_select = sorted(np.random.choice( + range(len(feature_name)), size=int(len(feature_name) * fr), replace=False)) + return feature_select diff --git a/python/ppc_model/datasets/data_reduction/sampling.py b/python/ppc_model/datasets/data_reduction/sampling.py new file mode 100644 index 00000000..0866e3dd --- /dev/null +++ b/python/ppc_model/datasets/data_reduction/sampling.py @@ -0,0 +1,86 @@ +import numpy as np + + +class Sampling: + + @staticmethod + def sample_selecting( + g_list: np.ndarray, + h_list: np.ndarray, + subsample: float = 0, + use_goss: bool = False, + top_rate: float = 0.2, + other_rate: float = 0.1 + ): + if use_goss: + instance, used_glist, used_hlist = Sampling._get_goss_sampling( + g_list, h_list, top_rate, other_rate) + elif subsample > 0 and subsample < 1: + instance, used_glist, used_hlist = Sampling._get_subsample_sampling( + g_list, h_list, subsample) + else: + instance, used_glist, used_hlist = Sampling._get_sampling( + g_list, h_list) + + return instance, used_glist, used_hlist + + def _get_goss_sampling(g_list, h_list, top_rate, other_rate): + + n = len(g_list) + instance, used_glist, used_hlist = Sampling._goss_sampleing( + n, top_rate, other_rate, g_list, h_list) + + return instance, used_glist, used_hlist + + def _get_subsample_sampling(g_list, h_list, subsample): + + rand_size = int(len(g_list) * subsample) + rand_idx = np.array(sorted( + np.random.choice(list(range(len(g_list))), size=rand_size, replace=False))) + used_glist = np.array(g_list)[(rand_idx)] + used_hlist = np.array(h_list)[(rand_idx)] + + # used_idx = {} + # for i in range(rand_size): + # used_idx[rand_idx[i]] = i + # curr_instance = np.array(list(used_idx.keys())) + + return rand_idx, used_glist, used_hlist + + def _get_sampling(g_list, h_list): + + used_glist, used_hlist = g_list, h_list + instance = np.array(list(range(len(g_list)))) + + return instance, used_glist, used_hlist + + @staticmethod + def _goss_sampleing(n, a, b, g_list, h_list): + top_size = int(n * a) + rand_size = int(n * b) + abs_g = np.abs(g_list) + + top_idx = np.argsort(abs_g)[-top_size:] + rand_idx = np.random.choice(np.argsort( + abs_g)[:-top_size], size=rand_size, replace=False) + used_idx = np.append(top_idx, rand_idx) + + fact = (1 - a) / b + rand_glist = np.array(g_list)[(rand_idx)] * fact + used_glist = np.append(np.array(g_list)[(top_idx)], rand_glist) + rand_hlist = np.array(h_list)[(rand_idx)] * fact + used_hlist = np.append(np.array(h_list)[(top_idx)], rand_hlist) + + return Sampling._sort_instance(used_idx, used_glist, used_hlist) + + @staticmethod + def _sort_instance(instance, g_list, h_list): + # 获取排序索引 + sorted_indices = np.argsort(instance) + + # 对所有数组进行排序 + sorted_idx = instance[sorted_indices] + sorted_glist = g_list[sorted_indices] + sorted_hlist = h_list[sorted_indices] + + return sorted_idx, sorted_glist, sorted_hlist diff --git a/python/ppc_model/datasets/data_reduction/test/__init__.py b/python/ppc_model/datasets/data_reduction/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/data_reduction/test/test_data_reduction.py b/python/ppc_model/datasets/data_reduction/test/test_data_reduction.py new file mode 100644 index 00000000..689be74f --- /dev/null +++ b/python/ppc_model/datasets/data_reduction/test/test_data_reduction.py @@ -0,0 +1,61 @@ +import unittest +import numpy as np + +from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection +from ppc_model.datasets.data_reduction.sampling import Sampling + + +class TestFeatureSelection(unittest.TestCase): + + feature_name = [f'x{i+1}' for i in range(30)] + + def test_fr_feature_select(self): + + feature_select = FeatureSelection.feature_selecting( + self.feature_name, [], 0.8) + self.assertEqual(len(feature_select), len(self.feature_name) * 0.8) + + def test_customized_feature_select(self): + + train_feats = ['x1', 'x3', 'x15', 'x27', 'x33'] + feature_select = FeatureSelection.feature_selecting( + self.feature_name, train_feats, 0.8) + self.assertEqual(len(feature_select), len( + set(self.feature_name).intersection(set(train_feats)))) + self.assertEqual(sorted([f'x{i+1}' for i in feature_select]), + sorted(set(self.feature_name).intersection(set(train_feats)))) + + def test_feature_select(self): + + feature_select = FeatureSelection.feature_selecting( + self.feature_name, [], 0) + self.assertEqual(len(feature_select), len(self.feature_name)) + self.assertEqual(feature_select, list(range(len(self.feature_name)))) + + +class TestSampling(unittest.TestCase): + + g_list = [np.random.rand() for i in range(500)] + h_list = [np.random.rand() for i in range(500)] + + def test_goss_sampling(self): + instance, used_glist, used_hlist = Sampling.sample_selecting( + self.g_list, self.h_list, use_goss=True) + self.assertEqual(len(instance), int( + len(self.g_list) * 0.2) + int(len(self.g_list) * 0.1)) + assert max(self.g_list) in used_glist + assert np.argmax(self.g_list) in instance + + def test_subsample_sampling(self): + instance, used_glist, used_hlist = Sampling.sample_selecting( + self.g_list, self.h_list, subsample=0.6) + self.assertEqual(len(instance), int(len(self.g_list) * 0.6)) + + def test_sampling(self): + instance, used_glist, used_hlist = Sampling.sample_selecting( + self.g_list, self.h_list) + self.assertEqual(len(instance), len(self.g_list)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py new file mode 100644 index 00000000..0278c51c --- /dev/null +++ b/python/ppc_model/datasets/dataset.py @@ -0,0 +1,232 @@ +import os +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split + +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.common.protocol import TaskRole +from ppc_model.common.model_result import ResultFileHandling, CommonMessage, SendMessage +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext + + +class SecureDataset: + def __init__(self, ctx: SecureLGBMContext, model_data=None, delimiter: str = ' '): + + self.eval_column_file = ctx.eval_column_file + self.iv_selected_file = ctx.iv_selected_file + self.selected_col_file = ctx.selected_col_file + self.is_label_holder = ctx.is_label_holder + self.algorithm_type = ctx.algorithm_type + self.test_size = ctx.lgbm_params.test_size + self.random_state = ctx.lgbm_params.random_state + self.eval_set_column = ctx.lgbm_params.eval_set_column + self.train_set_value = ctx.lgbm_params.train_set_value + self.eval_set_value = ctx.lgbm_params.eval_set_value + + self.ctx = ctx + self.train_X = None + self.test_X = None + self.train_y = None + self.test_y = None + self.train_idx = None + self.test_idx = None + self.feature_name = None + + if model_data is None: + self.model_data = pd.read_csv( + ctx.model_prepare_file, header=0, delimiter=delimiter) + else: + self.model_data = model_data + + self._construct_dataset() + + @staticmethod + def read_dataset(file_path, has_label: bool, delimiter: str = ' '): + df = pd.read_csv(file_path, header=0, delimiter=delimiter) + + if 'id' in df.columns: + df.drop('id', axis=1, inplace=True) + + field_list = df.columns.tolist() + + if has_label: + return field_list, df.iloc[:, 0].values, df.iloc[:, 1:].values + else: + return field_list, None, df.iloc[:, :].values + + @staticmethod + def simulate_dataset(data_size: int = 1000, feature_dim: int = 100, has_label: bool = True): + X = np.random.rand(data_size, feature_dim) + if has_label: + y = np.random.randint(2, size=data_size) + return SecureDataset.assembling_dataset(X, y) + return SecureDataset.assembling_dataset(X) + + @staticmethod + def assembling_dataset(X: np.ndarray, y: np.ndarray = None): + # 创建自定义列名 + num_features = X.shape[1] + column_names = [f'x{i+1}' for i in range(num_features)] + + # 创建数据框并使用自定义列名 + df = pd.DataFrame(X, columns=column_names) + + # 添加 id 列和 y 列 + df.insert(0, 'id', range(1, len(df) + 1)) + if y is not None: + df.insert(1, 'y', y) + + return df + + @staticmethod + def hetero_split_dataset(df: pd.DataFrame, split_point: int = None): + # 获取特征列名 + column_names = df.columns.tolist() + if 'id' in column_names: + column_names.remove('id') + if 'y' in column_names: + column_names.remove('y') + + # 计算切分点 + if not split_point: + split_point = (df.shape[1] - 2) // 2 + + # 划分特征列 + columns_with_y = ['id', 'y'] + column_names[:split_point] + columns_without_y = ['id'] + column_names[split_point:] + + # 创建两个数据集 + df_with_y = df[columns_with_y] + df_without_y = df[columns_without_y] + + return df_with_y, df_without_y + + def _random_split_dataset(self): + + train_data, test_data = train_test_split( + self.model_data, test_size=self.test_size, random_state=self.random_state) + + return train_data, test_data + + def _customized_split_dataset(self): + if self.ctx.role == TaskRole.ACTIVE_PARTY: + for partner_index in range(1, len(self.ctx.participant_id_list)): + byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, + f'{CommonMessage.EVAL_SET_FILE.value}', partner_index) + if not os.path.exists(self.eval_column_file) and byte_data != bytes(): + with open(self.eval_column_file, 'wb') as f: + f.write(byte_data) + with open(self.eval_column_file, 'rb') as f: + byte_data = f.read() + for partner_index in range(1, len(self.ctx.participant_id_list)): + SendMessage._send_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}', + byte_data, partner_index) + else: + if not os.path.exists(self.eval_column_file): + byte_data = bytes() + else: + with open(self.eval_column_file, 'rb') as f: + byte_data = f.read() + SendMessage._send_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}', + byte_data, 0) + byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, + f'{CommonMessage.EVAL_SET_FILE.value}', 0) + if not os.path.exists(self.eval_column_file): + with open(self.eval_column_file, 'wb') as f: + f.write(byte_data) + + eval_set_df = pd.read_csv(self.eval_column_file, header=0) + train_data = self.model_data[eval_set_df[self.eval_set_column] == self.train_set_value] + test_data = self.model_data[eval_set_df[self.eval_set_column] == self.eval_set_value] + + return train_data, test_data + + def _construct_model_dataset(self, train_data, test_data): + + self.train_idx = train_data['id'].values + self.test_idx = test_data['id'].values + + if self.is_label_holder and 'y' in train_data.columns: + self.train_y = train_data['y'].values + self.test_y = test_data['y'].values + self.train_X = train_data.drop(columns=['id', 'y']).values + self.test_X = test_data.drop(columns=['id', 'y']).values + self.feature_name = train_data.drop( + columns=['id', 'y']).columns.tolist() + else: + self.train_X = train_data.drop(columns=['id']).values + self.test_X = test_data.drop(columns=['id']).values + self.feature_name = train_data.drop( + columns=['id']).columns.tolist() + + def _construct_predict_dataset(self, test_data): + self.test_idx = test_data['id'].values + if self.is_label_holder and 'y' in test_data.columns: + self.test_y = test_data['y'].values + self.test_X = test_data.drop(columns=['id', 'y']).values + self.feature_name = test_data.drop( + columns=['id', 'y']).columns.tolist() + else: + self.test_X = test_data.drop(columns=['id']).values + self.feature_name = test_data.drop(columns=['id']).columns.tolist() + + def _dataset_fe_selected(self, file_path, feature_name): + iv_selected = pd.read_csv(file_path, header=0) + selected_list = iv_selected[feature_name][iv_selected['iv_selected'] == 1].tolist( + ) + + drop_columns = [] + for column in self.model_data.columns: + if column == 'id' or column == 'y': + continue + if column not in selected_list: + drop_columns.append(column) + + if len(drop_columns) > 0: + self.model_data = self.model_data.drop(columns=drop_columns) + + def _construct_dataset(self): + + if os.path.exists(self.iv_selected_file): + self._dataset_fe_selected(self.iv_selected_file, 'feature') + + if self.algorithm_type == AlgorithmType.Predict.name \ + and not os.path.exists(self.selected_col_file): + try: + self.ctx.remote_selected_col_file = os.path.join( + self.ctx.lgbm_params.training_job_id, self.ctx.SELECTED_COL_FILE) + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.selected_col_file, self.ctx.remote_selected_col_file) + self._dataset_fe_selected(self.selected_col_file, 'id') + except: + pass + + if 'id' not in self.model_data.columns: + if self.ctx.dataset_file_path is None: + import glob + pattern = os.path.join(self.ctx.workspace, 'd-*') + dataset_file_path = glob.glob(pattern)[0] + else: + dataset_file_path = self.ctx.dataset_file_path + dataset_id = pd.read_csv( + dataset_file_path, header=0, usecols=['id']) + if os.path.exists(self.ctx.psi_result_path): + psi_data = pd.read_csv(self.ctx.psi_result_path, header=0) + dataset_id = pd.merge(dataset_id, psi_data, on=[ + 'id']).sort_values(by='id', ascending=True) + self.model_data = pd.concat([dataset_id, self.model_data], axis=1) + + if self.algorithm_type == AlgorithmType.Train.name: + if self.eval_set_column: + train_data, test_data = self._customized_split_dataset() + else: + train_data, test_data = self._random_split_dataset() + self._construct_model_dataset(train_data, test_data) + + elif self.algorithm_type == AlgorithmType.Predict.name: + test_data = self.model_data + self._construct_predict_dataset(test_data) + else: + raise PpcException(PpcErrorCode.ALGORITHM_TYPE_ERROR.get_code(), + PpcErrorCode.ALGORITHM_TYPE_ERROR.get_message()) diff --git a/python/ppc_model/datasets/feature_binning/__init__.py b/python/ppc_model/datasets/feature_binning/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/feature_binning/feature_binning.py b/python/ppc_model/datasets/feature_binning/feature_binning.py new file mode 100644 index 00000000..91c5ab3d --- /dev/null +++ b/python/ppc_model/datasets/feature_binning/feature_binning.py @@ -0,0 +1,131 @@ +import numpy as np +import pandas as pd + +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.common.context import Context + + +class FeatureBinning: + def __init__(self, ctx: Context): + self.ctx = ctx + self.params = ctx.lgbm_params + self.data = None + self.data_bin = None + self.data_split = None + + @staticmethod + def binning_continuous_feature(feature: np.ndarray, max_bin: int, is_equal_freq: bool = True): + try: + if is_equal_freq: + # 等频分箱，不替换缺失值 + Xk_bin, Xk_split = pd.qcut( + feature, q=max_bin, retbins=True, labels=False) + else: + # 等距分箱，不替换缺失值 + Xk_bin, Xk_split = pd.cut( + feature, max_bin, retbins=True, labels=False) + # 将缺失值标记为 -1 + Xk_bin[np.isnan(feature)] = -1 + except ValueError: + unique_values = sorted(set(feature[~np.isnan(feature)])) + if len(unique_values) > 1000: + raise Exception( + 'Features with more than 1000 groups are not supported.') + if len(unique_values) == 2 and 0 in unique_values and 1 in unique_values: + Xk_bin = feature + Xk_split = [min(unique_values) - 0.01, 0.5, max(unique_values)] + elif len(unique_values) > max_bin: + Xk_bin, Xk_split = pd.cut( + feature, max_bin, labels=False, retbins=True) + else: + # 创建映射字典 + mapping_dict = {value: i for i, + value in enumerate(unique_values)} + # 使用map函数进行重新编号 + Xk_bin = pd.DataFrame(feature)[0].map(mapping_dict).values + Xk_split = [min(unique_values) - 0.01] + list(unique_values) + Xk_bin[np.isnan(feature)] = -1 + + return Xk_bin, Xk_split + + @staticmethod + def binning_categorical_feature(feature: np.ndarray): + unique_values = sorted(set(feature[~np.isnan(feature)])) + if len(unique_values) > 1000: + raise Exception( + 'Features with more than 1000 groups are not supported.') + mapping_dict = {value: i for i, value in enumerate(unique_values)} + # 使用map函数进行重新编号 + Xk_bin = pd.DataFrame(feature)[0].map(mapping_dict).values + Xk_split = [min(unique_values) - 0.01] + list(unique_values) + Xk_bin[np.isnan(feature)] = -1 + + return Xk_bin, Xk_split + + def data_binning(self, data: np.ndarray, data_split=None): + + self.data = data + self.data_split = data_split + + if self.ctx.algorithm_type == AlgorithmType.Train.name and self.data_split is None: + self._generate_data_binning() + else: + self._reuse_data_binning(data_split) + + return self.data_bin, self.data_split + + def _generate_data_binning(self): + + n = self.data.shape[0] + d = self.data.shape[1] + + X_bin = np.zeros((d, n), dtype='int16') + X_split = [] + + for idx, feature in enumerate(self.data.T): + if idx in self.params.my_categorical_idx: + Xk_bin, Xk_split = FeatureBinning.binning_categorical_feature( + feature) + else: + Xk_bin, Xk_split = FeatureBinning.binning_continuous_feature( + feature, self.params.max_bin) + + X_bin[idx] = Xk_bin + if isinstance(Xk_split, np.ndarray): + Xk_split = Xk_split.tolist() + X_split.append(Xk_split) + + self.data_bin = X_bin.T + self.data_split = X_split + + def _reuse_data_binning(self, data_split): + + self.data_split = data_split + + n = self.data.shape[0] + d = self.data.shape[1] + + test_X_bin = np.zeros((d, n), dtype='int16') + + for k in range(d): + bin_min = min(self.data[:, k]) - 1 + bin_max = max(self.data[:, k]) + 1 + if np.isnan(bin_min): + bin_min = min(self.data_split[k]) + if np.isnan(bin_max): + bin_max = max(self.data_split[k]) + + bin_min = min(bin_min, min(self.data_split[k])) - 1 + bin_max = max(bin_max, max(self.data_split[k])) + 1 + + if len(self.data_split[k]) > 2: + bins = np.concatenate( + ([bin_min], self.data_split[k][1:-1], [bin_max]), axis=0) + else: + bins = np.concatenate(([bin_min], [bin_max]), axis=0) + test_Xk_bin = pd.cut(self.data[:, k], bins, labels=False) + test_Xk_bin[np.isnan(test_Xk_bin)] = -1 + + test_X_bin[k] = test_Xk_bin + + self.data_bin = test_X_bin.T diff --git a/python/ppc_model/datasets/feature_binning/test/__init__.py b/python/ppc_model/datasets/feature_binning/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py new file mode 100644 index 00000000..e06b82b2 --- /dev/null +++ b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py @@ -0,0 +1,102 @@ +import unittest + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext + + +data_size = 1000 +feature_dim = 100 +has_label = True + + +class TestFeatureBinning(unittest.TestCase): + + components = Initializer(log_config_path='', config_path='') + components.config_data = {'JOB_TEMP_DIR': '/tmp'} + components.mock_logger = MockLogger() + + def test_train_feature_binning(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': None, + 'model_dict': { + 'objective': 'regression', + 'max_bin': 10, + 'n_estimators': 6, + 'max_depth': 3, + 'use_goss': 1 + } + } + + task_info = SecureLGBMContext(args, self.components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label) + secure_dataset = SecureDataset(task_info, model_data) + print(secure_dataset.train_idx.shape) + print(secure_dataset.train_X.shape) + print(secure_dataset.train_y.shape) + print(secure_dataset.test_idx.shape) + print(secure_dataset.test_X.shape) + print(secure_dataset.test_y.shape) + + feat_bin = FeatureBinning(task_info) + data_bin, data_split = feat_bin.data_binning(secure_dataset.train_X) + + self.assertEqual(data_bin.shape, secure_dataset.train_X.shape) + # print(data_bin) + # print(data_split) + + def test_test_feature_binning(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': None, + 'model_dict': { + 'objective': 'regression', + 'categorical_feature': [], + 'max_bin': 10, + 'n_estimators': 6, + 'max_depth': 3, + 'use_goss': 1 + } + } + + task_info = SecureLGBMContext(args, self.components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label) + secure_dataset = SecureDataset(task_info, model_data) + print(secure_dataset.train_idx.shape) + print(secure_dataset.train_X.shape) + print(secure_dataset.train_y.shape) + print(secure_dataset.test_idx.shape) + print(secure_dataset.test_X.shape) + print(secure_dataset.test_y.shape) + + feat_bin = FeatureBinning(task_info) + data_split = None + data_bin, data_split = feat_bin.data_binning( + secure_dataset.train_X, data_split) + + self.assertEqual(data_bin.shape, secure_dataset.train_X.shape) + # print(data_bin) + # print(data_split) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/ppc_model/datasets/test/__init__.py b/python/ppc_model/datasets/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/datasets/test/test_dataset.py b/python/ppc_model/datasets/test/test_dataset.py new file mode 100644 index 00000000..7a544cfc --- /dev/null +++ b/python/ppc_model/datasets/test/test_dataset.py @@ -0,0 +1,213 @@ +import unittest +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.datasets.dataset import SecureDataset + + +class TestSecureDataset(unittest.TestCase): + cancer = load_breast_cancer() + X = cancer.data + y = cancer.target + + df = SecureDataset.assembling_dataset(X, y) + assert (df.columns == ['id', 'y'] + [f'x{i + 1}' for i in range(30)]).all() + + # 模拟生成主动方和被动方数据集 + df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df) + assert (df_with_y.columns == ['id', 'y'] + + [f'x{i + 1}' for i in range(15)]).all() + assert (df_without_y.columns == ['id'] + + [f'x{i + 16}' for i in range(15)]).all() + + # 模拟自定义分组 + import os + + eval_column_file = './model_eval_column.csv' + if not os.path.exists(eval_column_file): + # 创建一个包含569行，2列的数据，其中415个为'INS'，154个为'OOS' + group_set = np.concatenate([['INS'] * 415, ['OOS'] * 154]) + np.random.shuffle(group_set) + eval_set_df = pd.DataFrame( + {'id': np.arange(1, 570), 'group': group_set}) + eval_set_df.to_csv(eval_column_file, index=None) + + df_with_y_file = './df_with_y.csv' + if not os.path.exists(df_with_y_file): + df_with_y.to_csv(df_with_y_file, index=None, sep=' ') + + df_without_y_file = './df_without_y.csv' + if not os.path.exists(df_without_y_file): + df_without_y.to_csv(df_without_y_file, index=None, sep=' ') + + iv_selected_file = './iv_selected.csv' + if not os.path.exists(iv_selected_file): + iv_selected = pd.DataFrame( + {'feature': [f'x{i + 1}' for i in range(30)], + 'iv_selected': np.random.binomial(n=1, p=0.5, size=30)}) + iv_selected.to_csv(iv_selected_file, index=None) + + components = Initializer(log_config_path='', config_path='') + components.config_data = {'JOB_TEMP_DIR': '/tmp', 'MAX_THREAD_WORKERS': 10} + components.mock_logger = MockLogger() + + def test_random_split_dataset(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': None, + 'model_dict': { + 'random_state': 2024 + } + } + task_info = SecureLGBMContext(args, self.components) + print(task_info.lgbm_params.get_all_params()) + + # 模拟构造主动方数据集 + dataset_with_y = SecureDataset(task_info, self.df_with_y) + assert (dataset_with_y.train_idx == + train_test_split(np.array(range(1, 570)), test_size=0.3, random_state=2024)[0]).all() + self.assertEqual(dataset_with_y.train_X.shape, (398, 15)) + self.assertEqual(dataset_with_y.test_X.shape, (171, 15)) + self.assertEqual(dataset_with_y.train_y.shape, (398,)) + self.assertEqual(dataset_with_y.test_y.shape, (171,)) + + # 构造被动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': False, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': None, + 'model_dict': { + 'random_state': 2024 + } + } + task_info = SecureLGBMContext(args, self.components) + print(task_info.lgbm_params.get_all_params()) + + # 模拟构造被动方数据集 + dataset_without_y = SecureDataset(task_info, self.df_without_y) + assert (dataset_without_y.train_idx == dataset_with_y.train_idx).all() + self.assertEqual(dataset_without_y.train_X.shape, (398, 15)) + self.assertEqual(dataset_without_y.test_X.shape, (171, 15)) + self.assertEqual(dataset_without_y.train_y, None) + self.assertEqual(dataset_without_y.test_y, None) + + def test_customized_split_dataset(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': None, + 'model_dict': { + 'eval_set_column': 'group', + 'train_set_value': 'INS', + 'eval_set_value': 'OOS' + } + } + task_info = SecureLGBMContext(args, self.components) + print(task_info.lgbm_params.get_all_params()) + + # 模拟构造主动方数据集 + task_info.eval_column_file = self.eval_column_file + task_info.model_prepare_file = self.df_with_y_file + eval_set_df = pd.read_csv(task_info.eval_column_file, header=0) + + dataset_with_y = SecureDataset(task_info) + assert (dataset_with_y.train_idx == + eval_set_df['id'][eval_set_df['group'] == 'INS']).all() + self.assertEqual(dataset_with_y.train_X.shape, (415, 15)) + self.assertEqual(dataset_with_y.test_X.shape, (154, 15)) + self.assertEqual(dataset_with_y.train_y.shape, (415,)) + self.assertEqual(dataset_with_y.test_y.shape, (154,)) + + def test_predict_dataset(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Predict', + 'algorithm_subtype': None, + 'model_dict': {} + } + task_info = SecureLGBMContext(args, self.components) + print(task_info.lgbm_params.get_all_params()) + + # 模拟构造主动方数据集 + task_info.model_prepare_file = self.df_with_y_file + dataset_with_y = SecureDataset(task_info) + + self.assertEqual(dataset_with_y.train_X, None) + self.assertEqual(dataset_with_y.test_X.shape, (569, 15)) + self.assertEqual(dataset_with_y.train_y, None) + self.assertEqual(dataset_with_y.test_y.shape, (569,)) + + def test_iv_selected_dataset(self): + + # 构造主动方参数配置 + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Predict', + 'algorithm_subtype': None, + 'model_dict': {} + } + task_info = SecureLGBMContext(args, self.components) + print(task_info.lgbm_params.get_all_params()) + + # 模拟构造主动方数据集 + task_info.model_prepare_file = self.df_with_y_file + task_info.iv_selected_file = './iv_selected.csv' + dataset_with_y = SecureDataset(task_info) + + self.assertEqual(dataset_with_y.train_X, None) + self.assertEqual(dataset_with_y.test_X.shape, (569, 9)) + self.assertEqual(dataset_with_y.train_y, None) + self.assertEqual(dataset_with_y.test_y.shape, (569,)) + + def test_read_dataset(self): + np.random.seed(0) + origin_data = np.random.randint(0, 100, size=(100, 10)) + columns = ['id'] + [f"x{i}" for i in range(2, 11)] + df = pd.DataFrame(origin_data, columns=columns) + csv_file = '/tmp/data_x1_to_x10.csv' + df.to_csv(csv_file, index=False) + field_list, label, feature = SecureDataset.read_dataset(csv_file, False, delimiter=',') + self.assertEqual(['id'] + field_list, columns) + field_list, label, feature = SecureDataset.read_dataset(csv_file, True, delimiter=',') + self.assertEqual(['id'] + field_list, columns) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/ppc_model/feature_engineering/__init__.py b/python/ppc_model/feature_engineering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/feature_engineering/feature_engineering_context.py b/python/ppc_model/feature_engineering/feature_engineering_context.py new file mode 100644 index 00000000..c3a793ea --- /dev/null +++ b/python/ppc_model/feature_engineering/feature_engineering_context.py @@ -0,0 +1,48 @@ +from enum import Enum + +import numpy as np + +from ppc_common.ppc_crypto.phe_factory import PheCipherFactory +from ppc_model.common.context import Context +from ppc_model.common.initializer import Initializer +from ppc_model.common.protocol import TaskRole +from ppc_model.common.model_setting import ModelSetting + + +class FeMessage(Enum): + ENC_LABELS = "ENC_LABELS" + AGGR_LABELS = "AGGR_LABELS" + WOE_FILE = "WOE_FILE" + IV_SELECTED_FILE = "IV_SELECTED_FILE" + + +class FeatureEngineeringContext(Context): + + def __init__(self, + args, + components: Initializer, + role: TaskRole, + feature: np.ndarray, + feature_name_list: list, + label: np.ndarray = None): + super().__init__(args['job_id'], + args['task_id'], + components, + role) + self.feature_name_list = feature_name_list + self.participant_id_list = args['participant_id_list'] + self.result_receiver_id_list = args['result_receiver_id_list'] + self.model_dict = args['model_dict'] + self.feature = feature + self.label = label + self.phe = PheCipherFactory.build_phe( + components.homo_algorithm, components.public_key_length) + self.codec = PheCipherFactory.build_codec(components.homo_algorithm) + self.model_setting = ModelSetting(self.model_dict) + self._parse_model_dict() + + def _parse_model_dict(self): + self.use_iv = self.model_setting.use_iv + self.iv_thresh = self.model_setting.iv_thresh + self.categorical = self.model_setting.categorical + self.group_num = self.model_setting.group_num diff --git a/python/ppc_model/feature_engineering/feature_engineering_engine.py b/python/ppc_model/feature_engineering/feature_engineering_engine.py new file mode 100644 index 00000000..16acdb7b --- /dev/null +++ b/python/ppc_model/feature_engineering/feature_engineering_engine.py @@ -0,0 +1,41 @@ +from ppc_model.common.base_context import BaseContext +from ppc_model.common.global_context import components +from ppc_model.common.protocol import ModelTask, TaskRole +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext +from ppc_model.feature_engineering.vertical.active_party import VerticalFeatureEngineeringActiveParty +from ppc_model.feature_engineering.vertical.passive_party import VerticalFeatureEngineeringPassiveParty +from ppc_model.interface.task_engine import TaskEngine + + +class FeatureEngineeringEngine(TaskEngine): + task_type = ModelTask.FEATURE_ENGINEERING + + @staticmethod + def run(args): + input_path = BaseContext.feature_engineering_input_path( + args['job_id'], components.config_data['JOB_TEMP_DIR']) + + if args['is_label_holder']: + field_list, label, feature = SecureDataset.read_dataset(input_path, True) + context = FeatureEngineeringContext( + args=args, + components=components, + role=TaskRole.ACTIVE_PARTY, + feature=feature, + feature_name_list=field_list[1:], + label=label + ) + vfe = VerticalFeatureEngineeringActiveParty(context) + else: + field_list, _, feature = SecureDataset.read_dataset(input_path, False) + context = FeatureEngineeringContext( + args=args, + components=components, + role=TaskRole.PASSIVE_PARTY, + feature=feature, + feature_name_list=field_list, + label=None + ) + vfe = VerticalFeatureEngineeringPassiveParty(context) + vfe.fit() diff --git a/python/ppc_model/feature_engineering/test/__init__.py b/python/ppc_model/feature_engineering/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py new file mode 100644 index 00000000..67251ad4 --- /dev/null +++ b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py @@ -0,0 +1,153 @@ +import multiprocessing +import threading +import traceback +import unittest + +import numpy as np + +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_mock.mock_objects import MockLogger, MockStorageClient +from ppc_model.common.initializer import Initializer +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.common.protocol import TaskRole +from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext +from ppc_model.feature_engineering.vertical.active_party import VerticalFeatureEngineeringActiveParty +from ppc_model.feature_engineering.vertical.passive_party import VerticalFeatureEngineeringPassiveParty +from ppc_model.network.stub import ModelStub + +ACTIVE_PARTY = 'ACTIVE_PARTY' + +PASSIVE_PARTY = 'PASSIVE_PARTY' + + +def construct_dataset(num_samples, num_features): + np.random.seed(0) + # 生成标签列 + labels = np.random.choice([0, 1], size=num_samples) + # 生成特征列 + features = np.random.rand(num_samples, num_features) + return labels, features + + +def mock_args(num_features, iv_thresh): + job_id = '0x12345678' + active_fields = ['a' + str(i) for i in range(num_features)] + passive_fields = ['b' + str(i) for i in range(num_features)] + + model_config_dict = { + 'use_iv': True, + 'iv_thresh': iv_thresh, + 'categorical': '0', + 'group_num': 100, + + } + + args_a = { + 'job_id': job_id, + 'task_id': job_id, + 'feature_name_list': active_fields, + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_dict': model_config_dict, + } + + args_b = { + 'job_id': job_id, + 'task_id': job_id, + 'feature_name_list': passive_fields, + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_dict': model_config_dict, + } + return args_a, args_b + + +class TestFeatureEngineering(unittest.TestCase): + + def setUp(self): + self._active_rpc_client = RpcClientMock() + self._passive_rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._active_stub = ModelStub( + agency_id=ACTIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._active_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._passive_stub = ModelStub( + agency_id=PASSIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._passive_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._active_rpc_client.set_message_handler( + self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler( + self._active_stub.on_message_received) + + def test_fit(self): + num_samples = 100000 + num_features = 100 + iv_thresh = 0.05 + labels, features = construct_dataset(num_samples, num_features) + args_a, args_b = mock_args(num_features, iv_thresh) + + active_components = Initializer(log_config_path='', config_path='') + active_components.homo_algorithm = 0 + active_components.stub = self._active_stub + active_components.config_data = {'JOB_TEMP_DIR': '/tmp'} + active_components.mock_logger = MockLogger() + active_components.storage_client = MockStorageClient() + active_context = FeatureEngineeringContext( + args=args_a, + components=active_components, + role=TaskRole.ACTIVE_PARTY, + feature=features, + feature_name_list=args_a['feature_name_list'], + label=labels + ) + active_vfe = VerticalFeatureEngineeringActiveParty(active_context) + + passive_components = Initializer(log_config_path='', config_path='') + passive_components.homo_algorithm = 0 + passive_components.stub = self._passive_stub + passive_components.config_data = {'JOB_TEMP_DIR': '/tmp'} + passive_components.mock_logger = MockLogger() + passive_components.storage_client = MockStorageClient() + passive_context = FeatureEngineeringContext( + args=args_b, + components=passive_components, + role=TaskRole.PASSIVE_PARTY, + feature=features, + feature_name_list=args_b['feature_name_list'], + label=None + ) + passive_vfe = VerticalFeatureEngineeringPassiveParty(passive_context) + + def active_worker(): + try: + active_vfe.fit() + except Exception as e: + active_components.logger().info(traceback.format_exc()) + + def passive_worker(): + try: + passive_vfe.fit() + except Exception as e: + active_components.logger().info(traceback.format_exc()) + + thread_fe_active = threading.Thread(target=active_worker, args=()) + thread_fe_active.start() + + thread_fe_passive = threading.Thread(target=passive_worker, args=()) + thread_fe_passive.start() + + thread_fe_active.join() + thread_fe_passive.join() + + +if __name__ == '__main__': + multiprocessing.set_start_method('spawn') + unittest.main() diff --git a/python/ppc_model/feature_engineering/vertical/__init__.py b/python/ppc_model/feature_engineering/vertical/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/feature_engineering/vertical/active_party.py b/python/ppc_model/feature_engineering/vertical/active_party.py new file mode 100644 index 00000000..6710c47d --- /dev/null +++ b/python/ppc_model/feature_engineering/vertical/active_party.py @@ -0,0 +1,202 @@ +import multiprocessing +import os +import time + +import pandas as pd + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import EncAggrLabelsList +from ppc_common.ppc_utils import utils +from ppc_model.common.protocol import PheMessage +from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext, FeMessage +from ppc_model.feature_engineering.vertical.utils import is_continuous_feature, calculate_woe_iv, \ + calculate_woe_iv_with_pos_event +from ppc_model.interface.model_base import VerticalModel +from ppc_model.network.stub import PushRequest, PullRequest + + +class VerticalFeatureEngineeringActiveParty(VerticalModel): + + def __init__(self, ctx: FeatureEngineeringContext): + super().__init__(ctx) + # 记录woe和iv详情 + self.woe_iv_df = pd.DataFrame(columns=['feature', 'bins', 'count', 'pos_event', 'pos_event_rate', + 'neg_event', 'neg_event_rate', 'woe', 'iv', 'iv_total']) + # 记录特征筛选情况 + self.iv_selected_df = pd.DataFrame(columns=['feature', 'iv_selected']) + + def fit(self, *args, **kwargs) -> None: + log = self.ctx.components.logger() + task_id = self.ctx.task_id + start_time = time.time() + if self.ctx.use_iv: + log.info(f"Start feature engineering, task_id: {task_id}, shape: {self.ctx.feature.shape}, " + f"feature_name_list: {self.ctx.feature_name_list}") + + if len(self.ctx.feature_name_list) != 0: + # 明文计算己方特征的woe/iv + self._compute_self_woe_iv() + + # 密态交互计算合作方特征的woe/iv + log.info(f"Start enc labels, task_id: {task_id}") + enc_start_time = time.time() + enc_labels = self.ctx.phe.encrypt_batch_parallel(self.ctx.label) + log.info( + f"Enc labels finished, task_id: {task_id}, count: {len(enc_labels)}, " + f"time_costs: {time.time() - enc_start_time}s") + + for i in range(1, len(self.ctx.participant_id_list)): + self._compute_partner_woe_iv(enc_labels, i) + + # 处理计算结果 + self._save_and_sync_fe_results() + + log.info( + f"Feature engineering finished, task_id: {task_id}, time_costs: {time.time() - start_time}s, " + f"iv_selected: {self.iv_selected_df}") + + def _compute_self_woe_iv(self): + log = self.ctx.components.logger() + start_time = time.time() + for i in range(self.ctx.feature.shape[1]): + field = self.ctx.feature_name_list[i] + is_continuous = is_continuous_feature(self.ctx.categorical, field) + grouped, iv_total = calculate_woe_iv(self.ctx.feature[:, i], + self.ctx.label, + self.ctx.group_num, + is_continuous) + for index, row in grouped.iterrows(): + self.woe_iv_df = pd.concat([self.woe_iv_df, pd.DataFrame({ + 'feature': field, + 'bins': index, + 'count': [row['count']], + 'pos_event': [row['pos_event']], + 'pos_event_rate': [row['pos_event_rate']], + 'neg_event': [row['neg_event']], + 'neg_event_rate': [row['neg_event_rate']], + 'woe': [row['woe']], + 'iv': [row['iv']], + 'iv_total': [row['iv_total']] + })], ignore_index=True) + + self.iv_selected_df.loc[len(self.iv_selected_df)] = {'feature': field, + 'iv_selected': int(iv_total >= self.ctx.iv_thresh)} + log.info( + f"Computing self woe/iv finished, task_id: {self.ctx.task_id}, time_costs: {time.time() - start_time}s") + + def _compute_partner_woe_iv(self, enc_labels, partner_index): + log = self.ctx.components.logger() + start_time = time.time() + + partner_id = self.ctx.participant_id_list[partner_index] + self._send_enc_labels(enc_labels, partner_id) + enc_aggr_labels = self._get_all_enc_aggr_labels(partner_id) + + # results = [] + # with ProcessPoolExecutor(max_workers=max(1, os.cpu_count() - 1)) as executor: + # futures = [executor.submit( + # self._process_one_feature, + # self.ctx.phe, + # field, + # count_list, + # enc_aggr_labels) for field, count_list, enc_aggr_labels in enc_aggr_labels] + # for future in as_completed(futures): + # results.append(future.result()) + + pool = multiprocessing.Pool() + tasks = [(self.ctx.phe, field, count_list, enc_aggr_labels) for field, count_list, enc_aggr_labels in + enc_aggr_labels] + results = pool.starmap(self._process_one_feature, tasks) + pool.close() + pool.join() + + for field, field_woe_iv_df, iv_total in results: + # 记录新字段的woe和iv + self.woe_iv_df = pd.concat( + [self.woe_iv_df, field_woe_iv_df], ignore_index=True) + self.iv_selected_df.loc[len(self.iv_selected_df)] = {'feature': field, + 'iv_selected': int(iv_total >= self.ctx.iv_thresh)} + log.info( + f"Computing {partner_id}'s woe/iv finished, task_id: {self.ctx.task_id}, " + f"time_costs: {time.time() - start_time}s") + + @staticmethod + def _process_one_feature(phe, field, count_list, enc_aggr_labels): + pos_event = phe.decrypt_batch(enc_aggr_labels) + field_woe_iv_df = pd.DataFrame({'bins': range(len(count_list)), 'count': count_list, + 'pos_event': pos_event, 'feature': field}) + field_woe_iv_df, iv_total = calculate_woe_iv_with_pos_event( + field_woe_iv_df) + return field, field_woe_iv_df, iv_total + + def _get_all_enc_aggr_labels(self, partner_id): + log = self.ctx.components.logger() + start_time = time.time() + data = self.ctx.components.stub.pull(PullRequest( + sender=partner_id, + task_id=self.ctx.task_id, + key=FeMessage.AGGR_LABELS.value + )) + + enc_aggr_labels_list_pb = EncAggrLabelsList() + utils.bytes_to_pb(enc_aggr_labels_list_pb, data) + public_key = self.ctx.codec.decode_enc_key(enc_aggr_labels_list_pb.public_key) + + res = [] + for enc_aggr_labels_pb in enc_aggr_labels_list_pb.enc_aggr_labels_list: + enc_aggr_labels = [self.ctx.codec.decode_cipher(public_key, + cipher.ciphertext, + cipher.exponent + ) for cipher in enc_aggr_labels_pb.cipher_list] + field = enc_aggr_labels_pb.field + res.append( + (field, list(enc_aggr_labels_pb.count_list), enc_aggr_labels)) + + log.info( + f"All enc aggr labels received, task_id: {self.ctx.task_id}, feature_num: {len(res)}, " + f"size: {len(data) / 1024}KB, time_costs: {time.time() - start_time}s") + return res + + def _send_enc_labels(self, enc_labels, receiver): + log = self.ctx.components.logger() + start_time = time.time() + + data = PheMessage.packing_data(self.ctx.codec, self.ctx.phe.public_key, enc_labels) + self.ctx.components.stub.push(PushRequest( + receiver=receiver, + task_id=self.ctx.task_id, + key=FeMessage.ENC_LABELS.value, + data=data + )) + log.info( + f"Sending enc labels to {receiver} finished, task_id: {self.ctx.task_id}, label_num: {len(enc_labels)}, " + f"size: {len(data) / 1024}KB, time_costs: {time.time() - start_time}s") + + def _save_and_sync_fe_results(self): + log = self.ctx.components.logger() + task_id = self.ctx.task_id + self.woe_iv_df.to_csv(self.ctx.woe_iv_file, sep=',', header=True, index=None) + self.iv_selected_df.to_csv(self.ctx.iv_selected_file, sep=',', header=True, index=None) + self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file, + self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE) + log.info(f"Saving fe results finished, task_id: {task_id}") + + with open(self.ctx.woe_iv_file, 'rb') as f: + woe_iv = f.read() + with open(self.ctx.iv_selected_file, 'rb') as f: + iv_selected = f.read() + for i in range(1, len(self.ctx.participant_id_list)): + partner_id = self.ctx.participant_id_list[i] + if partner_id in self.ctx.result_receiver_id_list: + self.ctx.components.stub.push(PushRequest( + receiver=partner_id, + task_id=self.ctx.task_id, + key=FeMessage.WOE_FILE.value, + data=woe_iv + )) + self.ctx.components.stub.push(PushRequest( + receiver=partner_id, + task_id=self.ctx.task_id, + key=FeMessage.IV_SELECTED_FILE.value, + data=iv_selected + )) + log.info(f"Sending fe results finished, task_id: {task_id}") diff --git a/python/ppc_model/feature_engineering/vertical/passive_party.py b/python/ppc_model/feature_engineering/vertical/passive_party.py new file mode 100644 index 00000000..8aa63996 --- /dev/null +++ b/python/ppc_model/feature_engineering/vertical/passive_party.py @@ -0,0 +1,177 @@ +import multiprocessing +import os +import time + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelCipher, EncAggrLabels, EncAggrLabelsList +from ppc_common.ppc_utils import utils +from ppc_model.common.protocol import PheMessage +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext, FeMessage +from ppc_model.feature_engineering.vertical.utils import is_continuous_feature +from ppc_model.interface.model_base import VerticalModel +from ppc_model.network.stub import PullRequest, PushRequest + + +class VerticalFeatureEngineeringPassiveParty(VerticalModel): + + def __init__(self, ctx: FeatureEngineeringContext): + super().__init__(ctx) + + def fit(self, *args, **kwargs) -> None: + log = self.ctx.components.logger() + + task_id = self.ctx.task_id + start_time = time.time() + if self.ctx.use_iv: + log.info(f"Start feature engineering, task_id: {task_id}, shape: {self.ctx.feature.shape}, " + f"feature_name_list: {self.ctx.feature_name_list}.") + + # 执行密文交互，计算正样本数量 + public_key, enc_labels = self._get_enc_labels() + + # 根据特征分箱，聚合加密标签 + aggr_labels_bytes_list = self._binning_and_aggregating_all(public_key, enc_labels) + + # 发送聚合的密文标签 + self._send_all_enc_aggr_labels(public_key, aggr_labels_bytes_list) + + self._get_and_save_result() + log.info( + f"Feature engineering finished, task_id: {task_id}, timecost: {time.time() - start_time}s") + + def _get_enc_labels(self): + log = self.ctx.components.logger() + start_time = time.time() + active_party = self.ctx.participant_id_list[0] + data = self.ctx.components.stub.pull(PullRequest( + sender=active_party, + task_id=self.ctx.task_id, + key=FeMessage.ENC_LABELS.value + )) + + public_key, enc_labels = PheMessage.unpacking_data(self.ctx.codec, data) + log.info(f"All enc labels received, task_id: {self.ctx.task_id}, label_num: {len(enc_labels)}, " + f"size: {len(data) / 1024}KB, timecost: {time.time() - start_time}s") + return public_key, enc_labels + + def _binning_and_aggregating_all(self, public_key, enc_labels) -> list: + log = self.ctx.components.logger() + start_time = time.time() + params = [] + for i in range(self.ctx.feature.shape[1]): + is_continuous = is_continuous_feature(self.ctx.categorical, self.ctx.feature_name_list[i]) + params.append({ + 'is_continuous': is_continuous, + 'feature_index': i, + 'field': self.ctx.feature_name_list[i], + 'feature': self.ctx.feature[:, i], + 'public_key': public_key, + 'enc_labels': enc_labels, + 'group_num': self.ctx.group_num, + 'codec': self.ctx.codec + }) + # aggr_labels_str = [] + # with ProcessPoolExecutor(max_workers=max(1, os.cpu_count() - 1)) as executor: + # futures = [executor.submit(self._binning_and_aggregating_one, param) for param in params] + # for future in as_completed(futures): + # aggr_labels_str.append(future.result()) + + pool = multiprocessing.Pool() + aggr_labels_str = pool.map(self._binning_and_aggregating_one, params) + pool.close() + pool.join() + + log.info(f"Feature binning and aggregating finished, task_id: {self.ctx.task_id}, " + f"feature_num: {len(params)}, timecost: {time.time() - start_time}s") + return aggr_labels_str + + @staticmethod + def _binning_and_aggregating_one(param): + feature = param['feature'] + if param['is_continuous']: + bins = FeatureBinning.binning_continuous_feature(feature, param['group_num'])[0] + else: + bins = FeatureBinning.binning_categorical_feature(feature)[0] + + enc_labels = param['enc_labels'] + data_dict = {} + for key, value in zip(bins, enc_labels): + if key in data_dict: + data_dict[key]['count'] += 1 + # 执行同态加法 + data_dict[key]['sum'] = data_dict[key]['sum'] + value + else: + data_dict[key] = {'count': 1, 'sum': value} + + count_list = [data_dict[key]['count'] for key in sorted(data_dict.keys())] + aggr_enc_labels = [data_dict[key]['sum'] for key in sorted(data_dict.keys())] + + return VerticalFeatureEngineeringPassiveParty._encode_enc_aggr_labels( + param['codec'], param['field'], count_list, aggr_enc_labels) + + @staticmethod + def _encode_enc_aggr_labels(codec, field, count_list, aggr_enc_labels): + enc_aggr_labels_pb = EncAggrLabels() + enc_aggr_labels_pb.field = field + for count in count_list: + enc_aggr_labels_pb.count_list.append(count) + for cipher in aggr_enc_labels: + model_cipher = ModelCipher() + model_cipher.ciphertext, model_cipher.exponent = \ + codec.encode_cipher(cipher, be_secure=False) + enc_aggr_labels_pb.cipher_list.append(model_cipher) + return utils.pb_to_bytes(enc_aggr_labels_pb) + + def _send_all_enc_aggr_labels(self, public_key, aggr_labels_bytes_list): + start_time = time.time() + enc_aggr_labels_list_pb = EncAggrLabelsList() + enc_aggr_labels_list_pb.public_key = self.ctx.codec.encode_enc_key(public_key) + + for aggr_labels_bytes in aggr_labels_bytes_list: + enc_aggr_labels_pb = EncAggrLabels() + utils.bytes_to_pb(enc_aggr_labels_pb, aggr_labels_bytes) + enc_aggr_labels_list_pb.enc_aggr_labels_list.append(enc_aggr_labels_pb) + + data = utils.pb_to_bytes(enc_aggr_labels_list_pb) + + self.ctx.components.logger().info( + f"Encoding all enc aggr labels finished, task_id: {self.ctx.task_id}, " + f"size: {len(data) / 1024}KB, timecost: {time.time() - start_time}s") + + self.ctx.components.stub.push(PushRequest( + receiver=self.ctx.participant_id_list[0], + task_id=self.ctx.task_id, + key=FeMessage.AGGR_LABELS.value, + data=data + )) + + self.ctx.components.logger().info( + f"Sending all enc aggr labels finished, task_id: {self.ctx.task_id}, " + f"feature_num: {len(aggr_labels_bytes_list)}, " + f"size: {len(data) / 1024}KB, timecost: {time.time() - start_time}s") + + def _get_and_save_result(self): + active_party = self.ctx.participant_id_list[0] + if self.ctx.components.stub.agency_id in self.ctx.result_receiver_id_list: + # 保存来自标签方的woe/iv结果 + data = self.ctx.components.stub.pull(PullRequest( + sender=active_party, + task_id=self.ctx.task_id, + key=FeMessage.WOE_FILE.value + )) + self.ctx.components.logger().info( + f"Result of woe/iv received, task_id: {self.ctx.task_id}, size: {len(data) / 1024}KB") + with open(self.ctx.woe_iv_file, 'wb') as f: + f.write(data) + self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file, + self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE) + + # 保存来自标签方的iv筛选结果 + data = self.ctx.components.stub.pull(PullRequest( + sender=active_party, + task_id=self.ctx.task_id, + key=FeMessage.IV_SELECTED_FILE.value)) + self.ctx.components.logger().info( + f"Result of iv_select received, task_id: {self.ctx.task_id}, size: {len(data) / 1024}KB") + with open(self.ctx.iv_selected_file, 'wb') as f: + f.write(data) diff --git a/python/ppc_model/feature_engineering/vertical/utils.py b/python/ppc_model/feature_engineering/vertical/utils.py new file mode 100644 index 00000000..cdeeceaf --- /dev/null +++ b/python/ppc_model/feature_engineering/vertical/utils.py @@ -0,0 +1,76 @@ +import numpy as np +import pandas as pd + +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning + + +def is_continuous_feature(categorical: str, field): + if categorical == '0': + return True + return field not in categorical.split(',') + + +def calculate_woe_iv_with_pos_event(grouped): + grouped['neg_event'] = grouped['count'] - grouped['pos_event'] + + # 避免出现无穷大woe + grouped['pos_event'] = grouped['pos_event'].astype(np.float64) + grouped['neg_event'] = grouped['neg_event'].astype(np.float64) + grouped.loc[grouped['neg_event'] == 0, 'pos_event'] += 0.5 + grouped.loc[grouped['neg_event'] == 0, 'neg_event'] = 0.5 + grouped.loc[grouped['pos_event'] == 0, 'neg_event'] += 0.5 + grouped.loc[grouped['pos_event'] == 0, 'pos_event'] = 0.5 + + # 计算WOE和IV值 + grouped['pos_event_rate'] = grouped['pos_event'] / \ + (grouped['pos_event'].sum()) + grouped['neg_event_rate'] = grouped['neg_event'] / \ + (grouped['neg_event'].sum()) + grouped['woe'] = np.log(grouped['pos_event_rate'] / + (grouped['neg_event_rate'])) + grouped['iv'] = (grouped['pos_event_rate'] - + grouped['neg_event_rate']) * grouped['woe'] + iv_total = grouped['iv'].sum() + grouped['iv_total'] = iv_total + return grouped, iv_total + + +def calculate_woe_iv(feature: np.ndarray, label: np.ndarray, num_bins: int = 10, is_continuous: bool = True, + is_equal_freq: bool = True): + # 将特征和目标变量合并 + combined = pd.DataFrame({'feature': feature, 'label': label}) + # 按特征值对数据集进行分箱 + if is_continuous: + combined['bins'] = FeatureBinning.binning_continuous_feature(feature, num_bins, is_equal_freq)[0] + else: + combined['bins'] = FeatureBinning.binning_categorical_feature(feature)[0] + # 计算每个分箱中的正负样本数量和总体样本数量 + grouped = combined.groupby('bins')['label'].agg(['count', 'sum']) + grouped = grouped.rename(columns={'sum': 'pos_event'}) + + return calculate_woe_iv_with_pos_event(grouped) + + +def calculate_woe_iv_with_pos_event(grouped): + grouped['neg_event'] = grouped['count'] - grouped['pos_event'] + + # 避免出现无穷大woe + grouped['pos_event'] = grouped['pos_event'].astype(np.float64) + grouped['neg_event'] = grouped['neg_event'].astype(np.float64) + grouped.loc[grouped['neg_event'] == 0, 'pos_event'] += 0.5 + grouped.loc[grouped['neg_event'] == 0, 'neg_event'] = 0.5 + grouped.loc[grouped['pos_event'] == 0, 'neg_event'] += 0.5 + grouped.loc[grouped['pos_event'] == 0, 'pos_event'] = 0.5 + + # 计算WOE和IV值 + grouped['pos_event_rate'] = grouped['pos_event'] / \ + (grouped['pos_event'].sum()) + grouped['neg_event_rate'] = grouped['neg_event'] / \ + (grouped['neg_event'].sum()) + grouped['woe'] = np.log(grouped['pos_event_rate'] / + (grouped['neg_event_rate'])) + grouped['iv'] = (grouped['pos_event_rate'] - + grouped['neg_event_rate']) * grouped['woe'] + iv_total = grouped['iv'].sum() + grouped['iv_total'] = iv_total + return grouped, iv_total diff --git a/python/ppc_model/interface/__init__.py b/python/ppc_model/interface/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/interface/model_base.py b/python/ppc_model/interface/model_base.py new file mode 100644 index 00000000..050980ae --- /dev/null +++ b/python/ppc_model/interface/model_base.py @@ -0,0 +1,33 @@ +from abc import ABC + +from pandas import DataFrame + + +class ModelBase(ABC): + mode: str + + def __init__(self, ctx): + self.ctx = ctx + + def fit( + self, + *args, + **kwargs + ) -> None: + pass + + def transform(self, transform_data: DataFrame) -> DataFrame: + pass + + def predict(self, predict_data: DataFrame) -> DataFrame: + pass + + def save_model(self, file_path): + pass + + def load_model(self, file_path): + pass + + +class VerticalModel(ModelBase): + mode = "VERTICAL" diff --git a/python/ppc_model/interface/rpc_client.py b/python/ppc_model/interface/rpc_client.py new file mode 100644 index 00000000..9473be6d --- /dev/null +++ b/python/ppc_model/interface/rpc_client.py @@ -0,0 +1,8 @@ +from abc import ABC + + +class RpcClient(ABC): + rpc_type: str + + def send(self, request): + ... diff --git a/python/ppc_model/interface/task_engine.py b/python/ppc_model/interface/task_engine.py new file mode 100644 index 00000000..123b0eb9 --- /dev/null +++ b/python/ppc_model/interface/task_engine.py @@ -0,0 +1,9 @@ +from abc import ABC + + +class TaskEngine(ABC): + task_type: str + + @staticmethod + def run(args: dict): + ... diff --git a/python/ppc_model/metrics/__init__.py b/python/ppc_model/metrics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/metrics/evaluation.py b/python/ppc_model/metrics/evaluation.py new file mode 100644 index 00000000..a759dc9d --- /dev/null +++ b/python/ppc_model/metrics/evaluation.py @@ -0,0 +1,276 @@ +import gc +import time +import random +import traceback +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from typing import Dict, Union, Tuple +from sklearn import metrics +from sklearn.metrics import accuracy_score +from sklearn.metrics import precision_recall_curve +from sklearn.metrics import roc_curve, auc + +from ppc_model.common.context import Context +from ppc_model.common.global_context import plot_lock +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import EvaluationType +from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import FeatureEvaluationResult + + +_Score = Union[float, Tuple[float, float]] + + +class Evaluation: + + def __init__(self, + ctx: Context, + dataset: SecureDataset, + train_praba:np.ndarray = None, + test_praba:np.ndarray = None) -> None: + + self.job_id = ctx.job_id + self.storage_client = ctx.components.storage_client + self.summary_evaluation_file = ctx.summary_evaluation_file + self.remote_summary_evaluation_file = ctx.remote_summary_evaluation_file + self.model_result_file = ctx.test_model_result_file + self.model_output_file = ctx.test_model_output_file + self.remote_model_output_file = ctx.remote_test_model_output_file + + self.metric_roc_file = ctx.test_metric_roc_file + self.metric_ks_file = ctx.test_metric_ks_file + self.metric_pr_file = ctx.test_metric_pr_file + self.metric_acc_file = ctx.test_metric_acc_file + self.metric_ks_table = ctx.test_metric_ks_table + self.remote_metric_roc_file = ctx.remote_test_metric_roc_file + self.remote_metric_ks_file = ctx.remote_test_metric_ks_file + self.remote_metric_pr_file = ctx.remote_test_metric_pr_file + self.remote_metric_acc_file = ctx.remote_test_metric_acc_file + self.remote_metric_ks_table = ctx.remote_test_metric_ks_table + + # if test_praba is None or dataset.test_y is None: + # raise Exception('test_praba or test_y is None') + + if test_praba is not None: + test_ks, test_auc = self.evaluation_file( + ctx, dataset.test_idx, dataset.test_y, test_praba, 'test') + if train_praba is not None: + train_ks, train_auc = self.evaluation_file( + ctx, dataset.train_idx, dataset.train_y, train_praba, 'train') + if dataset.train_y is not None: + self.summary_evaluation(dataset, test_ks, test_auc, train_ks, train_auc) + + @staticmethod + def fevaluation( + y_true: np.ndarray, + y_pred: np.ndarray, + decimal_num: int = 4 + ) -> Dict[str, _Score]: + auc = metrics.roc_auc_score(y_true, y_pred) + + y_pred_label = [0 if p <= 0.5 else 1 for p in y_pred] + acc = metrics.accuracy_score(y_true, y_pred_label) + recall = metrics.recall_score(y_true, y_pred_label) + precision = metrics.precision_score(y_true, y_pred_label) + + scores_dict = { + 'auc': auc, + 'acc': acc, + 'recall': recall, + 'precision': precision + } + for metric_name in scores_dict: + scores_dict[metric_name] = round( + scores_dict[metric_name], decimal_num) + return scores_dict + + def summary_evaluation(self, dataset, test_ks, test_auc, train_ks, train_auc): + train_evaluation = FeatureEvaluationResult( + type=EvaluationType.TRAIN, ks_value=train_ks, auc_value=train_auc, label_list=dataset.train_y) + test_evaluation = FeatureEvaluationResult( + type=EvaluationType.VALIDATION, ks_value=test_ks, auc_value=test_auc, label_list=dataset.test_y) + FeatureEvaluationResult.store_and_upload_summary( + [train_evaluation, test_evaluation], + self.summary_evaluation_file, self.remote_summary_evaluation_file, + self.storage_client) + + @staticmethod + def calculate_ks_and_stats(predicted_proba, actual_label, num_buckets=10): + # 合并预测概率和实际标签为一个 DataFrame + df = pd.DataFrame({'predicted_proba': predicted_proba.reshape(-1), 'actual_label': actual_label.reshape(-1)}) + # 根据预测概率降序排列 + df_sorted = df.sort_values(by='predicted_proba', ascending=False) + # 将数据划分为 num_buckets 个分组 + try: + df_sorted['bucket'] = pd.qcut(df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] + except Exception: + df_sorted['bucket'] = pd.cut(df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] + # 统计每个分组的信息 + stats = df_sorted.groupby('bucket').agg({ + 'actual_label': ['count', 'sum'], + 'predicted_proba': ['min', 'max'] + }) + # 计算其他指标 + stats.columns = ['count', 'positive_count', 'predict_proba_min', 'predict_proba_max'] + stats['positive_ratio'] = stats['positive_count'] / stats['count'] + stats['negative_ratio'] = 1 - stats['positive_ratio'] + stats['count_ratio'] = stats['count'] / stats['count'].sum() + # stats['累计坏客户占比'] = stats['坏客户数'].cumsum() / stats['坏客户数'].sum() + # 计算累计坏客户占比，从第 9 组开始计算 + stats['cum_positive_ratio'] = stats['positive_count'].iloc[::-1].cumsum()[::-1] / stats['positive_count'].sum() + stats = stats[['count_ratio', 'count', 'positive_count', + 'positive_ratio', 'negative_ratio', 'cum_positive_ratio']].reset_index() + stats.columns = ['分组', '样本占比', '样本数', '正样本数', '正样本比例', '负样本比例', '累积正样本占比'] + return stats + + def evaluation_file(self, ctx, data_index: np.ndarray, + y_true: np.ndarray, y_praba: np.ndarray, label: str = 'test'): + if label == 'train': + self.model_result_file = ctx.train_model_result_file + self.model_output_file = ctx.train_model_output_file + self.remote_model_output_file = ctx.remote_train_model_output_file + + self.metric_roc_file = ctx.train_metric_roc_file + self.metric_ks_file = ctx.train_metric_ks_file + self.metric_pr_file = ctx.train_metric_pr_file + self.metric_acc_file = ctx.train_metric_acc_file + self.metric_ks_table = ctx.train_metric_ks_table + self.remote_metric_roc_file = ctx.remote_train_metric_roc_file + self.remote_metric_ks_file = ctx.remote_train_metric_ks_file + self.remote_metric_pr_file = ctx.remote_train_metric_pr_file + self.remote_metric_acc_file = ctx.remote_train_metric_acc_file + self.remote_metric_ks_table = ctx.remote_train_metric_ks_table + + if y_true is not None: + # metrics plot + max_retry = 3 + retry_num = 0 + while retry_num < max_retry: + retry_num += 1 + try: + with plot_lock: + ks_value, auc_value = Evaluation.plot_two_class_graph(self, y_true, y_praba) + except: + ctx.components.logger().info(f'y_true = {len(y_true)}, {y_true[0:2]}') + ctx.components.logger().info(f'y_praba = {len(y_praba)}, {y_praba[0:2]}') + err = traceback.format_exc() + # ctx.components.logger().exception(err) + ctx.components.logger().info( + f'plot metrics in times-{retry_num} failed, traceback: {err}.') + time.sleep(random.uniform(0.1, 3)) + + ResultFileHandling._upload_file(self.storage_client, self.metric_roc_file, self.remote_metric_roc_file) + ResultFileHandling._upload_file(self.storage_client, self.metric_ks_file, self.remote_metric_ks_file) + ResultFileHandling._upload_file(self.storage_client, self.metric_pr_file, self.remote_metric_pr_file) + ResultFileHandling._upload_file(self.storage_client, self.metric_acc_file, self.remote_metric_acc_file) + + # ks table + ks_table = self.calculate_ks_and_stats(y_praba, y_true) + ks_table.to_csv(self.metric_ks_table, header=True, index=None) + ResultFileHandling._upload_file(self.storage_client, self.metric_ks_table, self.remote_metric_ks_table) + else: + ks_value = auc_value = None + + # predict result + self._parse_model_result(data_index, y_true, y_praba) + ResultFileHandling._upload_file(self.storage_client, self.model_output_file, self.remote_model_output_file) + + return ks_value, auc_value + + def _parse_model_result(self, data_index, y_true=None, y_praba=None): + + np.savetxt(self.model_result_file, y_praba, delimiter=',', fmt='%f') + + if y_true is None: + df = pd.DataFrame(np.column_stack((data_index, y_praba)), columns=['id', 'class_pred']) + else: + df = pd.DataFrame(np.column_stack((data_index, y_true, y_praba)), + columns=['id', 'class_label', 'class_pred']) + df['class_label'] = df['class_label'].astype(int) + + df['id'] = df['id'].astype(int) + df['class_pred'] = df['class_pred'].astype(float) + df = df.sort_values(by='id') + df.to_csv(self.model_output_file, index=None) + + def plot_two_class_graph(self, y_true, y_scores): + + y_label_probs = y_true + y_pred_probs = y_scores + # plt.cla() + plt.rcParams['figure.figsize'] = (12.0, 8.0) + + # plot ROC + fpr, tpr, thresholds = roc_curve(y_label_probs, y_pred_probs, pos_label=1) + auc_value = auc(fpr, tpr) + plt.figure(f'roc-{self.job_id}') + plt.title('ROC Curve') # give plot a title + plt.xlabel('False Positive Rate (1 - Specificity)') + plt.ylabel('True Positive Rate (Sensitivity)') + plt.plot([0, 1], [0, 1], 'k--', lw=2) + plt.plot(fpr, tpr, label='area = {0:0.5f}' + ''.format(auc_value)) + plt.legend(loc="lower right") + plt.savefig(self.metric_roc_file, dpi=1000) + # plt.show() + + plt.close('all') + gc.collect() + + # plot KS + plt.figure(f'ks-{self.job_id}') + threshold_x = np.sort(thresholds) + threshold_x[-1] = 1 + ks_value = max(abs(fpr - tpr)) + plt.title('KS Curve') + plt.xlabel('Threshold') + plt.plot(threshold_x, tpr, label='True Positive Rate') + plt.plot(threshold_x, fpr, label='False Positive Rate') + # 标记最大ks值 + x_index = np.argwhere(abs(fpr - tpr) == ks_value)[0, 0] + plt.plot((threshold_x[x_index], threshold_x[x_index]), (fpr[x_index], tpr[x_index]), + label='ks = {:.3f}'.format(ks_value), color='r', marker='o', markerfacecolor='r', markersize=5) + plt.legend(loc="lower right") + plt.savefig(self.metric_ks_file, dpi=1000) + # plt.show() + + plt.close('all') + gc.collect() + + # plot Precision Recall + plt.figure(f'pr-{self.job_id}') + plt.title('Precision/Recall Curve') + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.xlim(0.0, 1.0) + plt.ylim(0.0, 1.05) + precision, recall, thresholds = precision_recall_curve( + y_label_probs, y_pred_probs) + plt.plot(recall, precision) + plt.savefig(self.metric_pr_file, dpi=1000) + # plt.show() + + plt.close('all') + gc.collect() + + # plot accuracy + plt.figure(f'accuracy-{self.job_id}') + thresholds = np.linspace(0, 1, num=100) # 在0~1之间生成100个阈值 + accuracies = [] + for threshold in thresholds: + predicted_labels = (y_pred_probs >= threshold).astype(int) + accuracy = accuracy_score(y_label_probs, predicted_labels) + accuracies.append(accuracy) + plt.title('Accuracy Curve') + plt.xlabel('Threshold') + plt.ylabel('Accuracy') + plt.xlim(0.0, 1.0) + plt.ylim(0.0, 1.05) + plt.plot(thresholds, accuracies) + plt.savefig(self.metric_acc_file, dpi=1000) + # plt.show() + + plt.close('all') + gc.collect() + return (ks_value, auc_value) diff --git a/python/ppc_model/metrics/loss.py b/python/ppc_model/metrics/loss.py new file mode 100644 index 00000000..b2b4e6b3 --- /dev/null +++ b/python/ppc_model/metrics/loss.py @@ -0,0 +1,32 @@ +import numpy as np + + +class Loss: + pass + + +class BinaryLoss(Loss): + + def __init__(self, objective: str) -> None: + super().__init__() + self.objective = objective + + @staticmethod + def sigmoid(x: np.ndarray): + return 1 / (1 + np.exp(-x)) + + @staticmethod + def compute_gradient(y_true: np.ndarray, y_pred: np.ndarray): + return y_pred - y_true + + @staticmethod + def compute_hessian(y_pred: np.ndarray): + return y_pred * (1 - y_pred) + + @staticmethod + def compute_loss(y_true: np.ndarray, y_pred: np.ndarray): + '''binary_cross_entropy''' + # 避免log(0)错误 + epsilon = 1e-15 + y_pred = np.clip(y_pred, epsilon, 1 - epsilon) + return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred)) diff --git a/python/ppc_model/metrics/model_plot.py b/python/ppc_model/metrics/model_plot.py new file mode 100644 index 00000000..ca169859 --- /dev/null +++ b/python/ppc_model/metrics/model_plot.py @@ -0,0 +1,181 @@ +import gc +import time +import random +import traceback +import matplotlib.pyplot as plt +import networkx as nx +from networkx.drawing.nx_pydot import graphviz_layout + +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.global_context import plot_lock +from ppc_model.secure_lgbm.vertical.booster import VerticalBooster + + +class ModelPlot: + + def __init__(self, model: VerticalBooster) -> None: + + self.ctx = model.ctx + self.model = model + self._tree_id = 0 + self._leaf_id = None + self._G = None + self._split = None + self.storage_client = self.ctx.components.storage_client + + if model._trees is not None and \ + self.ctx.components.config_data['AGENCY_ID'] in self.ctx.result_receiver_id_list: + self.plot_tree() + + def plot_tree(self): + trees = self.model._trees + self._split = self.model._X_split + + for i, tree in enumerate(trees): + if i < 6: + tree_file_path = self.ctx.model_tree_prefix + '_' + str(self._tree_id)+'.svg' + remote_tree_file_path = self.ctx.remote_model_tree_prefix + '_' + str(self._tree_id)+'.svg' + self._tree_id += 1 + self._leaf_id = 0 + self._G = DiGraphTree() + if not isinstance(tree, list) or tree == 0: + continue + else: + self._graph_gtree(tree) + + max_retry = 3 + retry_num = 0 + while retry_num < max_retry: + retry_num += 1 + try: + with plot_lock: + self._G.tree_plot(figsize=(10, 5), save_filename=tree_file_path) + except: + self.ctx.components.logger().info(f'tree_id = {i}, tree = {tree}') + self.ctx.components.logger().info(f'G = {self._G}') + err = traceback.format_exc() + # self.ctx.components.logger().exception(err) + self.ctx.components.logger().info( + f'plot tree-{i} in times-{retry_num} failed, traceback: {err}.') + time.sleep(random.uniform(0.1, 3)) + + ResultFileHandling._upload_file(self.storage_client, tree_file_path, remote_tree_file_path) + + def _graph_gtree(self, tree, leaf_id=0, depth=0, orient=None, split_info=None): + self._leaf_id += 1 + self._G.add_node(self._leaf_id) + if split_info is not None: + if self.ctx.participant_id_list[split_info.agency_idx] == self.ctx.components.config_data['AGENCY_ID']: + feature = str(self.model.dataset.feature_name[split_info.agency_feature]) + value = str(round(float(self._split[split_info.agency_feature][split_info.value]), 4)) + else: + feature = str(split_info.feature) + value = str(split_info.value) + else: + feature = value = '' + + if isinstance(tree, list): + best_split_info, left_tree, right_tree = tree[0] + if leaf_id != 0: + if orient == 'left': + self._G.add_weighted_edges_from( + [(leaf_id, self._leaf_id, orient+'_'+feature+'_'+value+'_'+str(split_info.w_left))]) + elif orient == 'right': + self._G.add_weighted_edges_from( + [(leaf_id, self._leaf_id, orient+'_'+feature+'_'+value+'_'+str(split_info.w_right))]) + my_leaf_id = self._leaf_id + self._graph_gtree(left_tree, my_leaf_id, depth+1, 'left', best_split_info) + self._graph_gtree(right_tree, my_leaf_id, depth+1, 'right', best_split_info) + else: + if leaf_id != 0: + self._G.add_weighted_edges_from( + [(leaf_id, self._leaf_id, orient+'_'+feature+'_'+value+'_'+str(tree))]) + + +class DiGraphTree(nx.DiGraph): + + def __init__(self): + + super().__init__() + + def tree_leaves(self): + leaves_list = [x for x in self.nodes() if self.out_degree(x)==0 and self.in_degree(x)<=1] + return leaves_list + + def tree_dfs_nodes(self): + nodes_list = list(nx.dfs_preorder_nodes(self)) + return nodes_list + + def tree_dfs_leaves(self): + dfs_leaves = [x for x in self.tree_dfs_nodes() if x in self.tree_leaves()] + return dfs_leaves + + def tree_depth(self): + max_depth = max(nx.shortest_path_length(self, 0).values()) + return max_depth + + def tree_shortest_path(self, node0, node1): + path_length = nx.shortest_path_length(self, node0, node1) + return path_length + + def tree_plot(self, split=True, figsize=(20, 10), dpi=300, save_filename=None): + # plt.cla() + pos = graphviz_layout(self, prog='dot') + # pos = nx.nx_agraph.graphviz_layout(self, prog='dot') + edge_labels = nx.get_edge_attributes(self, 'weight') + + if split: + labels = {} + # leaves = self.tree_leaves() + leaves = [x for x in self.nodes() if self.out_degree(x)==0 and self.in_degree(x)<=1] + + if leaves == [0]: + leaves = [] + self.remove_node(0) + + for n in self.nodes(): + + if n in leaves: + # in_node = list(nx.all_neighbors(self, n))[0] + in_node = list(self.predecessors(n))[0] + weight = edge_labels[(in_node, n)] + try: + labels[n] = round(float(str(weight).split('_')[3]), 4) + except: + labels[n] = str(weight).split('_')[3] + else: + in_node = list(nx.neighbors(self, n))[0] + weight = edge_labels[(n, in_node)] + labels[n] = weight.split('_')[1] + ':' + weight.split('_')[2] + + # for key, value in edge_labels.items(): + # edge_labels[key] = round(float(value.split('_')[3]), 4) + + plt.figure(figsize=figsize, dpi=dpi) + nx.draw(self, pos, + node_size=1000, node_color='#72BFC5', node_shape='o', alpha=None, + with_labels=True, labels=labels, font_weight='normal', font_color='black') + # nx.draw_networkx_edge_labels(self, pos, edge_labels=edge_labels) + # plt.show() + if save_filename is not None: + plt.savefig(save_filename) + else: + plt.show() + + else: + labels = {n: n for n in self.nodes()} + for key, value in edge_labels.items(): + edge_labels[key] = value.split('_')[1] + '-' + value.split('_')[2] + \ + '-' + str(round(float(value.split('_')[3]), 4)) + + plt.figure(figsize=figsize, dpi=dpi) + nx.draw(self, pos, with_labels=True, labels=labels, font_weight='bold') + nx.draw_networkx_edge_labels(self, pos, edge_labels=edge_labels) + # plt.show() + if save_filename is not None: + plt.savefig(save_filename) + else: + plt.show() + + plt.close('all') + gc.collect() diff --git a/python/ppc_model/metrics/test/__init__.py b/python/ppc_model/metrics/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/metrics/test/test_metrics.py b/python/ppc_model/metrics/test/test_metrics.py new file mode 100644 index 00000000..3545aaa7 --- /dev/null +++ b/python/ppc_model/metrics/test/test_metrics.py @@ -0,0 +1,165 @@ +import os +import unittest +import numpy as np + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot, DiGraphTree +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + +data_size = 1000 +feature_dim = 20 + + +def mock_args(): + job_id = 'j-111' + task_id = 't-111' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'max_bin': 10, + 'n_estimators': 2, + 'max_depth': 3, + 'use_goss': 1, + 'feature_rate': 0.8, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + return args_a, args_b + + +class TestXgboostTraining(unittest.TestCase): + + args_a, args_b = mock_args() + + def test_active_metrics(self): + + active_components = Initializer(log_config_path='', config_path='') + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLGBMContext(self.args_a, active_components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + secure_dataset_a = SecureDataset(task_info_a, model_data) + booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) + print(secure_dataset_a.feature_name) + print(secure_dataset_a.train_idx.shape) + print(secure_dataset_a.train_X.shape) + print(secure_dataset_a.train_y.shape) + print(secure_dataset_a.test_idx.shape) + print(secure_dataset_a.test_X.shape) + print(secure_dataset_a.test_y.shape) + + # booster_a._train_praba = np.random.rand(len(secure_dataset_a.train_y)) + booster_a._test_praba = np.random.rand(len(secure_dataset_a.test_y)) + + Evaluation(task_info_a, secure_dataset_a, booster_a._train_praba, booster_a._test_praba) + + def test_passive_metrics(self): + + passive_components = Initializer(log_config_path='', config_path='') + passive_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} + passive_components.mock_logger = MockLogger() + task_info_b = SecureLGBMContext(self.args_b, passive_components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=False) + secure_dataset_b = SecureDataset(task_info_b, model_data) + booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) + print(secure_dataset_b.feature_name) + print(secure_dataset_b.train_idx.shape) + print(secure_dataset_b.train_X.shape) + print(secure_dataset_b.test_idx.shape) + print(secure_dataset_b.test_X.shape) + + # booster_b._train_praba = np.random.rand(len(secure_dataset_b.train_idx)) + booster_b._test_praba = np.random.rand(len(secure_dataset_b.test_idx)) + + Evaluation(task_info_b, secure_dataset_b, booster_b._train_praba, booster_b._test_praba) + + def test_model_plot(self): + + active_components = Initializer(log_config_path='', config_path='') + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLGBMContext(self.args_a, active_components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + secure_dataset_a = SecureDataset(task_info_a, model_data) + booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) + if os.path.exists(booster_a.ctx.model_data_file): + booster_a.load_model() + ModelPlot(booster_a) + + def test_digraphtree(self): + Gtree = DiGraphTree() + Gtree.add_node(0) + Gtree.add_nodes_from([1, 2]) + Gtree.add_weighted_edges_from( + [(0, 1, 'left_'+str(2)+'_'+str(3)+'_'+str(0.5)), + (0, 2, 'right_'+str(2)+'_'+str(3)+'_'+str(0.9))]) + Gtree.add_nodes_from([3, 4]) + Gtree.add_weighted_edges_from( + [(1, 3, 'left_'+str(20)+'_'+str(4)+'_'+str(0.5)), + (1, 4, 'right_'+str(20)+'_'+str(4)+'_'+str(0.9))]) + Gtree.add_nodes_from([5, 6]) + Gtree.add_weighted_edges_from( + [(2, 5, 'left_'+str(2)+'_'+str(7)+'_'+str(0.5)), + (2, 6, 'right_'+str(2)+'_'+str(7)+'_'+str(0.9))]) + Gtree.add_nodes_from([7, 8]) + Gtree.add_weighted_edges_from( + [(3, 7, 'left_'+str(1)+'_'+str(11)+'_'+str(0.5)), + (3, 8, 'right_'+str(1)+'_'+str(11)+'_'+str(0.9))]) + Gtree.add_nodes_from([9, 10]) + Gtree.add_weighted_edges_from( + [(4, 9, 'left_'+str(18)+'_'+str(2)+'_'+str(0.5)), + (4, 10, 'right_'+str(18)+'_'+str(2)+'_'+str(0.9))]) + Gtree.add_nodes_from([11, 12]) + Gtree.add_weighted_edges_from( + [(5, 11, 'left_'+str(23)+'_'+str(25)+'_'+str(0.5)), + (5, 12, 'right_'+str(23)+'_'+str(25)+'_'+str(0.9))]) + Gtree.add_nodes_from([13, 14]) + Gtree.add_weighted_edges_from( + [(6, 13, 'left_'+str(16)+'_'+str(10)+'_'+str(0.5)), + (6, 14, 'right_'+str(16)+'_'+str(10)+'_'+str(0.9))]) + + # Gtree.tree_plot() + # Gtree.tree_plot(split=False, figsize=(10, 5)) + # Gtree.tree_plot(figsize=(6, 3)) + Gtree.tree_plot(figsize=(10, 5), save_filename='tree.svg') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py new file mode 100644 index 00000000..935446d1 --- /dev/null +++ b/python/ppc_model/model_result/task_result_handler.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +from ppc_common.ppc_utils.utils import PpcException, PpcErrorCode +from ppc_common.ppc_utils import utils +from ppc_model.common.protocol import ModelTask +from ppc_common.ppc_ml.model.algorithm_info import ClassificationType +from ppc_model.common.model_result import ResultFileHandling +from ppc_common.ppc_ml.model.algorithm_info import EvaluationType +from ppc_model.common.base_context import BaseContext +from enum import Enum + + +class TaskResultRequest: + def __init__(self, job_id, task_type): + self.job_id = job_id + self.task_type = task_type + + +class DataType(Enum): + TEXT = "str", + TABLE = "table", + IMAGE = "image" + + +class DataItem: + DEFAULT_NAME_PROPERTY = "metricsName" + DEFAULT_DATA_PROPERTY = "metricsData" + DEFAULT_TYPE_PROPERTY = "metricsType" + + def __init__(self, name, data, type, + name_property=DEFAULT_NAME_PROPERTY, + data_property=DEFAULT_DATA_PROPERTY, + type_property=DEFAULT_TYPE_PROPERTY): + self.name = name + self.data = data + self.type = type + self.name_property = name_property + self.data_property = data_property + self.type_property = type_property + + def to_dict(self): + return {self.name_property: self.name, + self.data_property: self.data, + self.type_property: self.type.name} + + +class ResultFileMeta: + def __init__(self, table_file_name, retrieve_lines=-1): + self.table_file_name = table_file_name + self.retrieve_lines = retrieve_lines + + +class JobEvaluationResult: + DEFAULT_TRAIN_EVALUATION_FILES = { + EvaluationType.ROC: utils.MPC_TRAIN_METRIC_ROC_FILE, + EvaluationType.PR: utils.MPC_TRAIN_METRIC_PR_FILE, + EvaluationType.KS: utils.MPC_TRAIN_METRIC_KS_FILE, + EvaluationType.ACCURACY: utils.MPC_TRAIN_METRIC_ACCURACY_FILE, + EvaluationType.CONFUSION_MATRIX: utils.MPC_TRAIN_METRIC_CONFUSION_MATRIX_FILE} + + DEFAULT_VALIDATION_EVALUATION_FILES = { + EvaluationType.ROC: utils.MPC_TRAIN_SET_METRIC_ROC_FILE, + EvaluationType.PR: utils.MPC_TRAIN_SET_METRIC_PR_FILE, + EvaluationType.KS: utils.MPC_TRAIN_SET_METRIC_KS_FILE, + EvaluationType.ACCURACY: utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE} + + DEFAULT_EVAL_EVALUATION_FILES = { + EvaluationType.ROC: utils.MPC_EVAL_METRIC_ROC_FILE, + EvaluationType.PR: utils.MPC_EVAL_METRIC_PR_FILE, + EvaluationType.KS: utils.MPC_EVAL_METRIC_KS_FILE, + EvaluationType.ACCURACY: utils.MPC_EVAL_METRIC_ACCURACY_FILE + } + + def __init__(self, property_name, classification_type, + job_id, evaluation_files, components): + self.job_id = job_id + self.classification_type = classification_type + self.components = components + self.logger = self.components.logger() + self.classification_type = classification_type + self.property_name = property_name + self.evaluation_files = evaluation_files + self.evaluation_results = [] + try: + self._fetch_evaluation_result() + self._fetch_two_classifcation_evaluation_result() + self._fetch_multi_classifcation_evaluation_result() + except Exception as e: + pass + + def _fetch_evaluation_result(self): + self.logger.info( + f"fetch roc-evaluation from: {self.evaluation_files[EvaluationType.ROC]}") + self.evaluation_results.append(DataItem("ROC", ResultFileHandling.make_graph_data( + self.components, + self.job_id, + self.evaluation_files[EvaluationType.ROC]), + DataType.IMAGE)) + self.logger.info( + f"fetch pr-evaluation from: {self.evaluation_files[EvaluationType.PR]}") + self.evaluation_results.append(DataItem("Precision Recall", ResultFileHandling.make_graph_data( + self.components, + self.job_id, + self.evaluation_files[EvaluationType.PR]), DataType.IMAGE)) + + def _fetch_two_classifcation_evaluation_result(self): + if self.classification_type is not ClassificationType.TWO: + return + + self.logger.info( + f"fetch ks-evaluation from: {self.evaluation_files[EvaluationType.KS]}") + self.evaluation_results.append(DataItem("K-S", ResultFileHandling.make_graph_data( + self.components, + self.job_id, + self.evaluation_files[EvaluationType.KS]), + DataType.IMAGE)) + + self.logger.info( + f"fetch accuracy-evaluation from: {self.evaluation_files[EvaluationType.ACCURACY]}") + self.evaluation_results.append(DataItem("Accuracy", + ResultFileHandling.make_graph_data( + self.components, + self.job_id, + self.evaluation_files[EvaluationType.ACCURACY]), + DataType.IMAGE)) + + def _fetch_multi_classifcation_evaluation_result(self): + if self.classification_type is not ClassificationType.MULTI: + return + self.logger.info( + f"fetch confusion-matrix-evaluation from: {self.evaluation_files[EvaluationType.CONFUSION_MATRIX]}") + self.evaluation_results.append(DataItem("Confusion Matrix", + ResultFileHandling.make_graph_data(self.components, + self.job_id, + self.evaluation_files[EvaluationType.CONFUSION_MATRIX]), + DataType.IMAGE)) + + def load_ks_table(self, ks_table_file, ks_table_property): + ks_table_object = TableResult(components=self.components, + job_id=self.job_id, file_meta=ResultFileMeta(table_file_name=ks_table_file)) + self.ks_table = ks_table_object.to_dict() + self.ks_table_property = ks_table_property + + def to_dict(self): + evaluation_result_list = [] + for evaluation in self.evaluation_results: + evaluation_result_list.append(evaluation.to_dict()) + result = {self.property_name: evaluation_result_list} + if self.ks_table is not None: + result.update({self.ks_table_property: self.ks_table}) + return result + + +class TableResult: + def __init__(self, components, job_id, file_meta): + self.components = components + self.job_id = job_id + self.file_meta = file_meta + + def to_dict(self): + try: + df = ResultFileHandling.make_csv_data(self.components, self.job_id, + self.file_meta.table_file_name) + csv_columns = list(df.columns) + + if self.file_meta.retrieve_lines == -1 or df.shape[0] <= self.file_meta.retrieve_lines: + csv_data = df.values.tolist() + else: + csv_data = df.iloc[:self.file_meta.retrieve_lines].values.tolist( + ) + return {'columns': csv_columns, 'data': csv_data} + except Exception as e: + pass + + +class FeatureProcessingResult: + DEFAULT_FEATURE_PROCESSING_FILES = { + "PRPreview": ResultFileMeta("xgb_result_column_info_selected.csv"), + "FEPreview": ResultFileMeta("woe_iv.csv", 5)} + + def __init__(self, components, job_id, file_infos): + self.components = components + self.job_id = job_id + self.file_infos = file_infos + self.result = dict() + self._fetch_result() + + def _fetch_result(self): + for property in self.file_infos.keys(): + table_info = TableResult(self.components, + self.job_id, self.file_infos[property]).to_dict() + self.result.update({property: table_info}) + + def to_dict(self): + return self.result + + +class XGBJobResult: + DEFAULT_PROPERTY_NAME = "outputModelResult" + MODEL_RESULT = "ModelResult" + MODEL_RESULT_PATH = "modelResultPath" + TRAIN_RESULT_PATH = "trainResultPath" + TEST_RESULT_PATH = "testResultPath" + WOE_RESULT_PATH = "woeIVResultPath" + + def __init__(self, job_id, components, property_name=DEFAULT_PROPERTY_NAME): + self.job_id = job_id + self.components = components + self.logger = components.logger() + self.property_name = property_name + self.model_result_list = None + self.job_result = None + self.model_result_path = None + self.train_result_path = None + self.woe_iv_result_path = None + self.xgb_result_path = None + self.evaluation_table = None + self.feature_importance_table = None + self.iteration_metrics = None + + def fetch_model_result(self): + self.model_result_list = [] + i = 0 + # while True: + while i < 6: + try: + tree_data = DataItem(data=ResultFileHandling.make_graph_data(self.components, + self.job_id, + utils.XGB_TREE_PERFIX + '_' + str(i) + '.svg'), + name='tree-' + str(i), name_property="ModelPlotName", data_property="ModelPlotData", + type=DataType.IMAGE) + self.model_result_list.append(tree_data.to_dict()) + i += 1 + except Exception: + break + + def load_result(self, result_path, result_property): + self.result_property = result_property + job_result_object = TableResult(self.components, + self.job_id, ResultFileMeta(result_path, 5)) + self.job_result = job_result_object.to_dict() + + def load_model_result_path(self, predict: bool): + self.xgb_result_path = dict() + self.model_result_path = ResultFileHandling.get_remote_path( + self.components, self.job_id, BaseContext.MODEL_DATA_FILE) + self.xgb_result_path.update( + {XGBJobResult.MODEL_RESULT_PATH: self.model_result_path}) + + self.train_result_path = ResultFileHandling.get_remote_path( + self.components, self.job_id, BaseContext.TRAIN_MODEL_OUTPUT_FILE) + self.xgb_result_path.update( + {XGBJobResult.TRAIN_RESULT_PATH: self.train_result_path}) + + self.xgb_result_path.update( + {XGBJobResult.TEST_RESULT_PATH: ResultFileHandling.get_remote_path( + self.components, self.job_id, BaseContext.TEST_MODEL_OUTPUT_FILE)}) + + self.woe_iv_result_path = ResultFileHandling.get_remote_path( + self.components, self.job_id, BaseContext.WOE_IV_FILE) + self.xgb_result_path.update( + {XGBJobResult.WOE_RESULT_PATH: self.woe_iv_result_path}) + + def load_evaluation_table(self, evaluation_path, property): + evaluation_table_object = TableResult(self.components, + self.job_id, ResultFileMeta(evaluation_path)) + self.evaluation_table = {property: DataItem(name=property, data=evaluation_table_object.to_dict(), + type=DataType.TABLE).to_dict()} + + def load_feature_importance_table(self, feature_importance_path, property): + feature_importance_table = TableResult(self.components, + self.job_id, ResultFileMeta(feature_importance_path)) + self.feature_importance_table = {property: DataItem(name=property, data=feature_importance_table.to_dict(), + type=DataType.TABLE).to_dict()} + + def load_iteration_metrics(self, iteration_path, property): + try: + iteration_metrics_data = DataItem(data=ResultFileHandling.make_graph_data(self.components, self.job_id, utils.METRICS_OVER_ITERATION_FILE), + name='iteration_metrics', name_property="ModelPlotName", data_property="ModelPlotData", + type=DataType.IMAGE) + self.iteration_metrics = [] + self.iteration_property = property + self.iteration_metrics.append(iteration_metrics_data.to_dict()) + except: + pass + + def to_dict(self): + result = dict() + if self.model_result_list is not None: + result.update({self.property_name: self.model_result_list}) + if self.job_result is not None: + result.update({self.result_property: self.job_result}) + if self.evaluation_table is not None: + result.update(self.evaluation_table) + if self.feature_importance_table is not None: + result.update(self.feature_importance_table) + if self.iteration_metrics is not None: + result.update({self.iteration_property: self.iteration_metrics}) + if self.xgb_result_path is not None: + result.update( + {XGBJobResult.MODEL_RESULT: self.xgb_result_path}) + return result + + +class TaskResultHandler: + def __init__(self, task_result_request: TaskResultRequest, components): + self.task_result_request = task_result_request + self.components = components + self.logger = components.logger() + self.result_list = [] + self.predict = False + if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name: + self.predict = True + self.logger.info( + f"Init jobResultHandler for: {self.task_result_request.job_id}") + self._get_evaluation_result() + self._get_feature_processing_result() + + def get_response(self): + merged_result = dict() + for result in self.result_list: + merged_result.update(result.to_dict()) + response = {"jobPlanetResult": merged_result} + return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response) + + def _get_evaluation_result(self): + if self.task_result_request.task_type == ModelTask.XGB_TRAINING.name: + # the train evaluation result + self.train_evaluation_result = JobEvaluationResult( + property_name="outputMetricsGraphs", + classification_type=ClassificationType.TWO, + job_id=self.task_result_request.job_id, + evaluation_files=JobEvaluationResult.DEFAULT_TRAIN_EVALUATION_FILES, + components=self.components) + # load the ks table + self.train_evaluation_result.load_ks_table( + "mpc_train_metric_ks.csv", "TrainKSTable") + self.result_list.append(self.train_evaluation_result) + + self.validation_evaluation_result = JobEvaluationResult( + property_name="outputTrainMetricsGraphs", + classification_type=ClassificationType.TWO, + job_id=self.task_result_request.job_id, + evaluation_files=JobEvaluationResult.DEFAULT_VALIDATION_EVALUATION_FILES, + components=self.components) + # load the ks_table + self.validation_evaluation_result.load_ks_table( + "mpc_metric_ks.csv", "KSTable") + self.result_list.append(self.validation_evaluation_result) + + self.xgb_model = XGBJobResult( + self.task_result_request.job_id, self.components, XGBJobResult.DEFAULT_PROPERTY_NAME) + self.xgb_model.fetch_model_result() + # the ks-auc table + self.xgb_model.load_evaluation_table( + utils.MPC_XGB_EVALUATION_TABLE, "EvaluationTable") + # the feature-importance table + self.xgb_model.load_feature_importance_table( + utils.XGB_FEATURE_IMPORTANCE_TABLE, "FeatureImportance") + self.result_list.append(self.xgb_model) + # the metrics iteration graph + self.xgb_model.load_iteration_metrics( + utils.METRICS_OVER_ITERATION_FILE, "IterationGraph") + + if self.predict: + # the train evaluation result + self.predict_evaluation_result = JobEvaluationResult( + property_name="outputMetricsGraphs", + classification_type=ClassificationType.TWO, + job_id=self.task_result_request.job_id, + evaluation_files=JobEvaluationResult.DEFAULT_EVAL_EVALUATION_FILES, + components=self.components) + # load ks_table + self.predict_evaluation_result.load_ks_table( + "mpc_eval_metric_ks.csv", "KSTable") + self.result_list.append(self.predict_evaluation_result) + + # load xgb_result + self.xgb_result = XGBJobResult( + self.task_result_request.job_id, self.components, XGBJobResult.DEFAULT_PROPERTY_NAME) + self.xgb_result.load_result( + "xgb_train_output.csv", "outputTrainPreview") + self.xgb_result.load_model_result_path(self.predict) + self.result_list.append(self.xgb_result) + + def _get_feature_processing_result(self): + self.feature_processing_result = FeatureProcessingResult( + self.components, self.task_result_request.job_id, FeatureProcessingResult.DEFAULT_FEATURE_PROCESSING_FILES) + self.result_list.append(self.feature_processing_result) diff --git a/python/ppc_model/network/__init__.py b/python/ppc_model/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/network/grpc/__init__.py b/python/ppc_model/network/grpc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/network/grpc/grpc_client.py b/python/ppc_model/network/grpc/grpc_client.py new file mode 100644 index 00000000..8169f522 --- /dev/null +++ b/python/ppc_model/network/grpc/grpc_client.py @@ -0,0 +1,82 @@ +import os +import time +import traceback + +import grpc + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse +from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceStub +from ppc_common.ppc_utils import utils +from ppc_model.common.protocol import RpcType +from ppc_model.interface.rpc_client import RpcClient + + +class GrpcClient(RpcClient): + rpc_type = RpcType.GRPC + + def __init__(self, logger, endpoint: str, grpc_options, ssl_switch: int = 0, + ca_path=None, ssl_key_path=None, ssl_crt_path=None): + self._logger = logger + self._endpoint = endpoint + self._ssl_switch = ssl_switch + self._grpc_options = grpc_options + self._ca_path = ca_path + self._ssl_key_path = ssl_key_path + self._ssl_crt_path = ssl_crt_path + if self._ssl_switch == 0: + insecure_channel = grpc.insecure_channel( + self._endpoint, options=grpc_options) + self._client = ModelServiceStub(insecure_channel) + else: + channel = self._create_secure_channel(self._endpoint) + self._client = ModelServiceStub(channel) + + def _create_secure_channel(self, target): + grpc_root_crt = utils.load_credential_from_file( + os.path.abspath(self._ca_path)) + grpc_ssl_key = utils.load_credential_from_file( + os.path.abspath(self._ssl_key_path)) + grpc_ssl_crt = utils.load_credential_from_file( + os.path.abspath(self._ssl_crt_path)) + credentials = grpc.ssl_channel_credentials( + root_certificates=grpc_root_crt, + private_key=grpc_ssl_key, + certificate_chain=grpc_ssl_crt + ) + return grpc.secure_channel(target, credentials, options=self._grpc_options) + + @staticmethod + def _build_error_model_response(message: str): + model_response = ModelResponse() + model_response.base_response.error_code = -1 + model_response.base_response.message = message + return model_response + + def send(self, request: ModelRequest): + start_time = time.time() + try: + self._logger.debug( + f"start sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}") + response = self._client.MessageInteraction(request) + end_time = time.time() + if response.base_response.error_code != 0: + self._logger.warn( + f"[OnWarn]send data to {request.receiver} failed, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " + f"ret_code: {response.base_response.error_code}, message: {response.base_response.message}, " + f"time_costs: {str(end_time - start_time)}s") + else: + self._logger.info( + f"finish sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " + f"ret_code: {response.base_response.error_code}, message: {response.base_response.message}, " + f"time_costs: {str(end_time - start_time)}s") + except Exception: + end_time = time.time() + message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \ + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \ + f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s" + self._logger.warn(message) + response = self._build_error_model_response(message) + return response diff --git a/python/ppc_model/network/grpc/grpc_server.py b/python/ppc_model/network/grpc/grpc_server.py new file mode 100644 index 00000000..389ba602 --- /dev/null +++ b/python/ppc_model/network/grpc/grpc_server.py @@ -0,0 +1,17 @@ +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse +from ppc_model.common.global_context import components + + +class ModelService(ppc_model_pb2_grpc.ModelServiceServicer): + + def MessageInteraction(self, model_request: ModelRequest, context): + components.logger().info( + f"receive a package, sender: {model_request.sender}, task_id: {model_request.task_id}, " + f"key: {model_request.key}, seq: {model_request.seq}, slice_num: {model_request.slice_num}") + + components.stub.on_message_received(model_request) + model_response = ModelResponse() + model_response.base_response.error_code = 0 + model_response.base_response.message = 'success' + return model_response diff --git a/python/ppc_model/network/http/__init__.py b/python/ppc_model/network/http/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/network/http/body_schema.py b/python/ppc_model/network/http/body_schema.py new file mode 100644 index 00000000..aacb7616 --- /dev/null +++ b/python/ppc_model/network/http/body_schema.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import json + +from flask_restx import fields, reqparse + +from ppc_model.network.http.restx import api + +response_base = api.model('Response base info', { + 'errorCode': fields.Integer(description='return code'), + 'message': fields.String(description='return message') +}) + +response_task_status = api.inherit('Task status', response_base, { + 'data': fields.Raw(description='Task status data as key-value dictionary', example={ + 'status': 'RUNNING', + 'traffic_volume': '10MB', + 'time_costs': '30s' + }) +}) diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py new file mode 100644 index 00000000..3ebbac2f --- /dev/null +++ b/python/ppc_model/network/http/model_controller.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +from flask import request +from flask_restx import Resource +import time + +from ppc_common.ppc_utils import utils +from ppc_model.common.global_context import components +from ppc_model.common.protocol import ModelTask +from ppc_model.network.http.body_schema import response_task_status, response_base +from ppc_model.network.http.restx import api +from ppc_model.model_result.task_result_handler import TaskResultHandler +from ppc_model.model_result.task_result_handler import TaskResultRequest + +ns = api.namespace('ppc-model/pml/run-model-task', + description='Operations related to run model task') +ns2 = api.namespace('ppc-model/pml/record-model-log', + description='Operations related to record model log') +ns_get_job_result = api.namespace( + 'ppc-model/pml/get-job-result', description='Get the job result') + + +@ns.route('/') +class ModelCollection(Resource): + + @api.response(201, 'Task started successfully.', response_base) + def post(self, model_id): + """ + Run a specific task by task_id. + """ + args = request.get_json() + task_id = model_id + components.logger().info(f"run task request, task_id: {task_id}, args: {args}") + task_type = args['task_type'] + components.task_manager.run_task( + task_id, ModelTask(task_type), (args,)) + return utils.BASE_RESPONSE + + @api.response(200, 'Task status retrieved successfully.', response_task_status) + def get(self, model_id): + """ + Get the status of a specific task by task_id. + """ + response = utils.BASE_RESPONSE + task_id = model_id + status, traffic_volume, time_costs = components.task_manager.status(task_id) + response['data'] = { + 'status': status, + 'traffic_volume': traffic_volume, + 'time_costs': time_costs + } + return response + + @api.response(200, 'Task killed successfully.', response_base) + def delete(self, model_id): + """ + Kill a specific task by job_id. + """ + job_id = model_id + components.logger().info(f"kill request, job_id: {job_id}") + components.task_manager.kill_task(job_id) + return utils.BASE_RESPONSE + + +@ns2.route('/') +class ModelLogCollection(Resource): + @api.response(200, 'Task status retrieved successfully.', response_task_status) + def get(self, job_id): + log_content = components.task_manager.record_model_job_log(job_id) + return utils.make_response(utils.PpcErrorCode.SUCCESS.get_code(), + utils.PpcErrorCode.SUCCESS.get_msg(), log_content) + + +@ns_get_job_result.route('/') +class ModelResultCollection(Resource): + @api.response(201, 'Get task result successfully.', response_base) + def post(self, task_id): + """ + Get the result related to the task_id + """ + start_t = time.time() + args = request.get_json() + components.logger().info( + f"run task request, task_id: {task_id}, args: {args}") + user_name = args['user'] + task_type = args['jobType'] + components.logger().info( + f"get_job_direct_result_response, job: {task_id}") + task_result_request = TaskResultRequest(task_id, task_type) + job_result_handler = TaskResultHandler( + task_result_request=task_result_request, components=components) + response = job_result_handler.get_response() + components.logger().info( + f"get_job_direct_result_response success, user: {user_name}, job: {task_id}, timecost: {time.time() - start_t}s") + return response diff --git a/python/ppc_model/network/http/restx.py b/python/ppc_model/network/http/restx.py new file mode 100644 index 00000000..83dba0ce --- /dev/null +++ b/python/ppc_model/network/http/restx.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from flask_restx import Api + +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.global_context import components + +authorizations = { + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + } +} + +api = Api(version='1.0', title='Ppc Model Service', + authorizations=authorizations, security='apikey') + + +@api.errorhandler(PpcException) +def default_error_handler(e): + components.logger().exception(e) + info = e.to_dict() + response = {'errorCode': info['code'], 'message': info['message']} + components.logger().error(f"OnError: code: {info['code']}, message: {info['message']}") + return response, 500 + + +@api.errorhandler(BaseException) +def default_error_handler(e): + components.logger().exception(e) + message = 'unknown error.' + response = {'errorCode': PpcErrorCode.INTERNAL_ERROR, 'message': message} + components.logger().error(f"OnError: unknown error") + return response, 500 diff --git a/python/ppc_model/network/stub.py b/python/ppc_model/network/stub.py new file mode 100644 index 00000000..7db22b1e --- /dev/null +++ b/python/ppc_model/network/stub.py @@ -0,0 +1,218 @@ +import os +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Dict, Union + +from readerwriterlock import rwlock + +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + + +@dataclass +class PushRequest: + receiver: str # 数据接收方的机构ID + task_id: str + key: str # 数据键 + data: bytes # 二进制数据 + slice_size_MB: int = 2 # 切片大小，默认为2MB + + def slice_data(self): + """将 data 按 slice_size 进行切片""" + if not self.data: + return [b''] + slice_size = self.slice_size_MB * 1024 * 1024 + return [self.data[i:i + slice_size] for i in + range(0, len(self.data), slice_size)] + + +@dataclass +class PullRequest: + sender: str # 数据发送方的机构ID + task_id: str + key: str # 数据键 + + +class ModelStub: + def __init__( + self, + agency_id: str, + thread_event_manager: ThreadEventManager, + rpc_client, + send_retry_times: int = 3, + retry_interval_s: Union[int, float] = 5 + ) -> None: + self.agency_id = agency_id + self._thread_event_manager = thread_event_manager + self._rpc_client = rpc_client + self._executor = ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)) + self._send_retry_times = send_retry_times + self._retry_interval_s = retry_interval_s + # 缓存收到的消息 [task_id:[sender:[key:[seq: data]]]] + # 缓存清理由TaskManager完成 + self._received_data: Dict[str, Dict[str, + Dict[str, Dict[int, tuple[int, bytes]]]]] = {} + self._received_uuid: Dict[str, set[str]] = {} + self._traffic_volume: Dict[str, int] = {} + self._data_rw_lock = rwlock.RWLockWrite() + + def push(self, request: PushRequest) -> bytes: + """ + 发送消息 + param request: 消息请求 + """ + slices = request.slice_data() + futures = [] + for seq, data in enumerate(slices): + model_request = ModelRequest( + sender=self.agency_id, + receiver=request.receiver, + task_id=request.task_id, + key=request.key, + seq=seq, + slice_num=len(slices), + data=data + ) + future = self._executor.submit( + self._send_with_retry, model_request) + futures.append(future) + + ret = bytearray() + for future in futures: + ret.extend(future.result()) + + self._accumulate_traffic_volume(request.task_id, len(request.data)) + + return bytes(ret) + + def pull(self, pull_request: PullRequest) -> bytes: + """ + 接收消息 + param request: 待收消息元信息 + return 消息 + """ + task_id = pull_request.task_id + sender = pull_request.sender + key = pull_request.key + while not self._thread_event_manager.event_status(task_id): + if self._is_all_data_ready(pull_request): + ret = bytearray() + with self._data_rw_lock.gen_rlock(): + slice_num = len(self._received_data[task_id][sender][key]) + for seq in range(slice_num): + ret.extend( + self._received_data[task_id][sender][key][seq][1]) + # 缓存中删除已获取到的数据 + with self._data_rw_lock.gen_wlock(): + del self._received_data[task_id][sender][key] + self._accumulate_traffic_volume(task_id, len(ret)) + return bytes(ret) + # 任务还在执行, 休眠后继续尝试获取数据 + time.sleep(0.04) + + # 接收到杀任务的信号 + raise PpcException(PpcErrorCode.TASK_IS_KILLED.get_code(), + PpcErrorCode.TASK_IS_KILLED.get_msg()) + + def traffic_volume(self, task_id) -> float: + with self._data_rw_lock.gen_rlock(): + if task_id not in self._traffic_volume: + return 0 + return self._traffic_volume[task_id] / 1024 / 1024 + + def on_message_received(self, model_request: ModelRequest): + """ + 注册给服务端的回调，服务端收到消息后调用 + param model_request: 收到的消息 + """ + # 消息幂等 + if not self._is_new_data(model_request): + return + # 缓存数据 + self._handle_received_data(model_request) + + def cleanup_cache(self, task_id): + with self._data_rw_lock.gen_wlock(): + if task_id in self._received_data: + del self._received_data[task_id] + if task_id in self._received_uuid: + del self._received_uuid[task_id] + if task_id in self._traffic_volume: + del self._traffic_volume[task_id] + + def _is_new_data(self, model_request: ModelRequest) -> bool: + # 返回是否需要继续处理消息 + task_id = model_request.task_id + uuid = f"{task_id}:{model_request.sender}:{model_request.key}:{model_request.seq}" + with self._data_rw_lock.gen_wlock(): + if task_id in self._received_uuid and uuid in self._received_uuid[task_id]: + # 收到重复的消息 + return False + elif task_id in self._received_uuid and uuid not in self._received_uuid[task_id]: + # 收到task_id的新消息 + self._received_uuid[task_id].add(uuid) + else: + # 首次收到task_id的消息 + self._received_uuid[task_id] = {uuid} + return True + + def _handle_received_data(self, model_request: ModelRequest): + task_id = model_request.task_id + sender = model_request.sender + key = model_request.key + seq = model_request.seq + slice_num = model_request.slice_num + data = model_request.data + with self._data_rw_lock.gen_wlock(): + if task_id not in self._received_data: + self._received_data[task_id] = { + model_request.sender: {key: {seq: (slice_num, data)}}} + elif sender not in self._received_data[task_id]: + self._received_data[task_id][sender] = { + key: {seq: (slice_num, data)}} + elif key not in self._received_data[task_id][sender]: + self._received_data[task_id][sender][key] = { + seq: (slice_num, data)} + else: + self._received_data[task_id][sender][key][seq] = ( + slice_num, data) + + def _is_all_data_ready(self, pull_request: PullRequest): + task_id = pull_request.task_id + sender = pull_request.sender + key = pull_request.key + with self._data_rw_lock.gen_rlock(): + if task_id not in self._received_data: + return False + if sender not in self._received_data[task_id]: + return False + if key not in self._received_data[task_id][sender]: + return False + if len(self._received_data[task_id][sender][key]) == 0: + return False + _, first_value = next( + iter(self._received_data[task_id][sender][key].items())) + if first_value[0] != len(self._received_data[task_id][sender][key]): + return False + return True + + def _send_with_retry(self, model_request: ModelRequest): + retry_times = 0 + while retry_times <= self._send_retry_times: + model_response = self._rpc_client.send(model_request) + if model_response.base_response.error_code == 0: + return model_response.data + if retry_times <= self._send_retry_times: + retry_times += 1 + time.sleep(self._retry_interval_s) + else: + raise PpcException(PpcErrorCode.NETWORK_ERROR.get_code( + ), model_response.base_response.message) + + def _accumulate_traffic_volume(self, task_id, length): + with self._data_rw_lock.gen_wlock(): + if task_id not in self._traffic_volume: + self._traffic_volume[task_id] = 0 + self._traffic_volume[task_id] += length diff --git a/python/ppc_model/network/test/__init__.py b/python/ppc_model/network/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/network/test/stub_unittest.py b/python/ppc_model/network/test/stub_unittest.py new file mode 100644 index 00000000..63e8e5c7 --- /dev/null +++ b/python/ppc_model/network/test/stub_unittest.py @@ -0,0 +1,83 @@ +import unittest + +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.network.stub import ModelStub, PushRequest, PullRequest + + +class TestStub(unittest.TestCase): + def setUp(self): + super().__init__() + self._agency_id = 'TEST_AGENCY' + self._message_type = 'TEST_MESSAGE' + self._rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._stub = ModelStub( + agency_id=self._agency_id, + thread_event_manager=self._thread_event_manager, + rpc_client=self._rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._rpc_client.set_message_handler(self._stub.on_message_received) + + def test_push_pull(self): + task_id = '0x12345678' + byte_array = bytearray(31 * 1024 * 1024) + bytes_data = bytes(byte_array) + self._stub.push(PushRequest( + receiver=self._agency_id, + task_id=task_id, + key=self._message_type, + data=bytes_data + )) + self._stub.push(PushRequest( + receiver=self._agency_id, + task_id=task_id, + key=self._message_type + 'other', + data=bytes_data + )) + received_data = self._stub.pull(PullRequest( + sender=self._agency_id, + task_id=task_id, + key=self._message_type + )) + other_data = self._stub.pull(PullRequest( + sender=self._agency_id, + task_id=task_id, + key=self._message_type + 'other', + )) + self.assertEqual(bytes_data, received_data) + self.assertEqual(bytes_data, other_data) + + def test_bad_client(self): + rpc_client = RpcClientMock(need_failed=True) + stub = ModelStub( + agency_id=self._agency_id, + thread_event_manager=self._thread_event_manager, + rpc_client=rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + rpc_client.set_message_handler(stub.on_message_received) + + task_id = '0x12345678' + byte_array = bytearray(3 * 1024 * 1024) + bytes_data = bytes(byte_array) + stub.push(PushRequest( + receiver=self._agency_id, + task_id=task_id, + key=self._message_type, + data=bytes_data + )) + + received_data = stub.pull(PullRequest( + sender=self._agency_id, + task_id=task_id, + key=self._message_type + )) + self.assertEqual(bytes_data, received_data) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py new file mode 100644 index 00000000..baef888b --- /dev/null +++ b/python/ppc_model/ppc_model_app.py @@ -0,0 +1,112 @@ +# Note: here can't be refactored by autopep +import sys +sys.path.append("../") + +import multiprocessing +import os +from concurrent import futures +from threading import Thread + +import grpc +from cheroot.ssl.builtin import BuiltinSSLAdapter +from cheroot.wsgi import Server as WSGIServer +from flask import Flask, Blueprint +from paste.translogger import TransLogger + +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_utils import utils +from ppc_model.common.global_context import components +from ppc_model.common.protocol import ModelTask +from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine +from ppc_model.network.grpc.grpc_server import ModelService +from ppc_model.network.http.model_controller import ns as task_namespace +from ppc_model.network.http.model_controller import ns2 as log_namespace +from ppc_model.network.http.restx import api +from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine +from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine +from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine + +app = Flask(__name__) + + +def initialize_app(app): + # 初始化应用功能组件 + components.init_all() + + app.config.update(components.config_data) + blueprint = Blueprint('api', __name__, url_prefix='/api') + api.init_app(blueprint) + api.add_namespace(task_namespace) + api.add_namespace(log_namespace) + app.register_blueprint(blueprint) + + +def register_task_handler(): + task_manager = components.task_manager + task_manager.register_task_handler( + ModelTask.PREPROCESSING, PreprocessingEngine.run) + task_manager.register_task_handler( + ModelTask.FEATURE_ENGINEERING, FeatureEngineeringEngine.run) + task_manager.register_task_handler( + ModelTask.XGB_TRAINING, SecureLGBMTrainingEngine.run) + task_manager.register_task_handler( + ModelTask.XGB_PREDICTING, SecureLGBMPredictionEngine.run) + + +def model_serve(): + if app.config['SSL_SWITCH'] == 0: + ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), + options=components.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), ppc_serve) + address = "[::]:{}".format(app.config['RPC_PORT']) + ppc_serve.add_insecure_port(address) + else: + grpc_root_crt = utils.load_credential_from_file( + os.path.abspath(app.config['SSL_CA'])) + grpc_ssl_key = utils.load_credential_from_file( + os.path.abspath(app.config['SSL_KEY'])) + grpc_ssl_crt = utils.load_credential_from_file( + os.path.abspath(app.config['SSL_CRT'])) + server_credentials = grpc.ssl_server_credentials((( + grpc_ssl_key, + grpc_ssl_crt, + ),), grpc_root_crt, True) + + ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), + options=components.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), ppc_serve) + address = "[::]:{}".format(app.config['RPC_PORT']) + ppc_serve.add_secure_port(address, server_credentials) + + ppc_serve.start() + components.logger().info( + f"Starting model grpc server at ://{app.config['HOST']}:{app.config['RPC_PORT']}") + ppc_serve.wait_for_termination() + + +if __name__ == '__main__': + initialize_app(app) + register_task_handler() + + # 启动子进程不继承父进程的锁状态，防止死锁 + multiprocessing.set_start_method('spawn') + + Thread(target=model_serve).start() + + app.config['SECRET_KEY'] = os.urandom(24) + server = WSGIServer((app.config['HOST'], app.config['HTTP_PORT']), + TransLogger(app, setup_console_handler=False), numthreads=2) + + ssl_switch = app.config['SSL_SWITCH'] + protocol = 'http' + if ssl_switch == 1: + protocol = 'https' + server.ssl_adapter = BuiltinSSLAdapter( + certificate=app.config['SSL_CRT'], + private_key=app.config['SSL_KEY'], + certificate_chain=app.config['CA_CRT']) + + message = f"Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']}" + print(message) + components.logger().info(message) + server.start() diff --git a/python/ppc_model/preprocessing/__init__.py b/python/ppc_model/preprocessing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/preprocessing/local_processing/local_processing_party.py b/python/ppc_model/preprocessing/local_processing/local_processing_party.py new file mode 100644 index 00000000..6cc13675 --- /dev/null +++ b/python/ppc_model/preprocessing/local_processing/local_processing_party.py @@ -0,0 +1,100 @@ +import os +import time +from abc import ABC + +import pandas as pd + +from ppc_common.ppc_utils import utils +from ppc_model.preprocessing.local_processing.preprocessing import process_dataframe +from ppc_model.preprocessing.processing_context import ProcessingContext + + +class LocalProcessingParty(ABC): + + def __init__(self, ctx: ProcessingContext): + self.ctx = ctx + + def processing(self): + log = self.ctx.components.logger() + start = time.time() + need_psi = self.ctx.need_run_psi + job_id = self.ctx.job_id + log.info( + f"run data preprocessing job, job: {job_id}, need_psi: {need_psi}") + dataset_path = self.ctx.dataset_path + dataset_file_path = self.ctx.dataset_file_path + storage_client = self.ctx.components.storage_client + job_algorithm_type = self.ctx.job_algorithm_type + if job_algorithm_type == utils.AlgorithmType.Predict.name: + storage_client.download_file(os.path.join(self.ctx.training_job_id, self.ctx.PREPROCESSING_RESULT_FILE), + self.ctx.preprocessing_result_file) + psi_result_path = self.ctx.psi_result_path + model_prepare_file = self.ctx.model_prepare_file + storage_client.download_file(dataset_path, dataset_file_path) + if need_psi and (not utils.file_exists(psi_result_path)): + storage_client.download_file( + self.ctx.remote_psi_result_path, psi_result_path) + self.handle_local_psi_result(psi_result_path) + log.info( + f"prepare_xgb_after_psi, make_dataset_to_xgb_data_plus_psi_data, dataset_file_path={dataset_file_path}, " + f"psi_result_path={dataset_file_path}, model_prepare_file={model_prepare_file}") + self.make_dataset_to_xgb_data() + storage_client.upload_file( + model_prepare_file, job_id + os.sep + self.ctx.model_prepare_file) + log.info(f"upload model_prepare_file to hdfs, job_id={job_id}") + if job_algorithm_type == utils.AlgorithmType.Train.name: + log.info(f"upload column_info to hdfs, job_id={job_id}") + storage_client.upload_file(self.ctx.preprocessing_result_file, + job_id + os.sep + self.ctx.PREPROCESSING_RESULT_FILE) + log.info( + f"call prepare_xgb_after_psi success, job_id={job_id}, timecost: {time.time() - start}") + + def handle_local_psi_result(self, local_psi_result_path): + try: + log = self.ctx.components.logger() + log.info( + f"handle_local_psi_result: start handle_local_psi_result, psi_result_path={local_psi_result_path}") + with open(local_psi_result_path, 'r+', encoding='utf-8') as psi_result_file: + content = psi_result_file.read() + psi_result_file.seek(0, 0) + psi_result_file.write('id\n' + content) + log.info( + f"handle_local_psi_result: call handle_local_psi_result success, psi_result_path={local_psi_result_path}") + except BaseException as e: + log.exception( + f"handle_local_psi_result: handle_local_psi_result, psi_result_path={local_psi_result_path}, error:{e}") + raise e + + def make_dataset_to_xgb_data(self): + log = self.ctx.components.logger() + dataset_file_path = self.ctx.dataset_file_path + psi_result_file_path = self.ctx.psi_result_path + model_prepare_file = self.ctx.model_prepare_file + log.info(f"dataset_file_path:{dataset_file_path}") + log.info(f"model_prepare_file:{model_prepare_file}") + need_run_psi = self.ctx.need_run_psi + job_id = self.ctx.job_id + if not utils.file_exists(dataset_file_path): + raise FileNotFoundError( + f"dataset_file_path not found: {dataset_file_path}") + dataset_df = pd.read_csv(dataset_file_path) + if need_run_psi: + log.info(f"psi_result_file_path:{psi_result_file_path}") + psi_data = pd.read_csv(psi_result_file_path, + delimiter=utils.CSV_SEP) + dataset_df = pd.merge(dataset_df, psi_data, on=[ + 'id']).sort_values(by='id', ascending=True) + + ppc_job_type = self.ctx.job_algorithm_type + column_info = process_dataframe( + dataset_df, self.ctx.model_setting, model_prepare_file, ppc_job_type, job_id, self.ctx) + + column_info_pd = pd.DataFrame(column_info).transpose() + # 如果是训练任务先写本地 + log.info(f"jobid {job_id}, job_algorithm_type {ppc_job_type}") + if ppc_job_type == utils.AlgorithmType.Train.name: + log.info( + f"write {column_info} to {self.ctx.preprocessing_result_file}") + column_info_pd.to_csv( + self.ctx.preprocessing_result_file, sep=utils.CSV_SEP, header=True) + log.info("finish make_dataset_to_xgb_data_plus_psi_data") diff --git a/python/ppc_model/preprocessing/local_processing/preprocessing.py b/python/ppc_model/preprocessing/local_processing/preprocessing.py new file mode 100644 index 00000000..55082a62 --- /dev/null +++ b/python/ppc_model/preprocessing/local_processing/preprocessing.py @@ -0,0 +1,660 @@ +# -*- coding: utf-8 -*- +# from concurrent.futures import ProcessPoolExecutor, as_completed + +import json +import os + +import numpy as np +import pandas as pd +from sklearn.preprocessing import MinMaxScaler, StandardScaler +from ppc_model.common.model_setting import ModelSetting + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.global_context import components +from ppc_model.preprocessing.local_processing.psi_select import calculate_psi +from ppc_model.preprocessing.local_processing.standard_type_enum import standardType +# from ppc_common.ppc_mock.mock_objects import MockLogger + +# components.mock_logger = MockLogger() +# log = components.mock_logger +log = components.logger() + + +def process_train_dataframe(dataset_df: pd.DataFrame, column_info_dict: dict): + """ + 使用column_info对dataset_df进行处理 只保留column_info中的列 + + 参数: + - dataset_df: 待处理的DataFrame数据 + - column_info: 字段信息, 字典类型 + + 返回值: + - dataset_df_filled: 处理后的DataFrame数据 + """ + # dataset_df_filled = None + # Iterate over column_info_dict keys, if 'isExisted' is False, drop the column + for key, value in column_info_dict.items(): + if value.get('isExisted') is None: + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "column_info_dict isExisted is None") + if value.get('isExisted') is False: + dataset_df = dataset_df.drop(key, axis=1) + + return dataset_df + + +def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb_data_file_path: str, + ppc_job_type: str = utils.AlgorithmType.Train.name, job_id: str = None, ctx=None): + """ + 对数据集进行预处理的函数。 + 共执行6步操作 + 1. 去除唯一属性: 为了让无意义的参数不影响模型, 比如id, id不代表样本本身的规律, 所以要把这些属性删掉 + 2. 缺失值处理: 删除含有缺失值的特征,若变量的缺失率较高(大于80%)，覆盖率较低，且重要性较低，可以直接将变量删除. + 每一行的数据不一定拥有所有的模型标签，支持不填充，和均值插补, + 我们目前使用均值插补，以该属性存在值的平均值来插补缺失的值 + 连续性：普通均值 + 类别：单独类别，当做新类别 + 3. 离群值处理方法: 数据过大或过小会一个峡谷分析结果，要先调整因子值的离群值上下限，减少离群值的影响，防止下一步归一化后的偏差过大 + 我们用的3 \sigma 法，又叫做标准差法 + 4. 特征编码: 将特征编码为固定的值 比如one-hot就是将不同的type 编码为1 2 3 4 5...这样的值 + 5. 数据标准化: min-max标准化(归一化):最大值1. 最小值0或-1 以及z-score标准化 规范化:均值为0 标准差为1 + 6. 特征选择: 从给定的特征集合中选出相关特征子集的过程称为特征选择, 我们使用的是PSI 风控模型，群体稳定性指标 PSI-Population Stability Index + 参考 https://zhuanlan.zhihu.com/p/79682292 + + + 参数： + dataset_df (pandas.DataFrame): 输入的数据集。 + xgb_data_file_path (str): XGBoost数据文件路径。 + + 返回： + column_info: 处理后的数据集字段。 + """ + log.info( + f"jobid: {job_id}, xgb_data_file_path:{xgb_data_file_path}, ppc_job_type: {ppc_job_type}") + if model_setting is None: + log.error("model_setting is None") + raise PpcException( + PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), "model_setting is None") + + column_info = {} + + if ppc_job_type == utils.AlgorithmType.Predict.name: + column_info_fm = pd.read_csv( + ctx.preprocessing_result_file, index_col=0) + column_info_train_str = json.dumps( + column_info_fm.to_dict(orient='index')) + if column_info_train_str is None: + raise PpcException(-1, "column_info_train is None") + try: + # 对应orient='records' + # column_info_train = json.loads(column_info_train_str, orient='records') + column_info_train = json.loads(column_info_train_str) + except Exception as e: + log.error( + f"jobid: {job_id} column_info_train json.loads error, e:{e}") + raise PpcException(-1, "column_info_train json.loads error") + dataset_df = process_train_dataframe(dataset_df, column_info_train) + column_info = column_info_train + elif ppc_job_type == utils.AlgorithmType.Train.name: + # 如果是训练任务 先默认所有数据都存在 + column_info = {col: {'isExisted': True} for col in dataset_df.columns} + + if model_setting.eval_set_column is not None: + if model_setting.eval_set_column in dataset_df.columns: + eval_column = model_setting.eval_set_column + dataset_df[['id', eval_column]].to_csv(ctx.eval_column_file, index=None) + ctx.components.storage_client.upload_file(ctx.eval_column_file, job_id + os.sep + ctx.EVAL_COLUMN_FILE) + if model_setting.eval_set_column != model_setting.psi_select_col: + dataset_df = dataset_df.drop(columns=[eval_column]) + + categorical_cols = ['id', 'y'] + + # 判断 model_setting['categorical']是否为None + if model_setting.categorical is None: + log.error( + f"jobid: {job_id} model_setting['categorical'] is None, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "xgb_model_dict['categorical'] is None") + + if model_setting.fillna is None: + log.error( + f"jobid: {job_id} xgb_model_dict['fillna'] is None, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "xgb_model_dict['fillna'] is None") + + # 指定分类特征索引 + if model_setting.categorical != '0': + categoricals = model_setting.categorical.split(',') + categorical_cols.extend(categoricals) + # 去除categorical_cols中的重复元素 + categorical_cols = list(set(categorical_cols)) + + df_filled = dataset_df + # 预处理表格信息 包含每一列缺失值比例 缺失值是否被筛掉 psi筛选和相关性筛选 如果在某个阶段被筛掉则设置为0 保留则为1 + # 1.去除唯一属性 + if 'id' in df_filled.columns: + log.info(f"jobid: {job_id} move id column start.") + df_filled = df_filled.drop('id', axis=1) + log.info(f"jobid: {job_id} move id column finish.") + + # 2.1 缺失值筛选 + if ppc_job_type == utils.AlgorithmType.Train.name: + if 0 <= model_setting.na_select <= 1: + log.info(f"jobid: {job_id} run fillna start") + df_filled, column_info = process_na_dataframe( + df_filled, model_setting.na_select) + log.info(f"jobid: {job_id} run fillna finish") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['na_select'] is range not 0 to 1, xgb_model_dict:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['na_select'] range not 0 to 1") + elif ppc_job_type == utils.AlgorithmType.Predict.name: + log.info(f"jobid: {job_id} don't need run fillna for predict job.") + else: + log.error( + f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "ppc_job_type is not Train or Predict") + # 2.2 缺失值处理 + if model_setting.fillna == 1: + # 填充 + log.info(f"jobid: {job_id} run fillna with means start") + try: + df_filled = process_na_fill_dataframe( + df_filled, categorical_cols, model_setting.psi_select_col) + log.info(f"jobid: {job_id} run fillna with means finish") + except Exception as e: + log.error( + f"jobid: {job_id} process_na_fill_dataframe error, e:{e}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "process_na_fill_dataframe error") + elif model_setting.fillna == 0: + # 不填充 + log.info(f"jobid: {job_id} don't need run fillna ") + # 如果本身是None就不需要处理 + df_filled.replace('None', np.nan) + else: + log.error( + f"jobid: {job_id} xgb_model_dict['fillna'] is not 0 or 1, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "xgb_model_dict['fillna'] is not 0 or 1") + + # 6.1 特征选择 进行 psi稳定性指标筛选 计算特征相关性 降维可以减少模型的复杂度，提高模型的泛化能力 + if ppc_job_type == utils.AlgorithmType.Train.name: + if model_setting.psi_select_col in df_filled.columns.tolist() and model_setting.psi_select_col != 0: + log.info(f"jobid: {job_id} run psi_select_col start") + psi_select_base = model_setting.psi_select_base + psi_select_thresh = model_setting.psi_select_thresh + psi_select_bins = model_setting.psi_select_bins + psi_select_col = model_setting.psi_select_col + + if psi_select_base is None or psi_select_thresh is None or psi_select_bins is None: + log.error( + f"jobid: {job_id} psi_select_base or psi_select_thresh or psi_select_bins is None, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "psi_select_base or psi_select_thresh or psi_select_bins is None") + df_filled, psi_selected_cols = process_psi(df_filled, categorical_cols, psi_select_col, psi_select_base, + psi_select_thresh, psi_select_bins) + # 使用column_info和psi_selected_cols 追加psi_selected列 如果列还在则选中是1 否则是0 + for col in column_info.keys(): + if col in psi_selected_cols: + column_info[col]['psi_selected'] = 1 + column_info[col]['isExisted'] = True + else: + column_info[col]['psi_selected'] = 0 + column_info[col]['isExisted'] = False + log.info(f"jobid: {job_id} run psi_select_col finish") + elif model_setting.psi_select_col == 0 or model_setting.psi_select_col == "": + log.info(f"jobid: {job_id} don't need run psi_select_col") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['psi_select_col'] is not 0 or in col, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['psi_select_col'] is not 0 or in col") + elif ppc_job_type == utils.AlgorithmType.Predict.name: + log.info( + f"jobid: {job_id} don't need run psi_select_col for predict job.") + else: + log.error( + f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "ppc_job_type is not Train or Predict") + + # 6.2 特征选择 进行 corr_select 计算特征相关性 + if ppc_job_type == utils.AlgorithmType.Train.name: + if model_setting.corr_select > 0: + log.info(f"jobid: {job_id} run corr_select start") + corr_select = model_setting.corr_select + df_filled = remove_high_correlation_features( + df_filled, categorical_cols, corr_select) + # 设置相关性筛选的column_info + for col in column_info.keys(): + if col in df_filled.columns.tolist(): + column_info[col]['corr_selected'] = 1 + column_info[col]['isExisted'] = True + else: + column_info[col]['corr_selected'] = 0 + column_info[col]['isExisted'] = False + log.info(f"jobid: {job_id} run corr_select finish") + elif model_setting.corr_select == 0: + log.info(f"jobid: {job_id} don't need run corr_select") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['corr_select'] is not >= 0, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['corr_select'] is not >= 0") + elif ppc_job_type == utils.AlgorithmType.Predict.name: + log.info( + f"jobid: {job_id} don't need run corr_select for predict job.") + else: + log.error( + f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "ppc_job_type is not Train or Predict") + + # 3. 离群值处理 3-sigma 法 + if model_setting.filloutlier == 1: + log.info(f"jobid: {job_id} run filloutlier start") + df_filled = process_outliers(df_filled, categorical_cols) + log.info(f"jobid: {job_id} run filloutlier finish") + elif model_setting.filloutlier == 0: + log.info(f"jobid: {job_id} don't need run filloutlier") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['filloutlier'] is not 0 or 1, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['filloutlier'] is not 0 or 1") + + # 5.1 数据标准化 支持max-min normalized + if model_setting.normalized == 1: + log.info(f"jobid: {job_id} run normalized start") + df_filled = normalize_dataframe( + df_filled, categorical_cols, standardType.min_max.value) + log.info(f"jobid: {job_id} run normalized finish") + elif model_setting.normalized == 0: + log.info(f"jobid: {job_id} don't need run normalized") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['normalized'] is not 0 or 1, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['normalized'] is not 0 or 1") + + # 5.2 z-score标准化（规范化） standardized + if model_setting.standardized == 1: + log.info(f"jobid: {job_id} run standardized start") + df_filled = normalize_dataframe( + df_filled, categorical_cols, standardType.z_score.value) + log.info(f"jobid: {job_id} run standardized finish") + elif model_setting.standardized == 0: + log.info(f"jobid: {job_id} don't need run standardized") + else: + log.error( + f"jobid: {job_id} xgb_model_dict['standardized'] is not 0 or 1, model_setting:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "xgb_model_dict['standardized'] is not 0 or 1") + + # 4. 特征编码，对分类特征进行one-hot编码 这里会多一些列 所以放到最后 + if model_setting.one_hot == 1: + log.info(f"jobid: {job_id} run one_hot start") + df_filled = one_hot_encode_and_merge(df_filled, categorical_cols) + log.info(f"jobid: {job_id} run one_hot finish") + elif model_setting.one_hot == 0: + log.info(f"jobid: {job_id} don't need run one_hot") + else: + log.error( + f"jobid: {job_id} model_setting['one_hot'] is not 0 or 1, xgb_model_dict:{model_setting}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "model_setting['one_hot'] is not 0 or 1") + df_filled.to_csv(xgb_data_file_path, mode='w', + sep=utils.BLANK_SEP, header=True, index=None) + # log.info(f"jobid: {job_id} column_info:{column_info} in type: {ppc_job_type}, process_dataframe succeed") + log.info( + f"jobid: {job_id} in type: {ppc_job_type}, process_dataframe succeed") + return column_info + + +def process_na_dataframe(df: pd.DataFrame, na_select: float): + """ + 缺失值处理 如果小于阈值则移除该列 + + 参数 + - df: 待处理的DataFrame数据 + - na_select: 缺失值占比阈值 + + 返回值： + 处理后的DataFrame数据 + """ + missing_ratios = df.isnull().mean() + + # 剔除缺失值占比大于na_select的列 + selected_cols = missing_ratios[missing_ratios <= na_select].index + + column_info = {col: {'missing_ratio': missing_ratios[col], 'na_selected': 1 if col in selected_cols else 0, } for + col in df.columns} + # 如果没被选择 将isExisted设置为false + for col in column_info.keys(): + if col not in selected_cols: + column_info[col]['isExisted'] = False + else: + column_info[col]['isExisted'] = True + df_processed = df[selected_cols] + return df_processed, column_info + + +def process_na_fill_dataframe(df: pd.DataFrame, categorical_cols: list = None, psi_select_col: str = None): + """ + 处理DataFrame数据: + 1. 计算每一列的缺失值占比； + 2. 剔除缺失值占比大于阈值的列； + 3. 使用每列的均值填充剩余的缺失值。 + + 参数： + - df: 待处理的DataFrame数据 + - na_select: 缺失值占比阈值 + - categorical_cols: 分类特征列 + + + 返回值： + 处理后的DataFrame数据 + """ + # 计算每一列的缺失值占比 + + # 判断categorical_cols是否在df中，如果是，填充则用col的max+1, 否则用均值插补 + df_processed = df.copy() # Assign the sliced DataFrame to a new variable + for col in df_processed.columns.to_list(): + # 如果col是y，则忽略 + if col == 'y': + continue + elif psi_select_col and col == psi_select_col: + continue + if col in categorical_cols: + df_processed.fillna( + {col: df_processed[col].max() + 1}, inplace=True) + else: + df_processed.fillna({col: df_processed[col].mean()}, inplace=True) + return df_processed + + +def process_outliers(df: pd.DataFrame, categorical_cols: list): + """ + 处理DataFrame数据中的异常值 + 1. 计算每一列的均值和标准差； + 2. 对于超出均值+-3倍标准差的数据 使用均值填充。 + + 参数： + - df: 待处理的DataFrame数据 + - categorical_cols: 列表，包含分类列的名称 + + 返回值： + 处理后的DataFrame数据 + """ + # 计算每一列的均值和标准差 + means = df.mean() + threshold = 3 * df.std() + + # 定义处理异常值的函数 + def replace_outliers(col): + # 如果列是分类列，不处理 + if col.name in categorical_cols: + return col + else: + lower_bound = means[col.name] - threshold[col.name] + upper_bound = means[col.name] + threshold[col.name] + # 如果元素是空值或不是异常值，保持不变；否则使用均值填充 + return np.where(col.notna() & ((col < lower_bound) | (col > upper_bound)), means[col.name], col) + + # 应用处理异常值的函数到DataFrame的每一列 + df_processed = df.apply(replace_outliers) + return df_processed + + +def one_hot_encode_and_merge(df_filled: pd.DataFrame, categorical_features: list): + """ + 对DataFrame中指定的分类特征列进行One-Hot编码 并将编码后的结果合并到DataFrame中。 + + 参数： + - df: 待处理的DataFrame数据 + - categorical_features: 分类特征列表 需要进行One-Hot编码的列名 + + 返回值： + 处理后的DataFrame数据 + """ + categorical_cols_without_y_and_id = categorical_features.copy() + if 'id' in categorical_cols_without_y_and_id: + categorical_cols_without_y_and_id.remove('id') + if 'y' in categorical_cols_without_y_and_id: + categorical_cols_without_y_and_id.remove('y') + # 如果categorical_cols_without_y_and_id不在df_filled的col name中 报错 + if not set(categorical_cols_without_y_and_id).issubset(set(df_filled.columns)): + log.error( + f"categorical_cols_without_y_and_id is not in df_filled columns, categorical_cols_without_y_and_id:{categorical_cols_without_y_and_id}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), + "categorical_cols_without_y_and_id is not in df_filled columns") + df_merged = pd.get_dummies( + df_filled, columns=categorical_cols_without_y_and_id) + + return df_merged + + +def normalize_dataframe(df: pd.DataFrame, categorical_cols: list, type: standardType = standardType.min_max.value): + """ + 对DataFrame进行标准化，排除指定的分类特征 + + 参数： + - df: 待标准化的DataFrame数据 + - categorical_cols: 列表，包含不需要标准化的分类特征的名称 + + 返回值： + 标准化后的DataFrame数据 + """ + # 创建MinMaxScaler对象 + if type == standardType.min_max.value: + scaler = MinMaxScaler() + elif type == standardType.z_score.value: + scaler = StandardScaler() + else: + log.error(f"unspport type in normalize_dataframe type:{type}") + raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code( + ), "unspport type in normalize_dataframe type") + + # 获取数值变量的列名 + numeric_cols = df.select_dtypes(include=['int', 'float']).columns + + # 排除指定的分类特征 + numeric_cols = [col for col in numeric_cols if col not in categorical_cols] + + # 对数值变量进行标准化 + df_normalized = df.copy() + df_normalized[numeric_cols] = scaler.fit_transform( + df_normalized[numeric_cols]) + return df_normalized + +# def calculate_correlation(df_i_col_name, df_j_col_name, i, j, col_i, col_j, categorical_cols): +# if df_i_col_name in categorical_cols or df_j_col_name in categorical_cols: +# return None # 返回NaN表示忽略这个计算 +# common_index = col_i.index.intersection(col_j.index) +# return i,j,np.corrcoef(col_i.loc[common_index], col_j.loc[common_index])[0, 1] + + +def remove_high_correlation_features(df: pd.DataFrame, categorical_cols: list, corr_select: float): + """ + 删除DataFrame中相关系数大于corr_select的特征中的一个。 + + 参数： + - df: 待处理的DataFrame数据 + - categorical_cols: 列表，包含分类特征的名称 + - corr_select: 相关系数阈值 + + 返回值： + 处理后的DataFrame数据 + """ + # ===========原有逻辑============== + # 计算特征之间的相关系数矩阵 + # num_features = df.shape[1] + # correlation_matrix = np.zeros((num_features, num_features)) + + # for i in range(num_features): + # for j in range(i+1, num_features): + # # if i == j: + # # continue + # # 忽略categorical_cols的列 + # if df.columns[i] in categorical_cols or df.columns[j] in categorical_cols: + # continue + # # 当有缺失值时 去除该行对应的缺失值进行比较, 比如col1有缺失值, col2没有, 那么col1的缺失值对应的位置，和col2一起去掉，共同参与计算 + # col_i = df.iloc[:, i].dropna() + # col_j = df.iloc[:, j].dropna() + # common_index = col_i.index.intersection(col_j.index) + # correlation_matrix[i, j] = np.corrcoef(col_i.loc[common_index], col_j.loc[common_index])[0, 1] + # # print(f"correlation_matrix: {correlation_matrix}") + # # correlation_matrix[i, j] = np.corrcoef(df.iloc[:, i], df.iloc[:, j])[0, 1] + + # # 获取相关系数大于corr_select的特征对, 获取列名 + # high_correlation = np.argwhere(np.abs(correlation_matrix) > corr_select) + + # high_correlation_col_name = [] + # for i, j in high_correlation: + # high_correlation_col_name.append((df.columns[i], df.columns[j])) + # # 删除相关性大于corr_select的特征中的一个 + # for col_left, col_right in high_correlation_col_name: + # try: + # # 如果col_left和col_right都在df中, 删除右边, 否则不动 + # if col_left in df.columns and col_right in df.columns: + # df.drop(col_right, axis=1, inplace=True) + # # df.drop(df.columns[j], axis=1, inplace=True) + # except: + # log.warning( + # f"remove_high_correlation_features error, i:{i}, j:{j}, df[col_left]:{df[col_left]}, df[col_right]:{df[col_right]}") + # pass + # return df + # ===========原有逻辑============== + num_features = df.shape[1] + correlation_matrix = np.zeros((num_features, num_features)) + + df_copy = df.copy() + for col_name in categorical_cols: + if col_name in df_copy.columns: + df_copy.drop(col_name, axis=1, inplace=True) + correlation_matrix = df_copy.corr() + # ===========尝试已提交多线程============== + # 提前设置好需要的列 + # col_list = {} + # for i in range(num_features): + # for j in range(i+1, num_features): + # col_list[i,j]=df.iloc[:, i].dropna().copy(), df.iloc[:, j].dropna().copy() + # print(correlation_matrix) + # for i in range(num_features): + # for j in range(i+1, num_features): + # result = calculate_correlation(df.columns[i], df.columns[j], i, j, df.iloc[:, i], df.iloc[:, j], categorical_cols) + # if result != None: + # i,j, correlation_matrix_i_j = result + # correlation_matrix[i,j] = correlation_matrix_i_j + # with ProcessPoolExecutor() as executor: + # col_i, col_j = col_list[i,j] + # results = [executor.submit(calculate_correlation, df.columns[i], df.columns[j], i, j, col_i, col_j, categorical_cols) for j in range(i + 1, num_features)] + # for future in as_completed(results): + # result = future.result() + # # print(f"result: {result}, {type(result)}") + # if result != None: + # i,j, correlation_matrix_i_j = result + # correlation_matrix[i,j] = correlation_matrix_i_j + # print(f"correlation_matrix: {correlation_matrix}") + # ===========尝试已提交多线程============== + high_correlation = np.argwhere(np.abs(correlation_matrix) > corr_select) + + high_correlation_col_name = [] + for i, j in high_correlation: + if i >= j: + continue + high_correlation_col_name.append( + (df_copy.columns[i], df_copy.columns[j])) + # 删除相关性大于corr_select的特征中的一个 + for col_left, col_right in high_correlation_col_name: + try: + # 如果col_left和col_right都在df中, 删除右边, 否则不动 + if col_left in df.columns and col_right in df.columns: + df.drop(col_right, axis=1, inplace=True) + # df.drop(df.columns[j], axis=1, inplace=True) + except: + log.warn( + f"remove_high_correlation_features error, i:{i}, j:{j}, df[col_left]:{df[col_left]}, df[col_right]:{df[col_right]}") + pass + return df + + +def process_psi(df_filled: pd.DataFrame, categorical_cols: list, psi_select_col: str, psi_select_base: int, + psi_select_thresh: float, psi_select_bins: int): + """ + Preprocesses the data by calculating the Population Stability Index (PSI) for a given column. + + Args: + df_filled (pd.DataFrame): The input DataFrame with missing values filled. + categorical_cols (list): A list of column names that are categorical variables. + psi_select_col (str): The name of the column for which PSI is calculated. + psi_select_base (int): The base period for calculating PSI. + psi_select_thresh (float): The threshold value for PSI. Columns with PSI above this value will be selected. + psi_select_bins (int): The number of bins to use for calculating PSI. + + Returns: + df_filled (pd.DataFrame): The input DataFrame with the PSI values calculated. + """ + # TODO: 需要验证计算psi的正确性 当为空值时 公式中没有讲如何处理 这里先按照github上的代码处理 + + # 最终所有保留的列 + psi_selected_cols = [] + + for col_select in df_filled.columns: + if col_select == psi_select_col: + # 如果是psi选择的列 则一定不保留 + continue + elif col_select in categorical_cols: + # 如果是特征列 则一定保留 + psi_selected_cols.append(col_select) + else: + # 如果是数值列 则计算psi 判断是否保留 + # 先提出base和select的数据 + psi_select_col_base_value = df_filled[col_select][df_filled[psi_select_col] + == psi_select_base].values + max_psi = 0 + for col_base in set(df_filled[psi_select_col]): + if col_base == psi_select_base: + continue + else: + col_value = df_filled[col_select][df_filled[psi_select_col] + == col_base].values + + col_psi = calculate_psi(psi_select_col_base_value, col_value, buckettype='quantiles', + buckets=psi_select_bins, axis=1) + # 如果psi值大于阈值 则保留 + if col_psi > max_psi: + max_psi = col_psi + # 如果小于阈值则保留 + if max_psi < psi_select_thresh: + psi_selected_cols.append(col_select) + + df_filled = df_filled[psi_selected_cols] + log.info(f"process_psi psi_selected_cols:{psi_selected_cols}") + return df_filled, psi_selected_cols + + +def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame): + """ + union the column_info1 with the column_info2. + + Args: + column_info1 (DataFrame): The column_info1 to be merged. + column_info2 (DataFrame): The column_info2 to be merged. + + Returns: + column_info_merge (DataFrame): The union column_info. + """ + # 将column_info1和column_info2按照left_index=True, right_index=True的方式进行合并 如果列有缺失则赋值为None 行的顺序按照column_info1 + column_info_conbine = column_info1.merge( + column_info2, how='outer', left_index=True, right_index=True, sort=False) + col1_index_list = column_info1.index.to_list() + col2_index_list = column_info2.index.to_list() + merged_list = col1_index_list + \ + [item for item in col2_index_list if item not in col1_index_list] + column_info_conbine = column_info_conbine.reindex(merged_list) + return column_info_conbine diff --git a/python/ppc_model/preprocessing/local_processing/psi_select.py b/python/ppc_model/preprocessing/local_processing/psi_select.py new file mode 100644 index 00000000..5bd1223e --- /dev/null +++ b/python/ppc_model/preprocessing/local_processing/psi_select.py @@ -0,0 +1,95 @@ +import numpy as np + + +def scale_range(input, min, max): + input += -(np.min(input)) + input /= np.max(input) / (max - min) + input += min + return input + + +def sub_psi(e_perc, a_perc): + """ + Calculate the actual PSI value from comparing the values. + Update the actual value to a very small number if equal to zero + """ + if a_perc == 0: + a_perc = 0.0001 + if e_perc == 0: + e_perc = 0.0001 + + value = (e_perc - a_perc) * np.log(e_perc / a_perc) + return value + + +def calculate_psi(expected, actual, buckettype='bins', buckets=10, axis=0): + """ + Calculate the PSI (population stability index) across all variables + + Args: + expected: numpy matrix of original values + actual: numpy matrix of new values + buckettype: type of strategy for creating buckets, bins splits into even splits, quantiles splits into quantile buckets + buckets: number of quantiles to use in bucketing variables + axis: axis by which variables are defined, 0 for vertical, 1 for horizontal + + Returns: + psi_values: ndarray of psi values for each variable + + Author: + Matthew Burke + github.com/mwburke + mwburke.github.io.com + """ + + def psi(expected_array, actual_array, buckets): + """ + Calculate the PSI for a single variable + + Args: + expected_array: numpy array of original values + actual_array: numpy array of new values, same size as expected + buckets: number of percentile ranges to bucket the values into + + Returns: + psi_value: calculated PSI value + """ + + breakpoints = np.arange(0, buckets + 1) / (buckets) * 100 + + if buckettype == 'bins': + breakpoints = scale_range(breakpoints, np.min( + expected_array), np.max(expected_array)) + elif buckettype == 'quantiles': + breakpoints = np.stack( + [np.percentile(expected_array, b) for b in breakpoints]) + + expected_fractions = np.histogram(expected_array, breakpoints)[ + 0] / len(expected_array) + actual_fractions = np.histogram(actual_array, breakpoints)[ + 0] / len(actual_array) + psi_value = sum(sub_psi(expected_fractions[i], actual_fractions[i]) for i in range( + 0, len(expected_fractions))) + + return psi_value + + if len(actual.shape) == 1: + psi_values = np.empty(len(actual.shape)) + else: + psi_values = np.empty(actual.shape[1 - axis]) + + for i in range(0, len(psi_values)): + if len(psi_values) == 1: + psi_values = psi(expected, actual, buckets) + elif axis == 0: + if len(expected.shape) == 1: + psi_values[i] = psi(expected, actual[:, i], buckets) + else: + psi_values[i] = psi(expected[:, i], actual[:, i], buckets) + elif axis == 1: + if len(expected.shape) == 1: + psi_values[i] = psi(expected, actual[i, :], buckets) + else: + psi_values[i] = psi(expected[i, :], actual[i, :], buckets) + + return psi_values diff --git a/python/ppc_model/preprocessing/local_processing/standard_type_enum.py b/python/ppc_model/preprocessing/local_processing/standard_type_enum.py new file mode 100644 index 00000000..118e2528 --- /dev/null +++ b/python/ppc_model/preprocessing/local_processing/standard_type_enum.py @@ -0,0 +1,7 @@ +from enum import Enum, unique + + +@unique +class standardType(Enum): + min_max = "min-max" + z_score = "z-score" diff --git a/python/ppc_model/preprocessing/preprocessing_engine.py b/python/ppc_model/preprocessing/preprocessing_engine.py new file mode 100644 index 00000000..3ca7c6e3 --- /dev/null +++ b/python/ppc_model/preprocessing/preprocessing_engine.py @@ -0,0 +1,19 @@ +from ppc_model.preprocessing.local_processing.local_processing_party import LocalProcessingParty + +from ppc_model.common.global_context import components +from ppc_model.common.protocol import ModelTask +from ppc_model.interface.task_engine import TaskEngine +from ppc_model.preprocessing.processing_context import ProcessingContext + + +class PreprocessingEngine(TaskEngine): + task_type = ModelTask.PREPROCESSING + + @staticmethod + def run(args): + context = ProcessingContext( + args=args, + components=components, + ) + lpp = LocalProcessingParty(context) + lpp.processing() diff --git a/python/ppc_model/preprocessing/processing_context.py b/python/ppc_model/preprocessing/processing_context.py new file mode 100644 index 00000000..f46fd11c --- /dev/null +++ b/python/ppc_model/preprocessing/processing_context.py @@ -0,0 +1,27 @@ +import os + +from ppc_model.common.context import Context +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_utils import common_func +from ppc_model.common.model_setting import ModelSetting + + +class ProcessingContext(Context): + def __init__(self, + args, + components: Initializer): + super().__init__(args['job_id'], + args['task_id'], + components, + role=None) + self.dataset_path = args['dataset_path'] + self.dataset_file_path = os.path.join( + self.workspace, args['dataset_id']) + self.job_algorithm_type = args['algorithm_type'] + self.need_run_psi = args['need_run_psi'] + self.model_dict = args['model_dict'] + self.training_job_id = common_func.get_config_value( + "training_job_id", None, args, False) + if "psi_result_path" in args: + self.remote_psi_result_path = args["psi_result_path"] + self.model_setting = ModelSetting(self.model_dict) diff --git a/python/ppc_model/preprocessing/tests/test_preprocessing.py b/python/ppc_model/preprocessing/tests/test_preprocessing.py new file mode 100644 index 00000000..279e00e6 --- /dev/null +++ b/python/ppc_model/preprocessing/tests/test_preprocessing.py @@ -0,0 +1,672 @@ +from ppc_common.ppc_utils import utils +from local_processing.preprocessing import union_column_info, process_na_dataframe, process_na_fill_dataframe, process_outliers, one_hot_encode_and_merge, normalize_dataframe, process_train_dataframe, remove_high_correlation_features, process_psi, process_dataframe +import pandas as pd +import numpy as np +import pytest +import json +import sys + +from ppc_model.preprocessing.local_processing.standard_type_enum import standardType +# import pytest +import numpy as np +import pandas as pd +from ppc_model.preprocessing.local_processing.preprocessing import union_column_info, process_na_dataframe, process_na_fill_dataframe, process_outliers, one_hot_encode_and_merge, normalize_dataframe, process_train_dataframe, remove_high_correlation_features, process_psi, process_dataframe +from ppc_common.ppc_utils import utils + + +def test_process_na_dataframe(): + # Create a sample DataFrame with missing values + df = pd.DataFrame({ + 'col1': [1, 2, None, 4, 5], + 'col2': [None, 2, None, 4, None], + 'col3': [1, 2, 3, 4, 5], + 'col4': [1, None, 3, 4, 5], + 'col5': [1, 2, None, None, None], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [1, 2, 3.0, 4, 5], + 'col3': [1, 2, 3, 4, 5], + 'col4': [1, 6.0, 3, 4, 5], + 'y': [0, 1, 0, 1, 0] + }) + expected_column_info = {'col1': {'missing_ratio': 0.2, 'na_selected': 1, 'isExisted': True}, + 'col2': {'missing_ratio': 0.6, 'na_selected': 0, 'isExisted': False}, + 'col3': {'missing_ratio': 0.0, 'na_selected': 1, 'isExisted': True}, + 'col4': {'missing_ratio': 0.2, 'na_selected': 1, 'isExisted': True}, + 'col5': {'missing_ratio': 0.6, 'na_selected': 0, 'isExisted': False}, + 'y': {'missing_ratio': 0.0, 'na_selected': 1, 'isExisted': True}} + + # Call the function under test + processed_df, column_info = process_na_dataframe(df, 0.5) + assert column_info == expected_column_info + processed_df = process_na_fill_dataframe(processed_df, ['col2', 'col4']) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_process_nan_dataframe(): + # Create a sample DataFrame with missing values + df = pd.DataFrame({ + 'col1': [1, 2, None, 4, 5], + 'col2': [None, 2, None, 4, None], + 'col3': [1, 2, 3, 4, 5], + 'col4': [1, None, 3, 4, 5], + 'col5': [1, 2, None, None, None], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [1, 2, np.nan, 4, 5], + 'col2': [np.nan, 2, np.nan, 4, np.nan], + 'col3': [1, 2, 3, 4, 5], + 'col4': [1, np.nan, 3, 4, 5], + 'col5': [1, 2, np.nan, np.nan, np.nan], + 'y': [0, 1, 0, 1, 0] + }) + + # Call the function under test + processed_df = df.replace('None', np.nan) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_process_outliers(): + # Create a sample DataFrame with outliers + df = pd.DataFrame({ + 'col3': [100, 200, None, 400, 500, 100, 200, 300, 400, 500, 100, 200, None, 400, 500], + 'col5': [100000, None, 1, -1, 1, -1, 1, -1, 1, 1, -1, None, None, None, 1], + }) + # + # outlier = np.random.normal(df['col5'].mean() + 3 * df['col5'].std(), df['col5'].std()) + # df.at[1, 'col5'] = outlier + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col3': [100, 200, None, 400, 500, 100, 200, 300, 400, 500, 100, 200, None, 400, 500], + 'col5': [df['col5'].mean(), None, 1, -1, 1, -1, 1, -1, 1, 1, -1, None, None, None, 1], + }) + + # Call the function under test + processed_df = process_outliers(df, []) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_one_hot_encode_and_merge(): + # Create a sample DataFrame + df = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'y': [0, 1, 0, 1, 0], + 'col2_A': [True, False, False, True, False], + 'col2_B': [False, True, False, False, True], + 'col2_C': [False, False, True, False, False], + 'col3_X': [True, False, False, True, False], + 'col3_Y': [False, True, False, False, True], + 'col3_Z': [False, False, True, False, False] + }) + + # Call the function under test + processed_df = one_hot_encode_and_merge(df, ['col2', 'col3']) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_normalize_dataframe(): + # Create a sample DataFrame + df = pd.DataFrame({ + 'col1': [1, 2, None, 4, 5], + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'col4': [10, 20, 30, 40, 50], + 'col5': [100, 200, None, 400, 500] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [0.0, 0.25, None, 0.75, 1.0], + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'col4': [0.0, 0.25, 0.5, 0.75, 1.0], + 'col5': [0.0, 0.25, None, 0.75, 1.0] + }) + + # Call the function under test + processed_df = normalize_dataframe( + df, ['col2', 'col3'], standardType.min_max.value) + assert processed_df.equals(expected_output) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [-1.265, -0.632, None, 0.632, 1.265], + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'col4': [-1.414, -0.707, 0.0, 0.707, 1.414], + 'col5': [-1.265, -0.632, None, 0.632, 1.265] + }) + # Call the function under test + processed_df = normalize_dataframe( + df, ['col2', 'col3'], standardType.z_score.value) + # Assert that the processed DataFrame matches the expected output + print(expected_output) + print(processed_df) + assert processed_df.round(3).equals(expected_output) + + +def test_remove_high_correlation_features(): + # Create a sample DataFrame + df = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'col2': [2, 4, 6, 8, 10], + 'col3': [3, 6, 9, 12, 15], + 'col4': [4, 8, 12, 16, 20], + 'col5': [5, 10, 15, 20, 25], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'y': [0, 1, 0, 1, 0] + }) + + # Call the function under test + processed_df = remove_high_correlation_features(df, ['y'], 0.8) + + df = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'col2': [2, 4, None, 8, 10], + 'col3': [3, 6, 9, 12, 15], + 'col4': [4, None, 12, 16, 20], + 'col5': [5, 10, None, 20, 25], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [1, 2, 3, 4, 5], + 'y': [0, 1, 0, 1, 0] + }) + processed_df = remove_high_correlation_features(df, ['y'], 0.8) + print(f"processed_df:{processed_df}") + print(f"expected_output:{expected_output}") + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + df = pd.DataFrame({ + 'col1': [-1, 222, 3.4, -22, 5.1], + 'col2': [2, None, 6, 8, 10], + 'col3': [3, 6, 9, 12, 15], + 'col4': [4, None, 12, 16, 20], + 'col5': [5, 10, None, 20, 25], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col1': [-1, 222, 3.4, -22, 5.1], + 'col2': [2, None, 6, 8, 10], + 'y': [0, 1, 0, 1, 0] + }) + processed_df = remove_high_correlation_features(df, ['y'], 0.8) + + print(processed_df) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_process_psi(): + # Create a sample DataFrame + df_filled = pd.DataFrame({ + # 'col1': [0, 1, 0, 1, 0], + 'col1': [1, 2, 3, 4, 5], + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'col4': [10, 20, 30, 40, 50], + 'col5': [100, 200, 300, 400, 500], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'col2': ['A', 'B', 'C', 'A', 'B'], + 'col3': ['X', 'Y', 'Z', 'X', 'Y'], + 'y': [0, 1, 0, 1, 0] + }) + + # Call the function under test + processed_df, _ = process_psi( + df_filled, ['col2', 'col3', 'y'], 'col1', 1, 0.9, 5) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + # read test csv as DataFrame + test_file_path = "./癌症纵向训练数据集psi.csv" + df_filled = pd.read_csv(test_file_path) + expected_output_col = ['id', 'y', 'x1', 'x5', + 'x6', 'x9', 'x10', 'x11', 'x12', 'x14'] + processed_df, _ = process_psi( + df_filled, ['id', 'y', 'x1', 'x5', 'x23'], 'x15', 0, 0.3, 4) + + assert processed_df.columns.tolist() == expected_output_col + + +def test_process_dataframe(): + # Create a sample DataFrame + test_file_path = "./癌症纵向训练数据集psi.csv" + df_filled = pd.read_csv(test_file_path) + + # Create a mock JobContext object + xgb_model = { + "algorithm_subtype": "HeteroXGB", + "participants": 2, # 提供数据集机构数 + # XGBoost参数 + "use_psi": 0, # 0否，1是 + "use_goss": 0, # 0否，1是 + "test_dataset_percentage": 0.3, + "learning_rate": 0.1, + "num_trees": 6, + "max_depth": 3, + "max_bin": 4, # 分箱数（计算XGB直方图） + "threads": 8, + # 调度服务各方预处理 + "na_select": 0.8, # 缺失值筛选阈值，缺失值比例超过阈值则移除该特征（0表示只要有缺失值就移除，1表示仅移除全为缺失值的列） + "fillna": 0, # 是否缺失值填充（均值） + "psi_select_col": "x15", # PSI稳定性筛选时间列名称（0代表不进行PSI筛选，例如："month"） + "psi_select_base": 1, # PSI稳定性筛选时间基期（例如以0为基期，统计其他时间的psi，时间按照周/月份/季度等提前处理为0,1,2,...,n格式） + "psi_select_thresh": 0.3, # 若最大的逐月PSI>0.1，则剔除该特征 + "psi_select_bins": 4, # 计算逐月PSI时分箱数 + "filloutlier": 1, # 是否异常值处理（+-3倍标准差使用均值填充） + "normalized": 1, # 是否归一化，每个值减去最小值，然后除以最大值与最小值的差 + "standardized": 1, # 是否标准化，计算每个数据点与均值的差，然后除以标准差 + "one_hot": 0, # 是否进行onehot + # 指定分类特征索引，例如"x1,x12,x23" ("0"代表无分类特征)（分类特征需要在此处标注，使用onehot预处理/建模） + "categorical": "id,y,x1,x5,x23", + # 建模节点特征工程 + "use_iv": 0, # 是否计算woe/iv，并使用iv进行特征筛选 + 'group_num': 3, # 分箱数（计算woe分箱，等频） + 'iv_thresh': 0.1, # 使用iv进行特征筛选的阈值（仅保留iv大于阈值的特征） + 'corr_select': 0.8 # 计算特征相关性，相关性大于阈值特征仅保留iv最大的（如use_iv=0，则随机保留一个）（corr_select=0时不进行相关性筛选） + } + + xgb_dict = dict() + # 根据xgb_model 生成xgb_dict + for key, value in xgb_model.items(): + xgb_dict[key] = value + + # Call the function under test + column_info1 = process_dataframe( + df_filled, xgb_dict, "./xgb_data_file_path", utils.AlgorithmType.Train.name, "j-123456") + + xgb_model = { + "algorithm_subtype": "HeteroXGB", + "participants": 2, # 提供数据集机构数 + # XGBoost参数 + "use_psi": 0, # 0否，1是 + "use_goss": 0, # 0否，1是 + "test_dataset_percentage": 0.3, + "learning_rate": 0.1, + "num_trees": 6, + "max_depth": 3, + "max_bin": 4, # 分箱数（计算XGB直方图） + "threads": 8, + # 调度服务各方预处理 + "na_select": 0.8, # 缺失值筛选阈值，缺失值比例超过阈值则移除该特征（0表示只要有缺失值就移除，1表示仅移除全为缺失值的列） + "fillna": 1, # 是否缺失值填充（均值） + "psi_select_col": "x15", # PSI稳定性筛选时间列名称（0代表不进行PSI筛选，例如："month"） + "psi_select_base": 1, # PSI稳定性筛选时间基期（例如以0为基期，统计其他时间的psi，时间按照周/月份/季度等提前处理为0,1,2,...,n格式） + "psi_select_thresh": 0.3, # 若最大的逐月PSI>0.1，则剔除该特征 + "psi_select_bins": 4, # 计算逐月PSI时分箱数 + "filloutlier": 1, # 是否异常值处理（+-3倍标准差使用均值填充） + "normalized": 1, # 是否归一化，每个值减去最小值，然后除以最大值与最小值的差 + "standardized": 1, # 是否标准化，计算每个数据点与均值的差，然后除以标准差 + "one_hot": 0, # 是否进行onehot + # 指定分类特征索引，例如"x1,x12,x23" ("0"代表无分类特征)（分类特征需要在此处标注，使用onehot预处理/建模） + "categorical": "id,y,x1,x5,x23", + # 建模节点特征工程 + "use_iv": 0, # 是否计算woe/iv，并使用iv进行特征筛选 + 'group_num': 3, # 分箱数（计算woe分箱，等频） + 'iv_thresh': 0.1, # 使用iv进行特征筛选的阈值（仅保留iv大于阈值的特征） + 'corr_select': 0.8 # 计算特征相关性，相关性大于阈值特征仅保留iv最大的（如use_iv=0，则随机保留一个）（corr_select=0时不进行相关性筛选） + } + for key, value in xgb_model.items(): + xgb_dict[key] = value + column_info2 = process_dataframe( + df_filled, xgb_dict, "./xgb_data_file_path2", utils.AlgorithmType.Train.name, "j-123456") + assert column_info1 == column_info2 + expected_column_info = {'y': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x0': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x1': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x2': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 0, + 'isExisted': False + }, + 'x3': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 0, + 'isExisted': False + }, + 'x4': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 0, + 'corr_selected': 0, + 'isExisted': False + }, + 'x5': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x6': {'missing_ratio': 0.018, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x7': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 0, + 'corr_selected': 0, + 'isExisted': False + }, + 'x8': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x9': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x10': {'missing_ratio': 0.012, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x11': {'missing_ratio': 0.998, + 'na_selected': 0, + 'psi_selected': 0, + 'corr_selected': 0, + 'isExisted': False + }, + 'x12': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 0, + 'isExisted': False + }, + 'x13': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 0, + 'isExisted': False + }, + 'x14': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 1, + 'corr_selected': 1, + 'isExisted': True + }, + 'x15': {'missing_ratio': 0.0, + 'na_selected': 1, + 'psi_selected': 0, + 'corr_selected': 0, + 'isExisted': False + } + } + # 转成python字典 + expected_column_info_dict = dict(expected_column_info) + assert column_info1 == expected_column_info_dict + + +def test_process_train_dataframe(): + # Create a sample DataFrame + df = pd.DataFrame({ + 'id': [1, 2, 3, 4, 5], + 'x1': [10, 20, 30, 40, 50], + 'x2': [100, 200, 300, 400, 500], + 'x3': [1000, 2000, 3000, 4000, 5000], + 'x4': [10000, 20000, 30000, 40000, 50000], + 'x5': [100000, 200000, 300000, 400000, 500000], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'id': [1, 2, 3, 4, 5], + 'x2': [100, 200, 300, 400, 500], + 'x3': [1000, 2000, 3000, 4000, 5000] + }) + + # Define the column_info dictionary + column_info = { + 'id': {'isExisted': True}, + 'x1': {'isExisted': False}, + 'x2': {'isExisted': True}, + 'x3': {'isExisted': True}, + 'x4': {'isExisted': False}, + 'x5': {'isExisted': False}, + 'y': {'isExisted': False}, + } + + # Call the function under test + processed_df = process_train_dataframe(df, column_info) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + + +def test_process_train_dataframe_with_additional_columns(): + # Create a sample DataFrame + df = pd.DataFrame({ + 'id': [1, 2, 3, 4, 5], + 'x1': [10, 20, 30, 40, 50], + 'x2': [100, 200, 300, 400, 500], + 'x3': [1000, 2000, 3000, 4000, 5000], + 'x4': [10000, 20000, 30000, 40000, 50000], + 'x5': [100000, 200000, 300000, 400000, 500000], + 'y': [0, 1, 0, 1, 0] + }) + + # Define the expected output DataFrame + expected_output = pd.DataFrame({ + 'id': [1, 2, 3, 4, 5], + 'x1': [10, 20, 30, 40, 50], + 'x3': [1000, 2000, 3000, 4000, 5000], + 'x4': [10000, 20000, 30000, 40000, 50000], + 'x5': [100000, 200000, 300000, 400000, 500000] + }) + + # Define the column_info dictionary + column_info = { + 'id': {'isExisted': True}, + 'x1': {'isExisted': True}, + 'x2': {'isExisted': False}, + 'x3': {'isExisted': True}, + 'x4': {'isExisted': True}, + 'x5': {'isExisted': True}, + 'y': {'isExisted': False}, + } + + # Call the function under test + processed_df = process_train_dataframe(df, column_info) + + # Assert that the processed DataFrame matches the expected output + assert processed_df.equals(expected_output) + +def test_merge_column_info_from_file(): + col_info_file_path = "./test_column_info_merge.csv" + iv_info_file_path = "./test_column_info_iv.csv" + column_info_fm = pd.read_csv(col_info_file_path, index_col=0) + column_info_iv = pd.read_csv(iv_info_file_path, index_col=0) + union_df = union_column_info(column_info_fm, column_info_iv) + + col_str_expected = '{"y": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x0": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x1": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x2": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 1, "corr_selected": 0}, "x3": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 1, "corr_selected": 0}, "x4": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 0, "corr_selected": 0}, "x5": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x6": {"missing_ratio": 0.018, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x7": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 0, "corr_selected": 0}, "x8": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x9": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x10": {"missing_ratio": 0.012, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x11": {"missing_ratio": 0.998, "na_selected": 0, "isExisted": false, "psi_selected": 0, "corr_selected": 0}, "x12": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 1, "corr_selected": 0}, "x13": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 1, "corr_selected": 0}, "x14": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": true, "psi_selected": 1, "corr_selected": 1}, "x15": {"missing_ratio": 0.0, "na_selected": 1, "isExisted": false, "psi_selected": 0, "corr_selected": 0}}' + + # expected_df_file_path = './test_union_column.csv' + # expected_df = pd.read_csv(expected_df_file_path, index_col=0) + # assert expected_df.equals(union_df) + column_info_str = json.dumps(column_info_fm.to_dict(orient='index')) + assert column_info_str == col_str_expected + +def construct_dataset(num_samples, num_features, file_path): + np.random.seed(0) + # 生成标签列 + labels = np.random.choice([0, 1], size=num_samples) + # 生成特征列 + features = np.random.rand(num_samples, num_features) + # 将标签转换为DataFrame + labels_df = pd.DataFrame(labels, columns=['Label']) + + # 将特征转换为DataFrame + features_df = pd.DataFrame(features) + + # 合并标签和特征DataFrame + dataset_df = pd.concat([labels_df, features_df], axis=1) + + # 将DataFrame写入CSV文件 + dataset_df.to_csv(file_path, index=False) + + return labels, features + +def test_gen_file(): + num_samples = 400000 + num_features = 100 + file_path = "./dataset-{}-{}.csv".format(num_samples, num_features) + construct_dataset(num_samples, num_features, file_path) + +def test_large_process_train_dataframe(): + num_samples = 400000 + num_features = 100 + test_file_path = "./dataset-{}-{}.csv".format(num_samples, num_features) + df_filled = pd.read_csv(test_file_path) + + # Create a mock JobContext object + xgb_model = { + "algorithm_subtype": "HeteroXGB", + "participants": 2, # 提供数据集机构数 + # XGBoost参数 + "use_psi": 0, # 0否，1是 + "use_goss": 0, # 0否，1是 + "test_dataset_percentage": 0.3, + "learning_rate": 0.1, + "num_trees": 6, + "max_depth": 3, + "max_bin": 4, # 分箱数（计算XGB直方图） + "threads": 8, + # 调度服务各方预处理 + "na_select": 0.8, # 缺失值筛选阈值，缺失值比例超过阈值则移除该特征（0表示只要有缺失值就移除，1表示仅移除全为缺失值的列） + "fillna": 0, # 是否缺失值填充（均值） + "psi_select_col": 0, # PSI稳定性筛选时间列名称（0代表不进行PSI筛选，例如："month"） + "psi_select_base": 1, # PSI稳定性筛选时间基期（例如以0为基期，统计其他时间的psi，时间按照周/月份/季度等提前处理为0,1,2,...,n格式） + "psi_select_thresh": 0.3, # 若最大的逐月PSI>0.1，则剔除该特征 + "psi_select_bins": 4, # 计算逐月PSI时分箱数 + "filloutlier": 1, # 是否异常值处理（+-3倍标准差使用均值填充） + "normalized": 1, # 是否归一化，每个值减去最小值，然后除以最大值与最小值的差 + "standardized": 1, # 是否标准化，计算每个数据点与均值的差，然后除以标准差 + "one_hot": 0, # 是否进行onehot + # 指定分类特征索引，例如"x1,x12,x23" ("0"代表无分类特征)（分类特征需要在此处标注，使用onehot预处理/建模） + "categorical": "id,y,1,5,23", + # 建模节点特征工程 + "use_iv": 0, # 是否计算woe/iv，并使用iv进行特征筛选 + 'group_num': 3, # 分箱数（计算woe分箱，等频） + 'iv_thresh': 0.1, # 使用iv进行特征筛选的阈值（仅保留iv大于阈值的特征） + 'corr_select': 0.8 # 计算特征相关性，相关性大于阈值特征仅保留iv最大的（如use_iv=0，则随机保留一个）（corr_select=0时不进行相关性筛选） + } + + xgb_dict = dict() + # 根据xgb_model 生成xgb_dict + for key, value in xgb_model.items(): + xgb_dict[key] = value + + # Call the function under test + start_time = time.time() + column_info1 = process_dataframe( + df_filled, xgb_dict, "./xgb_data_file_path", utils.AlgorithmType.Train.name, "j-123456") + end_time = time.time() + print(f"test_large_process_train_dataframe time cost:{end_time-start_time}, num_samples: {num_samples}, num_features: {num_features}") + + + + + +# Run the tests +# pytest.main() +if __name__=="__main__": + import time + # test_large_process_train_dataframe() + time1 = time.time() + test_process_na_dataframe() + time2 = time.time() + test_process_nan_dataframe() + time3 = time.time() + test_process_outliers() + time4 = time.time() + test_one_hot_encode_and_merge() + time5 = time.time() + test_normalize_dataframe() + time6 = time.time() + test_remove_high_correlation_features() + time7 = time.time() + test_process_psi() + time8 = time.time() + test_process_dataframe() + time9 = time.time() + test_process_train_dataframe() + time10 = time.time() + test_process_train_dataframe_with_additional_columns() + time11 = time.time() + test_merge_column_info_from_file() + time12 = time.time() + print(f"test_process_na_dataframe time cost: {time2-time1}") + print(f"test_process_nan_dataframe time cost: {time3-time2}") + print(f"test_process_outliers time cost: {time4-time3}") + print(f"test_one_hot_encode_and_merge time cost: {time5-time4}") + print(f"test_normalize_dataframe time cosy: {time6-time5}") + print(f"test_remove_high_correlation_features time cost: {time7-time6}") + print(f"test_process_psi time cost: {time8-time7}") + print(f"test_process_dataframe time cost: {time9-time8}") + print(f"test_process_train_dataframe time cost: {time10-time9}") + print(f"test_process_train_dataframe_with_additional_columns time cost: {time11-time10}") + print(f"test_merge_column_info_from_file time cost: {time12-time11}") + print("All tests pass!") diff --git a/python/ppc_model/secure_lgbm/__init__.py b/python/ppc_model/secure_lgbm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lgbm/monitor/__init__.py b/python/ppc_model/secure_lgbm/monitor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lgbm/monitor/callback.py b/python/ppc_model/secure_lgbm/monitor/callback.py new file mode 100644 index 00000000..fa1247de --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/callback.py @@ -0,0 +1,83 @@ +from abc import ABC +from typing import ( + Callable, + Optional, + Sequence +) + +import numpy as np + +from ppc_model.secure_lgbm.monitor.core import _Model + + +class TrainingCallback(ABC): + def __init__(self) -> None: + pass + + def before_training(self, model: _Model) -> _Model: + """Run before training starts.""" + return model + + def after_training(self, model: _Model) -> _Model: + """Run after training is finished.""" + return model + + def before_iteration(self, model: _Model, epoch: int) -> bool: + """Run before each iteration. Returns True when training should stop.""" + return False + + def after_iteration(self, model: _Model, epoch: int) -> bool: + """Run after each iteration. Returns `True` when training should stop.""" + return False + + +class CallbackContainer: + """A special internal callback for invoking a list of other callbacks.""" + + def __init__( + self, + callbacks: Sequence[TrainingCallback], + feval: Optional[Callable] = None + ) -> None: + self.callbacks = set(callbacks) + for cb in callbacks: + if not isinstance(cb, TrainingCallback): + raise TypeError("callback must be an instance of `TrainingCallback`.") + + msg = ( + "feval must be callable object for monitoring. For builtin metrics" + ", passing them in training parameter invokes monitor automatically." + ) + if feval is not None and not callable(feval): + raise TypeError(msg) + + self.feval = feval + + def before_training(self, model: _Model) -> _Model: + for c in self.callbacks: + model = c.before_training(model=model) + return model + + def after_training(self, model: _Model) -> _Model: + for c in self.callbacks: + model = c.after_training(model=model) + return model + + def before_iteration( + self, + model: _Model + ) -> bool: + return any( + c.before_iteration(model, model.get_epoch()) for c in self.callbacks + ) + + def after_iteration( + self, + model: _Model, + pred: np.ndarray, + eval_on_test: bool = True + ) -> bool: + model.after_iteration(pred, eval_on_test) + model.eval(self.feval) + ret = any(c.after_iteration(model, model.get_epoch()) for c in self.callbacks) + return ret diff --git a/python/ppc_model/secure_lgbm/monitor/core.py b/python/ppc_model/secure_lgbm/monitor/core.py new file mode 100644 index 00000000..6e080a04 --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/core.py @@ -0,0 +1,144 @@ +import collections +from typing import Dict, Any, Union, Tuple, List, Callable + +import numpy as np +from sklearn import metrics + +_Score = Union[float, Tuple[float, float]] +_ScoreList = Union[List[float], List[Tuple[float, float]]] + +_BoosterParams = Dict[str, Any] +_Metric = Callable[[np.ndarray, np.ndarray], Dict[str, _Score]] + +""" +A dictionary containing the evaluation history: +{"metric_name": [0.5, ...]} +""" +_EvalsLog = Dict[str, _ScoreList] + + +class Booster: + """A Booster of XGBoost. + + Booster is the model of xgboost, that contains low level routines for + training, prediction and evaluation. + """ + + def __init__( + self, + y_true: np.ndarray, + test_y_true: np.ndarray, + workspace: str = None, + job_id: str = None, + storage_client: str = None + ) -> None: + self.params: _BoosterParams = {} + self.y_true = y_true + self.test_y_true = test_y_true + self.y_pred = None + self.test_y_pred = None + self.eval_on_test = True + self.epoch = 0 + self.workspace = workspace + self.job_id = job_id + self.storage_client = storage_client + self.history: _EvalsLog = collections.OrderedDict() + + def get_y_true(self) -> np.ndarray: + return self.y_true + + def get_test_y_true(self) -> np.ndarray: + return self.test_y_true + + def get_y_pred(self) -> np.ndarray: + return self.y_pred + + def get_test_y_pred(self) -> np.ndarray: + return self.test_y_pred + + def get_epoch(self) -> int: + return self.epoch + + def get_workspace(self) -> str: + return self.workspace + + def get_job_id(self) -> str: + return self.job_id + + def get_storage_client(self): + return self.storage_client + + def set_param( + self, + key: str, + value: Any, + ) -> None: + self.params[key] = value + + def get_param( + self, + key: str + ) -> Any: + return self.params[key] + + def get_history(self) -> _EvalsLog: + return self.history + + def after_iteration( + self, + pred: np.ndarray, + eval_on_test: bool = True + ) -> None: + if eval_on_test: + self.test_y_pred = pred + else: + self.y_pred = pred + self.eval_on_test = eval_on_test + self.epoch += 1 + + def _update_history( + self, + scores: Dict[str, _Score] + ) -> None: + for key, value in scores.items(): + if key in self.history: + self.history[key].append(value) + else: + self.history[key] = [value] + + def eval( + self, + feval: _Metric + ) -> Dict[str, _Score]: + if self.eval_on_test: + scores = feval(self.test_y_true, self.test_y_pred) + else: + scores = feval(self.y_true, self.y_pred) + self._update_history(scores) + return scores + + +_Model = Booster + + +def fevaluation( + y_true: np.ndarray, + y_pred: np.ndarray, + decimal_num: int = 4 +) -> Dict[str, _Score]: + auc = metrics.roc_auc_score(y_true, y_pred) + + y_pred_label = [0 if p <= 0.5 else 1 for p in y_pred] + acc = metrics.accuracy_score(y_true, y_pred_label) + recall = metrics.recall_score(y_true, y_pred_label) + precision = metrics.precision_score(y_true, y_pred_label) + + scores_dict = { + 'auc': auc, + 'acc': acc, + 'recall': recall, + 'precision': precision + } + for metric_name in scores_dict: + scores_dict[metric_name] = round(scores_dict[metric_name], decimal_num) + return scores_dict diff --git a/python/ppc_model/secure_lgbm/monitor/early_stopping.py b/python/ppc_model/secure_lgbm/monitor/early_stopping.py new file mode 100644 index 00000000..de718ee3 --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/early_stopping.py @@ -0,0 +1,122 @@ +from typing import Optional, cast + +import numpy + +from ppc_model.secure_lgbm.monitor.callback import TrainingCallback +from ppc_model.secure_lgbm.monitor.core import _Score, _ScoreList, _Model, _EvalsLog + + +class EarlyStopping(TrainingCallback): + """Callback function for early stopping + Parameters + ---------- + rounds : + Early stopping rounds. + metric_name : + Name of metric that is used for early stopping. + maximize : + Whether to maximize evaluation metric. None means auto (discouraged). + min_delta : + Minimum absolute change in score to be qualified as an improvement. + """ + + def __init__( + self, + rounds: int, + metric_name: str, + maximize: Optional[bool] = None, + save_best: Optional[bool] = True, + min_delta: float = 0.0, + ) -> None: + self.metric_name = metric_name + assert self.metric_name in ['auc', 'acc', 'recall', 'precision'] + self.rounds = rounds + self.maximize = maximize + self.save_best = save_best + self._min_delta = min_delta + if self._min_delta < 0: + raise ValueError("min_delta must be greater or equal to 0.") + self.stopping_history: _EvalsLog = {} + self.current_rounds: int = 0 + self.best_scores: dict = {} + super().__init__() + + def before_training(self, model: _Model) -> _Model: + return model + + def _update_rounds( + self, score: _Score, metric_name: str, model: _Model, epoch: int + ) -> bool: + def get_s(value: _Score) -> float: + """get score if it's cross validation history.""" + return value[0] if isinstance(value, tuple) else value + + def maximize(new: _Score, best: _Score) -> bool: + """New score should be greater than the old one.""" + return numpy.greater(get_s(new) - self._min_delta, get_s(best)) + + def minimize(new: _Score, best: _Score) -> bool: + """New score should be lesser than the old one.""" + return numpy.greater(get_s(best) - self._min_delta, get_s(new)) + + if self.maximize is None: + maximize_metrics = ( + "auc", + "aucpr", + "pre", + "pre@", + "map", + "ndcg", + "auc@", + "aucpr@", + "map@", + "ndcg@", + ) + if metric_name != "mape" and any(metric_name.startswith(x) for x in maximize_metrics): + self.maximize = True + else: + self.maximize = False + + if self.maximize: + improve_op = maximize + else: + improve_op = minimize + + if not self.stopping_history: # First round + self.current_rounds = 0 + self.stopping_history[metric_name] = cast(_ScoreList, [score]) + self.best_scores[metric_name] = cast(_ScoreList, [score]) + if self.save_best: + model.set_param('best_score', score) + model.set_param('best_iteration', epoch) + elif not improve_op(score, self.best_scores[metric_name][-1]): + # Not improved + self.stopping_history[metric_name].append(score) # type: ignore + self.current_rounds += 1 + else: # Improved + self.stopping_history[metric_name].append(score) # type: ignore + self.best_scores[metric_name].append(score) + self.current_rounds = 0 # reset + if self.save_best: + model.set_param('best_score', score) + model.set_param('best_iteration', epoch) + + if self.current_rounds >= self.rounds: + # Should stop + return True + return False + + def after_iteration( + self, model: _Model, epoch: int + ) -> bool: + history = model.get_history() + if len(history.keys()) < 1: + raise ValueError("Must have at least 1 validation dataset for early stopping.") + + metric_name = self.metric_name + # The latest score + score = history[metric_name][-1] + return self._update_rounds(score, metric_name, model, epoch) + + def after_training(self, model: _Model) -> _Model: + return model diff --git a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py new file mode 100644 index 00000000..c1038bbb --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py @@ -0,0 +1,123 @@ +import os +import time +import random +import traceback +from typing import Optional + +import matplotlib.pyplot as plt + +from ppc_common.ppc_utils.utils import METRICS_OVER_ITERATION_FILE +from ppc_model.common.global_context import plot_lock +from ppc_model.secure_lgbm.monitor.callback import TrainingCallback +from ppc_model.secure_lgbm.monitor.core import _Model + + +def _draw_figure(model: _Model): + scores = model.get_history() + path = model.get_workspace() + + iterations = [i + 1 for i in range(len(next(iter(scores.values()))))] + + # plt.cla() + plt.figure(figsize=(int(10 + len(iterations) / 5), 10)) + + for metric, values in scores.items(): + plt.plot(iterations, values, label=metric) + max_index = values.index(max(values)) + plt.scatter(max_index + 1, values[max_index], color='green') + plt.text(max_index + 1, values[max_index], f'{values[max_index]:.4f}', fontsize=9, ha='right') + + plt.legend() + plt.title('Metrics Over Iterations') + plt.xlabel('Iteration') + plt.ylabel('Metric Value') + plt.grid(True) + if len(iterations) <= 60: + plt.xticks(iterations, fontsize=10, rotation=45) + else: + plt.xticks(range(0, len(iterations), 5), fontsize=10, rotation=45) + plt.yticks(fontsize=12) + + file_path = os.path.join(path, METRICS_OVER_ITERATION_FILE) + plt.savefig(file_path, format='svg', dpi=300) + plt.close('all') + + +def _upload_figure(model: _Model): + storage_client = model.get_storage_client() + if storage_client is not None: + path = model.get_workspace() + job_id = model.get_job_id() + metrics_file_path = os.path.join(path, METRICS_OVER_ITERATION_FILE) + unique_file_path = os.path.join(job_id, METRICS_OVER_ITERATION_FILE) + storage_client.upload_file(metrics_file_path, unique_file_path) + + +def _fmt_metric( + metric_name: str, score: float +) -> str: + msg = f"\t{metric_name}:{score:.5f}" + return msg + + +class EvaluationMonitor(TrainingCallback): + """Print the evaluation result after each period iteration. + Parameters + ---------- + period : + How many epoches between printing. + """ + + def __init__(self, logger, period: int = 1) -> None: + self.logger = logger + self.period = period + assert period > 0 + # last error message, useful when early stopping and period are used together. + self._latest: Optional[str] = None + super().__init__() + + def after_iteration( + self, model: _Model, epoch: int + ) -> bool: + history = model.get_history() + if not history: + return False + + msg: str = f"[{model.get_job_id()}, epoch(iter): {epoch}]" + for metric_name, scores in history.items(): + if isinstance(scores[-1], tuple): + score = scores[-1][0] + else: + score = scores[-1] + msg += _fmt_metric(metric_name, score) + msg += "\n" + + if (epoch % self.period) == 0 or self.period == 1: + self.logger.info(msg) + self._latest = None + else: + # There is skipped message + self._latest = msg + + return False + + def after_training(self, model: _Model) -> _Model: + if self._latest is not None: + self.logger.info(self._latest) + max_retry = 3 + retry_num = 0 + while retry_num < max_retry: + retry_num += 1 + try: + with plot_lock: + _draw_figure(model) + except: + self.logger.info(f'scores = {model.get_history()}') + self.logger.info(f'path = {model.get_workspace()}') + err = traceback.format_exc() + # self.logger.exception(err) + self.logger.info( + f'plot moniter in times-{retry_num} failed, traceback: {err}.') + time.sleep(random.uniform(0.1, 3)) + _upload_figure(model) + return model diff --git a/python/ppc_model/secure_lgbm/monitor/feature/__init__.py b/python/ppc_model/secure_lgbm/monitor/feature/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py b/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py new file mode 100644 index 00000000..bb2530f2 --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +from enum import Enum +import pandas as pd + + +class EvaluationType(Enum): + """the evaluation type + """ + TRAIN = "train", + VALIDATION = "validation", + + +class EvaluationMetric: + """the evaluation metric + """ + + def __init__(self, value, desc): + self.value = value + self.desc = desc + + def set_desc(self, desc): + self.desc = desc + + +class FeatureEvaluationResult: + DEFAULT_SAMPLE_STAT_DESC = "总样本" + DEFAULT_POSITIVE_SAMPLE_STAT_DESC = "正样本" + DEFAULT_KS_STAT_DESC = "KS" + DEFAULT_AUC_STAT_DESC = "AUC" + + DEFAULT_TRAIN_EVALUATION_DESC = "训练集" + DEFAULT_VALIDATION_EVALUATION_DESC = "验证集" + DEFAULT_ROW_INDEX_LABEL_DESC = "分类" + + def __init__(self, type, type_desc=None, ks_value=0, auc_value=0, label_list=None): + self.type = type + self.type_desc = type_desc + if self.type_desc is None: + if self.type == EvaluationType.TRAIN: + self.type_desc = FeatureEvaluationResult.DEFAULT_TRAIN_EVALUATION_DESC + elif self.type == EvaluationType.VALIDATION: + self.type_desc = FeatureEvaluationResult.DEFAULT_VALIDATION_EVALUATION_DESC + else: + raise Exception( + f"Create FeatureEvaluationResult for unsupported evaluation type: {type}") + self.ks = EvaluationMetric( + ks_value, FeatureEvaluationResult.DEFAULT_KS_STAT_DESC) + self.auc = EvaluationMetric( + auc_value, FeatureEvaluationResult.DEFAULT_AUC_STAT_DESC) + self.positive_samples = EvaluationMetric( + 0, FeatureEvaluationResult.DEFAULT_POSITIVE_SAMPLE_STAT_DESC) + self.samples = EvaluationMetric( + 0, FeatureEvaluationResult.DEFAULT_SAMPLE_STAT_DESC) + + if label_list is not None: + self.set_sample_info(label_list) + + def set_sample_info(self, label_list): + self.samples.value = len(label_list) + for label in label_list: + self.positive_samples.value += label + + def columns(self): + return [FeatureEvaluationResult.DEFAULT_ROW_INDEX_LABEL_DESC, self.samples.desc, self.positive_samples.desc, self.ks.desc, self.auc.desc] + + def to_dict(self): + return {FeatureEvaluationResult.DEFAULT_ROW_INDEX_LABEL_DESC: self.type_desc, + self.samples.desc: self.samples.value, + self.positive_samples.desc: self.positive_samples.value, + self.ks.desc: self.ks.value, + self.auc.desc: self.auc.value} + + @staticmethod + def summary(evaluation_result_list): + if evaluation_result_list is None or len(evaluation_result_list) == 0: + return None + columns = None + rows = [] + for evaluation_metric in evaluation_result_list: + rows.append(evaluation_metric.to_dict()) + if columns is None: + columns = evaluation_metric.columns() + return pd.DataFrame(rows, columns=columns) + + @staticmethod + def store_and_upload_summary(evaluation_result_list, local_file_path, remote_file_path, storage_client): + df = FeatureEvaluationResult.summary(evaluation_result_list) + df.to_csv(local_file_path, index=False) + if storage_client is not None: + storage_client.upload_file(local_file_path, remote_file_path) diff --git a/python/ppc_model/secure_lgbm/monitor/feature/test/__init__.py b/python/ppc_model/secure_lgbm/monitor/feature/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lgbm/monitor/feature/test/feature_evalution_info_test.py b/python/ppc_model/secure_lgbm/monitor/feature/test/feature_evalution_info_test.py new file mode 100644 index 00000000..ff6f0061 --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/feature/test/feature_evalution_info_test.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +import unittest +import numpy as np +from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import FeatureEvaluationResult +from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import EvaluationMetric +from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import EvaluationType + + +def check_result(ut_obj, evaluation_result, expected_ks, expected_auc, expected_sample): + ut_obj.assertEqual(evaluation_result.ks.value, expected_ks) + ut_obj.assertEqual(evaluation_result.auc.value, expected_auc) + ut_obj.assertEqual(evaluation_result.samples.value, expected_sample) + + +class TestFeatureEvaluationResult(unittest.TestCase): + def test_default_table_meta(self): + sample_num = 1000000 + train_label_list = np.random.randint(0, 2, sample_num) + train_evaluation_result = FeatureEvaluationResult( + type=EvaluationType.TRAIN, label_list=train_label_list) + ks_value = 0.4126 + auc_value = 0.7685 + (train_evaluation_result.ks.value, + train_evaluation_result.auc.value) = (ks_value, auc_value) + check_result(self, train_evaluation_result, + ks_value, auc_value, sample_num) + + sample_num = 2000000 + validation_label_list = np.random.randint(0, 2, sample_num) + validation_evaluation_result = FeatureEvaluationResult( + type=EvaluationType.VALIDATION, label_list=validation_label_list) + ks_value = 0.3116 + auc_value = 0.6676 + (validation_evaluation_result.ks.value, + validation_evaluation_result.auc.value) = (ks_value, auc_value) + check_result(self, validation_evaluation_result, + ks_value, auc_value, sample_num) + + local_path = "evaluation_result_case_1.csv" + FeatureEvaluationResult.store_and_upload_summary( + [train_evaluation_result, validation_evaluation_result], local_path, None, None) + + def test_with_given_table_meta(self): + ks_value = 0.4126 + auc_value = 0.7685 + sample_num = 1000000 + train_label_list = np.random.randint(0, 2, sample_num) + train_evaluation_result = FeatureEvaluationResult( + type=EvaluationType.TRAIN, label_list=train_label_list, ks_value=ks_value, auc_value=auc_value) + check_result(self, train_evaluation_result, + ks_value, auc_value, sample_num) + train_evaluation_result.ks.desc = "KS值" + train_evaluation_result.auc.desc = "AUC值" + + ks_value = 0.3116 + auc_value = 0.6676 + sample_num = 2000000 + validation_label_list = np.random.randint(0, 2, sample_num) + validation_evaluation_result = FeatureEvaluationResult( + type=EvaluationType.VALIDATION, label_list=validation_label_list, ks_value=ks_value, auc_value=auc_value) + validation_evaluation_result.ks.desc = "KS值" + validation_evaluation_result.auc.desc = "AUC值" + check_result(self, validation_evaluation_result, + ks_value, auc_value, sample_num) + + local_path = "evaluation_result_case_2.csv" + + FeatureEvaluationResult.store_and_upload_summary( + [train_evaluation_result, validation_evaluation_result], local_path, None, None) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py b/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py new file mode 100644 index 00000000..6429807f --- /dev/null +++ b/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py @@ -0,0 +1,105 @@ +import time +import unittest + +import numpy as np + +from ppc_model_service import config +from ppc_model.secure_lgbm.monitor.callback import CallbackContainer +from ppc_model.secure_lgbm.monitor.core import Booster, fevaluation +from ppc_model.secure_lgbm.monitor.early_stopping import EarlyStopping +from ppc_model.secure_lgbm.monitor.evaluation_monitor import EvaluationMonitor + +log = config.get_logger() + + +class TestBooster(unittest.TestCase): + def setUp(self): + np.random.seed(int(time.time())) + self.y_true = np.random.randint(0, 2, 10000) + self.test_y_true = np.random.randint(0, 2, 10000) + self.y_pred = np.random.rand(10000) + self.booster = Booster(self.y_true, self.test_y_true) + + def test_set_get_param(self): + self.booster.set_param('learning_rate', 0.1) + self.assertEqual(self.booster.get_param('learning_rate'), 0.1) + + def test_after_iteration(self): + self.booster.after_iteration(self.y_pred, False) + np.testing.assert_array_equal(self.booster.get_y_pred(), self.y_pred) + self.assertEqual(self.booster.get_epoch(), 1) + + def test_eval(self): + self.booster.after_iteration(self.y_pred) + results = self.booster.eval(fevaluation) + self.assertIn('auc', results) + self.assertIn('acc', results) + self.assertIn('recall', results) + self.assertIn('precision', results) + self.assertIsInstance(results['auc'], float) + self.assertIsInstance(results['acc'], float) + self.assertIsInstance(results['recall'], float) + self.assertIsInstance(results['precision'], float) + + +class TestEarlyStopping(unittest.TestCase): + def setUp(self): + np.random.seed(int(time.time())) + self.y_true = np.random.randint(0, 2, 10000) + self.test_y_true = np.random.randint(0, 2, 10000) + self.y_pred = np.random.rand(10000) + self.model = Booster(self.y_true, self.test_y_true) + self.early_stopping = EarlyStopping(rounds=4, metric_name='auc', maximize=True) + + def test_early_stopping(self): + stop = False + while not stop: + np.random.seed(int(time.time()) + self.model.epoch) + y_pred = np.random.rand(10000) + self.model.after_iteration(y_pred) + self.model.eval(fevaluation) + stop = self.early_stopping.after_iteration(self.model, self.model.epoch) + print(self.model.epoch, stop) + + +class TestEvaluationMonitor(unittest.TestCase): + def setUp(self): + np.random.seed(int(time.time())) + self.y_true = np.random.randint(0, 2, 10000) + self.test_y_true = np.random.randint(0, 2, 10000) + self.y_pred = np.random.rand(10000) + self.model = Booster(self.y_true, self.test_y_true, '/tmp/') + self.monitor = EvaluationMonitor(log, period=2) + + def test_after_training(self): + np.random.seed(int(time.time())) + for i in range(10): + np.random.seed(int(time.time()) + self.model.epoch) + y_pred = np.random.rand(10000) + self.model.after_iteration(y_pred) + self.model.eval(fevaluation) + self.monitor.after_training(self.model) + + +class TestCallbackContainer(unittest.TestCase): + def setUp(self): + np.random.seed(int(time.time())) + self.y_true = np.random.randint(0, 2, 10000) + self.test_y_true = np.random.randint(0, 2, 10000) + self.y_pred = np.random.rand(10000) + self.model = Booster(self.y_true, self.test_y_true, 'tmp') + self.early_stopping = EarlyStopping(rounds=4, metric_name='auc', maximize=True) + self.monitor = EvaluationMonitor(log, period=2) + self.container = CallbackContainer([self.early_stopping, self.monitor], fevaluation) + + def test_callback_container(self): + stop = False + while not stop: + np.random.seed(int(time.time()) + self.model.epoch) + y_pred = np.random.rand(10000) + stop = self.container.after_iteration(self.model, y_pred, True) + print(self.model.epoch, stop) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py new file mode 100644 index 00000000..d6a52db8 --- /dev/null +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -0,0 +1,254 @@ +import os +from enum import Enum +from typing import Any, Dict +from sklearn.base import BaseEstimator + +from ppc_common.ppc_crypto.phe_factory import PheCipherFactory +from ppc_model.common.context import Context +from ppc_model.common.initializer import Initializer +from ppc_model.common.protocol import TaskRole +from ppc_common.ppc_utils import common_func +from ppc_model.common.model_setting import ModelSetting + + +class LGBMModel(BaseEstimator): + + def __init__( + self, + boosting_type: str = 'gbdt', + num_leaves: int = 31, + max_depth: int = -1, + learning_rate: float = 0.1, + n_estimators: int = 100, + subsample_for_bin: int = 200000, + objective: str = None, + min_split_gain: float = 0., + min_child_weight: float = 1e-3, + min_child_samples: int = 20, + subsample: float = 1., + subsample_freq: int = 0, + colsample_bytree: float = 1., + reg_alpha: float = 0., + reg_lambda: float = 0., + random_state: int = None, + n_jobs: int = None, + importance_type: str = 'split', + **kwargs + ): + + self.boosting_type = boosting_type + self.objective = objective + self.num_leaves = num_leaves + self.max_depth = max_depth + self.learning_rate = learning_rate + self.n_estimators = n_estimators + self.subsample_for_bin = subsample_for_bin + self.min_split_gain = min_split_gain + self.min_child_weight = min_child_weight + self.min_child_samples = min_child_samples + self.subsample = subsample + self.subsample_freq = subsample_freq + self.colsample_bytree = colsample_bytree + self.reg_alpha = reg_alpha + self.reg_lambda = reg_lambda + self.random_state = random_state + self.n_jobs = n_jobs + self.importance_type = importance_type + self._other_params: Dict[str, Any] = {} + self.set_params(**kwargs) + + def get_params(self, deep: bool = True) -> Dict[str, Any]: + """Get parameters for this estimator. + + Parameters + ---------- + deep : bool, optional (default=True) + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : dict + Parameter names mapped to their values. + """ + params = super().get_params(deep=deep) + params.update(self._other_params) + return params + + def set_model_setting(self, model_setting: ModelSetting) -> "LGBMModel": + # 获取对象的所有属性名 + attrs = dir(model_setting) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + setattr(self, attr, getattr(model_setting, attr)) + except Exception as e: + pass + return self + + def set_params(self, **params: Any) -> "LGBMModel": + """Set the parameters of this estimator. + + Parameters + ---------- + **params + Parameter names with their new values. + + Returns + ------- + self : object + Returns self. + """ + for key, value in params.items(): + setattr(self, key, value) + if hasattr(self, f"_{key}"): + setattr(self, f"_{key}", value) + self._other_params[key] = value + return self + + +class ModelTaskParams(LGBMModel): + def __init__( + self, + test_size: float = 0.3, + max_bin: int = 10, + use_goss: bool = False, + top_rate: float = 0.2, + other_rate: float = 0.1, + feature_rate: float = 1.0, + colsample_bylevel: float = 1.0, + gamma: float = 0, + loss_type: str = 'logistic', + eval_set_column: str = None, + train_set_value: str = None, + eval_set_value: str = None, + train_feats: str = None, + early_stopping_rounds: int = 5, + eval_metric: str = 'auc', + verbose_eval: int = 1, + categorical_feature: list = [], + silent: bool = False + ): + + super().__init__() + + self.test_size = test_size + self.max_bin = max_bin + self.use_goss = use_goss + self.top_rate = top_rate + self.other_rate = other_rate + self.feature_rate = feature_rate + self.colsample_bylevel = colsample_bylevel + self.gamma = gamma + self.loss_type = loss_type + self.eval_set_column = eval_set_column + self.train_set_value = train_set_value + self.eval_set_value = eval_set_value + self.train_feature = train_feats + self.early_stopping_rounds = early_stopping_rounds + self.eval_metric = eval_metric + self.verbose_eval = verbose_eval + self.silent = silent + self.λ = self.reg_lambda + self.lr = self.learning_rate + self.categorical_feature = categorical_feature + self.categorical_idx = [] + self.my_categorical_idx = [] + + +class SecureLGBMParams(ModelTaskParams): + + def __init__(self): + super().__init__() + + def _get_params(self): + """返回LGBMClassifier所有参数""" + return LGBMModel().get_params() + + def get_all_params(self): + """返回SecureLGBMParams所有参数""" + # 获取对象的所有属性名 + attrs = dir(self) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + # 使用getattr来获取属性的值 + value = getattr(self, attr) + # 检查value是否可调用（例如，方法或函数），如果是，则不打印其值 + if not callable(value): + params[attr] = value + except Exception as e: + pass + return params + + +class SecureLGBMContext(Context): + + def __init__(self, + args, + components: Initializer + ): + + if args['is_label_holder']: + role = TaskRole.ACTIVE_PARTY + else: + role = TaskRole.PASSIVE_PARTY + + super().__init__(args['job_id'], + args['task_id'], + components, + role) + + self.phe = PheCipherFactory.build_phe( + components.homo_algorithm, components.public_key_length) + self.codec = PheCipherFactory.build_codec(components.homo_algorithm) + self.is_label_holder = args['is_label_holder'] + self.result_receiver_id_list = args['result_receiver_id_list'] + self.participant_id_list = args['participant_id_list'] + self.model_predict_algorithm = common_func.get_config_value( + "model_predict_algorithm", None, args, False) + self.algorithm_type = args['algorithm_type'] + if 'dataset_id' in args and args['dataset_id'] is not None: + self.dataset_file_path = os.path.join( + self.workspace, args['dataset_id']) + else: + self.dataset_file_path = None + + self.lgbm_params = SecureLGBMParams() + model_setting = ModelSetting(args['model_dict']) + self.set_lgbm_params(model_setting) + if model_setting.train_features is not None and len(model_setting.train_features) > 0: + self.lgbm_params.train_feature = model_setting.train_features.split( + ',') + self.lgbm_params.n_estimators = model_setting.num_trees + self.lgbm_params.feature_rate = model_setting.colsample_bytree + self.lgbm_params.min_split_gain = model_setting.gamma + self.lgbm_params.random_state = model_setting.seed + + def set_lgbm_params(self, model_setting: ModelSetting): + """设置lgbm参数""" + self.lgbm_params.set_model_setting(model_setting) + + def get_lgbm_params(self): + """获取lgbm参数""" + return self.lgbm_params + + +class LGBMMessage(Enum): + FEATURE_NAME = "FEATURE_NAME" + INSTANCE = "INSTANCE" + ENC_GH_LIST = "ENC_GH_LIST" + ENC_GH_HIST = "ENC_GH_HIST" + SPLIT_INFO = 'SPLIT_INFO' + INSTANCE_MASK = "INSTANCE_MASK" + PREDICT_LEAF_MASK = "PREDICT_LEAF_MASK" + TEST_LEAF_MASK = "PREDICT_TEST_LEAF_MASK" + VALID_LEAF_MASK = "PREDICT_VALID_LEAF_MASK" + STOP_ITERATION = "STOP_ITERATION" + PREDICT_PRABA = "PREDICT_PRABA" diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py new file mode 100644 index 00000000..1f481a78 --- /dev/null +++ b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py @@ -0,0 +1,38 @@ +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.protocol import TaskRole, ModelTask +from ppc_model.common.global_context import components +from ppc_model.interface.task_engine import TaskEngine +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +class SecureLGBMPredictionEngine(TaskEngine): + task_type = ModelTask.XGB_PREDICTING + + @staticmethod + def run(args): + + task_info = SecureLGBMContext(args, components) + secure_dataset = SecureDataset(task_info) + + if task_info.role == TaskRole.ACTIVE_PARTY: + booster = VerticalLGBMActiveParty(task_info, secure_dataset) + elif task_info.role == TaskRole.PASSIVE_PARTY: + booster = VerticalLGBMPassiveParty(task_info, secure_dataset) + else: + raise PpcException(PpcErrorCode.ROLE_TYPE_ERROR.get_code(), + PpcErrorCode.ROLE_TYPE_ERROR.get_message()) + + booster.load_model() + booster.predict() + + # 获取测试集的预测概率值 + test_praba = booster.get_test_praba() + + # 获取测试集的预测值评估指标 + Evaluation(task_info, secure_dataset, test_praba=test_praba) + + ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py b/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py new file mode 100644 index 00000000..4fe502c8 --- /dev/null +++ b/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py @@ -0,0 +1,40 @@ +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.protocol import TaskRole, ModelTask +from ppc_model.common.global_context import components +from ppc_model.interface.task_engine import TaskEngine +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +class SecureLGBMTrainingEngine(TaskEngine): + task_type = ModelTask.XGB_TRAINING + + @staticmethod + def run(args): + + task_info = SecureLGBMContext(args, components) + secure_dataset = SecureDataset(task_info) + + if task_info.role == TaskRole.ACTIVE_PARTY: + booster = VerticalLGBMActiveParty(task_info, secure_dataset) + elif task_info.role == TaskRole.PASSIVE_PARTY: + booster = VerticalLGBMPassiveParty(task_info, secure_dataset) + else: + raise PpcException(PpcErrorCode.ROLE_TYPE_ERROR.get_code(), + PpcErrorCode.ROLE_TYPE_ERROR.get_message()) + + booster.fit() + booster.save_model() + + # 获取训练集和验证集的预测概率值 + train_praba = booster.get_train_praba() + test_praba = booster.get_test_praba() + + # 获取训练集和验证集的预测值评估指标 + Evaluation(task_info, secure_dataset, train_praba, test_praba) + ModelPlot(booster) + ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lgbm/test/__init__.py b/python/ppc_model/secure_lgbm/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lgbm/test/test_cipher_packing.py b/python/ppc_model/secure_lgbm/test/test_cipher_packing.py new file mode 100644 index 00000000..1f205a71 --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_cipher_packing.py @@ -0,0 +1,61 @@ +import time +import unittest +import numpy as np + +from ppc_common.ppc_crypto.paillier_cipher import PaillierCipher +from ppc_common.ppc_crypto.paillier_codec import PaillierCodec +from ppc_common.ppc_protos.generated.ppc_model_pb2 import CipherList, ModelCipher + + +paillier = PaillierCipher() + + +class TestCipherPacking: + + def test_cipher_list(self): + + data_list = np.random.randint(1, 10001, size=1000) + + start_time = time.time() + ciphers = paillier.encrypt_batch_parallel(data_list) + # ciphers = paillier.encrypt_batch(data_list) + print("enc:", time.time() - start_time, "seconds") + + start_time = time.time() + enc_data_pb = CipherList() + enc_data_pb.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + for cipher in ciphers: + paillier_cipher = ModelCipher() + paillier_cipher.ciphertext, paillier_cipher.exponent = PaillierCodec.encode_cipher(cipher) + enc_data_pb.cipher_list.append(paillier_cipher) + print("pack ciphers:", time.time() - start_time, "seconds") + + ciphers2 = [] + for i in range(100): + ciphers2.append(np.array(ciphers[10*i:10*(i+1)]).sum()) + + start_time = time.time() + enc_data_pb2 = CipherList() + enc_data_pb2.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + for cipher in ciphers2: + paillier_cipher2 = ModelCipher() + paillier_cipher2.ciphertext, paillier_cipher2.exponent = PaillierCodec.encode_cipher(cipher, be_secure=False) + enc_data_pb2.cipher_list.append(paillier_cipher2) + print("pack ciphers:", time.time() - start_time, "seconds") + + ciphers3 = [] + for i in range(100): + ciphers3.append(np.array(ciphers[10*i:10*(i+1)]).sum()) + + start_time = time.time() + enc_data_pb3 = CipherList() + enc_data_pb3.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + for cipher in ciphers3: + paillier_cipher3 = ModelCipher() + paillier_cipher3.ciphertext, paillier_cipher3.exponent = PaillierCodec.encode_cipher(cipher) + enc_data_pb3.cipher_list.append(paillier_cipher3) + print("pack ciphers:", time.time() - start_time, "seconds") + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/test/test_pack_gh.py b/python/ppc_model/secure_lgbm/test/test_pack_gh.py new file mode 100644 index 00000000..dba82280 --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_pack_gh.py @@ -0,0 +1,43 @@ +import unittest +import numpy as np + +from ppc_model.secure_lgbm.vertical.booster import VerticalBooster + + +class TestPackGH(unittest.TestCase): + + def test_pack_gh(self): + + g_list = np.array([-4, 2, -1.3, 0, 0, -15.3544564544]) + h_list = np.array([2, 1.5, -1.4, 0, -1.68, 1.2356564564]) + + gh_list = VerticalBooster.packing_gh(g_list, h_list) + + result_array = np.array( + [429496329600000000000000002000, 200000000000000000001500, + 429496599600000000004294965896, 0, + 4294965616, 429495194200000000000000001235], dtype=object) + + assert np.array_equal(gh_list, result_array) + + def test_unpack_gh(self): + + gh_list = np.array( + [429496329600000000000000002000, 200000000000000000001500, + 429496599600000000004294965896, 0, + 4294965616, 429495194200000000000000001235], dtype=object) + + gh_sum_list = np.array([sum(gh_list), sum(gh_list)*2]) + g_hist, h_hist = VerticalBooster.unpacking_gh(gh_sum_list) + + g_list = np.array([-4, 2, -1.3, 0, 0, -15.3544564544]) + h_list = np.array([2, 1.5, -1.4, 0, -1.68, 1.2356564564]) + result_g_hist = np.array([-18.654, -37.308]) + result_h_hist = np.array([1.655, 3.31]) + + assert np.array_equal(g_hist, result_g_hist) + assert np.array_equal(h_hist, result_h_hist) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/test/test_save_load_model.py b/python/ppc_model/secure_lgbm/test/test_save_load_model.py new file mode 100644 index 00000000..a899b805 --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_save_load_model.py @@ -0,0 +1,92 @@ +import unittest +import numpy as np + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical.booster import VerticalBooster + + +class TestSaveLoadModel(unittest.TestCase): + + n_estimators = 2 + max_depth = 3 + np.random.seed(2024) + + ACTIVE_PARTY = 'ACTIVE_PARTY' + PASSIVE_PARTY = 'PASSIVE_PARTY' + + job_id = 'j-123' + task_id = 't-123' + + model_dict = {} + + args = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + components = Initializer(log_config_path='', config_path='') + components.config_data = {'JOB_TEMP_DIR': '/tmp'} + components.mock_logger = MockLogger() + + def test_save_load_model(self): + + x_split = [[1, 2, 3, 5], [1, 2], [1.23, 3.45, 5.23]] + + trees = [] + for i in range(self.n_estimators): + tree = self._build_tree(self.max_depth) + trees.append(tree) + # print(trees) + + task_info = SecureLGBMContext(self.args, self.components) + booster = VerticalBooster(task_info, dataset=None) + booster._X_split = x_split + booster._trees = trees + booster.save_model() + + booster_predict = VerticalBooster(task_info, dataset=None) + booster_predict.load_model() + + assert x_split == booster_predict._X_split + assert trees == booster_predict._trees + + @staticmethod + def _build_tree(max_depth, depth=0, weight=0): + + if depth == max_depth: + return weight + + best_split_info = BestSplitInfo( + feature=np.random.randint(0,10), + value=np.random.randint(0,4), + best_gain=np.random.rand(), + w_left=np.random.rand(), + w_right=np.random.rand(), + agency_idx=np.random.randint(0,2), + agency_feature=np.random.randint(0,5) + ) + # print(best_split_info) + + if best_split_info.best_gain > 0.2: + left_tree = TestSaveLoadModel._build_tree( + max_depth, depth + 1, best_split_info.w_left) + right_tree = TestSaveLoadModel._build_tree( + max_depth, depth + 1, best_split_info.w_right) + + return [(best_split_info, left_tree, right_tree)] + else: + return weight + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py new file mode 100644 index 00000000..2797194c --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py @@ -0,0 +1,77 @@ +import unittest + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext + + +class TestSecureLGBMContext(unittest.TestCase): + + components = Initializer(log_config_path='', config_path='') + components.config_data = {'JOB_TEMP_DIR': '/tmp'} + components.mock_logger = MockLogger() + + def test_get_lgbm_params(self): + + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': None, + 'algorithm_subtype': None, + 'model_dict': {} + } + + task_info = SecureLGBMContext(args, self.components) + lgbm_params = task_info.get_lgbm_params() + # 打印LGBMModel默认参数 + print(lgbm_params._get_params()) + + # 默认自定义参数为空字典 + assert lgbm_params.get_params() == {} + # assert lgbm_params.get_all_params() != lgbm_params._get_params() + + def test_set_lgbm_params(self): + + args = { + 'job_id': 'j-123', + 'task_id': '1', + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': None, + 'algorithm_subtype': None, + 'model_dict': { + 'objective': 'regression', + 'n_estimators': 6, + 'max_depth': 3, + 'test_size': 0.2, + 'use_goss': 1 + } + } + + task_info = SecureLGBMContext(args, self.components) + lgbm_params = task_info.get_lgbm_params() + # 打印SecureLGBMParams自定义参数 + print(lgbm_params.get_params()) + # 打印SecureLGBMParams所有参数 + print(lgbm_params.get_all_params()) + + assert lgbm_params.get_params() == args['model_dict'] + self.assertEqual(lgbm_params.get_all_params()[ + 'learning_rate'], lgbm_params._get_params()['learning_rate']) + self.assertEqual(lgbm_params.learning_rate, + lgbm_params._get_params()['learning_rate']) + self.assertEqual(lgbm_params.n_estimators, + args['model_dict']['n_estimators']) + self.assertEqual(lgbm_params.test_size, + args['model_dict']['test_size']) + self.assertEqual(lgbm_params.use_goss, args['model_dict']['use_goss']) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py new file mode 100644 index 00000000..936a53ac --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py @@ -0,0 +1,172 @@ +import unittest +import threading +import traceback + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.network.stub import ModelStub +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + +data_size = 1000 +feature_dim = 20 + + +def mock_args(): + job_id = 'j-111' + task_id = 't-111' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'max_bin': 10, + 'n_estimators': 2, + 'max_depth': 3, + 'feature_rate': 1.0, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + return args_a, args_b + + +class TestXgboostTraining(unittest.TestCase): + + def setUp(self): + self._active_rpc_client = RpcClientMock() + self._passive_rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._active_stub = ModelStub( + agency_id=ACTIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._active_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._passive_stub = ModelStub( + agency_id=PASSIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._passive_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._active_rpc_client.set_message_handler(self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler(self._active_stub.on_message_received) + + def test_fit(self): + args_a, args_b = mock_args() + + active_components = Initializer(log_config_path='', config_path='') + active_components.stub = self._active_stub + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLGBMContext(args_a, active_components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + secure_dataset_a = SecureDataset(task_info_a, model_data) + booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) + print(secure_dataset_a.feature_name) + print(secure_dataset_a.train_idx.shape) + print(secure_dataset_a.train_X.shape) + print(secure_dataset_a.train_y.shape) + print(secure_dataset_a.test_idx.shape) + print(secure_dataset_a.test_X.shape) + print(secure_dataset_a.test_y.shape) + + passive_components = Initializer(log_config_path='', config_path='') + passive_components.stub = self._passive_stub + passive_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} + passive_components.mock_logger = MockLogger() + task_info_b = SecureLGBMContext(args_b, passive_components) + model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=False) + secure_dataset_b = SecureDataset(task_info_b, model_data) + booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) + print(secure_dataset_b.feature_name) + print(secure_dataset_b.train_idx.shape) + print(secure_dataset_b.train_X.shape) + print(secure_dataset_b.test_idx.shape) + print(secure_dataset_b.test_X.shape) + + def active_worker(): + try: + booster_a.fit() + booster_a.save_model() + train_praba = booster_a.get_train_praba() + test_praba = booster_a.get_test_praba() + Evaluation(task_info_a, secure_dataset_a, train_praba, test_praba) + # ModelPlot(booster_a) + ResultFileHandling(task_info_a) + booster_a.load_model() + booster_a.predict() + test_praba = booster_a.get_test_praba() + task_info_a.algorithm_type = 'PPC_PREDICT' + Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) + ResultFileHandling(task_info_a) + except Exception as e: + task_info_a.components.logger().info(traceback.format_exc()) + + def passive_worker(): + try: + booster_b.fit() + booster_b.save_model() + train_praba = booster_b.get_train_praba() + test_praba = booster_b.get_test_praba() + Evaluation(task_info_b, secure_dataset_b, train_praba, test_praba) + # ModelPlot(booster_b) + ResultFileHandling(task_info_b) + booster_b.load_model() + booster_b.predict() + test_praba = booster_b.get_test_praba() + task_info_b.algorithm_type = 'PPC_PREDICT' + Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) + ResultFileHandling(task_info_b) + except Exception as e: + task_info_b.components.logger().info(traceback.format_exc()) + + thread_lgbm_active = threading.Thread(target=active_worker, args=()) + thread_lgbm_active.start() + + thread_lgbm_passive = threading.Thread(target=passive_worker, args=()) + thread_lgbm_passive.start() + + thread_lgbm_active.join() + thread_lgbm_passive.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py new file mode 100644 index 00000000..7cddb6db --- /dev/null +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py @@ -0,0 +1,178 @@ +import unittest +import threading +import traceback +from sklearn.datasets import load_breast_cancer + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.network.stub import ModelStub +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + + +def mock_args(): + job_id = 'j-123' + task_id = 't-123' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'max_bin': 10, + 'n_estimators': 2, + 'max_depth': 3, + 'use_goss': 1, + 'feature_rate': 0.8, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + return args_a, args_b + + +class TestXgboostTraining(unittest.TestCase): + + cancer = load_breast_cancer() + X = cancer.data + y = cancer.target + + df = SecureDataset.assembling_dataset(X, y) + df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df) + + def setUp(self): + self._active_rpc_client = RpcClientMock() + self._passive_rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._active_stub = ModelStub( + agency_id=ACTIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._active_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._passive_stub = ModelStub( + agency_id=PASSIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._passive_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._active_rpc_client.set_message_handler( + self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler( + self._active_stub.on_message_received) + + def test_fit(self): + args_a, args_b = mock_args() + + active_components = Initializer(log_config_path='', config_path='') + active_components.stub = self._active_stub + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLGBMContext(args_a, active_components) + secure_dataset_a = SecureDataset(task_info_a, self.df_with_y) + booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) + print(secure_dataset_a.feature_name) + print(secure_dataset_a.train_idx.shape) + print(secure_dataset_a.train_X.shape) + print(secure_dataset_a.train_y.shape) + print(secure_dataset_a.test_idx.shape) + print(secure_dataset_a.test_X.shape) + print(secure_dataset_a.test_y.shape) + + passive_components = Initializer(log_config_path='', config_path='') + passive_components.stub = self._passive_stub + passive_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} + passive_components.mock_logger = MockLogger() + task_info_b = SecureLGBMContext(args_b, passive_components) + secure_dataset_b = SecureDataset(task_info_b, self.df_without_y) + booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) + print(secure_dataset_b.feature_name) + print(secure_dataset_b.train_idx.shape) + print(secure_dataset_b.train_X.shape) + print(secure_dataset_b.test_idx.shape) + print(secure_dataset_b.test_X.shape) + + def active_worker(): + try: + booster_a.fit() + booster_a.save_model() + train_praba = booster_a.get_train_praba() + test_praba = booster_a.get_test_praba() + Evaluation(task_info_a, secure_dataset_a, train_praba, test_praba) + ModelPlot(booster_a) + ResultFileHandling(task_info_a) + booster_a.load_model() + booster_a.predict() + test_praba = booster_a.get_test_praba() + task_info_a.algorithm_type = 'PPC_PREDICT' + Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) + ResultFileHandling(task_info_a) + except Exception as e: + task_info_a.components.logger().info(traceback.format_exc()) + + def passive_worker(): + try: + booster_b.fit() + booster_b.save_model() + train_praba = booster_b.get_train_praba() + test_praba = booster_b.get_test_praba() + Evaluation(task_info_b, secure_dataset_b, train_praba, test_praba) + ModelPlot(booster_b) + ResultFileHandling(task_info_b) + booster_b.load_model() + booster_b.predict() + test_praba = booster_b.get_test_praba() + task_info_b.algorithm_type = 'PPC_PREDICT' + Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) + ResultFileHandling(task_info_b) + except Exception as e: + task_info_b.components.logger().info(traceback.format_exc()) + + thread_lgbm_active = threading.Thread(target=active_worker, args=()) + thread_lgbm_active.start() + + thread_lgbm_passive = threading.Thread(target=passive_worker, args=()) + thread_lgbm_passive.start() + + thread_lgbm_active.join() + thread_lgbm_passive.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lgbm/vertical/__init__.py b/python/ppc_model/secure_lgbm/vertical/__init__.py new file mode 100644 index 00000000..7a33ec6e --- /dev/null +++ b/python/ppc_model/secure_lgbm/vertical/__init__.py @@ -0,0 +1,4 @@ +from ppc_model.secure_lgbm.vertical.active_party import VerticalLGBMActiveParty +from ppc_model.secure_lgbm.vertical.passive_party import VerticalLGBMPassiveParty + +__all__ = ["VerticalLGBMActiveParty", "VerticalLGBMPassiveParty"] diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py new file mode 100644 index 00000000..524c8266 --- /dev/null +++ b/python/ppc_model/secure_lgbm/vertical/active_party.py @@ -0,0 +1,461 @@ +import itertools +import time + +import numpy as np +from pandas import DataFrame + +from ppc_common.deps_services.serialize_type import SerializeType +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceStore +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceType +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo, IterationRequest +from ppc_common.ppc_utils import utils +from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection +from ppc_model.datasets.data_reduction.sampling import Sampling +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.loss import BinaryLoss +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext, LGBMMessage +from ppc_model.secure_lgbm.monitor.callback import CallbackContainer +from ppc_model.secure_lgbm.monitor.core import Booster +from ppc_model.secure_lgbm.monitor.early_stopping import EarlyStopping +from ppc_model.secure_lgbm.monitor.evaluation_monitor import EvaluationMonitor +from ppc_model.secure_lgbm.vertical.booster import VerticalBooster + + +class VerticalLGBMActiveParty(VerticalBooster): + + def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: + super().__init__(ctx, dataset) + self.params = ctx.lgbm_params + self._loss_func = BinaryLoss(self.params.objective) + self._all_feature_name = [dataset.feature_name] + self._all_feature_num = len(dataset.feature_name) + self.log = ctx.components.logger() + self.storage_client = ctx.components.storage_client + self.feature_importance_store = FeatureImportanceStore( + FeatureImportanceStore.DEFAULT_IMPORTANCE_LIST, None, self.log) + self.log.info(f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + + def fit( + self, + *args, + **kwargs, + ) -> None: + self.log.info( + f'task {self.ctx.task_id}: Starting the lgbm on the active party.') + self._init_active_data() + self._init_valid_data() + self._init_early_stop() + + for _ in range(self.params.n_estimators): + self._tree_id += 1 + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in active party.') + + # 初始化 + feature_select, instance, used_glist, used_hlist = self._init_each_tree() + self.log.info(f'task {self.ctx.task_id}: Sampling number: {len(instance)}, ' + f'feature select: {len(feature_select)}, {feature_select}.') + + # 构建 + tree = self._build_tree( + feature_select, instance, used_glist, used_hlist) + self._trees.append(tree) + # print('tree', tree) + + # 预测 + self._train_weights += self._predict_tree( + tree, self._X_bin, np.ones(self._X_bin.shape[0], dtype=bool), LGBMMessage.PREDICT_LEAF_MASK.value) + self._train_praba = self._loss_func.sigmoid(self._train_weights) + # print('train_praba', set(self._train_praba)) + + # 评估 + if not self.params.silent and self.dataset.train_y is not None: + auc = Evaluation.fevaluation(self.dataset.train_y, self._train_praba)['auc'] + self.log.info(f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, auc: {auc}.') + self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id}, ' + f'time_costs: {time.time() - start_time}s.') + + # 预测验证集 + self._test_weights += self._predict_tree( + tree, self._test_X_bin, np.ones(self._test_X_bin.shape[0], dtype=bool), + LGBMMessage.TEST_LEAF_MASK.value) + self._test_praba = self._loss_func.sigmoid(self._test_weights) + if not self.params.silent and self.dataset.test_y is not None: + auc = Evaluation.fevaluation(self.dataset.test_y, self._test_praba)['auc'] + self.log.info(f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, test auc: {auc}.') + if self._iteration_early_stop(): + self.log.info(f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") + break + + self._end_active_data() + + def transform(self, transform_data: DataFrame) -> DataFrame: + ... + + def predict(self, dataset: SecureDataset = None) -> np.ndarray: + start_time = time.time() + if dataset is None: + dataset = self.dataset + + self.params.my_categorical_idx = self._get_categorical_idx( + dataset.feature_name, self.params.categorical_feature) + + test_weights = self._init_weight(dataset.test_X.shape[0]) + test_X_bin = self._split_test_data(self.ctx, dataset.test_X, self._X_split) + + for tree in self._trees: + test_weights += self._predict_tree( + tree, test_X_bin, np.ones(test_X_bin.shape[0], dtype=bool), LGBMMessage.VALID_LEAF_MASK.value) + test_praba = self._loss_func.sigmoid(test_weights) + self._test_praba = test_praba + + if dataset.test_y is not None: + auc = Evaluation.fevaluation(dataset.test_y, test_praba)['auc'] + self.log.info(f'task {self.ctx.task_id}: predict test auc: {auc}.') + self.log.info(f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + + self._end_active_data(is_train=False) + + def _init_active_data(self): + + # 初始化预测值和权重 + self._train_praba = self._init_praba(self.dataset.train_X.shape[0]) + self._train_weights = self._init_weight(self.dataset.train_X.shape[0]) + self._tree_id = 0 + + # 初始化所有参与方的特征 + for i in range(1, len(self.ctx.participant_id_list)): + feature_name_bytes = self._receive_byte_data(self.ctx, LGBMMessage.FEATURE_NAME.value, i) + self._all_feature_name.append([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + self._all_feature_num += len([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + + self.log.info(f'task {self.ctx.task_id}: total feature number:{self._all_feature_num}, ' + f'total feature name: {self._all_feature_name}.') + self.params.categorical_idx = self._get_categorical_idx( + list(itertools.chain(*self._all_feature_name)), self.params.categorical_feature) + self.params.my_categorical_idx = self._get_categorical_idx( + self.dataset.feature_name, self.params.categorical_feature) + + # 更新feature_importance中的特征列表 + self.feature_importance_store.set_init(list(itertools.chain(*self._all_feature_name))) + + # 初始化分桶数据集 + feat_bin = FeatureBinning(self.ctx) + self._X_bin, self._X_split = feat_bin.data_binning(self.dataset.train_X) + + def _init_each_tree(self): + + if self.callback_container: + self.callback_container.before_iteration(self.model) + + gradient = self._loss_func.compute_gradient(self.dataset.train_y, self._train_praba) + hessian = self._loss_func.compute_hessian(self._train_praba) + + feature_select = FeatureSelection.feature_selecting( + list(itertools.chain(*self._all_feature_name)), + self.params.train_feature, self.params.feature_rate) + instance, used_glist, used_hlist = Sampling.sample_selecting( + gradient, hessian, self.params.subsample, + self.params.use_goss, self.params.top_rate, self.params.other_rate) + self._send_gh_instance_list(instance, used_glist, used_hlist) + + return feature_select, instance, used_glist, used_hlist + + def _send_gh_instance_list(self, instance, glist, hlist): + + self._leaf_id = 0 + gh_list = self.packing_gh(glist, hlist) + + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} ' + f'encrypt g & h in active party.') + enc_ghlist = self.ctx.phe.encrypt_batch_parallel((gh_list).astype('object')) + self.log.info(f'task {self.ctx.task_id}: Finished n_estimators-{self._tree_id} ' + f'encrypt gradient & hessian time_costs: {time.time() - start_time}.') + + for partner_index in range(1, len(self.ctx.participant_id_list)): + self._send_byte_data(self.ctx, f'{LGBMMessage.INSTANCE.value}_{self._tree_id}', + instance.astype('int64').tobytes(), partner_index) + self._send_enc_data(self.ctx, f'{LGBMMessage.ENC_GH_LIST.value}_{self._tree_id}', + enc_ghlist, partner_index) + + def _build_tree(self, feature_select, instance, glist, hlist, depth=0, weight=0): + + if depth == self.params.max_depth: + return weight + if self.params.max_depth < 0 and self._leaf_id >= self.params.num_leaves: + return weight + + self._leaf_id += 1 + print('tree', self._tree_id, 'leaf', self._leaf_id, 'instance', len(instance), + 'glist', len(glist), 'hlist', len(hlist)) + if self.params.colsample_bylevel > 0 and self.params.colsample_bylevel < 1: + feature_select_level = sorted(np.random.choice( + feature_select, size=int(len(feature_select) * self.params.colsample_bylevel), replace=False)) + best_split_info = self._find_best_split(feature_select_level, instance, glist, hlist) + else: + best_split_info = self._find_best_split(feature_select, instance, glist, hlist) + + if best_split_info.best_gain > 0 and best_split_info.best_gain > self.params.min_split_gain: + gain_list = {FeatureImportanceType.GAIN: best_split_info.best_gain, + FeatureImportanceType.WEIGHT: 1} + self.feature_importance_store.update_feature_importance(best_split_info.feature, gain_list) + left_mask, right_mask = self._get_leaf_mask(best_split_info, instance) + + if (abs(best_split_info.w_left) * sum(left_mask) / self.params.lr) < self.params.min_child_weight or \ + (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: + return weight + if sum(left_mask) < self.params.min_child_samples or sum(right_mask) < self.params.min_child_samples: + return weight + + left_tree = self._build_tree( + feature_select, instance[left_mask], glist[left_mask], + hlist[left_mask], depth + 1, best_split_info.w_left) + right_tree = self._build_tree( + feature_select, instance[right_mask], glist[right_mask], + hlist[right_mask], depth + 1, best_split_info.w_right) + + return [(best_split_info, left_tree, right_tree)] + else: + return weight + + def _predict_tree(self, tree, X_bin, leaf_mask, key_type): + if not isinstance(tree, list): + return tree * leaf_mask + else: + best_split_info, left_subtree, right_subtree = tree[0] + if self.ctx.participant_id_list[best_split_info.agency_idx] == \ + self.ctx.components.config_data['AGENCY_ID']: + if best_split_info.agency_feature in self.params.my_categorical_idx: + left_mask = X_bin[:, best_split_info.agency_feature] == best_split_info.value + else: + left_mask = X_bin[:, best_split_info.agency_feature] <= best_split_info.value + else: + left_mask = np.frombuffer( + self._receive_byte_data( + self.ctx, + f'{key_type}_{best_split_info.tree_id}_{best_split_info.leaf_id}', + best_split_info.agency_idx), dtype='bool') + right_mask = ~left_mask + left_weight = self._predict_tree(left_subtree, X_bin, leaf_mask * left_mask, key_type) + right_weight = self._predict_tree(right_subtree, X_bin, leaf_mask * right_mask, key_type) + return left_weight + right_weight + + def _find_best_split(self, feature_select, instance, glist, hlist): + + self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} in active party.') + grad_hist, hess_hist = self._get_gh_hist(instance, glist, hlist) + best_split_info = self._get_best_split_point(feature_select, glist, hlist, grad_hist, hess_hist) + # print('grad_hist_sum', [sum(sublist) for sublist in grad_hist]) + + best_split_info.tree_id = self._tree_id + best_split_info.leaf_id = self._leaf_id + if best_split_info.best_gain > 0: + agency_idx, agency_feature = self._get_best_split_agency( + self._all_feature_name, best_split_info.feature) + best_split_info.agency_idx = agency_idx + best_split_info.agency_feature = agency_feature + + for partner_index in range(1, len(self.ctx.participant_id_list)): + self._send_byte_data( + ctx=self.ctx, + key_type=f'{LGBMMessage.SPLIT_INFO.value}_{self._tree_id}_{self._leaf_id}', + byte_data=utils.pb_to_bytes(best_split_info), + partner_index=partner_index) + self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} in active party.') + # print('best_split_info', best_split_info) + return best_split_info + + def _get_gh_hist(self, instance, glist, hlist): + ghist, hhist = self._calculate_hist(self._X_bin, instance, glist, hlist) + + for partner_index in range(1, len(self.ctx.participant_id_list)): + partner_feature_name = self._all_feature_name[partner_index] + + partner_ghist = [None] * len(partner_feature_name) + partner_hhist = [None] * len(partner_feature_name) + _, gh_hist = self._receive_enc_data( + self.ctx, f'{LGBMMessage.ENC_GH_HIST.value}_{self._tree_id}_{self._leaf_id}', + partner_index, matrix_data=True) + + for feature_index in range(len(partner_feature_name)): + ghk_hist = np.array(self.ctx.phe.decrypt_batch(gh_hist[feature_index]), dtype='object') + gk_hist, hk_hist = self.unpacking_gh(ghk_hist) + partner_ghist[feature_index] = gk_hist + partner_hhist[feature_index] = hk_hist + + ghist.extend(partner_ghist) + hhist.extend(partner_hhist) + + return ghist, hhist + + @staticmethod + def _calculate_hist(X_bin, instance, used_glist, used_hlist): + + g_hist = [] + h_hist = [] + for k in range(X_bin.shape[1]): + Xk_bin = X_bin[instance, k] + gk_hist = [] + hk_hist = [] + sorted_x = sorted(set(X_bin[:, k])) + for v in sorted_x: + gk_hist.append(used_glist[Xk_bin == v].sum()) + hk_hist.append(used_hlist[Xk_bin == v].sum()) + g_hist.append(gk_hist) + h_hist.append(hk_hist) + + return g_hist, h_hist + + def _get_best_split_point(self, feature_select, glist, hlist, grad_hist, hess_hist): + + beat_feature, best_value, best_gain, best_wl, best_wr = None, None, 0, None, None + g = np.sum(glist) + h = np.sum(hlist) + + for feature in feature_select: + gl = 0 + hl = 0 + for value in range(len(grad_hist[feature])): + gl, hl = self._compute_gh_sum( + feature, value, self.params.categorical_idx, gl, hl, grad_hist, hess_hist) + gr = g - gl + hr = h - hl + + gain = self._compute_gain(g, h, gl, hl, gr, hr, self.params.λ) + wl, wr = self._compute_leaf_weight( + self.params.lr, self.params.λ, gl, hl, gr, hr, self.params.reg_alpha) + compare = bool(gain > best_gain) + # print('f', feature, 'v', value, 'gl', gl, 'gr', gr, 'hl', hl, 'hr', hr, + # 'gain', gain, 'wl', wl, 'wr', wr, 'compare', compare) + + if compare: + beat_feature = feature + best_value = value + best_wl = wl + best_wr = wr + best_gain = gain + + return BestSplitInfo(feature=beat_feature, + value=best_value, + best_gain=best_gain, + w_left=best_wl, + w_right=best_wr) + + @staticmethod + def _get_best_split_agency(all_feature_name, feature): + """Get the agency index and feature index of the best split point. + + Parameters + ---------- + all_feature_name : two-dimensional list + Feature list of all participating agency. + feature : int + Best split point global feature index. + + Returns + ------- + sublist_index : int + Agency index. + position_in_sublist : int + Feature index. + """ + count = 0 + for sublist_index, sublist in enumerate(all_feature_name): + if count + len(sublist) > feature: + position_in_sublist = feature - count + return sublist_index, position_in_sublist + count += len(sublist) + return None + + def _init_valid_data(self): + self._test_weights = self._init_weight(self.dataset.test_X.shape[0]) + self._test_X_bin = self._split_test_data(self.ctx, self.dataset.test_X, self._X_split) + + def _init_early_stop(self): + + callbacks = [] + early_stopping_rounds = self.params.early_stopping_rounds + if early_stopping_rounds != 0: + eval_metric = self.params.eval_metric + early_stopping = EarlyStopping(rounds=early_stopping_rounds, metric_name=eval_metric, save_best=True) + callbacks.append(early_stopping) + + verbose_eval = self.params.verbose_eval + if verbose_eval != 0: + evaluation_monitor = EvaluationMonitor(logger=self.log, period=verbose_eval) + callbacks.append(evaluation_monitor) + + callback_container = None + if len(callbacks) != 0: + callback_container = CallbackContainer(callbacks=callbacks, feval=Evaluation.fevaluation) + + model = Booster(y_true=self.dataset.train_y, test_y_true=self.dataset.test_y, + workspace=self.ctx.workspace, job_id=self.ctx.job_id, + storage_client=self.storage_client) + + if callback_container: + callback_container.before_training(model) + + self.model = model + self.callback_container = callback_container + + def _iteration_early_stop(self): + # check early stopping + early_stopping_rounds = self.params.early_stopping_rounds + if early_stopping_rounds != 0: + # evaluate the model using test sets + eval_on_test = True + pred = self._test_praba + else: + eval_on_test = False + pred = self._train_praba + stop = False + if self.callback_container: + stop = self.callback_container.after_iteration(model=self.model, + pred=pred, + eval_on_test=eval_on_test) + self.log.info(f"task {self.ctx.task_id}: after iteration {self._tree_id} iterations, stop: {stop}.") + + iteration_request = IterationRequest() + iteration_request.epoch = self._tree_id - 1 + iteration_request.stop = stop + + # send stop to passive + for partner_index in range(1, len(self.ctx.participant_id_list)): + self._send_byte_data( + ctx=self.ctx, + key_type=f'{LGBMMessage.STOP_ITERATION.value}_{self._tree_id}', + byte_data=utils.pb_to_bytes(iteration_request), + partner_index=partner_index) + + return stop + + def _end_active_data(self, is_train=True): + if is_train: + self.feature_importance_store.store( + serialize_type=SerializeType.CSV, local_file_path=self.ctx.feature_importance_file, + remote_file_path=self.ctx.remote_feature_importance_file, storage_client=self.storage_client) + + if self.callback_container: + self.callback_container.after_training(self.model) + + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_train', + self._train_praba.astype('float').tobytes(), partner_index) + + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_test', + self._test_praba.astype('float').tobytes(), partner_index) + + else: + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_predict', + self._test_praba.astype('float').tobytes(), partner_index) diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py new file mode 100644 index 00000000..468158cf --- /dev/null +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -0,0 +1,345 @@ +import os +import time +import random +import json +import numpy as np + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.interface.model_base import VerticalModel +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.common.protocol import PheMessage +from ppc_model.network.stub import PushRequest, PullRequest +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext, LGBMMessage + + +# 抽离sgb的公共部分 +class VerticalBooster(VerticalModel): + def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: + super().__init__(ctx) + self.dataset = dataset + self._stub = ctx.components.stub + + self._tree_id = None + self._leaf_id = None + self._X_bin = None + self._X_split = None + self._trees = [] + + self._train_weights = None + self._train_praba = None + self._test_weights = None + self._test_praba = None + + random.seed(ctx.lgbm_params.random_state) + np.random.seed(ctx.lgbm_params.random_state) + + def _build_tree(self, *args, **kwargs): + + raise NotImplementedError + + def _predict_tree(self, *args, **kwargs): + + raise NotImplementedError + + def _init_praba(self, n): + return np.full(n, 0.5) + + def _init_weight(self, n): + return np.zeros(n, dtype=float) + + @staticmethod + def _get_categorical_idx(feature_name, categorical_feature = []): + categorical_idx = [] + if len(categorical_feature) > 0: + for i in categorical_feature: + if i in feature_name: + categorical_idx.append(feature_name.index(i)) + return categorical_idx + + @staticmethod + def _compute_gh_sum(feature, value, categorical_idx, gl, hl, grad_hist, hess_hist): + if feature in categorical_idx: + gl = grad_hist[feature][value] + hl = hess_hist[feature][value] + else: + gl = gl + grad_hist[feature][value] + hl = hl + hess_hist[feature][value] + return gl, hl + + @staticmethod + def _compute_gain(g, h, gl, hl, gr, hr, λ): + if (h + λ) != 0 and (hl + λ) != 0 and (hr + λ) != 0: + return gl**2 / (hl + λ) + gr**2 / (hr + λ) - g**2 / (h + λ) + else: + return 0 + + @staticmethod + def _compute_leaf_weight(lr, λ, gl, hl, gr, hr, reg_alpha): + + weight_l = VerticalBooster._calulate_weight(lr, λ, gl, hl, reg_alpha) + weight_r = VerticalBooster._calulate_weight(lr, λ, gr, hr, reg_alpha) + + return weight_l, weight_r + + @staticmethod + def _calulate_weight(lr, λ, g, h, reg_alpha): + + # weight = lr * - g / (h + λ) + if (h + λ) != 0 and g > reg_alpha: + weight = lr * - (g - reg_alpha) / (h + λ) + elif (h + λ) != 0 and g < -reg_alpha: + weight = lr * - (g + reg_alpha) / (h + λ) + else: + weight = 0 + + return weight + + @staticmethod + def _get_leaf_instance(X, instance, feature, value, my_categorical_idx): + + if feature in my_categorical_idx: + left_mask = X[instance, feature] == value + right_mask = ~left_mask + else: + left_mask = X[instance, feature] <= value + right_mask = ~left_mask + + return left_mask, right_mask + + def _get_leaf_mask(self, split_info, instance): + + if self.ctx.participant_id_list[split_info.agency_idx] == self.ctx.components.config_data['AGENCY_ID']: + left_mask, right_mask = self._get_leaf_instance( + self._X_bin, instance, split_info.agency_feature, split_info.value, self.params.my_categorical_idx) + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != self.ctx.components.config_data['AGENCY_ID']: + self._send_byte_data( + self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', + left_mask.astype('bool').tobytes(), partner_index) + else: + left_mask = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', + split_info.agency_idx), dtype='bool') + right_mask = ~left_mask + + return left_mask, right_mask + + def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=False): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + if matrix_data: + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=PheMessage.packing_2dim_data(ctx.codec, ctx.phe.public_key, enc_data) + )) + else: + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=PheMessage.packing_data(ctx.codec, ctx.phe.public_key, enc_data) + )) + + log.info( + f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " + f"data_length: {len(enc_data)}, time_costs: {time.time() - start_time}s") + + def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + byte_data = self._stub.pull(PullRequest( + sender=partner_id, + task_id=ctx.task_id, + key=key_type + )) + + if matrix_data: + public_key, enc_data = PheMessage.unpacking_2dim_data(ctx.codec, byte_data) + else: + public_key, enc_data = PheMessage.unpacking_data(ctx.codec, byte_data) + + log.info( + f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + return public_key, enc_data + + def _send_byte_data(self, ctx, key_type, byte_data, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=byte_data + )) + + log.info( + f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + + def _receive_byte_data(self, ctx, key_type, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + byte_data = self._stub.pull(PullRequest( + sender=partner_id, + task_id=ctx.task_id, + key=key_type + )) + + log.info( + f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + return byte_data + + @staticmethod + def _split_test_data(ctx, test_X, X_split): + feat_bin = FeatureBinning(ctx) + return feat_bin.data_binning(test_X, X_split)[0] + + def save_model(self, file_path=None): + log = self.ctx.components.logger() + if file_path is not None: + self.ctx.feature_bin_file = os.path.join(file_path, self.ctx.FEATURE_BIN_FILE) + self.ctx.model_data_file = os.path.join(file_path, self.ctx.MODEL_DATA_FILE) + + if self._X_split is not None and not os.path.exists(self.ctx.feature_bin_file): + X_split_dict = {k: v for k, v in zip(self.dataset.feature_name, self._X_split)} + with open(self.ctx.feature_bin_file, 'w') as f: + json.dump(X_split_dict, f) + ResultFileHandling._upload_file(self.ctx.components.storage_client, + self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) + log.info(f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished.") + + if not os.path.exists(self.ctx.model_data_file): + serial_trees = [self._serial_tree(tree) for tree in self._trees] + with open(self.ctx.model_data_file, 'w') as f: + json.dump(serial_trees, f) + ResultFileHandling._upload_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + log.info(f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.") + + def load_model(self, file_path=None): + log = self.ctx.components.logger() + if file_path is not None: + self.ctx.feature_bin_file = os.path.join(file_path, self.ctx.FEATURE_BIN_FILE) + self.ctx.model_data_file = os.path.join(file_path, self.ctx.MODEL_DATA_FILE) + if self.ctx.algorithm_type == AlgorithmType.Predict.name: + self.ctx.remote_feature_bin_file = os.path.join( + self.ctx.lgbm_params.training_job_id, self.ctx.FEATURE_BIN_FILE) + self.ctx.remote_model_data_file = os.path.join( + self.ctx.lgbm_params.training_job_id, self.ctx.MODEL_DATA_FILE) + + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + + with open(self.ctx.feature_bin_file, 'r') as f: + X_split_dict = json.load(f) + feature_name = list(X_split_dict.keys()) + x_split = list(X_split_dict.values()) + log.info(f"task {self.ctx.task_id}: Load x_split from {self.ctx.feature_bin_file} finished.") + assert len(feature_name) == len(self.dataset.feature_name) + + with open(self.ctx.model_data_file, 'r') as f: + serial_trees = json.load(f) + log.info(f"task {self.ctx.task_id}: Load serial_trees from {self.ctx.model_data_file} finished.") + + trees = [self._deserial_tree(tree) for tree in serial_trees] + self._X_split = x_split + # self.my_feature_name = feature_name + self._trees = trees + + @staticmethod + def _serial_tree(tree): + if isinstance(tree, list): + best_split_info, left_tree, right_tree = tree[0] + best_split_info_list = [] + for field in best_split_info.DESCRIPTOR.fields: + best_split_info_list.append(getattr(best_split_info, field.name)) + left_tree = VerticalBooster._serial_tree(left_tree) + right_tree = VerticalBooster._serial_tree(right_tree) + best_split_info_list.extend([left_tree, right_tree]) + return best_split_info_list + else: + return tree + + @staticmethod + def _deserial_tree(tree_list): + if isinstance(tree_list, list): + best_split_info_list = tree_list[:-2] + left_tree, right_tree = tree_list[-2:] + best_split_info = BestSplitInfo() + for i, field in enumerate(best_split_info.DESCRIPTOR.fields): + setattr(best_split_info, field.name, best_split_info_list[i]) + left_tree = VerticalBooster._deserial_tree(left_tree) + right_tree = VerticalBooster._deserial_tree(right_tree) + return [(best_split_info, left_tree, right_tree)] + else: + return tree_list + + def get_trees(self): + return self._trees + + def get_x_split(self): + return self._X_split + + def get_train_praba(self): + return self._train_praba + + def get_test_praba(self): + return self._test_praba + + @staticmethod + def packing_gh(g_list: np.ndarray, h_list: np.ndarray, expand=1000, mod_length=32, pack_length=20): + ''' + 1. 转正整数 + g和h的梯度值为浮点数, 取值范围: [-1 ~ 1] + 浮点数转整数默乘以 1000(取3位小数) + 按照最高数据量100w样本, g/h求和值上限为 1000 * 10**6 = 10**9 + 基于g/h上限, 负数模运算转正数需要加上 2**32 (4.29*10**9) + + 2. packing + g/h负数模运算转为正数后最大值为 2**32-1, 100w样本求和需要预留10**6位 + packing g和h时, 对g乘以10**20, 为h预留总计20位长度。 + ''' + mod_n = 2 ** mod_length + pos_int_glist = ((g_list * expand).astype('int64') + mod_n) % mod_n + pos_int_hlist = ((h_list * expand).astype('int64') + mod_n) % mod_n + + gh_list = pos_int_glist.astype('object') * 10**pack_length + pos_int_hlist.astype('object') + + return gh_list + + @staticmethod + def unpacking_gh(gh_sum_list: np.ndarray, expand=1000, mod_length=32, pack_length=20): + ''' + 1. 拆解g_pos_int_sum和h_pos_int_sum + 2. 还原g_sum和h_sum + ''' + + mod_n = 2 ** mod_length + g_pos_int_sum = (gh_sum_list // 10**pack_length) % mod_n + h_pos_int_sum = (gh_sum_list % 10**pack_length) % mod_n + + g_pos_int_sum[g_pos_int_sum > 2**(mod_length-1)] -= mod_n + h_pos_int_sum[g_pos_int_sum > 2**(mod_length-1)] -= mod_n + + g_hist = (g_pos_int_sum / expand).astype('float') + h_hist = (h_pos_int_sum / expand).astype('float') + + return g_hist, h_hist diff --git a/python/ppc_model/secure_lgbm/vertical/passive_party.py b/python/ppc_model/secure_lgbm/vertical/passive_party.py new file mode 100644 index 00000000..2f09f6d1 --- /dev/null +++ b/python/ppc_model/secure_lgbm/vertical/passive_party.py @@ -0,0 +1,271 @@ +import multiprocessing +import time +import numpy as np +from pandas import DataFrame + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo, IterationRequest +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext, LGBMMessage +from ppc_model.secure_lgbm.vertical.booster import VerticalBooster + + +class VerticalLGBMPassiveParty(VerticalBooster): + + def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: + super().__init__(ctx, dataset) + self.params = ctx.lgbm_params + self.log = ctx.components.logger() + self.log.info(f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + + def fit( + self, + *args, + **kwargs, + ) -> None: + self.log.info( + f'task {self.ctx.task_id}: Starting the lgbm on the passive party.') + self._init_passive_data() + self._test_X_bin = self._split_test_data(self.ctx, self.dataset.test_X, self._X_split) + + for _ in range(self.params.n_estimators): + self._tree_id += 1 + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in passive party.') + + # 初始化 + instance, used_ghlist, public_key = self._receive_gh_instance_list() + self.ctx.phe.public_key = public_key + self.log.info(f'task {self.ctx.task_id}: Sampling number: {len(instance)}.') + + # 构建 + tree = self._build_tree(instance, used_ghlist) + self._trees.append(tree) + + # 预测 + self._predict_tree(tree, self._X_bin, LGBMMessage.PREDICT_LEAF_MASK.value) + self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id}, ' + f'time_costs: {time.time() - start_time}s.') + + # 预测验证集 + self._predict_tree(tree, self._test_X_bin, LGBMMessage.TEST_LEAF_MASK.value) + if self._iteration_early_stop(): + self.log.info(f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") + break + + self._end_passive_data() + + def transform(self, transform_data: DataFrame) -> DataFrame: + ... + + def predict(self, dataset: SecureDataset = None) -> np.ndarray: + start_time = time.time() + if dataset is None: + dataset = self.dataset + + self.params.my_categorical_idx = self._get_categorical_idx( + dataset.feature_name, self.params.categorical_feature) + + test_X_bin = self._split_test_data(self.ctx, dataset.test_X, self._X_split) + + [self._predict_tree( + tree, test_X_bin, LGBMMessage.VALID_LEAF_MASK.value) for tree in self._trees] + self.log.info(f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + + self._end_passive_data(is_train=False) + + def _init_passive_data(self): + + # 初始化tree id + self._tree_id = 0 + + # 初始化参与方特征 + self._send_byte_data(self.ctx, LGBMMessage.FEATURE_NAME.value, + b''.join(s.encode('utf-8') + b' ' for s in self.dataset.feature_name), 0) + self.params.my_categorical_idx = self._get_categorical_idx( + self.dataset.feature_name, self.params.categorical_feature) + + # 初始化分桶数据集 + feat_bin = FeatureBinning(self.ctx) + self._X_bin, self._X_split = feat_bin.data_binning( + self.dataset.train_X) + + def _receive_gh_instance_list(self): + + self._leaf_id = 0 + + instance = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LGBMMessage.INSTANCE.value}_{self._tree_id}', 0), dtype=np.int64) + public_key, gh = self._receive_enc_data( + self.ctx, f'{LGBMMessage.ENC_GH_LIST.value}_{self._tree_id}', 0) + + return instance, np.array(gh), public_key + + def _build_tree(self, instance, ghlist, depth=0, weight=0): + + if depth == self.params.max_depth: + return weight + if self.params.max_depth < 0 and self._leaf_id >= self.params.num_leaves: + return weight + + self._leaf_id += 1 + print('tree', self._tree_id, 'leaf', self._leaf_id, 'instance', len(instance), + 'ghlist', len(ghlist)) + best_split_info = self._find_best_split(instance, ghlist) + + if best_split_info.best_gain > 0 and best_split_info.best_gain > self.params.min_split_gain: + left_mask, right_mask = self._get_leaf_mask(best_split_info, instance) + + if (abs(best_split_info.w_left) * sum(left_mask) / self.params.lr) < self.params.min_child_weight or \ + (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: + return weight + if sum(left_mask) < self.params.min_child_samples or sum(right_mask) < self.params.min_child_samples: + return weight + + left_tree = self._build_tree( + instance[left_mask], ghlist[left_mask], + depth + 1, best_split_info.w_left) + right_tree = self._build_tree( + instance[right_mask], ghlist[right_mask], + depth + 1, best_split_info.w_right) + + return [(best_split_info, left_tree, right_tree)] + else: + return weight + + def _predict_tree(self, tree, X_bin, key_type): + if not isinstance(tree, list): + return None + else: + best_split_info, left_subtree, right_subtree = tree[0] + if self.ctx.participant_id_list[best_split_info.agency_idx] == \ + self.ctx.components.config_data['AGENCY_ID']: + if best_split_info.agency_feature in self.params.my_categorical_idx: + left_mask = X_bin[:, best_split_info.agency_feature] == best_split_info.value + else: + left_mask = X_bin[:, best_split_info.agency_feature] <= best_split_info.value + self._send_byte_data( + self.ctx, + f'{key_type}_{best_split_info.tree_id}_{best_split_info.leaf_id}', + left_mask.astype('bool').tobytes(), 0) + else: + pass + left_weight = self._predict_tree(left_subtree, X_bin, key_type) + right_weight = self._predict_tree(right_subtree, X_bin, key_type) + return [left_weight, right_weight] + + def _find_best_split(self, instance, ghlist): + + self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} in passive party.') + if len(instance) > 200000: + self._get_gh_hist_parallel(instance, ghlist) + else: + self._get_gh_hist(instance, ghlist) + + best_split_info_byte = self._receive_byte_data( + self.ctx, f'{LGBMMessage.SPLIT_INFO.value}_{self._tree_id}_{self._leaf_id}', 0) + best_split_info = BestSplitInfo() + utils.bytes_to_pb(best_split_info, best_split_info_byte) + + self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} in passive party.') + return best_split_info + + def _get_gh_hist_parallel(self, instance, ghlist): + + params = [] + for i in range(len(self.dataset.feature_name)): + params.append({ + 'bins': self._X_bin[:, i], + 'xk_bin': self._X_bin[:, i][instance], + 'enc_gh_list': ghlist, + 'phe': self.ctx.phe + }) + + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Start n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} calculate hist in passive party.') + + # gh_hist = [] + # with ProcessPoolExecutor() as executor: + # futures = [executor.submit(self._calculate_hist, context) for context in params] + # for future in as_completed(futures): + # gh_hist.append(future.result()) + + pool = multiprocessing.Pool() + gh_hist = pool.map(self._calculate_hist, params) + pool.close() + pool.join() + + self.log.info(f'task {self.ctx.task_id}: End n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} calculate hist time_costs: {time.time() - start_time}s.') + self._send_enc_data(self.ctx, + f'{LGBMMessage.ENC_GH_HIST.value}_{self._tree_id}_{self._leaf_id}', + gh_hist, 0, matrix_data=True) + + def _get_gh_hist(self, instance, ghlist): + + gh_hist = [] + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Start n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} calculate hist in passive party.') + + for i in range(len(self.dataset.feature_name)): + param = { + 'bins': self._X_bin[:, i], + 'xk_bin': self._X_bin[:, i][instance], + 'enc_gh_list': ghlist, + 'phe': self.ctx.phe + } + gh_hist.append(self._calculate_hist(param)) + + self.log.info(f'task {self.ctx.task_id}: Start n_estimators-{self._tree_id} ' + f'leaf-{self._leaf_id} calculate hist time_costs: {time.time() - start_time}s.') + self._send_enc_data(self.ctx, + f'{LGBMMessage.ENC_GH_HIST.value}_{self._tree_id}_{self._leaf_id}', + gh_hist, 0, matrix_data=True) + + @staticmethod + def _calculate_hist(param): + bins = param['bins'] + gh_list = param['enc_gh_list'] + phe = param['phe'] + xk_bin = param['xk_bin'] + ghk_hist = [] + sorted_bins = sorted(set(bins)) + for v in sorted_bins: + # 处理gk_hist中部分分桶没有样本，直接结算值为明文0的情况 + if len(gh_list[xk_bin == v]) == 0: + ghk_hist.append(phe.encrypt(0)) + else: + ghk_hist.append(gh_list[xk_bin == v].sum()) + return ghk_hist + + def _iteration_early_stop(self): + + iteration_request_byte = self._receive_byte_data( + self.ctx, f'{LGBMMessage.STOP_ITERATION.value}_{self._tree_id}', 0) + iteration_request = IterationRequest() + utils.bytes_to_pb(iteration_request, iteration_request_byte) + + return iteration_request.stop + + def _end_passive_data(self, is_train=True): + + if self.ctx.components.config_data['AGENCY_ID'] in self.ctx.result_receiver_id_list: + if is_train: + self._train_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_train', 0), dtype=np.float) + + self._test_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_test', 0), dtype=np.float) + + else: + self._test_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LGBMMessage.PREDICT_PRABA.value}_predict', 0), dtype=np.float) diff --git a/python/ppc_model/task/__init__.py b/python/ppc_model/task/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py new file mode 100644 index 00000000..d6a308d0 --- /dev/null +++ b/python/ppc_model/task/task_manager.py @@ -0,0 +1,183 @@ +import datetime +import logging +import os +import threading +import time +from typing import Callable, Union + +from readerwriterlock import rwlock + +from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.protocol import ModelTask, TaskStatus, LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER +from ppc_model.network.stub import ModelStub + + +class TaskManager: + def __init__(self, logger, + thread_event_manager: ThreadEventManager, + stub: ModelStub, + task_timeout_h: Union[int, float]): + self.logger = logger + self._thread_event_manager = thread_event_manager + self._stub = stub + self._task_timeout_s = task_timeout_h * 3600 + self._rw_lock = rwlock.RWLockWrite() + self._tasks: dict[str, list] = {} + self._jobs: dict[str, set] = {} + self._handlers = {} + self._async_executor = AsyncThreadExecutor( + event_manager=self._thread_event_manager, logger=logger) + self._cleanup_thread = threading.Thread(target=self._loop_cleanup) + self._cleanup_thread.daemon = True + self._cleanup_thread.start() + + def register_task_handler(self, task_type: ModelTask, task_handler: Callable): + """ + 注册任务的执行入口 + param task_type: 任务类型 + param task_handler: 任务执行入口 + """ + self._handlers[task_type.value] = task_handler + + def run_task(self, task_id: str, task_type: ModelTask, args=()): + """ + 发起任务 + param task_id: 任务ID + param task_type: 任务类型 + param args: 各任务参数 + """ + job_id = args[0]['job_id'] + with self._rw_lock.gen_wlock(): + if task_id in self._tasks: + self.logger.info(f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id][0]}") + return + self._tasks[task_id] = [TaskStatus.RUNNING.value, datetime.datetime.now(), 0, args[0]['job_id']] + if job_id in self._jobs: + self._jobs[job_id].add(task_id) + else: + self._jobs[job_id] = {task_id} + self.logger.info(LOG_START_FLAG_FORMATTER.format(job_id=job_id)) + self.logger.info(f"Run task, job_id: {job_id}, task_id: {task_id}") + self._async_executor.execute(task_id, self._handlers[task_type.value], self._on_task_finish, args) + + def kill_task(self, job_id: str): + """ + 终止任务 + """ + task_ids = [] + with self._rw_lock.gen_rlock(): + if job_id not in self._jobs: + return + for task_id in self._jobs[job_id]: + task_ids.append(task_id) + + for task_id in task_ids: + self.kill_one_task(task_id) + + def kill_one_task(self, task_id: str): + with self._rw_lock.gen_rlock(): + if task_id not in self._tasks or self._tasks[task_id][0] != TaskStatus.RUNNING.value: + return + + self.logger.info(f"Kill task, task_id: {task_id}") + self._async_executor.kill(task_id) + + with self._rw_lock.gen_wlock(): + self._tasks[task_id][0] = TaskStatus.FAILED.value + + def status(self, task_id: str) -> [str, float, float]: + """ + 返回: 任务状态, 通讯量(MB), 执行耗时(s) + """ + with self._rw_lock.gen_rlock(): + if task_id not in self._tasks: + raise PpcException( + PpcErrorCode.TASK_NOT_FOUND.get_code(), + PpcErrorCode.TASK_NOT_FOUND.get_msg()) + status = self._tasks[task_id][0] + traffic_volume = self._stub.traffic_volume(task_id) + time_costs = self._tasks[task_id][2] + return status, traffic_volume, time_costs + + def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None): + with self._rw_lock.gen_wlock(): + time_costs = (datetime.datetime.now() - + self._tasks[task_id][1]).total_seconds() + self._tasks[task_id][2] = time_costs + if is_succeeded: + self._tasks[task_id][0] = TaskStatus.COMPLETED.value + self.logger.info(f"Task {task_id} completed, job_id: {self._tasks[task_id][3]}, " + f"time_costs: {time_costs}s") + else: + self._tasks[task_id][0] = TaskStatus.FAILED.value + self.logger.warn(f"Task {task_id} failed, job_id: {self._tasks[task_id][3]}, " + f"time_costs: {time_costs}s, error: {e}") + self.logger.info(LOG_END_FLAG_FORMATTER.format( + job_id=self._tasks[task_id][3])) + + def _loop_cleanup(self): + while True: + self._terminate_timeout_tasks() + self._cleanup_finished_tasks() + time.sleep(5) + + def _terminate_timeout_tasks(self): + tasks_to_kill = [] + with self._rw_lock.gen_rlock(): + for task_id, value in self._tasks.items(): + alive_time = (datetime.datetime.now() - + value[1]).total_seconds() + if alive_time >= self._task_timeout_s and value[0] == TaskStatus.RUNNING.value: + tasks_to_kill.append(task_id) + + for task_id in tasks_to_kill: + self.logger.warn(f"Task is timeout, task_id: {task_id}") + self.kill_one_task(task_id) + + def _cleanup_finished_tasks(self): + tasks_to_cleanup = [] + with self._rw_lock.gen_rlock(): + for task_id, value in self._tasks.items(): + alive_time = (datetime.datetime.now() - + value[1]).total_seconds() + if alive_time >= self._task_timeout_s + 3600: + tasks_to_cleanup.append((task_id, value[3])) + with self._rw_lock.gen_wlock(): + for task_id, job_id in tasks_to_cleanup: + if task_id in self._tasks: + del self._tasks[task_id] + if job_id in self._jobs: + del self._jobs[job_id] + self._thread_event_manager.remove_event(task_id) + self._stub.cleanup_cache(task_id) + self.logger.info(f"Cleanup task cache, task_id: {task_id}, job_id: {job_id}") + + def record_model_job_log(self, job_id): + log_file = self._get_log_file_path() + if log_file is None or log_file == "": + current_working_dir = os.getcwd() + relative_log_path = "logs/ppcs-model4ef-node.log" + log_file = os.path.join(current_working_dir, relative_log_path) + + start_keyword = LOG_START_FLAG_FORMATTER.format(job_id=job_id) + end_keyword = LOG_END_FLAG_FORMATTER.format(job_id=job_id) + with open(log_file, 'r') as file: + log_data = file.read() + start_index = log_data.find(start_keyword) + end_index = log_data.rfind(end_keyword) + + if start_index == -1 or end_index == -1: + return f"{job_id} not found in log data" + + end_index += len(end_keyword) + return log_data[start_index:end_index] + + def _get_log_file_path(self): + log_file_path = None + for handler in self.logger.handlers: + if isinstance(handler, logging.FileHandler): + log_file_path = handler.baseFilename + break + return log_file_path diff --git a/python/ppc_model/task/test/__init__.py b/python/ppc_model/task/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/task/test/task_manager_unittest.py b/python/ppc_model/task/test/task_manager_unittest.py new file mode 100644 index 00000000..64f2a86f --- /dev/null +++ b/python/ppc_model/task/test/task_manager_unittest.py @@ -0,0 +1,155 @@ +import time +import unittest + +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_model.common.protocol import ModelTask +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.task.task_manager import TaskManager +from ppc_model.network.stub import ModelStub, PushRequest, PullRequest + +rpc_client = RpcClientMock() +thread_event_manager = ThreadEventManager() +stub = ModelStub( + agency_id='TEST_AGENCY', + thread_event_manager=thread_event_manager, + rpc_client=rpc_client, + send_retry_times=3, + retry_interval_s=0.1 +) +rpc_client.set_message_handler(stub.on_message_received) + + +def my_send_task(args): + print("start my_send_task") + time.sleep(1) + byte_array = bytearray(31 * 1024 * 1024) + bytes_data = bytes(byte_array) + stub.push(PushRequest( + receiver=args['receiver'], + task_id=args['task_id'], + key=args['key'], + data=bytes_data + )) + time.sleep(1) + + +def my_receive_task(args): + print("start my_receive_task") + stub.pull(PullRequest( + sender=args['sender'], + task_id=args['task_id'], + key=args['key'], + )) + time.sleep(1) + print("finish my_receive_task") + + +def my_failed_task(args): + print("start my_failed_task") + time.sleep(1) + raise Exception('For Test') + + +def my_long_task(args): + print("start my_long_task") + stub.pull(PullRequest( + sender=args['sender'], + task_id=args['task_id'], + key='not_ready', + )) + print("finish my_receive_task") + + +def my_timeout_task(args): + print("start my_timeout_task") + stub.pull(PullRequest( + sender=args['sender'], + task_id=args['task_id'], + key='not_ready', + )) + print("finish my_timeout_task") + + +class TestTaskManager(unittest.TestCase): + + def setUp(self): + self._task_manager = TaskManager( + logger=MockLogger(), + thread_event_manager=thread_event_manager, + stub=stub, + task_timeout_h=0.0005 + ) + self._task_manager.register_task_handler( + ModelTask.FEATURE_ENGINEERING, my_send_task) + self._task_manager.register_task_handler( + ModelTask.PREPROCESSING, my_receive_task) + self._task_manager.register_task_handler( + ModelTask.XGB_TRAINING, my_failed_task) + self._task_manager.register_task_handler( + ModelTask.XGB_PREDICTING, my_long_task) + + def test_run_task(self): + args = { + 'receiver': 'TEST_AGENCY', + 'sender': 'TEST_AGENCY', + 'task_id': '0x12345678', + 'job_id': '0x123456789', + 'key': 'TEST_MESSAGE', + } + self._task_manager.run_task( + "my_send_task", ModelTask.FEATURE_ENGINEERING, (args,)) + self.assertEqual(self._task_manager.status( + "my_send_task")[0], 'RUNNING') + self._task_manager.run_task( + "my_receive_task", ModelTask.PREPROCESSING, (args,)) + self.assertEqual(self._task_manager.status( + "my_receive_task")[0], 'RUNNING') + self._task_manager.run_task( + "my_failed_task", ModelTask.XGB_TRAINING, (args,)) + self.assertEqual(self._task_manager.status( + "my_failed_task")[0], 'RUNNING') + time.sleep(3) + self.assertEqual(self._task_manager.status( + "my_send_task")[0], 'COMPLETED') + self.assertEqual(self._task_manager.status( + "my_receive_task")[0], 'COMPLETED') + self.assertEqual(self._task_manager.status( + "my_failed_task")[0], 'FAILED') + time.sleep(1) + + def test_kill_task(self): + args = { + 'receiver': 'TEST_AGENCY', + 'sender': 'TEST_AGENCY', + 'task_id': 'my_long_task', + 'job_id': '0x123456789', + 'key': 'TEST_MESSAGE', + } + self._task_manager.run_task("my_long_task", ModelTask.XGB_PREDICTING, (args,)) + self.assertEqual(self._task_manager.status("my_long_task")[0], 'RUNNING') + self._task_manager.kill_task("0x123456789") + time.sleep(1) + self.assertEqual(self._task_manager.status( + "my_long_task")[0], 'FAILED') + + self._task_manager.register_task_handler( + ModelTask.XGB_PREDICTING, my_timeout_task) + args = { + 'receiver': 'TEST_AGENCY', + 'sender': 'TEST_AGENCY', + 'task_id': 'my_timeout_task', + 'job_id': '0x123456789', + 'key': 'TEST_MESSAGE', + } + self._task_manager.run_task( + "my_timeout_task", ModelTask.XGB_PREDICTING, (args,)) + self.assertEqual(self._task_manager.status( + "my_timeout_task")[0], 'RUNNING') + time.sleep(6) + self.assertEqual(self._task_manager.status( + "my_timeout_task")[0], 'FAILED') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/tools/start.sh b/python/ppc_model/tools/start.sh new file mode 100644 index 00000000..3c1c818b --- /dev/null +++ b/python/ppc_model/tools/start.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath +LOG_DIR=/data/app/logs/ppcs-model4ef/ + +# kill crypto process +crypto_pro_num=`ps -ef | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | wc -l` +for i in $( seq 1 $crypto_pro_num ) +do + crypto_pid=`ps -ef | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | awk 'NR==1{print}'` + kill -9 $crypto_pid +done + +sleep 1 + +nohup python ppc_model_app.py > start.out 2>&1 & + +check_service() { + try_times=5 + i=0 + while [ -z `ps -ef | grep ${1} | grep python | grep -v grep | awk '{print $2}'` ]; do + sleep 1 + ((i = i + 1)) + if [ $i -lt ${try_times} ]; then + echo -e "\033[32m.\033[0m\c" + else + echo -e "\033[31m\nServer ${1} isn't running. \033[0m" + return + fi + done + + echo -e "\033[32mServer ${1} started \033[0m" +} + +sleep 5 +check_service ppc_model_app.py +rm -rf logs +ln -s ${LOG_DIR} logs \ No newline at end of file diff --git a/python/ppc_model/tools/stop.sh b/python/ppc_model/tools/stop.sh new file mode 100644 index 00000000..3b290668 --- /dev/null +++ b/python/ppc_model/tools/stop.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +# kill crypto process +crypto_pro_num=`ps -ef | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | wc -l` +for i in $( seq 1 $crypto_pro_num ) +do + crypto_pid=`ps -ef | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | awk 'NR==1{print}'` + kill -9 $crypto_pid +done + +sleep 1 + +ppc_model_app_pid=`ps aux |grep ppc_model_app.py |grep -v grep |awk '{print $2}'` +kill -9 $ppc_model_app_pid + +echo -e "\033[32mServer ppc_model_app.py killed. \033[0m" diff --git a/python/ppc_model_gateway/__init__.py b/python/ppc_model_gateway/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model_gateway/clients/__init__.py b/python/ppc_model_gateway/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model_gateway/clients/client_manager.py b/python/ppc_model_gateway/clients/client_manager.py new file mode 100644 index 00000000..e05f89fb --- /dev/null +++ b/python/ppc_model_gateway/clients/client_manager.py @@ -0,0 +1,57 @@ +import os + +import grpc + +from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceStub +from ppc_common.ppc_utils import utils +from ppc_model_gateway import config + + +class ClientManager: + def __init__(self, config_data, grpc_options): + self._config_data = config_data + self._grpc_options = grpc_options + channel = grpc.insecure_channel( + self._config_data['NODE_ENDPOINT'], options=self._grpc_options) + self.node_stub = ModelServiceStub(channel) + self.agency_stub_dict = {} + self._create_partner_stubs() + + def _create_partner_stubs(self): + for agency_id, endpoint in self._get_agency_dict().items(): + if self._config_data['SSL_SWITCH'] == 0: + channel = grpc.insecure_channel( + endpoint, options=self._grpc_options) + self.agency_stub_dict[agency_id] = ModelServiceStub(channel) + else: + channel = self._create_secure_channel(endpoint) + self.agency_stub_dict[agency_id] = ModelServiceStub(channel) + + def _get_agency_dict(self) -> dict: + agency_dict = {} + for entry in self._config_data.get('AGENCY_LIST', []): + if ':' in entry: + key, value = entry.split(":", 1) + key = key.strip() + value = value.strip() + agency_dict[key] = value + return agency_dict + + def _create_secure_channel(self, target): + grpc_root_crt = utils.load_credential_from_file( + os.path.abspath(self._config_data['SSL_CA'])) + grpc_ssl_key = utils.load_credential_from_file( + os.path.abspath(self._config_data['SSL_KEY'])) + grpc_ssl_crt = utils.load_credential_from_file( + os.path.abspath(self._config_data['SSL_CRT'])) + + credentials = grpc.ssl_channel_credentials( + root_certificates=grpc_root_crt, + private_key=grpc_ssl_key, + certificate_chain=grpc_ssl_crt + ) + + return grpc.secure_channel(target, credentials, options=self._grpc_options) + + +client_manager = ClientManager(config.CONFIG_DATA, config.grpc_options) diff --git a/python/ppc_model_gateway/conf/application-sample.yml b/python/ppc_model_gateway/conf/application-sample.yml new file mode 100644 index 00000000..3c7abf8c --- /dev/null +++ b/python/ppc_model_gateway/conf/application-sample.yml @@ -0,0 +1,19 @@ +HOST: "0.0.0.0" + +# 0: off, 1: on +SSL_SWITCH: 1 +SSL_CA: "cert/ca.crt" +SSL_KEY: "cert/node.key" +SSL_CRT: "cert/node.crt" + +# 0801配置 +NODE_TO_PARTNER_RPC_PORT: 43454 +PARTNER_TO_NODE_RPC_PORT: 43451 + +NODE_ENDPOINT: "127.0.0.1:43472" + +AGENCY_LIST: + #- "SG: [@IDC_PPCS_SG_MODEL_GATEWAY_ENDPOINT]" + - "WeBank: 127.0.0.1:43451" + +MAX_MESSAGE_LENGTH_MB: 100 diff --git a/python/ppc_model_gateway/conf/logging.conf b/python/ppc_model_gateway/conf/logging.conf new file mode 100644 index 00000000..bb34233c --- /dev/null +++ b/python/ppc_model_gateway/conf/logging.conf @@ -0,0 +1,40 @@ +[loggers] +keys=root,grpc + +[logger_root] +level=INFO +handlers=consoleHandler,fileHandler + +[logger_grpc] +level = DEBUG +handlers = accessHandler +qualname = grpc +propagate = 0 + +[handlers] +keys=fileHandler,consoleHandler,accessHandler + +[handler_accessHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/logs/ppcs-modelgateway/appmonitor.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_fileHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/logs/ppcs-modelgateway/ppcs-modelgateway-gateway.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_consoleHandler] +class=StreamHandler +args=(sys.stdout,) +level=ERROR +formatter=simpleFormatter + +[formatters] +keys=simpleFormatter + +[formatter_simpleFormatter] +format=[%(levelname)s][%(asctime)s %(msecs)03d][%(process)d][%(filename)s:%(lineno)d] %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/python/ppc_model_gateway/config.py b/python/ppc_model_gateway/config.py new file mode 100644 index 00000000..1b9edb23 --- /dev/null +++ b/python/ppc_model_gateway/config.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +import logging +import logging.config +import os + +import yaml + +from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize + +path = os.getcwd() +log_dir = os.sep.join([path, 'logs']) +chain_log_dir = os.sep.join([path, 'bin', 'logs']) +print(f"log_dir: {log_dir}") +print(f"chain_log_dir: {chain_log_dir}") +if not os.path.exists(log_dir): + os.makedirs(log_dir) +if not os.path.exists(chain_log_dir): + os.makedirs(chain_log_dir) +logging_conf_path = os.path.normpath('logging.conf') +logging.config.fileConfig(logging_conf_path) + + +def get_logger(name=None): + log = logging.getLogger(name) + return log + + +config_path = "application.yml" + +CONFIG_DATA = {} +agency_dict = {} + + +def read_config(): + with open(config_path, 'rb') as f: + global CONFIG_DATA + CONFIG_DATA = yaml.safe_load(f.read()) + + +read_config() + +grpc_options = [ + ('grpc.ssl_target_name_override', 'PPCS MODEL GATEWAY'), + ('grpc.max_send_message_length', + CONFIG_DATA['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024), + ('grpc.max_receive_message_length', + CONFIG_DATA['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024), + ('grpc.keepalive_time_ms', 15000), # 每 15 秒发送一次心跳 + ('grpc.keepalive_timeout_ms', 5000), # 等待心跳回应的超时时间为 5 秒 + ('grpc.keepalive_permit_without_calls', True), # 即使没有调用也允许发送心跳 + ('grpc.http2.min_time_between_pings_ms', 15000), # 心跳之间最小时间间隔为 15 秒 + ('grpc.http2.max_pings_without_data', 0), # 在发送数据前不限制心跳次数 + # 在没有数据传输的情况下，确保心跳包之间至少有 20 秒的间隔 + ('grpc.http2.min_ping_interval_without_data_ms', 20000), + ("grpc.so_reuseport", 1), + ("grpc.use_local_subchannel_pool", 1), + ('grpc.enable_retries', 1), + ('grpc.service_config', + '{ "retryPolicy":{ "maxAttempts": 4, "initialBackoff": "0.1s", "maxBackoff": "1s", "backoffMutiplier": 2, "retryableStatusCodes": [ "UNAVAILABLE" ] } }') +] diff --git a/python/ppc_model_gateway/endpoints/__init__.py b/python/ppc_model_gateway/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model_gateway/endpoints/node_to_partner.py b/python/ppc_model_gateway/endpoints/node_to_partner.py new file mode 100644 index 00000000..71f6aa07 --- /dev/null +++ b/python/ppc_model_gateway/endpoints/node_to_partner.py @@ -0,0 +1,36 @@ +import time +import traceback + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest +from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceServicer +from ppc_model_gateway import config +from ppc_model_gateway.clients.client_manager import client_manager +from ppc_model_gateway.endpoints.response_builder import build_error_model_response + +log = config.get_logger() + + +class NodeToPartnerService(ModelServiceServicer): + + def MessageInteraction(self, request: ModelRequest, context): + start_time = time.time() + try: + log.debug( + f"start sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}") + # 根据接收方的机构ID路由消息 + client = client_manager.agency_stub_dict[request.receiver] + response = client.MessageInteraction(request) + end_time = time.time() + log.info( + f"finish sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " + f"ret_code: {response.base_response.error_code}, time_costs: {str(end_time - start_time)}s") + except Exception: + end_time = time.time() + message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \ + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \ + f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s" + log.warn(message) + response = build_error_model_response(message) + return response diff --git a/python/ppc_model_gateway/endpoints/partner_to_node.py b/python/ppc_model_gateway/endpoints/partner_to_node.py new file mode 100644 index 00000000..e4aebfce --- /dev/null +++ b/python/ppc_model_gateway/endpoints/partner_to_node.py @@ -0,0 +1,34 @@ +import time +import traceback + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest +from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceServicer +from ppc_model_gateway import config +from ppc_model_gateway.clients.client_manager import client_manager +from ppc_model_gateway.endpoints.response_builder import build_error_model_response + +log = config.get_logger() + + +class PartnerToNodeService(ModelServiceServicer): + + def MessageInteraction(self, request: ModelRequest, context): + start_time = time.time() + try: + log.debug( + f"start sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}") + response = client_manager.node_stub.MessageInteraction(request) + end_time = time.time() + log.info( + f"finish sending data to {request.receiver}, task_id: {request.task_id}, " + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " + f"ret_code: {response.base_response.error_code}, time_costs: {str(end_time - start_time)}s") + except Exception: + end_time = time.time() + message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \ + f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \ + f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s" + log.warn(message) + response = build_error_model_response(message) + return response diff --git a/python/ppc_model_gateway/endpoints/response_builder.py b/python/ppc_model_gateway/endpoints/response_builder.py new file mode 100644 index 00000000..f97fa21a --- /dev/null +++ b/python/ppc_model_gateway/endpoints/response_builder.py @@ -0,0 +1,8 @@ +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelResponse + + +def build_error_model_response(message: str): + model_response = ModelResponse() + model_response.base_response.error_code = -1 + model_response.base_response.message = message + return model_response diff --git a/python/ppc_model_gateway/ppc_model_gateway_app.py b/python/ppc_model_gateway/ppc_model_gateway_app.py new file mode 100644 index 00000000..f64a96c4 --- /dev/null +++ b/python/ppc_model_gateway/ppc_model_gateway_app.py @@ -0,0 +1,90 @@ +import os +# Note: here can't be refactored by autopep +import sys +sys.path.append("../") + +from concurrent import futures +from threading import Thread + +import grpc + +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_utils import utils +from ppc_model_gateway import config +from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService +from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService + +log = config.get_logger() + + +def node_to_partner_serve(): + rpc_port = config.CONFIG_DATA['NODE_TO_PARTNER_RPC_PORT'] + + ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), + options=config.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(NodeToPartnerService(), ppc_serve) + address = "[::]:{}".format(rpc_port) + ppc_serve.add_insecure_port(address) + + ppc_serve.start() + + start_message = f'Start ppc model gateway internal rpc server at {rpc_port}' + print(start_message) + log.info(start_message) + ppc_serve.wait_for_termination() + + +def partner_to_node_serve(): + rpc_port = config.CONFIG_DATA['PARTNER_TO_NODE_RPC_PORT'] + + if config.CONFIG_DATA['SSL_SWITCH'] == 0: + ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), + options=config.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(PartnerToNodeService(), ppc_serve) + address = "[::]:{}".format(rpc_port) + ppc_serve.add_insecure_port(address) + else: + grpc_root_crt = utils.load_credential_from_file( + os.path.abspath(config.CONFIG_DATA['SSL_CA'])) + grpc_ssl_key = utils.load_credential_from_file( + os.path.abspath(config.CONFIG_DATA['SSL_KEY'])) + grpc_ssl_crt = utils.load_credential_from_file( + os.path.abspath(config.CONFIG_DATA['SSL_CRT'])) + server_credentials = grpc.ssl_server_credentials((( + grpc_ssl_key, + grpc_ssl_crt, + ),), grpc_root_crt, True) + + ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), + options=config.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(PartnerToNodeService(), ppc_serve) + address = "[::]:{}".format(rpc_port) + ppc_serve.add_secure_port(address, server_credentials) + + ppc_serve.start() + + start_message = f'Start ppc model gateway external rpc server at {rpc_port}' + print(start_message) + log.info(start_message) + ppc_serve.wait_for_termination() + + +if __name__ == '__main__': + log = config.get_logger() + + # 设置守护线程 + node_to_partner_serve_thread = Thread(target=node_to_partner_serve) + partner_to_node_serve_thread = Thread(target=partner_to_node_serve) + + node_to_partner_serve_thread.daemon = True + partner_to_node_serve_thread.daemon = True + + node_to_partner_serve_thread.start() + partner_to_node_serve_thread.start() + + node_to_partner_serve_thread.join() + partner_to_node_serve_thread.join() + + message = f'Start ppc model gateway successfully.' + print(message) + log.info(message) diff --git a/python/ppc_model_gateway/test/__init__.py b/python/ppc_model_gateway/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model_gateway/test/client.py b/python/ppc_model_gateway/test/client.py new file mode 100644 index 00000000..a941eb1b --- /dev/null +++ b/python/ppc_model_gateway/test/client.py @@ -0,0 +1,41 @@ + +import grpc +import sys +import os + +from ppc_model_gateway import config +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest + + +def generate_bytes(size_in_mb): + size_in_bytes = size_in_mb * 1024 * 1023 + return os.urandom(size_in_bytes) + + +def send_data(): + channel = grpc.insecure_channel( + f'localhost:{port}', options=config.grpc_options) + stub = ppc_model_pb2_grpc.ModelServiceStub(channel) + + request = ModelRequest() + + request.task_id = "task_id" + request.receiver = receiver + request.key = 'key' + request.seq = 0 + request.slice_num = 1 + request.data = bytes(generate_bytes( + config.CONFIG_DATA['MAX_MESSAGE_LENGTH_MB'])) + + response = stub.MessageInteraction(request) + print("Received response:", response.base_response.message) + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print("Usage: python client.py ") + sys.exit(1) + port = int(sys.argv[1]) + receiver = sys.argv[2] + send_data() diff --git a/python/ppc_model_gateway/test/server.py b/python/ppc_model_gateway/test/server.py new file mode 100644 index 00000000..1df95ecf --- /dev/null +++ b/python/ppc_model_gateway/test/server.py @@ -0,0 +1,36 @@ +import os +from concurrent import futures +import grpc +import sys + +from ppc_model_gateway import config +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelResponse + + +class ModelService(ppc_model_pb2_grpc.ModelServiceServicer): + def MessageInteraction(self, request, context): + response = ModelResponse() + response.base_response.error_code = 0 + response.base_response.message = "Data received successfully." + response.data = request.data + return response + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor( + max_workers=max(1, os.cpu_count() - 1)), + options=config.grpc_options) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), server) + server.add_insecure_port(f'[::]:{port}') + server.start() + print(f'Start serve successfully at {port}.') + server.wait_for_termination() + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: python server.py ") + sys.exit(1) + port = sys.argv[1] + serve() diff --git a/python/ppc_model_gateway/tools/gen_cert.sh b/python/ppc_model_gateway/tools/gen_cert.sh new file mode 100644 index 00000000..199b7844 --- /dev/null +++ b/python/ppc_model_gateway/tools/gen_cert.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +print_usage() { + echo "Usage: $0 [--ca] [--node ] [-h|--help]" + echo " --ca Generate CA key and certificate" + echo " --node Generate sets of certificates and place them in nodeX directories" + echo " -h, --help Display this help message" +} + +generate_ca() { + echo "Generating CA key and certificate..." + # 生成CA私钥 + openssl genrsa -out ca.key 2048 + # 生成自签名的CA证书 + openssl req -x509 -new -nodes -key ca.key -sha256 -days 36500 -out ca.crt -subj "/CN=PPCS CA" +} + +generate_node_certificates() { + local number=$1 + echo "Generating $number sets of node certificates..." + for ((i=1; i<=$number; i++)); do + # 生成节点私钥 + openssl genrsa -out node.key 2048 + # 生成证书签署请求(CSR) + openssl req -new -key node.key -out node.csr -subj "/CN=PPCS MODEL GATEWAY" + # 使用CA证书签署CSR以生成节点证书 + openssl x509 -req -in node.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out node.crt -days 36500 -sha256 + # 清理CSR文件 + rm node.csr + # 创建节点目录并将证书移入 + mkdir -p "node$i" + mv node.key node.crt "node$i/" + echo "Generated certificate set $i and placed in node$i directory" + done +} + +# 检查参数 +if [[ $# -eq 0 ]]; then + echo "Error: No arguments provided." + print_usage + exit 1 +fi + +while [[ "$1" != "" ]]; do + case $1 in + --ca) + generate_ca + ;; + --node) + shift + if [[ $1 =~ ^[0-9]+$ ]]; then + generate_node_certificates $1 + else + echo "Error: --node argument expects a number." + print_usage + exit 1 + fi + ;; + -h | --help) + print_usage + exit 0 + ;; + *) + echo "Error: Invalid argument: $1" + print_usage + exit 1 + ;; + esac + shift +done diff --git a/python/ppc_model_gateway/tools/start.sh b/python/ppc_model_gateway/tools/start.sh new file mode 100644 index 00000000..2a3fb1d9 --- /dev/null +++ b/python/ppc_model_gateway/tools/start.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath +LOG_DIR=/data/app/logs/ppcs-modelgateway/ + +export PYTHONPATH=$dirpath/../ +source /data/app/ppcs-modelgateway/gateway_env/bin/deactivate +source /data/app/ppcs-modelgateway/gateway_env/bin/activate +sleep 1 + +rm -rf $dirpath/../success +nohup python $dirpath/ppc_model_gateway_app.py > start.out 2>&1 & + +check_service() { + try_times=5 + i=0 + while [ -z `ps -ef | grep ${1} | grep python | grep -v grep | awk '{print $2}'` ]; do + sleep 1 + ((i = i + 1)) + if [ $i -lt ${try_times} ]; then + echo -e "\033[32m.\033[0m\c" + else + echo -e "\033[31m\nServer ${1} isn't running. \033[0m" + return + fi + done + echo -e "\033[32mServer ${1} started \033[0m" + echo "success" > $dirpath/../success +} + +sleep 5 +check_service ppc_model_gateway_app.py +rm -rf logs +ln -s ${LOG_DIR} logs diff --git a/python/ppc_model_gateway/tools/stop.sh b/python/ppc_model_gateway/tools/stop.sh new file mode 100644 index 00000000..fd17dcea --- /dev/null +++ b/python/ppc_model_gateway/tools/stop.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +sleep 1 + +ppc_model_gateway_app_pid=`ps aux |grep ppc_model_gateway_app.py |grep -v grep |awk '{print $2}'` +kill -9 $ppc_model_gateway_app_pid + +echo -e "\033[32mServer ppc_model_gateway_app.py killed. \033[0m" diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 00000000..14e6b826 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,65 @@ +cx-Oracle==8.3.0 +click>=8.0 +pytest +cheroot==8.5.2 +flask_restx==1.3.0 +configobj~=5.0.6 +Flask_SQLAlchemy==3.1.0 +cryptography~=41.0.5 +pandas +hypothesis~=5.48.0 +parsimonious~=0.8.1 +SQLAlchemy==2.0.16 +argcomplete~=1.12.2 +cytoolz~=0.10.1 +six~=1.15.0 +attrdict~=2.0.1 +Flask~=2.2.5 +pymitter~=0.3.0 +requests~=2.31.0 +requests_toolbelt==0.9.1 +lru_dict==1.1.6 +promise~=2.3 +#protobuf==3.19.0 +# protobuf>=4.21.6,<5.0dev +protobuf>=5.27.1 +pycryptodome==3.9.9 +PyJWT==2.4.0 +PyYAML==5.4.1 +# sha3==0.2.1 +mysqlclient==2.1.0 +waitress==2.1.2 +sqlparse~=0.4.1 +ecdsa==0.19.0 +toolz~=0.11.1 +tenacity==7.0.0 +coincurve~=13.0.0 +google~=3.0.0 +paste~=3.5.0 +func_timeout==4.3.0 +cheroot==8.5.2 +prefect==0.14.15 +gmssl~=3.2.1 +readerwriterlock~=1.0.4 +jsoncomment~=0.2.3 +matplotlib~=3.2.2 +seaborn~=0.10.1 +sqlvalidator==0.0.17 +requests-toolbelt==0.9.1 +hdfs +scikit-learn~=0.24.2 +gmpy2 +openpyxl +networkx +pydot +snowland-smx +numpy==1.23.1 +graphviz~=0.20.1 +grpcio==1.62.1 +grpcio-tools==1.62.1 +xlrd~=1.0.0 +MarkupSafe>=2.1.1 +Werkzeug==2.3.8 +urllib3==1.26.18 +phe +chardet diff --git a/python/tools/fake_id_data.py b/python/tools/fake_id_data.py new file mode 100644 index 00000000..d78cc9ac --- /dev/null +++ b/python/tools/fake_id_data.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +import argparse +from enum import Enum +import datetime +import random +import string +import sys + + +class IDType(Enum): + CreditID = "credit_id", + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class WithTimeType(Enum): + Empty = "none", + Random = "random", + ALL = "all", + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + @classmethod + def value_of(cls, label): + if label in cls.Empty.value: + return cls.Empty + elif label in cls.Random.value: + return cls.Random + elif label in cls.ALL.value: + return cls.ALL + else: + raise NotImplementedError + + +def parse_args(): + parser = argparse.ArgumentParser(prog=sys.argv[0]) + parser.add_argument("-r", '--dataset_file', + help='the file to store the faked data', required=True) + parser.add_argument("-p", '--peer_dataset_size', + help='the peer dataset size', default=0, required=False) + parser.add_argument("-j", '--join_id_count', + help='the joined id count', default=0, required=False) + parser.add_argument("-c", '--id_count', + help='the id count', required=False) + parser.add_argument("-f", '--id_file', + help='the id file', required=False) + + parser.add_argument("-t", '--with_time', + help=f'generate id information with time({WithTimeType.Empty.value} means not with time, random means only generate a random time for one id, all means generate (end_date-start_date) times for one id)', default=WithTimeType.Empty.value[0], required=False) + + parser.add_argument("-s", '--start_date', + help='the start time(only useful when --with_time is setted)', required=False) + parser.add_argument("-e", '--end_date', + help='the end time(only useful when --with_time is setted)', required=False) + parser.add_argument("-I", '--id_type', + help=f'the id type, currently only support {IDType.CreditID.value}', + default=IDType.CreditID.value, required=False) + args = parser.parse_args() + return args + + +def generate_credit_id(): + code = random.choice(string.digits + string.ascii_uppercase) + code += random.choice(string.digits + string.ascii_uppercase) + code += ''.join(random.choices(string.digits, k=6)) + code += ''.join(random.choices(string.digits, k=9)) + code += random.choice(string.digits + string.ascii_uppercase) + return code + + +def generate_id(id_type): + if IDType.has_value(id_type) is False: + error_msg = f"Unsupported id type: {id_type}" + raise Exception(error_msg) + id_type = IDType(id_type) + if id_type == IDType.CreditID: + return generate_credit_id() + + +def generate_random_time(start_date, end_date): + start_time = datetime.time(8, 0, 0) + random_date = start_date + \ + (end_date - start_date) * random.random() + random_time = datetime.datetime.combine( + random_date, start_time) + datetime.timedelta(seconds=random.randint(0, 86399)) + return random_time.strftime("%Y-%m-%d") + + +def write_line_data(fp, with_time, id_data, start_date, end_date): + if with_time is WithTimeType.Random: + line_data = id_data + "," + \ + generate_random_time(start_date, end_date) + "\n" + fp.write(line_data) + elif with_time is WithTimeType.ALL: + for i in range((end_date - start_date).days + 1): + day = start_date + datetime.timedelta(days=i) + line_data = id_data + "," + day.strftime("%Y-%m-%d") + "\n" + fp.write(line_data) + else: + fp.write(id_data + "\n") + + +def generate_header(dataset_file, with_time): + with open(dataset_file, "a+") as fp: + # with the header + fp.write("id") + if with_time is not WithTimeType.Empty: + fp.write(",time") + fp.write("\n") + + +def generate_dataset(id_count, id_type, dataset_file, with_time, start_date, end_date, joined_fp, joined_count, id_file): + with open(dataset_file, "a+") as f: + # generate with id_count + if id_file is None: + epoch = int(id_count) * 0.1 + for i in range(int(id_count)): + if i % epoch == 0: + print( + f"#### generate {epoch}/{id_count} records for: {dataset_file}") + id_data = generate_id(id_type) + write_line_data(f, with_time, id_data, start_date, end_date) + if joined_fp is not None and joined_count > 0: + joined_count -= 1 + write_line_data(joined_fp, with_time, + id_data, start_date, end_date) + # generate with id_file + else: + with open(id_file, "r") as id_fp: + # skip the header + id_data = id_fp.readline().strip() + if id_data is not None: + id_data = id_fp.readline().strip() + while id_data is not None and id_data != '': + write_line_data(f, with_time, id_data, + start_date, end_date) + if joined_fp is not None and joined_count > 0: + joined_count -= 1 + write_line_data(joined_fp, with_time, + id_data, start_date, end_date) + id_data = id_fp.readline().strip() + + print(f"##### fake_data for {dataset_file} success") + + +def check_args(args): + if args.id_count is None and args.id_file is None: + raise Exception("Must set id_count or id_file") + + +def fake_data(args): + check_args(args) + end_date = datetime.datetime.now() + if args.end_date is not None: + end_date = datetime.datetime.strptime(args.end_date, "%Y-%m-%d") + start_date = datetime.datetime.now() + datetime.timedelta(days=-2 * 365) + if args.start_date is not None: + start_date = datetime.datetime.strptime(args.start_date, "%Y-%m-%d") + with_time = WithTimeType.Empty + if args.with_time is not None: + with_time = WithTimeType.value_of(args.with_time) + + joined_dataset_path = args.dataset_file + ".peer" + joined_dataset_fp = None + if int(args.join_id_count) > 0: + generate_header(joined_dataset_path, with_time) + if int(args.peer_dataset_size) > int(args.join_id_count): + expected_peer_data_size = int( + args.peer_dataset_size) - int(args.join_id_count) + generate_dataset(expected_peer_data_size, args.id_type, + joined_dataset_path, with_time, start_date, end_date, None, 0, None) + joined_dataset_fp = open(joined_dataset_path, "a+") + + generate_header(args.dataset_file, with_time) + generate_dataset(args.id_count, args.id_type, + args.dataset_file, with_time, start_date, end_date, joined_dataset_fp, int(args.join_id_count), args.id_file) + if joined_dataset_fp is not None: + joined_dataset_fp.close() + + +if __name__ == "__main__": + args = parse_args() + fake_data(args) diff --git a/python/tools/fake_ml_train_data.py b/python/tools/fake_ml_train_data.py new file mode 100644 index 00000000..65a152d6 --- /dev/null +++ b/python/tools/fake_ml_train_data.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +import numpy as np +import sys +import os +import argparse +from enum import Enum +import random + + +class DataType(Enum): + TRAIN = "train", + PREDICT = "predict", + ALL = "all" + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + @classmethod + def value_of(cls, label): + if label in cls.TRAIN.value: + return cls.TRAIN + elif label in cls.PREDICT.value: + return cls.PREDICT + elif label in cls.ALL.value: + return cls.ALL + else: + raise NotImplementedError + + +def parse_args(): + parser = argparse.ArgumentParser(prog=sys.argv[0]) + parser.add_argument("-f", '--feature_size', + help='the feature size', required=True) + parser.add_argument("-s", '--sample_capacity', + help='the faked data size(in GB, default 1GB)', default=1, required=False) + parser.add_argument("-d", '--id_file', + help='the id file', required=False) + parser.add_argument("-S", '--sample_file', + help='the file to store the faked data', required=True) + parser.add_argument("-i", "--start_id", + help="the start id", required=False, default=0) + parser.add_argument("-I", "--ignore_header", + help="ignore the file header", required=False, default=False) + parser.add_argument("-m", "--with_missing_value", help="with missing value", + required=False, default=False) + parser.add_argument("-p", "--missing_percent", required=False, + default=10, help="the missing value percentage") + parser.add_argument("-t", "--data_type", required=False, + default=DataType.ALL.value, help="the dataType, now support 'all'/'predict'/'train'") + parser.add_argument("-F", "--feature_prefix", required=False, + default="x", help="the feature prefix") + args = parser.parse_args() + return args + + +def generate_id_list(id_fp, granularity, id): + if id_fp is None: + return (granularity, np.array(range(id, id + granularity), dtype=str)) + lines = id_fp.readlines(granularity * 1024) + data = [line.split(",")[0].strip() for line in lines] + return (len(data), data) + + +class FileInfo: + def __init__(self, fp, fmt_setting): + self.fp = fp + self.fmt_setting = fmt_setting + + +def generate_header_for_given_data(file_name, ignore_header, data_type, feature_size, feature_prefix): + if ignore_header: + return + # write the header + f = open(file_name, "a") + # write the header + fmt_setting = "%s" + if data_type == DataType.PREDICT: + f.write("id") + if data_type == DataType.TRAIN: + f.write("id,y") + fmt_setting = "%s,%s" + for j in range(feature_size): + fmt_setting = fmt_setting + ",%s" + f.write(f",{feature_prefix}{str(j)}") + f.write("\n") + return FileInfo(f, fmt_setting) + + +def generate_header(args, data_type, feature_size): + train_file_info = None + predict_file_info = None + if data_type == DataType.PREDICT: + predict_file_info = generate_header_for_given_data( + args.sample_file, args.ignore_header, data_type, feature_size, args.feature_prefix) + if data_type == DataType.TRAIN: + train_file_info = generate_header_for_given_data( + args.sample_file, args.ignore_header, data_type, feature_size, args.feature_prefix) + if data_type == DataType.ALL: + train_file_info = generate_header_for_given_data( + args.sample_file, args.ignore_header, DataType.TRAIN, feature_size, args.feature_prefix) + predict_file_info = generate_header_for_given_data( + args.sample_file + ".predict", args.ignore_header, DataType.PREDICT, feature_size, args.feature_prefix) + return (train_file_info, predict_file_info) + + +def fake_data(args): + data_type = DataType.value_of(args.data_type) + sample_capacity_bytes = None + if args.sample_capacity is not None: + sample_capacity_bytes = int(args.sample_capacity) * 1024 + feature_size = int(args.feature_size) + granularity = 100 + id = 0 + (train_file_info, predict_file_info) = generate_header( + args, data_type, feature_size) + if args.start_id is not None: + id = int(args.start_id) + id_fp = None + if args.id_file is not None: + id_fp = open(args.id_file, "r") + # skip the header + id_fp.readline() + # write the header + while True: + (rows, id_list) = generate_id_list(id_fp, granularity, id) + if rows == 0: + break + id += rows + feature_sample = np.random.standard_normal( + (rows, feature_size)) + if args.with_missing_value: + for i in range(int(args.missing_percent)): + selected_line = random.randrange(rows) + selected_feature = random.randrange(feature_size) + feature_sample[selected_line, selected_feature] = np.nan + if train_file_info is not None: + train_sample = feature_sample.astype("str") + y_list = np.random.randint(0, 2, rows) + # insert the y_list + train_sample = np.insert(train_sample, 0, values=y_list, axis=1) + # insert the id + train_sample = np.insert(train_sample.astype( + "str"), 0, values=id_list, axis=1) + np.set_printoptions(suppress=True, threshold=np.inf) + np.savetxt(train_file_info.fp, train_sample.astype( + "str"), fmt=train_file_info.fmt_setting) + if predict_file_info is not None: + predict_sample = feature_sample.astype("str") + # insert the id + predict_sample = np.insert(predict_sample.astype( + "str"), 0, values=id_list, axis=1) + np.set_printoptions(suppress=True, threshold=np.inf) + np.savetxt(predict_file_info.fp, predict_sample.astype( + "str"), fmt=predict_file_info.fmt_setting) + # check the file size + if args.id_file is None: + file_size = os.stat(args.sample_file).st_size + if file_size >= sample_capacity_bytes: + print(f"#### final id: {id}") + break + if train_file_info is not None: + train_file_info.fp.close() + if predict_file_info is not None: + predict_file_info.fp.close() + + +if __name__ == "__main__": + args = parse_args() + fake_data(args) diff --git a/python/tools/requirements.txt b/python/tools/requirements.txt new file mode 100644 index 00000000..24ce15ab --- /dev/null +++ b/python/tools/requirements.txt @@ -0,0 +1 @@ +numpy From b21a3feecee94e561081622a01f2947ae2c0df7b Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 21 Aug 2024 22:57:48 +0800 Subject: [PATCH 003/120] add changelog and ci (#3) * add changelog * add workflow * use vcpkg as submodule * fix ut --- .github/stale.yml | 17 + .github/workflows/cpp_workflow.yml | 75 ++++ .gitmodules | 3 + ChangeLog.md | 21 ++ README.md | 26 +- cpp/CMakeLists.txt | 2 +- cpp/cmake/InstallBcosUtilities.cmake | 2 +- cpp/cmake/Options.cmake | 4 + cpp/ppc-crypto-core/src/CMakeLists.txt | 3 +- cpp/ppc-gateway/test/unittests/MockCache.h | 104 +++--- cpp/ppc-mpc/CMakeLists.txt | 10 +- cpp/ppc-pir/tests/CMakeLists.txt | 2 +- cpp/ppc-pir/tests/FakeOtPIRFactory.h | 72 ++-- cpp/ppc-pir/tests/TestBaseOT.cpp | 390 +++++++++++---------- cpp/ppc-pir/tests/data/AysPreDataset.csv | 3 + cpp/ppc-psi/tests/CMakeLists.txt | 2 +- cpp/vcpkg | 1 + cpp/vcpkg-configuration.json | 2 +- cpp/vcpkg.json | 9 +- 19 files changed, 450 insertions(+), 298 deletions(-) create mode 100644 .github/stale.yml create mode 100644 .github/workflows/cpp_workflow.yml create mode 100644 .gitmodules create mode 100644 ChangeLog.md create mode 100644 cpp/ppc-pir/tests/data/AysPreDataset.csv create mode 160000 cpp/vcpkg diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 00000000..e6e32871 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,17 @@ +# Number of days of inactivity before an issue becomes stale +daysUntilStale: 120 +# Number of days of inactivity before a stale issue is closed +daysUntilClose: 7 +# Issues with these labels will never be considered stale +exemptLabels: + - pinned + - security +# Label to use when marking an issue as stale +staleLabel: wontfix +# Comment to post when marking an issue as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. +# Comment to post when closing a stale issue. Set to `false` to disable +closeComment: false diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml new file mode 100644 index 00000000..bdec90b6 --- /dev/null +++ b/.github/workflows/cpp_workflow.yml @@ -0,0 +1,75 @@ +name: WeDPR-Component ci(cpp) +on: + push: + paths-ignore: + - "docs/**" + - "Changelog.md" + - "README.md" + pull_request: + paths-ignore: + - "docs/**" + - "Changelog.md" + - "README.md" + release: + types: [published, push] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + build: + name: build + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [macos-12, ubuntu-20.04] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 5 + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-28 + override: true + - name: Prepare vcpkg + if: runner.os != 'Windows' + uses: friendlyanon/setup-vcpkg@v1 + with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 } + - uses: actions/cache@v2 + id: deps_cache + with: + path: | + deps/ + c:/vcpkg + !c:/vcpkg/.git + !c:/vcpkg/buildtrees + !c:/vcpkg/packages + !c:/vcpkg/downloads + key: build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + restore-keys: | + build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + build-${{ matrix.os }}-${{ github.base_ref }}- + build-${{ matrix.os }}- + - name: Build for linux + if: runner.os == 'Linux' + run: | + sudo apt install -y lcov ccache wget libgmp-dev python3-dev + export GCC='gcc-10' + export CXX='g++-10' + bash cpp/tools/install_depends.sh -o ubuntu + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + make -j3 + - name: Build for macos + if: runner.os == 'macOS' + run: | + bash cpp/tools/install_depends.sh -o macos + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + make -j3 + - name: Test + if: runner.os != 'Windows' + run: | + cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest + make cov \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..b647ee05 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "cpp/vcpkg"] + path = cpp/vcpkg + url = https://github.com/microsoft/vcpkg diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 00000000..8b7b54d4 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,21 @@ +### 1.0.0-rc1 +(2024-08-21) + +**新增** + +#### 隐私计算组件 + +- **PSI**: 实现多种类型隐私求交集算法，包括CM2020, RA2018, ECDH-PSI, ECDH-Multi-PSI等 +- **MPC**: 安全多方计算相关组件 +- **MPCSQL**: 基于安全多方计算协议，支持联合分析查询任务 +- **PIR**: 匿踪查询组件 + +#### 隐私计算互联互通 + +- ECDH PSI算法与隐语互联互通 + +#### 隐私建模组件 + +- 2+方的多方联合XGB组件（训练 + 离线预测） +- 2+方多方特征工程组件（特征分箱，WOE/IV计算等） +- 预处理组件 \ No newline at end of file diff --git a/README.md b/README.md index b984af1d..8bdd4c71 100644 --- a/README.md +++ b/README.md @@ -1 +1,25 @@ -# WeDPR-Component \ No newline at end of file +# WeDPR + +![](https://wedpr-lab.readthedocs.io/zh_CN/latest/_static/images/wedpr_logo.png) + + +[![CodeFactor](https://www.codefactor.io/repository/github/webankblockchain/wedpr-component/badge?s=a4c3fb6ffd39e7618378fe13b6bd06c5846cc103)](https://www.codefactor.io/repository/github/webankblockchain/wedpr-component) +[![contributors](https://img.shields.io/github/contributors/WeBankBlockchain/WeDPR)](https://github.com/WeBankBlockchain/WeDPR-Component/graphs/contributors) +[![GitHub activity](https://img.shields.io/github/commit-activity/m/WeBankBlockchain/WeDPR-Component)](https://github.com/WeBankBlockchain/WeDPR-Component/pulse) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) + +微众银行多方大数据隐私计算平台[WeDPR](https://github.com/WeBankBlockchain/WeDPR)核心组件库，包括： + +- 丰富的隐私计算算法组件，支持隐私求交集、匿踪查询、多方联合分析(隐私SQL)、数据预处理、隐私建模和预测等能力，以满足多样化的业务场景 +- 统一网关：稳定的跨机构网络通信组件，支持基于Rip协议的路由转发协议 + + +## 技术文档 + +- [文档](https://wedpr-lab.readthedocs.io/zh-cn/latest/) +- [代码](https://github.com/WeBankBlockchain/WeDPR-Component) + + +## License + +WeDPR-Component的开源协议为Apache License 2.0, 详情参见[LICENSE](LICENSE)。 \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 42aef4a7..f15fcba6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -55,6 +55,7 @@ include(BuildInfoGenerator) find_package(OpenSSL REQUIRED) include(InstallBcosUtilities) +find_package(Boost COMPONENTS unit_test_framework) if(BUILD_SDK) ##### the sdk-dependencies ##### # find JNI @@ -89,7 +90,6 @@ if(NOT BUILD_SDK AND NOT BUILD_UDF) ##### the full-dependencies ##### find_package(TBB REQUIRED) find_package(jsoncpp REQUIRED) - find_package(Boost REQUIRED unit_test_framework) find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) # tcmalloc diff --git a/cpp/cmake/InstallBcosUtilities.cmake b/cpp/cmake/InstallBcosUtilities.cmake index 5fc51ace..83ad5079 100644 --- a/cpp/cmake/InstallBcosUtilities.cmake +++ b/cpp/cmake/InstallBcosUtilities.cmake @@ -1,3 +1,3 @@ -find_package(Boost COMPONENTS log filesystem chrono thread serialization iostreams system) +find_package(Boost COMPONENTS log filesystem chrono thread serialization iostreams system ) find_package(ZLIB REQUIRED) find_package(bcos-utilities REQUIRED) \ No newline at end of file diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index ab407a53..74a16d18 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -93,6 +93,10 @@ elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") endif () endif () +if(ENABLE_SSE) + list(APPEND VCPKG_MANIFEST_FEATURES "sse") +endif() + set(ENABLE_CPU_FEATURES OFF) # only ENABLE_CPU_FEATURES for aarch64 and x86 if ("${ARCHITECTURE}" MATCHES "aarch64") diff --git a/cpp/ppc-crypto-core/src/CMakeLists.txt b/cpp/ppc-crypto-core/src/CMakeLists.txt index ed0fa3f2..d40633ab 100644 --- a/cpp/ppc-crypto-core/src/CMakeLists.txt +++ b/cpp/ppc-crypto-core/src/CMakeLists.txt @@ -5,4 +5,5 @@ find_package(OpenSSL REQUIRED) message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}") message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}") -target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file +find_package(unofficial-sodium REQUIRED) +target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC unofficial-sodium::sodium ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-gateway/test/unittests/MockCache.h b/cpp/ppc-gateway/test/unittests/MockCache.h index f052b652..a974faed 100644 --- a/cpp/ppc-gateway/test/unittests/MockCache.h +++ b/cpp/ppc-gateway/test/unittests/MockCache.h @@ -21,66 +21,66 @@ #include "ppc-framework/storage/CacheStorage.h" #include -namespace ppc::mock -{ -class MockCache : public storage::CacheStorage -{ +namespace ppc::mock { +class MockCache : public storage::CacheStorage { public: - using Ptr = std::shared_ptr; - MockCache() = default; - ~MockCache() override {} + using Ptr = std::shared_ptr; + MockCache() = default; + ~MockCache() override {} - /// Note: all these interfaces throws exception when error happened - /** - * @brief: check whether the key exists - * @param _key: key - * @return whether the key exists - */ - bool exists(const std::string& _key) override { return m_kv.find(_key) != m_kv.end(); } - - /** - * @brief: set key value - * @param _expirationTime: timeout of key, seconds - */ - void setValue(const std::string& _key, const std::string& _value, - int32_t _expirationSeconds = -1) override - { - m_kv.emplace(_key, _value); - } + /// Note: all these interfaces throws exception when error happened + /** + * @brief: check whether the key exists + * @param _key: key + * @return whether the key exists + */ + bool exists(const std::string &_key) override { + return m_kv.find(_key) != m_kv.end(); + } + /** + * @brief: set key value + * @param _expirationTime: timeout of key, seconds + */ + void setValue(const std::string &_key, const std::string &_value, + int32_t _expirationSeconds = -1) override { + m_kv.emplace(_key, _value); + } - /** - * @brief: get value by key - * @param _key: key - * @return value - */ - Optional getValue(const std::string& _key) override - { - auto it = m_kv.find(_key); - if (it == m_kv.end()) - { - return std::nullopt; - } - - return it->second; + /** + * @brief: get value by key + * @param _key: key + * @return value + */ + std::optional getValue(const std::string &_key) override { + auto it = m_kv.find(_key); + if (it == m_kv.end()) { + return std::nullopt; } - /** - * @brief: set a timeout on key - * @param _expirationTime: timeout of key, ms - * @return whether setting is successful - */ - bool expireKey(const std::string& _key, uint32_t _expirationTime) override { return true; } + return it->second; + } + + /** + * @brief: set a timeout on key + * @param _expirationTime: timeout of key, ms + * @return whether setting is successful + */ + bool expireKey(const std::string &_key, uint32_t _expirationTime) override { + return true; + } - /** - * @brief: delete key - * @param _key: key - * @return the number of key deleted - */ - uint64_t deleteKey(const std::string& _key) override { return m_kv.erase(_key); } + /** + * @brief: delete key + * @param _key: key + * @return the number of key deleted + */ + uint64_t deleteKey(const std::string &_key) override { + return m_kv.erase(_key); + } private: - std::unordered_map> m_kv; + std::unordered_map> m_kv; }; -} // namespace ppc::mock \ No newline at end of file +} // namespace ppc::mock \ No newline at end of file diff --git a/cpp/ppc-mpc/CMakeLists.txt b/cpp/ppc-mpc/CMakeLists.txt index cdb60d56..53720a81 100644 --- a/cpp/ppc-mpc/CMakeLists.txt +++ b/cpp/ppc-mpc/CMakeLists.txt @@ -1,9 +1,9 @@ project(ppc-mpc VERSION ${VERSION}) add_subdirectory(src) -if (TESTS) - enable_testing() - set(CTEST_OUTPUT_ON_FAILURE TRUE) - add_subdirectory(tests) -endif() +#if (TESTS) +# enable_testing() +# set(CTEST_OUTPUT_ON_FAILURE TRUE) +# add_subdirectory(tests) +#endif() diff --git a/cpp/ppc-pir/tests/CMakeLists.txt b/cpp/ppc-pir/tests/CMakeLists.txt index c7255ab0..6a969962 100644 --- a/cpp/ppc-pir/tests/CMakeLists.txt +++ b/cpp/ppc-pir/tests/CMakeLists.txt @@ -9,4 +9,4 @@ target_include_directories(${TEST_BINARY_NAME} PRIVATE .) # target_link_libraries(${TEST_BINARY_NAME} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) -add_test(NAME test-ays WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file +add_test(NAME test-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-pir/tests/FakeOtPIRFactory.h b/cpp/ppc-pir/tests/FakeOtPIRFactory.h index 7393a0b8..c38a1b71 100644 --- a/cpp/ppc-pir/tests/FakeOtPIRFactory.h +++ b/cpp/ppc-pir/tests/FakeOtPIRFactory.h @@ -19,10 +19,10 @@ */ #pragma once +#include "ppc-crypto-core/src/hash/BLAKE2bHash.h" +#include "ppc-crypto-core/src/hash/Sha512Hash.h" #include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" #include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" -#include "ppc-crypto/src/hash/BLAKE2bHash.h" -#include "ppc-crypto/src/hash/Sha512Hash.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-io/src/DataResourceLoaderImpl.h" #include "ppc-tools/src/config/PPCConfig.h" @@ -38,55 +38,51 @@ using namespace ppc::io; using namespace ppc::front; using namespace ppc::tools; -namespace ppc::test -{ +namespace ppc::test { -class FakeOtPIRImpl : public OtPIRImpl -{ +class FakeOtPIRImpl : public OtPIRImpl { public: - using Ptr = std::shared_ptr; - FakeOtPIRImpl(OtPIRConfig::Ptr const& _config, unsigned _idleTimeMs = 0) - : OtPIRImpl(_config, _idleTimeMs) - { - m_enableOutputExists = true; - } - ~FakeOtPIRImpl() override = default; + using Ptr = std::shared_ptr; + FakeOtPIRImpl(OtPIRConfig::Ptr const &_config, unsigned _idleTimeMs = 0) + : OtPIRImpl(_config, _idleTimeMs) { + m_enableOutputExists = true; + } + ~FakeOtPIRImpl() override = default; }; -class FakeOtPIRFactory : public OtPIRFactory -{ +class FakeOtPIRFactory : public OtPIRFactory { public: - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; - FakeOtPIRFactory() + FakeOtPIRFactory() : m_front(std::make_shared()), m_dataResourceLoader(std::make_shared( nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), - m_threadPool(std::make_shared("ot-pir", 4)) - { - auto hashImpl = std::make_shared(); - auto eccCrypto = std::make_shared(hashImpl, ppc::protocol::ECCCurve::P256); - m_cryptoBox = std::make_shared(hashImpl, eccCrypto); - } + m_threadPool(std::make_shared("ot-pir", 4)) { + auto hashImpl = std::make_shared(); + auto eccCrypto = std::make_shared( + hashImpl, ppc::protocol::ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } - ~FakeOtPIRFactory() override = default; + ~FakeOtPIRFactory() override = default; - OtPIRImpl::Ptr createOtPIR(std::string const& _selfParty) - { - auto config = std::make_shared( - _selfParty, m_front, m_cryptoBox, m_threadPool, m_dataResourceLoader, 1); + OtPIRImpl::Ptr createOtPIR(std::string const &_selfParty) { + auto config = + std::make_shared(_selfParty, m_front, m_cryptoBox, + m_threadPool, m_dataResourceLoader, 1); - return std::make_shared(config); - } + return std::make_shared(config); + } - DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } - FakeFront::Ptr front() { return m_front; } - CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } private: - FakeFront::Ptr m_front; - DataResourceLoaderImpl::Ptr m_dataResourceLoader; - ThreadPool::Ptr m_threadPool; - CryptoBox::Ptr m_cryptoBox; + FakeFront::Ptr m_front; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; + CryptoBox::Ptr m_cryptoBox; }; -} // namespace ppc::test \ No newline at end of file +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/ppc-pir/tests/TestBaseOT.cpp b/cpp/ppc-pir/tests/TestBaseOT.cpp index 7aa264e7..9f5477fb 100644 --- a/cpp/ppc-pir/tests/TestBaseOT.cpp +++ b/cpp/ppc-pir/tests/TestBaseOT.cpp @@ -18,19 +18,18 @@ * @date 2023-03-13 */ #include "FakeOtPIRFactory.h" +#include "ppc-crypto-core/src/hash/HashFactoryImpl.h" +#include "ppc-crypto/src/ecc/EccCryptoFactoryImpl.h" +#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" #include "ppc-framework/protocol/Protocol.h" +#include "ppc-pir/src/BaseOT.h" #include "ppc-pir/src/Common.h" #include "ppc-pir/src/OtPIRImpl.h" -#include "ppc-pir/src/BaseOT.h" +#include "test-utils/TaskMock.h" #include #include #include #include -#include "ppc-crypto/src/ecc/OpenSSLEccCrypto.h" -#include "ppc-crypto/src/ecc/EccCryptoFactoryImpl.h" -#include "ppc-crypto/src/hash/HashFactoryImpl.h" -#include "test-utils/TaskMock.h" - using namespace ppc::pir; using namespace ppc::crypto; @@ -38,213 +37,216 @@ using namespace ppc::pir; using namespace bcos; using namespace bcos::test; - -namespace ppc::test -{ +namespace ppc::test { BOOST_FIXTURE_TEST_SUITE(OtPIRest, TestPromptFixture) - -BOOST_AUTO_TEST_CASE(testBaseOT) -{ - // 统计函数执行时间 - std::cout<< "testBaseOT" << std::endl; - - auto eccFactory = std::make_shared(); - auto hashFactory = std::make_shared(); - auto hash = hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512); - EccCrypto::Ptr ecc = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hash); - // auto aysService = std::make_shared(); - // auto aes = std::make_shared( - // OpenSSLAES::AESType::AES128, - // SymCrypto::OperationMode::CBC, _seed, bcos::bytes()); - - - std::string datasetPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; - // std::cout<< "aysService->prepareDataset" << std::endl; - std::string prefix = "testmsg1"; - bcos::bytes sendObfuscatedHash(prefix.begin(), prefix.end()); - - - uint32_t obfuscatedOrder = 10; - auto baseOT = std::make_shared(ecc, hash); - auto messageKeypair = baseOT->prepareDataset(sendObfuscatedHash, datasetPath); - // for(auto iter = messageKeypair.begin(); iter != messageKeypair.end(); ++iter) - // { - // std::string pairKey(iter->first.begin(), iter->first.end()); - // std::string pairValue(iter->second.begin(), iter->second.end()); - // // for(uint32_t i = 0; i < messageKeypair.size(); ++i) { - // // std::cout<< "pairKey:"<< pairKey << std::endl; - // // std::cout<< "pairValue:"<< pairValue << std::endl; - // } - auto start = std::chrono::high_resolution_clock::now(); - - std::string choice = "testmsg1100"; - // std::cout<< "baseOT->senderGenerateCipher" << std::endl; - auto senderMessage = baseOT->senderGenerateCipher(bcos::bytes(choice.begin(), choice.end()), obfuscatedOrder); - // std::cout<< "baseOT->receiverGenerateMessage" << std::endl; - auto receiverMessage = baseOT->receiverGenerateMessage(senderMessage.pointX, senderMessage.pointY, messageKeypair, senderMessage.pointZ); - - // std::cout<< "baseOT->finishSender" << std::endl; - auto result = baseOT->finishSender(senderMessage.scalarBlidingB, receiverMessage.pointWList, receiverMessage.encryptMessagePair, receiverMessage.encryptCipher); - - auto end = std::chrono::high_resolution_clock::now(); - - auto duration = std::chrono::duration_cast(end - start); - std::cout << "执行时间 time: " << duration.count() << " microseconds" << std::endl; - - if(result.size() == 0){ - std::cout<< "final result: message not found" << std::endl; - } - else { - std::cout<< "final result: " << std::string(result.begin(), result.end()) << std::endl; - } - // for(uint32_t i = 0; i < result.size(); ++i) - // { - // std::cout<< std::string(result[i].begin(), result[i].end()) << std::endl; - // } - - BOOST_CHECK(true); +BOOST_AUTO_TEST_CASE(testBaseOT) { + // 统计函数执行时间 + std::cout << "testBaseOT" << std::endl; + + auto eccFactory = std::make_shared(); + auto hashFactory = std::make_shared(); + auto hash = + hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512); + EccCrypto::Ptr ecc = + eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hash); + // auto aysService = std::make_shared(); + // auto aes = std::make_shared( + // OpenSSLAES::AESType::AES128, + // SymCrypto::OperationMode::CBC, _seed, bcos::bytes()); + + std::string datasetPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; + // std::cout<< "aysService->prepareDataset" << std::endl; + std::string prefix = "testmsg1"; + bcos::bytes sendObfuscatedHash(prefix.begin(), prefix.end()); + + uint32_t obfuscatedOrder = 10; + auto baseOT = std::make_shared(ecc, hash); + auto messageKeypair = baseOT->prepareDataset(sendObfuscatedHash, datasetPath); + // for(auto iter = messageKeypair.begin(); iter != messageKeypair.end(); + // ++iter) + // { + // std::string pairKey(iter->first.begin(), iter->first.end()); + // std::string pairValue(iter->second.begin(), iter->second.end()); + // // for(uint32_t i = 0; i < messageKeypair.size(); ++i) { + // // std::cout<< "pairKey:"<< pairKey << std::endl; + // // std::cout<< "pairValue:"<< pairValue << std::endl; + // } + auto start = std::chrono::high_resolution_clock::now(); + + std::string choice = "testmsg1100"; + // std::cout<< "baseOT->senderGenerateCipher" << std::endl; + auto senderMessage = baseOT->senderGenerateCipher( + bcos::bytes(choice.begin(), choice.end()), obfuscatedOrder); + // std::cout<< "baseOT->receiverGenerateMessage" << std::endl; + auto receiverMessage = baseOT->receiverGenerateMessage( + senderMessage.pointX, senderMessage.pointY, messageKeypair, + senderMessage.pointZ); + + // std::cout<< "baseOT->finishSender" << std::endl; + auto result = baseOT->finishSender( + senderMessage.scalarBlidingB, receiverMessage.pointWList, + receiverMessage.encryptMessagePair, receiverMessage.encryptCipher); + + auto end = std::chrono::high_resolution_clock::now(); + + auto duration = + std::chrono::duration_cast(end - start); + std::cout << "执行时间 time: " << duration.count() << " microseconds" + << std::endl; + + if (result.size() == 0) { + std::cout << "final result: message not found" << std::endl; + } else { + std::cout << "final result: " << std::string(result.begin(), result.end()) + << std::endl; + } + // for(uint32_t i = 0; i < result.size(); ++i) + // { + // std::cout<< std::string(result[i].begin(), result[i].end()) << + // std::endl; + // } + + BOOST_CHECK(true); } - void testOTPIR(FakeOtPIRFactory::Ptr _factory, OtPIRImpl::Ptr _sender, - OtPIRImpl::Ptr _receiver, ppc::protocol::Task::ConstPtr _senderPirTask, - ppc::protocol::Task::ConstPtr _receiverPirTask, - std::vector const& _expectedPIRResult, bool _expectedSuccess, - int _expectedErrorCode = 0) -{ - std::atomic flag = 0; - - _sender->asyncRunTask(_senderPirTask, [_senderPirTask, _expectedSuccess, _expectedErrorCode, - &flag](ppc::protocol::TaskResult::Ptr&& _response) { - if (_expectedSuccess) - { - BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); - BOOST_CHECK(_response->taskID() == _senderPirTask->id()); - auto result = _response->error(); - BOOST_CHECK(result == nullptr || result->errorCode() == 0); + OtPIRImpl::Ptr _receiver, + ppc::protocol::Task::ConstPtr _senderPirTask, + ppc::protocol::Task::ConstPtr _receiverPirTask, + std::vector const &_expectedPIRResult, + bool _expectedSuccess, int _expectedErrorCode = 0) { + std::atomic flag = 0; + + _sender->asyncRunTask( + _senderPirTask, [_senderPirTask, _expectedSuccess, _expectedErrorCode, + &flag](ppc::protocol::TaskResult::Ptr &&_response) { + if (_expectedSuccess) { + BOOST_CHECK(_response->error() == nullptr || + _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _senderPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } else { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); } - else - { - BOOST_CHECK(_response->error() != nullptr); - auto result = _response->error(); - BOOST_CHECK(result != nullptr); - BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); + flag++; + }); + _sender->start(); + + _receiver->asyncRunTask( + _receiverPirTask, [_receiverPirTask, _expectedSuccess, + &flag](ppc::protocol::TaskResult::Ptr &&_response) { + if (_expectedSuccess) { + BOOST_CHECK(_response->error() == nullptr || + _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _receiverPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } else { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); } flag++; - }); - _sender->start(); - - _receiver->asyncRunTask(_receiverPirTask, - [_receiverPirTask, _expectedSuccess, &flag](ppc::protocol::TaskResult::Ptr&& _response) { - if (_expectedSuccess) - { - BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); - BOOST_CHECK(_response->taskID() == _receiverPirTask->id()); - auto result = _response->error(); - BOOST_CHECK(result == nullptr || result->errorCode() == 0); - } - else - { - BOOST_CHECK(_response->error() != nullptr); - auto result = _response->error(); - BOOST_CHECK(result != nullptr); - } - flag++; - }); - _receiver->start(); - - // wait for the task finish and check - while (flag < 2) - { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - _sender->stop(); - _receiver->stop(); - - // if (_expectedSuccess && !_expectedPIRResult.empty()) - // { - // checkTaskPIRResult(_factory->resourceLoader(), _receiverPirTask, _expectedPIRResult.size(), - // _expectedPIRResult); - // } + }); + _receiver->start(); + + // wait for the task finish and check + while (flag < 2) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + _sender->stop(); + _receiver->stop(); + + // if (_expectedSuccess && !_expectedPIRResult.empty()) + // { + // checkTaskPIRResult(_factory->resourceLoader(), _receiverPirTask, + // _expectedPIRResult.size(), + // _expectedPIRResult); + // } } - -void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, - bool _syncResults, PartyResource::Ptr _senderParty, PartyResource::Ptr _receiverParty, - std::vector const& _expectedPIRResult, bool _expectedSuccess, - int _expectedErrorCode = 0) -{ - auto factory = std::make_shared(); - - // fake the sender - std::string senderAgencyName = "sender"; - auto senderPIR = factory->createOtPIR("sender"); - - // fake the receiver - std::string receiverAgencyName = _receiverParty->id(); - auto receiverPIR = factory->createOtPIR(receiverAgencyName); - - // register the server-pir into the front - factory->front()->registerOTPIR(senderAgencyName, senderPIR); - factory->front()->registerOTPIR(receiverAgencyName, receiverPIR); - - // trigger the pir task - auto senderPIRTask = std::make_shared(senderAgencyName); - senderPIRTask->setId(_taskID); - senderPIRTask->setParam(_params); - senderPIRTask->setSelf(_senderParty); - senderPIRTask->addParty(_receiverParty); - senderPIRTask->setSyncResultToPeer(_syncResults); - senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); - senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); - - auto receiverPIRTask = std::make_shared(receiverAgencyName); - receiverPIRTask->setId(_taskID); - receiverPIRTask->setParam(_params); - receiverPIRTask->setSelf(_receiverParty); - receiverPIRTask->addParty(_senderParty); - receiverPIRTask->setSyncResultToPeer(_syncResults); - receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); - receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); - - testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, - _expectedPIRResult, _expectedSuccess, _expectedErrorCode); +void testOTPIRImplFunc(const std::string &_taskID, const std::string &_params, + bool _syncResults, PartyResource::Ptr _senderParty, + PartyResource::Ptr _receiverParty, + std::vector const &_expectedPIRResult, + bool _expectedSuccess, int _expectedErrorCode = 0) { + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = "sender"; + auto senderPIR = factory->createOtPIR("sender"); + + // fake the receiver + std::string receiverAgencyName = _receiverParty->id(); + auto receiverPIR = factory->createOtPIR(receiverAgencyName); + + // register the server-pir into the front + factory->front()->registerOTPIR(senderAgencyName, senderPIR); + factory->front()->registerOTPIR(receiverAgencyName, receiverPIR); + + // trigger the pir task + auto senderPIRTask = std::make_shared(senderAgencyName); + senderPIRTask->setId(_taskID); + senderPIRTask->setParam(_params); + senderPIRTask->setSelf(_senderParty); + senderPIRTask->addParty(_receiverParty); + senderPIRTask->setSyncResultToPeer(_syncResults); + senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + auto receiverPIRTask = std::make_shared(receiverAgencyName); + receiverPIRTask->setId(_taskID); + receiverPIRTask->setParam(_params); + receiverPIRTask->setSelf(_receiverParty); + receiverPIRTask->addParty(_senderParty); + receiverPIRTask->setSyncResultToPeer(_syncResults); + receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, + _expectedPIRResult, _expectedSuccess, _expectedErrorCode); } -BOOST_AUTO_TEST_CASE(testNormalOtPIRCase) -{ - std::string otSearchPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; - std::string outputPath = "../../../ppc-pir/tests/data/output.csv"; +BOOST_AUTO_TEST_CASE(testNormalOtPIRCase) { + std::string otSearchPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; + std::string outputPath = "../../../ppc-pir/tests/data/output.csv"; - uint32_t count = 513; - // prepareInputs(senderPath, count, receiverPath, count, count); + uint32_t count = 513; + // prepareInputs(senderPath, count, receiverPath, count, count); - auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, "sender", - "senderPartyResource", "sender_inputs", protocol::DataResourceType::FILE, ""); - auto senderOutputDesc = std::make_shared(); - senderOutputDesc->setPath(outputPath); - senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, + "sender", "senderPartyResource", "sender_inputs", + protocol::DataResourceType::FILE, ""); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(outputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); - auto receiverParty = mockParty((uint16_t)ppc::protocol::PartyType::Server, "receiver", - "receiverPartyResource", "receiver_inputs", DataResourceType::FILE, otSearchPath); + auto receiverParty = + mockParty((uint16_t)ppc::protocol::PartyType::Server, "receiver", + "receiverPartyResource", "receiver_inputs", + DataResourceType::FILE, otSearchPath); - // auto receiverOutputDesc = std::make_shared(); - // receiverOutputDesc->setPath(outputPath); - // receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + // auto receiverOutputDesc = std::make_shared(); + // receiverOutputDesc->setPath(outputPath); + // receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); - std::vector expectedResult; - for (uint32_t i = 0; i < count; i++) - { - expectedResult.emplace_back(std::to_string(100000 + i)); - } + std::vector expectedResult; + for (uint32_t i = 0; i < count; i++) { + expectedResult.emplace_back(std::to_string(100000 + i)); + } - std::string jobParams = "{\"searchId\":\"testmsg1100\",\"requestAgencyId\":\"receiver\",\"prefixLength\":6}"; + std::string jobParams = "{\"searchId\":\"testmsg1100\",\"requestAgencyId\":" + "\"receiver\",\"prefixLength\":6}"; - testOTPIRImplFunc( - "0x12345678", jobParams, true, senderParty, receiverParty, expectedResult, true, 0); + testOTPIRImplFunc("0x12345678", jobParams, true, senderParty, receiverParty, + expectedResult, true, 0); } BOOST_AUTO_TEST_SUITE_END() -} // namespace ppc::test +} // namespace ppc::test diff --git a/cpp/ppc-pir/tests/data/AysPreDataset.csv b/cpp/ppc-pir/tests/data/AysPreDataset.csv new file mode 100644 index 00000000..590b61f2 --- /dev/null +++ b/cpp/ppc-pir/tests/data/AysPreDataset.csv @@ -0,0 +1,3 @@ +id,x1 +1,test +2,test2 diff --git a/cpp/ppc-psi/tests/CMakeLists.txt b/cpp/ppc-psi/tests/CMakeLists.txt index f2fa68c3..26b00337 100644 --- a/cpp/ppc-psi/tests/CMakeLists.txt +++ b/cpp/ppc-psi/tests/CMakeLists.txt @@ -6,6 +6,6 @@ set(TEST_BINARY_NAME test-ppc-psi) add_executable(${TEST_BINARY_NAME} ${SOURCES}) target_include_directories(${TEST_BINARY_NAME} PRIVATE .) -target_link_libraries(${TEST_BINARY_NAME} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) +target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST}) add_test(NAME test-ppc-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/vcpkg b/cpp/vcpkg new file mode 160000 index 00000000..51b14cd4 --- /dev/null +++ b/cpp/vcpkg @@ -0,0 +1 @@ +Subproject commit 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json index 71d2a9ae..5913a7c1 100644 --- a/cpp/vcpkg-configuration.json +++ b/cpp/vcpkg-configuration.json @@ -3,7 +3,7 @@ { "kind": "git", "repository": "https://github.com/FISCO-BCOS/registry", - "baseline": "a3508ded2bb7f83d95dd3f7406b05b2500a1fdbe", + "baseline": "070f336149afdac5cc9ace97df01de7ee31aab30", "packages": [ "openssl", "bcos-utilities", diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index c85989f4..34c765ca 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -65,7 +65,7 @@ }, { "name": "libhdfs3", - "version": "2024-04-27" + "version": "2024-04-27" }, { "name": "tbb", @@ -75,5 +75,10 @@ "name": "tarscpp", "version": "3.0.3-1#1" } - ] + ], + "features": { + "sse": { + "description": "Enable SSE4.2 for libhdfs3" + } + } } \ No newline at end of file From ccec2073842042b8d1cd425f0d0f516f4e00b3fc Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 01:34:17 +0800 Subject: [PATCH 004/120] fix ci --- .github/workflows/cpp_workflow.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index bdec90b6..8d9f76e8 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -72,4 +72,10 @@ jobs: if: runner.os != 'Windows' run: | cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest - make cov \ No newline at end of file + make cov + - uses: webiny/action-post-run@3.1.0 + id: post-run-command + with: + if: runner.os == 'Linux' + run: cat vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log + working-directory: /home/runner/work/WeDPR-Component/WeDPR-Component/ \ No newline at end of file From e5b27b7c4ba191744912f218eab89aef1c1d8530 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 11:18:14 +0800 Subject: [PATCH 005/120] refactor cmake (#5) * refactor cmake * add build sdk ci --- .github/workflows/cpp_workflow.yml | 67 +++++++++++- cpp/CMakeLists.txt | 109 ++++++------------ cpp/cmake/Dependencies.cmake | 51 +++++++++ cpp/cmake/FindGSasl.cmake | 27 ----- cpp/cmake/InstallBcosUtilities.cmake | 2 +- cpp/cmake/Options.cmake | 146 ++++++++++++++----------- cpp/ppc-crypto-core/src/CMakeLists.txt | 4 +- cpp/ppc-crypto/src/CMakeLists.txt | 4 +- cpp/vcpkg.json | 78 +++++++------ 9 files changed, 280 insertions(+), 208 deletions(-) create mode 100644 cpp/cmake/Dependencies.cmake delete mode 100644 cpp/cmake/FindGSasl.cmake diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index 8d9f76e8..5f75d524 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -19,7 +19,7 @@ concurrency: jobs: build: - name: build + name: build all runs-on: ${{ matrix.os }} continue-on-error: true strategy: @@ -73,9 +73,64 @@ jobs: run: | cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest make cov - - uses: webiny/action-post-run@3.1.0 - id: post-run-command + + build_sdk: + name: build sdk + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, windows-2019, macos-12] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 5 + - uses: actions-rs/toolchain@v1 with: - if: runner.os == 'Linux' - run: cat vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log - working-directory: /home/runner/work/WeDPR-Component/WeDPR-Component/ \ No newline at end of file + toolchain: nightly-2022-07-28 + override: true + - name: Prepare vcpkg + if: runner.os != 'Windows' + uses: friendlyanon/setup-vcpkg@v1 + with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 } + - uses: actions/cache@v2 + id: deps_cache + with: + path: | + deps/ + c:/vcpkg + !c:/vcpkg/.git + !c:/vcpkg/buildtrees + !c:/vcpkg/packages + !c:/vcpkg/downloads + key: build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + restore-keys: | + build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + build-${{ matrix.os }}-${{ github.base_ref }}- + build-${{ matrix.os }}- + - name: Build for windows + if: runner.os == 'Windows' + run: | + mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_SDK=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ../ + cmake --build . --parallel 3 + - name: Build for linux + if: runner.os == 'Linux' + run: | + sudo apt install -y lcov ccache wget libgmp-dev python3-dev + export GCC='gcc-10' + export CXX='g++-10' + bash cpp/tools/install_depends.sh -o ubuntu + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + make -j3 + - name: Build for macos + if: runner.os == 'macOS' + run: | + bash cpp/tools/install_depends.sh -o macos + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + make -j3 + - name: Test + if: runner.os != 'Windows' + run: | + cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest + make cov \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f15fcba6..0fb668fc 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -35,6 +35,13 @@ if(WIN32) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") endif() + + +# basic settings +include(Options) +configure_project() + + # vcpkg init if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) find_package(Git REQUIRED) @@ -46,88 +53,40 @@ endif() include(Version) project(WeDPR-Component VERSION ${VERSION}) -# basic settings -include(Options) -configure_project() include(CompilerSettings) include(BuildInfoGenerator) - -find_package(OpenSSL REQUIRED) -include(InstallBcosUtilities) -find_package(Boost COMPONENTS unit_test_framework) -if(BUILD_SDK) - ##### the sdk-dependencies ##### - # find JNI - set(JAVA_AWT_LIBRARY NotNeeded) - set(JAVA_JVM_LIBRARY NotNeeded) - find_package(JNI REQUIRED) - include_directories(${JNI_INCLUDE_DIRS}) -endif() - -# ipp-crypto -#if(ENABLE_IPP_CRYPTO) -# hunter_add_package(ipp-crypto) -#endif() - -if(ENABLE_CPU_FEATURES) - find_package(CpuFeatures REQUIRED) -endif() - include(IncludeDirectories) +# the target settings include(TargetSettings) - -if(BUILD_SDK) - add_subdirectory(ppc-crypto-c-sdk) - set(JNI_SOURCE_PATH ppc-crypto-c-sdk/bindings/java/src/main/c) - add_subdirectory(${JNI_SOURCE_PATH}) +# dependencies +include(Dependencies) + +########### set the sources ########### +set(JNI_SOURCE_PATH ppc-crypto-c-sdk/bindings/java/src/main/c) +set(SDK_SOURCE_LIST ppc-homo ppc-crypto-core ppc-crypto-c-sdk ${JNI_SOURCE_PATH}) +# Note: the udf depends on mysql, not enabled in the full node mode +set(UDF_SOURCE_LIST ${SDK_SOURCE_LIST} ppc-udf) +set(ALL_SOURCE_LIST + ${SDK_SOURCE_LIST} ppc-crypto + libhelper libinitializer ppc-io ppc-protocol + ppc-gateway ppc-front ppc-tars-protocol + ppc-tools ppc-storage ppc-psi ppc-rpc + ppc-http ppc-mpc ppc-pir + ${CEM_SOURCE} ppc-main) +set(CEM_SOURCE "") +if(BUILD_CEM) + set(CEM_SOURCE "ppc-cem") endif() -add_subdirectory(ppc-crypto-core) -add_subdirectory(ppc-homo) - -# when BUILD_SDK, the following modules no need to compile -if(NOT BUILD_SDK AND NOT BUILD_UDF) - ##### the full-dependencies ##### - find_package(TBB REQUIRED) - find_package(jsoncpp REQUIRED) - - find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) - # tcmalloc - include(ProjectTCMalloc) - - find_package(SEAL REQUIRED) - find_package(Kuku REQUIRED) - - # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 - include(ProjectAPSI) - # Wedpr Crypto - include(ProjectWedprCrypto) - include(FindGSasl) - include(Installlibhdfs3) - - add_subdirectory(ppc-crypto) - add_subdirectory(libhelper) - add_subdirectory(libinitializer) - add_subdirectory(ppc-io) - add_subdirectory(ppc-protocol) - add_subdirectory(ppc-gateway) - add_subdirectory(ppc-front) - add_subdirectory(ppc-tars-protocol) - add_subdirectory(ppc-tools) - add_subdirectory(ppc-storage) - add_subdirectory(ppc-psi) - add_subdirectory(ppc-rpc) - add_subdirectory(ppc-http) - add_subdirectory(ppc-mpc) - add_subdirectory(ppc-pir) - if(BUILD_CEM) - add_subdirectory(ppc-cem) - endif () - add_subdirectory(ppc-main) -endif() -if(BUILD_UDF) - add_subdirectory(ppc-udf) + +if(BUILD_ALL) + add_sources("${ALL_SOURCE_LIST}") +elseif(BUILD_UDF) + add_sources("${UDF_SOURCE_LIST}") +elseif(BUILD_SDK) + add_sources("${SDK_SOURCE_LIST}") endif() +########### set the sources end ########### if (TESTS) enable_testing() diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake new file mode 100644 index 00000000..4354187e --- /dev/null +++ b/cpp/cmake/Dependencies.cmake @@ -0,0 +1,51 @@ + +# ipp-crypto +#if(ENABLE_IPP_CRYPTO) +# hunter_add_package(ipp-crypto) +#endif() + +######## common dependencies ######## +find_package(OpenSSL REQUIRED) +include(InstallBcosUtilities) + +if (TESTS) + find_package(Boost COMPONENTS unit_test_framework) +endif() + +# cpp_features +if(ENABLE_CPU_FEATURES) + find_package(CpuFeatures REQUIRED) +endif() +find_package(unofficial-sodium CONFIG REQUIRED) +######## common dependencies end ######## + + +##### the full-dependencies ##### +if(BUILD_ALL) + find_package(TBB REQUIRED) + find_package(jsoncpp REQUIRED) + + find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) + # tcmalloc + include(ProjectTCMalloc) + + find_package(SEAL REQUIRED) + find_package(Kuku REQUIRED) + + # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 + include(ProjectAPSI) + # Wedpr Crypto + include(ProjectWedprCrypto) + include(Installlibhdfs3) +endif() +##### the full-dependencies end ##### + +##### the sdk-dependencies ##### +if(BUILD_SDK) + # find JNI + set(JAVA_AWT_LIBRARY NotNeeded) + set(JAVA_JVM_LIBRARY NotNeeded) + find_package(JNI REQUIRED) + include_directories(${JNI_INCLUDE_DIRS}) +endif() +##### the sdk-dependencies end##### \ No newline at end of file diff --git a/cpp/cmake/FindGSasl.cmake b/cpp/cmake/FindGSasl.cmake deleted file mode 100644 index caefeea1..00000000 --- a/cpp/cmake/FindGSasl.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# - Try to find the GNU sasl library (gsasl) -# -# Once done this will define -# -# GSASL_FOUND - System has gnutls -# GSASL_INCLUDE_DIR - The gnutls include directory -# GSASL_LIBRARIES - The libraries needed to use gnutls -# GSASL_DEFINITIONS - Compiler switches required for using gnutls - - -IF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) - # in cache already - SET(GSasl_FIND_QUIETLY TRUE) -ENDIF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) - -FIND_PATH(GSASL_INCLUDE_DIR gsasl.h) - -FIND_LIBRARY(GSASL_LIBRARIES gsasl) -FIND_LIBRARY(GSASL_STATIC_LIBRARIES NAMES "libgsasl.a") - -INCLUDE(FindPackageHandleStandardArgs) - -# handle the QUIETLY and REQUIRED arguments and set GSASL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSASL DEFAULT_MSG GSASL_LIBRARIES GSASL_INCLUDE_DIR) - -MARK_AS_ADVANCED(GSASL_INCLUDE_DIR GSASL_LIBRARIES) \ No newline at end of file diff --git a/cpp/cmake/InstallBcosUtilities.cmake b/cpp/cmake/InstallBcosUtilities.cmake index 83ad5079..6177ba72 100644 --- a/cpp/cmake/InstallBcosUtilities.cmake +++ b/cpp/cmake/InstallBcosUtilities.cmake @@ -1,3 +1,3 @@ find_package(Boost COMPONENTS log filesystem chrono thread serialization iostreams system ) find_package(ZLIB REQUIRED) -find_package(bcos-utilities REQUIRED) \ No newline at end of file +find_package(bcos-utilities CONFIG REQUIRED) \ No newline at end of file diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 74a16d18..0588372d 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -9,6 +9,11 @@ macro(default_option O DEF) set(${O} ${DEF}) endif () endmacro() +macro(add_sources source_list) + foreach(source ${source_list}) + add_subdirectory(${source}) + endforeach() +endmacro() # common settings if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "4") @@ -44,89 +49,102 @@ macro(configure_project) default_option(ENABLE_DEMO OFF) # sdk + default_option(BUILD_ALL ON) default_option(BUILD_SDK OFF) default_option(BUILD_UDF OFF) - if(BUILD_UDF) - set(BUILD_SDK ON) - endif() # Suffix like "-rc1" e.t.c. to append to versions wherever needed. if (NOT DEFINED VERSION_SUFFIX) set(VERSION_SUFFIX "") endif () - print_config(${NAME}) -endmacro() -# for boost-ssl enable/disable native -set(ARCH_NATIVE OFF) -if ("${ARCHITECTURE}" MATCHES "aarch64" OR "${ARCHITECTURE}" MATCHES "arm64") - set(ARCH_NATIVE ON) -endif () + # for boost-ssl enable/disable native + set(ARCH_NATIVE OFF) + if ("${ARCHITECTURE}" MATCHES "aarch64" OR "${ARCHITECTURE}" MATCHES "arm64") + set(ARCH_NATIVE ON) + endif () -set(VISIBILITY_FLAG " -fvisibility=hidden -fvisibility-inlines-hidden") -if (BUILD_UDF) - set(VISIBILITY_FLAG "") -endif() -if (BUILD_SDK) - set(VISIBILITY_FLAG "") -endif() -set(MARCH_TYPE "-march=x86-64 -mtune=generic ${VISIBILITY_FLAG}") -if (ARCH_NATIVE) - set(MARCH_TYPE "-march=native -mtune=native ${VISIBILITY_FLAG}") -endif () + set(VISIBILITY_FLAG " -fvisibility=hidden -fvisibility-inlines-hidden") -# for enable sse4.2(hdfs used) -set(ENABLE_SSE OFF) -# for enable/disable ipp-crypto -if (APPLE) - EXECUTE_PROCESS(COMMAND sysctl -a COMMAND grep "machdep.cpu.*features" COMMAND tr -d '\n' OUTPUT_VARIABLE SUPPORTED_INSTRUCTIONS) - message("* SUPPORTED_INSTRUCTIONS: ${SUPPORTED_INSTRUCTIONS}") - # detect sse4.2 - if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*SSE4.2.*") - set(ENABLE_SSE ON) + set(MARCH_TYPE "-march=x86-64 -mtune=generic ${VISIBILITY_FLAG}") + if (ARCH_NATIVE) + set(MARCH_TYPE "-march=native -mtune=native ${VISIBILITY_FLAG}") endif () -elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") - # detect sse4_2 - FILE(READ "/proc/cpuinfo" SUPPORTED_INSTRUCTIONS) - if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*sse4_2.*") - set(ENABLE_SSE ON) + + # for enable sse4.2(hdfs used) + set(ENABLE_SSE OFF) + # for enable/disable ipp-crypto + if (APPLE) + EXECUTE_PROCESS(COMMAND sysctl -a COMMAND grep "machdep.cpu.*features" COMMAND tr -d '\n' OUTPUT_VARIABLE SUPPORTED_INSTRUCTIONS) + message("* SUPPORTED_INSTRUCTIONS: ${SUPPORTED_INSTRUCTIONS}") + # detect sse4.2 + if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*SSE4.2.*") + set(ENABLE_SSE ON) + endif () + elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # detect sse4_2 + FILE(READ "/proc/cpuinfo" SUPPORTED_INSTRUCTIONS) + if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*sse4_2.*") + set(ENABLE_SSE ON) + endif () endif () -endif () -if(ENABLE_SSE) - list(APPEND VCPKG_MANIFEST_FEATURES "sse") -endif() + set(ENABLE_CPU_FEATURES OFF) + # only ENABLE_CPU_FEATURES for aarch64 and x86 + if ("${ARCHITECTURE}" MATCHES "aarch64") + add_definitions(-DARCH) + set(ENABLE_CPU_FEATURES ON) + endif () -set(ENABLE_CPU_FEATURES OFF) -# only ENABLE_CPU_FEATURES for aarch64 and x86 -if ("${ARCHITECTURE}" MATCHES "aarch64") - add_definitions(-DARCH) - set(ENABLE_CPU_FEATURES ON) -endif () + if ("${ARCHITECTURE}" MATCHES "x86_64") + add_definitions(-DX86) + set(ENABLE_CPU_FEATURES ON) + endif () -if ("${ARCHITECTURE}" MATCHES "x86_64") - add_definitions(-DX86) - set(ENABLE_CPU_FEATURES ON) -endif () + if (ENABLE_CPU_FEATURES) + add_definitions(-DENABLE_CPU_FEATURES) + endif () -if (ENABLE_CPU_FEATURES) - add_definitions(-DENABLE_CPU_FEATURES) -endif () + # Enable CONN_PSI Joint Running With Ant Company + if (ENABLE_CONN) + add_definitions(-DENABLE_CONN) + endif () -# Enable CONN_PSI Joint Running With Ant Company -if (ENABLE_CONN) - add_definitions(-DENABLE_CONN) -endif () + set(ENABLE_IPP_CRYPTO OFF) + # Note: only ENABLE_CRYPTO_MB for x86_64 + # if ("${ARCHITECTURE}" MATCHES "x86_64") + # set(ENABLE_IPP_CRYPTO ON) + # add_definitions(-DENABLE_CRYPTO_MB) + # endif () -set(ENABLE_IPP_CRYPTO OFF) -# Note: only ENABLE_CRYPTO_MB for x86_64 -# if ("${ARCHITECTURE}" MATCHES "x86_64") -# set(ENABLE_IPP_CRYPTO ON) -# add_definitions(-DENABLE_CRYPTO_MB) -# endif () + # fix the boost beast build failed for [call to 'async_teardown' is ambiguous] + add_definitions(-DBOOST_ASIO_DISABLE_CONCEPTS) -# fix the boost beast build failed for [call to 'async_teardown' is ambiguous] -add_definitions(-DBOOST_ASIO_DISABLE_CONCEPTS) + ####### options settings ###### + if (BUILD_UDF) + set(VISIBILITY_FLAG "") + set(BUILD_ALL OFF) + endif() + if (BUILD_SDK) + set(VISIBILITY_FLAG "") + set(BUILD_ALL OFF) + endif() + if (BUILD_ALL) + # install all dependencies + list(APPEND VCPKG_MANIFEST_FEATURES "all") + endif() + if(ENABLE_SSE) + # enable sse for libhdfs3 + list(APPEND VCPKG_MANIFEST_FEATURES "sse") + endif() + # cpp_features + if(ENABLE_CPU_FEATURES) + list(APPEND VCPKG_MANIFEST_FEATURES "cpufeatures") + message("##### append cpp_features: ${VCPKG_MANIFEST_FEATURES}") + endif() + ####### options settings ###### + print_config("WeDPR-Component") +endmacro() macro(print_config NAME) message("") diff --git a/cpp/ppc-crypto-core/src/CMakeLists.txt b/cpp/ppc-crypto-core/src/CMakeLists.txt index d40633ab..3387e7a2 100644 --- a/cpp/ppc-crypto-core/src/CMakeLists.txt +++ b/cpp/ppc-crypto-core/src/CMakeLists.txt @@ -5,5 +5,5 @@ find_package(OpenSSL REQUIRED) message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}") message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}") -find_package(unofficial-sodium REQUIRED) -target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC unofficial-sodium::sodium ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file +target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC unofficial-sodium::sodium + unofficial-sodium::sodium_config_public ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/ppc-crypto/src/CMakeLists.txt b/cpp/ppc-crypto/src/CMakeLists.txt index b699d88c..954f1ca6 100644 --- a/cpp/ppc-crypto/src/CMakeLists.txt +++ b/cpp/ppc-crypto/src/CMakeLists.txt @@ -1,12 +1,12 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${CRYPTO_TARGET} ${SRCS}) -find_package(unofficial-sodium REQUIRED) + find_package(OpenSSL REQUIRED) message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}") message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}") -target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB}) +target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium unofficial-sodium::sodium_config_public TBB::tbb ${CPU_FEATURES_LIB}) if (ENABLE_IPP_CRYPTO) find_package(ipp-crypto REQUIRED) diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 34c765ca..635e6990 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -16,42 +16,15 @@ "name": "openssl", "version>=": "1.1.1-tassl" }, - "tarscpp", "libsodium", + { + "name": "libsodium", + "version>=": "1.0.18" + }, { "name": "bcos-utilities", "version>=": "1.0.0" } - , - { - "name": "bcos-boostssl", - "version>=": "3.2.3" - }, - { - "name": "seal", - "version>=": "4.0.0", - "features": ["no-throw-tran"] - }, - { - "name": "kuku", - "version>=": "2.1" - }, - { - "name": "redis-plus-plus", - "version>=": "1.3.6" - }, - { - "name": "mysql-connector-cpp", - "version>=": "8.0.32" - }, - { - "name": "cpu-features", - "version>=": "0.9.0" - }, - "libhdfs3", - "tarscpp", - "tbb", - "libxml2" ], "builtin-baseline": "51b14cd4e1230dd51c11ffeff6f7d53c61cc5297", "overrides": [ @@ -74,11 +47,54 @@ { "name": "tarscpp", "version": "3.0.3-1#1" + }, + { + "name": "libsodium", + "version": "1.0.18#9" } ], "features": { "sse": { "description": "Enable SSE4.2 for libhdfs3" + }, + "cpufeatures":{ + "description": "Enable cpu features", + "dependencies": [ + { + "name": "cpu-features", + "version>=": "0.9.0" + } + ] + }, + "all": { + "description": "all dependencies", + "dependencies": [ + { + "name": "bcos-boostssl", + "version>=": "3.2.3" + }, + { + "name": "seal", + "version>=": "4.0.0", + "features": ["no-throw-tran"] + }, + { + "name": "kuku", + "version>=": "2.1" + }, + { + "name": "redis-plus-plus", + "version>=": "1.3.6" + }, + { + "name": "mysql-connector-cpp", + "version>=": "8.0.32" + }, + "libhdfs3", + "tarscpp", + "tbb", + "libxml2" + ] } } } \ No newline at end of file From ae2217b66cafef2abe55a2d96ec9ce19fadc9b8e Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 11:22:32 +0800 Subject: [PATCH 006/120] fix ci && upload artifact --- .github/workflows/cpp_workflow.yml | 96 ++++++++++++++++++- cpp/CMakeLists.txt | 2 +- cpp/cmake/CompilerSettings.cmake | 20 ++-- cpp/cmake/Coverage.cmake | 41 ++++++++ cpp/cmake/Dependencies.cmake | 12 +-- cpp/cmake/Options.cmake | 3 +- .../bindings/java/src/main/c/CMakeLists.txt | 3 +- 7 files changed, 152 insertions(+), 25 deletions(-) create mode 100644 cpp/cmake/Coverage.cmake diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index 5f75d524..3ec7f9c1 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -16,6 +16,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + RUST_BACKTRACE: 1 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: build: @@ -72,7 +75,7 @@ jobs: if: runner.os != 'Windows' run: | cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest - make cov + make coverage build_sdk: name: build sdk @@ -112,7 +115,7 @@ jobs: - name: Build for windows if: runner.os == 'Windows' run: | - mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_SDK=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ../ + mkdir -p cpp/build && cd cpp/build && cmake -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_SDK=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ../ cmake --build . --parallel 3 - name: Build for linux if: runner.os == 'Linux' @@ -132,5 +135,90 @@ jobs: - name: Test if: runner.os != 'Windows' run: | - cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest - make cov \ No newline at end of file + cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test + make coverage + - uses: actions/upload-artifact@v2 + if: runner.os == 'macos' + with: + name: libppc-crypto-sdk-jni.dylib + path: ./cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib + - uses: actions/upload-artifact@v2 + if: runner.os == 'Windows' + with: + name: libppc-crypto-sdk-jni.dylib + path: D:\a\WeDPR-Component\cpp\ppc-crypto-c-sdk\bindings\java\src\main\resources\META-INF\native\Release\ppc-crypto-sdk-jni.dll + + build_centos: + name: build_centos full node + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + container: docker.io/centos:7 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 5 + - uses: actions/cache@v2 + id: deps_cache + with: + path: | + /home/runner/.ccache + /Users/runner/.ccache/ + deps/ + key: centos-notest-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + restore-keys: | + centos-notest-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }} + centos-notest-${{ matrix.os }}-${{ github.base_ref }}- + centos-notest-${{ matrix.os }}- + - name: Prepare centos tools + run: | + sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo + yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel + yum install -y devtoolset-10 devtoolset-11 llvm-toolset-7 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel + yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm + yum install -y git + - name: Prepare vcpkg + if: runner.os != 'Windows' + uses: friendlyanon/setup-vcpkg@v1 + with: { committish: 7e3dcf74e37034eea358934a90a11d618520e139 } + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-07-28 + override: true + - name: Build + run: | + alias cmake='cmake3' + . /opt/rh/devtoolset-10/enable + . /opt/rh/rh-perl530/enable + export LIBCLANG_PATH=/opt/rh/llvm-toolset-7/root/lib64/ + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + . /opt/rh/llvm-toolset-7/enable + mkdir -p cpp/build + cd cpp/build + cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + cmake3 --build . --parallel 3 + - name: Test + run: | + export OMP_NUM_THREADS=1 + cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test + - uses: actions/upload-artifact@v2 + with: + name: ppc-air-node-centos-x64 + path: ./cpp/build/bin/ppc-air-node + - uses: actions/upload-artifact@v2 + with: + name: ppc-pro-node-centos-x64 + path: ./cpp/build/bin/ppc-pro-node + - uses: actions/upload-artifact@v2 + with: + name: ppc-gateway-service-centos-x64 + path: ./cpp/build/bin/ppc-gateway-service + - uses: actions/upload-artifact@v2 + with: + name: libppc-crypto-sdk-jni.so + path: ./cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0fb668fc..698343ec 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -96,5 +96,5 @@ endif () # for code coverage if (COVERAGE) include(Coverage) - config_coverage("coverage" "'/usr*' 'boost/*'") + config_coverage("coverage" "") endif () diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake index 005d130a..5e1140e3 100644 --- a/cpp/cmake/CompilerSettings.cmake +++ b/cpp/cmake/CompilerSettings.cmake @@ -111,16 +111,16 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA if (COVERAGE) set(TESTS ON) - if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") - set(CMAKE_CXX_FLAGS "-g --coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_C_FLAGS "-g --coverage ${CMAKE_C_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "--coverage ${CMAKE_EXE_LINKER_FLAGS}") - elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - add_compile_options(-Wno-unused-command-line-argument) - set(CMAKE_CXX_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_C_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_C_FLAGS}") - endif() + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") + set(CMAKE_CXX_FLAGS "-g --coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-g --coverage ${CMAKE_C_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "--coverage ${CMAKE_EXE_LINKER_FLAGS}") + elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + add_compile_options(-Wno-unused-command-line-argument) + set(CMAKE_CXX_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "-g -fprofile-arcs -ftest-coverage ${CMAKE_C_FLAGS}") + endif() endif () elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC") diff --git a/cpp/cmake/Coverage.cmake b/cpp/cmake/Coverage.cmake new file mode 100644 index 00000000..b76a97e2 --- /dev/null +++ b/cpp/cmake/Coverage.cmake @@ -0,0 +1,41 @@ +# ------------------------------------------------------------------------------ +# Copyright (C) 2021 FISCO BCOS. +# SPDX-License-Identifier: Apache-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------ +# File: Coverage.cmake +# Function: Define coverage related functions +# ------------------------------------------------------------------------------ +# REMOVE_FILE_PATTERN eg.: '/usr*' '${CMAKE_SOURCE_DIR}/deps**' '${CMAKE_SOURCE_DIR}/evmc*' ‘${CMAKE_SOURCE_DIR}/fisco-bcos*’ +function(config_coverage TARGET REMOVE_FILE_PATTERN) + find_program(LCOV_TOOL lcov) + message(STATUS "lcov tool: ${LCOV_TOOL}") + if (LCOV_TOOL) + message(STATUS "coverage dir: " ${CMAKE_BINARY_DIR}) + message(STATUS "coverage TARGET: " ${TARGET}) + message(STATUS "coverage REMOVE_FILE_PATTERN: " ${REMOVE_FILE_PATTERN}) + if (APPLE) + add_custom_target(${TARGET} + COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/ + COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info + COMMAND genhtml --keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info) + else() + add_custom_target(${TARGET} + COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/ + COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info + COMMAND genhtml --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info) + endif() + else () + message(FATAL_ERROR "Can't find lcov tool. Please install lcov") + endif() +endfunction() \ No newline at end of file diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake index 4354187e..ff1000b4 100644 --- a/cpp/cmake/Dependencies.cmake +++ b/cpp/cmake/Dependencies.cmake @@ -41,11 +41,9 @@ endif() ##### the full-dependencies end ##### ##### the sdk-dependencies ##### -if(BUILD_SDK) - # find JNI - set(JAVA_AWT_LIBRARY NotNeeded) - set(JAVA_JVM_LIBRARY NotNeeded) - find_package(JNI REQUIRED) - include_directories(${JNI_INCLUDE_DIRS}) -endif() +# find JNI +set(JAVA_AWT_LIBRARY NotNeeded) +set(JAVA_JVM_LIBRARY NotNeeded) +find_package(JNI REQUIRED) +include_directories(${JNI_INCLUDE_DIRS}) ##### the sdk-dependencies end##### \ No newline at end of file diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 0588372d..1b4245ab 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -140,7 +140,6 @@ macro(configure_project) # cpp_features if(ENABLE_CPU_FEATURES) list(APPEND VCPKG_MANIFEST_FEATURES "cpufeatures") - message("##### append cpp_features: ${VCPKG_MANIFEST_FEATURES}") endif() ####### options settings ###### print_config("WeDPR-Component") @@ -154,6 +153,8 @@ macro(print_config NAME) message("-- CMake Cmake version and location ${CMAKE_VERSION} (${CMAKE_COMMAND})") message("-- Compiler C++ compiler version ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") message("-- CMAKE_BUILD_TYPE Build type ${CMAKE_BUILD_TYPE}") + message("-- VCPKG_MANIFEST_FEATURES VCPKG manifest features ${VCPKG_MANIFEST_FEATURES}") + message("-- CMAKE_TOOLCHAIN_FILE Cmake toolchain file ${CMAKE_TOOLCHAIN_FILE}") message("-- TARGET_PLATFORM Target platform ${CMAKE_SYSTEM_NAME} ${ARCHITECTURE}") message("-- BUILD_STATIC Build static ${BUILD_STATIC}") message("-- COVERAGE Build code coverage ${COVERAGE}") diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt index 3741b8c1..94a27864 100644 --- a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt +++ b/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt @@ -11,5 +11,4 @@ target_link_libraries(${PPC_CRYPTO_SDK_JNI_STATIC_TARGET} PUBLIC ${PPC_CRYPTO_C_ SET(LIBRARY_OUTPUT_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../resources/META-INF/native/) message(STATUS "CMAKE_INSTALL_INCLUDEDIR => ${CMAKE_INSTALL_INCLUDEDIR}") -message(STATUS "CMAKE_CURRENT_SOURCE_DIR => ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS "LIB_DIR_PATH => ${CMAKE_CURRENT_SOURCE_DIR}/../resources/META-INF/native/") \ No newline at end of file +message(STATUS "CMAKE_CURRENT_SOURCE_DIR => ${CMAKE_CURRENT_SOURCE_DIR}") \ No newline at end of file From c0bdf753597ee4bca88bcbb5f76fd8165443f609 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 14:29:52 +0800 Subject: [PATCH 007/120] format python code (#6) --- python/ppc_common/ppc_crypto/ihc_cipher.py | 27 ++-- .../ppc_common/ppc_crypto/paillier_cipher.py | 12 +- .../ppc_common/ppc_crypto/paillier_codec.py | 2 +- .../ppc_crypto/test/phe_unittest.py | 8 +- .../ppc_protos/generated/ppc_model_pb2.py | 58 ++++---- python/ppc_common/ppc_utils/http_utils.py | 4 +- .../ppc_utils/ppc_model_config_parser.py | 2 +- python/ppc_model/common/base_context.py | 135 ++++++++++++------ python/ppc_model/common/global_context.py | 3 +- python/ppc_model/common/model_result.py | 63 ++++---- python/ppc_model/common/protocol.py | 3 +- python/ppc_model/datasets/dataset.py | 16 ++- .../test/test_feature_binning.py | 16 ++- .../ppc_model/datasets/test/test_dataset.py | 8 +- .../feature_engineering_engine.py | 6 +- .../vertical/active_party.py | 12 +- .../vertical/passive_party.py | 24 ++-- .../feature_engineering/vertical/utils.py | 6 +- python/ppc_model/metrics/evaluation.py | 90 +++++++----- python/ppc_model/metrics/model_plot.py | 50 ++++--- python/ppc_model/metrics/test/test_metrics.py | 45 +++--- .../network/http/model_controller.py | 6 +- python/ppc_model/network/http/restx.py | 3 +- python/ppc_model/network/stub.py | 3 +- python/ppc_model/ppc_model_app.py | 50 +++---- .../local_processing/preprocessing.py | 6 +- .../preprocessing/tests/test_preprocessing.py | 27 ++-- .../ppc_model/secure_lgbm/monitor/callback.py | 6 +- .../secure_lgbm/monitor/early_stopping.py | 3 +- .../secure_lgbm/monitor/evaluation_monitor.py | 3 +- .../monitor/train_callback_unittest.py | 12 +- .../secure_lgbm_prediction_engine.py | 2 +- .../secure_lgbm/test/test_cipher_packing.py | 24 ++-- .../secure_lgbm/test/test_pack_gh.py | 6 +- .../secure_lgbm/test/test_save_load_model.py | 14 +- .../test_secure_lgbm_performance_training.py | 24 ++-- .../test/test_secure_lgbm_training.py | 12 +- .../secure_lgbm/vertical/active_party.py | 101 ++++++++----- .../ppc_model/secure_lgbm/vertical/booster.py | 77 ++++++---- .../secure_lgbm/vertical/passive_party.py | 38 +++-- python/ppc_model/task/task_manager.py | 12 +- .../task/test/task_manager_unittest.py | 6 +- .../ppc_model_gateway_app.py | 27 ++-- python/ppc_model_gateway/test/server.py | 3 +- 44 files changed, 640 insertions(+), 415 deletions(-) diff --git a/python/ppc_common/ppc_crypto/ihc_cipher.py b/python/ppc_common/ppc_crypto/ihc_cipher.py index 7cc385ff..54061208 100644 --- a/python/ppc_common/ppc_crypto/ihc_cipher.py +++ b/python/ppc_common/ppc_crypto/ihc_cipher.py @@ -10,19 +10,19 @@ @dataclass class IhcCiphertext(): __slots__ = ['c_left', 'c_right'] - + def __init__(self, c_left: int, c_right: int) -> None: self.c_left = c_left self.c_right = c_right - + def __add__(self, other): cipher_left = self.c_left + other.c_left cipher_right = self.c_right + other.c_right return IhcCiphertext(cipher_left, cipher_right) - + def __eq__(self, other): return self.c_left == other.c_left and self.c_right == other.c_right - + def encode(self) -> bytes: # 计算每个整数的字节长度 len_c_left = (self.c_left.bit_length() + 7) // 8 @@ -37,17 +37,20 @@ def encode(self) -> bytes: # 返回所有数据 return len_bytes + c_left_bytes + c_right_bytes - + @classmethod def decode(cls, encoded_data: bytes): # 解码整数的长度 len_c_left, len_c_right = struct.unpack('>II', encoded_data[:8]) # 根据长度解码整数 - c_left = int.from_bytes(encoded_data[8:8 + len_c_left], byteorder='big') - c_right = int.from_bytes(encoded_data[8 + len_c_left:8 + len_c_left + len_c_right], byteorder='big') + c_left = int.from_bytes( + encoded_data[8:8 + len_c_left], byteorder='big') + c_right = int.from_bytes( + encoded_data[8 + len_c_left:8 + len_c_left + len_c_right], byteorder='big') return cls(c_left, c_right) - + + class IhcCipher(PheCipher): def __init__(self, key_length: int = 256, iter_round: int = 16) -> None: super().__init__(key_length) @@ -56,9 +59,9 @@ def __init__(self, key_length: int = 256, iter_round: int = 16) -> None: self.private_key = key self.iter_round = iter_round self.key_length = key_length - + self.max_mod = 1 << key_length - + def encrypt(self, number: int) -> IhcCiphertext: random_u = secrets.randbits(self.key_length) x_this = number @@ -70,7 +73,7 @@ def encrypt(self, number: int) -> IhcCiphertext: # cipher = IhcCiphertext(x_this, x_last, self.max_mod) cipher = IhcCiphertext(x_this, x_last) return cipher - + def decrypt(self, cipher: IhcCiphertext) -> int: x_this = cipher.c_right x_last = cipher.c_left @@ -79,7 +82,7 @@ def decrypt(self, cipher: IhcCiphertext) -> int: x_last = x_this x_this = x_tmp return x_this - + def encrypt_batch(self, numbers) -> list: return [self.encrypt(num) for num in numbers] diff --git a/python/ppc_common/ppc_crypto/paillier_cipher.py b/python/ppc_common/ppc_crypto/paillier_cipher.py index d2e0232a..822093be 100644 --- a/python/ppc_common/ppc_crypto/paillier_cipher.py +++ b/python/ppc_common/ppc_crypto/paillier_cipher.py @@ -29,17 +29,21 @@ def decrypt_batch(self, ciphers) -> list: def encrypt_batch_parallel(self, numbers) -> list: num_cores = os.cpu_count() batch_size = math.ceil(len(numbers) / num_cores) - batches = [numbers[i:i + batch_size] for i in range(0, len(numbers), batch_size)] + batches = [numbers[i:i + batch_size] + for i in range(0, len(numbers), batch_size)] with ProcessPoolExecutor(max_workers=num_cores) as executor: - futures = [executor.submit(self.encrypt_batch, batch) for batch in batches] + futures = [executor.submit(self.encrypt_batch, batch) + for batch in batches] result = [future.result() for future in futures] return [item for sublist in result for item in sublist] def decrypt_batch_parallel(self, ciphers) -> list: num_cores = os.cpu_count() batch_size = math.ceil(len(ciphers) / num_cores) - batches = [ciphers[i:i + batch_size] for i in range(0, len(ciphers), batch_size)] + batches = [ciphers[i:i + batch_size] + for i in range(0, len(ciphers), batch_size)] with ProcessPoolExecutor(max_workers=num_cores) as executor: - futures = [executor.submit(self.decrypt_batch, batch) for batch in batches] + futures = [executor.submit(self.decrypt_batch, batch) + for batch in batches] result = [future.result() for future in futures] return [item for sublist in result for item in sublist] diff --git a/python/ppc_common/ppc_crypto/paillier_codec.py b/python/ppc_common/ppc_crypto/paillier_codec.py index fb5f3bf0..d66f3c95 100644 --- a/python/ppc_common/ppc_crypto/paillier_codec.py +++ b/python/ppc_common/ppc_crypto/paillier_codec.py @@ -24,7 +24,7 @@ def decode_enc_key(public_key_bytes: bytes) -> PaillierPublicKey: @staticmethod def encode_cipher(cipher: EncryptedNumber, be_secure=True) -> Tuple[bytes, bytes]: return PaillierCodec._int_to_bytes(cipher.ciphertext(be_secure=be_secure)), \ - PaillierCodec._int_to_bytes(cipher.exponent) + PaillierCodec._int_to_bytes(cipher.exponent) @staticmethod def decode_cipher(public_key: PaillierPublicKey, ciphertext: bytes, exponent: bytes) -> EncryptedNumber: diff --git a/python/ppc_common/ppc_crypto/test/phe_unittest.py b/python/ppc_common/ppc_crypto/test/phe_unittest.py index 5d036b75..a8ef12a7 100644 --- a/python/ppc_common/ppc_crypto/test/phe_unittest.py +++ b/python/ppc_common/ppc_crypto/test/phe_unittest.py @@ -30,7 +30,7 @@ def test_enc_and_dec_parallel(self): print("dec_p:", end_time - start_time, "seconds") self.assertListEqual(list(inputs), list(outputs)) - + def test_ihc_enc_and_dec_parallel(self): ihc = IhcCipher(key_length=256) try_size = 100000 @@ -48,17 +48,17 @@ def test_ihc_enc_and_dec_parallel(self): cipher_start = ciphers[0] for i in range(1, len(ciphers)): cipher_left = (cipher_start.c_left + ciphers[i].c_left) - cipher_right = (cipher_start.c_right + ciphers[i].c_right ) + cipher_right = (cipher_start.c_right + ciphers[i].c_right) # IhcCiphertext(cipher_left, cipher_right, cipher_start.max_mod) IhcCiphertext(cipher_left, cipher_right) end_time = time.time() print(f"size:{try_size}, add_p raw with class: {end_time - start_time} seconds, average times: {(end_time - start_time)/try_size * 1000 * 1000} us") - + start_time = time.time() cipher_start = ciphers[0] for i in range(1, len(ciphers)): cipher_left = (cipher_start.c_left + ciphers[i].c_left) - cipher_right = (cipher_start.c_right + ciphers[i].c_right ) + cipher_right = (cipher_start.c_right + ciphers[i].c_right) # IhcCiphertext(cipher_left, cipher_right) end_time = time.time() print(f"size:{try_size}, add_p raw: {end_time - start_time} seconds, average times: {(end_time - start_time)/try_size * 1000 * 1000} us") diff --git a/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py b/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py index 57f7ca3a..ced82b9e 100644 --- a/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py +++ b/python/ppc_common/ppc_protos/generated/ppc_model_pb2.py @@ -12,40 +12,38 @@ _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fppc_model.proto\x12\tppc.model\"|\n\x0cModelRequest\x12\x0e\n\x06sender\x18\x01 \x01(\t\x12\x10\n\x08receiver\x18\x02 \x01(\t\x12\x0f\n\x07task_id\x18\x03 \x01(\t\x12\x0b\n\x03key\x18\x04 \x01(\t\x12\x0b\n\x03seq\x18\x05 \x01(\x03\x12\x11\n\tslice_num\x18\x06 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x07 \x01(\x0c\"3\n\x0c\x42\x61seResponse\x12\x12\n\nerror_code\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"M\n\rModelResponse\x12.\n\rbase_response\x18\x01 \x01(\x0b\x32\x17.ppc.model.BaseResponse\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\"#\n\rPlainBoolList\x12\x12\n\nplain_list\x18\x01 \x03(\x08\"\xb1\x01\n\rBestSplitInfo\x12\x0f\n\x07tree_id\x18\x01 \x01(\x03\x12\x0f\n\x07leaf_id\x18\x02 \x01(\x03\x12\x0f\n\x07\x66\x65\x61ture\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x03\x12\x12\n\nagency_idx\x18\x05 \x01(\x03\x12\x16\n\x0e\x61gency_feature\x18\x06 \x01(\x03\x12\x11\n\tbest_gain\x18\x07 \x01(\x02\x12\x0e\n\x06w_left\x18\x08 \x01(\x02\x12\x0f\n\x07w_right\x18\t \x01(\x02\"3\n\x0bModelCipher\x12\x12\n\nciphertext\x18\x01 \x01(\x0c\x12\x10\n\x08\x65xponent\x18\x02 \x01(\x0c\"M\n\nCipherList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12+\n\x0b\x63ipher_list\x18\x02 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"=\n\x0e\x43ipher1DimList\x12+\n\x0b\x63ipher_list\x18\x01 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"W\n\x0e\x43ipher2DimList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\x31\n\x0e\x63ipher_1d_list\x18\x02 \x03(\x0b\x32\x19.ppc.model.Cipher1DimList\"_\n\rEncAggrLabels\x12\r\n\x05\x66ield\x18\x01 \x01(\t\x12\x12\n\ncount_list\x18\x02 \x03(\x03\x12+\n\x0b\x63ipher_list\x18\x03 \x03(\x0b\x32\x16.ppc.model.ModelCipher\"_\n\x11\x45ncAggrLabelsList\x12\x12\n\npublic_key\x18\x01 \x01(\x0c\x12\x36\n\x14\x65nc_aggr_labels_list\x18\x02 \x03(\x0b\x32\x18.ppc.model.EncAggrLabels\"/\n\x10IterationRequest\x12\r\n\x05\x65poch\x18\x01 \x01(\x03\x12\x0c\n\x04stop\x18\x02 \x01(\x08\x32Y\n\x0cModelService\x12I\n\x12MessageInteraction\x12\x17.ppc.model.ModelRequest\x1a\x18.ppc.model.ModelResponse\"\x00\x42\x08P\x01\xa2\x02\x03PPCb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ppc_model_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - _globals['DESCRIPTOR']._options = None - _globals['DESCRIPTOR']._serialized_options = b'P\001\242\002\003PPC' - _globals['_MODELREQUEST']._serialized_start=30 - _globals['_MODELREQUEST']._serialized_end=154 - _globals['_BASERESPONSE']._serialized_start=156 - _globals['_BASERESPONSE']._serialized_end=207 - _globals['_MODELRESPONSE']._serialized_start=209 - _globals['_MODELRESPONSE']._serialized_end=286 - _globals['_PLAINBOOLLIST']._serialized_start=288 - _globals['_PLAINBOOLLIST']._serialized_end=323 - _globals['_BESTSPLITINFO']._serialized_start=326 - _globals['_BESTSPLITINFO']._serialized_end=503 - _globals['_MODELCIPHER']._serialized_start=505 - _globals['_MODELCIPHER']._serialized_end=556 - _globals['_CIPHERLIST']._serialized_start=558 - _globals['_CIPHERLIST']._serialized_end=635 - _globals['_CIPHER1DIMLIST']._serialized_start=637 - _globals['_CIPHER1DIMLIST']._serialized_end=698 - _globals['_CIPHER2DIMLIST']._serialized_start=700 - _globals['_CIPHER2DIMLIST']._serialized_end=787 - _globals['_ENCAGGRLABELS']._serialized_start=789 - _globals['_ENCAGGRLABELS']._serialized_end=884 - _globals['_ENCAGGRLABELSLIST']._serialized_start=886 - _globals['_ENCAGGRLABELSLIST']._serialized_end=981 - _globals['_ITERATIONREQUEST']._serialized_start=983 - _globals['_ITERATIONREQUEST']._serialized_end=1030 - _globals['_MODELSERVICE']._serialized_start=1032 - _globals['_MODELSERVICE']._serialized_end=1121 + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'P\001\242\002\003PPC' + _globals['_MODELREQUEST']._serialized_start = 30 + _globals['_MODELREQUEST']._serialized_end = 154 + _globals['_BASERESPONSE']._serialized_start = 156 + _globals['_BASERESPONSE']._serialized_end = 207 + _globals['_MODELRESPONSE']._serialized_start = 209 + _globals['_MODELRESPONSE']._serialized_end = 286 + _globals['_PLAINBOOLLIST']._serialized_start = 288 + _globals['_PLAINBOOLLIST']._serialized_end = 323 + _globals['_BESTSPLITINFO']._serialized_start = 326 + _globals['_BESTSPLITINFO']._serialized_end = 503 + _globals['_MODELCIPHER']._serialized_start = 505 + _globals['_MODELCIPHER']._serialized_end = 556 + _globals['_CIPHERLIST']._serialized_start = 558 + _globals['_CIPHERLIST']._serialized_end = 635 + _globals['_CIPHER1DIMLIST']._serialized_start = 637 + _globals['_CIPHER1DIMLIST']._serialized_end = 698 + _globals['_CIPHER2DIMLIST']._serialized_start = 700 + _globals['_CIPHER2DIMLIST']._serialized_end = 787 + _globals['_ENCAGGRLABELS']._serialized_start = 789 + _globals['_ENCAGGRLABELS']._serialized_end = 884 + _globals['_ENCAGGRLABELSLIST']._serialized_start = 886 + _globals['_ENCAGGRLABELSLIST']._serialized_end = 981 + _globals['_ITERATIONREQUEST']._serialized_start = 983 + _globals['_ITERATIONREQUEST']._serialized_end = 1030 + _globals['_MODELSERVICE']._serialized_start = 1032 + _globals['_MODELSERVICE']._serialized_end = 1121 # @@protoc_insertion_point(module_scope) diff --git a/python/ppc_common/ppc_utils/http_utils.py b/python/ppc_common/ppc_utils/http_utils.py index 4489240d..8dc43a56 100644 --- a/python/ppc_common/ppc_utils/http_utils.py +++ b/python/ppc_common/ppc_utils/http_utils.py @@ -28,7 +28,8 @@ def send_get_request(endpoint, uri, params=None, headers=None): else: url = f"http://{endpoint}" log.debug(f"send a get request, url: {url}, params: {params}") - response = requests.get(url=url, params=params, headers=headers, timeout=30) + response = requests.get(url=url, params=params, + headers=headers, timeout=30) log.debug(f"response: {response.text}") check_response(response) response_data = json.loads(response.text) @@ -99,4 +100,3 @@ def send_upload_request(endpoint, uri, params=None, headers=None, data=None): except JSONDecodeError: response_data = response.text return response_data - diff --git a/python/ppc_common/ppc_utils/ppc_model_config_parser.py b/python/ppc_common/ppc_utils/ppc_model_config_parser.py index ab2cce26..04ea012f 100644 --- a/python/ppc_common/ppc_utils/ppc_model_config_parser.py +++ b/python/ppc_common/ppc_utils/ppc_model_config_parser.py @@ -471,4 +471,4 @@ def generate_mpc_predict_algorithm(algorithm_name, layers, participants, is_psi) # PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_msg()) # except BaseException as e: # raise PpcException(PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_code(), - # PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_msg()) \ No newline at end of file + # PpcErrorCode.ALGORITHM_PPC_MODEL_THREADS_ERROR.get_msg()) diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py index a6c1b582..43f1e873 100644 --- a/python/ppc_model/common/base_context.py +++ b/python/ppc_model/common/base_context.py @@ -27,56 +27,101 @@ def __init__(self, job_id: str, job_temp_dir: str): self.workspace = os.path.join(job_temp_dir, self.job_id) if not os.path.exists(self.workspace): os.makedirs(self.workspace) - self.psi_result_path = os.path.join(self.workspace, self.PSI_RESULT_FILE) - self.model_prepare_file = os.path.join(self.workspace, self.MODEL_PREPARE_FILE) - self.preprocessing_result_file = os.path.join(self.workspace, self.PREPROCESSING_RESULT_FILE) - self.eval_column_file = os.path.join(self.workspace, self.EVAL_COLUMN_FILE) + self.psi_result_path = os.path.join( + self.workspace, self.PSI_RESULT_FILE) + self.model_prepare_file = os.path.join( + self.workspace, self.MODEL_PREPARE_FILE) + self.preprocessing_result_file = os.path.join( + self.workspace, self.PREPROCESSING_RESULT_FILE) + self.eval_column_file = os.path.join( + self.workspace, self.EVAL_COLUMN_FILE) self.woe_iv_file = os.path.join(self.workspace, self.WOE_IV_FILE) - self.iv_selected_file = os.path.join(self.workspace, self.IV_SELECTED_FILE) - self.selected_col_file = os.path.join(self.workspace, self.SELECTED_COL_FILE) - self.remote_selected_col_file = os.path.join(self.job_id, self.SELECTED_COL_FILE) + self.iv_selected_file = os.path.join( + self.workspace, self.IV_SELECTED_FILE) + self.selected_col_file = os.path.join( + self.workspace, self.SELECTED_COL_FILE) + self.remote_selected_col_file = os.path.join( + self.job_id, self.SELECTED_COL_FILE) - self.summary_evaluation_file = os.path.join(self.workspace, utils.MPC_XGB_EVALUATION_TABLE) - self.feature_importance_file = os.path.join(self.workspace, utils.XGB_FEATURE_IMPORTANCE_TABLE) - self.feature_bin_file = os.path.join(self.workspace, self.FEATURE_BIN_FILE) - self.model_data_file = os.path.join(self.workspace, self.MODEL_DATA_FILE) - self.test_model_result_file = os.path.join(self.workspace, self.TEST_MODEL_RESULT_FILE) - self.test_model_output_file = os.path.join(self.workspace, self.TEST_MODEL_OUTPUT_FILE) - self.train_model_result_file = os.path.join(self.workspace, self.TRAIN_MODEL_RESULT_FILE) - self.train_model_output_file = os.path.join(self.workspace, self.TRAIN_MODEL_OUTPUT_FILE) + self.summary_evaluation_file = os.path.join( + self.workspace, utils.MPC_XGB_EVALUATION_TABLE) + self.feature_importance_file = os.path.join( + self.workspace, utils.XGB_FEATURE_IMPORTANCE_TABLE) + self.feature_bin_file = os.path.join( + self.workspace, self.FEATURE_BIN_FILE) + self.model_data_file = os.path.join( + self.workspace, self.MODEL_DATA_FILE) + self.test_model_result_file = os.path.join( + self.workspace, self.TEST_MODEL_RESULT_FILE) + self.test_model_output_file = os.path.join( + self.workspace, self.TEST_MODEL_OUTPUT_FILE) + self.train_model_result_file = os.path.join( + self.workspace, self.TRAIN_MODEL_RESULT_FILE) + self.train_model_output_file = os.path.join( + self.workspace, self.TRAIN_MODEL_OUTPUT_FILE) - self.train_metric_roc_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) - self.train_metric_ks_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_FILE) - self.train_metric_pr_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_PR_FILE) - self.train_metric_acc_file = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) - self.test_metric_roc_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_ROC_FILE) - self.test_metric_ks_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_KS_FILE) - self.test_metric_pr_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_PR_FILE) - self.test_metric_acc_file = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) - self.train_metric_ks_table = os.path.join(self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) - self.test_metric_ks_table = os.path.join(self.workspace, utils.MPC_TRAIN_METRIC_KS_TABLE) - self.model_tree_prefix = os.path.join(self.workspace, utils.XGB_TREE_PERFIX) - self.metrics_iteration_file = os.path.join(self.workspace, utils.METRICS_OVER_ITERATION_FILE) + self.train_metric_roc_file = os.path.join( + self.workspace, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) + self.train_metric_ks_file = os.path.join( + self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_FILE) + self.train_metric_pr_file = os.path.join( + self.workspace, utils.MPC_TRAIN_SET_METRIC_PR_FILE) + self.train_metric_acc_file = os.path.join( + self.workspace, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) + self.test_metric_roc_file = os.path.join( + self.workspace, utils.MPC_TRAIN_METRIC_ROC_FILE) + self.test_metric_ks_file = os.path.join( + self.workspace, utils.MPC_TRAIN_METRIC_KS_FILE) + self.test_metric_pr_file = os.path.join( + self.workspace, utils.MPC_TRAIN_METRIC_PR_FILE) + self.test_metric_acc_file = os.path.join( + self.workspace, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) + self.train_metric_ks_table = os.path.join( + self.workspace, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) + self.test_metric_ks_table = os.path.join( + self.workspace, utils.MPC_TRAIN_METRIC_KS_TABLE) + self.model_tree_prefix = os.path.join( + self.workspace, utils.XGB_TREE_PERFIX) + self.metrics_iteration_file = os.path.join( + self.workspace, utils.METRICS_OVER_ITERATION_FILE) - self.remote_summary_evaluation_file = os.path.join(self.job_id, utils.MPC_XGB_EVALUATION_TABLE) - self.remote_feature_importance_file = os.path.join(self.job_id, utils.XGB_FEATURE_IMPORTANCE_TABLE) - self.remote_feature_bin_file = os.path.join(self.job_id, self.FEATURE_BIN_FILE) - self.remote_model_data_file = os.path.join(self.job_id, self.MODEL_DATA_FILE) - self.remote_test_model_output_file = os.path.join(self.job_id, self.TEST_MODEL_OUTPUT_FILE) - self.remote_train_model_output_file = os.path.join(self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) + self.remote_summary_evaluation_file = os.path.join( + self.job_id, utils.MPC_XGB_EVALUATION_TABLE) + self.remote_feature_importance_file = os.path.join( + self.job_id, utils.XGB_FEATURE_IMPORTANCE_TABLE) + self.remote_feature_bin_file = os.path.join( + self.job_id, self.FEATURE_BIN_FILE) + self.remote_model_data_file = os.path.join( + self.job_id, self.MODEL_DATA_FILE) + self.remote_test_model_output_file = os.path.join( + self.job_id, self.TEST_MODEL_OUTPUT_FILE) + self.remote_train_model_output_file = os.path.join( + self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) - self.remote_train_metric_roc_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) - self.remote_train_metric_ks_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_FILE) - self.remote_train_metric_pr_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_PR_FILE) - self.remote_train_metric_acc_file = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) - self.remote_test_metric_roc_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_ROC_FILE) - self.remote_test_metric_ks_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_KS_FILE) - self.remote_test_metric_pr_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_PR_FILE) - self.remote_test_metric_acc_file = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) - self.remote_train_metric_ks_table = os.path.join(self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) - self.remote_test_metric_ks_table = os.path.join(self.job_id, utils.MPC_TRAIN_METRIC_KS_TABLE) - self.remote_model_tree_prefix = os.path.join(self.job_id, utils.XGB_TREE_PERFIX) - self.remote_metrics_iteration_file = os.path.join(self.job_id, utils.METRICS_OVER_ITERATION_FILE) + self.remote_train_metric_roc_file = os.path.join( + self.job_id, utils.MPC_TRAIN_SET_METRIC_ROC_FILE) + self.remote_train_metric_ks_file = os.path.join( + self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_FILE) + self.remote_train_metric_pr_file = os.path.join( + self.job_id, utils.MPC_TRAIN_SET_METRIC_PR_FILE) + self.remote_train_metric_acc_file = os.path.join( + self.job_id, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE) + self.remote_test_metric_roc_file = os.path.join( + self.job_id, utils.MPC_TRAIN_METRIC_ROC_FILE) + self.remote_test_metric_ks_file = os.path.join( + self.job_id, utils.MPC_TRAIN_METRIC_KS_FILE) + self.remote_test_metric_pr_file = os.path.join( + self.job_id, utils.MPC_TRAIN_METRIC_PR_FILE) + self.remote_test_metric_acc_file = os.path.join( + self.job_id, utils.MPC_TRAIN_METRIC_ACCURACY_FILE) + self.remote_train_metric_ks_table = os.path.join( + self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_TABLE) + self.remote_test_metric_ks_table = os.path.join( + self.job_id, utils.MPC_TRAIN_METRIC_KS_TABLE) + self.remote_model_tree_prefix = os.path.join( + self.job_id, utils.XGB_TREE_PERFIX) + self.remote_metrics_iteration_file = os.path.join( + self.job_id, utils.METRICS_OVER_ITERATION_FILE) @staticmethod def feature_engineering_input_path(job_id: str, job_temp_dir: str): diff --git a/python/ppc_model/common/global_context.py b/python/ppc_model/common/global_context.py index 13437d90..d552bef5 100644 --- a/python/ppc_model/common/global_context.py +++ b/python/ppc_model/common/global_context.py @@ -7,7 +7,8 @@ # config_path = '{}/../application.yml'.format(dirName) config_path = "application.yml" -components = Initializer(log_config_path='logging.conf', config_path=config_path) +components = Initializer( + log_config_path='logging.conf', config_path=config_path) # matplotlib 线程不安全，并行任务绘图增加全局锁 plot_lock = threading.Lock() diff --git a/python/ppc_model/common/model_result.py b/python/ppc_model/common/model_result.py index 5b156948..0cb55142 100644 --- a/python/ppc_model/common/model_result.py +++ b/python/ppc_model/common/model_result.py @@ -25,22 +25,27 @@ def __init__(self, ctx: Context) -> None: # Synchronization result file if (len(ctx.result_receiver_id_list) == 1 and ctx.participant_id_list[0] != ctx.result_receiver_id_list[0]) \ - or len(ctx.result_receiver_id_list) > 1: + or len(ctx.result_receiver_id_list) > 1: self._sync_result_files() - + def _process_fe_result(self): if os.path.exists(self.ctx.preprocessing_result_file): - column_info_fm = pd.read_csv(self.ctx.preprocessing_result_file, index_col=0) + column_info_fm = pd.read_csv( + self.ctx.preprocessing_result_file, index_col=0) if os.path.exists(self.ctx.iv_selected_file): - column_info_iv_fm = pd.read_csv(self.ctx.iv_selected_file, index_col=0) - merged_df = self.union_column_info(column_info_fm, column_info_iv_fm) + column_info_iv_fm = pd.read_csv( + self.ctx.iv_selected_file, index_col=0) + merged_df = self.union_column_info( + column_info_fm, column_info_iv_fm) else: merged_df = column_info_fm merged_df.fillna("None", inplace=True) - merged_df.to_csv(self.ctx.selected_col_file, sep=utils.CSV_SEP, header=True, index_label='id') + merged_df.to_csv(self.ctx.selected_col_file, + sep=utils.CSV_SEP, header=True, index_label='id') # 存储column_info到hdfs给前端展示 - self._upload_file(self.ctx.components.storage_client, self.ctx.selected_col_file, self.ctx.remote_selected_col_file) + self._upload_file(self.ctx.components.storage_client, + self.ctx.selected_col_file, self.ctx.remote_selected_col_file) @staticmethod def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame): @@ -55,10 +60,12 @@ def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame): column_info_merge (DataFrame): The union column_info. """ # 将column_info1和column_info2按照left_index=True, right_index=True的方式进行合并 如果列有缺失则赋值为None 行的顺序按照column_info1 - column_info_conbine = column_info1.merge(column_info2, how='outer', left_index=True, right_index=True, sort=False) + column_info_conbine = column_info1.merge( + column_info2, how='outer', left_index=True, right_index=True, sort=False) col1_index_list = column_info1.index.to_list() col2_index_list = column_info2.index.to_list() - merged_list = col1_index_list + [item for item in col2_index_list if item not in col1_index_list] + merged_list = col1_index_list + \ + [item for item in col2_index_list if item not in col1_index_list] column_info_conbine = column_info_conbine.reindex(merged_list) return column_info_conbine @@ -71,6 +78,7 @@ def _upload_file(storage_client, local_file, remote_file): def _download_file(storage_client, local_file, remote_file): if storage_client is not None and not os.path.exists(local_file): storage_client.download_file(remote_file, local_file) + @staticmethod def make_graph_data(components, job_id, graph_file_name): graph_format = 'svg+xml' @@ -119,37 +127,39 @@ def make_csv_data(components, job_id, csv_file_name): def _remove_workspace(self): if os.path.exists(self.ctx.workspace): shutil.rmtree(self.ctx.workspace) - self.log.info(f'job {self.ctx.job_id}: {self.ctx.workspace} has been removed.') + self.log.info( + f'job {self.ctx.job_id}: {self.ctx.workspace} has been removed.') else: - self.log.info(f'job {self.ctx.job_id}: {self.ctx.workspace} does not exist.') + self.log.info( + f'job {self.ctx.job_id}: {self.ctx.workspace} does not exist.') def _sync_result_files(self): if self.ctx.algorithm_type == AlgorithmType.Train.name: - self.sync_result_file(self.ctx, self.ctx.metrics_iteration_file, + self.sync_result_file(self.ctx, self.ctx.metrics_iteration_file, self.ctx.remote_metrics_iteration_file, 'f1') - self.sync_result_file(self.ctx, self.ctx.feature_importance_file, + self.sync_result_file(self.ctx, self.ctx.feature_importance_file, self.ctx.remote_feature_importance_file, 'f2') - self.sync_result_file(self.ctx, self.ctx.summary_evaluation_file, + self.sync_result_file(self.ctx, self.ctx.summary_evaluation_file, self.ctx.remote_summary_evaluation_file, 'f3') - self.sync_result_file(self.ctx, self.ctx.train_metric_ks_table, + self.sync_result_file(self.ctx, self.ctx.train_metric_ks_table, self.ctx.remote_train_metric_ks_table, 'f4') - self.sync_result_file(self.ctx, self.ctx.train_metric_roc_file, + self.sync_result_file(self.ctx, self.ctx.train_metric_roc_file, self.ctx.remote_train_metric_roc_file, 'f5') - self.sync_result_file(self.ctx, self.ctx.train_metric_ks_file, + self.sync_result_file(self.ctx, self.ctx.train_metric_ks_file, self.ctx.remote_train_metric_ks_file, 'f6') - self.sync_result_file(self.ctx, self.ctx.train_metric_pr_file, + self.sync_result_file(self.ctx, self.ctx.train_metric_pr_file, self.ctx.remote_train_metric_pr_file, 'f7') - self.sync_result_file(self.ctx, self.ctx.train_metric_acc_file, + self.sync_result_file(self.ctx, self.ctx.train_metric_acc_file, self.ctx.remote_train_metric_acc_file, 'f8') - self.sync_result_file(self.ctx, self.ctx.test_metric_ks_table, + self.sync_result_file(self.ctx, self.ctx.test_metric_ks_table, self.ctx.remote_test_metric_ks_table, 'f9') - self.sync_result_file(self.ctx, self.ctx.test_metric_roc_file, + self.sync_result_file(self.ctx, self.ctx.test_metric_roc_file, self.ctx.remote_test_metric_roc_file, 'f10') - self.sync_result_file(self.ctx, self.ctx.test_metric_ks_file, + self.sync_result_file(self.ctx, self.ctx.test_metric_ks_file, self.ctx.remote_test_metric_ks_file, 'f11') - self.sync_result_file(self.ctx, self.ctx.test_metric_pr_file, + self.sync_result_file(self.ctx, self.ctx.test_metric_pr_file, self.ctx.remote_test_metric_pr_file, 'f12') - self.sync_result_file(self.ctx, self.ctx.test_metric_acc_file, + self.sync_result_file(self.ctx, self.ctx.test_metric_acc_file, self.ctx.remote_test_metric_acc_file, 'f13') @staticmethod @@ -163,11 +173,12 @@ def sync_result_file(ctx, local_file, remote_file, key_file): byte_data, partner_index) else: if ctx.components.config_data['AGENCY_ID'] in ctx.result_receiver_id_list: - byte_data = SendMessage._receive_byte_data(ctx.components.stub, ctx, + byte_data = SendMessage._receive_byte_data(ctx.components.stub, ctx, f'{CommonMessage.SYNC_FILE.value}_{key_file}', 0) with open(local_file, 'wb') as f: f.write(byte_data) - ResultFileHandling._upload_file(ctx.components.storage_client, local_file, remote_file) + ResultFileHandling._upload_file( + ctx.components.storage_client, local_file, remote_file) class CommonMessage(Enum): diff --git a/python/ppc_model/common/protocol.py b/python/ppc_model/common/protocol.py index c09a4da4..2ec8f65d 100644 --- a/python/ppc_model/common/protocol.py +++ b/python/ppc_model/common/protocol.py @@ -37,7 +37,8 @@ def packing_data(codec, public_key, cipher_list): for cipher in cipher_list: model_cipher = ModelCipher() - model_cipher.ciphertext, model_cipher.exponent = codec.encode_cipher(cipher) + model_cipher.ciphertext, model_cipher.exponent = codec.encode_cipher( + cipher) enc_data_pb.cipher_list.append(model_cipher) return utils.pb_to_bytes(enc_data_pb) diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py index 0278c51c..a8db9c58 100644 --- a/python/ppc_model/datasets/dataset.py +++ b/python/ppc_model/datasets/dataset.py @@ -112,7 +112,7 @@ def _random_split_dataset(self): def _customized_split_dataset(self): if self.ctx.role == TaskRole.ACTIVE_PARTY: for partner_index in range(1, len(self.ctx.participant_id_list)): - byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, + byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}', partner_index) if not os.path.exists(self.eval_column_file) and byte_data != bytes(): with open(self.eval_column_file, 'wb') as f: @@ -130,15 +130,17 @@ def _customized_split_dataset(self): byte_data = f.read() SendMessage._send_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}', byte_data, 0) - byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, - f'{CommonMessage.EVAL_SET_FILE.value}', 0) + byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx, + f'{CommonMessage.EVAL_SET_FILE.value}', 0) if not os.path.exists(self.eval_column_file): with open(self.eval_column_file, 'wb') as f: f.write(byte_data) - - eval_set_df = pd.read_csv(self.eval_column_file, header=0) - train_data = self.model_data[eval_set_df[self.eval_set_column] == self.train_set_value] - test_data = self.model_data[eval_set_df[self.eval_set_column] == self.eval_set_value] + + eval_set_df = pd.read_csv(self.eval_column_file, header=0) + train_data = self.model_data[eval_set_df[self.eval_set_column] + == self.train_set_value] + test_data = self.model_data[eval_set_df[self.eval_set_column] + == self.eval_set_value] return train_data, test_data diff --git a/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py index e06b82b2..20592c87 100644 --- a/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py +++ b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py @@ -31,16 +31,17 @@ def test_train_feature_binning(self): 'algorithm_type': 'Train', 'algorithm_subtype': None, 'model_dict': { - 'objective': 'regression', - 'max_bin': 10, - 'n_estimators': 6, - 'max_depth': 3, - 'use_goss': 1 + 'objective': 'regression', + 'max_bin': 10, + 'n_estimators': 6, + 'max_depth': 3, + 'use_goss': 1 } } task_info = SecureLGBMContext(args, self.components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label) secure_dataset = SecureDataset(task_info, model_data) print(secure_dataset.train_idx.shape) print(secure_dataset.train_X.shape) @@ -79,7 +80,8 @@ def test_test_feature_binning(self): } task_info = SecureLGBMContext(args, self.components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label) secure_dataset = SecureDataset(task_info, model_data) print(secure_dataset.train_idx.shape) print(secure_dataset.train_X.shape) diff --git a/python/ppc_model/datasets/test/test_dataset.py b/python/ppc_model/datasets/test/test_dataset.py index 7a544cfc..e7519454 100644 --- a/python/ppc_model/datasets/test/test_dataset.py +++ b/python/ppc_model/datasets/test/test_dataset.py @@ -48,7 +48,7 @@ class TestSecureDataset(unittest.TestCase): iv_selected_file = './iv_selected.csv' if not os.path.exists(iv_selected_file): iv_selected = pd.DataFrame( - {'feature': [f'x{i + 1}' for i in range(30)], + {'feature': [f'x{i + 1}' for i in range(30)], 'iv_selected': np.random.binomial(n=1, p=0.5, size=30)}) iv_selected.to_csv(iv_selected_file, index=None) @@ -203,9 +203,11 @@ def test_read_dataset(self): df = pd.DataFrame(origin_data, columns=columns) csv_file = '/tmp/data_x1_to_x10.csv' df.to_csv(csv_file, index=False) - field_list, label, feature = SecureDataset.read_dataset(csv_file, False, delimiter=',') + field_list, label, feature = SecureDataset.read_dataset( + csv_file, False, delimiter=',') self.assertEqual(['id'] + field_list, columns) - field_list, label, feature = SecureDataset.read_dataset(csv_file, True, delimiter=',') + field_list, label, feature = SecureDataset.read_dataset( + csv_file, True, delimiter=',') self.assertEqual(['id'] + field_list, columns) diff --git a/python/ppc_model/feature_engineering/feature_engineering_engine.py b/python/ppc_model/feature_engineering/feature_engineering_engine.py index 16acdb7b..0b3bb88f 100644 --- a/python/ppc_model/feature_engineering/feature_engineering_engine.py +++ b/python/ppc_model/feature_engineering/feature_engineering_engine.py @@ -17,7 +17,8 @@ def run(args): args['job_id'], components.config_data['JOB_TEMP_DIR']) if args['is_label_holder']: - field_list, label, feature = SecureDataset.read_dataset(input_path, True) + field_list, label, feature = SecureDataset.read_dataset( + input_path, True) context = FeatureEngineeringContext( args=args, components=components, @@ -28,7 +29,8 @@ def run(args): ) vfe = VerticalFeatureEngineeringActiveParty(context) else: - field_list, _, feature = SecureDataset.read_dataset(input_path, False) + field_list, _, feature = SecureDataset.read_dataset( + input_path, False) context = FeatureEngineeringContext( args=args, components=components, diff --git a/python/ppc_model/feature_engineering/vertical/active_party.py b/python/ppc_model/feature_engineering/vertical/active_party.py index 6710c47d..82523420 100644 --- a/python/ppc_model/feature_engineering/vertical/active_party.py +++ b/python/ppc_model/feature_engineering/vertical/active_party.py @@ -139,7 +139,8 @@ def _get_all_enc_aggr_labels(self, partner_id): enc_aggr_labels_list_pb = EncAggrLabelsList() utils.bytes_to_pb(enc_aggr_labels_list_pb, data) - public_key = self.ctx.codec.decode_enc_key(enc_aggr_labels_list_pb.public_key) + public_key = self.ctx.codec.decode_enc_key( + enc_aggr_labels_list_pb.public_key) res = [] for enc_aggr_labels_pb in enc_aggr_labels_list_pb.enc_aggr_labels_list: @@ -160,7 +161,8 @@ def _send_enc_labels(self, enc_labels, receiver): log = self.ctx.components.logger() start_time = time.time() - data = PheMessage.packing_data(self.ctx.codec, self.ctx.phe.public_key, enc_labels) + data = PheMessage.packing_data( + self.ctx.codec, self.ctx.phe.public_key, enc_labels) self.ctx.components.stub.push(PushRequest( receiver=receiver, task_id=self.ctx.task_id, @@ -174,8 +176,10 @@ def _send_enc_labels(self, enc_labels, receiver): def _save_and_sync_fe_results(self): log = self.ctx.components.logger() task_id = self.ctx.task_id - self.woe_iv_df.to_csv(self.ctx.woe_iv_file, sep=',', header=True, index=None) - self.iv_selected_df.to_csv(self.ctx.iv_selected_file, sep=',', header=True, index=None) + self.woe_iv_df.to_csv(self.ctx.woe_iv_file, + sep=',', header=True, index=None) + self.iv_selected_df.to_csv( + self.ctx.iv_selected_file, sep=',', header=True, index=None) self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file, self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE) log.info(f"Saving fe results finished, task_id: {task_id}") diff --git a/python/ppc_model/feature_engineering/vertical/passive_party.py b/python/ppc_model/feature_engineering/vertical/passive_party.py index 8aa63996..4b3ffd47 100644 --- a/python/ppc_model/feature_engineering/vertical/passive_party.py +++ b/python/ppc_model/feature_engineering/vertical/passive_party.py @@ -30,7 +30,8 @@ def fit(self, *args, **kwargs) -> None: public_key, enc_labels = self._get_enc_labels() # 根据特征分箱，聚合加密标签 - aggr_labels_bytes_list = self._binning_and_aggregating_all(public_key, enc_labels) + aggr_labels_bytes_list = self._binning_and_aggregating_all( + public_key, enc_labels) # 发送聚合的密文标签 self._send_all_enc_aggr_labels(public_key, aggr_labels_bytes_list) @@ -49,7 +50,8 @@ def _get_enc_labels(self): key=FeMessage.ENC_LABELS.value )) - public_key, enc_labels = PheMessage.unpacking_data(self.ctx.codec, data) + public_key, enc_labels = PheMessage.unpacking_data( + self.ctx.codec, data) log.info(f"All enc labels received, task_id: {self.ctx.task_id}, label_num: {len(enc_labels)}, " f"size: {len(data) / 1024}KB, timecost: {time.time() - start_time}s") return public_key, enc_labels @@ -59,7 +61,8 @@ def _binning_and_aggregating_all(self, public_key, enc_labels) -> list: start_time = time.time() params = [] for i in range(self.ctx.feature.shape[1]): - is_continuous = is_continuous_feature(self.ctx.categorical, self.ctx.feature_name_list[i]) + is_continuous = is_continuous_feature( + self.ctx.categorical, self.ctx.feature_name_list[i]) params.append({ 'is_continuous': is_continuous, 'feature_index': i, @@ -89,7 +92,8 @@ def _binning_and_aggregating_all(self, public_key, enc_labels) -> list: def _binning_and_aggregating_one(param): feature = param['feature'] if param['is_continuous']: - bins = FeatureBinning.binning_continuous_feature(feature, param['group_num'])[0] + bins = FeatureBinning.binning_continuous_feature( + feature, param['group_num'])[0] else: bins = FeatureBinning.binning_categorical_feature(feature)[0] @@ -103,8 +107,10 @@ def _binning_and_aggregating_one(param): else: data_dict[key] = {'count': 1, 'sum': value} - count_list = [data_dict[key]['count'] for key in sorted(data_dict.keys())] - aggr_enc_labels = [data_dict[key]['sum'] for key in sorted(data_dict.keys())] + count_list = [data_dict[key]['count'] + for key in sorted(data_dict.keys())] + aggr_enc_labels = [data_dict[key]['sum'] + for key in sorted(data_dict.keys())] return VerticalFeatureEngineeringPassiveParty._encode_enc_aggr_labels( param['codec'], param['field'], count_list, aggr_enc_labels) @@ -125,12 +131,14 @@ def _encode_enc_aggr_labels(codec, field, count_list, aggr_enc_labels): def _send_all_enc_aggr_labels(self, public_key, aggr_labels_bytes_list): start_time = time.time() enc_aggr_labels_list_pb = EncAggrLabelsList() - enc_aggr_labels_list_pb.public_key = self.ctx.codec.encode_enc_key(public_key) + enc_aggr_labels_list_pb.public_key = self.ctx.codec.encode_enc_key( + public_key) for aggr_labels_bytes in aggr_labels_bytes_list: enc_aggr_labels_pb = EncAggrLabels() utils.bytes_to_pb(enc_aggr_labels_pb, aggr_labels_bytes) - enc_aggr_labels_list_pb.enc_aggr_labels_list.append(enc_aggr_labels_pb) + enc_aggr_labels_list_pb.enc_aggr_labels_list.append( + enc_aggr_labels_pb) data = utils.pb_to_bytes(enc_aggr_labels_list_pb) diff --git a/python/ppc_model/feature_engineering/vertical/utils.py b/python/ppc_model/feature_engineering/vertical/utils.py index cdeeceaf..32faa2fc 100644 --- a/python/ppc_model/feature_engineering/vertical/utils.py +++ b/python/ppc_model/feature_engineering/vertical/utils.py @@ -41,9 +41,11 @@ def calculate_woe_iv(feature: np.ndarray, label: np.ndarray, num_bins: int = 10, combined = pd.DataFrame({'feature': feature, 'label': label}) # 按特征值对数据集进行分箱 if is_continuous: - combined['bins'] = FeatureBinning.binning_continuous_feature(feature, num_bins, is_equal_freq)[0] + combined['bins'] = FeatureBinning.binning_continuous_feature( + feature, num_bins, is_equal_freq)[0] else: - combined['bins'] = FeatureBinning.binning_categorical_feature(feature)[0] + combined['bins'] = FeatureBinning.binning_categorical_feature(feature)[ + 0] # 计算每个分箱中的正负样本数量和总体样本数量 grouped = combined.groupby('bins')['label'].agg(['count', 'sum']) grouped = grouped.rename(columns={'sum': 'pos_event'}) diff --git a/python/ppc_model/metrics/evaluation.py b/python/ppc_model/metrics/evaluation.py index a759dc9d..2bf12ebb 100644 --- a/python/ppc_model/metrics/evaluation.py +++ b/python/ppc_model/metrics/evaluation.py @@ -24,11 +24,11 @@ class Evaluation: - def __init__(self, - ctx: Context, - dataset: SecureDataset, - train_praba:np.ndarray = None, - test_praba:np.ndarray = None) -> None: + def __init__(self, + ctx: Context, + dataset: SecureDataset, + train_praba: np.ndarray = None, + test_praba: np.ndarray = None) -> None: self.job_id = ctx.job_id self.storage_client = ctx.components.storage_client @@ -59,7 +59,8 @@ def __init__(self, train_ks, train_auc = self.evaluation_file( ctx, dataset.train_idx, dataset.train_y, train_praba, 'train') if dataset.train_y is not None: - self.summary_evaluation(dataset, test_ks, test_auc, train_ks, train_auc) + self.summary_evaluation( + dataset, test_ks, test_auc, train_ks, train_auc) @staticmethod def fevaluation( @@ -98,33 +99,39 @@ def summary_evaluation(self, dataset, test_ks, test_auc, train_ks, train_auc): @staticmethod def calculate_ks_and_stats(predicted_proba, actual_label, num_buckets=10): # 合并预测概率和实际标签为一个 DataFrame - df = pd.DataFrame({'predicted_proba': predicted_proba.reshape(-1), 'actual_label': actual_label.reshape(-1)}) + df = pd.DataFrame({'predicted_proba': predicted_proba.reshape(-1), + 'actual_label': actual_label.reshape(-1)}) # 根据预测概率降序排列 df_sorted = df.sort_values(by='predicted_proba', ascending=False) # 将数据划分为 num_buckets 个分组 try: - df_sorted['bucket'] = pd.qcut(df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] + df_sorted['bucket'] = pd.qcut( + df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] except Exception: - df_sorted['bucket'] = pd.cut(df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] + df_sorted['bucket'] = pd.cut( + df_sorted['predicted_proba'], num_buckets, retbins=True, labels=False)[0] # 统计每个分组的信息 stats = df_sorted.groupby('bucket').agg({ 'actual_label': ['count', 'sum'], 'predicted_proba': ['min', 'max'] }) # 计算其他指标 - stats.columns = ['count', 'positive_count', 'predict_proba_min', 'predict_proba_max'] + stats.columns = ['count', 'positive_count', + 'predict_proba_min', 'predict_proba_max'] stats['positive_ratio'] = stats['positive_count'] / stats['count'] stats['negative_ratio'] = 1 - stats['positive_ratio'] stats['count_ratio'] = stats['count'] / stats['count'].sum() # stats['累计坏客户占比'] = stats['坏客户数'].cumsum() / stats['坏客户数'].sum() # 计算累计坏客户占比，从第 9 组开始计算 - stats['cum_positive_ratio'] = stats['positive_count'].iloc[::-1].cumsum()[::-1] / stats['positive_count'].sum() + stats['cum_positive_ratio'] = stats['positive_count'].iloc[::- + 1].cumsum()[::-1] / stats['positive_count'].sum() stats = stats[['count_ratio', 'count', 'positive_count', - 'positive_ratio', 'negative_ratio', 'cum_positive_ratio']].reset_index() - stats.columns = ['分组', '样本占比', '样本数', '正样本数', '正样本比例', '负样本比例', '累积正样本占比'] + 'positive_ratio', 'negative_ratio', 'cum_positive_ratio']].reset_index() + stats.columns = ['分组', '样本占比', '样本数', + '正样本数', '正样本比例', '负样本比例', '累积正样本占比'] return stats - def evaluation_file(self, ctx, data_index: np.ndarray, + def evaluation_file(self, ctx, data_index: np.ndarray, y_true: np.ndarray, y_praba: np.ndarray, label: str = 'test'): if label == 'train': self.model_result_file = ctx.train_model_result_file @@ -150,45 +157,55 @@ def evaluation_file(self, ctx, data_index: np.ndarray, retry_num += 1 try: with plot_lock: - ks_value, auc_value = Evaluation.plot_two_class_graph(self, y_true, y_praba) + ks_value, auc_value = Evaluation.plot_two_class_graph( + self, y_true, y_praba) except: - ctx.components.logger().info(f'y_true = {len(y_true)}, {y_true[0:2]}') - ctx.components.logger().info(f'y_praba = {len(y_praba)}, {y_praba[0:2]}') + ctx.components.logger().info( + f'y_true = {len(y_true)}, {y_true[0:2]}') + ctx.components.logger().info( + f'y_praba = {len(y_praba)}, {y_praba[0:2]}') err = traceback.format_exc() # ctx.components.logger().exception(err) ctx.components.logger().info( f'plot metrics in times-{retry_num} failed, traceback: {err}.') time.sleep(random.uniform(0.1, 3)) - - ResultFileHandling._upload_file(self.storage_client, self.metric_roc_file, self.remote_metric_roc_file) - ResultFileHandling._upload_file(self.storage_client, self.metric_ks_file, self.remote_metric_ks_file) - ResultFileHandling._upload_file(self.storage_client, self.metric_pr_file, self.remote_metric_pr_file) - ResultFileHandling._upload_file(self.storage_client, self.metric_acc_file, self.remote_metric_acc_file) + + ResultFileHandling._upload_file( + self.storage_client, self.metric_roc_file, self.remote_metric_roc_file) + ResultFileHandling._upload_file( + self.storage_client, self.metric_ks_file, self.remote_metric_ks_file) + ResultFileHandling._upload_file( + self.storage_client, self.metric_pr_file, self.remote_metric_pr_file) + ResultFileHandling._upload_file( + self.storage_client, self.metric_acc_file, self.remote_metric_acc_file) # ks table ks_table = self.calculate_ks_and_stats(y_praba, y_true) ks_table.to_csv(self.metric_ks_table, header=True, index=None) - ResultFileHandling._upload_file(self.storage_client, self.metric_ks_table, self.remote_metric_ks_table) + ResultFileHandling._upload_file( + self.storage_client, self.metric_ks_table, self.remote_metric_ks_table) else: ks_value = auc_value = None - + # predict result self._parse_model_result(data_index, y_true, y_praba) - ResultFileHandling._upload_file(self.storage_client, self.model_output_file, self.remote_model_output_file) + ResultFileHandling._upload_file( + self.storage_client, self.model_output_file, self.remote_model_output_file) return ks_value, auc_value def _parse_model_result(self, data_index, y_true=None, y_praba=None): - + np.savetxt(self.model_result_file, y_praba, delimiter=',', fmt='%f') if y_true is None: - df = pd.DataFrame(np.column_stack((data_index, y_praba)), columns=['id', 'class_pred']) + df = pd.DataFrame(np.column_stack( + (data_index, y_praba)), columns=['id', 'class_pred']) else: - df = pd.DataFrame(np.column_stack((data_index, y_true, y_praba)), + df = pd.DataFrame(np.column_stack((data_index, y_true, y_praba)), columns=['id', 'class_label', 'class_pred']) df['class_label'] = df['class_label'].astype(int) - + df['id'] = df['id'].astype(int) df['class_pred'] = df['class_pred'].astype(float) df = df.sort_values(by='id') @@ -202,7 +219,8 @@ def plot_two_class_graph(self, y_true, y_scores): plt.rcParams['figure.figsize'] = (12.0, 8.0) # plot ROC - fpr, tpr, thresholds = roc_curve(y_label_probs, y_pred_probs, pos_label=1) + fpr, tpr, thresholds = roc_curve( + y_label_probs, y_pred_probs, pos_label=1) auc_value = auc(fpr, tpr) plt.figure(f'roc-{self.job_id}') plt.title('ROC Curve') # give plot a title @@ -210,12 +228,12 @@ def plot_two_class_graph(self, y_true, y_scores): plt.ylabel('True Positive Rate (Sensitivity)') plt.plot([0, 1], [0, 1], 'k--', lw=2) plt.plot(fpr, tpr, label='area = {0:0.5f}' - ''.format(auc_value)) + ''.format(auc_value)) plt.legend(loc="lower right") plt.savefig(self.metric_roc_file, dpi=1000) # plt.show() - plt.close('all') + plt.close('all') gc.collect() # plot KS @@ -230,12 +248,12 @@ def plot_two_class_graph(self, y_true, y_scores): # 标记最大ks值 x_index = np.argwhere(abs(fpr - tpr) == ks_value)[0, 0] plt.plot((threshold_x[x_index], threshold_x[x_index]), (fpr[x_index], tpr[x_index]), - label='ks = {:.3f}'.format(ks_value), color='r', marker='o', markerfacecolor='r', markersize=5) + label='ks = {:.3f}'.format(ks_value), color='r', marker='o', markerfacecolor='r', markersize=5) plt.legend(loc="lower right") plt.savefig(self.metric_ks_file, dpi=1000) # plt.show() - plt.close('all') + plt.close('all') gc.collect() # plot Precision Recall @@ -251,7 +269,7 @@ def plot_two_class_graph(self, y_true, y_scores): plt.savefig(self.metric_pr_file, dpi=1000) # plt.show() - plt.close('all') + plt.close('all') gc.collect() # plot accuracy @@ -271,6 +289,6 @@ def plot_two_class_graph(self, y_true, y_scores): plt.savefig(self.metric_acc_file, dpi=1000) # plt.show() - plt.close('all') + plt.close('all') gc.collect() return (ks_value, auc_value) diff --git a/python/ppc_model/metrics/model_plot.py b/python/ppc_model/metrics/model_plot.py index ca169859..bdae534b 100644 --- a/python/ppc_model/metrics/model_plot.py +++ b/python/ppc_model/metrics/model_plot.py @@ -12,9 +12,9 @@ class ModelPlot: - + def __init__(self, model: VerticalBooster) -> None: - + self.ctx = model.ctx self.model = model self._tree_id = 0 @@ -24,17 +24,19 @@ def __init__(self, model: VerticalBooster) -> None: self.storage_client = self.ctx.components.storage_client if model._trees is not None and \ - self.ctx.components.config_data['AGENCY_ID'] in self.ctx.result_receiver_id_list: + self.ctx.components.config_data['AGENCY_ID'] in self.ctx.result_receiver_id_list: self.plot_tree() - + def plot_tree(self): trees = self.model._trees self._split = self.model._X_split for i, tree in enumerate(trees): if i < 6: - tree_file_path = self.ctx.model_tree_prefix + '_' + str(self._tree_id)+'.svg' - remote_tree_file_path = self.ctx.remote_model_tree_prefix + '_' + str(self._tree_id)+'.svg' + tree_file_path = self.ctx.model_tree_prefix + \ + '_' + str(self._tree_id)+'.svg' + remote_tree_file_path = self.ctx.remote_model_tree_prefix + \ + '_' + str(self._tree_id)+'.svg' self._tree_id += 1 self._leaf_id = 0 self._G = DiGraphTree() @@ -49,9 +51,11 @@ def plot_tree(self): retry_num += 1 try: with plot_lock: - self._G.tree_plot(figsize=(10, 5), save_filename=tree_file_path) + self._G.tree_plot( + figsize=(10, 5), save_filename=tree_file_path) except: - self.ctx.components.logger().info(f'tree_id = {i}, tree = {tree}') + self.ctx.components.logger().info( + f'tree_id = {i}, tree = {tree}') self.ctx.components.logger().info(f'G = {self._G}') err = traceback.format_exc() # self.ctx.components.logger().exception(err) @@ -59,15 +63,18 @@ def plot_tree(self): f'plot tree-{i} in times-{retry_num} failed, traceback: {err}.') time.sleep(random.uniform(0.1, 3)) - ResultFileHandling._upload_file(self.storage_client, tree_file_path, remote_tree_file_path) + ResultFileHandling._upload_file( + self.storage_client, tree_file_path, remote_tree_file_path) def _graph_gtree(self, tree, leaf_id=0, depth=0, orient=None, split_info=None): self._leaf_id += 1 self._G.add_node(self._leaf_id) if split_info is not None: if self.ctx.participant_id_list[split_info.agency_idx] == self.ctx.components.config_data['AGENCY_ID']: - feature = str(self.model.dataset.feature_name[split_info.agency_feature]) - value = str(round(float(self._split[split_info.agency_feature][split_info.value]), 4)) + feature = str( + self.model.dataset.feature_name[split_info.agency_feature]) + value = str( + round(float(self._split[split_info.agency_feature][split_info.value]), 4)) else: feature = str(split_info.feature) value = str(split_info.value) @@ -84,8 +91,10 @@ def _graph_gtree(self, tree, leaf_id=0, depth=0, orient=None, split_info=None): self._G.add_weighted_edges_from( [(leaf_id, self._leaf_id, orient+'_'+feature+'_'+value+'_'+str(split_info.w_right))]) my_leaf_id = self._leaf_id - self._graph_gtree(left_tree, my_leaf_id, depth+1, 'left', best_split_info) - self._graph_gtree(right_tree, my_leaf_id, depth+1, 'right', best_split_info) + self._graph_gtree(left_tree, my_leaf_id, depth + + 1, 'left', best_split_info) + self._graph_gtree(right_tree, my_leaf_id, depth + + 1, 'right', best_split_info) else: if leaf_id != 0: self._G.add_weighted_edges_from( @@ -99,7 +108,8 @@ def __init__(self): super().__init__() def tree_leaves(self): - leaves_list = [x for x in self.nodes() if self.out_degree(x)==0 and self.in_degree(x)<=1] + leaves_list = [x for x in self.nodes() if self.out_degree( + x) == 0 and self.in_degree(x) <= 1] return leaves_list def tree_dfs_nodes(self): @@ -107,7 +117,8 @@ def tree_dfs_nodes(self): return nodes_list def tree_dfs_leaves(self): - dfs_leaves = [x for x in self.tree_dfs_nodes() if x in self.tree_leaves()] + dfs_leaves = [x for x in self.tree_dfs_nodes() + if x in self.tree_leaves()] return dfs_leaves def tree_depth(self): @@ -127,7 +138,8 @@ def tree_plot(self, split=True, figsize=(20, 10), dpi=300, save_filename=None): if split: labels = {} # leaves = self.tree_leaves() - leaves = [x for x in self.nodes() if self.out_degree(x)==0 and self.in_degree(x)<=1] + leaves = [x for x in self.nodes() if self.out_degree(x) == + 0 and self.in_degree(x) <= 1] if leaves == [0]: leaves = [] @@ -146,7 +158,8 @@ def tree_plot(self, split=True, figsize=(20, 10), dpi=300, save_filename=None): else: in_node = list(nx.neighbors(self, n))[0] weight = edge_labels[(n, in_node)] - labels[n] = weight.split('_')[1] + ':' + weight.split('_')[2] + labels[n] = weight.split( + '_')[1] + ':' + weight.split('_')[2] # for key, value in edge_labels.items(): # edge_labels[key] = round(float(value.split('_')[3]), 4) @@ -169,7 +182,8 @@ def tree_plot(self, split=True, figsize=(20, 10), dpi=300, save_filename=None): '-' + str(round(float(value.split('_')[3]), 4)) plt.figure(figsize=figsize, dpi=dpi) - nx.draw(self, pos, with_labels=True, labels=labels, font_weight='bold') + nx.draw(self, pos, with_labels=True, + labels=labels, font_weight='bold') nx.draw_networkx_edge_labels(self, pos, edge_labels=edge_labels) # plt.show() if save_filename is not None: diff --git a/python/ppc_model/metrics/test/test_metrics.py b/python/ppc_model/metrics/test/test_metrics.py index 3545aaa7..5382d5d8 100644 --- a/python/ppc_model/metrics/test/test_metrics.py +++ b/python/ppc_model/metrics/test/test_metrics.py @@ -66,13 +66,14 @@ class TestXgboostTraining(unittest.TestCase): args_a, args_b = mock_args() def test_active_metrics(self): - + active_components = Initializer(log_config_path='', config_path='') active_components.config_data = { 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} active_components.mock_logger = MockLogger() task_info_a = SecureLGBMContext(self.args_a, active_components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=True) secure_dataset_a = SecureDataset(task_info_a, model_data) booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) print(secure_dataset_a.feature_name) @@ -86,7 +87,8 @@ def test_active_metrics(self): # booster_a._train_praba = np.random.rand(len(secure_dataset_a.train_y)) booster_a._test_praba = np.random.rand(len(secure_dataset_a.test_y)) - Evaluation(task_info_a, secure_dataset_a, booster_a._train_praba, booster_a._test_praba) + Evaluation(task_info_a, secure_dataset_a, + booster_a._train_praba, booster_a._test_praba) def test_passive_metrics(self): @@ -95,7 +97,8 @@ def test_passive_metrics(self): 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} passive_components.mock_logger = MockLogger() task_info_b = SecureLGBMContext(self.args_b, passive_components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=False) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=False) secure_dataset_b = SecureDataset(task_info_b, model_data) booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) print(secure_dataset_b.feature_name) @@ -107,7 +110,8 @@ def test_passive_metrics(self): # booster_b._train_praba = np.random.rand(len(secure_dataset_b.train_idx)) booster_b._test_praba = np.random.rand(len(secure_dataset_b.test_idx)) - Evaluation(task_info_b, secure_dataset_b, booster_b._train_praba, booster_b._test_praba) + Evaluation(task_info_b, secure_dataset_b, + booster_b._train_praba, booster_b._test_praba) def test_model_plot(self): @@ -116,7 +120,8 @@ def test_model_plot(self): 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} active_components.mock_logger = MockLogger() task_info_a = SecureLGBMContext(self.args_a, active_components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=True) secure_dataset_a = SecureDataset(task_info_a, model_data) booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) if os.path.exists(booster_a.ctx.model_data_file): @@ -128,32 +133,32 @@ def test_digraphtree(self): Gtree.add_node(0) Gtree.add_nodes_from([1, 2]) Gtree.add_weighted_edges_from( - [(0, 1, 'left_'+str(2)+'_'+str(3)+'_'+str(0.5)), - (0, 2, 'right_'+str(2)+'_'+str(3)+'_'+str(0.9))]) + [(0, 1, 'left_'+str(2)+'_'+str(3)+'_'+str(0.5)), + (0, 2, 'right_'+str(2)+'_'+str(3)+'_'+str(0.9))]) Gtree.add_nodes_from([3, 4]) Gtree.add_weighted_edges_from( - [(1, 3, 'left_'+str(20)+'_'+str(4)+'_'+str(0.5)), - (1, 4, 'right_'+str(20)+'_'+str(4)+'_'+str(0.9))]) + [(1, 3, 'left_'+str(20)+'_'+str(4)+'_'+str(0.5)), + (1, 4, 'right_'+str(20)+'_'+str(4)+'_'+str(0.9))]) Gtree.add_nodes_from([5, 6]) Gtree.add_weighted_edges_from( - [(2, 5, 'left_'+str(2)+'_'+str(7)+'_'+str(0.5)), - (2, 6, 'right_'+str(2)+'_'+str(7)+'_'+str(0.9))]) + [(2, 5, 'left_'+str(2)+'_'+str(7)+'_'+str(0.5)), + (2, 6, 'right_'+str(2)+'_'+str(7)+'_'+str(0.9))]) Gtree.add_nodes_from([7, 8]) Gtree.add_weighted_edges_from( - [(3, 7, 'left_'+str(1)+'_'+str(11)+'_'+str(0.5)), - (3, 8, 'right_'+str(1)+'_'+str(11)+'_'+str(0.9))]) + [(3, 7, 'left_'+str(1)+'_'+str(11)+'_'+str(0.5)), + (3, 8, 'right_'+str(1)+'_'+str(11)+'_'+str(0.9))]) Gtree.add_nodes_from([9, 10]) Gtree.add_weighted_edges_from( - [(4, 9, 'left_'+str(18)+'_'+str(2)+'_'+str(0.5)), - (4, 10, 'right_'+str(18)+'_'+str(2)+'_'+str(0.9))]) + [(4, 9, 'left_'+str(18)+'_'+str(2)+'_'+str(0.5)), + (4, 10, 'right_'+str(18)+'_'+str(2)+'_'+str(0.9))]) Gtree.add_nodes_from([11, 12]) Gtree.add_weighted_edges_from( - [(5, 11, 'left_'+str(23)+'_'+str(25)+'_'+str(0.5)), - (5, 12, 'right_'+str(23)+'_'+str(25)+'_'+str(0.9))]) + [(5, 11, 'left_'+str(23)+'_'+str(25)+'_'+str(0.5)), + (5, 12, 'right_'+str(23)+'_'+str(25)+'_'+str(0.9))]) Gtree.add_nodes_from([13, 14]) Gtree.add_weighted_edges_from( - [(6, 13, 'left_'+str(16)+'_'+str(10)+'_'+str(0.5)), - (6, 14, 'right_'+str(16)+'_'+str(10)+'_'+str(0.9))]) + [(6, 13, 'left_'+str(16)+'_'+str(10)+'_'+str(0.5)), + (6, 14, 'right_'+str(16)+'_'+str(10)+'_'+str(0.9))]) # Gtree.tree_plot() # Gtree.tree_plot(split=False, figsize=(10, 5)) diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py index 3ebbac2f..f8b0130f 100644 --- a/python/ppc_model/network/http/model_controller.py +++ b/python/ppc_model/network/http/model_controller.py @@ -30,7 +30,8 @@ def post(self, model_id): """ args = request.get_json() task_id = model_id - components.logger().info(f"run task request, task_id: {task_id}, args: {args}") + components.logger().info( + f"run task request, task_id: {task_id}, args: {args}") task_type = args['task_type'] components.task_manager.run_task( task_id, ModelTask(task_type), (args,)) @@ -43,7 +44,8 @@ def get(self, model_id): """ response = utils.BASE_RESPONSE task_id = model_id - status, traffic_volume, time_costs = components.task_manager.status(task_id) + status, traffic_volume, time_costs = components.task_manager.status( + task_id) response['data'] = { 'status': status, 'traffic_volume': traffic_volume, diff --git a/python/ppc_model/network/http/restx.py b/python/ppc_model/network/http/restx.py index 83dba0ce..553337ae 100644 --- a/python/ppc_model/network/http/restx.py +++ b/python/ppc_model/network/http/restx.py @@ -22,7 +22,8 @@ def default_error_handler(e): components.logger().exception(e) info = e.to_dict() response = {'errorCode': info['code'], 'message': info['message']} - components.logger().error(f"OnError: code: {info['code']}, message: {info['message']}") + components.logger().error( + f"OnError: code: {info['code']}, message: {info['message']}") return response, 500 diff --git a/python/ppc_model/network/stub.py b/python/ppc_model/network/stub.py index 7db22b1e..d472847b 100644 --- a/python/ppc_model/network/stub.py +++ b/python/ppc_model/network/stub.py @@ -47,7 +47,8 @@ def __init__( self.agency_id = agency_id self._thread_event_manager = thread_event_manager self._rpc_client = rpc_client - self._executor = ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)) + self._executor = ThreadPoolExecutor( + max_workers=max(1, os.cpu_count() - 1)) self._send_retry_times = send_retry_times self._retry_interval_s = retry_interval_s # 缓存收到的消息 [task_id:[sender:[key:[seq: data]]]] diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py index baef888b..79060c82 100644 --- a/python/ppc_model/ppc_model_app.py +++ b/python/ppc_model/ppc_model_app.py @@ -1,30 +1,28 @@ # Note: here can't be refactored by autopep +from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine +from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine +from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine +from ppc_model.network.http.restx import api +from ppc_model.network.http.model_controller import ns2 as log_namespace +from ppc_model.network.http.model_controller import ns as task_namespace +from ppc_model.network.grpc.grpc_server import ModelService +from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine +from ppc_model.common.protocol import ModelTask +from ppc_model.common.global_context import components +from ppc_common.ppc_utils import utils +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from paste.translogger import TransLogger +from flask import Flask, Blueprint +from cheroot.wsgi import Server as WSGIServer +from cheroot.ssl.builtin import BuiltinSSLAdapter +import grpc +from threading import Thread +from concurrent import futures +import os +import multiprocessing import sys sys.path.append("../") -import multiprocessing -import os -from concurrent import futures -from threading import Thread - -import grpc -from cheroot.ssl.builtin import BuiltinSSLAdapter -from cheroot.wsgi import Server as WSGIServer -from flask import Flask, Blueprint -from paste.translogger import TransLogger - -from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc -from ppc_common.ppc_utils import utils -from ppc_model.common.global_context import components -from ppc_model.common.protocol import ModelTask -from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine -from ppc_model.network.grpc.grpc_server import ModelService -from ppc_model.network.http.model_controller import ns as task_namespace -from ppc_model.network.http.model_controller import ns2 as log_namespace -from ppc_model.network.http.restx import api -from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine -from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine -from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine app = Flask(__name__) @@ -57,7 +55,8 @@ def model_serve(): if app.config['SSL_SWITCH'] == 0: ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), options=components.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), ppc_serve) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + ModelService(), ppc_serve) address = "[::]:{}".format(app.config['RPC_PORT']) ppc_serve.add_insecure_port(address) else: @@ -74,7 +73,8 @@ def model_serve(): ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), options=components.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), ppc_serve) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + ModelService(), ppc_serve) address = "[::]:{}".format(app.config['RPC_PORT']) ppc_serve.add_secure_port(address, server_credentials) diff --git a/python/ppc_model/preprocessing/local_processing/preprocessing.py b/python/ppc_model/preprocessing/local_processing/preprocessing.py index 55082a62..f0d39faf 100644 --- a/python/ppc_model/preprocessing/local_processing/preprocessing.py +++ b/python/ppc_model/preprocessing/local_processing/preprocessing.py @@ -103,8 +103,10 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb if model_setting.eval_set_column is not None: if model_setting.eval_set_column in dataset_df.columns: eval_column = model_setting.eval_set_column - dataset_df[['id', eval_column]].to_csv(ctx.eval_column_file, index=None) - ctx.components.storage_client.upload_file(ctx.eval_column_file, job_id + os.sep + ctx.EVAL_COLUMN_FILE) + dataset_df[['id', eval_column]].to_csv( + ctx.eval_column_file, index=None) + ctx.components.storage_client.upload_file( + ctx.eval_column_file, job_id + os.sep + ctx.EVAL_COLUMN_FILE) if model_setting.eval_set_column != model_setting.psi_select_col: dataset_df = dataset_df.drop(columns=[eval_column]) diff --git a/python/ppc_model/preprocessing/tests/test_preprocessing.py b/python/ppc_model/preprocessing/tests/test_preprocessing.py index 279e00e6..244c6a41 100644 --- a/python/ppc_model/preprocessing/tests/test_preprocessing.py +++ b/python/ppc_model/preprocessing/tests/test_preprocessing.py @@ -534,6 +534,7 @@ def test_process_train_dataframe_with_additional_columns(): # Assert that the processed DataFrame matches the expected output assert processed_df.equals(expected_output) + def test_merge_column_info_from_file(): col_info_file_path = "./test_column_info_merge.csv" iv_info_file_path = "./test_column_info_iv.csv" @@ -548,7 +549,8 @@ def test_merge_column_info_from_file(): # assert expected_df.equals(union_df) column_info_str = json.dumps(column_info_fm.to_dict(orient='index')) assert column_info_str == col_str_expected - + + def construct_dataset(num_samples, num_features, file_path): np.random.seed(0) # 生成标签列 @@ -557,24 +559,26 @@ def construct_dataset(num_samples, num_features, file_path): features = np.random.rand(num_samples, num_features) # 将标签转换为DataFrame labels_df = pd.DataFrame(labels, columns=['Label']) - + # 将特征转换为DataFrame features_df = pd.DataFrame(features) - + # 合并标签和特征DataFrame dataset_df = pd.concat([labels_df, features_df], axis=1) - + # 将DataFrame写入CSV文件 dataset_df.to_csv(file_path, index=False) - + return labels, features + def test_gen_file(): num_samples = 400000 num_features = 100 file_path = "./dataset-{}-{}.csv".format(num_samples, num_features) construct_dataset(num_samples, num_features, file_path) + def test_large_process_train_dataframe(): num_samples = 400000 num_features = 100 @@ -624,15 +628,13 @@ def test_large_process_train_dataframe(): column_info1 = process_dataframe( df_filled, xgb_dict, "./xgb_data_file_path", utils.AlgorithmType.Train.name, "j-123456") end_time = time.time() - print(f"test_large_process_train_dataframe time cost:{end_time-start_time}, num_samples: {num_samples}, num_features: {num_features}") + print( + f"test_large_process_train_dataframe time cost:{end_time-start_time}, num_samples: {num_samples}, num_features: {num_features}") + - - - - # Run the tests # pytest.main() -if __name__=="__main__": +if __name__ == "__main__": import time # test_large_process_train_dataframe() time1 = time.time() @@ -667,6 +669,7 @@ def test_large_process_train_dataframe(): print(f"test_process_psi time cost: {time8-time7}") print(f"test_process_dataframe time cost: {time9-time8}") print(f"test_process_train_dataframe time cost: {time10-time9}") - print(f"test_process_train_dataframe_with_additional_columns time cost: {time11-time10}") + print( + f"test_process_train_dataframe_with_additional_columns time cost: {time11-time10}") print(f"test_merge_column_info_from_file time cost: {time12-time11}") print("All tests pass!") diff --git a/python/ppc_model/secure_lgbm/monitor/callback.py b/python/ppc_model/secure_lgbm/monitor/callback.py index fa1247de..cd145668 100644 --- a/python/ppc_model/secure_lgbm/monitor/callback.py +++ b/python/ppc_model/secure_lgbm/monitor/callback.py @@ -42,7 +42,8 @@ def __init__( self.callbacks = set(callbacks) for cb in callbacks: if not isinstance(cb, TrainingCallback): - raise TypeError("callback must be an instance of `TrainingCallback`.") + raise TypeError( + "callback must be an instance of `TrainingCallback`.") msg = ( "feval must be callable object for monitoring. For builtin metrics" @@ -79,5 +80,6 @@ def after_iteration( ) -> bool: model.after_iteration(pred, eval_on_test) model.eval(self.feval) - ret = any(c.after_iteration(model, model.get_epoch()) for c in self.callbacks) + ret = any(c.after_iteration(model, model.get_epoch()) + for c in self.callbacks) return ret diff --git a/python/ppc_model/secure_lgbm/monitor/early_stopping.py b/python/ppc_model/secure_lgbm/monitor/early_stopping.py index de718ee3..c6140680 100644 --- a/python/ppc_model/secure_lgbm/monitor/early_stopping.py +++ b/python/ppc_model/secure_lgbm/monitor/early_stopping.py @@ -111,7 +111,8 @@ def after_iteration( ) -> bool: history = model.get_history() if len(history.keys()) < 1: - raise ValueError("Must have at least 1 validation dataset for early stopping.") + raise ValueError( + "Must have at least 1 validation dataset for early stopping.") metric_name = self.metric_name # The latest score diff --git a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py index c1038bbb..432149a7 100644 --- a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py +++ b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py @@ -25,7 +25,8 @@ def _draw_figure(model: _Model): plt.plot(iterations, values, label=metric) max_index = values.index(max(values)) plt.scatter(max_index + 1, values[max_index], color='green') - plt.text(max_index + 1, values[max_index], f'{values[max_index]:.4f}', fontsize=9, ha='right') + plt.text(max_index + 1, values[max_index], + f'{values[max_index]:.4f}', fontsize=9, ha='right') plt.legend() plt.title('Metrics Over Iterations') diff --git a/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py b/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py index 6429807f..6d995868 100644 --- a/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py +++ b/python/ppc_model/secure_lgbm/monitor/train_callback_unittest.py @@ -49,7 +49,8 @@ def setUp(self): self.test_y_true = np.random.randint(0, 2, 10000) self.y_pred = np.random.rand(10000) self.model = Booster(self.y_true, self.test_y_true) - self.early_stopping = EarlyStopping(rounds=4, metric_name='auc', maximize=True) + self.early_stopping = EarlyStopping( + rounds=4, metric_name='auc', maximize=True) def test_early_stopping(self): stop = False @@ -58,7 +59,8 @@ def test_early_stopping(self): y_pred = np.random.rand(10000) self.model.after_iteration(y_pred) self.model.eval(fevaluation) - stop = self.early_stopping.after_iteration(self.model, self.model.epoch) + stop = self.early_stopping.after_iteration( + self.model, self.model.epoch) print(self.model.epoch, stop) @@ -88,9 +90,11 @@ def setUp(self): self.test_y_true = np.random.randint(0, 2, 10000) self.y_pred = np.random.rand(10000) self.model = Booster(self.y_true, self.test_y_true, 'tmp') - self.early_stopping = EarlyStopping(rounds=4, metric_name='auc', maximize=True) + self.early_stopping = EarlyStopping( + rounds=4, metric_name='auc', maximize=True) self.monitor = EvaluationMonitor(log, period=2) - self.container = CallbackContainer([self.early_stopping, self.monitor], fevaluation) + self.container = CallbackContainer( + [self.early_stopping, self.monitor], fevaluation) def test_callback_container(self): stop = False diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py index 1f481a78..203f71e1 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py @@ -34,5 +34,5 @@ def run(args): # 获取测试集的预测值评估指标 Evaluation(task_info, secure_dataset, test_praba=test_praba) - + ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lgbm/test/test_cipher_packing.py b/python/ppc_model/secure_lgbm/test/test_cipher_packing.py index 1f205a71..1120c6b3 100644 --- a/python/ppc_model/secure_lgbm/test/test_cipher_packing.py +++ b/python/ppc_model/secure_lgbm/test/test_cipher_packing.py @@ -20,39 +20,45 @@ def test_cipher_list(self): ciphers = paillier.encrypt_batch_parallel(data_list) # ciphers = paillier.encrypt_batch(data_list) print("enc:", time.time() - start_time, "seconds") - + start_time = time.time() enc_data_pb = CipherList() - enc_data_pb.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + enc_data_pb.public_key = PaillierCodec.encode_enc_key( + paillier.public_key) for cipher in ciphers: paillier_cipher = ModelCipher() - paillier_cipher.ciphertext, paillier_cipher.exponent = PaillierCodec.encode_cipher(cipher) + paillier_cipher.ciphertext, paillier_cipher.exponent = PaillierCodec.encode_cipher( + cipher) enc_data_pb.cipher_list.append(paillier_cipher) print("pack ciphers:", time.time() - start_time, "seconds") ciphers2 = [] for i in range(100): ciphers2.append(np.array(ciphers[10*i:10*(i+1)]).sum()) - + start_time = time.time() enc_data_pb2 = CipherList() - enc_data_pb2.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + enc_data_pb2.public_key = PaillierCodec.encode_enc_key( + paillier.public_key) for cipher in ciphers2: paillier_cipher2 = ModelCipher() - paillier_cipher2.ciphertext, paillier_cipher2.exponent = PaillierCodec.encode_cipher(cipher, be_secure=False) + paillier_cipher2.ciphertext, paillier_cipher2.exponent = PaillierCodec.encode_cipher( + cipher, be_secure=False) enc_data_pb2.cipher_list.append(paillier_cipher2) print("pack ciphers:", time.time() - start_time, "seconds") ciphers3 = [] for i in range(100): ciphers3.append(np.array(ciphers[10*i:10*(i+1)]).sum()) - + start_time = time.time() enc_data_pb3 = CipherList() - enc_data_pb3.public_key = PaillierCodec.encode_enc_key(paillier.public_key) + enc_data_pb3.public_key = PaillierCodec.encode_enc_key( + paillier.public_key) for cipher in ciphers3: paillier_cipher3 = ModelCipher() - paillier_cipher3.ciphertext, paillier_cipher3.exponent = PaillierCodec.encode_cipher(cipher) + paillier_cipher3.ciphertext, paillier_cipher3.exponent = PaillierCodec.encode_cipher( + cipher) enc_data_pb3.cipher_list.append(paillier_cipher3) print("pack ciphers:", time.time() - start_time, "seconds") diff --git a/python/ppc_model/secure_lgbm/test/test_pack_gh.py b/python/ppc_model/secure_lgbm/test/test_pack_gh.py index dba82280..798de369 100644 --- a/python/ppc_model/secure_lgbm/test/test_pack_gh.py +++ b/python/ppc_model/secure_lgbm/test/test_pack_gh.py @@ -15,16 +15,16 @@ def test_pack_gh(self): result_array = np.array( [429496329600000000000000002000, 200000000000000000001500, - 429496599600000000004294965896, 0, + 429496599600000000004294965896, 0, 4294965616, 429495194200000000000000001235], dtype=object) - + assert np.array_equal(gh_list, result_array) def test_unpack_gh(self): gh_list = np.array( [429496329600000000000000002000, 200000000000000000001500, - 429496599600000000004294965896, 0, + 429496599600000000004294965896, 0, 4294965616, 429495194200000000000000001235], dtype=object) gh_sum_list = np.array([sum(gh_list), sum(gh_list)*2]) diff --git a/python/ppc_model/secure_lgbm/test/test_save_load_model.py b/python/ppc_model/secure_lgbm/test/test_save_load_model.py index a899b805..e42c4369 100644 --- a/python/ppc_model/secure_lgbm/test/test_save_load_model.py +++ b/python/ppc_model/secure_lgbm/test/test_save_load_model.py @@ -56,7 +56,7 @@ def test_save_load_model(self): booster_predict = VerticalBooster(task_info, dataset=None) booster_predict.load_model() - + assert x_split == booster_predict._X_split assert trees == booster_predict._trees @@ -66,15 +66,15 @@ def _build_tree(max_depth, depth=0, weight=0): if depth == max_depth: return weight - best_split_info = BestSplitInfo( - feature=np.random.randint(0,10), - value=np.random.randint(0,4), + best_split_info = BestSplitInfo( + feature=np.random.randint(0, 10), + value=np.random.randint(0, 4), best_gain=np.random.rand(), w_left=np.random.rand(), w_right=np.random.rand(), - agency_idx=np.random.randint(0,2), - agency_feature=np.random.randint(0,5) - ) + agency_idx=np.random.randint(0, 2), + agency_feature=np.random.randint(0, 5) + ) # print(best_split_info) if best_split_info.best_gain > 0.2: diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py index 936a53ac..e992415d 100644 --- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py @@ -84,8 +84,10 @@ def setUp(self): send_retry_times=3, retry_interval_s=0.1 ) - self._active_rpc_client.set_message_handler(self._passive_stub.on_message_received) - self._passive_rpc_client.set_message_handler(self._active_stub.on_message_received) + self._active_rpc_client.set_message_handler( + self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler( + self._active_stub.on_message_received) def test_fit(self): args_a, args_b = mock_args() @@ -96,7 +98,8 @@ def test_fit(self): 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} active_components.mock_logger = MockLogger() task_info_a = SecureLGBMContext(args_a, active_components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=True) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=True) secure_dataset_a = SecureDataset(task_info_a, model_data) booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) print(secure_dataset_a.feature_name) @@ -113,7 +116,8 @@ def test_fit(self): 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} passive_components.mock_logger = MockLogger() task_info_b = SecureLGBMContext(args_b, passive_components) - model_data = SecureDataset.simulate_dataset(data_size, feature_dim, has_label=False) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=False) secure_dataset_b = SecureDataset(task_info_b, model_data) booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) print(secure_dataset_b.feature_name) @@ -128,14 +132,16 @@ def active_worker(): booster_a.save_model() train_praba = booster_a.get_train_praba() test_praba = booster_a.get_test_praba() - Evaluation(task_info_a, secure_dataset_a, train_praba, test_praba) + Evaluation(task_info_a, secure_dataset_a, + train_praba, test_praba) # ModelPlot(booster_a) ResultFileHandling(task_info_a) booster_a.load_model() booster_a.predict() test_praba = booster_a.get_test_praba() task_info_a.algorithm_type = 'PPC_PREDICT' - Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) + Evaluation(task_info_a, secure_dataset_a, + test_praba=test_praba) ResultFileHandling(task_info_a) except Exception as e: task_info_a.components.logger().info(traceback.format_exc()) @@ -146,14 +152,16 @@ def passive_worker(): booster_b.save_model() train_praba = booster_b.get_train_praba() test_praba = booster_b.get_test_praba() - Evaluation(task_info_b, secure_dataset_b, train_praba, test_praba) + Evaluation(task_info_b, secure_dataset_b, + train_praba, test_praba) # ModelPlot(booster_b) ResultFileHandling(task_info_b) booster_b.load_model() booster_b.predict() test_praba = booster_b.get_test_praba() task_info_b.algorithm_type = 'PPC_PREDICT' - Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) + Evaluation(task_info_b, secure_dataset_b, + test_praba=test_praba) ResultFileHandling(task_info_b) except Exception as e: task_info_b.components.logger().info(traceback.format_exc()) diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py index 7cddb6db..ea0806ce 100644 --- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py @@ -134,14 +134,16 @@ def active_worker(): booster_a.save_model() train_praba = booster_a.get_train_praba() test_praba = booster_a.get_test_praba() - Evaluation(task_info_a, secure_dataset_a, train_praba, test_praba) + Evaluation(task_info_a, secure_dataset_a, + train_praba, test_praba) ModelPlot(booster_a) ResultFileHandling(task_info_a) booster_a.load_model() booster_a.predict() test_praba = booster_a.get_test_praba() task_info_a.algorithm_type = 'PPC_PREDICT' - Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) + Evaluation(task_info_a, secure_dataset_a, + test_praba=test_praba) ResultFileHandling(task_info_a) except Exception as e: task_info_a.components.logger().info(traceback.format_exc()) @@ -152,14 +154,16 @@ def passive_worker(): booster_b.save_model() train_praba = booster_b.get_train_praba() test_praba = booster_b.get_test_praba() - Evaluation(task_info_b, secure_dataset_b, train_praba, test_praba) + Evaluation(task_info_b, secure_dataset_b, + train_praba, test_praba) ModelPlot(booster_b) ResultFileHandling(task_info_b) booster_b.load_model() booster_b.predict() test_praba = booster_b.get_test_praba() task_info_b.algorithm_type = 'PPC_PREDICT' - Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) + Evaluation(task_info_b, secure_dataset_b, + test_praba=test_praba) ResultFileHandling(task_info_b) except Exception as e: task_info_b.components.logger().info(traceback.format_exc()) diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py index 524c8266..6451db6b 100644 --- a/python/ppc_model/secure_lgbm/vertical/active_party.py +++ b/python/ppc_model/secure_lgbm/vertical/active_party.py @@ -35,7 +35,8 @@ def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: self.storage_client = ctx.components.storage_client self.feature_importance_store = FeatureImportanceStore( FeatureImportanceStore.DEFAULT_IMPORTANCE_LIST, None, self.log) - self.log.info(f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + self.log.info( + f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') def fit( self, @@ -51,7 +52,8 @@ def fit( for _ in range(self.params.n_estimators): self._tree_id += 1 start_time = time.time() - self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in active party.') + self.log.info( + f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in active party.') # 初始化 feature_select, instance, used_glist, used_hlist = self._init_each_tree() @@ -72,21 +74,27 @@ def fit( # 评估 if not self.params.silent and self.dataset.train_y is not None: - auc = Evaluation.fevaluation(self.dataset.train_y, self._train_praba)['auc'] - self.log.info(f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, auc: {auc}.') + auc = Evaluation.fevaluation( + self.dataset.train_y, self._train_praba)['auc'] + self.log.info( + f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, auc: {auc}.') self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id}, ' f'time_costs: {time.time() - start_time}s.') # 预测验证集 self._test_weights += self._predict_tree( - tree, self._test_X_bin, np.ones(self._test_X_bin.shape[0], dtype=bool), + tree, self._test_X_bin, np.ones( + self._test_X_bin.shape[0], dtype=bool), LGBMMessage.TEST_LEAF_MASK.value) self._test_praba = self._loss_func.sigmoid(self._test_weights) if not self.params.silent and self.dataset.test_y is not None: - auc = Evaluation.fevaluation(self.dataset.test_y, self._test_praba)['auc'] - self.log.info(f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, test auc: {auc}.') + auc = Evaluation.fevaluation( + self.dataset.test_y, self._test_praba)['auc'] + self.log.info( + f'task {self.ctx.task_id}: n_estimators-{self._tree_id}, test auc: {auc}.') if self._iteration_early_stop(): - self.log.info(f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") + self.log.info( + f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") break self._end_active_data() @@ -103,7 +111,8 @@ def predict(self, dataset: SecureDataset = None) -> np.ndarray: dataset.feature_name, self.params.categorical_feature) test_weights = self._init_weight(dataset.test_X.shape[0]) - test_X_bin = self._split_test_data(self.ctx, dataset.test_X, self._X_split) + test_X_bin = self._split_test_data( + self.ctx, dataset.test_X, self._X_split) for tree in self._trees: test_weights += self._predict_tree( @@ -114,7 +123,8 @@ def predict(self, dataset: SecureDataset = None) -> np.ndarray: if dataset.test_y is not None: auc = Evaluation.fevaluation(dataset.test_y, test_praba)['auc'] self.log.info(f'task {self.ctx.task_id}: predict test auc: {auc}.') - self.log.info(f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + self.log.info( + f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') self._end_active_data(is_train=False) @@ -127,9 +137,12 @@ def _init_active_data(self): # 初始化所有参与方的特征 for i in range(1, len(self.ctx.participant_id_list)): - feature_name_bytes = self._receive_byte_data(self.ctx, LGBMMessage.FEATURE_NAME.value, i) - self._all_feature_name.append([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) - self._all_feature_num += len([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + feature_name_bytes = self._receive_byte_data( + self.ctx, LGBMMessage.FEATURE_NAME.value, i) + self._all_feature_name.append( + [s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + self._all_feature_num += len([s.decode('utf-8') + for s in feature_name_bytes.split(b' ') if s]) self.log.info(f'task {self.ctx.task_id}: total feature number:{self._all_feature_num}, ' f'total feature name: {self._all_feature_name}.') @@ -139,18 +152,21 @@ def _init_active_data(self): self.dataset.feature_name, self.params.categorical_feature) # 更新feature_importance中的特征列表 - self.feature_importance_store.set_init(list(itertools.chain(*self._all_feature_name))) + self.feature_importance_store.set_init( + list(itertools.chain(*self._all_feature_name))) # 初始化分桶数据集 feat_bin = FeatureBinning(self.ctx) - self._X_bin, self._X_split = feat_bin.data_binning(self.dataset.train_X) + self._X_bin, self._X_split = feat_bin.data_binning( + self.dataset.train_X) def _init_each_tree(self): if self.callback_container: self.callback_container.before_iteration(self.model) - gradient = self._loss_func.compute_gradient(self.dataset.train_y, self._train_praba) + gradient = self._loss_func.compute_gradient( + self.dataset.train_y, self._train_praba) hessian = self._loss_func.compute_hessian(self._train_praba) feature_select = FeatureSelection.feature_selecting( @@ -171,7 +187,8 @@ def _send_gh_instance_list(self, instance, glist, hlist): start_time = time.time() self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} ' f'encrypt g & h in active party.') - enc_ghlist = self.ctx.phe.encrypt_batch_parallel((gh_list).astype('object')) + enc_ghlist = self.ctx.phe.encrypt_batch_parallel( + (gh_list).astype('object')) self.log.info(f'task {self.ctx.task_id}: Finished n_estimators-{self._tree_id} ' f'encrypt gradient & hessian time_costs: {time.time() - start_time}.') @@ -194,18 +211,22 @@ def _build_tree(self, feature_select, instance, glist, hlist, depth=0, weight=0) if self.params.colsample_bylevel > 0 and self.params.colsample_bylevel < 1: feature_select_level = sorted(np.random.choice( feature_select, size=int(len(feature_select) * self.params.colsample_bylevel), replace=False)) - best_split_info = self._find_best_split(feature_select_level, instance, glist, hlist) + best_split_info = self._find_best_split( + feature_select_level, instance, glist, hlist) else: - best_split_info = self._find_best_split(feature_select, instance, glist, hlist) + best_split_info = self._find_best_split( + feature_select, instance, glist, hlist) if best_split_info.best_gain > 0 and best_split_info.best_gain > self.params.min_split_gain: gain_list = {FeatureImportanceType.GAIN: best_split_info.best_gain, FeatureImportanceType.WEIGHT: 1} - self.feature_importance_store.update_feature_importance(best_split_info.feature, gain_list) - left_mask, right_mask = self._get_leaf_mask(best_split_info, instance) + self.feature_importance_store.update_feature_importance( + best_split_info.feature, gain_list) + left_mask, right_mask = self._get_leaf_mask( + best_split_info, instance) if (abs(best_split_info.w_left) * sum(left_mask) / self.params.lr) < self.params.min_child_weight or \ - (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: + (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: return weight if sum(left_mask) < self.params.min_child_samples or sum(right_mask) < self.params.min_child_samples: return weight @@ -229,9 +250,11 @@ def _predict_tree(self, tree, X_bin, leaf_mask, key_type): if self.ctx.participant_id_list[best_split_info.agency_idx] == \ self.ctx.components.config_data['AGENCY_ID']: if best_split_info.agency_feature in self.params.my_categorical_idx: - left_mask = X_bin[:, best_split_info.agency_feature] == best_split_info.value + left_mask = X_bin[:, + best_split_info.agency_feature] == best_split_info.value else: - left_mask = X_bin[:, best_split_info.agency_feature] <= best_split_info.value + left_mask = X_bin[:, + best_split_info.agency_feature] <= best_split_info.value else: left_mask = np.frombuffer( self._receive_byte_data( @@ -239,8 +262,10 @@ def _predict_tree(self, tree, X_bin, leaf_mask, key_type): f'{key_type}_{best_split_info.tree_id}_{best_split_info.leaf_id}', best_split_info.agency_idx), dtype='bool') right_mask = ~left_mask - left_weight = self._predict_tree(left_subtree, X_bin, leaf_mask * left_mask, key_type) - right_weight = self._predict_tree(right_subtree, X_bin, leaf_mask * right_mask, key_type) + left_weight = self._predict_tree( + left_subtree, X_bin, leaf_mask * left_mask, key_type) + right_weight = self._predict_tree( + right_subtree, X_bin, leaf_mask * right_mask, key_type) return left_weight + right_weight def _find_best_split(self, feature_select, instance, glist, hlist): @@ -248,7 +273,8 @@ def _find_best_split(self, feature_select, instance, glist, hlist): self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} ' f'leaf-{self._leaf_id} in active party.') grad_hist, hess_hist = self._get_gh_hist(instance, glist, hlist) - best_split_info = self._get_best_split_point(feature_select, glist, hlist, grad_hist, hess_hist) + best_split_info = self._get_best_split_point( + feature_select, glist, hlist, grad_hist, hess_hist) # print('grad_hist_sum', [sum(sublist) for sublist in grad_hist]) best_split_info.tree_id = self._tree_id @@ -271,7 +297,8 @@ def _find_best_split(self, feature_select, instance, glist, hlist): return best_split_info def _get_gh_hist(self, instance, glist, hlist): - ghist, hhist = self._calculate_hist(self._X_bin, instance, glist, hlist) + ghist, hhist = self._calculate_hist( + self._X_bin, instance, glist, hlist) for partner_index in range(1, len(self.ctx.participant_id_list)): partner_feature_name = self._all_feature_name[partner_index] @@ -283,7 +310,8 @@ def _get_gh_hist(self, instance, glist, hlist): partner_index, matrix_data=True) for feature_index in range(len(partner_feature_name)): - ghk_hist = np.array(self.ctx.phe.decrypt_batch(gh_hist[feature_index]), dtype='object') + ghk_hist = np.array(self.ctx.phe.decrypt_batch( + gh_hist[feature_index]), dtype='object') gk_hist, hk_hist = self.unpacking_gh(ghk_hist) partner_ghist[feature_index] = gk_hist partner_hhist[feature_index] = hk_hist @@ -374,7 +402,8 @@ def _get_best_split_agency(all_feature_name, feature): def _init_valid_data(self): self._test_weights = self._init_weight(self.dataset.test_X.shape[0]) - self._test_X_bin = self._split_test_data(self.ctx, self.dataset.test_X, self._X_split) + self._test_X_bin = self._split_test_data( + self.ctx, self.dataset.test_X, self._X_split) def _init_early_stop(self): @@ -382,17 +411,20 @@ def _init_early_stop(self): early_stopping_rounds = self.params.early_stopping_rounds if early_stopping_rounds != 0: eval_metric = self.params.eval_metric - early_stopping = EarlyStopping(rounds=early_stopping_rounds, metric_name=eval_metric, save_best=True) + early_stopping = EarlyStopping( + rounds=early_stopping_rounds, metric_name=eval_metric, save_best=True) callbacks.append(early_stopping) verbose_eval = self.params.verbose_eval if verbose_eval != 0: - evaluation_monitor = EvaluationMonitor(logger=self.log, period=verbose_eval) + evaluation_monitor = EvaluationMonitor( + logger=self.log, period=verbose_eval) callbacks.append(evaluation_monitor) callback_container = None if len(callbacks) != 0: - callback_container = CallbackContainer(callbacks=callbacks, feval=Evaluation.fevaluation) + callback_container = CallbackContainer( + callbacks=callbacks, feval=Evaluation.fevaluation) model = Booster(y_true=self.dataset.train_y, test_y_true=self.dataset.test_y, workspace=self.ctx.workspace, job_id=self.ctx.job_id, @@ -419,7 +451,8 @@ def _iteration_early_stop(self): stop = self.callback_container.after_iteration(model=self.model, pred=pred, eval_on_test=eval_on_test) - self.log.info(f"task {self.ctx.task_id}: after iteration {self._tree_id} iterations, stop: {stop}.") + self.log.info( + f"task {self.ctx.task_id}: after iteration {self._tree_id} iterations, stop: {stop}.") iteration_request = IterationRequest() iteration_request.epoch = self._tree_id - 1 diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index 468158cf..c191b6f2 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -51,7 +51,7 @@ def _init_weight(self, n): return np.zeros(n, dtype=float) @staticmethod - def _get_categorical_idx(feature_name, categorical_feature = []): + def _get_categorical_idx(feature_name, categorical_feature=[]): categorical_idx = [] if len(categorical_feature) > 0: for i in categorical_feature: @@ -86,7 +86,7 @@ def _compute_leaf_weight(lr, λ, gl, hl, gr, hr, reg_alpha): @staticmethod def _calulate_weight(lr, λ, g, h, reg_alpha): - + # weight = lr * - g / (h + λ) if (h + λ) != 0 and g > reg_alpha: weight = lr * - (g - reg_alpha) / (h + λ) @@ -117,15 +117,15 @@ def _get_leaf_mask(self, split_info, instance): for partner_index in range(0, len(self.ctx.participant_id_list)): if self.ctx.participant_id_list[partner_index] != self.ctx.components.config_data['AGENCY_ID']: self._send_byte_data( - self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', + self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', left_mask.astype('bool').tobytes(), partner_index) else: left_mask = np.frombuffer( self._receive_byte_data( - self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', + self.ctx, f'{LGBMMessage.INSTANCE_MASK.value}_{self._tree_id}_{self._leaf_id}', split_info.agency_idx), dtype='bool') right_mask = ~left_mask - + return left_mask, right_mask def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=False): @@ -138,14 +138,16 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal receiver=partner_id, task_id=ctx.task_id, key=key_type, - data=PheMessage.packing_2dim_data(ctx.codec, ctx.phe.public_key, enc_data) + data=PheMessage.packing_2dim_data( + ctx.codec, ctx.phe.public_key, enc_data) )) else: self._stub.push(PushRequest( receiver=partner_id, task_id=ctx.task_id, key=key_type, - data=PheMessage.packing_data(ctx.codec, ctx.phe.public_key, enc_data) + data=PheMessage.packing_data( + ctx.codec, ctx.phe.public_key, enc_data) )) log.info( @@ -164,9 +166,11 @@ def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): )) if matrix_data: - public_key, enc_data = PheMessage.unpacking_2dim_data(ctx.codec, byte_data) + public_key, enc_data = PheMessage.unpacking_2dim_data( + ctx.codec, byte_data) else: - public_key, enc_data = PheMessage.unpacking_data(ctx.codec, byte_data) + public_key, enc_data = PheMessage.unpacking_data( + ctx.codec, byte_data) log.info( f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " @@ -213,51 +217,60 @@ def _split_test_data(ctx, test_X, X_split): def save_model(self, file_path=None): log = self.ctx.components.logger() if file_path is not None: - self.ctx.feature_bin_file = os.path.join(file_path, self.ctx.FEATURE_BIN_FILE) - self.ctx.model_data_file = os.path.join(file_path, self.ctx.MODEL_DATA_FILE) - + self.ctx.feature_bin_file = os.path.join( + file_path, self.ctx.FEATURE_BIN_FILE) + self.ctx.model_data_file = os.path.join( + file_path, self.ctx.MODEL_DATA_FILE) + if self._X_split is not None and not os.path.exists(self.ctx.feature_bin_file): - X_split_dict = {k: v for k, v in zip(self.dataset.feature_name, self._X_split)} + X_split_dict = {k: v for k, v in zip( + self.dataset.feature_name, self._X_split)} with open(self.ctx.feature_bin_file, 'w') as f: json.dump(X_split_dict, f) - ResultFileHandling._upload_file(self.ctx.components.storage_client, + ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) - log.info(f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished.") - + log.info( + f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished.") + if not os.path.exists(self.ctx.model_data_file): serial_trees = [self._serial_tree(tree) for tree in self._trees] with open(self.ctx.model_data_file, 'w') as f: json.dump(serial_trees, f) - ResultFileHandling._upload_file(self.ctx.components.storage_client, + ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.model_data_file, self.ctx.remote_model_data_file) - log.info(f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.") - + log.info( + f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.") + def load_model(self, file_path=None): log = self.ctx.components.logger() if file_path is not None: - self.ctx.feature_bin_file = os.path.join(file_path, self.ctx.FEATURE_BIN_FILE) - self.ctx.model_data_file = os.path.join(file_path, self.ctx.MODEL_DATA_FILE) + self.ctx.feature_bin_file = os.path.join( + file_path, self.ctx.FEATURE_BIN_FILE) + self.ctx.model_data_file = os.path.join( + file_path, self.ctx.MODEL_DATA_FILE) if self.ctx.algorithm_type == AlgorithmType.Predict.name: self.ctx.remote_feature_bin_file = os.path.join( self.ctx.lgbm_params.training_job_id, self.ctx.FEATURE_BIN_FILE) self.ctx.remote_model_data_file = os.path.join( self.ctx.lgbm_params.training_job_id, self.ctx.MODEL_DATA_FILE) - - ResultFileHandling._download_file(self.ctx.components.storage_client, + + ResultFileHandling._download_file(self.ctx.components.storage_client, self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) - ResultFileHandling._download_file(self.ctx.components.storage_client, + ResultFileHandling._download_file(self.ctx.components.storage_client, self.ctx.model_data_file, self.ctx.remote_model_data_file) with open(self.ctx.feature_bin_file, 'r') as f: X_split_dict = json.load(f) feature_name = list(X_split_dict.keys()) x_split = list(X_split_dict.values()) - log.info(f"task {self.ctx.task_id}: Load x_split from {self.ctx.feature_bin_file} finished.") + log.info( + f"task {self.ctx.task_id}: Load x_split from {self.ctx.feature_bin_file} finished.") assert len(feature_name) == len(self.dataset.feature_name) with open(self.ctx.model_data_file, 'r') as f: serial_trees = json.load(f) - log.info(f"task {self.ctx.task_id}: Load serial_trees from {self.ctx.model_data_file} finished.") + log.info( + f"task {self.ctx.task_id}: Load serial_trees from {self.ctx.model_data_file} finished.") trees = [self._deserial_tree(tree) for tree in serial_trees] self._X_split = x_split @@ -270,7 +283,8 @@ def _serial_tree(tree): best_split_info, left_tree, right_tree = tree[0] best_split_info_list = [] for field in best_split_info.DESCRIPTOR.fields: - best_split_info_list.append(getattr(best_split_info, field.name)) + best_split_info_list.append( + getattr(best_split_info, field.name)) left_tree = VerticalBooster._serial_tree(left_tree) right_tree = VerticalBooster._serial_tree(right_tree) best_split_info_list.extend([left_tree, right_tree]) @@ -312,7 +326,7 @@ def packing_gh(g_list: np.ndarray, h_list: np.ndarray, expand=1000, mod_length=3 浮点数转整数默乘以 1000(取3位小数) 按照最高数据量100w样本, g/h求和值上限为 1000 * 10**6 = 10**9 基于g/h上限, 负数模运算转正数需要加上 2**32 (4.29*10**9) - + 2. packing g/h负数模运算转为正数后最大值为 2**32-1, 100w样本求和需要预留10**6位 packing g和h时, 对g乘以10**20, 为h预留总计20位长度。 @@ -320,9 +334,10 @@ def packing_gh(g_list: np.ndarray, h_list: np.ndarray, expand=1000, mod_length=3 mod_n = 2 ** mod_length pos_int_glist = ((g_list * expand).astype('int64') + mod_n) % mod_n pos_int_hlist = ((h_list * expand).astype('int64') + mod_n) % mod_n - - gh_list = pos_int_glist.astype('object') * 10**pack_length + pos_int_hlist.astype('object') - + + gh_list = pos_int_glist.astype( + 'object') * 10**pack_length + pos_int_hlist.astype('object') + return gh_list @staticmethod diff --git a/python/ppc_model/secure_lgbm/vertical/passive_party.py b/python/ppc_model/secure_lgbm/vertical/passive_party.py index 2f09f6d1..321f3651 100644 --- a/python/ppc_model/secure_lgbm/vertical/passive_party.py +++ b/python/ppc_model/secure_lgbm/vertical/passive_party.py @@ -17,7 +17,8 @@ def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: super().__init__(ctx, dataset) self.params = ctx.lgbm_params self.log = ctx.components.logger() - self.log.info(f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + self.log.info( + f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') def fit( self, @@ -27,31 +28,37 @@ def fit( self.log.info( f'task {self.ctx.task_id}: Starting the lgbm on the passive party.') self._init_passive_data() - self._test_X_bin = self._split_test_data(self.ctx, self.dataset.test_X, self._X_split) + self._test_X_bin = self._split_test_data( + self.ctx, self.dataset.test_X, self._X_split) for _ in range(self.params.n_estimators): self._tree_id += 1 start_time = time.time() - self.log.info(f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in passive party.') + self.log.info( + f'task {self.ctx.task_id}: Starting n_estimators-{self._tree_id} in passive party.') # 初始化 instance, used_ghlist, public_key = self._receive_gh_instance_list() self.ctx.phe.public_key = public_key - self.log.info(f'task {self.ctx.task_id}: Sampling number: {len(instance)}.') + self.log.info( + f'task {self.ctx.task_id}: Sampling number: {len(instance)}.') # 构建 tree = self._build_tree(instance, used_ghlist) self._trees.append(tree) # 预测 - self._predict_tree(tree, self._X_bin, LGBMMessage.PREDICT_LEAF_MASK.value) + self._predict_tree(tree, self._X_bin, + LGBMMessage.PREDICT_LEAF_MASK.value) self.log.info(f'task {self.ctx.task_id}: Ending n_estimators-{self._tree_id}, ' f'time_costs: {time.time() - start_time}s.') # 预测验证集 - self._predict_tree(tree, self._test_X_bin, LGBMMessage.TEST_LEAF_MASK.value) + self._predict_tree(tree, self._test_X_bin, + LGBMMessage.TEST_LEAF_MASK.value) if self._iteration_early_stop(): - self.log.info(f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") + self.log.info( + f"task {self.ctx.task_id}: lgbm early stop after {self._tree_id} iterations.") break self._end_passive_data() @@ -67,11 +74,13 @@ def predict(self, dataset: SecureDataset = None) -> np.ndarray: self.params.my_categorical_idx = self._get_categorical_idx( dataset.feature_name, self.params.categorical_feature) - test_X_bin = self._split_test_data(self.ctx, dataset.test_X, self._X_split) + test_X_bin = self._split_test_data( + self.ctx, dataset.test_X, self._X_split) [self._predict_tree( tree, test_X_bin, LGBMMessage.VALID_LEAF_MASK.value) for tree in self._trees] - self.log.info(f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + self.log.info( + f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') self._end_passive_data(is_train=False) @@ -116,10 +125,11 @@ def _build_tree(self, instance, ghlist, depth=0, weight=0): best_split_info = self._find_best_split(instance, ghlist) if best_split_info.best_gain > 0 and best_split_info.best_gain > self.params.min_split_gain: - left_mask, right_mask = self._get_leaf_mask(best_split_info, instance) + left_mask, right_mask = self._get_leaf_mask( + best_split_info, instance) if (abs(best_split_info.w_left) * sum(left_mask) / self.params.lr) < self.params.min_child_weight or \ - (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: + (abs(best_split_info.w_right) * sum(right_mask) / self.params.lr) < self.params.min_child_weight: return weight if sum(left_mask) < self.params.min_child_samples or sum(right_mask) < self.params.min_child_samples: return weight @@ -143,9 +153,11 @@ def _predict_tree(self, tree, X_bin, key_type): if self.ctx.participant_id_list[best_split_info.agency_idx] == \ self.ctx.components.config_data['AGENCY_ID']: if best_split_info.agency_feature in self.params.my_categorical_idx: - left_mask = X_bin[:, best_split_info.agency_feature] == best_split_info.value + left_mask = X_bin[:, + best_split_info.agency_feature] == best_split_info.value else: - left_mask = X_bin[:, best_split_info.agency_feature] <= best_split_info.value + left_mask = X_bin[:, + best_split_info.agency_feature] <= best_split_info.value self._send_byte_data( self.ctx, f'{key_type}_{best_split_info.tree_id}_{best_split_info.leaf_id}', diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py index d6a308d0..d3ea8e33 100644 --- a/python/ppc_model/task/task_manager.py +++ b/python/ppc_model/task/task_manager.py @@ -51,16 +51,19 @@ def run_task(self, task_id: str, task_type: ModelTask, args=()): job_id = args[0]['job_id'] with self._rw_lock.gen_wlock(): if task_id in self._tasks: - self.logger.info(f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id][0]}") + self.logger.info( + f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id][0]}") return - self._tasks[task_id] = [TaskStatus.RUNNING.value, datetime.datetime.now(), 0, args[0]['job_id']] + self._tasks[task_id] = [TaskStatus.RUNNING.value, + datetime.datetime.now(), 0, args[0]['job_id']] if job_id in self._jobs: self._jobs[job_id].add(task_id) else: self._jobs[job_id] = {task_id} self.logger.info(LOG_START_FLAG_FORMATTER.format(job_id=job_id)) self.logger.info(f"Run task, job_id: {job_id}, task_id: {task_id}") - self._async_executor.execute(task_id, self._handlers[task_type.value], self._on_task_finish, args) + self._async_executor.execute( + task_id, self._handlers[task_type.value], self._on_task_finish, args) def kill_task(self, job_id: str): """ @@ -152,7 +155,8 @@ def _cleanup_finished_tasks(self): del self._jobs[job_id] self._thread_event_manager.remove_event(task_id) self._stub.cleanup_cache(task_id) - self.logger.info(f"Cleanup task cache, task_id: {task_id}, job_id: {job_id}") + self.logger.info( + f"Cleanup task cache, task_id: {task_id}, job_id: {job_id}") def record_model_job_log(self, job_id): log_file = self._get_log_file_path() diff --git a/python/ppc_model/task/test/task_manager_unittest.py b/python/ppc_model/task/test/task_manager_unittest.py index 64f2a86f..060d0d6c 100644 --- a/python/ppc_model/task/test/task_manager_unittest.py +++ b/python/ppc_model/task/test/task_manager_unittest.py @@ -126,8 +126,10 @@ def test_kill_task(self): 'job_id': '0x123456789', 'key': 'TEST_MESSAGE', } - self._task_manager.run_task("my_long_task", ModelTask.XGB_PREDICTING, (args,)) - self.assertEqual(self._task_manager.status("my_long_task")[0], 'RUNNING') + self._task_manager.run_task( + "my_long_task", ModelTask.XGB_PREDICTING, (args,)) + self.assertEqual(self._task_manager.status( + "my_long_task")[0], 'RUNNING') self._task_manager.kill_task("0x123456789") time.sleep(1) self.assertEqual(self._task_manager.status( diff --git a/python/ppc_model_gateway/ppc_model_gateway_app.py b/python/ppc_model_gateway/ppc_model_gateway_app.py index f64a96c4..ef55c4e6 100644 --- a/python/ppc_model_gateway/ppc_model_gateway_app.py +++ b/python/ppc_model_gateway/ppc_model_gateway_app.py @@ -1,18 +1,16 @@ +from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService +from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService +from ppc_model_gateway import config +from ppc_common.ppc_utils import utils +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +import grpc +from threading import Thread +from concurrent import futures import os # Note: here can't be refactored by autopep import sys sys.path.append("../") -from concurrent import futures -from threading import Thread - -import grpc - -from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc -from ppc_common.ppc_utils import utils -from ppc_model_gateway import config -from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService -from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService log = config.get_logger() @@ -22,7 +20,8 @@ def node_to_partner_serve(): ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), options=config.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(NodeToPartnerService(), ppc_serve) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + NodeToPartnerService(), ppc_serve) address = "[::]:{}".format(rpc_port) ppc_serve.add_insecure_port(address) @@ -40,7 +39,8 @@ def partner_to_node_serve(): if config.CONFIG_DATA['SSL_SWITCH'] == 0: ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), options=config.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(PartnerToNodeService(), ppc_serve) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + PartnerToNodeService(), ppc_serve) address = "[::]:{}".format(rpc_port) ppc_serve.add_insecure_port(address) else: @@ -57,7 +57,8 @@ def partner_to_node_serve(): ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)), options=config.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(PartnerToNodeService(), ppc_serve) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + PartnerToNodeService(), ppc_serve) address = "[::]:{}".format(rpc_port) ppc_serve.add_secure_port(address, server_credentials) diff --git a/python/ppc_model_gateway/test/server.py b/python/ppc_model_gateway/test/server.py index 1df95ecf..03d2a93a 100644 --- a/python/ppc_model_gateway/test/server.py +++ b/python/ppc_model_gateway/test/server.py @@ -21,7 +21,8 @@ def serve(): server = grpc.server(futures.ThreadPoolExecutor( max_workers=max(1, os.cpu_count() - 1)), options=config.grpc_options) - ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(ModelService(), server) + ppc_model_pb2_grpc.add_ModelServiceServicer_to_server( + ModelService(), server) server.add_insecure_port(f'[::]:{port}') server.start() print(f'Start serve successfully at {port}.') From 07ef9b7fa3a1d260702077b82de651013f56aace Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:16:40 +0800 Subject: [PATCH 008/120] update global plot and sync files --- python/ppc_model/common/global_context.py | 4 --- python/ppc_model/common/initializer.py | 7 +++- python/ppc_model/common/model_result.py | 35 +++---------------- python/ppc_model/metrics/evaluation.py | 3 +- python/ppc_model/metrics/model_plot.py | 3 +- .../secure_lgbm/monitor/evaluation_monitor.py | 4 +-- .../secure_lgbm/secure_lgbm_context.py | 19 ++++++++++ .../test_secure_lgbm_performance_training.py | 6 ++-- .../test/test_secure_lgbm_training.py | 12 ++++--- .../secure_lgbm/vertical/active_party.py | 3 +- 10 files changed, 47 insertions(+), 49 deletions(-) diff --git a/python/ppc_model/common/global_context.py b/python/ppc_model/common/global_context.py index d552bef5..186fe3f0 100644 --- a/python/ppc_model/common/global_context.py +++ b/python/ppc_model/common/global_context.py @@ -1,5 +1,4 @@ import os -import threading from ppc_model.common.initializer import Initializer @@ -9,6 +8,3 @@ components = Initializer( log_config_path='logging.conf', config_path=config_path) - -# matplotlib 线程不安全，并行任务绘图增加全局锁 -plot_lock = threading.Lock() diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py index 6dd6258f..829e0182 100644 --- a/python/ppc_model/common/initializer.py +++ b/python/ppc_model/common/initializer.py @@ -1,6 +1,7 @@ import logging import logging.config import os +import threading import yaml @@ -14,7 +15,7 @@ class Initializer: - def __init__(self, log_config_path, config_path): + def __init__(self, log_config_path, config_path, plot_lock=None): self.log_config_path = log_config_path self.config_path = config_path self.config_data = None @@ -27,6 +28,10 @@ def __init__(self, log_config_path, config_path): self.mock_logger = None self.public_key_length = 2048 self.homo_algorithm = 0 + # matplotlib 线程不安全，并行任务绘图增加全局锁 + self.plot_lock = plot_lock + if plot_lock is None: + self.plot_lock = threading.Lock() def init_all(self): self.init_log() diff --git a/python/ppc_model/common/model_result.py b/python/ppc_model/common/model_result.py index 0cb55142..d42f4106 100644 --- a/python/ppc_model/common/model_result.py +++ b/python/ppc_model/common/model_result.py @@ -20,14 +20,14 @@ def __init__(self, ctx: Context) -> None: if ctx.algorithm_type == AlgorithmType.Train.name: self._process_fe_result() - # remove job workspace - # self._remove_workspace() - # Synchronization result file if (len(ctx.result_receiver_id_list) == 1 and ctx.participant_id_list[0] != ctx.result_receiver_id_list[0]) \ or len(ctx.result_receiver_id_list) > 1: self._sync_result_files() + # remove job workspace + self._remove_workspace() + def _process_fe_result(self): if os.path.exists(self.ctx.preprocessing_result_file): column_info_fm = pd.read_csv( @@ -134,33 +134,8 @@ def _remove_workspace(self): f'job {self.ctx.job_id}: {self.ctx.workspace} does not exist.') def _sync_result_files(self): - if self.ctx.algorithm_type == AlgorithmType.Train.name: - self.sync_result_file(self.ctx, self.ctx.metrics_iteration_file, - self.ctx.remote_metrics_iteration_file, 'f1') - self.sync_result_file(self.ctx, self.ctx.feature_importance_file, - self.ctx.remote_feature_importance_file, 'f2') - self.sync_result_file(self.ctx, self.ctx.summary_evaluation_file, - self.ctx.remote_summary_evaluation_file, 'f3') - self.sync_result_file(self.ctx, self.ctx.train_metric_ks_table, - self.ctx.remote_train_metric_ks_table, 'f4') - self.sync_result_file(self.ctx, self.ctx.train_metric_roc_file, - self.ctx.remote_train_metric_roc_file, 'f5') - self.sync_result_file(self.ctx, self.ctx.train_metric_ks_file, - self.ctx.remote_train_metric_ks_file, 'f6') - self.sync_result_file(self.ctx, self.ctx.train_metric_pr_file, - self.ctx.remote_train_metric_pr_file, 'f7') - self.sync_result_file(self.ctx, self.ctx.train_metric_acc_file, - self.ctx.remote_train_metric_acc_file, 'f8') - self.sync_result_file(self.ctx, self.ctx.test_metric_ks_table, - self.ctx.remote_test_metric_ks_table, 'f9') - self.sync_result_file(self.ctx, self.ctx.test_metric_roc_file, - self.ctx.remote_test_metric_roc_file, 'f10') - self.sync_result_file(self.ctx, self.ctx.test_metric_ks_file, - self.ctx.remote_test_metric_ks_file, 'f11') - self.sync_result_file(self.ctx, self.ctx.test_metric_pr_file, - self.ctx.remote_test_metric_pr_file, 'f12') - self.sync_result_file(self.ctx, self.ctx.test_metric_acc_file, - self.ctx.remote_test_metric_acc_file, 'f13') + for key, value in self.ctx.sync_file_list.items(): + self.sync_result_file(self.ctx, value[0], value[1], key) @staticmethod def sync_result_file(ctx, local_file, remote_file, key_file): diff --git a/python/ppc_model/metrics/evaluation.py b/python/ppc_model/metrics/evaluation.py index 2bf12ebb..b2f5dac2 100644 --- a/python/ppc_model/metrics/evaluation.py +++ b/python/ppc_model/metrics/evaluation.py @@ -12,7 +12,6 @@ from sklearn.metrics import roc_curve, auc from ppc_model.common.context import Context -from ppc_model.common.global_context import plot_lock from ppc_model.datasets.dataset import SecureDataset from ppc_model.common.model_result import ResultFileHandling from ppc_model.secure_lgbm.monitor.feature.feature_evaluation_info import EvaluationType @@ -156,7 +155,7 @@ def evaluation_file(self, ctx, data_index: np.ndarray, while retry_num < max_retry: retry_num += 1 try: - with plot_lock: + with ctx.components.plot_lock: ks_value, auc_value = Evaluation.plot_two_class_graph( self, y_true, y_praba) except: diff --git a/python/ppc_model/metrics/model_plot.py b/python/ppc_model/metrics/model_plot.py index bdae534b..dd3632a2 100644 --- a/python/ppc_model/metrics/model_plot.py +++ b/python/ppc_model/metrics/model_plot.py @@ -7,7 +7,6 @@ from networkx.drawing.nx_pydot import graphviz_layout from ppc_model.common.model_result import ResultFileHandling -from ppc_model.common.global_context import plot_lock from ppc_model.secure_lgbm.vertical.booster import VerticalBooster @@ -50,7 +49,7 @@ def plot_tree(self): while retry_num < max_retry: retry_num += 1 try: - with plot_lock: + with self.ctx.components.plot_lock: self._G.tree_plot( figsize=(10, 5), save_filename=tree_file_path) except: diff --git a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py index 432149a7..edda3ba8 100644 --- a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py +++ b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py @@ -7,7 +7,6 @@ import matplotlib.pyplot as plt from ppc_common.ppc_utils.utils import METRICS_OVER_ITERATION_FILE -from ppc_model.common.global_context import plot_lock from ppc_model.secure_lgbm.monitor.callback import TrainingCallback from ppc_model.secure_lgbm.monitor.core import _Model @@ -110,8 +109,7 @@ def after_training(self, model: _Model) -> _Model: while retry_num < max_retry: retry_num += 1 try: - with plot_lock: - _draw_figure(model) + _draw_figure(model) except: self.logger.info(f'scores = {model.get_history()}') self.logger.info(f'path = {model.get_workspace()}') diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py index d6a52db8..5475185b 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -3,6 +3,7 @@ from typing import Any, Dict from sklearn.base import BaseEstimator +from ppc_common.ppc_utils.utils import AlgorithmType from ppc_common.ppc_crypto.phe_factory import PheCipherFactory from ppc_model.common.context import Context from ppc_model.common.initializer import Initializer @@ -231,6 +232,10 @@ def __init__(self, self.lgbm_params.min_split_gain = model_setting.gamma self.lgbm_params.random_state = model_setting.seed + self.sync_file_list = {} + if self.algorithm_type == AlgorithmType.Train.name: + self.set_sync_file() + def set_lgbm_params(self, model_setting: ModelSetting): """设置lgbm参数""" self.lgbm_params.set_model_setting(model_setting) @@ -239,6 +244,20 @@ def get_lgbm_params(self): """获取lgbm参数""" return self.lgbm_params + def set_sync_file(self): + self.sync_file_list['metrics_iteration'] = [self.metrics_iteration_file, self.remote_metrics_iteration_file] + self.sync_file_list['feature_importance'] = [self.feature_importance_file, self.remote_feature_importance_file] + self.sync_file_list['summary_evaluation'] = [self.summary_evaluation_file, self.remote_summary_evaluation_file] + self.sync_file_list['train_ks_table'] = [self.train_metric_ks_table, self.remote_train_metric_ks_table] + self.sync_file_list['train_metric_roc'] = [self.train_metric_roc_file, self.remote_train_metric_roc_file] + self.sync_file_list['train_metric_ks'] = [self.train_metric_ks_file, self.remote_train_metric_ks_file] + self.sync_file_list['train_metric_pr'] = [self.train_metric_pr_file, self.remote_train_metric_pr_file] + self.sync_file_list['train_metric_acc'] = [self.train_metric_acc_file, self.remote_train_metric_acc_file] + self.sync_file_list['test_ks_table'] = [self.test_metric_ks_table, self.remote_test_metric_ks_table] + self.sync_file_list['test_metric_roc'] = [self.test_metric_roc_file, self.remote_test_metric_roc_file] + self.sync_file_list['test_metric_ks'] = [self.test_metric_ks_file, self.remote_test_metric_ks_file] + self.sync_file_list['test_metric_pr'] = [self.test_metric_pr_file, self.remote_test_metric_pr_file] + self.sync_file_list['test_metric_acc'] = [self.test_metric_acc_file, self.remote_test_metric_acc_file] class LGBMMessage(Enum): FEATURE_NAME = "FEATURE_NAME" diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py index e992415d..0c1925ca 100644 --- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py @@ -139,7 +139,8 @@ def active_worker(): booster_a.load_model() booster_a.predict() test_praba = booster_a.get_test_praba() - task_info_a.algorithm_type = 'PPC_PREDICT' + task_info_a.algorithm_type = 'Predict' + task_info_a.sync_file_list = {} Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) ResultFileHandling(task_info_a) @@ -159,7 +160,8 @@ def passive_worker(): booster_b.load_model() booster_b.predict() test_praba = booster_b.get_test_praba() - task_info_b.algorithm_type = 'PPC_PREDICT' + task_info_b.algorithm_type = 'Predict' + task_info_b.sync_file_list = {} Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) ResultFileHandling(task_info_b) diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py index ea0806ce..d8e20c90 100644 --- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py @@ -97,8 +97,9 @@ def setUp(self): def test_fit(self): args_a, args_b = mock_args() + plot_lock = threading.Lock() - active_components = Initializer(log_config_path='', config_path='') + active_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock) active_components.stub = self._active_stub active_components.config_data = { 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} @@ -114,7 +115,8 @@ def test_fit(self): print(secure_dataset_a.test_X.shape) print(secure_dataset_a.test_y.shape) - passive_components = Initializer(log_config_path='', config_path='') + passive_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock) + passive_components.stub = self._passive_stub passive_components.stub = self._passive_stub passive_components.config_data = { 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} @@ -141,7 +143,8 @@ def active_worker(): booster_a.load_model() booster_a.predict() test_praba = booster_a.get_test_praba() - task_info_a.algorithm_type = 'PPC_PREDICT' + task_info_a.algorithm_type = 'Predict' + task_info_a.sync_file_list = {} Evaluation(task_info_a, secure_dataset_a, test_praba=test_praba) ResultFileHandling(task_info_a) @@ -161,7 +164,8 @@ def passive_worker(): booster_b.load_model() booster_b.predict() test_praba = booster_b.get_test_praba() - task_info_b.algorithm_type = 'PPC_PREDICT' + task_info_b.algorithm_type = 'Predict' + task_info_b.sync_file_list = {} Evaluation(task_info_b, secure_dataset_b, test_praba=test_praba) ResultFileHandling(task_info_b) diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py index 6451db6b..4078ec35 100644 --- a/python/ppc_model/secure_lgbm/vertical/active_party.py +++ b/python/ppc_model/secure_lgbm/vertical/active_party.py @@ -475,7 +475,8 @@ def _end_active_data(self, is_train=True): remote_file_path=self.ctx.remote_feature_importance_file, storage_client=self.storage_client) if self.callback_container: - self.callback_container.after_training(self.model) + with self.ctx.components.plot_lock: + self.callback_container.after_training(self.model) for partner_index in range(1, len(self.ctx.participant_id_list)): if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: From 6416da86dda36fc5a251216b0e9b3774650419c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zachma=28=E9=A9=AC=E6=99=A8=29?= Date: Thu, 22 Aug 2024 20:21:50 +0800 Subject: [PATCH 009/120] =?UTF-8?q?[new-feature]=E5=8C=BF=E8=B8=AA?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E8=8A=82=E7=82=B9=E5=92=8CSDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + java/ppc-pir-services/build.gradle | 96 +++++++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 60756 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 + java/ppc-pir-services/gradlew | 240 ++++++++++++++++++ java/ppc-pir-services/gradlew.bat | 91 +++++++ java/ppc-pir-services/script/start.sh | 126 +++++++++ java/ppc-pir-services/script/stop.sh | 45 ++++ java/ppc-pir-services/sdk/build.gradle | 26 ++ .../main/java/com/wedpr/pir/demo/Demo.java | 51 ++++ .../java/com/wedpr/pir/sdk/PirClient.java | 72 ++++++ .../com/wedpr/pir/sdk/crypto/IdHashVec.java | 51 ++++ .../pir/sdk/entity/body/PirDataBody.java | 17 ++ .../pir/sdk/entity/body/PirResultBody.java | 14 + .../pir/sdk/entity/body/ServerResultBody.java | 19 ++ .../pir/sdk/entity/body/ServerResultList.java | 17 ++ .../pir/sdk/entity/body/SimpleEntity.java | 15 ++ .../pir/sdk/entity/param/PirJobParam.java | 65 +++++ .../entity/request/ClientDecryptRequest.java | 17 ++ .../sdk/entity/request/ClientOTRequest.java | 17 ++ .../sdk/entity/request/PirBaseRequest.java | 21 ++ .../sdk/entity/request/ServerJobRequest.java | 19 ++ .../sdk/entity/request/ServerOTRequest.java | 19 ++ .../response/ClientDecryptResponse.java | 14 + .../entity/response/ClientJobResponse.java | 26 ++ .../sdk/entity/response/ClientOTResponse.java | 20 ++ .../entity/response/PirResultResponse.java | 15 ++ .../sdk/entity/response/ServerOTResponse.java | 18 ++ .../com/wedpr/pir/sdk/enums/ParamEnum.java | 35 +++ .../pir/sdk/exception/WedprException.java | 23 ++ .../pir/sdk/exception/WedprStatusEnum.java | 72 ++++++ .../wedpr/pir/sdk/gateway/GatewayClient.java | 80 ++++++ .../com/wedpr/pir/sdk/helper/AESHelper.java | 76 ++++++ .../wedpr/pir/sdk/helper/BasicTypeHelper.java | 15 ++ .../pir/sdk/helper/ConvertTypeHelper.java | 44 ++++ .../pir/sdk/helper/CryptoOperatorHelper.java | 51 ++++ .../pir/sdk/service/ClientDecryptService.java | 69 +++++ .../pir/sdk/service/ClientOTService.java | 94 +++++++ .../wedpr/pir/sdk/service/PirJobService.java | 64 +++++ java/ppc-pir-services/settings.gradle | 11 + .../webank/wedpr/pir/WedprPirApplication.java | 18 ++ .../wedpr/pir/crypto/entity/PirTable.java | 17 ++ .../pir/crypto/service/PirOTService.java | 25 ++ .../pir/crypto/service/PirTableService.java | 21 ++ .../crypto/service/impl/PirOTServiceImpl.java | 139 ++++++++++ .../service/impl/PirTableServiceImpl.java | 67 +++++ .../wedpr/pir/http/common/Constant.java | 10 + .../wedpr/pir/http/config/JacksonConfig.java | 59 +++++ .../pir/http/controller/PirController.java | 54 ++++ .../wedpr/pir/http/service/PirAppService.java | 50 ++++ .../main/resources/application-dev.properties | 9 + .../resources/application-mysql.properties | 6 + .../src/main/resources/application.properties | 7 + .../src/main/resources/log4j2.xml | 111 ++++++++ 54 files changed, 2366 insertions(+) create mode 100644 java/ppc-pir-services/build.gradle create mode 100644 java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar create mode 100644 java/ppc-pir-services/gradle/wrapper/gradle-wrapper.properties create mode 100644 java/ppc-pir-services/gradlew create mode 100644 java/ppc-pir-services/gradlew.bat create mode 100644 java/ppc-pir-services/script/start.sh create mode 100644 java/ppc-pir-services/script/stop.sh create mode 100644 java/ppc-pir-services/sdk/build.gradle create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java create mode 100644 java/ppc-pir-services/settings.gradle create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java create mode 100644 java/ppc-pir-services/src/main/resources/application-dev.properties create mode 100644 java/ppc-pir-services/src/main/resources/application-mysql.properties create mode 100644 java/ppc-pir-services/src/main/resources/application.properties create mode 100644 java/ppc-pir-services/src/main/resources/log4j2.xml diff --git a/.gitignore b/.gitignore index bb7e592f..15029804 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,9 @@ instance/ # dotenv .env +# javaenv +.gradle + # virtualenv venv/ ENV/ diff --git a/java/ppc-pir-services/build.gradle b/java/ppc-pir-services/build.gradle new file mode 100644 index 00000000..ce80f537 --- /dev/null +++ b/java/ppc-pir-services/build.gradle @@ -0,0 +1,96 @@ +plugins { + id 'java' + id 'com.github.sherter.google-java-format' version '0.8' +} + +group 'cn.webank.wedpr' +version '3.0.0' + +repositories { + maven { + url = "https://mirrors.huaweicloud.com/repository/maven/" + } +} + +configurations.all { + exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" + exclude group: "org.slf4j", module: "slf4j-log4j12" +} + +dependencies { + implementation 'org.springframework.boot:spring-boot-starter-web:2.7.12' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.7.11' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.0' + implementation 'com.mysql:mysql-connector-j:8.0.33' + implementation 'org.mybatis:mybatis:3.5.13' + implementation 'org.mybatis.spring.boot:mybatis-spring-boot-starter:2.2.2' + implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.19.0' + implementation 'commons-codec:commons-codec:1.15' + implementation 'com.squareup.okhttp3:okhttp:4.9.1' + implementation project(":ppc-pir-sdk") + + compileOnly 'org.projectlombok:lombok:1.18.24' + annotationProcessor 'org.projectlombok:lombok:1.18.24' +} + +jar { + destinationDir file('dist/app') + archiveName project.name + "-" + project.version + '.jar' + exclude '**/*.xml' + exclude '**/*.toml' + exclude '**/*.properties' + exclude '**/*.yml' + exclude '**/*.crt' + exclude '**/*.key' + exclude '**/*.sql' + exclude '**/*.pem' + + doLast { + copy { + from file('src/main/resources/') + into 'dist/conf' + } + copy { + from configurations.runtimeClasspath + into 'dist/libs' + } + copy { + from file('.').listFiles().findAll { File f -> (f.name.endsWith('.sh') || f.name.endsWith('.env')) } + into 'dist' + } + copy { + from file('script/') + into 'dist' + } + } +} + +sourceSets { + main { + java { + srcDir 'src/main/java' + } + resources { + srcDir 'src/main/resources' + srcDir 'src/main/java' + } + } +} + +test { + useJUnitPlatform() +} + + +googleJavaFormat { + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' +} + +clean { + file("dist/").deleteDir() +} + + + diff --git a/java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar b/java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..249e5832f090a2944b7473328c07c9755baa3196 GIT binary patch literal 60756 zcmb5WV{~QRw(p$^Dz@00IL3?^hro$gg*4VI_WAaTyVM5Foj~O|-84 z$;06hMwt*rV;^8iB z1~&0XWpYJmG?Ts^K9PC62H*`G}xom%S%yq|xvG~FIfP=9*f zZoDRJBm*Y0aId=qJ?7dyb)6)JGWGwe)MHeNSzhi)Ko6J<-m@v=a%NsP537lHe0R* z`If4$aaBA#S=w!2z&m>{lpTy^Lm^mg*3?M&7HFv}7K6x*cukLIGX;bQG|QWdn{%_6 zHnwBKr84#B7Z+AnBXa16a?or^R?+>$4`}{*a_>IhbjvyTtWkHw)|ay)ahWUd-qq$~ zMbh6roVsj;_qnC-R{G+Cy6bApVOinSU-;(DxUEl!i2)1EeQ9`hrfqj(nKI7?Z>Xur zoJz-a`PxkYit1HEbv|jy%~DO^13J-ut986EEG=66S}D3!L}Efp;Bez~7tNq{QsUMm zh9~(HYg1pA*=37C0}n4g&bFbQ+?-h-W}onYeE{q;cIy%eZK9wZjSwGvT+&Cgv z?~{9p(;bY_1+k|wkt_|N!@J~aoY@|U_RGoWX<;p{Nu*D*&_phw`8jYkMNpRTWx1H* z>J-Mi_!`M468#5Aix$$u1M@rJEIOc?k^QBc?T(#=n&*5eS#u*Y)?L8Ha$9wRWdH^3D4|Ps)Y?m0q~SiKiSfEkJ!=^`lJ(%W3o|CZ zSrZL-Xxc{OrmsQD&s~zPfNJOpSZUl%V8tdG%ei}lQkM+z@-4etFPR>GOH9+Y_F<3=~SXln9Kb-o~f>2a6Xz@AS3cn^;c_>lUwlK(n>z?A>NbC z`Ud8^aQy>wy=$)w;JZzA)_*Y$Z5hU=KAG&htLw1Uh00yE!|Nu{EZkch zY9O6x7Y??>!7pUNME*d!=R#s)ghr|R#41l!c?~=3CS8&zr6*aA7n9*)*PWBV2w+&I zpW1-9fr3j{VTcls1>ua}F*bbju_Xq%^v;-W~paSqlf zolj*dt`BBjHI)H9{zrkBo=B%>8}4jeBO~kWqO!~Thi!I1H(in=n^fS%nuL=X2+s!p}HfTU#NBGiwEBF^^tKU zbhhv+0dE-sbK$>J#t-J!B$TMgN@Wh5wTtK2BG}4BGfsZOoRUS#G8Cxv|6EI*n&Xxq zt{&OxCC+BNqz$9b0WM7_PyBJEVObHFh%%`~!@MNZlo*oXDCwDcFwT~Rls!aApL<)^ zbBftGKKBRhB!{?fX@l2_y~%ygNFfF(XJzHh#?`WlSL{1lKT*gJM zs>bd^H9NCxqxn(IOky5k-wALFowQr(gw%|`0991u#9jXQh?4l|l>pd6a&rx|v=fPJ z1mutj{YzpJ_gsClbWFk(G}bSlFi-6@mwoQh-XeD*j@~huW4(8ub%^I|azA)h2t#yG z7e_V_<4jlM3D(I+qX}yEtqj)cpzN*oCdYHa!nm%0t^wHm)EmFP*|FMw!tb@&`G-u~ zK)=Sf6z+BiTAI}}i{*_Ac$ffr*Wrv$F7_0gJkjx;@)XjYSh`RjAgrCck`x!zP>Ifu z&%he4P|S)H*(9oB4uvH67^0}I-_ye_!w)u3v2+EY>eD3#8QR24<;7?*hj8k~rS)~7 zSXs5ww)T(0eHSp$hEIBnW|Iun<_i`}VE0Nc$|-R}wlSIs5pV{g_Dar(Zz<4X3`W?K z6&CAIl4U(Qk-tTcK{|zYF6QG5ArrEB!;5s?tW7 zrE3hcFY&k)+)e{+YOJ0X2uDE_hd2{|m_dC}kgEKqiE9Q^A-+>2UonB+L@v3$9?AYw zVQv?X*pK;X4Ovc6Ev5Gbg{{Eu*7{N3#0@9oMI~}KnObQE#Y{&3mM4`w%wN+xrKYgD zB-ay0Q}m{QI;iY`s1Z^NqIkjrTlf`B)B#MajZ#9u41oRBC1oM1vq0i|F59> z#StM@bHt|#`2)cpl_rWB($DNJ3Lap}QM-+A$3pe}NyP(@+i1>o^fe-oxX#Bt`mcQc zb?pD4W%#ep|3%CHAYnr*^M6Czg>~L4?l16H1OozM{P*en298b+`i4$|w$|4AHbzqB zHpYUsHZET$Z0ztC;U+0*+amF!@PI%^oUIZy{`L{%O^i{Xk}X0&nl)n~tVEpcAJSJ} zverw15zP1P-O8h9nd!&hj$zuwjg?DoxYIw{jWM zW5_pj+wFy8Tsa9g<7Qa21WaV&;ejoYflRKcz?#fSH_)@*QVlN2l4(QNk| z4aPnv&mrS&0|6NHq05XQw$J^RR9T{3SOcMKCXIR1iSf+xJ0E_Wv?jEc*I#ZPzyJN2 zUG0UOXHl+PikM*&g$U@g+KbG-RY>uaIl&DEtw_Q=FYq?etc!;hEC_}UX{eyh%dw2V zTTSlap&5>PY{6I#(6`j-9`D&I#|YPP8a;(sOzgeKDWsLa!i-$frD>zr-oid!Hf&yS z!i^cr&7tN}OOGmX2)`8k?Tn!!4=tz~3hCTq_9CdiV!NIblUDxHh(FJ$zs)B2(t5@u z-`^RA1ShrLCkg0)OhfoM;4Z{&oZmAec$qV@ zGQ(7(!CBk<5;Ar%DLJ0p0!ResC#U<+3i<|vib1?{5gCebG7$F7URKZXuX-2WgF>YJ^i zMhHDBsh9PDU8dlZ$yJKtc6JA#y!y$57%sE>4Nt+wF1lfNIWyA`=hF=9Gj%sRwi@vd z%2eVV3y&dvAgyuJ=eNJR+*080dbO_t@BFJO<@&#yqTK&+xc|FRR;p;KVk@J3$S{p` zGaMj6isho#%m)?pOG^G0mzOAw0z?!AEMsv=0T>WWcE>??WS=fII$t$(^PDPMU(P>o z_*0s^W#|x)%tx8jIgZY~A2yG;US0m2ZOQt6yJqW@XNY_>_R7(Nxb8Ged6BdYW6{prd!|zuX$@Q2o6Ona8zzYC1u!+2!Y$Jc9a;wy+pXt}o6~Bu1oF1c zp7Y|SBTNi@=I(K%A60PMjM#sfH$y*c{xUgeSpi#HB`?|`!Tb&-qJ3;vxS!TIzuTZs-&%#bAkAyw9m4PJgvey zM5?up*b}eDEY+#@tKec)-c(#QF0P?MRlD1+7%Yk*jW;)`f;0a-ZJ6CQA?E%>i2Dt7T9?s|9ZF|KP4;CNWvaVKZ+Qeut;Jith_y{v*Ny6Co6!8MZx;Wgo z=qAi%&S;8J{iyD&>3CLCQdTX*$+Rx1AwA*D_J^0>suTgBMBb=*hefV+Ars#mmr+YsI3#!F@Xc1t4F-gB@6aoyT+5O(qMz*zG<9Qq*f0w^V!03rpr*-WLH}; zfM{xSPJeu6D(%8HU%0GEa%waFHE$G?FH^kMS-&I3)ycx|iv{T6Wx}9$$D&6{%1N_8 z_CLw)_9+O4&u94##vI9b-HHm_95m)fa??q07`DniVjAy`t7;)4NpeyAY(aAk(+T_O z1om+b5K2g_B&b2DCTK<>SE$Ode1DopAi)xaJjU>**AJK3hZrnhEQ9E`2=|HHe<^tv z63e(bn#fMWuz>4erc47}!J>U58%<&N<6AOAewyzNTqi7hJc|X{782&cM zHZYclNbBwU6673=!ClmxMfkC$(CykGR@10F!zN1Se83LR&a~$Ht&>~43OX22mt7tcZUpa;9@q}KDX3O&Ugp6< zLZLfIMO5;pTee1vNyVC$FGxzK2f>0Z-6hM82zKg44nWo|n}$Zk6&;5ry3`(JFEX$q zK&KivAe${e^5ZGc3a9hOt|!UOE&OocpVryE$Y4sPcs4rJ>>Kbi2_subQ9($2VN(3o zb~tEzMsHaBmBtaHAyES+d3A(qURgiskSSwUc9CfJ@99&MKp2sooSYZu+-0t0+L*!I zYagjOlPgx|lep9tiU%ts&McF6b0VE57%E0Ho%2oi?=Ks+5%aj#au^OBwNwhec zta6QAeQI^V!dF1C)>RHAmB`HnxyqWx?td@4sd15zPd*Fc9hpDXP23kbBenBxGeD$k z;%0VBQEJ-C)&dTAw_yW@k0u?IUk*NrkJ)(XEeI z9Y>6Vel>#s_v@=@0<{4A{pl=9cQ&Iah0iD0H`q)7NeCIRz8zx;! z^OO;1+IqoQNak&pV`qKW+K0^Hqp!~gSohcyS)?^P`JNZXw@gc6{A3OLZ?@1Uc^I2v z+X!^R*HCm3{7JPq{8*Tn>5;B|X7n4QQ0Bs79uTU%nbqOJh`nX(BVj!#f;#J+WZxx4 z_yM&1Y`2XzhfqkIMO7tB3raJKQS+H5F%o83bM+hxbQ zeeJm=Dvix$2j|b4?mDacb67v-1^lTp${z=jc1=j~QD>7c*@+1?py>%Kj%Ejp7Y-!? z8iYRUlGVrQPandAaxFfks53@2EC#0)%mrnmGRn&>=$H$S8q|kE_iWko4`^vCS2aWg z#!`RHUGyOt*k?bBYu3*j3u0gB#v(3tsije zgIuNNWNtrOkx@Pzs;A9un+2LX!zw+p3_NX^Sh09HZAf>m8l@O*rXy_82aWT$Q>iyy zqO7Of)D=wcSn!0+467&!Hl))eff=$aneB?R!YykdKW@k^_uR!+Q1tR)+IJb`-6=jj zymzA>Sv4>Z&g&WWu#|~GcP7qP&m*w-S$)7Xr;(duqCTe7p8H3k5>Y-n8438+%^9~K z3r^LIT_K{i7DgEJjIocw_6d0!<;wKT`X;&vv+&msmhAAnIe!OTdybPctzcEzBy88_ zWO{6i4YT%e4^WQZB)KHCvA(0tS zHu_Bg+6Ko%a9~$EjRB90`P(2~6uI@SFibxct{H#o&y40MdiXblu@VFXbhz>Nko;7R z70Ntmm-FePqhb%9gL+7U8@(ch|JfH5Fm)5${8|`Lef>LttM_iww6LW2X61ldBmG0z zax3y)njFe>j*T{i0s8D4=L>X^j0)({R5lMGVS#7(2C9@AxL&C-lZQx~czI7Iv+{%1 z2hEG>RzX4S8x3v#9sgGAnPzptM)g&LB}@%E>fy0vGSa(&q0ch|=ncKjNrK z`jA~jObJhrJ^ri|-)J^HUyeZXz~XkBp$VhcTEcTdc#a2EUOGVX?@mYx#Vy*!qO$Jv zQ4rgOJ~M*o-_Wptam=~krnmG*p^j!JAqoQ%+YsDFW7Cc9M%YPiBOrVcD^RY>m9Pd< zu}#9M?K{+;UIO!D9qOpq9yxUquQRmQNMo0pT`@$pVt=rMvyX)ph(-CCJLvUJy71DI zBk7oc7)-%ngdj~s@76Yse3L^gV0 z2==qfp&Q~L(+%RHP0n}+xH#k(hPRx(!AdBM$JCfJ5*C=K3ts>P?@@SZ_+{U2qFZb>4kZ{Go37{# zSQc+-dq*a-Vy4?taS&{Ht|MLRiS)Sn14JOONyXqPNnpq&2y~)6wEG0oNy>qvod$FF z`9o&?&6uZjhZ4_*5qWVrEfu(>_n2Xi2{@Gz9MZ8!YmjYvIMasE9yVQL10NBrTCczq zcTY1q^PF2l!Eraguf{+PtHV3=2A?Cu&NN&a8V(y;q(^_mFc6)%Yfn&X&~Pq zU1?qCj^LF(EQB1F`8NxNjyV%fde}dEa(Hx=r7$~ts2dzDwyi6ByBAIx$NllB4%K=O z$AHz1<2bTUb>(MCVPpK(E9wlLElo(aSd(Os)^Raum`d(g9Vd_+Bf&V;l=@mM=cC>) z)9b0enb)u_7V!!E_bl>u5nf&Rl|2r=2F3rHMdb7y9E}}F82^$Rf+P8%dKnOeKh1vs zhH^P*4Ydr^$)$h@4KVzxrHyy#cKmWEa9P5DJ|- zG;!Qi35Tp7XNj60=$!S6U#!(${6hyh7d4q=pF{`0t|N^|L^d8pD{O9@tF~W;#Je*P z&ah%W!KOIN;SyAEhAeTafJ4uEL`(RtnovM+cb(O#>xQnk?dzAjG^~4$dFn^<@-Na3 z395;wBnS{t*H;Jef2eE!2}u5Ns{AHj>WYZDgQJt8v%x?9{MXqJsGP|l%OiZqQ1aB! z%E=*Ig`(!tHh>}4_z5IMpg{49UvD*Pp9!pxt_gdAW%sIf3k6CTycOT1McPl=_#0?8 zVjz8Hj*Vy9c5-krd-{BQ{6Xy|P$6LJvMuX$* zA+@I_66_ET5l2&gk9n4$1M3LN8(yEViRx&mtd#LD}AqEs?RW=xKC(OCWH;~>(X6h!uDxXIPH06xh z*`F4cVlbDP`A)-fzf>MuScYsmq&1LUMGaQ3bRm6i7OsJ|%uhTDT zlvZA1M}nz*SalJWNT|`dBm1$xlaA>CCiQ zK`xD-RuEn>-`Z?M{1%@wewf#8?F|(@1e0+T4>nmlSRrNK5f)BJ2H*$q(H>zGD0>eL zQ!tl_Wk)k*e6v^m*{~A;@6+JGeWU-q9>?+L_#UNT%G?4&BnOgvm9@o7l?ov~XL+et zbGT)|G7)KAeqb=wHSPk+J1bdg7N3$vp(ekjI1D9V$G5Cj!=R2w=3*4!z*J-r-cyeb zd(i2KmX!|Lhey!snRw z?#$Gu%S^SQEKt&kep)up#j&9}e+3=JJBS(s>MH+|=R(`8xK{mmndWo_r`-w1#SeRD&YtAJ#GiVI*TkQZ}&aq<+bU2+coU3!jCI6E+Ad_xFW*ghnZ$q zAoF*i&3n1j#?B8x;kjSJD${1jdRB;)R*)Ao!9bd|C7{;iqDo|T&>KSh6*hCD!rwv= zyK#F@2+cv3=|S1Kef(E6Niv8kyLVLX&e=U;{0x{$tDfShqkjUME>f8d(5nzSkY6@! z^-0>DM)wa&%m#UF1F?zR`8Y3X#tA!*7Q$P3lZJ%*KNlrk_uaPkxw~ zxZ1qlE;Zo;nb@!SMazSjM>;34ROOoygo%SF);LL>rRonWwR>bmSd1XD^~sGSu$Gg# zFZ`|yKU0%!v07dz^v(tY%;So(e`o{ZYTX`hm;@b0%8|H>VW`*cr8R%3n|ehw2`(9B+V72`>SY}9^8oh$En80mZK9T4abVG*to;E z1_S6bgDOW?!Oy1LwYy=w3q~KKdbNtyH#d24PFjX)KYMY93{3-mPP-H>@M-_>N~DDu zENh~reh?JBAK=TFN-SfDfT^=+{w4ea2KNWXq2Y<;?(gf(FgVp8Zp-oEjKzB%2Iqj;48GmY3h=bcdYJ}~&4tS`Q1sb=^emaW$IC$|R+r-8V- zf0$gGE(CS_n4s>oicVk)MfvVg#I>iDvf~Ov8bk}sSxluG!6#^Z_zhB&U^`eIi1@j( z^CK$z^stBHtaDDHxn+R;3u+>Lil^}fj?7eaGB z&5nl^STqcaBxI@v>%zG|j))G(rVa4aY=B@^2{TFkW~YP!8!9TG#(-nOf^^X-%m9{Z zCC?iC`G-^RcBSCuk=Z`(FaUUe?hf3{0C>>$?Vs z`2Uud9M+T&KB6o4o9kvdi^Q=Bw!asPdxbe#W-Oaa#_NP(qpyF@bVxv5D5))srkU#m zj_KA+#7sqDn*Ipf!F5Byco4HOSd!Ui$l94|IbW%Ny(s1>f4|Mv^#NfB31N~kya9!k zWCGL-$0ZQztBate^fd>R!hXY_N9ZjYp3V~4_V z#eB)Kjr8yW=+oG)BuNdZG?jaZlw+l_ma8aET(s+-x+=F-t#Qoiuu1i`^x8Sj>b^U} zs^z<()YMFP7CmjUC@M=&lA5W7t&cxTlzJAts*%PBDAPuqcV5o7HEnqjif_7xGt)F% zGx2b4w{@!tE)$p=l3&?Bf#`+!-RLOleeRk3 z7#pF|w@6_sBmn1nECqdunmG^}pr5(ZJQVvAt$6p3H(16~;vO>?sTE`Y+mq5YP&PBo zvq!7#W$Gewy`;%6o^!Dtjz~x)T}Bdk*BS#=EY=ODD&B=V6TD2z^hj1m5^d6s)D*wk zu$z~D7QuZ2b?5`p)E8e2_L38v3WE{V`bVk;6fl#o2`) z99JsWhh?$oVRn@$S#)uK&8DL8>An0&S<%V8hnGD7Z^;Y(%6;^9!7kDQ5bjR_V+~wp zfx4m3z6CWmmZ<8gDGUyg3>t8wgJ5NkkiEm^(sedCicP^&3D%}6LtIUq>mXCAt{9eF zNXL$kGcoUTf_Lhm`t;hD-SE)m=iBnxRU(NyL}f6~1uH)`K!hmYZjLI%H}AmEF5RZt z06$wn63GHnApHXZZJ}s^s)j9(BM6e*7IBK6Bq(!)d~zR#rbxK9NVIlgquoMq z=eGZ9NR!SEqP6=9UQg#@!rtbbSBUM#ynF);zKX+|!Zm}*{H z+j=d?aZ2!?@EL7C~%B?6ouCKLnO$uWn;Y6Xz zX8dSwj732u(o*U3F$F=7xwxm>E-B+SVZH;O-4XPuPkLSt_?S0)lb7EEg)Mglk0#eS z9@jl(OnH4juMxY+*r03VDfPx_IM!Lmc(5hOI;`?d37f>jPP$?9jQQIQU@i4vuG6MagEoJrQ=RD7xt@8E;c zeGV*+Pt+t$@pt!|McETOE$9k=_C!70uhwRS9X#b%ZK z%q(TIUXSS^F0`4Cx?Rk07C6wI4!UVPeI~-fxY6`YH$kABdOuiRtl73MqG|~AzZ@iL&^s?24iS;RK_pdlWkhcF z@Wv-Om(Aealfg)D^adlXh9Nvf~Uf@y;g3Y)i(YP zEXDnb1V}1pJT5ZWyw=1i+0fni9yINurD=EqH^ciOwLUGi)C%Da)tyt=zq2P7pV5-G zR7!oq28-Fgn5pW|nlu^b!S1Z#r7!Wtr{5J5PQ>pd+2P7RSD?>(U7-|Y z7ZQ5lhYIl_IF<9?T9^IPK<(Hp;l5bl5tF9>X-zG14_7PfsA>6<$~A338iYRT{a@r_ zuXBaT=`T5x3=s&3=RYx6NgG>No4?5KFBVjE(swfcivcIpPQFx5l+O;fiGsOrl5teR z_Cm+;PW}O0Dwe_(4Z@XZ)O0W-v2X><&L*<~*q3dg;bQW3g7)a#3KiQP>+qj|qo*Hk z?57>f2?f@`=Fj^nkDKeRkN2d$Z@2eNKpHo}ksj-$`QKb6n?*$^*%Fb3_Kbf1(*W9K>{L$mud2WHJ=j0^=g30Xhg8$#g^?36`p1fm;;1@0Lrx+8t`?vN0ZorM zSW?rhjCE8$C|@p^sXdx z|NOHHg+fL;HIlqyLp~SSdIF`TnSHehNCU9t89yr@)FY<~hu+X`tjg(aSVae$wDG*C zq$nY(Y494R)hD!i1|IIyP*&PD_c2FPgeY)&mX1qujB1VHPG9`yFQpLFVQ0>EKS@Bp zAfP5`C(sWGLI?AC{XEjLKR4FVNw(4+9b?kba95ukgR1H?w<8F7)G+6&(zUhIE5Ef% z=fFkL3QKA~M@h{nzjRq!Y_t!%U66#L8!(2-GgFxkD1=JRRqk=n%G(yHKn%^&$dW>; zSjAcjETMz1%205se$iH_)ZCpfg_LwvnsZQAUCS#^FExp8O4CrJb6>JquNV@qPq~3A zZ<6dOU#6|8+fcgiA#~MDmcpIEaUO02L5#T$HV0$EMD94HT_eXLZ2Zi&(! z&5E>%&|FZ`)CN10tM%tLSPD*~r#--K(H-CZqIOb99_;m|D5wdgJ<1iOJz@h2Zkq?} z%8_KXb&hf=2Wza(Wgc;3v3TN*;HTU*q2?#z&tLn_U0Nt!y>Oo>+2T)He6%XuP;fgn z-G!#h$Y2`9>Jtf}hbVrm6D70|ERzLAU>3zoWhJmjWfgM^))T+2u$~5>HF9jQDkrXR z=IzX36)V75PrFjkQ%TO+iqKGCQ-DDXbaE;C#}!-CoWQx&v*vHfyI>$HNRbpvm<`O( zlx9NBWD6_e&J%Ous4yp~s6)Ghni!I6)0W;9(9$y1wWu`$gs<$9Mcf$L*piP zPR0Av*2%ul`W;?-1_-5Zy0~}?`e@Y5A&0H!^ApyVTT}BiOm4GeFo$_oPlDEyeGBbh z1h3q&Dx~GmUS|3@4V36&$2uO8!Yp&^pD7J5&TN{?xphf*-js1fP?B|`>p_K>lh{ij zP(?H%e}AIP?_i^f&Li=FDSQ`2_NWxL+BB=nQr=$ zHojMlXNGauvvwPU>ZLq!`bX-5F4jBJ&So{kE5+ms9UEYD{66!|k~3vsP+mE}x!>%P za98bAU0!h0&ka4EoiDvBM#CP#dRNdXJcb*(%=<(g+M@<)DZ!@v1V>;54En?igcHR2 zhubQMq}VSOK)onqHfczM7YA@s=9*ow;k;8)&?J3@0JiGcP! zP#00KZ1t)GyZeRJ=f0^gc+58lc4Qh*S7RqPIC6GugG1gXe$LIQMRCo8cHf^qXgAa2 z`}t>u2Cq1CbSEpLr~E=c7~=Qkc9-vLE%(v9N*&HF`(d~(0`iukl5aQ9u4rUvc8%m) zr2GwZN4!s;{SB87lJB;veebPmqE}tSpT>+`t?<457Q9iV$th%i__Z1kOMAswFldD6 ztbOvO337S5o#ZZgN2G99_AVqPv!?Gmt3pzgD+Hp3QPQ`9qJ(g=kjvD+fUSS3upJn! zqoG7acIKEFRX~S}3|{EWT$kdz#zrDlJU(rPkxjws_iyLKU8+v|*oS_W*-guAb&Pj1 z35Z`3z<&Jb@2Mwz=KXucNYdY#SNO$tcVFr9KdKm|%^e-TXzs6M`PBper%ajkrIyUe zp$vVxVs9*>Vp4_1NC~Zg)WOCPmOxI1V34QlG4!aSFOH{QqSVq1^1)- z0P!Z?tT&E-ll(pwf0?=F=yOzik=@nh1Clxr9}Vij89z)ePDSCYAqw?lVI?v?+&*zH z)p$CScFI8rrwId~`}9YWPFu0cW1Sf@vRELs&cbntRU6QfPK-SO*mqu|u~}8AJ!Q$z znzu}50O=YbjwKCuSVBs6&CZR#0FTu)3{}qJJYX(>QPr4$RqWiwX3NT~;>cLn*_&1H zaKpIW)JVJ>b{uo2oq>oQt3y=zJjb%fU@wLqM{SyaC6x2snMx-}ivfU<1- znu1Lh;i$3Tf$Kh5Uk))G!D1UhE8pvx&nO~w^fG)BC&L!_hQk%^p`Kp@F{cz>80W&T ziOK=Sq3fdRu*V0=S53rcIfWFazI}Twj63CG(jOB;$*b`*#B9uEnBM`hDk*EwSRdwP8?5T?xGUKs=5N83XsR*)a4|ijz|c{4tIU+4j^A5C<#5 z*$c_d=5ml~%pGxw#?*q9N7aRwPux5EyqHVkdJO=5J>84!X6P>DS8PTTz>7C#FO?k#edkntG+fJk8ZMn?pmJSO@`x-QHq;7^h6GEXLXo1TCNhH z8ZDH{*NLAjo3WM`xeb=X{((uv3H(8&r8fJJg_uSs_%hOH%JDD?hu*2NvWGYD+j)&` zz#_1%O1wF^o5ryt?O0n;`lHbzp0wQ?rcbW(F1+h7_EZZ9{>rePvLAPVZ_R|n@;b$;UchU=0j<6k8G9QuQf@76oiE*4 zXOLQ&n3$NR#p4<5NJMVC*S);5x2)eRbaAM%VxWu9ohlT;pGEk7;002enCbQ>2r-us z3#bpXP9g|mE`65VrN`+3mC)M(eMj~~eOf)do<@l+fMiTR)XO}422*1SL{wyY(%oMpBgJagtiDf zz>O6(m;};>Hi=t8o{DVC@YigqS(Qh+ix3Rwa9aliH}a}IlOCW1@?%h_bRbq-W{KHF z%Vo?-j@{Xi@=~Lz5uZP27==UGE15|g^0gzD|3x)SCEXrx`*MP^FDLl%pOi~~Il;dc z^hrwp9sYeT7iZ)-ajKy@{a`kr0-5*_!XfBpXwEcFGJ;%kV$0Nx;apKrur zJN2J~CAv{Zjj%FolyurtW8RaFmpn&zKJWL>(0;;+q(%(Hx!GMW4AcfP0YJ*Vz!F4g z!ZhMyj$BdXL@MlF%KeInmPCt~9&A!;cRw)W!Hi@0DY(GD_f?jeV{=s=cJ6e}JktJw zQORnxxj3mBxfrH=x{`_^Z1ddDh}L#V7i}$njUFRVwOX?qOTKjfPMBO4y(WiU<)epb zvB9L=%jW#*SL|Nd_G?E*_h1^M-$PG6Pc_&QqF0O-FIOpa4)PAEPsyvB)GKasmBoEt z?_Q2~QCYGH+hW31x-B=@5_AN870vY#KB~3a*&{I=f);3Kv7q4Q7s)0)gVYx2#Iz9g(F2;=+Iy4 z6KI^8GJ6D@%tpS^8boU}zpi=+(5GfIR)35PzrbuXeL1Y1N%JK7PG|^2k3qIqHfX;G zQ}~JZ-UWx|60P5?d1e;AHx!_;#PG%d=^X(AR%i`l0jSpYOpXoKFW~7ip7|xvN;2^? zsYC9fanpO7rO=V7+KXqVc;Q5z%Bj})xHVrgoR04sA2 zl~DAwv=!(()DvH*=lyhIlU^hBkA0$e*7&fJpB0|oB7)rqGK#5##2T`@_I^|O2x4GO z;xh6ROcV<9>?e0)MI(y++$-ksV;G;Xe`lh76T#Htuia+(UrIXrf9?

L(tZ$0BqX1>24?V$S+&kLZ`AodQ4_)P#Q3*4xg8}lMV-FLwC*cN$< zt65Rf%7z41u^i=P*qO8>JqXPrinQFapR7qHAtp~&RZ85$>ob|Js;GS^y;S{XnGiBc zGa4IGvDl?x%gY`vNhv8wgZnP#UYI-w*^4YCZnxkF85@ldepk$&$#3EAhrJY0U)lR{F6sM3SONV^+$;Zx8BD&Eku3K zKNLZyBni3)pGzU0;n(X@1fX8wYGKYMpLmCu{N5-}epPDxClPFK#A@02WM3!myN%bkF z|GJ4GZ}3sL{3{qXemy+#Uk{4>Kf8v11;f8I&c76+B&AQ8udd<8gU7+BeWC`akUU~U zgXoxie>MS@rBoyY8O8Tc&8id!w+_ooxcr!1?#rc$-|SBBtH6S?)1e#P#S?jFZ8u-Bs&k`yLqW|{j+%c#A4AQ>+tj$Y z^CZajspu$F%73E68Lw5q7IVREED9r1Ijsg#@DzH>wKseye>hjsk^{n0g?3+gs@7`i zHx+-!sjLx^fS;fY!ERBU+Q zVJ!e0hJH%P)z!y%1^ZyG0>PN@5W~SV%f>}c?$H8r;Sy-ui>aruVTY=bHe}$e zi&Q4&XK!qT7-XjCrDaufT@>ieQ&4G(SShUob0Q>Gznep9fR783jGuUynAqc6$pYX; z7*O@@JW>O6lKIk0G00xsm|=*UVTQBB`u1f=6wGAj%nHK_;Aqmfa!eAykDmi-@u%6~ z;*c!pS1@V8r@IX9j&rW&d*}wpNs96O2Ute>%yt{yv>k!6zfT6pru{F1M3P z2WN1JDYqoTB#(`kE{H676QOoX`cnqHl1Yaru)>8Ky~VU{)r#{&s86Vz5X)v15ULHA zAZDb{99+s~qI6;-dQ5DBjHJP@GYTwn;Dv&9kE<0R!d z8tf1oq$kO`_sV(NHOSbMwr=To4r^X$`sBW4$gWUov|WY?xccQJN}1DOL|GEaD_!@& z15p?Pj+>7d`@LvNIu9*^hPN)pwcv|akvYYq)ks%`G>!+!pW{-iXPZsRp8 z35LR;DhseQKWYSD`%gO&k$Dj6_6q#vjWA}rZcWtQr=Xn*)kJ9kacA=esi*I<)1>w^ zO_+E>QvjP)qiSZg9M|GNeLtO2D7xT6vsj`88sd!94j^AqxFLi}@w9!Y*?nwWARE0P znuI_7A-saQ+%?MFA$gttMV-NAR^#tjl_e{R$N8t2NbOlX373>e7Ox=l=;y#;M7asp zRCz*CLnrm$esvSb5{T<$6CjY zmZ(i{Rs_<#pWW>(HPaaYj`%YqBra=Ey3R21O7vUbzOkJJO?V`4-D*u4$Me0Bx$K(lYo`JO}gnC zx`V}a7m-hLU9Xvb@K2ymioF)vj12<*^oAqRuG_4u%(ah?+go%$kOpfb`T96P+L$4> zQ#S+sA%VbH&mD1k5Ak7^^dZoC>`1L%i>ZXmooA!%GI)b+$D&ziKrb)a=-ds9xk#~& z7)3iem6I|r5+ZrTRe_W861x8JpD`DDIYZNm{$baw+$)X^Jtjnl0xlBgdnNY}x%5za zkQ8E6T<^$sKBPtL4(1zi_Rd(tVth*3Xs!ulflX+70?gb&jRTnI8l+*Aj9{|d%qLZ+ z>~V9Z;)`8-lds*Zgs~z1?Fg?Po7|FDl(Ce<*c^2=lFQ~ahwh6rqSjtM5+$GT>3WZW zj;u~w9xwAhOc<kF}~`CJ68 z?(S5vNJa;kriPlim33{N5`C{9?NWhzsna_~^|K2k4xz1`xcui*LXL-1#Y}Hi9`Oo!zQ>x-kgAX4LrPz63uZ+?uG*84@PKq-KgQlMNRwz=6Yes) zY}>YN+qP}nwr$(CZQFjUOI=-6J$2^XGvC~EZ+vrqWaOXB$k?%Suf5k=4>AveC1aJ! ziaW4IS%F$_Babi)kA8Y&u4F7E%99OPtm=vzw$$ zEz#9rvn`Iot_z-r3MtV>k)YvErZ<^Oa${`2>MYYODSr6?QZu+be-~MBjwPGdMvGd!b!elsdi4% z`37W*8+OGulab8YM?`KjJ8e+jM(tqLKSS@=jimq3)Ea2EB%88L8CaM+aG7;27b?5` z4zuUWBr)f)k2o&xg{iZ$IQkJ+SK>lpq4GEacu~eOW4yNFLU!Kgc{w4&D$4ecm0f}~ zTTzquRW@`f0}|IILl`!1P+;69g^upiPA6F{)U8)muWHzexRenBU$E^9X-uIY2%&1w z_=#5*(nmxJ9zF%styBwivi)?#KMG96-H@hD-H_&EZiRNsfk7mjBq{L%!E;Sqn!mVX*}kXhwH6eh;b42eD!*~upVG@ z#smUqz$ICm!Y8wY53gJeS|Iuard0=;k5i5Z_hSIs6tr)R4n*r*rE`>38Pw&lkv{_r!jNN=;#?WbMj|l>cU(9trCq; z%nN~r^y7!kH^GPOf3R}?dDhO=v^3BeP5hF|%4GNQYBSwz;x({21i4OQY->1G=KFyu z&6d`f2tT9Yl_Z8YACZaJ#v#-(gcyeqXMhYGXb=t>)M@fFa8tHp2x;ODX=Ap@a5I=U z0G80^$N0G4=U(>W%mrrThl0DjyQ-_I>+1Tdd_AuB3qpYAqY54upwa3}owa|x5iQ^1 zEf|iTZxKNGRpI>34EwkIQ2zHDEZ=(J@lRaOH>F|2Z%V_t56Km$PUYu^xA5#5Uj4I4RGqHD56xT%H{+P8Ag>e_3pN$4m8n>i%OyJFPNWaEnJ4McUZPa1QmOh?t8~n& z&RulPCors8wUaqMHECG=IhB(-tU2XvHP6#NrLVyKG%Ee*mQ5Ps%wW?mcnriTVRc4J`2YVM>$ixSF2Xi+Wn(RUZnV?mJ?GRdw%lhZ+t&3s7g!~g{%m&i<6 z5{ib-<==DYG93I(yhyv4jp*y3#*WNuDUf6`vTM%c&hiayf(%=x@4$kJ!W4MtYcE#1 zHM?3xw63;L%x3drtd?jot!8u3qeqctceX3m;tWetK+>~q7Be$h>n6riK(5@ujLgRS zvOym)k+VAtyV^mF)$29Y`nw&ijdg~jYpkx%*^ z8dz`C*g=I?;clyi5|!27e2AuSa$&%UyR(J3W!A=ZgHF9OuKA34I-1U~pyD!KuRkjA zbkN!?MfQOeN>DUPBxoy5IX}@vw`EEB->q!)8fRl_mqUVuRu|C@KD-;yl=yKc=ZT0% zB$fMwcC|HE*0f8+PVlWHi>M`zfsA(NQFET?LrM^pPcw`cK+Mo0%8*x8@65=CS_^$cG{GZQ#xv($7J z??R$P)nPLodI;P!IC3eEYEHh7TV@opr#*)6A-;EU2XuogHvC;;k1aI8asq7ovoP!* z?x%UoPrZjj<&&aWpsbr>J$Er-7!E(BmOyEv!-mbGQGeJm-U2J>74>o5x`1l;)+P&~ z>}f^=Rx(ZQ2bm+YE0u=ZYrAV@apyt=v1wb?R@`i_g64YyAwcOUl=C!i>=Lzb$`tjv zOO-P#A+)t-JbbotGMT}arNhJmmGl-lyUpMn=2UacVZxmiG!s!6H39@~&uVokS zG=5qWhfW-WOI9g4!R$n7!|ViL!|v3G?GN6HR0Pt_L5*>D#FEj5wM1DScz4Jv@Sxnl zB@MPPmdI{(2D?;*wd>3#tjAirmUnQoZrVv`xM3hARuJksF(Q)wd4P$88fGYOT1p6U z`AHSN!`St}}UMBT9o7i|G`r$ zrB=s$qV3d6$W9@?L!pl0lf%)xs%1ko^=QY$ty-57=55PvP(^6E7cc zGJ*>m2=;fOj?F~yBf@K@9qwX0hA803Xw+b0m}+#a(>RyR8}*Y<4b+kpp|OS+!whP( zH`v{%s>jsQI9rd$*vm)EkwOm#W_-rLTHcZRek)>AtF+~<(did)*oR1|&~1|e36d-d zgtm5cv1O0oqgWC%Et@P4Vhm}Ndl(Y#C^MD03g#PH-TFy+7!Osv1z^UWS9@%JhswEq~6kSr2DITo59+; ze=ZC}i2Q?CJ~Iyu?vn|=9iKV>4j8KbxhE4&!@SQ^dVa-gK@YfS9xT(0kpW*EDjYUkoj! zE49{7H&E}k%5(>sM4uGY)Q*&3>{aitqdNnRJkbOmD5Mp5rv-hxzOn80QsG=HJ_atI-EaP69cacR)Uvh{G5dTpYG7d zbtmRMq@Sexey)||UpnZ?;g_KMZq4IDCy5}@u!5&B^-=6yyY{}e4Hh3ee!ZWtL*s?G zxG(A!<9o!CL+q?u_utltPMk+hn?N2@?}xU0KlYg?Jco{Yf@|mSGC<(Zj^yHCvhmyx z?OxOYoxbptDK()tsJ42VzXdINAMWL$0Gcw?G(g8TMB)Khw_|v9`_ql#pRd2i*?CZl z7k1b!jQB=9-V@h%;Cnl7EKi;Y^&NhU0mWEcj8B|3L30Ku#-9389Q+(Yet0r$F=+3p z6AKOMAIi|OHyzlHZtOm73}|ntKtFaXF2Fy|M!gOh^L4^62kGUoWS1i{9gsds_GWBc zLw|TaLP64z3z9?=R2|T6Xh2W4_F*$cq>MtXMOy&=IPIJ`;!Tw?PqvI2b*U1)25^<2 zU_ZPoxg_V0tngA0J+mm?3;OYw{i2Zb4x}NedZug!>EoN3DC{1i)Z{Z4m*(y{ov2%- zk(w>+scOO}MN!exSc`TN)!B=NUX`zThWO~M*ohqq;J2hx9h9}|s#?@eR!=F{QTrq~ zTcY|>azkCe$|Q0XFUdpFT=lTcyW##i;-e{}ORB4D?t@SfqGo_cS z->?^rh$<&n9DL!CF+h?LMZRi)qju!meugvxX*&jfD!^1XB3?E?HnwHP8$;uX{Rvp# zh|)hM>XDv$ZGg=$1{+_bA~u-vXqlw6NH=nkpyWE0u}LQjF-3NhATL@9rRxMnpO%f7 z)EhZf{PF|mKIMFxnC?*78(}{Y)}iztV12}_OXffJ;ta!fcFIVjdchyHxH=t%ci`Xd zX2AUB?%?poD6Zv*&BA!6c5S#|xn~DK01#XvjT!w!;&`lDXSJT4_j$}!qSPrb37vc{ z9^NfC%QvPu@vlxaZ;mIbn-VHA6miwi8qJ~V;pTZkKqqOii<1Cs}0i?uUIss;hM4dKq^1O35y?Yp=l4i zf{M!@QHH~rJ&X~8uATV><23zZUbs-J^3}$IvV_ANLS08>k`Td7aU_S1sLsfi*C-m1 z-e#S%UGs4E!;CeBT@9}aaI)qR-6NU@kvS#0r`g&UWg?fC7|b^_HyCE!8}nyh^~o@< zpm7PDFs9yxp+byMS(JWm$NeL?DNrMCNE!I^ko-*csB+dsf4GAq{=6sfyf4wb>?v1v zmb`F*bN1KUx-`ra1+TJ37bXNP%`-Fd`vVQFTwWpX@;s(%nDQa#oWhgk#mYlY*!d>( zE&!|ySF!mIyfING+#%RDY3IBH_fW$}6~1%!G`suHub1kP@&DoAd5~7J55;5_noPI6eLf{t;@9Kf<{aO0`1WNKd?<)C-|?C?)3s z>wEq@8=I$Wc~Mt$o;g++5qR+(6wt9GI~pyrDJ%c?gPZe)owvy^J2S=+M^ z&WhIE`g;;J^xQLVeCtf7b%Dg#Z2gq9hp_%g)-%_`y*zb; zn9`f`mUPN-Ts&fFo(aNTsXPA|J!TJ{0hZp0^;MYHLOcD=r_~~^ymS8KLCSeU3;^QzJNqS z5{5rEAv#l(X?bvwxpU;2%pQftF`YFgrD1jt2^~Mt^~G>T*}A$yZc@(k9orlCGv&|1 zWWvVgiJsCAtamuAYT~nzs?TQFt<1LSEx!@e0~@yd6$b5!Zm(FpBl;(Cn>2vF?k zOm#TTjFwd2D-CyA!mqR^?#Uwm{NBemP>(pHmM}9;;8`c&+_o3#E5m)JzfwN?(f-a4 zyd%xZc^oQx3XT?vcCqCX&Qrk~nu;fxs@JUoyVoi5fqpi&bUhQ2y!Ok2pzsFR(M(|U zw3E+kH_zmTRQ9dUMZWRE%Zakiwc+lgv7Z%|YO9YxAy`y28`Aw;WU6HXBgU7fl@dnt z-fFBV)}H-gqP!1;V@Je$WcbYre|dRdp{xt!7sL3Eoa%IA`5CAA%;Wq8PktwPdULo! z8!sB}Qt8#jH9Sh}QiUtEPZ6H0b*7qEKGJ%ITZ|vH)5Q^2m<7o3#Z>AKc%z7_u`rXA zqrCy{-{8;9>dfllLu$^M5L z-hXs))h*qz%~ActwkIA(qOVBZl2v4lwbM>9l70Y`+T*elINFqt#>OaVWoja8RMsep z6Or3f=oBnA3vDbn*+HNZP?8LsH2MY)x%c13@(XfuGR}R?Nu<|07{$+Lc3$Uv^I!MQ z>6qWgd-=aG2Y^24g4{Bw9ueOR)(9h`scImD=86dD+MnSN4$6 z^U*o_mE-6Rk~Dp!ANp#5RE9n*LG(Vg`1)g6!(XtDzsov$Dvz|Gv1WU68J$CkshQhS zCrc|cdkW~UK}5NeaWj^F4MSgFM+@fJd{|LLM)}_O<{rj z+?*Lm?owq?IzC%U%9EBga~h-cJbIu=#C}XuWN>OLrc%M@Gu~kFEYUi4EC6l#PR2JS zQUkGKrrS#6H7}2l0F@S11DP`@pih0WRkRJl#F;u{c&ZC{^$Z+_*lB)r)-bPgRFE;* zl)@hK4`tEP=P=il02x7-C7p%l=B`vkYjw?YhdJU9!P!jcmY$OtC^12w?vy3<<=tlY zUwHJ_0lgWN9vf>1%WACBD{UT)1qHQSE2%z|JHvP{#INr13jM}oYv_5#xsnv9`)UAO zuwgyV4YZ;O)eSc3(mka6=aRohi!HH@I#xq7kng?Acdg7S4vDJb6cI5fw?2z%3yR+| zU5v@Hm}vy;${cBp&@D=HQ9j7NcFaOYL zj-wV=eYF{|XTkFNM2uz&T8uH~;)^Zo!=KP)EVyH6s9l1~4m}N%XzPpduPg|h-&lL` zAXspR0YMOKd2yO)eMFFJ4?sQ&!`dF&!|niH*!^*Ml##o0M(0*uK9&yzekFi$+mP9s z>W9d%Jb)PtVi&-Ha!o~Iyh@KRuKpQ@)I~L*d`{O8!kRObjO7=n+Gp36fe!66neh+7 zW*l^0tTKjLLzr`x4`_8&on?mjW-PzheTNox8Hg7Nt@*SbE-%kP2hWYmHu#Fn@Q^J(SsPUz*|EgOoZ6byg3ew88UGdZ>9B2Tq=jF72ZaR=4u%1A6Vm{O#?@dD!(#tmR;eP(Fu z{$0O%=Vmua7=Gjr8nY%>ul?w=FJ76O2js&17W_iq2*tb!i{pt#`qZB#im9Rl>?t?0c zicIC}et_4d+CpVPx)i4~$u6N-QX3H77ez z?ZdvXifFk|*F8~L(W$OWM~r`pSk5}#F?j_5u$Obu9lDWIknO^AGu+Blk7!9Sb;NjS zncZA?qtASdNtzQ>z7N871IsPAk^CC?iIL}+{K|F@BuG2>qQ;_RUYV#>hHO(HUPpk@ z(bn~4|F_jiZi}Sad;_7`#4}EmD<1EiIxa48QjUuR?rC}^HRocq`OQPM@aHVKP9E#q zy%6bmHygCpIddPjE}q_DPC`VH_2m;Eey&ZH)E6xGeStOK7H)#+9y!%-Hm|QF6w#A( zIC0Yw%9j$s-#odxG~C*^MZ?M<+&WJ+@?B_QPUyTg9DJGtQN#NIC&-XddRsf3n^AL6 zT@P|H;PvN;ZpL0iv$bRb7|J{0o!Hq+S>_NrH4@coZtBJu#g8#CbR7|#?6uxi8d+$g z87apN>EciJZ`%Zv2**_uiET9Vk{pny&My;+WfGDw4EVL#B!Wiw&M|A8f1A@ z(yFQS6jfbH{b8Z-S7D2?Ixl`j0{+ZnpT=;KzVMLW{B$`N?Gw^Fl0H6lT61%T2AU**!sX0u?|I(yoy&Xveg7XBL&+>n6jd1##6d>TxE*Vj=8lWiG$4=u{1UbAa5QD>5_ z;Te^42v7K6Mmu4IWT6Rnm>oxrl~b<~^e3vbj-GCdHLIB_>59}Ya+~OF68NiH=?}2o zP(X7EN=quQn&)fK>M&kqF|<_*H`}c zk=+x)GU>{Af#vx&s?`UKUsz})g^Pc&?Ka@t5$n$bqf6{r1>#mWx6Ep>9|A}VmWRnowVo`OyCr^fHsf# zQjQ3Ttp7y#iQY8l`zEUW)(@gGQdt(~rkxlkefskT(t%@i8=|p1Y9Dc5bc+z#n$s13 zGJk|V0+&Ekh(F};PJzQKKo+FG@KV8a<$gmNSD;7rd_nRdc%?9)p!|B-@P~kxQG}~B zi|{0}@}zKC(rlFUYp*dO1RuvPC^DQOkX4<+EwvBAC{IZQdYxoq1Za!MW7%p7gGr=j zzWnAq%)^O2$eItftC#TTSArUyL$U54-O7e|)4_7%Q^2tZ^0-d&3J1}qCzR4dWX!)4 zzIEKjgnYgMus^>6uw4Jm8ga6>GBtMjpNRJ6CP~W=37~||gMo_p@GA@#-3)+cVYnU> zE5=Y4kzl+EbEh%dhQokB{gqNDqx%5*qBusWV%!iprn$S!;oN_6E3?0+umADVs4ako z?P+t?m?};gev9JXQ#Q&KBpzkHPde_CGu-y z<{}RRAx=xlv#mVi+Ibrgx~ujW$h{?zPfhz)Kp7kmYS&_|97b&H&1;J-mzrBWAvY} zh8-I8hl_RK2+nnf&}!W0P+>5?#?7>npshe<1~&l_xqKd0_>dl_^RMRq@-Myz&|TKZBj1=Q()) zF{dBjv5)h=&Z)Aevx}+i|7=R9rG^Di!sa)sZCl&ctX4&LScQ-kMncgO(9o6W6)yd< z@Rk!vkja*X_N3H=BavGoR0@u0<}m-7|2v!0+2h~S2Q&a=lTH91OJsvms2MT~ zY=c@LO5i`mLpBd(vh|)I&^A3TQLtr>w=zoyzTd=^f@TPu&+*2MtqE$Avf>l>}V|3-8Fp2hzo3y<)hr_|NO(&oSD z!vEjTWBxbKTiShVl-U{n*B3#)3a8$`{~Pk}J@elZ=>Pqp|MQ}jrGv7KrNcjW%TN_< zZz8kG{#}XoeWf7qY?D)L)8?Q-b@Na&>i=)(@uNo zr;cH98T3$Iau8Hn*@vXi{A@YehxDE2zX~o+RY`)6-X{8~hMpc#C`|8y> zU8Mnv5A0dNCf{Ims*|l-^ z(MRp{qoGohB34|ggDI*p!Aw|MFyJ|v+<+E3brfrI)|+l3W~CQLPbnF@G0)P~Ly!1TJLp}xh8uW`Q+RB-v`MRYZ9Gam3cM%{ zb4Cb*f)0deR~wtNb*8w-LlIF>kc7DAv>T0D(a3@l`k4TFnrO+g9XH7;nYOHxjc4lq zMmaW6qpgAgy)MckYMhl?>sq;-1E)-1llUneeA!ya9KM$)DaNGu57Z5aE>=VST$#vb zFo=uRHr$0M{-ha>h(D_boS4zId;3B|Tpqo|?B?Z@I?G(?&Iei+-{9L_A9=h=Qfn-U z1wIUnQe9!z%_j$F_{rf&`ZFSott09gY~qrf@g3O=Y>vzAnXCyL!@(BqWa)Zqt!#_k zfZHuwS52|&&)aK;CHq9V-t9qt0au{$#6c*R#e5n3rje0hic7c7m{kW$p(_`wB=Gw7 z4k`1Hi;Mc@yA7dp@r~?@rfw)TkjAW++|pkfOG}0N|2guek}j8Zen(!+@7?qt_7ndX zB=BG6WJ31#F3#Vk3=aQr8T)3`{=p9nBHlKzE0I@v`{vJ}h8pd6vby&VgFhzH|q;=aonunAXL6G2y(X^CtAhWr*jI zGjpY@raZDQkg*aMq}Ni6cRF z{oWv}5`nhSAv>usX}m^GHt`f(t8@zHc?K|y5Zi=4G*UG1Sza{$Dpj%X8 zzEXaKT5N6F5j4J|w#qlZP!zS7BT)9b+!ZSJdToqJts1c!)fwih4d31vfb{}W)EgcA zH2pZ^8_k$9+WD2n`6q5XbOy8>3pcYH9 z07eUB+p}YD@AH!}p!iKv><2QF-Y^&xx^PAc1F13A{nUeCDg&{hnix#FiO!fe(^&%Qcux!h znu*S!s$&nnkeotYsDthh1dq(iQrE|#f_=xVgfiiL&-5eAcC-> z5L0l|DVEM$#ulf{bj+Y~7iD)j<~O8CYM8GW)dQGq)!mck)FqoL^X zwNdZb3->hFrbHFm?hLvut-*uK?zXn3q1z|UX{RZ;-WiLoOjnle!xs+W0-8D)kjU#R z+S|A^HkRg$Ij%N4v~k`jyHffKaC~=wg=9)V5h=|kLQ@;^W!o2^K+xG&2n`XCd>OY5Ydi= zgHH=lgy++erK8&+YeTl7VNyVm9-GfONlSlVb3)V9NW5tT!cJ8d7X)!b-$fb!s76{t z@d=Vg-5K_sqHA@Zx-L_}wVnc@L@GL9_K~Zl(h5@AR#FAiKad8~KeWCo@mgXIQ#~u{ zgYFwNz}2b6Vu@CP0XoqJ+dm8px(5W5-Jpis97F`+KM)TuP*X8H@zwiVKDKGVp59pI zifNHZr|B+PG|7|Y<*tqap0CvG7tbR1R>jn70t1X`XJixiMVcHf%Ez*=xm1(CrTSDt z0cle!+{8*Ja&EOZ4@$qhBuKQ$U95Q%rc7tg$VRhk?3=pE&n+T3upZg^ZJc9~c2es% zh7>+|mrmA-p&v}|OtxqmHIBgUxL~^0+cpfkSK2mhh+4b=^F1Xgd2)}U*Yp+H?ls#z zrLxWg_hm}AfK2XYWr!rzW4g;+^^&bW%LmbtRai9f3PjU${r@n`JThy-cphbcwn)rq9{A$Ht`lmYKxOacy z6v2R(?gHhD5@&kB-Eg?4!hAoD7~(h>(R!s1c1Hx#s9vGPePUR|of32bS`J5U5w{F) z>0<^ktO2UHg<0{oxkdOQ;}coZDQph8p6ruj*_?uqURCMTac;>T#v+l1Tc~%^k-Vd@ zkc5y35jVNc49vZpZx;gG$h{%yslDI%Lqga1&&;mN{Ush1c7p>7e-(zp}6E7f-XmJb4nhk zb8zS+{IVbL$QVF8pf8}~kQ|dHJAEATmmnrb_wLG}-yHe>W|A&Y|;muy-d^t^<&)g5SJfaTH@P1%euONny=mxo+C z4N&w#biWY41r8k~468tvuYVh&XN&d#%QtIf9;iVXfWY)#j=l`&B~lqDT@28+Y!0E+MkfC}}H*#(WKKdJJq=O$vNYCb(ZG@p{fJgu;h z21oHQ(14?LeT>n5)s;uD@5&ohU!@wX8w*lB6i@GEH0pM>YTG+RAIWZD;4#F1&F%Jp zXZUml2sH0!lYJT?&sA!qwez6cXzJEd(1ZC~kT5kZSp7(@=H2$Azb_*W&6aA|9iwCL zdX7Q=42;@dspHDwYE?miGX#L^3xD&%BI&fN9^;`v4OjQXPBaBmOF1;#C)8XA(WFlH zycro;DS2?(G&6wkr6rqC>rqDv3nfGw3hmN_9Al>TgvmGsL8_hXx09};l9Ow@)F5@y z#VH5WigLDwZE4nh^7&@g{1FV^UZ%_LJ-s<{HN*2R$OPg@R~Z`c-ET*2}XB@9xvAjrK&hS=f|R8Gr9 zr|0TGOsI7RD+4+2{ZiwdVD@2zmg~g@^D--YL;6UYGSM8i$NbQr4!c7T9rg!8;TM0E zT#@?&S=t>GQm)*ua|?TLT2ktj#`|R<_*FAkOu2Pz$wEc%-=Y9V*$&dg+wIei3b*O8 z2|m$!jJG!J!ZGbbIa!(Af~oSyZV+~M1qGvelMzPNE_%5?c2>;MeeG2^N?JDKjFYCy z7SbPWH-$cWF9~fX%9~v99L!G(wi!PFp>rB!9xj7=Cv|F+7CsGNwY0Q_J%FID%C^CBZQfJ9K(HK%k31j~e#&?hQ zNuD6gRkVckU)v+53-fc} z7ZCzYN-5RG4H7;>>Hg?LU9&5_aua?A0)0dpew1#MMlu)LHe(M;OHjHIUl7|%%)YPo z0cBk;AOY00%Fe6heoN*$(b<)Cd#^8Iu;-2v@>cE-OB$icUF9EEoaC&q8z9}jMTT2I z8`9;jT%z0;dy4!8U;GW{i`)3!c6&oWY`J3669C!tM<5nQFFrFRglU8f)5Op$GtR-3 zn!+SPCw|04sv?%YZ(a7#L?vsdr7ss@WKAw&A*}-1S|9~cL%uA+E~>N6QklFE>8W|% zyX-qAUGTY1hQ-+um`2|&ji0cY*(qN!zp{YpDO-r>jPk*yuVSay<)cUt`t@&FPF_&$ zcHwu1(SQ`I-l8~vYyUxm@D1UEdFJ$f5Sw^HPH7b!9 zzYT3gKMF((N(v0#4f_jPfVZ=ApN^jQJe-X$`A?X+vWjLn_%31KXE*}5_}d8 zw_B1+a#6T1?>M{ronLbHIlEsMf93muJ7AH5h%;i99<~JX^;EAgEB1uHralD*!aJ@F zV2ruuFe9i2Q1C?^^kmVy921eb=tLDD43@-AgL^rQ3IO9%+vi_&R2^dpr}x{bCVPej z7G0-0o64uyWNtr*loIvslyo0%)KSDDKjfThe0hcqs)(C-MH1>bNGBDRTW~scy_{w} zp^aq8Qb!h9Lwielq%C1b8=?Z=&U)ST&PHbS)8Xzjh2DF?d{iAv)Eh)wsUnf>UtXN( zL7=$%YrZ#|^c{MYmhn!zV#t*(jdmYdCpwqpZ{v&L8KIuKn`@IIZfp!uo}c;7J57N` zAxyZ-uA4=Gzl~Ovycz%MW9ZL7N+nRo&1cfNn9(1H5eM;V_4Z_qVann7F>5f>%{rf= zPBZFaV@_Sobl?Fy&KXyzFDV*FIdhS5`Uc~S^Gjo)aiTHgn#<0C=9o-a-}@}xDor;D zZyZ|fvf;+=3MZd>SR1F^F`RJEZo+|MdyJYQAEauKu%WDol~ayrGU3zzbHKsnHKZ*z zFiwUkL@DZ>!*x05ql&EBq@_Vqv83&?@~q5?lVmffQZ+V-=qL+!u4Xs2Z2zdCQ3U7B&QR9_Iggy} z(om{Y9eU;IPe`+p1ifLx-XWh?wI)xU9ik+m#g&pGdB5Bi<`PR*?92lE0+TkRuXI)z z5LP!N2+tTc%cB6B1F-!fj#}>S!vnpgVU~3!*U1ej^)vjUH4s-bd^%B=ItQqDCGbrEzNQi(dJ`J}-U=2{7-d zK8k^Rlq2N#0G?9&1?HSle2vlkj^KWSBYTwx`2?9TU_DX#J+f+qLiZCqY1TXHFxXZqYMuD@RU$TgcnCC{_(vwZ-*uX)~go#%PK z@}2Km_5aQ~(<3cXeJN6|F8X_1@L%@xTzs}$_*E|a^_URF_qcF;Pfhoe?FTFwvjm1o z8onf@OY@jC2tVcMaZS;|T!Ks(wOgPpRzRnFS-^RZ4E!9dsnj9sFt609a|jJbb1Dt@ z<=Gal2jDEupxUSwWu6zp<<&RnAA;d&4gKVG0iu6g(DsST(4)z6R)zDpfaQ}v{5ARt zyhwvMtF%b-YazR5XLz+oh=mn;y-Mf2a8>7?2v8qX;19y?b>Z5laGHvzH;Nu9S`B8} zI)qN$GbXIQ1VL3lnof^6TS~rvPVg4V?Dl2Bb*K2z4E{5vy<(@@K_cN@U>R!>aUIRnb zL*)=787*cs#zb31zBC49x$`=fkQbMAef)L2$dR{)6BAz!t5U_B#1zZG`^neKSS22oJ#5B=gl%U=WeqL9REF2g zZnfCb0?quf?Ztj$VXvDSWoK`0L=Zxem2q}!XWLoT-kYMOx)!7fcgT35uC~0pySEme z`{wGWTkGr7>+Kb^n;W?BZH6ZP(9tQX%-7zF>vc2}LuWDI(9kh1G#7B99r4x6;_-V+k&c{nPUrR zAXJGRiMe~aup{0qzmLNjS_BC4cB#sXjckx{%_c&^xy{M61xEb>KW_AG5VFXUOjAG4 z^>Qlm9A#1N{4snY=(AmWzatb!ngqiqPbBZ7>Uhb3)dTkSGcL#&SH>iMO-IJBPua`u zo)LWZ>=NZLr758j{%(|uQuZ)pXq_4c!!>s|aDM9#`~1bzK3J1^^D#<2bNCccH7~-X}Ggi!pIIF>uFx%aPARGQsnC8ZQc8lrQ5o~smqOg>Ti^GNme94*w z)JZy{_{#$jxGQ&`M z!OMvZMHR>8*^>eS%o*6hJwn!l8VOOjZQJvh)@tnHVW&*GYPuxqXw}%M!(f-SQf`=L z5;=5w2;%82VMH6Xi&-K3W)o&K^+vJCepWZ-rW%+Dc6X3(){z$@4zjYxQ|}8UIojeC zYZpQ1dU{fy=oTr<4VX?$q)LP}IUmpiez^O&N3E_qPpchGTi5ZM6-2ScWlQq%V&R2Euz zO|Q0Hx>lY1Q1cW5xHv5!0OGU~PVEqSuy#fD72d#O`N!C;o=m+YioGu-wH2k6!t<~K zSr`E=W9)!g==~x9VV~-8{4ZN9{~-A9zJpRe%NGg$+MDuI-dH|b@BD)~>pPCGUNNzY zMDg||0@XGQgw`YCt5C&A{_+J}mvV9Wg{6V%2n#YSRN{AP#PY?1FF1#|vO_%e+#`|2*~wGAJaeRX6=IzFNeWhz6gJc8+(03Ph4y6ELAm=AkN7TOgMUEw*N{= z_)EIDQx5q22oUR+_b*tazu9+pX|n1c*IB-}{DqIj z-?E|ks{o3AGRNb;+iKcHkZvYJvFsW&83RAPs1Oh@IWy%l#5x2oUP6ZCtv+b|q>jsf zZ_9XO;V!>n`UxH1LvH8)L4?8raIvasEhkpQoJ`%!5rBs!0Tu(s_D{`4opB;57)pkX z4$A^8CsD3U5*!|bHIEqsn~{q+Ddj$ME@Gq4JXtgVz&7l{Ok!@?EA{B3P~NAqb9)4? zkQo30A^EbHfQ@87G5&EQTd`frrwL)&Yw?%-W@uy^Gn23%j?Y!Iea2xw<-f;esq zf%w5WN@E1}zyXtYv}}`U^B>W`>XPmdLj%4{P298|SisrE;7HvXX;A}Ffi8B#3Lr;1 zHt6zVb`8{#+e$*k?w8|O{Uh|&AG}|DG1PFo1i?Y*cQm$ZwtGcVgMwtBUDa{~L1KT-{jET4w60>{KZ27vXrHJ;fW{6| z=|Y4!&UX020wU1>1iRgB@Q#m~1^Z^9CG1LqDhYBrnx%IEdIty z!46iOoKlKs)c}newDG)rWUikD%j`)p z_w9Ph&e40=(2eBy;T!}*1p1f1SAUDP9iWy^u^Ubdj21Kn{46;GR+hwLO=4D11@c~V zI8x&(D({K~Df2E)Nx_yQvYfh4;MbMJ@Z}=Dt3_>iim~QZ*hZIlEs0mEb z_54+&*?wMD`2#vsQRN3KvoT>hWofI_Vf(^C1ff-Ike@h@saEf7g}<9T`W;HAne-Nd z>RR+&SP35w)xKn8^U$7))PsM!jKwYZ*RzEcG-OlTrX3}9a{q%#Un5E5W{{hp>w~;` zGky+3(vJvQyGwBo`tCpmo0mo((?nM8vf9aXrrY1Ve}~TuVkB(zeds^jEfI}xGBCM2 zL1|#tycSaWCurP+0MiActG3LCas@_@tao@(R1ANlwB$4K53egNE_;!&(%@Qo$>h`^1S_!hN6 z)vZtG$8fN!|BXBJ=SI>e(LAU(y(i*PHvgQ2llulxS8>qsimv7yL}0q_E5WiAz7)(f zC(ahFvG8&HN9+6^jGyLHM~$)7auppeWh_^zKk&C_MQ~8;N??OlyH~azgz5fe^>~7F zl3HnPN3z-kN)I$4@`CLCMQx3sG~V8hPS^}XDXZrQA>}mQPw%7&!sd(Pp^P=tgp-s^ zjl}1-KRPNWXgV_K^HkP__SR`S-|OF0bR-N5>I%ODj&1JUeAQ3$9i;B~$S6}*^tK?= z**%aCiH7y?xdY?{LgVP}S0HOh%0%LI$wRx;$T|~Y8R)Vdwa}kGWv8?SJVm^>r6+%I z#lj1aR94{@MP;t-scEYQWc#xFA30^}?|BeX*W#9OL;Q9#WqaaM546j5j29((^_8Nu z4uq}ESLr~r*O7E7$D{!k9W>`!SLoyA53i9QwRB{!pHe8um|aDE`Cg0O*{jmor)^t)3`>V>SWN-2VJcFmj^1?~tT=JrP`fVh*t zXHarp=8HEcR#vFe+1a%XXuK+)oFs`GDD}#Z+TJ}Ri`FvKO@ek2ayn}yaOi%(8p%2$ zpEu)v0Jym@f}U|-;}CbR=9{#<^z28PzkkTNvyKvJDZe+^VS2bES3N@Jq!-*}{oQlz z@8bgC_KnDnT4}d#&Cpr!%Yb?E!brx0!eVOw~;lLwUoz#Np%d$o%9scc3&zPm`%G((Le|6o1 zM(VhOw)!f84zG^)tZ1?Egv)d8cdNi+T${=5kV+j;Wf%2{3g@FHp^Gf*qO0q!u$=m9 zCaY`4mRqJ;FTH5`a$affE5dJrk~k`HTP_7nGTY@B9o9vvnbytaID;^b=Tzp7Q#DmD zC(XEN)Ktn39z5|G!wsVNnHi) z%^q94!lL|hF`IijA^9NR0F$@h7k5R^ljOW(;Td9grRN0Mb)l_l7##{2nPQ@?;VjXv zaLZG}yuf$r$<79rVPpXg?6iiieX|r#&`p#Con2i%S8*8F}(E) zI5E6c3tG*<;m~6>!&H!GJ6zEuhH7mkAzovdhLy;)q z{H2*8I^Pb}xC4s^6Y}6bJvMu=8>g&I)7!N!5QG$xseeU#CC?ZM-TbjsHwHgDGrsD= z{%f;@Sod+Ch66Ko2WF~;Ty)v>&x^aovCbCbD7>qF*!?BXmOV3(s|nxsb*Lx_2lpB7 zokUnzrk;P=T-&kUHO}td+Zdj!3n&NR?K~cRU zAXU!DCp?51{J4w^`cV#ye}(`SQhGQkkMu}O3M*BWt4UsC^jCFUy;wTINYmhD$AT;4 z?Xd{HaJjP`raZ39qAm;%beDbrLpbRf(mkKbANan7XsL>_pE2oo^$TgdidjRP!5-`% zv0d!|iKN$c0(T|L0C~XD0aS8t{*&#LnhE;1Kb<9&=c2B+9JeLvJr*AyyRh%@jHej=AetOMSlz^=!kxX>>B{2B1uIrQyfd8KjJ+DBy!h)~*(!|&L4^Q_07SQ~E zcemVP`{9CwFvPFu7pyVGCLhH?LhEVb2{7U+Z_>o25#+3<|8%1T^5dh}*4(kfJGry} zm%r#hU+__Z;;*4fMrX=Bkc@7|v^*B;HAl0((IBPPii%X9+u3DDF6%bI&6?Eu$8&aWVqHIM7mK6?Uvq$1|(-T|)IV<>e?!(rY zqkmO1MRaLeTR=)io(0GVtQT@s6rN%C6;nS3@eu;P#ry4q;^O@1ZKCJyp_Jo)Ty^QW z+vweTx_DLm{P-XSBj~Sl<%_b^$=}odJ!S2wAcxenmzFGX1t&Qp8Vxz2VT`uQsQYtdn&_0xVivIcxZ_hnrRtwq4cZSj1c-SG9 z7vHBCA=fd0O1<4*=lu$6pn~_pVKyL@ztw1swbZi0B?spLo56ZKu5;7ZeUml1Ws1?u zqMf1p{5myAzeX$lAi{jIUqo1g4!zWLMm9cfWcnw`k6*BR^?$2(&yW?>w;G$EmTA@a z6?y#K$C~ZT8+v{87n5Dm&H6Pb_EQ@V0IWmG9cG=O;(;5aMWWrIPzz4Q`mhK;qQp~a z+BbQrEQ+w{SeiuG-~Po5f=^EvlouB@_|4xQXH@A~KgpFHrwu%dwuCR)=B&C(y6J4J zvoGk9;lLs9%iA-IJGU#RgnZZR+@{5lYl8(e1h6&>Vc_mvg0d@);X zji4T|n#lB!>pfL|8tQYkw?U2bD`W{na&;*|znjmalA&f;*U++_aBYerq;&C8Kw7mI z7tsG*?7*5j&dU)Lje;^{D_h`%(dK|pB*A*1(Jj)w^mZ9HB|vGLkF1GEFhu&rH=r=8 zMxO42e{Si6$m+Zj`_mXb&w5Q(i|Yxyg?juUrY}78uo@~3v84|8dfgbPd0iQJRdMj< zncCNGdMEcsxu#o#B5+XD{tsg*;j-eF8`mp~K8O1J!Z0+>0=7O=4M}E?)H)ENE;P*F z$Ox?ril_^p0g7xhDUf(q652l|562VFlC8^r8?lQv;TMvn+*8I}&+hIQYh2 z1}uQQaag&!-+DZ@|C+C$bN6W;S-Z@)d1|en+XGvjbOxCa-qAF*LA=6s(Jg+g;82f$ z(Vb)8I)AH@cdjGFAR5Rqd0wiNCu!xtqWbcTx&5kslzTb^7A78~Xzw1($UV6S^VWiP zFd{Rimd-0CZC_Bu(WxBFW7+k{cOW7DxBBkJdJ;VsJ4Z@lERQr%3eVv&$%)b%<~ zCl^Y4NgO}js@u{|o~KTgH}>!* z_iDNqX2(As7T0xivMH|3SC1ivm8Q}6Ffcd7owUKN5lHAtzMM4<0v+ykUT!QiowO;`@%JGv+K$bBx@*S7C8GJVqQ_K>12}M`f_Ys=S zKFh}HM9#6Izb$Y{wYzItTy+l5U2oL%boCJn?R3?jP@n$zSIwlmyGq30Cw4QBO|14` zW5c);AN*J3&eMFAk$SR~2k|&+&Bc$e>s%c{`?d~85S-UWjA>DS5+;UKZ}5oVa5O(N zqqc@>)nee)+4MUjH?FGv%hm2{IlIF-QX}ym-7ok4Z9{V+ZHVZQl$A*x!(q%<2~iVv znUa+BX35&lCb#9VE-~Y^W_f;Xhl%vgjwdjzMy$FsSIj&ok}L+X`4>J=9BkN&nu^E*gbhj3(+D>C4E z@Fwq_=N)^bKFSHTzZk?-gNU$@l}r}dwGyh_fNi=9b|n}J>&;G!lzilbWF4B}BBq4f zYIOl?b)PSh#XTPp4IS5ZR_2C!E)Z`zH0OW%4;&~z7UAyA-X|sh9@~>cQW^COA9hV4 zXcA6qUo9P{bW1_2`eo6%hgbN%(G-F1xTvq!sc?4wN6Q4`e9Hku zFwvlAcRY?6h^Fj$R8zCNEDq8`=uZB8D-xn)tA<^bFFy}4$vA}Xq0jAsv1&5!h!yRA zU()KLJya5MQ`q&LKdH#fwq&(bNFS{sKlEh_{N%{XCGO+po#(+WCLmKW6&5iOHny>g z3*VFN?mx!16V5{zyuMWDVP8U*|BGT$(%IO|)?EF|OI*sq&RovH!N%=>i_c?K*A>>k zyg1+~++zY4Q)J;VWN0axhoIKx;l&G$gvj(#go^pZskEVj8^}is3Jw26LzYYVos0HX zRPvmK$dVxM8(Tc?pHFe0Z3uq){{#OK3i-ra#@+;*=ui8)y6hsRv z4Fxx1c1+fr!VI{L3DFMwXKrfl#Q8hfP@ajgEau&QMCxd{g#!T^;ATXW)nUg&$-n25 zruy3V!!;{?OTobo|0GAxe`Acn3GV@W=&n;~&9 zQM>NWW~R@OYORkJAo+eq1!4vzmf9K%plR4(tB@TR&FSbDoRgJ8qVcH#;7lQub*nq&?Z>7WM=oeEVjkaG zT#f)=o!M2DO5hLR+op>t0CixJCIeXH*+z{-XS|%jx)y(j&}Wo|3!l7{o)HU3m7LYyhv*xF&tq z%IN7N;D4raue&&hm0xM=`qv`+TK@;_xAcGKuK(2|75~ar2Yw)geNLSmVxV@x89bQu zpViVKKnlkwjS&&c|-X6`~xdnh}Ps)Hs z4VbUL^{XNLf7_|Oi>tA%?SG5zax}esF*FH3d(JH^Gvr7Rp*n=t7frH!U;!y1gJB^i zY_M$KL_}mW&XKaDEi9K-wZR|q*L32&m+2n_8lq$xRznJ7p8}V>w+d@?uB!eS3#u<} zIaqi!b!w}a2;_BfUUhGMy#4dPx>)_>yZ`ai?Rk`}d0>~ce-PfY-b?Csd(28yX22L% zI7XI>OjIHYTk_@Xk;Gu^F52^Gn6E1&+?4MxDS2G_#PQ&yXPXP^<-p|2nLTb@AAQEY zI*UQ9Pmm{Kat}wuazpjSyXCdnrD&|C1c5DIb1TnzF}f4KIV6D)CJ!?&l&{T)e4U%3HTSYqsQ zo@zWB1o}ceQSV)<4G<)jM|@@YpL+XHuWsr5AYh^Q{K=wSV99D~4RRU52FufmMBMmd z_H}L#qe(}|I9ZyPRD6kT>Ivj&2Y?qVZq<4bG_co_DP`sE*_Xw8D;+7QR$Uq(rr+u> z8bHUWbV19i#)@@G4bCco@Xb<8u~wVDz9S`#k@ciJtlu@uP1U0X?yov8v9U3VOig2t zL9?n$P3=1U_Emi$#slR>N5wH-=J&T=EdUHA}_Z zZIl3nvMP*AZS9{cDqFanrA~S5BqxtNm9tlu;^`)3X&V4tMAkJ4gEIPl= zoV!Gyx0N{3DpD@)pv^iS*dl2FwANu;1;%EDl}JQ7MbxLMAp>)UwNwe{=V}O-5C*>F zu?Ny+F64jZn<+fKjF01}8h5H_3pey|;%bI;SFg$w8;IC<8l|3#Lz2;mNNik6sVTG3 z+Su^rIE#40C4a-587$U~%KedEEw1%r6wdvoMwpmlXH$xPnNQN#f%Z7|p)nC>WsuO= z4zyqapLS<8(UJ~Qi9d|dQijb_xhA2)v>la)<1md5s^R1N&PiuA$^k|A<+2C?OiHbj z>Bn$~t)>Y(Zb`8hW7q9xQ=s>Rv81V+UiuZJc<23HplI88isqRCId89fb`Kt|CxVIg znWcwprwXnotO>3s&Oypkte^9yJjlUVVxSe%_xlzmje|mYOVPH^vjA=?6xd0vaj0Oz zwJ4OJNiFdnHJX3rw&inskjryukl`*fRQ#SMod5J|KroJRsVXa5_$q7whSQ{gOi*s0 z1LeCy|JBWRsDPn7jCb4s(p|JZiZ8+*ExC@Vj)MF|*Vp{B(ziccSn`G1Br9bV(v!C2 z6#?eqpJBc9o@lJ#^p-`-=`4i&wFe>2)nlPK1p9yPFzJCzBQbpkcR>={YtamIw)3nt z(QEF;+)4`>8^_LU)_Q3 zC5_7lgi_6y>U%m)m@}Ku4C}=l^J=<<7c;99ec3p{aR+v=diuJR7uZi%aQv$oP?dn?@6Yu_+*^>T0ptf(oobdL;6)N-I!TO`zg^Xbv3#L0I~sn@WGk-^SmPh5>W+LB<+1PU}AKa?FCWF|qMNELOgdxR{ zbqE7@jVe+FklzdcD$!(A$&}}H*HQFTJ+AOrJYnhh}Yvta(B zQ_bW4Rr;R~&6PAKwgLWXS{Bnln(vUI+~g#kl{r+_zbngT`Y3`^Qf=!PxN4IYX#iW4 zucW7@LLJA9Zh3(rj~&SyN_pjO8H&)|(v%!BnMWySBJV=eSkB3YSTCyIeJ{i;(oc%_hk{$_l;v>nWSB)oVeg+blh=HB5JSlG_r7@P z3q;aFoZjD_qS@zygYqCn=;Zxjo!?NK!%J$ z52lOP`8G3feEj+HTp@Tnn9X~nG=;tS+z}u{mQX_J0kxtr)O30YD%oo)L@wy`jpQYM z@M>Me=95k1p*FW~rHiV1CIfVc{K8r|#Kt(ApkXKsDG$_>76UGNhHExFCw#Ky9*B-z zNq2ga*xax!HMf_|Vp-86r{;~YgQKqu7%szk8$hpvi_2I`OVbG1doP(`gn}=W<8%Gn z%81#&WjkH4GV;4u43EtSW>K_Ta3Zj!XF?;SO3V#q=<=>Tc^@?A`i;&`-cYj|;^ zEo#Jl5zSr~_V-4}y8pnufXLa80vZY4z2ko7fj>DR)#z=wWuS1$$W!L?(y}YC+yQ|G z@L&`2upy3f>~*IquAjkVNU>}c10(fq#HdbK$~Q3l6|=@-eBbo>B9(6xV`*)sae58*f zym~RRVx;xoCG3`JV`xo z!lFw)=t2Hy)e!IFs?0~7osWk(d%^wxq&>_XD4+U#y&-VF%4z?XH^i4w`TxpF{`XhZ z%G}iEzf!T(l>g;W9<~K+)$g!{UvhW{E0Lis(S^%I8OF&%kr!gJ&fMOpM=&=Aj@wuL zBX?*6i51Qb$uhkwkFYkaD_UDE+)rh1c;(&Y=B$3)J&iJfQSx!1NGgPtK!$c9OtJuu zX(pV$bfuJpRR|K(dp@^j}i&HeJOh@|7lWo8^$*o~Xqo z5Sb+!EtJ&e@6F+h&+_1ETbg7LfP5GZjvIUIN3ibCOldAv z)>YdO|NH$x7AC8dr=<2ekiY1%fN*r~e5h6Yaw<{XIErujKV~tiyrvV_DV0AzEknC- zR^xKM3i<1UkvqBj3C{wDvytOd+YtDSGu!gEMg+!&|8BQrT*|p)(dwQLEy+ zMtMzij3zo40)CA!BKZF~yWg?#lWhqD3@qR)gh~D{uZaJO;{OWV8XZ_)J@r3=)T|kt zUS1pXr6-`!Z}w2QR7nP%d?ecf90;K_7C3d!UZ`N(TZoWNN^Q~RjVhQG{Y<%E1PpV^4 z-m-K+$A~-+VDABs^Q@U*)YvhY4Znn2^w>732H?NRK(5QSS$V@D7yz2BVX4)f5A04~$WbxGOam22>t&uD)JB8-~yiQW6ik;FGblY_I>SvB_z2?PS z*Qm&qbKI{H1V@YGWzpx`!v)WeLT02};JJo*#f$a*FH?IIad-^(;9XC#YTWN6;Z6+S zm4O1KH=#V@FJw7Pha0!9Vb%ZIM$)a`VRMoiN&C|$YA3~ZC*8ayZRY^fyuP6$n%2IU z$#XceYZeqLTXw(m$_z|33I$B4k~NZO>pP6)H_}R{E$i%USGy{l{-jOE;%CloYPEU+ zRFxOn4;7lIOh!7abb23YKD+_-?O z0FP9otcAh+oSj;=f#$&*ExUHpd&e#bSF%#8*&ItcL2H$Sa)?pt0Xtf+t)z$_u^wZi z44oE}r4kIZGy3!Mc8q$B&6JqtnHZ>Znn!Zh@6rgIu|yU+zG8q`q9%B18|T|oN3zMq z`l&D;U!OL~%>vo&q0>Y==~zLiCZk4v%s_7!9DxQ~id1LLE93gf*gg&2$|hB#j8;?3 z5v4S;oM6rT{Y;I+#FdmNw z){d%tNM<<#GN%n9ox7B=3#;u7unZ~tLB_vRZ52a&2=IM)2VkXm=L+Iqq~uk#Dug|x z>S84e+A7EiOY5lj*!q?6HDkNh~0g;0Jy(al!ZHHDtur9T$y-~)94HelX1NHjXWIM7UAe}$?jiz z9?P4`I0JM=G5K{3_%2jPLC^_Mlw?-kYYgb7`qGa3@dn|^1fRMwiyM@Ch z;CB&o7&&?c5e>h`IM;Wnha0QKnEp=$hA8TJgR-07N~U5(>9vJzeoFsSRBkDq=x(YgEMpb=l4TDD`2 zwVJpWGTA_u7}?ecW7s6%rUs&NXD3+n;jB86`X?8(l3MBo6)PdakI6V6a}22{)8ilT zM~T*mU}__xSy|6XSrJ^%lDAR3Lft%+yxC|ZUvSO_nqMX!_ul3;R#*{~4DA=h$bP)%8Yv9X zyp><|e8=_ttI}ZAwOd#dlnSjck#6%273{E$kJuCGu=I@O)&6ID{nWF5@gLb16sj|&Sb~+du4e4O_%_o`Ix4NRrAsyr1_}MuP94s>de8cH-OUkVPk3+K z&jW)It9QiU-ti~AuJkL`XMca8Oh4$SyJ=`-5WU<{cIh+XVH#e4d&zive_UHC!pN>W z3TB;Mn5i)9Qn)#6@lo4QpI3jFYc0~+jS)4AFz8fVC;lD^+idw^S~Qhq>Tg(!3$yLD zzktzoFrU@6s4wwCMz}edpF5i5Q1IMmEJQHzp(LAt)pgN3&O!&d?3W@6U4)I^2V{;- z6A(?zd93hS*uQmnh4T)nHnE{wVhh(=MMD(h(P4+^p83Om6t<*cUW>l(qJzr%5vp@K zN27ka(L{JX=1~e2^)F^i=TYj&;<7jyUUR2Bek^A8+3Up*&Xwc{)1nRR5CT8vG>ExV zHnF3UqXJOAno_?bnhCX-&kwI~Ti8t4`n0%Up>!U`ZvK^w2+0Cs-b9%w%4`$+To|k= zKtgc&l}P`*8IS>8DOe?EB84^kx4BQp3<7P{Pq}&p%xF_81pg!l2|u=&I{AuUgmF5n zJQCTLv}%}xbFGYtKfbba{CBo)lWW%Z>i(_NvLhoQZ*5-@2l&x>e+I~0Nld3UI9tdL zRzu8}i;X!h8LHVvN?C+|M81e>Jr38%&*9LYQec9Ax>?NN+9(_>XSRv&6hlCYB`>Qm z1&ygi{Y()OU4@D_jd_-7vDILR{>o|7-k)Sjdxkjgvi{@S>6GqiF|o`*Otr;P)kLHN zZkpts;0zw_6;?f(@4S1FN=m!4^mv~W+lJA`&7RH%2$)49z0A+8@0BCHtj|yH--AEL z0tW6G%X-+J+5a{5*WKaM0QDznf;V?L5&uQw+yegDNDP`hA;0XPYc6e0;Xv6|i|^F2WB)Z$LR|HR4 zTQsRAby9(^Z@yATyOgcfQw7cKyr^3Tz7lc7+JEwwzA7)|2x+PtEb>nD(tpxJQm)Kn zW9K_*r!L%~N*vS8<5T=iv|o!zTe9k_2jC_j*7ik^M_ zaf%k{WX{-;0*`t`G!&`eW;gChVXnJ-Rn)To8vW-?>>a%QU1v`ZC=U)f8iA@%JG0mZ zDqH;~mgBnrCP~1II<=V9;EBL)J+xzCoiRBaeH&J6rL!{4zIY8tZka?_FBeQeNO3q6 zyG_alW54Ba&wQf{&F1v-r1R6ID)PTsqjIBc+5MHkcW5Fnvi~{-FjKe)t1bl}Y;z@< z=!%zvpRua>>t_x}^}z0<7MI!H2v6|XAyR9!t50q-A)xk0nflgF4*OQlCGK==4S|wc zRMsSscNhRzHMBU8TdcHN!q^I}x0iXJ%uehac|Zs_B$p@CnF)HeXPpB_Za}F{<@6-4 zl%kml@}kHQ(ypD8FsPJ2=14xXJE|b20RUIgs!2|R3>LUMGF6X*B_I|$`Qg=;zm7C z{mEDy9dTmPbued7mlO@phdmAmJ7p@GR1bjCkMw6*G7#4+`k>fk1czdJUB!e@Q(~6# zwo%@p@V5RL0ABU2LH7Asq^quDUho@H>eTZH9f*no9fY0T zD_-9px3e}A!>>kv5wk91%C9R1J_Nh!*&Kk$J3KNxC}c_@zlgpJZ+5L)Nw|^p=2ue}CJtm;uj*Iqr)K})kA$xtNUEvX;4!Px*^&9T_`IN{D z{6~QY=Nau6EzpvufB^hflc#XIsSq0Y9(nf$d~6ZwK}fal92)fr%T3=q{0mP-EyP_G z)UR5h@IX}3Qll2b0oCAcBF>b*@Etu*aTLPU<%C>KoOrk=x?pN!#f_Og-w+;xbFgjQ zXp`et%lDBBh~OcFnMKMUoox0YwBNy`N0q~bSPh@+enQ=4RUw1) zpovN`QoV>vZ#5LvC;cl|6jPr}O5tu!Ipoyib8iXqy}TeJ;4+_7r<1kV0v5?Kv>fYp zg>9L`;XwXa&W7-jf|9~uP2iyF5`5AJ`Q~p4eBU$MCC00`rcSF>`&0fbd^_eqR+}mK z4n*PMMa&FOcc)vTUR zlDUAn-mh`ahi_`f`=39JYTNVjsTa_Y3b1GOIi)6dY)D}xeshB0T8Eov5%UhWd1)u}kjEQ|LDo{tqKKrYIfVz~@dp!! zMOnah@vp)%_-jDTUG09l+;{CkDCH|Q{NqX*uHa1YxFShy*1+;J`gywKaz|2Q{lG8x zP?KBur`}r`!WLKXY_K;C8$EWG>jY3UIh{+BLv0=2)KH%P}6xE2kg)%(-uA6lC?u8}{K(#P*c zE9C8t*u%j2r_{;Rpe1A{9nNXU;b_N0vNgyK!EZVut~}+R2rcbsHilqsOviYh-pYX= zHw@53nlmwYI5W5KP>&`dBZe0Jn?nAdC^HY1wlR6$u^PbpB#AS&5L6zqrXN&7*N2Q` z+Rae1EwS)H=aVSIkr8Ek^1jy2iS2o7mqm~Mr&g5=jjt7VxwglQ^`h#Mx+x2v|9ZAwE$i_9918MjJxTMr?n!bZ6n$}y11u8I9COTU`Z$Fi z!AeAQLMw^gp_{+0QTEJrhL424pVDp%wpku~XRlD3iv{vQ!lAf!_jyqd_h}+Tr1XG| z`*FT*NbPqvHCUsYAkFnM`@l4u_QH&bszpUK#M~XLJt{%?00GXY?u_{gj3Hvs!=N(I z(=AuWPijyoU!r?aFTsa8pLB&cx}$*%;K$e*XqF{~*rA-qn)h^!(-;e}O#B$|S~c+U zN4vyOK0vmtx$5K!?g*+J@G1NmlEI=pyZXZ69tAv=@`t%ag_Hk{LP~OH9iE)I= zaJ69b4kuCkV0V zo(M0#>phpQ_)@j;h%m{-a*LGi(72TP)ws2w*@4|C-3+;=5DmC4s7Lp95%n%@Ko zfdr3-a7m*dys9iIci$A=4NPJ`HfJ;hujLgU)ZRuJI`n;Pw|yksu!#LQnJ#dJysgNb z@@qwR^wrk(jbq4H?d!lNyy72~Dnn87KxsgQ!)|*m(DRM+eC$wh7KnS-mho3|KE)7h zK3k;qZ;K1Lj6uEXLYUYi)1FN}F@-xJ z@@3Hb84sl|j{4$3J}aTY@cbX@pzB_qM~APljrjju6P0tY{C@ zpUCOz_NFmALMv1*blCcwUD3?U6tYs+N%cmJ98D%3)%)Xu^uvzF zS5O!sc#X6?EwsYkvPo6A%O8&y8sCCQH<%f2togVwW&{M;PR!a(ZT_A+jVAbf{@5kL zB@Z(hb$3U{T_}SKA_CoQVU-;j>2J=L#lZ~aQCFg-d<9rzs$_gO&d5N6eFSc z1ml8)P*FSi+k@!^M9nDWR5e@ATD8oxtDu=36Iv2!;dZzidIS(PCtEuXAtlBb1;H%Z zwnC^Ek*D)EX4#Q>R$$WA2sxC_t(!!6Tr?C#@{3}n{<^o;9id1RA&-Pig1e-2B1XpG zliNjgmd3c&%A}s>qf{_j#!Z`fu0xIwm4L0)OF=u(OEmp;bLCIaZX$&J_^Z%4Sq4GZ zPn6sV_#+6pJmDN_lx@1;Zw6Md_p0w9h6mHtzpuIEwNn>OnuRSC2=>fP^Hqgc)xu^4 z<3!s`cORHJh#?!nKI`Et7{3C27+EuH)Gw1f)aoP|B3y?fuVfvpYYmmukx0ya-)TQX zR{ggy5cNf4X|g)nl#jC9p>7|09_S7>1D2GTRBUTW zAkQ=JMRogZqG#v;^=11O6@rPPwvJkr{bW-Qg8`q8GoD#K`&Y+S#%&B>SGRL>;ZunM@49!}Uy zN|bBCJ%sO;@3wl0>0gbl3L@1^O60ONObz8ZI7nder>(udj-jt`;yj^nTQ$L9`OU9W zX4alF#$|GiR47%x@s&LV>2Sz2R6?;2R~5k6V>)nz!o_*1Y!$p>BC5&?hJg_MiE6UBy>RkVZj`9UWbRkN-Hk!S`=BS3t3uyX6)7SF#)71*}`~Ogz z1rap5H6~dhBJ83;q-Y<5V35C2&F^JI-it(=5D#v!fAi9p#UwV~2tZQI+W(Dv?1t9? zfh*xpxxO{-(VGB>!Q&0%^YW_F!@aZS#ucP|YaD#>wd1Fv&Z*SR&mc;asi}1G) z_H>`!akh-Zxq9#io(7%;a$)w+{QH)Y$?UK1Dt^4)up!Szcxnu}kn$0afcfJL#IL+S z5gF_Y30j;{lNrG6m~$Ay?)*V9fZuU@3=kd40=LhazjFrau>(Y>SJNtOz>8x_X-BlA zIpl{i>OarVGj1v(4?^1`R}aQB&WCRQzS~;7R{tDZG=HhgrW@B`W|#cdyj%YBky)P= zpxuOZkW>S6%q7U{VsB#G(^FMsH5QuGXhb(sY+!-R8Bmv6Sx3WzSW<1MPPN1!&PurYky(@`bP9tz z52}LH9Q?+FF5jR6-;|+GVdRA!qtd;}*-h&iIw3Tq3qF9sDIb1FFxGbo&fbG5n8$3F zyY&PWL{ys^dTO}oZ#@sIX^BKW*bon=;te9j5k+T%wJ zNJtoN1~YVj4~YRrlZl)b&kJqp+Z`DqT!la$x&&IxgOQw#yZd-nBP3!7FijBXD|IsU8Zl^ zc6?MKpJQ+7ka|tZQLfchD$PD|;K(9FiLE|eUZX#EZxhG!S-63C$jWX1Yd!6-Yxi-u zjULIr|0-Q%D9jz}IF~S%>0(jOqZ(Ln<$9PxiySr&2Oic7vb<8q=46)Ln%Z|<*z5&> z3f~Zw@m;vR(bESB<=Jqkxn(=#hQw42l(7)h`vMQQTttz9XW6^|^8EK7qhju4r_c*b zJIi`)MB$w@9epwdIfnEBR+?~);yd6C(LeMC& zn&&N*?-g&BBJcV;8&UoZi4Lmxcj16ojlxR~zMrf=O_^i1wGb9X-0@6_rpjPYemIin zmJb+;lHe;Yp=8G)Q(L1bzH*}I>}uAqhj4;g)PlvD9_e_ScR{Ipq|$8NvAvLD8MYr}xl=bU~)f%B3E>r3Bu9_t|ThF3C5~BdOve zEbk^r&r#PT&?^V1cb{72yEWH}TXEE}w>t!cY~rA+hNOTK8FAtIEoszp!qqptS&;r$ zaYV-NX96-h$6aR@1xz6_E0^N49mU)-v#bwtGJm)ibygzJ8!7|WIrcb`$XH~^!a#s& z{Db-0IOTFq#9!^j!n_F}#Z_nX{YzBK8XLPVmc&X`fT7!@$U-@2KM9soGbmOSAmqV z{nr$L^MBo_u^Joyf0E^=eo{Rt0{{e$IFA(#*kP@SQd6lWT2-#>` zP1)7_@IO!9lk>Zt?#CU?cuhiLF&)+XEM9B)cS(gvQT!X3`wL*{fArTS;Ak`J<84du zALKPz4}3nlG8Fo^MH0L|oK2-4xIY!~Oux~1sw!+It)&D3p;+N8AgqKI`ld6v71wy8I!eP0o~=RVcFQR2Gr(eP_JbSytoQ$Yt}l*4r@A8Me94y z8cTDWhqlq^qoAhbOzGBXv^Wa4vUz$(7B!mX`T=x_ueKRRDfg&Uc-e1+z4x$jyW_Pm zp?U;-R#xt^Z8Ev~`m`iL4*c#65Nn)q#=Y0l1AuD&+{|8-Gsij3LUZXpM0Bx0u7WWm zH|%yE@-#XEph2}-$-thl+S;__ciBxSSzHveP%~v}5I%u!z_l_KoW{KRx2=eB33umE zIYFtu^5=wGU`Jab8#}cnYry@9p5UE#U|VVvx_4l49JQ;jQdp(uw=$^A$EA$LM%vmE zvdEOaIcp5qX8wX{mYf0;#51~imYYPn4=k&#DsKTxo{_Mg*;S495?OBY?#gv=edYC* z^O@-sd-qa+U24xvcbL0@C7_6o!$`)sVr-jSJE4XQUQ$?L7}2(}Eixqv;L8AdJAVqc zq}RPgpnDb@E_;?6K58r3h4-!4rT4Ab#rLHLX?eMOfluJk=3i1@Gt1i#iA=O`M0@x! z(HtJP9BMHXEzuD93m|B&woj0g6T?f#^)>J>|I4C5?Gam>n9!8CT%~aT;=oco5d6U8 zMXl(=W;$ND_8+DD*?|5bJ!;8ebESXMUKBAf7YBwNVJibGaJ*(2G`F%wx)grqVPjudiaq^Kl&g$8A2 zWMxMr@_$c}d+;_B`#kUX-t|4VKH&_f^^EP0&=DPLW)H)UzBG%%Tra*5 z%$kyZe3I&S#gfie^z5)!twG={3Cuh)FdeA!Kj<-9** zvT*5%Tb`|QbE!iW-XcOuy39>D3oe6x{>&<#E$o8Ac|j)wq#kQzz|ATd=Z0K!p2$QE zPu?jL8Lb^y3_CQE{*}sTDe!2!dtlFjq&YLY@2#4>XS`}v#PLrpvc4*@q^O{mmnr5D zmyJq~t?8>FWU5vZdE(%4cuZuao0GNjp3~Dt*SLaxI#g_u>hu@k&9Ho*#CZP~lFJHj z(e!SYlLigyc?&5-YxlE{uuk$9b&l6d`uIlpg_z15dPo*iU&|Khx2*A5Fp;8iK_bdP z?T6|^7@lcx2j0T@x>X7|kuuBSB7<^zeY~R~4McconTxA2flHC0_jFxmSTv-~?zVT| zG_|yDqa9lkF*B6_{j=T>=M8r<0s;@z#h)3BQ4NLl@`Xr__o7;~M&dL3J8fP&zLfDfy z);ckcTev{@OUlZ`bCo(-3? z1u1xD`PKgSg?RqeVVsF<1SLF;XYA@Bsa&cY!I48ZJn1V<3d!?s=St?TLo zC0cNr`qD*M#s6f~X>SCNVkva^9A2ZP>CoJ9bvgXe_c}WdX-)pHM5m7O zrHt#g$F0AO+nGA;7dSJ?)|Mo~cf{z2L)Rz!`fpi73Zv)H=a5K)*$5sf_IZypi($P5 zsPwUc4~P-J1@^3C6-r9{V-u0Z&Sl7vNfmuMY4yy*cL>_)BmQF!8Om9Dej%cHxbIzA zhtV0d{=%cr?;bpBPjt@4w=#<>k5ee=TiWAXM2~tUGfm z$s&!Dm0R^V$}fOR*B^kGaipi~rx~A2cS0;t&khV1a4u38*XRUP~f za!rZMtay8bsLt6yFYl@>-y^31(*P!L^^s@mslZy(SMsv9bVoX`O#yBgEcjCmGpyc* zeH$Dw6vB5P*;jor+JOX@;6K#+xc)Z9B8M=x2a@Wx-{snPGpRmOC$zpsqW*JCh@M2Y z#K+M(>=#d^>Of9C`))h<=Bsy)6zaMJ&x-t%&+UcpLjV`jo4R2025 zXaG8EA!0lQa)|dx-@{O)qP6`$rhCkoQqZ`^SW8g-kOwrwsK8 z3ms*AIcyj}-1x&A&vSq{r=QMyp3CHdWH35!sad#!Sm>^|-|afB+Q;|Iq@LFgqIp#Z zD1%H+3I?6RGnk&IFo|u+E0dCxXz4yI^1i!QTu7uvIEH>i3rR{srcST`LIRwdV1P;W z+%AN1NIf@xxvVLiSX`8ILA8MzNqE&7>%jMzGt9wm78bo9<;h*W84i29^w!>V>{N+S zd`5Zmz^G;f=icvoOZfK5#1ctx*~UwD=ab4DGQXehQ!XYnak*dee%YN$_ZPL%KZuz$ zD;$PpT;HM^$KwtQm@7uvT`i6>Hae1CoRVM2)NL<2-k2PiX=eAx+-6j#JI?M}(tuBW zkF%jjLR)O`gI2fcPBxF^HeI|DWwQWHVR!;;{BXXHskxh8F@BMDn`oEi-NHt;CLymW z=KSv5)3dyzec0T5B*`g-MQ<;gz=nIWKUi9ko<|4I(-E0k$QncH>E4l z**1w&#={&zv4Tvhgz#c29`m|;lU-jmaXFMC11 z*dlXDMEOG>VoLMc>!rApwOu2prKSi*!w%`yzGmS+k(zm*CsLK*wv{S_0WX^8A-rKy zbk^Gf_92^7iB_uUF)EE+ET4d|X|>d&mdN?x@vxKAQk`O+r4Qdu>XGy(a(19g;=jU} zFX{O*_NG>!$@jh!U369Lnc+D~qch3uT+_Amyi}*k#LAAwh}k8IPK5a-WZ81ufD>l> z$4cF}GSz>ce`3FAic}6W4Z7m9KGO?(eWqi@L|5Hq0@L|&2flN1PVl}XgQ2q*_n2s3 zt5KtowNkTYB5b;SVuoXA@i5irXO)A&%7?V`1@HGCB&)Wgk+l|^XXChq;u(nyPB}b3 zY>m5jkxpZgi)zfbgv&ec4Zqdvm+D<?Im*mXweS9H+V>)zF#Zp3)bhl$PbISY{5=_z!8&*Jv~NYtI-g!>fDs zmvL5O^U%!^VaKA9gvKw|5?-jk>~%CVGvctKmP$kpnpfN{D8@X*Aazi$txfa%vd-|E z>kYmV66W!lNekJPom29LdZ%(I+ZLZYTXzTg*to~m?7vp%{V<~>H+2}PQ?PPAq`36R z<%wR8v6UkS>Wt#hzGk#44W<%9S=nBfB);6clKwnxY}T*w21Qc3_?IJ@4gYzC7s;WP zVQNI(M=S=JT#xsZy7G`cR(BP9*je0bfeN8JN5~zY(DDs0t{LpHOIbN);?T-69Pf3R zSNe*&p2%AwXHL>__g+xd4Hlc_vu<25H?(`nafS%)3UPP7_4;gk-9ckt8SJRTv5v0M z_Hww`qPudL?ajIR&X*;$y-`<)6dxx1U~5eGS13CB!lX;3w7n&lDDiArbAhSycd}+b zya_3p@A`$kQy;|NJZ~s44Hqo7Hwt}X86NK=(ey>lgWTtGL6k@Gy;PbO!M%1~Wcn2k zUFP|*5d>t-X*RU8g%>|(wwj*~#l4z^Aatf^DWd1Wj#Q*AY0D^V@sC`M zjJc6qXu0I7Y*2;;gGu!plAFzG=J;1%eIOdn zQA>J&e05UN*7I5@yRhK|lbBSfJ+5Uq;!&HV@xfPZrgD}kE*1DSq^=%{o%|LChhl#0 zlMb<^a6ixzpd{kNZr|3jTGeEzuo}-eLT-)Q$#b{!vKx8Tg}swCni>{#%vDY$Ww$84 zew3c9BBovqb}_&BRo#^!G(1Eg((BScRZ}C)Oz?y`T5wOrv);)b^4XR8 zhJo7+<^7)qB>I;46!GySzdneZ>n_E1oWZY;kf94#)s)kWjuJN1c+wbVoNQcmnv}{> zN0pF+Sl3E}UQ$}slSZeLJrwT>Sr}#V(dVaezCQl2|4LN`7L7v&siYR|r7M(*JYfR$ zst3=YaDw$FSc{g}KHO&QiKxuhEzF{f%RJLKe3p*7=oo`WNP)M(9X1zIQPP0XHhY3c znrP{$4#Ol$A0s|4S7Gx2L23dv*Gv2o;h((XVn+9+$qvm}s%zi6nI-_s6?mG! zj{DV;qesJb&owKeEK?=J>UcAlYckA7Sl+I&IN=yasrZOkejir*kE@SN`fk<8Fgx*$ zy&fE6?}G)d_N`){P~U@1jRVA|2*69)KSe_}!~?+`Yb{Y=O~_+@!j<&oVQQMnhoIRU zA0CyF1OFfkK44n*JD~!2!SCPM;PRSk%1XL=0&rz00wxPs&-_eapJy#$h!eqY%nS0{ z!aGg58JIJPF3_ci%n)QSVpa2H`vIe$RD43;#IRfDV&Ibit z+?>HW4{2wOfC6Fw)}4x}i1maDxcE1qi@BS*qcxD2gE@h3#4cgU*D-&3z7D|tVZWt= z-Cy2+*Cm@P4GN_TPUtaVyVesbVDazF@)j8VJ4>XZv!f%}&eO1SvIgr}4`A*3#vat< z_MoByL(qW6L7SFZ#|Gc1fFN)L2PxY+{B8tJp+pxRyz*87)vXR}*=&ahXjBlQKguuf zX6x<<6fQulE^C*KH8~W%ptpaC0l?b=_{~*U4?5Vt;dgM4t_{&UZ1C2j?b>b+5}{IF_CUyvz-@QZPMlJ)r_tS$9kH%RPv#2_nMb zRLj5;chJ72*U`Z@Dqt4$@_+k$%|8m(HqLG!qT4P^DdfvGf&){gKnGCX#H0!;W=AGP zbA&Z`-__a)VTS}kKFjWGk z%|>yE?t*EJ!qeQ%dPk$;xIQ+P0;()PCBDgjJm6Buj{f^awNoVx+9<|lg3%-$G(*f) zll6oOkN|yamn1uyl2*N-lnqRI1cvs_JxLTeahEK=THV$Sz*gQhKNb*p0fNoda#-&F zB-qJgW^g}!TtM|0bS2QZekW7_tKu%GcJ!4?lObt0z_$mZ4rbQ0o=^curCs3bJK6sq z9fu-aW-l#>z~ca(B;4yv;2RZ?tGYAU)^)Kz{L|4oPj zdOf_?de|#yS)p2v8-N||+XL=O*%3+y)oI(HbM)Ds?q8~HPzIP(vs*G`iddbWq}! z(2!VjP&{Z1w+%eUq^ '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/java/ppc-pir-services/gradlew.bat b/java/ppc-pir-services/gradlew.bat new file mode 100644 index 00000000..f127cfd4 --- /dev/null +++ b/java/ppc-pir-services/gradlew.bat @@ -0,0 +1,91 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/java/ppc-pir-services/script/start.sh b/java/ppc-pir-services/script/start.sh new file mode 100644 index 00000000..fd8114ca --- /dev/null +++ b/java/ppc-pir-services/script/start.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +JAVA_HOME=$JAVA_HOME +APP_MAIN=cn.webank.wedpr.http.PpcsPirApplication +CLASSPATH='conf/:app/*:libs/*' +CURRENT_DIR=`pwd` +LOG_DIR=${CURRENT_DIR}/logs +CONF_DIR=${CURRENT_DIR}/conf +os_platform=`uname -s` + + +if [ "${JAVA_HOME}" = "" ];then + echo "JAVA_HOME has not been configured" + exit -1 +fi + +log_path="/data/app/ppc/wedpr-pir-node/logs/wedpr-pir.log" +# 检查文件是否存在 +if [ ! -f "$log_path" ]; then + # 文件不存在，创建文件 + touch "$log_path" +fi + +# 初始化全局变量，用于标识系统的PID（0表示未启动） +tradePortalPID=0 +start_timestamp=0 +APP_NAME="wedpr-pir" + +getTradeProtalPID(){ + javaps=`$JAVA_HOME/bin/jps -l | grep $APP_MAIN` + if [ -n "$javaps" ]; then + tradePortalPID=`echo $javaps | awk '{print $1}'` + else + tradePortalPID=0 + fi +} + +#JAVA_OPTS=" -Dfile.encoding=UTF-8 -Dcom.sun.management.jmxremote.port=${JMXREMOTE_PORT} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +JAVA_OPTS=" -Dlog4j.configurationfile=${CONF_DIR}/log4j2.xml -Dindex.log.home=${LOG_DIR} -Dconfig=${CONF_DIR}/" +JAVA_OPTS+=" -Xmx3072m -Xms3072m -XX:NewSize=1024m -XX:MaxNewSize=1024m -XX:PermSize=512m -XX:MaxPermSize=512m" +JAVA_OPTS+=" -XX:CompileThreshold=20000 -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70" +JAVA_OPTS+=" -XX:+DisableExplicitGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${LOG_DIR}/jvm.log" +JAVA_OPTS+=" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOG_DIR}/ -XX:ErrorFile=${LOG_DIR}/heap_error.log" + +function get_start_time() { + start_time=$(date "+%Y-%m-%d %H:%M:%S") + if [[ "${os_platform}" = "Darwin" ]];then + start_timestamp=$(date -j -f "%Y-%m-%d %H:%M:%S" "${start_time}" +%s) + elif [[ "${os_platform}" = "Linux" ]];then + start_timestamp=$(date -d "${start_time}" +%s) + else + echo -e "\033[31m[ERROR] Server start fail!\033[0m" + echo "check platform...[failed]" + echo "===============================================================================================" + kill $tradePortalPID + exit 1 + fi +} + +function waiting_for_start() { + echo "" + i=0 + while [ $i -le 30 ] + do + for j in '\\' '|' '/' '-' + do + printf "%c%c%c%c%c Waiting for server started %c%c%c%c%c\r" \ + "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" + check_time=$(cat ${dirpath}/logs/${APP_NAME}.log | grep -a "JVM running for" | tail -n1 | awk -F "]" '{print $3}' | awk -F "[" '{print $2}' | awk -F " " '{print $1, $2}') + if [ -n "$check_time" ]; then + if [[ "${os_platform}" = "Darwin" ]];then + timestamp=$(date -j -f "%Y-%m-%d %H:%M:%S" "${check_time}" +%s) + elif [[ "${os_platform}" = "Linux" ]];then + timestamp=$(date -d "${check_time}" +%s) + else + echo -e "\033[31m[ERROR] Server start fail!\033[0m" + echo "check platform...[failed]" + echo "===============================================================================================" + kill $tradePortalPID + exit 1 + fi + if [[ ${timestamp} -gt ${start_timestamp} ]]; then + echo "" + echo -e "\033[32m[INFO] Server start Successful!\033[0m" + echo "(PID=$tradePortalPID)...[Success]" + echo "===============================================================================================" + exit 0 + fi + fi + sleep 1 + done + let i=i+5 + done + echo "" + echo -e "\033[31m[ERROR] Server start fail!\033[0m" + echo "timeout...[failed]" + echo "===============================================================================================" + kill $tradePortalPID + exit 1 +} + +start(){ + getTradeProtalPID + echo "===============================================================================================" + if [ $tradePortalPID -ne 0 ]; then + echo "$APP_MAIN already started(PID=$tradePortalPID)" + echo "===============================================================================================" + else + get_start_time + nohup $JAVA_HOME/bin/java $JAVA_OPTS -cp $CLASSPATH $APP_MAIN >start.out 2>&1 & + sleep 1 + getTradeProtalPID + if [ $tradePortalPID -ne 0 ]; then + waiting_for_start + else + echo -e "\033[31m[ERROR] Server start fail!\033[0m" + echo "[Failed]" + echo "===============================================================================================" + fi + fi +} + +start diff --git a/java/ppc-pir-services/script/stop.sh b/java/ppc-pir-services/script/stop.sh new file mode 100644 index 00000000..ac04bbde --- /dev/null +++ b/java/ppc-pir-services/script/stop.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +rm -rf start.out + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +JAVA_HOME=$JAVA_HOME +APP_MAIN=cn.webank.wedpr.http.PpcsPirApplication +CURRENT_DIR=`pwd` +CONF_DIR=${CURRENT_DIR}/conf + +# 初始化全局变量，用于标识系统的PID（0表示未启动） +tradePortalPID=0 + +getTradeProtalPID(){ + javaps=`$JAVA_HOME/bin/jps -l | grep $APP_MAIN` + if [ -n "$javaps" ]; then + tradePortalPID=`echo $javaps | awk '{print $1}'` + else + tradePortalPID=0 + fi +} + +stop(){ + getTradeProtalPID + echo "===============================================================================================" + if [ $tradePortalPID -ne 0 ]; then + echo -n "Stopping $APP_MAIN(PID=$tradePortalPID)..." + kill $tradePortalPID > /dev/null + if [ $? -eq 0 ]; then + echo "[Success]" + echo "===============================================================================================" + else + echo "[Failed]" + echo "===============================================================================================" + fi + getTradeProtalPID + else + echo "$APP_MAIN is not running" + echo "===============================================================================================" + fi +} + +stop diff --git a/java/ppc-pir-services/sdk/build.gradle b/java/ppc-pir-services/sdk/build.gradle new file mode 100644 index 00000000..355c101d --- /dev/null +++ b/java/ppc-pir-services/sdk/build.gradle @@ -0,0 +1,26 @@ +plugins { + id 'java' + id 'com.github.sherter.google-java-format' +} + +repositories { + maven { + url = "https://mirrors.huaweicloud.com/repository/maven/" + } +} + +project.version="1.0.0" + +dependencies { + implementation 'com.squareup.okhttp3:okhttp:4.9.1' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.0' + + compileOnly 'org.projectlombok:lombok:1.18.24' + annotationProcessor 'org.projectlombok:lombok:1.18.24' +} + +googleJavaFormat { + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' +} \ No newline at end of file diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java new file mode 100644 index 00000000..41f5ee8d --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java @@ -0,0 +1,51 @@ +package com.wedpr.pir.demo; + +import com.wedpr.pir.sdk.PirClient; +import com.wedpr.pir.sdk.entity.param.PirJobParam; +import com.wedpr.pir.sdk.entity.response.PirResultResponse; +import com.wedpr.pir.sdk.enums.ParamEnum; +import com.wedpr.pir.sdk.exception.WedprException; +import java.util.Arrays; + +/** + * @author zachma + * @date 2024/8/22 + */ +public class Demo { + public static void main(String[] args) throws WedprException { + testIdObfuscation(); + testIdFilter(); + } + + private static void testIdFilter() throws WedprException { + PirJobParam pirJobParam = new PirJobParam(); + pirJobParam.setJobId("j-12345678912"); + pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); + pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idFilter.getValue()); + pirJobParam.setParticipateAgencyId("1002"); + pirJobParam.setDatasetId("t_login_token"); + pirJobParam.setJobCreator("1001"); + pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); + pirJobParam.setSearchIdList(Arrays.asList(new String[] {"1", "20", "3", "456"})); + + PirClient pirClient = new PirClient(pirJobParam); + PirResultResponse pirResultResponse = pirClient.executePirJob(); + System.out.println(pirResultResponse); + } + + private static void testIdObfuscation() throws WedprException { + PirJobParam pirJobParam = new PirJobParam(); + pirJobParam.setJobId("j-12345678912"); + pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); + pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idObfuscation.getValue()); + pirJobParam.setParticipateAgencyId("1002"); + pirJobParam.setDatasetId("t_login_token"); + pirJobParam.setJobCreator("1001"); + pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); + pirJobParam.setSearchIdList(Arrays.asList(new String[] {"1", "20", "3", "456"})); + + PirClient pirClient = new PirClient(pirJobParam); + PirResultResponse pirResultResponse = pirClient.executePirJob(); + System.out.println(pirResultResponse); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java new file mode 100644 index 00000000..b9e95e29 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java @@ -0,0 +1,72 @@ +package com.wedpr.pir.sdk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.org.slf4j.internal.Logger; +import com.sun.org.slf4j.internal.LoggerFactory; +import com.wedpr.pir.sdk.entity.body.SimpleEntity; +import com.wedpr.pir.sdk.entity.param.PirJobParam; +import com.wedpr.pir.sdk.entity.request.ServerJobRequest; +import com.wedpr.pir.sdk.entity.response.ClientOTResponse; +import com.wedpr.pir.sdk.entity.response.PirResultResponse; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import com.wedpr.pir.sdk.gateway.GatewayClient; +import com.wedpr.pir.sdk.service.PirJobService; +import java.util.Objects; + +/** + * @author zachma + * @date 2024/8/19 + */ +public class PirClient { + private static final Logger logger = LoggerFactory.getLogger(PirClient.class); + private final PirJobService pirJobService; + + public PirClient(PirJobParam pirJobParam) throws WedprException { + pirJobParam.check(); + this.pirJobService = new PirJobService(pirJobParam); + } + + public PirResultResponse executePirJob() throws WedprException { + try { + ClientOTResponse otParamResponse = pirJobService.requesterOtCipher(); + PirJobParam pirJobParam = pirJobService.getPirJobParam(); + ServerJobRequest serverJobRequest = + generatePirJobRequestFromJobAndOTParam(pirJobParam, otParamResponse); + logger.debug("executePirJob start: ", serverJobRequest); + + String pirUrl = pirJobParam.getGatewayUrl(); + ObjectMapper objectMapper = new ObjectMapper(); + String body = objectMapper.writeValueAsString(serverJobRequest); + SimpleEntity otResult = + GatewayClient.sendPostRequestWithRetry(pirUrl, body, SimpleEntity.class); + logger.debug("executePirJob success return: ", otResult); + if (Objects.isNull(otResult) + || !otResult.getCode().equals(WedprStatusEnum.SUCCESS.getCode())) { + throw new WedprException(WedprStatusEnum.TASK_EXECUTE_ERROR); + } + return pirJobService.requesterOtRecover(otParamResponse, pirJobParam, otResult); + } catch (Exception e) { + logger.error("executePirJob cause exception: ", e.getMessage()); + throw new WedprException(WedprStatusEnum.CALL_PSI_ERROR); + } + } + + /** 根据 ClientOTResponse 和 PirJobParam 构建PirJobRequest */ + private ServerJobRequest generatePirJobRequestFromJobAndOTParam( + PirJobParam pirJobParam, ClientOTResponse otParamResponse) { + ServerJobRequest serverJobRequest = new ServerJobRequest(); + serverJobRequest.setJobId(pirJobParam.getJobId()); + serverJobRequest.setParticipateAgencyId(pirJobParam.getParticipateAgencyId()); + serverJobRequest.setDatasetId(pirJobParam.getDatasetId()); + serverJobRequest.setJobCreator(pirJobParam.getJobCreator()); + serverJobRequest.setJobAlgorithmType(pirJobParam.getJobAlgorithmType()); + serverJobRequest.setX(otParamResponse.getX()); + serverJobRequest.setY(otParamResponse.getY()); + serverJobRequest.setDataBodyList(otParamResponse.getDataBodyList()); + for (int i = 0; i < serverJobRequest.getDataBodyList().size(); i++) { + serverJobRequest.getDataBodyList().get(i).setIdIndex(0); + } + return serverJobRequest; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java new file mode 100644 index 00000000..1aed44d6 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java @@ -0,0 +1,51 @@ +package com.wedpr.pir.sdk.crypto; + +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * @author zachma + * @date 2024/8/20 + */ +public class IdHashVec { + public static List getIdHashVec(int obfuscationOrder, Integer idIndex, String searchId) + throws WedprException { + + List clientDataBodyList = new ArrayList<>(); + String searchIdTemp; + for (int i = 0; i < obfuscationOrder + 1; i++) { + if (idIndex == i) { + searchIdTemp = searchId; + } else { + String uuid = UUID.randomUUID().toString(); + searchIdTemp = makeHash(uuid.getBytes()); + } + clientDataBodyList.add(searchIdTemp); + } + return clientDataBodyList; + } + + private static String makeHash(byte[] data) throws WedprException { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(data); + + StringBuilder hexString = new StringBuilder(); + for (byte b : hashBytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + throw new WedprException(WedprStatusEnum.WRONG_HASH_ERROR); + } + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java new file mode 100644 index 00000000..0bf38256 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java @@ -0,0 +1,17 @@ +package com.wedpr.pir.sdk.entity.body; + +import java.math.BigInteger; +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +public class PirDataBody { + BigInteger z0; + String filter; + int idIndex; + List idHashList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java new file mode 100644 index 00000000..c8bf5524 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java @@ -0,0 +1,14 @@ +package com.wedpr.pir.sdk.entity.body; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/21 + */ +@Data +public class PirResultBody { + String searchId; + Boolean searchExist; + String searchValue; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java new file mode 100644 index 00000000..f71b9896 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java @@ -0,0 +1,19 @@ +package com.wedpr.pir.sdk.entity.body; + +import java.math.BigInteger; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ServerResultBody { + BigInteger e; + BigInteger w; + String c; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java new file mode 100644 index 00000000..3f448180 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java @@ -0,0 +1,17 @@ +package com.wedpr.pir.sdk.entity.body; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ServerResultList { + List resultBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java new file mode 100644 index 00000000..7726c772 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java @@ -0,0 +1,15 @@ +package com.wedpr.pir.sdk.entity.body; + +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/21 + */ +@Data +public class SimpleEntity { + String code; + String message; + ServerOTResponse data; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java new file mode 100644 index 00000000..1812d9c4 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java @@ -0,0 +1,65 @@ +package com.wedpr.pir.sdk.entity.param; + +import com.wedpr.pir.sdk.entity.request.PirBaseRequest; +import com.wedpr.pir.sdk.enums.ParamEnum; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import com.wedpr.pir.sdk.helper.BasicTypeHelper; +import java.util.Arrays; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author zachma + * @date 2024/8/19 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class PirJobParam extends PirBaseRequest { + + private Integer filterLength = 4; + + private String gatewayUrl; + + private List searchIdList; + + public void check() throws WedprException { + if (BasicTypeHelper.isStringEmpty(gatewayUrl)) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "访问gatewayUrl不能为空"); + } + + if (BasicTypeHelper.isStringEmpty(getJobId())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobId 不能为空"); + } + + if (Arrays.stream(ParamEnum.JobType.values()) + .noneMatch(enumValue -> enumValue.getValue().equals(getJobType()))) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobType输入错误"); + } + + if (BasicTypeHelper.isStringEmpty(getParticipateAgencyId())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "数据机构Id 不能为空"); + } + + if (BasicTypeHelper.isStringEmpty(getDatasetId())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "datasetId 不能为空"); + } + + if (Arrays.stream(ParamEnum.AlgorithmType.values()) + .noneMatch(enumValue -> enumValue.getValue().equals(getJobAlgorithmType()))) { + throw new WedprException( + WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobAlgorithmType输入错误"); + } + + if (searchIdList == null || searchIdList.size() == 0) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "searchId列表不能为空"); + } + + if (getJobAlgorithmType().equals(ParamEnum.AlgorithmType.idObfuscation.getValue()) + && getObfuscationOrder() == -1) { + throw new WedprException( + WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "哈希披露算法中, obfuscationOrder未设置"); + } + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java new file mode 100644 index 00000000..7a3ecfb3 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java @@ -0,0 +1,17 @@ +package com.wedpr.pir.sdk.entity.request; + +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import java.math.BigInteger; +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/21 + */ +@Data +public class ClientDecryptRequest { + BigInteger b; + List dataBodyList; + ServerOTResponse serverResult; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java new file mode 100644 index 00000000..b4d54dcb --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java @@ -0,0 +1,17 @@ +package com.wedpr.pir.sdk.entity.request; + +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/20 + */ +@Data +public class ClientOTRequest { + int filterLength = 4; + + int obfuscationOrder; + + List dataBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java new file mode 100644 index 00000000..028db290 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java @@ -0,0 +1,21 @@ +package com.wedpr.pir.sdk.entity.request; + +import lombok.Data; + +@Data +public class PirBaseRequest { + // 任务ID + String jobId; + // 任务类型(0: 查询存在性, 1: 查询对应值) + String jobType; + // 数据方机构id + String participateAgencyId; + // 数据集id + String datasetId; + // 发起方用户名 + String jobCreator; + // 匿踪算法类型(0: hash披露算法, 1: hash混淆算法) + String jobAlgorithmType; + // 查询范围 + Integer obfuscationOrder = 9; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java new file mode 100644 index 00000000..0f91aa94 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java @@ -0,0 +1,19 @@ +package com.wedpr.pir.sdk.entity.request; + +import com.wedpr.pir.sdk.entity.body.PirDataBody; +import java.math.BigInteger; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author zachma + * @date 2024/8/18 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class ServerJobRequest extends PirBaseRequest { + BigInteger x; + BigInteger y; + List dataBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java new file mode 100644 index 00000000..a1ad74de --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java @@ -0,0 +1,19 @@ +package com.wedpr.pir.sdk.entity.request; + +import com.wedpr.pir.sdk.entity.body.PirDataBody; +import java.math.BigInteger; +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +public class ServerOTRequest { + String jobType; + String datasetId; + BigInteger x; + BigInteger y; + List dataBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java new file mode 100644 index 00000000..cf720cb8 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java @@ -0,0 +1,14 @@ +package com.wedpr.pir.sdk.entity.response; + +import com.wedpr.pir.sdk.entity.body.PirResultBody; +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/21 + */ +@Data +public class ClientDecryptResponse { + List detail; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java new file mode 100644 index 00000000..d367cf54 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java @@ -0,0 +1,26 @@ +package com.wedpr.pir.sdk.entity.response; + +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +public class ClientJobResponse { + private String code; + private String message; + private Object data; + + public ClientJobResponse(WedprStatusEnum wedprStatusEnum) { + this.code = wedprStatusEnum.getCode(); + this.message = wedprStatusEnum.getMessage(); + } + + public ClientJobResponse(WedprStatusEnum wedprStatusEnum, Object data) { + this.code = wedprStatusEnum.getCode(); + this.message = wedprStatusEnum.getMessage(); + this.data = data; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java new file mode 100644 index 00000000..a26ce00b --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java @@ -0,0 +1,20 @@ +package com.wedpr.pir.sdk.entity.response; + +import com.wedpr.pir.sdk.entity.body.PirDataBody; +import java.math.BigInteger; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/20 + */ +@Data +@AllArgsConstructor +public class ClientOTResponse { + BigInteger b; + BigInteger x; + BigInteger y; + List dataBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java new file mode 100644 index 00000000..fd78b0df --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java @@ -0,0 +1,15 @@ +package com.wedpr.pir.sdk.entity.response; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/21 + */ +@Data +public class PirResultResponse { + String jobId; + String jobType; + // List detail; + ClientDecryptResponse detail; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java new file mode 100644 index 00000000..45d0a312 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java @@ -0,0 +1,18 @@ +package com.wedpr.pir.sdk.entity.response; + +import com.wedpr.pir.sdk.entity.body.ServerResultList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ServerOTResponse { + List resultBodyList; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java new file mode 100644 index 00000000..f3a2b321 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java @@ -0,0 +1,35 @@ +package com.wedpr.pir.sdk.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author zachma + * @date 2024/8/18 + */ +public class ParamEnum { + @Getter + @AllArgsConstructor + public enum JobType { + SearchValue("0"), + SearchExist("1"); + private String value; + } + + @Getter + @AllArgsConstructor + public enum AlgorithmType { + idFilter("0"), + idObfuscation("1"); + + private String value; + } + + @Getter + @AllArgsConstructor + public enum HttpType { + HttpTimeout(180), + HttpMaxRetries(4); + private Integer value; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java new file mode 100644 index 00000000..3565318d --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java @@ -0,0 +1,23 @@ +package com.wedpr.pir.sdk.exception; + +/** + * @author zachma + * @date 2024/8/18 + */ +public class WedprException extends Exception { + private final WedprStatusEnum status; + + public WedprException(WedprStatusEnum status) { + super(status.getMessage()); + this.status = status; + } + + public WedprException(WedprStatusEnum status, String message) { + super(message); + this.status = status; + } + + public WedprStatusEnum getStatus() { + return status; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java new file mode 100644 index 00000000..f1070a74 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java @@ -0,0 +1,72 @@ +package com.wedpr.pir.sdk.exception; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author zachma + * @date 2024/8/18 + */ +@AllArgsConstructor +@Getter +public enum WedprStatusEnum { + SUCCESS("0", "成功"), + FAILURE("1", "系统内部错误"), + INTERNAL_SERVER_ERROR("3", "系统内部错误"), + SYSTEM_CONFIG_ERROR("4", "系统配置参数错误"), + HANDLE_FILE_ERROR("5", "处理文件错误"), + TASK_EXECUTE_ERROR("6", "任务执行失败"), + + // 1000 未定义错误 + SYSTEM_EXCEPTION("1000", "系统抛出异常，请检查日志记录"), + CLIENT_PARAM_EXCEPTION("1001", "发起方参数错误"), + + // 2000 参数解析错误 + INVALID_INPUT_VALUE("2000", "输入参数异常"), + JSON_ERROR("2001", "输入参数JSON解析失败"), + INVALID_SYSTEM_KEY("2002", "系统密钥解析失败"), + INVALID_CUSTOMER_TYPE("2003", "未定义的客户类型"), + INVALID_HASH_VALUE("2004", "输入的哈希值不合法"), + INVALID_INPUT_LENGTH("2005", "输入参数列表长度不合法"), + INVALID_INPUT_PREFIX("2006", "输入参数前缀不匹配"), + INVALID_TIMESTAMP_VALUE("2007", "输入时间戳不合法"), + JSON_GET_ERROR("2008", "JSON解析返回值失败"), + INVALID_TRANS_TYPE("2009", "不合法的交易类型"), + INVALID_FILE_PATH("2010", "不合法交集文件路径"), + INVALID_ROLE_VALUE("2011", "pir.Role设置错误"), + + // 3000 db错误 + DB_ERROR("3000", "DB查询失败"), + // DB_TIMEOUT("3000", "DB访问超时"), + DB_INSERT_FAILURE("3001", "DB插入失败"), + DB_DELETE_FAILURE("3002", "DB删除失败"), + UNENCRYPTED_DATA("3003", "查询的数据未被加密"), + DB_MISSED_USER("3004", "无法写入未命中用户"), + + // 4000 被请求服务端异常 + SERVICE_UNAVAILABLE("4000", "对方PIR服务访问失败"), + INVALID_ENCRYPTION_KEY("4001", "服务端加密密钥异常"), + SEARCHED_DB_UNAVAILABLE("4002", "服务端db无法访问"), + HTTP_CALL_ERROR("4003", "HTTP调用异常"), + CALL_PSI_ERROR("4004", "调用PIR服务异常"), + UPLOAD_FPS_ERROR("4005", "上传文件至Fps失败"), + DOWNLOAD_FPS_ERROR("4006", "下载Fps文件失败"), + CLIENT_LOCAL_FILE("4007", "本地初始化./client/local.txt没有设置"), + SERVER_LOCAL_FILE("4008", "本地初始化./server/local.txt没有设置"), + + // 5000 密码学运算错误 + AES_ENCRYPT_ERROR("5000", "AES加密明文信息错误"), + AES_DECRYPT_ERROR("5001", "AES解密密文信息错误"), + MATCH_RESULT_ERROR("5002", "找不到匹配结果查询值"), + ENCODE_BIG_NUMBER_ERROR("5003", "转换输入哈希到BIG NUMBER类型失败"), + DECODE_ECC_POINT_ERROR("5004", "解码密文到椭圆曲线上点失败"), + MATCH_CIPHER_ERROR("5005", "计算匹配结果失败"), + CLIENT_CIPHER_ERROR("5006", "计算客户端密文失败"), + CIPHER_LENGTH_ERROR("5007", "根据前缀索引到的密文太多"), + WRONG_HASH_ERROR("5008", "输入哈希算法错误"), + + CRYPTO_SPE_ERROR("5099", "密码运算通用错误"); + + private String code; + private String message; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java new file mode 100644 index 00000000..ce58e8e2 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java @@ -0,0 +1,80 @@ +package com.wedpr.pir.sdk.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.wedpr.pir.sdk.enums.ParamEnum; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * @author zachma + * @date 2024/8/20 + */ +public class GatewayClient { + + private static final MediaType JSON_MEDIA_TYPE = + MediaType.get("application/json; charset=utf-8"); + + private static String sendGetRequest(String url) throws IOException, WedprException { + int timeout = ParamEnum.HttpType.HttpTimeout.getValue(); + OkHttpClient httpClient = + new OkHttpClient() + .newBuilder() + .connectTimeout(timeout, TimeUnit.SECONDS) + .readTimeout(timeout, TimeUnit.SECONDS) + .writeTimeout(timeout, TimeUnit.SECONDS) + .build(); + + Request getRequest = new Request.Builder().url(url).get().build(); + + Call call = httpClient.newCall(getRequest); + Response response = call.execute(); + String responseBody = ""; + if (response.isSuccessful() && Objects.nonNull(response.body())) { + responseBody = response.body().string(); + } else { + throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); + } + return responseBody; + } + + private static Response sendPostRequest(String url, String jsonBody) throws IOException { + OkHttpClient httpClient = new OkHttpClient().newBuilder().build(); + RequestBody body = RequestBody.create(jsonBody, JSON_MEDIA_TYPE); + Request request = new Request.Builder().post(body).url(url).build(); + Call call = httpClient.newCall(request); + return call.execute(); + } + + public static T sendPostRequestWithRetry(String url, String body, Class responseType) + throws WedprException { + int maxRetries = ParamEnum.HttpType.HttpMaxRetries.getValue(); + int retries = 1; + while (retries <= maxRetries) { + try { + Response response = sendPostRequest(url, body); + if (response.isSuccessful() && Objects.nonNull(response.body())) { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(response.body().string(), responseType); + } else { + throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); + } + + } catch (Exception e) { + retries++; + if (retries > maxRetries) { + throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); + } + } + } + return null; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java new file mode 100644 index 00000000..8288ccb9 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java @@ -0,0 +1,76 @@ +package com.wedpr.pir.sdk.helper; + +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Base64; +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +/** + * @author zachma + * @date 2024/8/21 + */ +public class AESHelper { + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + + private static final String AES_ALGORITHM_TRANSFORMATION = "AES/GCM/NoPadding"; + + public static String generateRandomKey() { + // 随机生成 16 位字符串格式的密钥 + byte[] keyBytes = new byte[16]; + SECURE_RANDOM.nextBytes(keyBytes); + return Base64.getEncoder().encodeToString(keyBytes); + } + + public static String encrypt(String plainText, String keyString) throws WedprException { + try { + byte[] iv = new byte[12]; + SecureRandom secureRandom = new SecureRandom(); + secureRandom.nextBytes(iv); + byte[] contentBytes = plainText.getBytes(StandardCharsets.UTF_8); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM_TRANSFORMATION); + GCMParameterSpec params = new GCMParameterSpec(128, iv); + cipher.init(Cipher.ENCRYPT_MODE, getSecretKey(keyString), params); + byte[] encryptData = cipher.doFinal(contentBytes); + assert encryptData.length == contentBytes.length + 16; + byte[] message = new byte[12 + contentBytes.length + 16]; + System.arraycopy(iv, 0, message, 0, 12); + System.arraycopy(encryptData, 0, message, 12, encryptData.length); + return Base64.getEncoder().encodeToString(message); + } catch (Exception e) { + throw new WedprException(WedprStatusEnum.AES_ENCRYPT_ERROR); + } + } + + public static String decrypt(String encryptedText, String keyString) throws WedprException { + try { + byte[] content = Base64.getDecoder().decode(encryptedText); + if (content.length < 12 + 16) { + throw new IllegalArgumentException(); + } + GCMParameterSpec params = new GCMParameterSpec(128, content, 0, 12); + Cipher cipher = Cipher.getInstance(AES_ALGORITHM_TRANSFORMATION); + cipher.init(Cipher.DECRYPT_MODE, getSecretKey(keyString), params); + byte[] decryptData = cipher.doFinal(content, 12, content.length - 12); + return new String(decryptData, StandardCharsets.UTF_8); + } catch (Exception e) { + throw new WedprException(WedprStatusEnum.AES_DECRYPT_ERROR); + } + } + + private static SecretKeySpec getSecretKey(String key) throws NoSuchAlgorithmException { + KeyGenerator kg = KeyGenerator.getInstance("AES"); + // 初始化密钥生成器，AES要求密钥长度为128位、192位、256位 + SecureRandom secureRandom = SecureRandom.getInstance("SHA1PRNG"); + secureRandom.setSeed(key.getBytes(StandardCharsets.UTF_8)); + kg.init(128, secureRandom); + SecretKey secretKey = kg.generateKey(); + return new SecretKeySpec(secretKey.getEncoded(), "AES"); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java new file mode 100644 index 00000000..3e8aebc0 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java @@ -0,0 +1,15 @@ +package com.wedpr.pir.sdk.helper; + +/** + * @author zachma + * @date 2024/8/21 + */ +public class BasicTypeHelper { + public static boolean isStringEmpty(String str) { + return str == null || str.length() == 0; + } + + public static boolean isStringNotEmpty(String str) { + return !isStringEmpty(str); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java new file mode 100644 index 00000000..6e5728fd --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java @@ -0,0 +1,44 @@ +package com.wedpr.pir.sdk.helper; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * @author zachma + * @date 2024/8/19 + */ +public class ConvertTypeHelper { + public static int bytesToInt(byte[] bytes, ByteOrder byteOrder) { + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(byteOrder); + return buffer.getInt(); + } + + public static byte[] intToBytes(int number, ByteOrder byteOrder) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); + buffer.putInt(number); + return buffer.array(); + } + + public static BigInteger bytesToBigInteger(byte[] bytes) { + return new BigInteger(1, bytes); + } + + public static byte[] bigIntegerToBytes(BigInteger number) { + byte[] bytes = number.toByteArray(); + if (bytes[0] == 0) { + byte[] tmpBytes = new byte[bytes.length - 1]; + System.arraycopy(bytes, 1, tmpBytes, 0, tmpBytes.length); + return tmpBytes; + } + return bytes; + } + + public static String bytesToHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02X ", b)); + } + return sb.toString().trim(); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java new file mode 100644 index 00000000..187cfcb7 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java @@ -0,0 +1,51 @@ +package com.wedpr.pir.sdk.helper; + +import java.math.BigInteger; +import java.security.SecureRandom; + +/** + * @author zachma + * @date 2024/8/19 + */ +public class CryptoOperatorHelper { + private static final BigInteger DEFAULT_G = + new BigInteger( + "9020881489161854992071763483314773468341853433975756385639545080944698236944020124874820917267762049756743282301106459062535797137327360192691469027152272"); + private static final BigInteger DEFAULT_N = + new BigInteger( + "102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885212534669604964926915306738569799518792945024759514373214412797317972739022405456550476153212687312211184540248262330559143446510677062823907392904449451177"); + private static final BigInteger DEFAULT_FI = + new BigInteger( + "102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885192126363163670343672910761259882348623401714459980712242233796355982147797162316532450768783823909695360736554767341443201861573989081253763975895939627220"); + + private static final SecureRandom RANDOM = new SecureRandom(); + + /** 生成随机数 * */ + public static BigInteger getRandomInt() { + BigInteger num = new BigInteger(DEFAULT_N.bitLength(), RANDOM); + while (num.compareTo(DEFAULT_N) >= 0) { + num = new BigInteger(DEFAULT_N.bitLength(), RANDOM); + } + return num; + } + + /** b*G mod N */ + public static BigInteger powMod(BigInteger b) { + return DEFAULT_G.modPow(b, DEFAULT_N); + } + + /** a^b mod N */ + public static BigInteger OTPow(BigInteger a, BigInteger b) { + return a.modPow(b, DEFAULT_N); + } + + /** a*b mod FI */ + public static BigInteger mulMod(BigInteger a, BigInteger b) { + return a.multiply(b).mod(DEFAULT_FI); + } + + /** a*b mod N */ + public static BigInteger OTMul(BigInteger a, BigInteger b) { + return a.multiply(b).mod(DEFAULT_N); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java new file mode 100644 index 00000000..6e1d2f2a --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java @@ -0,0 +1,69 @@ +package com.wedpr.pir.sdk.service; + +import com.sun.org.slf4j.internal.Logger; +import com.sun.org.slf4j.internal.LoggerFactory; +import com.wedpr.pir.sdk.entity.body.PirResultBody; +import com.wedpr.pir.sdk.entity.body.ServerResultBody; +import com.wedpr.pir.sdk.entity.request.ClientDecryptRequest; +import com.wedpr.pir.sdk.entity.response.ClientDecryptResponse; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.helper.AESHelper; +import com.wedpr.pir.sdk.helper.BasicTypeHelper; +import com.wedpr.pir.sdk.helper.ConvertTypeHelper; +import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * @author zachma + * @date 2024/8/21 + */ +public class ClientDecryptService { + private static final Logger logger = LoggerFactory.getLogger(ClientDecryptService.class); + private void decryptServerResultList( + PirResultBody pirResultBody, List serverResultList, BigInteger b) { + for (ServerResultBody body : serverResultList) { + BigInteger e = body.getE(); + BigInteger w = body.getW(); + String cipherStr = body.getC(); + BigInteger w1 = CryptoOperatorHelper.OTPow(w, b); + try { + // 对整数AES密钥OT解密，报错后不处理 + BigInteger messageRecover = w1.xor(e); + byte[] convertedBytes = ConvertTypeHelper.bigIntegerToBytes(messageRecover); + String convertedStr = new String(convertedBytes, StandardCharsets.UTF_8); + String decryptedText = AESHelper.decrypt(cipherStr, convertedStr); + pirResultBody.setSearchValue(decryptedText); + } catch (WedprException ignored) { + + } + } + } + + protected ClientDecryptResponse runDecryptOTparam(ClientDecryptRequest clientDecryptRequest) { + logger.debug("runDecryptOTparam Start: ", clientDecryptRequest); + List pirResultBodyArrayList = new ArrayList<>(); + for (int i = 0; + i < clientDecryptRequest.getServerResult().getResultBodyList().size(); + i++) { + PirResultBody pirResultBody = new PirResultBody(); + pirResultBody.setSearchId(clientDecryptRequest.getDataBodyList().get(i)); + List serverResultlist = + clientDecryptRequest + .getServerResult() + .getResultBodyList() + .get(i) + .getResultBodyList(); + decryptServerResultList(pirResultBody, serverResultlist, clientDecryptRequest.getB()); + pirResultBody.setSearchExist( + BasicTypeHelper.isStringNotEmpty(pirResultBody.getSearchValue())); + pirResultBodyArrayList.add(pirResultBody); + } + logger.debug("runDecryptOTparam Success: ", pirResultBodyArrayList); + ClientDecryptResponse clientDecryptResponse = new ClientDecryptResponse(); + clientDecryptResponse.setDetail(pirResultBodyArrayList); + return clientDecryptResponse; + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java new file mode 100644 index 00000000..63d096df --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java @@ -0,0 +1,94 @@ +package com.wedpr.pir.sdk.service; + +import com.sun.org.slf4j.internal.Logger; +import com.sun.org.slf4j.internal.LoggerFactory; +import com.wedpr.pir.sdk.crypto.IdHashVec; +import com.wedpr.pir.sdk.entity.body.PirDataBody; +import com.wedpr.pir.sdk.entity.request.ClientOTRequest; +import com.wedpr.pir.sdk.entity.response.ClientOTResponse; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.helper.ConvertTypeHelper; +import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * @author zachma + * @date 2024/8/20 + */ +public class ClientOTService { + private static final Logger logger = LoggerFactory.getLogger(ClientOTService.class); + private final Random rand = new Random(); + + /* hash披露, 请求方选择id，生成随机数a、b */ + protected ClientOTResponse runClientOTParam(ClientOTRequest clientOTRequest) { + logger.debug("runClientOTParam start: ", clientOTRequest); + int filterLength = clientOTRequest.getFilterLength(); + BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); + BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); + + BigInteger x = CryptoOperatorHelper.powMod(blindingA); + BigInteger y = CryptoOperatorHelper.powMod(blindingB); + BigInteger blindingC = CryptoOperatorHelper.mulMod(blindingA, blindingB); + + List pirDataArrayList = new ArrayList<>(); + for (String searchId : clientOTRequest.getDataBodyList()) { + String filter = + searchId.length() < filterLength + ? searchId + : searchId.substring(0, filterLength); + BigInteger z0 = calculateZ0(searchId, blindingC); + PirDataBody pirDataBody = new PirDataBody(); + pirDataBody.setFilter(filter); + pirDataBody.setZ0(z0); + pirDataArrayList.add(pirDataBody); + } + logger.debug("runClientOTParam success: ", pirDataArrayList); + return new ClientOTResponse(blindingB, x, y, pirDataArrayList); + } + + /* hash筛选, 请求方选择顺序\delta\in \{0,1,..,m-1\}，生成随机数a、b */ + protected ClientOTResponse runClientOTCipher(ClientOTRequest clientOTRequest) + throws WedprException { + logger.debug("runClientOTCipher start: ", clientOTRequest); + + int obfuscationOrder = clientOTRequest.getObfuscationOrder(); + BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); + BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); + + BigInteger x = CryptoOperatorHelper.powMod(blindingA); + BigInteger y = CryptoOperatorHelper.powMod(blindingB); + BigInteger blindingC = CryptoOperatorHelper.mulMod(blindingA, blindingB); + + List pirDataArrayList = new ArrayList<>(); + for (String searchId : clientOTRequest.getDataBodyList()) { + int idIndex = rand.nextInt(obfuscationOrder + 1); + BigInteger z0 = calculateIndexZ0(idIndex, blindingC); + List idHashVecList = + IdHashVec.getIdHashVec(obfuscationOrder, idIndex, searchId); + + PirDataBody pirDataBody = new PirDataBody(); + pirDataBody.setZ0(z0); + pirDataBody.setIdIndex(idIndex); + pirDataBody.setIdHashList(idHashVecList); + pirDataArrayList.add(pirDataBody); + } + logger.debug("runClientOTCipher success: ", pirDataArrayList); + return new ClientOTResponse(blindingB, x, y, pirDataArrayList); + } + + private BigInteger calculateZ0(String searchId, BigInteger blindingC) { + byte[] idBytes = searchId.getBytes(StandardCharsets.UTF_8); + BigInteger idNumber = ConvertTypeHelper.bytesToBigInteger(idBytes); + return CryptoOperatorHelper.powMod(blindingC.subtract(idNumber)); + } + + private BigInteger calculateIndexZ0(Integer idIndex, BigInteger blindingC) { + // 将整数转长整数 + BigInteger idNumber = BigInteger.valueOf(idIndex); + return CryptoOperatorHelper.powMod(blindingC.subtract(idNumber)); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java new file mode 100644 index 00000000..5d6c0e1d --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java @@ -0,0 +1,64 @@ +package com.wedpr.pir.sdk.service; + +import com.wedpr.pir.sdk.entity.body.SimpleEntity; +import com.wedpr.pir.sdk.entity.param.PirJobParam; +import com.wedpr.pir.sdk.entity.request.ClientDecryptRequest; +import com.wedpr.pir.sdk.entity.request.ClientOTRequest; +import com.wedpr.pir.sdk.entity.response.ClientDecryptResponse; +import com.wedpr.pir.sdk.entity.response.ClientOTResponse; +import com.wedpr.pir.sdk.entity.response.PirResultResponse; +import com.wedpr.pir.sdk.enums.ParamEnum; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import lombok.Getter; + +/** + * @author zachma + * @date 2024/8/20 + */ +public class PirJobService { + + @Getter private final PirJobParam pirJobParam; + private final ClientOTService clientOTService; + private final ClientDecryptService clientDecryptService; + + public PirJobService(PirJobParam pirJobParam) { + this.pirJobParam = pirJobParam; + this.clientOTService = new ClientOTService(); + this.clientDecryptService = new ClientDecryptService(); + } + + public ClientOTResponse requesterOtCipher() throws WedprException { + ClientOTRequest clientOTRequest = new ClientOTRequest(); + ClientOTResponse otParamResponse; + clientOTRequest.setDataBodyList(pirJobParam.getSearchIdList()); + if (pirJobParam.getJobAlgorithmType().equals(ParamEnum.AlgorithmType.idFilter.getValue())) { + clientOTRequest.setFilterLength(pirJobParam.getFilterLength()); + otParamResponse = clientOTService.runClientOTParam(clientOTRequest); + } else if (pirJobParam + .getJobAlgorithmType() + .equals(ParamEnum.AlgorithmType.idObfuscation.getValue())) { + otParamResponse = clientOTService.runClientOTCipher(clientOTRequest); + } else { + throw new WedprException(WedprStatusEnum.INVALID_INPUT_VALUE); + } + return otParamResponse; + } + + public PirResultResponse requesterOtRecover( + ClientOTResponse otParamResponse, PirJobParam clientJobRequest, SimpleEntity otResult) { + ClientDecryptRequest clientDecryptRequest = new ClientDecryptRequest(); + clientDecryptRequest.setB(otParamResponse.getB()); + clientDecryptRequest.setDataBodyList(clientJobRequest.getSearchIdList()); + clientDecryptRequest.setServerResult(otResult.getData()); + + ClientDecryptResponse clientDecryptResponse = + clientDecryptService.runDecryptOTparam(clientDecryptRequest); + + PirResultResponse pirResultResponse = new PirResultResponse(); + pirResultResponse.setJobId(clientJobRequest.getJobId()); + pirResultResponse.setJobType(clientJobRequest.getJobType()); + pirResultResponse.setDetail(clientDecryptResponse); + return pirResultResponse; + } +} diff --git a/java/ppc-pir-services/settings.gradle b/java/ppc-pir-services/settings.gradle new file mode 100644 index 00000000..301af393 --- /dev/null +++ b/java/ppc-pir-services/settings.gradle @@ -0,0 +1,11 @@ +pluginManagement { + repositories { + gradlePluginPortal() + } +} + +rootProject.name = "ppc-pir-services" + +include ':ppc-pir-sdk' +project(":ppc-pir-sdk").projectDir = file("sdk") + diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java new file mode 100644 index 00000000..9d016653 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java @@ -0,0 +1,18 @@ +package com.webank.wedpr.pir; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * @author zachma + * @date 2024/8/18 + */ +@SpringBootApplication +@EnableAutoConfiguration +public class WedprPirApplication { + public static void main(String[] args) { + SpringApplication.run(WedprPirApplication.class, args); + System.out.println("Start WedprPirApplication Server successfully!"); + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java new file mode 100644 index 00000000..d153c4a8 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java @@ -0,0 +1,17 @@ +package com.webank.wedpr.pir.crypto.entity; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/8/19 + */ +@Data +public class PirTable { + + private Integer id; + + private String pirKey; + + private String pirValue; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java new file mode 100644 index 00000000..68be68e0 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java @@ -0,0 +1,25 @@ +package com.webank.wedpr.pir.crypto.service; + +import com.wedpr.pir.sdk.entity.request.ServerOTRequest; +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import com.wedpr.pir.sdk.exception.WedprException; + +/** + * @author zachma + * @date 2024/8/19 + */ +public interface PirOTService { + /** + * @param serverOTRequest + * @return PirOTResponse + * @throws WedprException + */ + ServerOTResponse runServerOTParam(ServerOTRequest serverOTRequest) throws WedprException; + + /** + * @param serverOTRequest + * @return PirOTResponse + * @throws WedprException + */ + ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) throws WedprException; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java new file mode 100644 index 00000000..a70db535 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java @@ -0,0 +1,21 @@ +package com.webank.wedpr.pir.crypto.service; + +import com.webank.wedpr.pir.crypto.entity.PirTable; +import java.util.List; + +/** + * @author zachma + * @date 2024/8/19 + */ +public interface PirTableService { + /** 根据id查询 */ + List idFilterTable(String datasetId, String filter); + + List idFilterTable(String datasetId, String filter, String[] params); + + List idObfuscationTable(String datasetId, String searchId); + + List idObfuscationTable(String datasetId, String searchId, String[] params); + + List objectsToPirTableList(List params); +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java new file mode 100644 index 00000000..a61fa4c3 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java @@ -0,0 +1,139 @@ +package com.webank.wedpr.pir.crypto.service.impl; + +import com.webank.wedpr.pir.crypto.entity.PirTable; +import com.webank.wedpr.pir.crypto.service.PirOTService; +import com.webank.wedpr.pir.crypto.service.PirTableService; +import com.wedpr.pir.sdk.entity.body.ServerResultBody; +import com.wedpr.pir.sdk.entity.body.ServerResultList; +import com.wedpr.pir.sdk.entity.request.ServerOTRequest; +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.helper.AESHelper; +import com.wedpr.pir.sdk.helper.ConvertTypeHelper; +import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Service +public class PirOTServiceImpl implements PirOTService { + private static final Logger logger = LoggerFactory.getLogger(PirOTServiceImpl.class); + @Autowired PirTableService pirTableService; + + public ServerOTResponse runServerOTParam(ServerOTRequest serverOTRequest) + throws WedprException { + logger.info("Server start runServerOTParam."); + String datasetId = serverOTRequest.getDatasetId(); + BigInteger x = serverOTRequest.getX(); + BigInteger y = serverOTRequest.getY(); + List serverResultlist = new ArrayList<>(); + for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { + BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); + String filter = serverOTRequest.getDataBodyList().get(i).getFilter(); + List objects = pirTableService.idFilterTable(datasetId, filter); + List pirTables = pirTableService.objectsToPirTableList(objects); + List serverResultBodyTempList = new ArrayList<>(); + for (PirTable pirTable : pirTables) { + ServerResultBody serverResultBody = + cipherPirTableContentIdSearch(pirTable, x, y, z0); + serverResultBodyTempList.add(serverResultBody); + } + serverResultlist.add(new ServerResultList(serverResultBodyTempList)); + } + logger.info("Server runServerOTParam success"); + return new ServerOTResponse(serverResultlist); + } + + public ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) + throws WedprException { + logger.info("Server start runServerOTCipher."); + String datasetId = serverOTRequest.getDatasetId(); + BigInteger x = serverOTRequest.getX(); + BigInteger y = serverOTRequest.getY(); + + List serverResultlist = new ArrayList<>(); + for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { + BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); + List idHashList = serverOTRequest.getDataBodyList().get(i).getIdHashList(); + List serverResultBodyTempList = new ArrayList<>(); + + for (int j = 0; j < idHashList.size(); j++) { + List objects = + pirTableService.idObfuscationTable(datasetId, idHashList.get(j)); + List pirTables = pirTableService.objectsToPirTableList(objects); + for (PirTable pirTable : pirTables) { + ServerResultBody serverResultBody = + cipherPirTableContentIdObfuscation(pirTable, x, y, z0, j); + serverResultBodyTempList.add(serverResultBody); + } + } + serverResultlist.add(new ServerResultList(serverResultBodyTempList)); + } + logger.info("Server runServerOTCipher success."); + return new ServerOTResponse(serverResultlist); + } + + private ServerResultBody cipherPirTableContentIdSearch( + PirTable pirTable, BigInteger x, BigInteger y, BigInteger z0) throws WedprException { + String uid = pirTable.getPirKey(); + String message = pirTable.getPirValue(); + byte[] uidBytes = uid.getBytes(StandardCharsets.UTF_8); + BigInteger uidNum = ConvertTypeHelper.bytesToBigInteger(uidBytes); + + BigInteger blindingR = CryptoOperatorHelper.getRandomInt(); + BigInteger blindingS = CryptoOperatorHelper.getRandomInt(); + BigInteger w = + CryptoOperatorHelper.OTMul( + CryptoOperatorHelper.OTPow(x, blindingS), + CryptoOperatorHelper.powMod(blindingR)); + BigInteger z1 = CryptoOperatorHelper.OTMul(z0, CryptoOperatorHelper.powMod(uidNum)); + BigInteger key = + CryptoOperatorHelper.OTMul( + CryptoOperatorHelper.OTPow(z1, blindingS), + CryptoOperatorHelper.OTPow(y, blindingR)); + + String aesKey = AESHelper.generateRandomKey(); + BigInteger aesNum = + ConvertTypeHelper.bytesToBigInteger(aesKey.getBytes(StandardCharsets.UTF_8)); + BigInteger messageCipherNum = key.xor(aesNum); + String c = AESHelper.encrypt(message, aesKey); + + return new ServerResultBody(messageCipherNum, w, c); + } + + private ServerResultBody cipherPirTableContentIdObfuscation( + PirTable pirTable, BigInteger x, BigInteger y, BigInteger z0, int index) + throws WedprException { + String message = pirTable.getPirValue(); + BigInteger blindingR = CryptoOperatorHelper.getRandomInt(); + BigInteger blindingS = CryptoOperatorHelper.getRandomInt(); + BigInteger w = + CryptoOperatorHelper.OTMul( + CryptoOperatorHelper.OTPow(x, blindingS), + CryptoOperatorHelper.powMod(blindingR)); + BigInteger z1 = + CryptoOperatorHelper.OTMul( + z0, CryptoOperatorHelper.powMod(BigInteger.valueOf(index))); + BigInteger key = + CryptoOperatorHelper.OTMul( + CryptoOperatorHelper.OTPow(z1, blindingS), + CryptoOperatorHelper.OTPow(y, blindingR)); + + String aesKey = AESHelper.generateRandomKey(); + BigInteger aesNum = + ConvertTypeHelper.bytesToBigInteger(aesKey.getBytes(StandardCharsets.UTF_8)); + BigInteger messageCipherNum = key.xor(aesNum); + String c = AESHelper.encrypt(message, aesKey); + + return new ServerResultBody(messageCipherNum, w, c); + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java new file mode 100644 index 00000000..7d319b5f --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java @@ -0,0 +1,67 @@ +package com.webank.wedpr.pir.crypto.service.impl; + +import com.webank.wedpr.pir.crypto.entity.PirTable; +import com.webank.wedpr.pir.crypto.service.PirTableService; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.Query; +import org.springframework.stereotype.Service; + +/** + * @author zachma + * @date 2024/8/19 + */ +@Service +public class PirTableServiceImpl implements PirTableService { + + @PersistenceContext private EntityManager entityManager; + + @Override + public List idFilterTable(String datasetId, String filter) { + String sqlFormat = "SELECT * FROM %s WHERE id LIKE CONCAT(%s, '%%')"; + String sql = String.format(sqlFormat, datasetId, filter); + Query query = entityManager.createNativeQuery(sql); + return query.getResultList(); + } + + @Override + public List idFilterTable(String datasetId, String filter, String[] params) { + String sqlFormat = "SELECT %s FROM %s WHERE id LIKE CONCAT(%s, '%%')"; + String sql = String.format(sqlFormat, String.join(", ", params), datasetId, filter); + Query query = entityManager.createNativeQuery(sql); + return query.getResultList(); + } + + @Override + public List idObfuscationTable(String datasetId, String searchId) { + String sqlFormat = "SELECT * FROM %s WHERE id = %s"; + String sql = String.format(sqlFormat, datasetId, searchId); + Query query = entityManager.createNativeQuery(sql); + return query.getResultList(); + } + + @Override + public List idObfuscationTable(String datasetId, String searchId, String[] params) { + String sqlFormat = "SELECT %s FROM %s WHERE id = %s"; + String sql = String.format(sqlFormat, String.join(", ", params), datasetId, searchId); + Query query = entityManager.createNativeQuery(sql); + return query.getResultList(); + } + + public List objectsToPirTableList(List params) { + List result = new LinkedList<>(); + for (int i = 0; i < params.size(); i++) { + Object[] temp = (Object[]) params.get(i); + PirTable pirTable = new PirTable(); + pirTable.setId(i + 1); + pirTable.setPirKey(String.valueOf(temp[0])); + Object[] objects = Arrays.copyOfRange(temp, 1, temp.length); + pirTable.setPirValue(Arrays.toString(objects)); + result.add(pirTable); + } + return result; + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java new file mode 100644 index 00000000..9ddfd9bb --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java @@ -0,0 +1,10 @@ +package com.webank.wedpr.pir.http.common; + +/** + * @author zachma + * @date 2024/8/18 + */ +public class Constant { + + public static final String PIR_API_PREFIX = "/api/pir/v3"; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java new file mode 100644 index 00000000..ef6adad2 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java @@ -0,0 +1,59 @@ +/** Copyright (C) @2014-2022 Webank */ +package com.webank.wedpr.pir.http.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalTimeSerializer; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JacksonConfig { + @Value("${spring.jackson.local-date-time-format:yyyy-MM-dd HH:mm:ss}") + String localDateTimeFormat; + + @Value("${spring.jackson.local-date-format:yyyy-MM-dd}") + String localDateFormat; + + @Value("${spring.jackson.local-time-format:HH:mm:ss}") + String localTimeFormat; + + @Bean + public ObjectMapper objectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + JavaTimeModule javaTimeModule = new JavaTimeModule(); + javaTimeModule.addSerializer( + LocalDateTime.class, + new LocalDateTimeSerializer(DateTimeFormatter.ofPattern(localDateTimeFormat))); + javaTimeModule.addSerializer( + LocalDate.class, + new LocalDateSerializer(DateTimeFormatter.ofPattern(localDateFormat))); + javaTimeModule.addSerializer( + LocalTime.class, + new LocalTimeSerializer(DateTimeFormatter.ofPattern(localTimeFormat))); + + javaTimeModule.addDeserializer( + LocalDateTime.class, + new LocalDateTimeDeserializer(DateTimeFormatter.ofPattern(localDateTimeFormat))); + javaTimeModule.addDeserializer( + LocalDate.class, + new LocalDateDeserializer(DateTimeFormatter.ofPattern(localDateFormat))); + javaTimeModule.addDeserializer( + LocalTime.class, + new LocalTimeDeserializer(DateTimeFormatter.ofPattern(localTimeFormat))); + objectMapper.registerModule(javaTimeModule); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java new file mode 100644 index 00000000..c33e3320 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java @@ -0,0 +1,54 @@ +package com.webank.wedpr.pir.http.controller; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.wedpr.pir.http.common.Constant; +import com.webank.wedpr.pir.http.service.PirAppService; +import com.wedpr.pir.sdk.entity.request.ServerJobRequest; +import com.wedpr.pir.sdk.entity.response.ClientJobResponse; +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +/** + * @author zachma + * @date 2024/8/18 + */ +@RestController +@Slf4j +public class PirController { + private static final Logger logger = LoggerFactory.getLogger(PirController.class); + + @Autowired private ObjectMapper objectMapper; + @Autowired private PirAppService pirAppService; + + @PostMapping(Constant.PIR_API_PREFIX + "/server") + public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serverJobRequest) { + try { + logger.info( + "WedprServerjob: serverJobRequest: {}.", + objectMapper.writeValueAsString(serverJobRequest)); + // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 + ServerOTResponse serverOTResponse = pirAppService.providerOtCipher(serverJobRequest); + pirAppService.reportToAdm(); + return new ClientJobResponse(WedprStatusEnum.SUCCESS, serverOTResponse); + } catch (Exception e) { + logger.warn( + "匿踪任务失败, jobID: {}, response: {}", + serverJobRequest.getJobId(), + WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); + return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, e.getMessage()); + } + } + + // @PostMapping(Constant.PIR_API_PREFIX + "/client") + // public ClientJobResponse pirClientController(@RequestBody ClientJobRequest + // clientJobRequest) { + // + // } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java new file mode 100644 index 00000000..ca73bdc0 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java @@ -0,0 +1,50 @@ +package com.webank.wedpr.pir.http.service; + +import com.webank.wedpr.pir.crypto.service.PirOTService; +import com.wedpr.pir.sdk.entity.request.ServerJobRequest; +import com.wedpr.pir.sdk.entity.request.ServerOTRequest; +import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import com.wedpr.pir.sdk.enums.ParamEnum; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @author zachma + * @date 2024/8/18 + */ +@Service +public class PirAppService { + + @Autowired private PirOTService pirOTService; + + public ServerOTResponse providerOtCipher(ServerJobRequest serverJobRequest) + throws WedprException { + // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 + ServerOTRequest serverOTRequest = new ServerOTRequest(); + serverOTRequest.setJobType(serverJobRequest.getJobType()); + serverOTRequest.setDatasetId(serverJobRequest.getDatasetId()); + serverOTRequest.setX(serverJobRequest.getX()); + serverOTRequest.setY(serverJobRequest.getY()); + serverOTRequest.setDataBodyList(serverJobRequest.getDataBodyList()); + + ServerOTResponse otResultResponse; + if (serverJobRequest + .getJobAlgorithmType() + .equals(ParamEnum.AlgorithmType.idFilter.getValue())) { + otResultResponse = pirOTService.runServerOTParam(serverOTRequest); + } else if (serverJobRequest + .getJobAlgorithmType() + .equals(ParamEnum.AlgorithmType.idObfuscation.getValue())) { + otResultResponse = pirOTService.runServerOTCipher(serverOTRequest); + } else { + throw new WedprException(WedprStatusEnum.INVALID_INPUT_VALUE); + } + return otResultResponse; + } + + public void reportToAdm() { + // TODO: 任务结束上报 + } +} diff --git a/java/ppc-pir-services/src/main/resources/application-dev.properties b/java/ppc-pir-services/src/main/resources/application-dev.properties new file mode 100644 index 00000000..86233b92 --- /dev/null +++ b/java/ppc-pir-services/src/main/resources/application-dev.properties @@ -0,0 +1,9 @@ +### Spring boot configuration +server.address=0.0.0.0 +server.port=5831 +server.session.timeout=60 +banner.charset=UTF-8 +spring.jackson.local-date-time-format=yyyy-MM-dd HH:mm:ss +spring.jackson.local-date-format=yyyy-MM-dd +spring.jackson.local-time-format=HH:mm:ss +spring.jackson.time-zone=GMT+8 \ No newline at end of file diff --git a/java/ppc-pir-services/src/main/resources/application-mysql.properties b/java/ppc-pir-services/src/main/resources/application-mysql.properties new file mode 100644 index 00000000..4c82652f --- /dev/null +++ b/java/ppc-pir-services/src/main/resources/application-mysql.properties @@ -0,0 +1,6 @@ +spring.datasource.url=jdbc:mysql://139.159.202.235:3306/ppc1?serverTimezone=Asia/Shanghai&useUnicode=true&autoReconnect=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&useFractionalSeconds=true&useUnbufferedInput=false&useSSL=false +spring.datasource.username=root +spring.datasource.password=Wedpr2023 +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5InnoDBDialect +spring.jpa.database=mysql diff --git a/java/ppc-pir-services/src/main/resources/application.properties b/java/ppc-pir-services/src/main/resources/application.properties new file mode 100644 index 00000000..80c5d0e0 --- /dev/null +++ b/java/ppc-pir-services/src/main/resources/application.properties @@ -0,0 +1,7 @@ + +spring.profiles.active=dev +# mysql or dm or gauss or kingbase or shentong or postgresql +spring.profiles.include=mysql + +server.shutdown=graceful +spring.lifecycle.timeout-per-shutdown-phase=30s diff --git a/java/ppc-pir-services/src/main/resources/log4j2.xml b/java/ppc-pir-services/src/main/resources/log4j2.xml new file mode 100644 index 00000000..b3c6d126 --- /dev/null +++ b/java/ppc-pir-services/src/main/resources/log4j2.xml @@ -0,0 +1,111 @@ + + + + + + info + log location + ./logs + + + wedpr-pir + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cb24aa1afb0af445b06c41e900a578cd7876ec4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zachma=28=E9=A9=AC=E6=99=A8=29?= Date: Mon, 26 Aug 2024 11:21:10 +0800 Subject: [PATCH 010/120] =?UTF-8?q?=E7=AE=80=E5=8C=96=E4=BE=9D=E8=B5=96PIR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/wedpr/pir/demo/Demo.java | 4 +- .../java/com/wedpr/pir/sdk/PirClient.java | 6 --- .../com/wedpr/pir/sdk/crypto/IdHashVec.java | 51 ------------------- .../com/wedpr/pir/sdk/helper/AESHelper.java | 3 +- .../pir/sdk/helper/CryptoOperatorHelper.java | 47 +++++++++++++++++ .../pir/sdk/service/ClientDecryptService.java | 5 -- .../pir/sdk/service/ClientOTService.java | 11 +--- .../pir/http/controller/PirController.java | 7 --- .../wedpr/pir/http/service/PirAppService.java | 4 -- 9 files changed, 51 insertions(+), 87 deletions(-) delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java index 41f5ee8d..f88a121b 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java @@ -26,7 +26,7 @@ private static void testIdFilter() throws WedprException { pirJobParam.setDatasetId("t_login_token"); pirJobParam.setJobCreator("1001"); pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); - pirJobParam.setSearchIdList(Arrays.asList(new String[] {"1", "20", "3", "456"})); + pirJobParam.setSearchIdList(Arrays.asList("1", "20", "3", "456")); PirClient pirClient = new PirClient(pirJobParam); PirResultResponse pirResultResponse = pirClient.executePirJob(); @@ -42,7 +42,7 @@ private static void testIdObfuscation() throws WedprException { pirJobParam.setDatasetId("t_login_token"); pirJobParam.setJobCreator("1001"); pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); - pirJobParam.setSearchIdList(Arrays.asList(new String[] {"1", "20", "3", "456"})); + pirJobParam.setSearchIdList(Arrays.asList("1", "20", "3", "456")); PirClient pirClient = new PirClient(pirJobParam); PirResultResponse pirResultResponse = pirClient.executePirJob(); diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java index b9e95e29..ecf6ba75 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java @@ -1,8 +1,6 @@ package com.wedpr.pir.sdk; import com.fasterxml.jackson.databind.ObjectMapper; -import com.sun.org.slf4j.internal.Logger; -import com.sun.org.slf4j.internal.LoggerFactory; import com.wedpr.pir.sdk.entity.body.SimpleEntity; import com.wedpr.pir.sdk.entity.param.PirJobParam; import com.wedpr.pir.sdk.entity.request.ServerJobRequest; @@ -19,7 +17,6 @@ * @date 2024/8/19 */ public class PirClient { - private static final Logger logger = LoggerFactory.getLogger(PirClient.class); private final PirJobService pirJobService; public PirClient(PirJobParam pirJobParam) throws WedprException { @@ -33,21 +30,18 @@ public PirResultResponse executePirJob() throws WedprException { PirJobParam pirJobParam = pirJobService.getPirJobParam(); ServerJobRequest serverJobRequest = generatePirJobRequestFromJobAndOTParam(pirJobParam, otParamResponse); - logger.debug("executePirJob start: ", serverJobRequest); String pirUrl = pirJobParam.getGatewayUrl(); ObjectMapper objectMapper = new ObjectMapper(); String body = objectMapper.writeValueAsString(serverJobRequest); SimpleEntity otResult = GatewayClient.sendPostRequestWithRetry(pirUrl, body, SimpleEntity.class); - logger.debug("executePirJob success return: ", otResult); if (Objects.isNull(otResult) || !otResult.getCode().equals(WedprStatusEnum.SUCCESS.getCode())) { throw new WedprException(WedprStatusEnum.TASK_EXECUTE_ERROR); } return pirJobService.requesterOtRecover(otParamResponse, pirJobParam, otResult); } catch (Exception e) { - logger.error("executePirJob cause exception: ", e.getMessage()); throw new WedprException(WedprStatusEnum.CALL_PSI_ERROR); } } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java deleted file mode 100644 index 1aed44d6..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/crypto/IdHashVec.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.wedpr.pir.sdk.crypto; - -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * @author zachma - * @date 2024/8/20 - */ -public class IdHashVec { - public static List getIdHashVec(int obfuscationOrder, Integer idIndex, String searchId) - throws WedprException { - - List clientDataBodyList = new ArrayList<>(); - String searchIdTemp; - for (int i = 0; i < obfuscationOrder + 1; i++) { - if (idIndex == i) { - searchIdTemp = searchId; - } else { - String uuid = UUID.randomUUID().toString(); - searchIdTemp = makeHash(uuid.getBytes()); - } - clientDataBodyList.add(searchIdTemp); - } - return clientDataBodyList; - } - - private static String makeHash(byte[] data) throws WedprException { - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] hashBytes = digest.digest(data); - - StringBuilder hexString = new StringBuilder(); - for (byte b : hashBytes) { - String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) { - hexString.append('0'); - } - hexString.append(hex); - } - return hexString.toString(); - } catch (NoSuchAlgorithmException e) { - throw new WedprException(WedprStatusEnum.WRONG_HASH_ERROR); - } - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java index 8288ccb9..f3260060 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java @@ -31,8 +31,7 @@ public static String generateRandomKey() { public static String encrypt(String plainText, String keyString) throws WedprException { try { byte[] iv = new byte[12]; - SecureRandom secureRandom = new SecureRandom(); - secureRandom.nextBytes(iv); + SECURE_RANDOM.nextBytes(iv); byte[] contentBytes = plainText.getBytes(StandardCharsets.UTF_8); Cipher cipher = Cipher.getInstance(AES_ALGORITHM_TRANSFORMATION); GCMParameterSpec params = new GCMParameterSpec(128, iv); diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java index 187cfcb7..51c89168 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java @@ -1,7 +1,15 @@ package com.wedpr.pir.sdk.helper; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; + import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; /** * @author zachma @@ -48,4 +56,43 @@ public static BigInteger mulMod(BigInteger a, BigInteger b) { public static BigInteger OTMul(BigInteger a, BigInteger b) { return a.multiply(b).mod(DEFAULT_N); } + + /** + * 批量获取searchID的Hash值 + * */ + public static List getIdHashVec(int obfuscationOrder, Integer idIndex, String searchId) + throws WedprException { + + List clientDataBodyList = new ArrayList<>(); + String searchIdTemp; + for (int i = 0; i < obfuscationOrder + 1; i++) { + if (idIndex == i) { + searchIdTemp = searchId; + } else { + String uuid = UUID.randomUUID().toString(); + searchIdTemp = makeHash(uuid.getBytes()); + } + clientDataBodyList.add(searchIdTemp); + } + return clientDataBodyList; + } + + private static String makeHash(byte[] data) throws WedprException { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = digest.digest(data); + + StringBuilder hexString = new StringBuilder(); + for (byte b : hashBytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } catch (NoSuchAlgorithmException e) { + throw new WedprException(WedprStatusEnum.WRONG_HASH_ERROR); + } + } } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java index 6e1d2f2a..2321dec4 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java @@ -1,7 +1,5 @@ package com.wedpr.pir.sdk.service; -import com.sun.org.slf4j.internal.Logger; -import com.sun.org.slf4j.internal.LoggerFactory; import com.wedpr.pir.sdk.entity.body.PirResultBody; import com.wedpr.pir.sdk.entity.body.ServerResultBody; import com.wedpr.pir.sdk.entity.request.ClientDecryptRequest; @@ -21,7 +19,6 @@ * @date 2024/8/21 */ public class ClientDecryptService { - private static final Logger logger = LoggerFactory.getLogger(ClientDecryptService.class); private void decryptServerResultList( PirResultBody pirResultBody, List serverResultList, BigInteger b) { for (ServerResultBody body : serverResultList) { @@ -43,7 +40,6 @@ private void decryptServerResultList( } protected ClientDecryptResponse runDecryptOTparam(ClientDecryptRequest clientDecryptRequest) { - logger.debug("runDecryptOTparam Start: ", clientDecryptRequest); List pirResultBodyArrayList = new ArrayList<>(); for (int i = 0; i < clientDecryptRequest.getServerResult().getResultBodyList().size(); @@ -61,7 +57,6 @@ protected ClientDecryptResponse runDecryptOTparam(ClientDecryptRequest clientDec BasicTypeHelper.isStringNotEmpty(pirResultBody.getSearchValue())); pirResultBodyArrayList.add(pirResultBody); } - logger.debug("runDecryptOTparam Success: ", pirResultBodyArrayList); ClientDecryptResponse clientDecryptResponse = new ClientDecryptResponse(); clientDecryptResponse.setDetail(pirResultBodyArrayList); return clientDecryptResponse; diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java index 63d096df..7d867479 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java @@ -1,8 +1,5 @@ package com.wedpr.pir.sdk.service; -import com.sun.org.slf4j.internal.Logger; -import com.sun.org.slf4j.internal.LoggerFactory; -import com.wedpr.pir.sdk.crypto.IdHashVec; import com.wedpr.pir.sdk.entity.body.PirDataBody; import com.wedpr.pir.sdk.entity.request.ClientOTRequest; import com.wedpr.pir.sdk.entity.response.ClientOTResponse; @@ -20,12 +17,10 @@ * @date 2024/8/20 */ public class ClientOTService { - private static final Logger logger = LoggerFactory.getLogger(ClientOTService.class); private final Random rand = new Random(); /* hash披露, 请求方选择id，生成随机数a、b */ protected ClientOTResponse runClientOTParam(ClientOTRequest clientOTRequest) { - logger.debug("runClientOTParam start: ", clientOTRequest); int filterLength = clientOTRequest.getFilterLength(); BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); @@ -46,15 +41,12 @@ protected ClientOTResponse runClientOTParam(ClientOTRequest clientOTRequest) { pirDataBody.setZ0(z0); pirDataArrayList.add(pirDataBody); } - logger.debug("runClientOTParam success: ", pirDataArrayList); return new ClientOTResponse(blindingB, x, y, pirDataArrayList); } /* hash筛选, 请求方选择顺序\delta\in \{0,1,..,m-1\}，生成随机数a、b */ protected ClientOTResponse runClientOTCipher(ClientOTRequest clientOTRequest) throws WedprException { - logger.debug("runClientOTCipher start: ", clientOTRequest); - int obfuscationOrder = clientOTRequest.getObfuscationOrder(); BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); @@ -68,7 +60,7 @@ protected ClientOTResponse runClientOTCipher(ClientOTRequest clientOTRequest) int idIndex = rand.nextInt(obfuscationOrder + 1); BigInteger z0 = calculateIndexZ0(idIndex, blindingC); List idHashVecList = - IdHashVec.getIdHashVec(obfuscationOrder, idIndex, searchId); + CryptoOperatorHelper.getIdHashVec(obfuscationOrder, idIndex, searchId); PirDataBody pirDataBody = new PirDataBody(); pirDataBody.setZ0(z0); @@ -76,7 +68,6 @@ protected ClientOTResponse runClientOTCipher(ClientOTRequest clientOTRequest) pirDataBody.setIdHashList(idHashVecList); pirDataArrayList.add(pirDataBody); } - logger.debug("runClientOTCipher success: ", pirDataArrayList); return new ClientOTResponse(blindingB, x, y, pirDataArrayList); } diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java index c33e3320..361c63b0 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java @@ -35,7 +35,6 @@ public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serve objectMapper.writeValueAsString(serverJobRequest)); // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 ServerOTResponse serverOTResponse = pirAppService.providerOtCipher(serverJobRequest); - pirAppService.reportToAdm(); return new ClientJobResponse(WedprStatusEnum.SUCCESS, serverOTResponse); } catch (Exception e) { logger.warn( @@ -45,10 +44,4 @@ public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serve return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, e.getMessage()); } } - - // @PostMapping(Constant.PIR_API_PREFIX + "/client") - // public ClientJobResponse pirClientController(@RequestBody ClientJobRequest - // clientJobRequest) { - // - // } } diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java index ca73bdc0..4d0db22f 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java @@ -43,8 +43,4 @@ public ServerOTResponse providerOtCipher(ServerJobRequest serverJobRequest) } return otResultResponse; } - - public void reportToAdm() { - // TODO: 任务结束上报 - } } From 3f202aeb448218600b643c6e53586384b6ddf934 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 20:06:39 +0800 Subject: [PATCH 011/120] rename ppc-crypto-c-sdk to wedpr-component-c-sdk --- cpp/ppc-crypto-c-sdk/CMakeLists.txt | 23 ------------------ cpp/wedpr-component-c-sdk/CMakeLists.txt | 15 ++++++++++++ .../bindings/java/build.gradle | 0 .../java/gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../bindings/java/gradlew | 0 .../bindings/java/gradlew.bat | 0 .../bindings/java/settings.gradle | 0 .../wedpr/sdk/jni/jmh/BenchmarkMain.java | 0 .../wedpr/sdk/jni/jmh/PaillierBenchmark.java | 0 .../bindings/java/src/main/c/CMakeLists.txt | 0 .../bindings/java/src/main/c/jni/Common.cpp | 0 .../bindings/java/src/main/c/jni/Common.h | 0 .../java/src/main/c/jni/JNIException.cpp | 0 .../java/src/main/c/jni/JNIException.h | 0 ...om_webank_wedpr_sdk_jni_crypto_FastOre.cpp | 0 .../com_webank_wedpr_sdk_jni_crypto_FastOre.h | 0 ...dpr_sdk_jni_crypto_SymmetricEncryption.cpp | 0 ...wedpr_sdk_jni_crypto_SymmetricEncryption.h | 0 ...k_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp | 0 ...ank_wedpr_sdk_jni_homo_NativeFloatingIhc.h | 0 ...k_jni_homo_NativeFloatingPointPaillier.cpp | 0 ...sdk_jni_homo_NativeFloatingPointPaillier.h | 0 .../com_webank_wedpr_sdk_jni_homo_NativeIhc.h | 0 ...bank_wedpr_sdk_jni_homo_NativePaillier.cpp | 0 ...webank_wedpr_sdk_jni_homo_NativePaillier.h | 0 .../webank/wedpr/sdk/demo/GenerateKeys.java | 0 .../sdk/jni/codec/FloatingPointNumber.java | 0 .../wedpr/sdk/jni/codec/NumberCodec.java | 0 .../sdk/jni/codec/NumberCodecException.java | 0 .../wedpr/sdk/jni/codec/NumberCodecImpl.java | 0 .../wedpr/sdk/jni/common/JniException.java | 0 .../wedpr/sdk/jni/common/JniLibLoader.java | 0 .../wedpr/sdk/jni/common/Utilities.java | 0 .../webank/wedpr/sdk/jni/crypto/FastOre.java | 0 .../sdk/jni/crypto/SymmetricEncryption.java | 0 .../wedpr/sdk/jni/homo/NativeFloatingIhc.java | 0 .../jni/homo/NativeFloatingPointPaillier.java | 0 .../wedpr/sdk/jni/homo/NativePaillier.java | 0 .../resources/META-INF/native/win/file.list | 0 .../sdk/jni/test/NativeFloatingIhcTest.java | 0 .../test/NativeFloatingPointPaillierTest.java | 0 .../sdk/jni/test/NativePaillierTest.java | 0 .../wedpr/sdk/jni/test/NumberCodecTest.java | 0 .../sdk/jni/test/SymmetricEncryptionTest.java | 0 .../ppc-crypto-c-sdk/CMakeLists.txt | 9 +++++++ .../ppc-crypto-c-sdk/fast_ore.cpp | 0 .../ppc-crypto-c-sdk/fast_ore.h | 0 .../ppc-crypto-c-sdk/floating_point_ihc.cpp | 0 .../ppc-crypto-c-sdk/floating_point_ihc.h | 0 .../floating_point_paillier.cpp | 0 .../floating_point_paillier.h | 0 .../ppc-crypto-c-sdk/homo_ihc.cpp | 0 .../ppc-crypto-c-sdk/homo_ihc.h | 0 .../ppc-crypto-c-sdk/homo_paillier.cpp | 0 .../ppc-crypto-c-sdk/homo_paillier.h | 0 .../ppc-crypto-c-sdk/symmetric_encryption.cpp | 0 .../ppc-crypto-c-sdk/symmetric_encryption.h | 0 .../ppc-crypto-c-sdk/utils/error.cpp | 0 .../ppc-crypto-c-sdk/utils/error.h | 0 .../ppc-crypto-c-sdk/utils/utilities.cpp | 0 .../ppc-crypto-c-sdk/utils/utilities.h | 0 .../tests/CMakeLists.txt | 0 .../tests/TestFastOre.cpp | 0 .../tests/TestHomoPaillier.cpp | 0 .../tests/main.cpp | 0 66 files changed, 24 insertions(+), 23 deletions(-) delete mode 100644 cpp/ppc-crypto-c-sdk/CMakeLists.txt create mode 100644 cpp/wedpr-component-c-sdk/CMakeLists.txt rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/build.gradle (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/gradle/wrapper/gradle-wrapper.jar (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/gradle/wrapper/gradle-wrapper.properties (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/gradlew (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/gradlew.bat (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/settings.gradle (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/CMakeLists.txt (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/Common.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/Common.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/JNIException.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/JNIException.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/main/resources/META-INF/native/win/file.list (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java (100%) create mode 100644 cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/fast_ore.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/fast_ore.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/floating_point_ihc.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/floating_point_ihc.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/floating_point_paillier.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/floating_point_paillier.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/homo_ihc.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/homo_ihc.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/homo_paillier.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/homo_paillier.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/symmetric_encryption.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/symmetric_encryption.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/utils/error.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/utils/error.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/utils/utilities.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/ppc-crypto-c-sdk/utils/utilities.h (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/tests/CMakeLists.txt (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/tests/TestFastOre.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/tests/TestHomoPaillier.cpp (100%) rename cpp/{ppc-crypto-c-sdk => wedpr-component-c-sdk}/tests/main.cpp (100%) diff --git a/cpp/ppc-crypto-c-sdk/CMakeLists.txt b/cpp/ppc-crypto-c-sdk/CMakeLists.txt deleted file mode 100644 index e55eb59d..00000000 --- a/cpp/ppc-crypto-c-sdk/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -project(ppc-crypto-c-sdk VERSION ${VERSION}) - -# export windows dll symbol -if(WIN32) - message(STATUS "Compile on Windows") - set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") -endif() - -file(GLOB_RECURSE SRCS ppc-crypto-c-sdk/*.cpp ppc-crypto-c-sdk/*.c) - -# generate the static lib -add_library(${PPC_CRYPTO_C_SDK_STATIC_TARGET} ${SRCS}) -target_link_libraries(${PPC_CRYPTO_C_SDK_STATIC_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) - -# generate the shared lib -add_library(${PPC_CRYPTO_C_SDK_TARGET} SHARED ${SRCS}) -target_link_libraries(${PPC_CRYPTO_C_SDK_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) - -if (TESTS) - enable_testing() - set(CTEST_OUTPUT_ON_FAILURE TRUE) - add_subdirectory(tests) -endif() \ No newline at end of file diff --git a/cpp/wedpr-component-c-sdk/CMakeLists.txt b/cpp/wedpr-component-c-sdk/CMakeLists.txt new file mode 100644 index 00000000..474f39c6 --- /dev/null +++ b/cpp/wedpr-component-c-sdk/CMakeLists.txt @@ -0,0 +1,15 @@ +project(wedpr-component-c-sdk VERSION ${VERSION}) + +# export windows dll symbol +if(WIN32) + message(STATUS "Compile on Windows") + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") +endif() +add_subdirectory(ppc-crypto-c-sdk) +add_subdirectory(ppc-front-c-sdk) + +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif() \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/build.gradle b/cpp/wedpr-component-c-sdk/bindings/java/build.gradle similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/build.gradle rename to cpp/wedpr-component-c-sdk/bindings/java/build.gradle diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar b/cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar rename to cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties b/cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties rename to cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradlew b/cpp/wedpr-component-c-sdk/bindings/java/gradlew similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/gradlew rename to cpp/wedpr-component-c-sdk/bindings/java/gradlew diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat b/cpp/wedpr-component-c-sdk/bindings/java/gradlew.bat similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/gradlew.bat rename to cpp/wedpr-component-c-sdk/bindings/java/gradlew.bat diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle b/cpp/wedpr-component-c-sdk/bindings/java/settings.gradle similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/settings.gradle rename to cpp/wedpr-component-c-sdk/bindings/java/settings.gradle diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java b/cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java b/cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/CMakeLists.txt rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/CMakeLists.txt diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/Common.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/JNIException.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h b/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java b/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list b/cpp/wedpr-component-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list rename to cpp/wedpr-component-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java b/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java b/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java b/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java b/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java diff --git a/cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java b/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java similarity index 100% rename from cpp/ppc-crypto-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java rename to cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt new file mode 100644 index 00000000..61caffc7 --- /dev/null +++ b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt @@ -0,0 +1,9 @@ +file(GLOB_RECURSE SRCS *.cpp *.c) + +# generate the static lib +add_library(${PPC_CRYPTO_C_SDK_STATIC_TARGET} ${SRCS}) +target_link_libraries(${PPC_CRYPTO_C_SDK_STATIC_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) + +# generate the shared lib +add_library(${PPC_CRYPTO_C_SDK_TARGET} SHARED ${SRCS}) +target_link_libraries(${PPC_CRYPTO_C_SDK_TARGET} PUBLIC ${PAILLIER_TARGET} ${IHC_TARGET} ${CRYPTO_CORE_TARGET}) \ No newline at end of file diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/fast_ore.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_ihc.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/homo_paillier.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/error.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.h diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp diff --git a/cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h b/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.h similarity index 100% rename from cpp/ppc-crypto-c-sdk/ppc-crypto-c-sdk/utils/utilities.h rename to cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.h diff --git a/cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt b/cpp/wedpr-component-c-sdk/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto-c-sdk/tests/CMakeLists.txt rename to cpp/wedpr-component-c-sdk/tests/CMakeLists.txt diff --git a/cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp b/cpp/wedpr-component-c-sdk/tests/TestFastOre.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/tests/TestFastOre.cpp rename to cpp/wedpr-component-c-sdk/tests/TestFastOre.cpp diff --git a/cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp b/cpp/wedpr-component-c-sdk/tests/TestHomoPaillier.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/tests/TestHomoPaillier.cpp rename to cpp/wedpr-component-c-sdk/tests/TestHomoPaillier.cpp diff --git a/cpp/ppc-crypto-c-sdk/tests/main.cpp b/cpp/wedpr-component-c-sdk/tests/main.cpp similarity index 100% rename from cpp/ppc-crypto-c-sdk/tests/main.cpp rename to cpp/wedpr-component-c-sdk/tests/main.cpp From 4cad628f524bbe1232d2e72e8d82d184691d8e7c Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 22 Aug 2024 22:19:18 +0800 Subject: [PATCH 012/120] add interface for wedpr-front-c-sdk --- .github/workflows/cpp_workflow.yml | 6 +- cpp/CMakeLists.txt | 17 ++- cpp/cmake/IncludeDirectories.cmake | 2 +- cpp/cmake/Options.cmake | 1 + cpp/cmake/TargetSettings.cmake | 14 ++- .../CMakeLists.txt | 6 +- .../bindings/java/build.gradle | 0 .../java/gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../bindings/java/gradlew | 0 .../bindings/java/gradlew.bat | 0 .../bindings/java/settings.gradle | 0 .../wedpr/sdk/jni/jmh/BenchmarkMain.java | 0 .../wedpr/sdk/jni/jmh/PaillierBenchmark.java | 0 .../bindings/java/src/main/c/CMakeLists.txt | 0 .../bindings/java/src/main/c/jni/Common.cpp | 0 .../bindings/java/src/main/c/jni/Common.h | 0 .../java/src/main/c/jni/JNIException.cpp | 0 .../java/src/main/c/jni/JNIException.h | 0 ...om_webank_wedpr_sdk_jni_crypto_FastOre.cpp | 0 .../com_webank_wedpr_sdk_jni_crypto_FastOre.h | 0 ...dpr_sdk_jni_crypto_SymmetricEncryption.cpp | 0 ...wedpr_sdk_jni_crypto_SymmetricEncryption.h | 0 ...k_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp | 0 ...ank_wedpr_sdk_jni_homo_NativeFloatingIhc.h | 0 ...k_jni_homo_NativeFloatingPointPaillier.cpp | 0 ...sdk_jni_homo_NativeFloatingPointPaillier.h | 0 .../com_webank_wedpr_sdk_jni_homo_NativeIhc.h | 0 ...bank_wedpr_sdk_jni_homo_NativePaillier.cpp | 0 ...webank_wedpr_sdk_jni_homo_NativePaillier.h | 0 .../webank/wedpr/sdk/demo/GenerateKeys.java | 0 .../sdk/jni/codec/FloatingPointNumber.java | 0 .../wedpr/sdk/jni/codec/NumberCodec.java | 0 .../sdk/jni/codec/NumberCodecException.java | 0 .../wedpr/sdk/jni/codec/NumberCodecImpl.java | 0 .../wedpr/sdk/jni/common/JniException.java | 0 .../wedpr/sdk/jni/common/JniLibLoader.java | 0 .../wedpr/sdk/jni/common/Utilities.java | 0 .../webank/wedpr/sdk/jni/crypto/FastOre.java | 0 .../sdk/jni/crypto/SymmetricEncryption.java | 0 .../wedpr/sdk/jni/homo/NativeFloatingIhc.java | 0 .../jni/homo/NativeFloatingPointPaillier.java | 0 .../wedpr/sdk/jni/homo/NativePaillier.java | 0 .../resources/META-INF/native/win/file.list | 0 .../sdk/jni/test/NativeFloatingIhcTest.java | 0 .../test/NativeFloatingPointPaillierTest.java | 0 .../sdk/jni/test/NativePaillierTest.java | 0 .../wedpr/sdk/jni/test/NumberCodecTest.java | 0 .../sdk/jni/test/SymmetricEncryptionTest.java | 0 .../ppc-crypto-c-sdk/CMakeLists.txt | 0 .../ppc-crypto-c-sdk/fast_ore.cpp | 0 .../ppc-crypto-c-sdk/fast_ore.h | 0 .../ppc-crypto-c-sdk/floating_point_ihc.cpp | 0 .../ppc-crypto-c-sdk/floating_point_ihc.h | 0 .../floating_point_paillier.cpp | 0 .../floating_point_paillier.h | 0 .../ppc-crypto-c-sdk/homo_ihc.cpp | 0 .../ppc-crypto-c-sdk/homo_ihc.h | 0 .../ppc-crypto-c-sdk/homo_paillier.cpp | 0 .../ppc-crypto-c-sdk/homo_paillier.h | 0 .../ppc-crypto-c-sdk/symmetric_encryption.cpp | 0 .../ppc-crypto-c-sdk/symmetric_encryption.h | 0 .../ppc-crypto-c-sdk/utils/error.cpp | 0 .../ppc-crypto-c-sdk/utils/error.h | 0 .../ppc-crypto-c-sdk/utils/utilities.cpp | 0 .../ppc-crypto-c-sdk/utils/utilities.h | 0 .../tests/CMakeLists.txt | 0 .../tests/TestFastOre.cpp | 0 .../tests/TestHomoPaillier.cpp | 0 .../tests/main.cpp | 0 .../wedpr-front-cpp-sdk/CMakeLists.txt | 9 ++ .../wedpr-front-cpp-sdk/wedpr_front_c.cpp | 92 ++++++++++++++++ .../wedpr-front-cpp-sdk/wedpr_front_c.h | 98 ++++++++++++++++++ .../wedpr-front-cpp-sdk/wedpr_front_common.h | 28 +++++ .../wedpr-front-cpp-sdk/wedpr_front_config.h | 56 ++++++++++ .../wedpr-front-cpp-sdk/wedpr_msg.h | 81 +++++++++++++++ 76 files changed, 398 insertions(+), 12 deletions(-) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/CMakeLists.txt (74%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/build.gradle (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/gradle/wrapper/gradle-wrapper.jar (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/gradle/wrapper/gradle-wrapper.properties (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/gradlew (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/gradlew.bat (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/settings.gradle (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/CMakeLists.txt (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/Common.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/Common.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/JNIException.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/JNIException.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/main/resources/META-INF/native/win/file.list (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/CMakeLists.txt (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/fast_ore.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/fast_ore.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/floating_point_ihc.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/floating_point_ihc.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/floating_point_paillier.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/floating_point_paillier.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/homo_ihc.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/homo_ihc.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/homo_paillier.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/homo_paillier.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/symmetric_encryption.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/symmetric_encryption.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/utils/error.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/utils/error.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/utils/utilities.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/ppc-crypto-c-sdk/utils/utilities.h (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/tests/CMakeLists.txt (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/tests/TestFastOre.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/tests/TestHomoPaillier.cpp (100%) rename cpp/{wedpr-component-c-sdk => wedpr-component-sdk}/tests/main.cpp (100%) create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h create mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index 3ec7f9c1..ce682ad2 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -141,12 +141,12 @@ jobs: if: runner.os == 'macos' with: name: libppc-crypto-sdk-jni.dylib - path: ./cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib + path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib - uses: actions/upload-artifact@v2 if: runner.os == 'Windows' with: name: libppc-crypto-sdk-jni.dylib - path: D:\a\WeDPR-Component\cpp\ppc-crypto-c-sdk\bindings\java\src\main\resources\META-INF\native\Release\ppc-crypto-sdk-jni.dll + path: D:\a\WeDPR-Component\cpp\wedpr-component-sdk\bindings\java\src\main\resources\META-INF\native\Release\ppc-crypto-sdk-jni.dll build_centos: name: build_centos full node @@ -221,4 +221,4 @@ jobs: - uses: actions/upload-artifact@v2 with: name: libppc-crypto-sdk-jni.so - path: ./cpp/ppc-crypto-c-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so \ No newline at end of file + path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 698343ec..75586dc4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -63,8 +63,8 @@ include(TargetSettings) include(Dependencies) ########### set the sources ########### -set(JNI_SOURCE_PATH ppc-crypto-c-sdk/bindings/java/src/main/c) -set(SDK_SOURCE_LIST ppc-homo ppc-crypto-core ppc-crypto-c-sdk ${JNI_SOURCE_PATH}) +set(JNI_SOURCE_PATH wedpr-component-sdk/bindings/java/src/main/c) +set(SDK_SOURCE_LIST ppc-homo ppc-crypto-core wedpr-component-sdk ${JNI_SOURCE_PATH}) # Note: the udf depends on mysql, not enabled in the full node mode set(UDF_SOURCE_LIST ${SDK_SOURCE_LIST} ppc-udf) set(ALL_SOURCE_LIST @@ -79,6 +79,19 @@ if(BUILD_CEM) set(CEM_SOURCE "ppc-cem") endif() +if(BUILD_WEDPR_TOOLKIT) + # fetch the python dependencies + option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON) + message(STATUS "Python fetch dependencies: ${FETCH_PYTHON_DEPS}") + include(python) + if(WIN32) + message(STATUS "Getting SWIG for Windows: ...") + include(swig) + message(STATUS "Getting SWIG for Windows: ...DONE") + endif() + add_subdirectory(wedpr-toolkit-wrapper) +endif() + if(BUILD_ALL) add_sources("${ALL_SOURCE_LIST}") elseif(BUILD_UDF) diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index c22d3cce..7bd1fb86 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -2,7 +2,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-crypto-c-sdk) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-c-sdk) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-tars-protocol) set(VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include") diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 1b4245ab..bf631af3 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -52,6 +52,7 @@ macro(configure_project) default_option(BUILD_ALL ON) default_option(BUILD_SDK OFF) default_option(BUILD_UDF OFF) + default_option(BUILD_WEDPR_TOOLKIT OFF) # Suffix like "-rc1" e.t.c. to append to versions wherever needed. if (NOT DEFINED VERSION_SUFFIX) diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 61b17f44..da9e9626 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -85,17 +85,22 @@ if (ENABLE_CPU_FEATURES) set(CPU_FEATURES_LIB CpuFeatures::cpu_features) endif () -#====== ppc-crypto-c-sdk =========== +#====== wedpr-component-sdk =========== set(PPC_CRYPTO_C_SDK_STATIC_TARGET ppc-crypto-c-sdk-static) set(PPC_CRYPTO_C_SDK_TARGET ppc-crypto-c-sdk) +set(PPC_FRONT_C_SDK_STATIC_TARGET ppc-front-c-sdk-static) +set(PPC_FRONT_C_SDK_TARGET ppc-front-c-sdk) + # add suffix for arm if(ARCH_NATIVE) message(STATUS "Building arm architecture, CMAKE_HOST_SYSTEM_PROCESSOR => ${CMAKE_HOST_SYSTEM_PROCESSOR}") set(PPC_CRYPTO_C_SDK_STATIC_TARGET "ppc-crypto-c-sdk-aarch64") set(PPC_CRYPTO_C_SDK_TARGET "ppc-crypto-c-sdk-static-aarch64") + set(PPC_FRONT_C_SDK_STATIC_TARGET ppc-front-c-sdk-static-aarch64) + set(PPC_FRONT_C_SDK_TARGET ppc-front-c-sdk-aarch64) endif() -#====== ppc-crypto-c-sdk =========== +#====== wedpr-component-sdk =========== #====== ppc-crypto-sdk-jni =========== @@ -119,4 +124,7 @@ if(ARCH_NATIVE) endif() # ========== ppc-udf =========== -set(BOOST_UNIT_TEST Boost::unit_test_framework) \ No newline at end of file +set(BOOST_UNIT_TEST Boost::unit_test_framework) + +# ==== the swig wrapper ===== +set(WEDPR_PYTHON_TOOLKIT "wedpr_python_toolkit") \ No newline at end of file diff --git a/cpp/wedpr-component-c-sdk/CMakeLists.txt b/cpp/wedpr-component-sdk/CMakeLists.txt similarity index 74% rename from cpp/wedpr-component-c-sdk/CMakeLists.txt rename to cpp/wedpr-component-sdk/CMakeLists.txt index 474f39c6..8f389e80 100644 --- a/cpp/wedpr-component-c-sdk/CMakeLists.txt +++ b/cpp/wedpr-component-sdk/CMakeLists.txt @@ -1,4 +1,4 @@ -project(wedpr-component-c-sdk VERSION ${VERSION}) +project(wedpr-component-sdk VERSION ${VERSION}) # export windows dll symbol if(WIN32) @@ -6,10 +6,10 @@ if(WIN32) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") endif() add_subdirectory(ppc-crypto-c-sdk) -add_subdirectory(ppc-front-c-sdk) +add_subdirectory(wedpr-front-cpp-sdk) if (TESTS) enable_testing() set(CTEST_OUTPUT_ON_FAILURE TRUE) add_subdirectory(tests) -endif() \ No newline at end of file +endif() diff --git a/cpp/wedpr-component-c-sdk/bindings/java/build.gradle b/cpp/wedpr-component-sdk/bindings/java/build.gradle similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/build.gradle rename to cpp/wedpr-component-sdk/bindings/java/build.gradle diff --git a/cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar b/cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar rename to cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar diff --git a/cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties b/cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties rename to cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties diff --git a/cpp/wedpr-component-c-sdk/bindings/java/gradlew b/cpp/wedpr-component-sdk/bindings/java/gradlew similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/gradlew rename to cpp/wedpr-component-sdk/bindings/java/gradlew diff --git a/cpp/wedpr-component-c-sdk/bindings/java/gradlew.bat b/cpp/wedpr-component-sdk/bindings/java/gradlew.bat similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/gradlew.bat rename to cpp/wedpr-component-sdk/bindings/java/gradlew.bat diff --git a/cpp/wedpr-component-c-sdk/bindings/java/settings.gradle b/cpp/wedpr-component-sdk/bindings/java/settings.gradle similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/settings.gradle rename to cpp/wedpr-component-sdk/bindings/java/settings.gradle diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java b/cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java rename to cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java b/cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java rename to cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/CMakeLists.txt b/cpp/wedpr-component-sdk/bindings/java/src/main/c/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/CMakeLists.txt rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/CMakeLists.txt diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/Common.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/JNIException.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h b/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h rename to cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java b/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java rename to cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list b/cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list rename to cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java b/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java rename to cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java b/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java rename to cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java b/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java rename to cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java b/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java rename to cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java diff --git a/cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java b/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java similarity index 100% rename from cpp/wedpr-component-c-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java rename to cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/CMakeLists.txt rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/CMakeLists.txt diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/fast_ore.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_ihc.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/floating_point_paillier.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_ihc.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/homo_paillier.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/symmetric_encryption.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/error.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.h diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.cpp rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.cpp diff --git a/cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.h b/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.h similarity index 100% rename from cpp/wedpr-component-c-sdk/ppc-crypto-c-sdk/utils/utilities.h rename to cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.h diff --git a/cpp/wedpr-component-c-sdk/tests/CMakeLists.txt b/cpp/wedpr-component-sdk/tests/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-c-sdk/tests/CMakeLists.txt rename to cpp/wedpr-component-sdk/tests/CMakeLists.txt diff --git a/cpp/wedpr-component-c-sdk/tests/TestFastOre.cpp b/cpp/wedpr-component-sdk/tests/TestFastOre.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/tests/TestFastOre.cpp rename to cpp/wedpr-component-sdk/tests/TestFastOre.cpp diff --git a/cpp/wedpr-component-c-sdk/tests/TestHomoPaillier.cpp b/cpp/wedpr-component-sdk/tests/TestHomoPaillier.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/tests/TestHomoPaillier.cpp rename to cpp/wedpr-component-sdk/tests/TestHomoPaillier.cpp diff --git a/cpp/wedpr-component-c-sdk/tests/main.cpp b/cpp/wedpr-component-sdk/tests/main.cpp similarity index 100% rename from cpp/wedpr-component-c-sdk/tests/main.cpp rename to cpp/wedpr-component-sdk/tests/main.cpp diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt new file mode 100644 index 00000000..f2fbbf92 --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt @@ -0,0 +1,9 @@ +file(GLOB_RECURSE SRCS *.cpp *.c) + +# generate the static lib +add_library(${PPC_FRONT_C_SDK_STATIC_TARGET} ${SRCS}) +target_link_libraries(${PPC_FRONT_C_SDK_STATIC_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) + +# generate the shared lib +add_library(${PPC_FRONT_C_SDK_TARGET} SHARED ${SRCS}) +target_link_libraries(${PPC_FRONT_C_SDK_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp new file mode 100644 index 00000000..e3b59381 --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file wedpr_front_c.h + * @author: yujiechen + * @date 2024-08-22 + */ +#include "wedpr_front_c.h" + +/** + * @brief create the wedpr_front using specified config + * + * @param config the config used to build the wedpr_front + * @return void* the created wedpr_front + */ +void* wedpr_front_create(struct wedpr_front_config* config) +{ + return nullptr; +} + +/** + * @brief start the wedpr_front + * + * @param front the front to start + */ +void wedpr_front_start(void* front) {} + +/** + * @brief stop the wedpr_front + * + * @param front the front to stop + */ +void wedpr_front_stop(void* front) {} + +/** + * @brief destroy the wedpr_front + * + * @param front the front to destroy + */ +void wedpr_front_destroy(void* front) {} + +/** + * @brief register the topic handler + * + * @param front the front object + * @param topic the topic + * @param callback the callback called when receive specified topic + */ +void register_topic_handler(void* front, InputBuffer const* topic, wedpr_msg_handler_cb callback) {} + +/** + * @brief async send message + * + * @param front the front to send the message + * @param routerPolicy the router policy: + * 0: route by nodeID + * 1: route by component + * 2: route by agency + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ +void async_send_message(void* front, int routerPolicy, InputBuffer const* topic, + InputBuffer const* dstInst, InputBuffer const* dstNodeID, uint8_t componentType, + InputBuffer const* payload, int seq, long timeout, wedpr_msg_handler_cb callback) +{} + +// the sync interface for async_send_message +wedpr_msg* push(void* front, int routerPolicy, InputBuffer const* topic, InputBuffer const* dstInst, + InputBuffer const* dstNodeID, uint8_t componentType, InputBuffer const* payload, int seq, + long timeout) +{ + return nullptr; +} diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h new file mode 100644 index 00000000..ed918ce8 --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h @@ -0,0 +1,98 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file wedpr_front_c.h + * @author: yujiechen + * @date 2024-08-22 + */ +#ifndef __WEDPR_FRONT_C_H__ +#define __WEDPR_FRONT_C_H__ +#include "ppc-framework/libwrapper/Buffer.h" +#include "wedpr_front_common.h" +#include "wedpr_front_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief create the wedpr_front using specified config + * + * @param config the config used to build the wedpr_front + * @return void* the created wedpr_front + */ +void* wedpr_front_create(struct wedpr_front_config* config); + +/** + * @brief start the wedpr_front + * + * @param front the front to start + */ +void wedpr_front_start(void* front); + +/** + * @brief stop the wedpr_front + * + * @param front the front to stop + */ +void wedpr_front_stop(void* front); + +/** + * @brief destroy the wedpr_front + * + * @param front the front to destroy + */ +void wedpr_front_destroy(void* front); + +/** + * @brief register the topic handler + * + * @param front the front object + * @param topic the topic + * @param callback the callback called when receive specified topic + */ +void register_topic_handler(void* front, InputBuffer const* topic, wedpr_msg_handler_cb callback); + +/** + * @brief async send message + * + * @param front the front to send the message + * @param routerPolicy the router policy: + * 0: route by nodeID + * 1: route by component + * 2: route by agency + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ +void async_send_message(void* front, int routerPolicy, InputBuffer const* topic, + InputBuffer const* dstInst, InputBuffer const* dstNodeID, uint8_t componentType, + InputBuffer const* payload, int seq, long timeout, wedpr_msg_handler_cb callback); + +// the sync interface for async_send_message +wedpr_msg* push(void* front, int routerPolicy, InputBuffer const* topic, InputBuffer const* dstInst, + InputBuffer const* dstNodeID, uint8_t componentType, InputBuffer const* payload, int seq, + long timeout); + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h new file mode 100644 index 00000000..22473223 --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h @@ -0,0 +1,28 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file wedpr_front_common.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#ifndef __WEDPR_FRONT_COMMON_H__ +#define __WEDPR_FRONT_COMMON_H__ + +#include "wedpr_msg.h" + +typedef void (*wedpr_msg_handler_cb)(struct wedpr_msg* response, void* context); + +#endif diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h new file mode 100644 index 00000000..cbe665d3 --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file wedpr_front_config.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#ifndef __WEDPR_FRONT_CONFIG_H__ +#define __WEDPR_FRONT_CONFIG_H__ +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief the gateway endpoint information + * + */ +struct wedpr_gateway_endpoint +{ + InputBuffer const* host; + uint16_t port; +}; + +struct wedpr_gateway_info +{ + struct wedpr_gateway_endpoint* gatewayEndpoints; + uint16_t gatewayCount; +}; + +struct wedpr_front_config +{ + int threadPoolSize; + // the agency id + InputBuffer const* agencyID; + // the gateway-endpoints + struct wedpr_gateway_info* gateway_info; +}; +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h new file mode 100644 index 00000000..ae6f280e --- /dev/null +++ b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file wedpr_front_common.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#ifndef __WEDPR_MSG_H__ +#define __WEDPR_MSG_H__ + +#include "ppc-framework/libwrapper/Buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct gateway_optional_header +{ + // the componentType + uint8_t componentType; + // the source nodeID that send the message + OutputBuffer* srcNode; + // the target nodeID that should receive the message + OutputBuffer* dstNode; + // the target agency that need receive the message + OutputBuffer* dstInst; +}; + +struct gateway_msg_header +{ + // the msg version, used to support compatibility + uint8_t version; + // the traceID + OutputBuffer* traceID; + // the srcGwNode + OutputBuffer* srcGwNode; + // the dstGwNode + OutputBuffer* dstGwNode; + // the packetType + uint16_t packetType; + // the seq + uint32_t seq; + // the ttl + int16_t ttl; + // the ext(contains the router policy and response flag) + uint16_t ext; + //// the optional field(used to route between components and nodes) + struct gateway_optional_header* optionalFields; +}; + +struct front_payload +{ + // the front payload version, used to support compatibility + uint8_t version; + // the topic + OutputBuffer* topic; + OutputBuffer* data; +}; + +struct wedpr_msg +{ + gateway_msg_header* header; + front_payload* payload; +}; +#ifdef __cplusplus +} +#endif +#endif From ceb0f929c3f0d78e68bb289add1e938e73f8f7b8 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 23 Aug 2024 21:11:14 +0800 Subject: [PATCH 013/120] remove wedpr-front-cpp-sdk --- cpp/cmake/IncludeDirectories.cmake | 2 +- cpp/cmake/TargetSettings.cmake | 6 +- cpp/cmake/Version.cmake | 1 + cpp/ppc-framework/Common.h | 12 ++ cpp/ppc-framework/protocol/Message.h | 195 ++++++++++++++++++ cpp/ppc-front/ppc-front/Common.h | 1 - cpp/ppc-protocol/src/PPCMessage.h | 5 +- cpp/ppc-tools/src/codec/CodecUtility.h | 13 +- .../Utilities.h} | 51 ++--- cpp/wedpr-component-sdk/CMakeLists.txt | 1 - .../wedpr-front-cpp-sdk/CMakeLists.txt | 9 - .../wedpr-front-cpp-sdk/wedpr_front_c.cpp | 92 --------- .../wedpr-front-cpp-sdk/wedpr_front_c.h | 98 --------- .../wedpr-front-cpp-sdk/wedpr_front_common.h | 28 --- .../wedpr-front-cpp-sdk/wedpr_msg.h | 81 -------- 15 files changed, 230 insertions(+), 365 deletions(-) create mode 100644 cpp/ppc-framework/protocol/Message.h rename cpp/{wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h => ppc-utilities/Utilities.h} (50%) delete mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt delete mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp delete mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h delete mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h delete mode 100644 cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index 7bd1fb86..f63512f5 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -2,7 +2,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-c-sdk) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-sdk) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-tars-protocol) set(VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include") diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index da9e9626..7f29bb81 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -90,15 +90,12 @@ set(PPC_CRYPTO_C_SDK_STATIC_TARGET ppc-crypto-c-sdk-static) set(PPC_CRYPTO_C_SDK_TARGET ppc-crypto-c-sdk) set(PPC_FRONT_C_SDK_STATIC_TARGET ppc-front-c-sdk-static) -set(PPC_FRONT_C_SDK_TARGET ppc-front-c-sdk) # add suffix for arm if(ARCH_NATIVE) message(STATUS "Building arm architecture, CMAKE_HOST_SYSTEM_PROCESSOR => ${CMAKE_HOST_SYSTEM_PROCESSOR}") set(PPC_CRYPTO_C_SDK_STATIC_TARGET "ppc-crypto-c-sdk-aarch64") set(PPC_CRYPTO_C_SDK_TARGET "ppc-crypto-c-sdk-static-aarch64") - set(PPC_FRONT_C_SDK_STATIC_TARGET ppc-front-c-sdk-static-aarch64) - set(PPC_FRONT_C_SDK_TARGET ppc-front-c-sdk-aarch64) endif() #====== wedpr-component-sdk =========== @@ -127,4 +124,5 @@ endif() set(BOOST_UNIT_TEST Boost::unit_test_framework) # ==== the swig wrapper ===== -set(WEDPR_PYTHON_TOOLKIT "wedpr_python_toolkit") \ No newline at end of file +set(WEDPR_PYTHON_TOOLKIT "wedpr_python_toolkit") +set(WEDPR_PYTHON_TOOLKIT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TOOLKIT}) \ No newline at end of file diff --git a/cpp/cmake/Version.cmake b/cpp/cmake/Version.cmake index 84833fb8..206620b2 100644 --- a/cpp/cmake/Version.cmake +++ b/cpp/cmake/Version.cmake @@ -1 +1,2 @@ set(VERSION "1.0.0") +set(PYTHON_TOOLKIT_VERSION "1.0.0") diff --git a/cpp/ppc-framework/Common.h b/cpp/ppc-framework/Common.h index ea80b504..6034c5b6 100644 --- a/cpp/ppc-framework/Common.h +++ b/cpp/ppc-framework/Common.h @@ -44,6 +44,18 @@ namespace ppc { \ } +#define CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, length) \ + do \ + { \ + if (offset > length) \ + { \ + throw std::out_of_range("Out of range error, offset:" + std::to_string(offset) + \ + " ,length: " + std::to_string(length) + \ + " ,file: " + __FILE__ + " ,func: " + __func__ + \ + " ,line: " + std::to_string(__LINE__)); \ + } \ + } while (0); + DERIVE_PPC_EXCEPTION(OpenFileFailed); DERIVE_PPC_EXCEPTION(DataSchemaNotSetted); DERIVE_PPC_EXCEPTION(UnsupportedDataSchema); diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h new file mode 100644 index 00000000..0aa7443a --- /dev/null +++ b/cpp/ppc-framework/protocol/Message.h @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Message.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include +#include +#include + +namespace ppc::protocol +{ +class MessageOptionalHeader +{ +public: + using Ptr = std::shared_ptr; + MessageOptionalHeader() = default; + virtual ~MessageOptionalHeader() = default; + + virtual bcos::bytes encode() const = 0; + // the componentType + virtual uint8_t componentType() const { return m_componentType; } + // the source nodeID that send the message + virtual bcos::bytes const& srcNode() const { return m_srcNode; } + // the target nodeID that should receive the message + virtual bcos::bytes const& dstNode() const { return m_dstNode; } + // the target agency that need receive the message + virtual bcos::bytes const& dstInst() const { return m_dstInst; } + +protected: + virtual uint32_t decode(bcos::bytesConstRef data) = 0; + +protected: + // the componentType + uint8_t m_componentType; + // the source nodeID that send the message + bcos::bytes m_srcNode; + // the target nodeID that should receive the message + bcos::bytes m_dstNode; + // the target agency that need receive the message + bcos::bytes m_dstInst; +}; +class MessageHeader +{ +public: + using Ptr = std::shared_ptr; + MessageHeader() = default; + virtual ~MessageHeader() = default; + + virtual bcos::bytes encode() const = 0; + + // the msg version, used to support compatibility + virtual uint8_t version() const { return m_version; } + // the traceID + virtual std::string const& traceID() const { return m_traceID; } + // the srcGwNode + virtual bcos::bytes const& srcGwNode() const { return m_srcGwNode; } + // the dstGwNode + virtual bcos::bytes const& dstGwNode() const { return m_dstGwNode; } + // the packetType + virtual uint16_t packetType() const { return m_packetType; } + // the ttl + virtual int16_t ttl() const { return m_ttl; } + // the ext(contains the router policy and response flag) + virtual uint16_t ext() const { return m_ext; } + //// the optional field(used to route between components and nodes) + virtual MessageOptionalHeader::Ptr optionalFields() const { return m_optionalFields; } + + virtual uint32_t length() const { return m_length; } + + virtual void setVersion(uint16_t version) { m_version = version; } + virtual void setPacketType(uint16_t packetType) { m_packetType = packetType; } + // the seq is the traceID + virtual void setTraceID(std::string traceID) { m_traceID = traceID; } + virtual void setExt(uint16_t ext) { m_ext = ext; } + + uint64_t packetLen() const { return m_packetLen; } + uint16_t headerLen() const { return m_headerLen; } + + virtual bool isRespPacket() const = 0; + virtual void setRespPacket() = 0; + +protected: + virtual uint32_t decode(bcos::bytesConstRef data) = 0; + +protected: + // the msg version, used to support compatibility + uint8_t m_version; + // the traceID + std::string m_traceID; + // the srcGwNode + bcos::bytes m_srcGwNode; + // the dstGwNode + bcos::bytes m_dstGwNode; + // the packetType + uint16_t m_packetType; + // the ttl + int16_t m_ttl; + // the ext(contains the router policy and response flag) + uint16_t m_ext; + //// the optional field(used to route between components and nodes) + MessageOptionalHeader::Ptr m_optionalFields; + uint64_t m_packetLen; + uint16_t m_headerLen; +}; + +class MessagePayload +{ +public: + using Ptr = std::shared_ptr; + MessagePayload() = default; + virtual ~MessagePayload() = default; + + virtual bcos::bytes encode() const = 0; + + // the version + virtual uint8_t version() const { return m_version; } + // the topic + virtual std::string const& topic() const { return m_topic; } + virtual bcos::bytes const& data() const { return m_data; } + virtual uint32_t length() const { return m_length; } + +protected: + virtual uint32_t decode(uint32_t startPos, bcos::bytesConstRef data) = 0; + +protected: + // the front payload version, used to support compatibility + uint8_t m_version; + // the topic + std::string m_topic; + bcos::bytes m_data; + uint32_t m_length; +}; + +class Message : virtual public bcos::boostssl::MessageFace +{ +public: + using Ptr = std::shared_ptr; + Message() = default; + ~Message() override {} + + virtual MessageHeader::Ptr header() const { return m_header; } + virtual MessagePayLoad::Ptr payload() const { return m_payload; } + + uint16_t version() const override { return m_header->version(); } + void setVersion(uint16_t version) override{m_header->setVersion(version)} uint16_t + packetType() const override + { + return m_header->packetType(); + } + void setPacketType(uint16_t packetType) override { m_header->setPacketType(packetType); } + std::string const& seq() const override { return m_header->traceID(); } + void setSeq(std::string traceID) override { m_header->setTraceID(traceID); } + uint16_t ext() const override { return m_header->ext(); } + void setExt(uint16_t ext) override { m_header->setExt(except); } + + bool isRespPacket() const override { return m_header->isRespPacket(); } + void setRespPacket() override { m_header->setRespPacket(); } + + virtual uint32_t length() const override { return m_header->packetLen(); } + std::shared_ptr payload() const override { return m_payload; } + void setPayload(std::shared_ptr _payload) override + { + m_payload = std::move(_payload); + } + + +protected: + MessageHeader::Ptr m_header; + std::shared_ptr m_payload; +}; + +class MessageBuilder +{ +public: + MessageBuilder() = default; + virtual ~MessageBuilder() = default; + + virtual Message::Ptr build() = 0; +} +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/Common.h b/cpp/ppc-front/ppc-front/Common.h index 677ed467..a221d179 100644 --- a/cpp/ppc-front/ppc-front/Common.h +++ b/cpp/ppc-front/ppc-front/Common.h @@ -32,7 +32,6 @@ namespace ppc::front { - #define FRONT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[FRONT]" #define HOLDING_MESSAGE_TIMEOUT_M 30 diff --git a/cpp/ppc-protocol/src/PPCMessage.h b/cpp/ppc-protocol/src/PPCMessage.h index c62f890f..ca34657b 100644 --- a/cpp/ppc-protocol/src/PPCMessage.h +++ b/cpp/ppc-protocol/src/PPCMessage.h @@ -64,10 +64,7 @@ class PPCMessage : public PPCMessageFace std::shared_ptr data() const override { return m_data; } // Note: here directly use passed-in _data, make-sure _data not changed before send the message void setData(std::shared_ptr _data) override { m_data = _data; } - std::map header() override - { - return decodeMap(m_header); - } + std::map header() override { return decodeMap(m_header); } void setHeader(std::map _header) override { m_header = encodeMap(_header); diff --git a/cpp/ppc-tools/src/codec/CodecUtility.h b/cpp/ppc-tools/src/codec/CodecUtility.h index 5d37ddf0..ce7a208a 100644 --- a/cpp/ppc-tools/src/codec/CodecUtility.h +++ b/cpp/ppc-tools/src/codec/CodecUtility.h @@ -20,21 +20,10 @@ #pragma once #include "openssl/bn.h" #include "ppc-framework/libwrapper/BigNum.h" +#include "ppc-utilities/Utilities.h" #include #include #include - -#define CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, length) \ - do \ - { \ - if (offset > length) \ - { \ - throw std::out_of_range("Out of range error, offset:" + std::to_string(offset) + \ - " ,length: " + std::to_string(length) + \ - " ,file: " + __FILE__ + " ,func: " + __func__ + \ - " ,line: " + std::to_string(__LINE__)); \ - } \ - } while (0); namespace ppc { inline bcos::byte* encodeBuffer( diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h b/cpp/ppc-utilities/Utilities.h similarity index 50% rename from cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h rename to cpp/ppc-utilities/Utilities.h index cbe665d3..b9b8e0ae 100644 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_config.h +++ b/cpp/ppc-utilities/Utilities.h @@ -13,44 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * @file wedpr_front_config.h + * @file Utilitiles.cpp * @author: yujiechen - * @date 2024-08-22 + * @date 2024-08-23 */ +#pragma once -#ifndef __WEDPR_FRONT_CONFIG_H__ -#define __WEDPR_FRONT_CONFIG_H__ -#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/Common.h" -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief the gateway endpoint information - * - */ -struct wedpr_gateway_endpoint +namespace ppc { - InputBuffer const* host; - uint16_t port; -}; - -struct wedpr_gateway_info -{ - struct wedpr_gateway_endpoint* gatewayEndpoints; - uint16_t gatewayCount; -}; - -struct wedpr_front_config +inline uint64_t decodeNetworkBuffer( + bcos::bytes& _result, bcos::byte const* buffer, unsigned int bufferLen, uint64_t const offset) { - int threadPoolSize; - // the agency id - InputBuffer const* agencyID; - // the gateway-endpoints - struct wedpr_gateway_info* gateway_info; -}; -#ifdef __cplusplus + CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, bufferLen); + auto dataLen = + boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)buffer + offset)); + offset += 2; + CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, bufferLen); + buffer.insert( + buffer.end(), (bcos::byte*)_buffer + offset, (bcos::byte*)_buffer + offset + dataLen); + offset += dataLen; + return offset; } -#endif -#endif \ No newline at end of file +} // namespace ppc \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/CMakeLists.txt b/cpp/wedpr-component-sdk/CMakeLists.txt index 8f389e80..fae969e6 100644 --- a/cpp/wedpr-component-sdk/CMakeLists.txt +++ b/cpp/wedpr-component-sdk/CMakeLists.txt @@ -6,7 +6,6 @@ if(WIN32) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON") endif() add_subdirectory(ppc-crypto-c-sdk) -add_subdirectory(wedpr-front-cpp-sdk) if (TESTS) enable_testing() diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt deleted file mode 100644 index f2fbbf92..00000000 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -file(GLOB_RECURSE SRCS *.cpp *.c) - -# generate the static lib -add_library(${PPC_FRONT_C_SDK_STATIC_TARGET} ${SRCS}) -target_link_libraries(${PPC_FRONT_C_SDK_STATIC_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) - -# generate the shared lib -add_library(${PPC_FRONT_C_SDK_TARGET} SHARED ${SRCS}) -target_link_libraries(${PPC_FRONT_C_SDK_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp deleted file mode 100644 index e3b59381..00000000 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright (C) 2023 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file wedpr_front_c.h - * @author: yujiechen - * @date 2024-08-22 - */ -#include "wedpr_front_c.h" - -/** - * @brief create the wedpr_front using specified config - * - * @param config the config used to build the wedpr_front - * @return void* the created wedpr_front - */ -void* wedpr_front_create(struct wedpr_front_config* config) -{ - return nullptr; -} - -/** - * @brief start the wedpr_front - * - * @param front the front to start - */ -void wedpr_front_start(void* front) {} - -/** - * @brief stop the wedpr_front - * - * @param front the front to stop - */ -void wedpr_front_stop(void* front) {} - -/** - * @brief destroy the wedpr_front - * - * @param front the front to destroy - */ -void wedpr_front_destroy(void* front) {} - -/** - * @brief register the topic handler - * - * @param front the front object - * @param topic the topic - * @param callback the callback called when receive specified topic - */ -void register_topic_handler(void* front, InputBuffer const* topic, wedpr_msg_handler_cb callback) {} - -/** - * @brief async send message - * - * @param front the front to send the message - * @param routerPolicy the router policy: - * 0: route by nodeID - * 1: route by component - * 2: route by agency - * @param topic the topic - * @param dstInst the dst agency(must set when 'route by agency' and 'route by - * component') - * @param dstNodeID the dst nodeID(must set when 'route by nodeID') - * @param componentType the componentType(must set when 'route by component') - * @param payload the payload to send - * @param seq the message seq - * @param timeout timeout - * @param callback callback - */ -void async_send_message(void* front, int routerPolicy, InputBuffer const* topic, - InputBuffer const* dstInst, InputBuffer const* dstNodeID, uint8_t componentType, - InputBuffer const* payload, int seq, long timeout, wedpr_msg_handler_cb callback) -{} - -// the sync interface for async_send_message -wedpr_msg* push(void* front, int routerPolicy, InputBuffer const* topic, InputBuffer const* dstInst, - InputBuffer const* dstNodeID, uint8_t componentType, InputBuffer const* payload, int seq, - long timeout) -{ - return nullptr; -} diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h deleted file mode 100644 index ed918ce8..00000000 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_c.h +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright (C) 2023 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file wedpr_front_c.h - * @author: yujiechen - * @date 2024-08-22 - */ -#ifndef __WEDPR_FRONT_C_H__ -#define __WEDPR_FRONT_C_H__ -#include "ppc-framework/libwrapper/Buffer.h" -#include "wedpr_front_common.h" -#include "wedpr_front_config.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief create the wedpr_front using specified config - * - * @param config the config used to build the wedpr_front - * @return void* the created wedpr_front - */ -void* wedpr_front_create(struct wedpr_front_config* config); - -/** - * @brief start the wedpr_front - * - * @param front the front to start - */ -void wedpr_front_start(void* front); - -/** - * @brief stop the wedpr_front - * - * @param front the front to stop - */ -void wedpr_front_stop(void* front); - -/** - * @brief destroy the wedpr_front - * - * @param front the front to destroy - */ -void wedpr_front_destroy(void* front); - -/** - * @brief register the topic handler - * - * @param front the front object - * @param topic the topic - * @param callback the callback called when receive specified topic - */ -void register_topic_handler(void* front, InputBuffer const* topic, wedpr_msg_handler_cb callback); - -/** - * @brief async send message - * - * @param front the front to send the message - * @param routerPolicy the router policy: - * 0: route by nodeID - * 1: route by component - * 2: route by agency - * @param topic the topic - * @param dstInst the dst agency(must set when 'route by agency' and 'route by - * component') - * @param dstNodeID the dst nodeID(must set when 'route by nodeID') - * @param componentType the componentType(must set when 'route by component') - * @param payload the payload to send - * @param seq the message seq - * @param timeout timeout - * @param callback callback - */ -void async_send_message(void* front, int routerPolicy, InputBuffer const* topic, - InputBuffer const* dstInst, InputBuffer const* dstNodeID, uint8_t componentType, - InputBuffer const* payload, int seq, long timeout, wedpr_msg_handler_cb callback); - -// the sync interface for async_send_message -wedpr_msg* push(void* front, int routerPolicy, InputBuffer const* topic, InputBuffer const* dstInst, - InputBuffer const* dstNodeID, uint8_t componentType, InputBuffer const* payload, int seq, - long timeout); - -#ifdef __cplusplus -} -#endif -#endif \ No newline at end of file diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h deleted file mode 100644 index 22473223..00000000 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_front_common.h +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (C) 2023 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file wedpr_front_common.h - * @author: yujiechen - * @date 2024-08-22 - */ - -#ifndef __WEDPR_FRONT_COMMON_H__ -#define __WEDPR_FRONT_COMMON_H__ - -#include "wedpr_msg.h" - -typedef void (*wedpr_msg_handler_cb)(struct wedpr_msg* response, void* context); - -#endif diff --git a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h b/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h deleted file mode 100644 index ae6f280e..00000000 --- a/cpp/wedpr-component-sdk/wedpr-front-cpp-sdk/wedpr_msg.h +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright (C) 2023 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file wedpr_front_common.h - * @author: yujiechen - * @date 2024-08-22 - */ - -#ifndef __WEDPR_MSG_H__ -#define __WEDPR_MSG_H__ - -#include "ppc-framework/libwrapper/Buffer.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct gateway_optional_header -{ - // the componentType - uint8_t componentType; - // the source nodeID that send the message - OutputBuffer* srcNode; - // the target nodeID that should receive the message - OutputBuffer* dstNode; - // the target agency that need receive the message - OutputBuffer* dstInst; -}; - -struct gateway_msg_header -{ - // the msg version, used to support compatibility - uint8_t version; - // the traceID - OutputBuffer* traceID; - // the srcGwNode - OutputBuffer* srcGwNode; - // the dstGwNode - OutputBuffer* dstGwNode; - // the packetType - uint16_t packetType; - // the seq - uint32_t seq; - // the ttl - int16_t ttl; - // the ext(contains the router policy and response flag) - uint16_t ext; - //// the optional field(used to route between components and nodes) - struct gateway_optional_header* optionalFields; -}; - -struct front_payload -{ - // the front payload version, used to support compatibility - uint8_t version; - // the topic - OutputBuffer* topic; - OutputBuffer* data; -}; - -struct wedpr_msg -{ - gateway_msg_header* header; - front_payload* payload; -}; -#ifdef __cplusplus -} -#endif -#endif From ef3a682f8a6a11f3b89d3585822c63903a1d1343 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sat, 24 Aug 2024 00:35:57 +0800 Subject: [PATCH 014/120] add Message and router implement --- cpp/cmake/IncludeDirectories.cmake | 1 + cpp/ppc-framework/Common.h | 15 + cpp/ppc-framework/gateway/GatewayProtocol.h | 37 +++ cpp/ppc-framework/protocol/Message.h | 150 +++++---- cpp/ppc-framework/protocol/MessagePayload.h | 68 ++++ cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp | 299 ++++++++++++++++++ cpp/ppc-gateway/ppc-gateway/p2p/Service.h | 86 +++++ .../ppc-gateway/p2p/router/RouterManager.cpp | 189 +++++++++++ .../ppc-gateway/p2p/router/RouterManager.h | 72 +++++ .../p2p/router/RouterTableImpl.cpp | 273 ++++++++++++++++ .../ppc-gateway/p2p/router/RouterTableImpl.h | 132 ++++++++ .../p2p/router/RouterTableInterface.h | 92 ++++++ cpp/ppc-protocol/src/JsonTaskImpl.h | 23 +- cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp | 146 +++++++++ cpp/ppc-protocol/src/v1/MessageHeaderImpl.h | 82 +++++ cpp/ppc-protocol/src/v1/MessageImpl.cpp | 63 ++++ cpp/ppc-protocol/src/v1/MessageImpl.h | 84 +++++ .../src/v1/MessagePayloadImpl.cpp | 67 ++++ cpp/ppc-protocol/src/v1/MessagePayloadImpl.h | 54 ++++ .../ppc-tars-protocol/tars/RouterTable.tars | 13 + cpp/ppc-tools/src/codec/CodecUtility.h | 1 - cpp/ppc-utilities/Utilities.h | 18 +- cpp/vcpkg-configuration.json | 4 +- 23 files changed, 1884 insertions(+), 85 deletions(-) create mode 100644 cpp/ppc-framework/gateway/GatewayProtocol.h create mode 100644 cpp/ppc-framework/protocol/MessagePayload.h create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/Service.h create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h create mode 100644 cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h create mode 100644 cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp create mode 100644 cpp/ppc-protocol/src/v1/MessageHeaderImpl.h create mode 100644 cpp/ppc-protocol/src/v1/MessageImpl.cpp create mode 100644 cpp/ppc-protocol/src/v1/MessageImpl.h create mode 100644 cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp create mode 100644 cpp/ppc-protocol/src/v1/MessagePayloadImpl.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index f63512f5..d6be1b25 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -1,5 +1,6 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) +include_directories(${CMAKE_BINARY_DIR}/generated/) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-sdk) diff --git a/cpp/ppc-framework/Common.h b/cpp/ppc-framework/Common.h index 6034c5b6..2ae44dfd 100644 --- a/cpp/ppc-framework/Common.h +++ b/cpp/ppc-framework/Common.h @@ -59,10 +59,25 @@ namespace ppc DERIVE_PPC_EXCEPTION(OpenFileFailed); DERIVE_PPC_EXCEPTION(DataSchemaNotSetted); DERIVE_PPC_EXCEPTION(UnsupportedDataSchema); +DERIVE_PPC_EXCEPTION(WeDPRException); constexpr static int MAX_PORT = 65535; constexpr static int DEFAULT_SECURITY_PARAM = 128; +constexpr static size_t RSA_PUBLIC_KEY_PREFIX = 18; +constexpr static size_t RSA_PUBLIC_KEY_TRUNC = 8; +constexpr static size_t RSA_PUBLIC_KEY_TRUNC_LENGTH = 26; + +inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept +{ + if (p2pId.length() < RSA_PUBLIC_KEY_TRUNC_LENGTH) + { + return p2pId; + } + return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC); +} + + #if ENABLE_CPU_FEATURES #if X86 static const cpu_features::X86Features CPU_FEATURES = cpu_features::GetX86Info().features; diff --git a/cpp/ppc-framework/gateway/GatewayProtocol.h b/cpp/ppc-framework/gateway/GatewayProtocol.h new file mode 100644 index 00000000..c3c0a94d --- /dev/null +++ b/cpp/ppc-framework/gateway/GatewayProtocol.h @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Message.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include + +namespace ppc::gateway +{ +enum class GatewayPacketType : uint16_t +{ + P2PMessage = 0x00, + RouterTableSyncSeq = 0x10, + RouterTableResponse = 0x11, + RouterTableRequest = 0x12 +}; + +enum class GatewayMsgExtFlag : uint16_t +{ + Response = 0x1, +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 0aa7443a..40c5f614 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -18,9 +18,12 @@ * @date 2024-08-22 */ #pragma once +#include "../Common.h" #include #include +#include #include +#include namespace ppc::protocol { @@ -31,18 +34,24 @@ class MessageOptionalHeader MessageOptionalHeader() = default; virtual ~MessageOptionalHeader() = default; - virtual bcos::bytes encode() const = 0; + virtual void encode(bcos::bytes& buffer) const = 0; + virtual int64_t decode(bcos::bytesConstRef data, uint64_t const _offset) = 0; + // the componentType virtual uint8_t componentType() const { return m_componentType; } + virtual void setComponentType(uint8_t componentType) { m_componentType = componentType; } + // the source nodeID that send the message virtual bcos::bytes const& srcNode() const { return m_srcNode; } + virtual void setSrcNode(bcos::bytes const& srcNode) { m_srcNode = srcNode; } + // the target nodeID that should receive the message virtual bcos::bytes const& dstNode() const { return m_dstNode; } + virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; } + // the target agency that need receive the message virtual bcos::bytes const& dstInst() const { return m_dstInst; } - -protected: - virtual uint32_t decode(bcos::bytesConstRef data) = 0; + virtual void setDstInst(bcos::bytes const& dstInst) { m_dstInst = dstInst; } protected: // the componentType @@ -54,6 +63,7 @@ class MessageOptionalHeader // the target agency that need receive the message bcos::bytes m_dstInst; }; + class MessageHeader { public: @@ -61,41 +71,51 @@ class MessageHeader MessageHeader() = default; virtual ~MessageHeader() = default; - virtual bcos::bytes encode() const = 0; + virtual void encode(bcos::bytes& buffer) const = 0; + virtual int64_t decode(bcos::bytesConstRef data) = 0; // the msg version, used to support compatibility virtual uint8_t version() const { return m_version; } + virtual void setVersion(uint16_t version) { m_version = version; } // the traceID virtual std::string const& traceID() const { return m_traceID; } + virtual void setTraceID(std::string traceID) { m_traceID = traceID; } + // the srcGwNode - virtual bcos::bytes const& srcGwNode() const { return m_srcGwNode; } + virtual std::string const& srcGwNode() const { return m_srcGwNode; } + virtual void setSrcGwNode(std::string const& srcGwNode) { m_srcGwNode = srcGwNode; } + // the dstGwNode - virtual bcos::bytes const& dstGwNode() const { return m_dstGwNode; } + virtual std::string const& dstGwNode() const { return m_dstGwNode; } + virtual void setDstGwNode(std::string const& dstGwNode) { m_dstGwNode = dstGwNode; } + // the packetType virtual uint16_t packetType() const { return m_packetType; } + virtual void setPacketType(uint16_t packetType) { m_packetType = packetType; } // the ttl virtual int16_t ttl() const { return m_ttl; } + virtual void setTTL(uint16_t ttl) { m_ttl = ttl; } + // the ext(contains the router policy and response flag) virtual uint16_t ext() const { return m_ext; } - //// the optional field(used to route between components and nodes) - virtual MessageOptionalHeader::Ptr optionalFields() const { return m_optionalFields; } - - virtual uint32_t length() const { return m_length; } - - virtual void setVersion(uint16_t version) { m_version = version; } - virtual void setPacketType(uint16_t packetType) { m_packetType = packetType; } - // the seq is the traceID - virtual void setTraceID(std::string traceID) { m_traceID = traceID; } virtual void setExt(uint16_t ext) { m_ext = ext; } + //// the optional field(used to route between components and nodes) + virtual MessageOptionalHeader::Ptr optionalField() const { return m_optionalField; } + void setOptionalField(MessageOptionalHeader::Ptr optionalField) + { + m_optionalField = std::move(optionalField); + } - uint64_t packetLen() const { return m_packetLen; } - uint16_t headerLen() const { return m_headerLen; } + virtual uint16_t length() const { return m_length; } virtual bool isRespPacket() const = 0; virtual void setRespPacket() = 0; -protected: - virtual uint32_t decode(bcos::bytesConstRef data) = 0; + + // Note: only for log + std::string_view srcP2PNodeIDView() const { return printP2PIDElegantly(m_srcGwNode); } + // Note: only for log + std::string_view dstP2PNodeIDView() const { return printP2PIDElegantly(m_dstGwNode); } protected: // the msg version, used to support compatibility @@ -103,9 +123,9 @@ class MessageHeader // the traceID std::string m_traceID; // the srcGwNode - bcos::bytes m_srcGwNode; + std::string m_srcGwNode; // the dstGwNode - bcos::bytes m_dstGwNode; + std::string m_dstGwNode; // the packetType uint16_t m_packetType; // the ttl @@ -113,37 +133,8 @@ class MessageHeader // the ext(contains the router policy and response flag) uint16_t m_ext; //// the optional field(used to route between components and nodes) - MessageOptionalHeader::Ptr m_optionalFields; - uint64_t m_packetLen; - uint16_t m_headerLen; -}; - -class MessagePayload -{ -public: - using Ptr = std::shared_ptr; - MessagePayload() = default; - virtual ~MessagePayload() = default; - - virtual bcos::bytes encode() const = 0; - - // the version - virtual uint8_t version() const { return m_version; } - // the topic - virtual std::string const& topic() const { return m_topic; } - virtual bcos::bytes const& data() const { return m_data; } - virtual uint32_t length() const { return m_length; } - -protected: - virtual uint32_t decode(uint32_t startPos, bcos::bytesConstRef data) = 0; - -protected: - // the front payload version, used to support compatibility - uint8_t m_version; - // the topic - std::string m_topic; - bcos::bytes m_data; - uint32_t m_length; + MessageOptionalHeader::Ptr m_optionalField; + uint16_t mutable m_length; }; class Message : virtual public bcos::boostssl::MessageFace @@ -154,36 +145,48 @@ class Message : virtual public bcos::boostssl::MessageFace ~Message() override {} virtual MessageHeader::Ptr header() const { return m_header; } - virtual MessagePayLoad::Ptr payload() const { return m_payload; } + virtual void setHeader(MessageHeader::Ptr header) { m_header = std::move(header); } + /// the overloaed implementation === uint16_t version() const override { return m_header->version(); } - void setVersion(uint16_t version) override{m_header->setVersion(version)} uint16_t - packetType() const override - { - return m_header->packetType(); - } + void setVersion(uint16_t version) override { m_header->setVersion(version); } + uint16_t packetType() const override { return m_header->packetType(); } void setPacketType(uint16_t packetType) override { m_header->setPacketType(packetType); } std::string const& seq() const override { return m_header->traceID(); } void setSeq(std::string traceID) override { m_header->setTraceID(traceID); } uint16_t ext() const override { return m_header->ext(); } - void setExt(uint16_t ext) override { m_header->setExt(except); } + void setExt(uint16_t ext) override { m_header->setExt(ext); } bool isRespPacket() const override { return m_header->isRespPacket(); } void setRespPacket() override { m_header->setRespPacket(); } - virtual uint32_t length() const override { return m_header->packetLen(); } + virtual uint32_t length() const override + { + return m_header->length() + (m_payload ? m_payload->size() : 0); + } + std::shared_ptr payload() const override { return m_payload; } void setPayload(std::shared_ptr _payload) override { m_payload = std::move(_payload); } - protected: MessageHeader::Ptr m_header; std::shared_ptr m_payload; }; +class MessageHeaderBuilder +{ +public: + using Ptr = std::shared_ptr; + MessageHeaderBuilder() = default; + virtual ~MessageHeaderBuilder() = default; + + virtual MessageHeader::Ptr build(bcos::bytesConstRef _data) = 0; + virtual MessageHeader::Ptr build() = 0; +}; + class MessageBuilder { public: @@ -191,5 +194,28 @@ class MessageBuilder virtual ~MessageBuilder() = default; virtual Message::Ptr build() = 0; + virtual Message::Ptr build(bcos::bytesConstRef buffer) = 0; +}; + +inline std::string printMessage(Message::Ptr const& _msg) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("from", _msg->header()->srcP2PNodeIDView()) + << LOG_KV("to", _msg->header()->dstP2PNodeIDView()) + << LOG_KV("ttl", _msg->header()->ttl()) + << LOG_KV("rsp", _msg->header()->isRespPacket()) + << LOG_KV("traceID", _msg->header()->traceID()) + << LOG_KV("packetType", _msg->header()->packetType()) + << LOG_KV("length", _msg->length()); + return stringstream.str(); +} + +inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("rsp", _msg->isRespPacket()) << LOG_KV("traceID", _msg->seq()) + << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length()); + return stringstream.str(); } + } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h new file mode 100644 index 00000000..4329aae2 --- /dev/null +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessagePayload.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include +#include + +namespace ppc::protocol +{ +class MessagePayload +{ +public: + using Ptr = std::shared_ptr; + MessagePayload() = default; + virtual ~MessagePayload() = default; + + virtual int64_t encode(bcos::bytes& buffer) const = 0; + virtual int64_t decode(bcos::bytesConstRef data) = 0; + + // the version + virtual uint8_t version() const { return m_version; } + virtual void setVersion(uint8_t version) { m_version = version; } + // the topic + virtual std::string const& topic() const { return m_topic; } + virtual void setTopic(std::string&& topic) { m_topic = std::move(topic); } + virtual void setTopic(std::string const& topic) { m_topic = topic; } + // data + virtual bcos::bytes const& data() const { return m_data; } + virtual void setData(bcos::bytes&& data) { m_data = std::move(data); } + virtual void setData(bcos::bytes const& data) { m_data = data; } + // the length + virtual int64_t length() const { return m_length; } + +protected: + // the front payload version, used to support compatibility + uint8_t m_version; + // the topic + std::string m_topic; + bcos::bytes m_data; + int64_t mutable m_length; +}; + +class MessagePayloadBuilder +{ +public: + using Ptr = std::shared_ptr; + MessagePayloadBuilder() = default; + virtual ~MessagePayloadBuilder() = default; + virtual MessagePayload::Ptr build() = 0; + virtual MessagePayload::Ptr build(bcos::bytesConstRef buffer) = 0; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp new file mode 100644 index 00000000..469e5b27 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -0,0 +1,299 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Service.cpp + * @author: yujiechen + * @date 2024-08-26 + */ + +#include "Service.h" +#include "bcos-boostssl/websocket/WsError.h" +#include "ppc-framework/Common.h" + +using namespace bcos; +using namespace ppc; +using namespace ppc::gateway; +using namespace ppc::protocol; +using namespace bcos::boostssl::ws; +using namespace bcos::boostssl; + +Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _routerTableFactory, + int unreachableDistance, std::string _moduleName) + : WsService(_moduleName) +{ + m_nodeID = _nodeID; + m_routerTableFactory = _routerTableFactory; + // create the local router + m_routerTable = m_routerTableFactory->createRouterTable(); + m_routerTable->setNodeID(m_nodeID); + m_routerTable->setUnreachableDistance(unreachableDistance); + + GATEWAY_LOG(INFO) << LOG_DESC("create P2PService") << LOG_KV("module", _moduleName); + WsService::registerConnectHandler( + boost::bind(&Service::onP2PConnect, this, boost::placeholders::_1)); + WsService::registerDisconnectHandler( + boost::bind(&Service::onP2PDisconnect, this, boost::placeholders::_1)); +} + + +void Service::onP2PConnect(WsSession::Ptr _session) +{ + GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect") << LOG_KV("p2pid", _session->nodeId()) + << LOG_KV("endpoint", _session->endPoint()); + + + RecursiveGuard l(x_nodeID2Session); + auto it = m_nodeID2Session.find(_session->nodeId()); + // the session already connected + if (it != m_nodeID2Session.end() && it->second->isConnected()) + { + GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect, drop the duplicated connection") + << LOG_KV("nodeID", _session->nodeId()) + << LOG_KV("endpoint", _session->endPoint()); + _session->drop(WsError::UserDisconnect); + updateNodeIDInfo(_session); + return; + } + // the node-self + if (_session->nodeId() == m_nodeID) + { + updateNodeIDInfo(_session); + GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect, drop the node-self connection") + << LOG_KV("nodeID", _session->nodeId()) + << LOG_KV("endpoint", _session->endPoint()); + _session->drop(WsError::UserDisconnect); + return; + } + // the new session + updateNodeIDInfo(_session); + if (it != m_nodeID2Session.end()) + { + it->second = _session; + } + else + { + m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); + } + GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect established") << LOG_KV("p2pid", _session->nodeId()) + << LOG_KV("endpoint", _session->endPoint()); +} + + +void Service::updateNodeIDInfo(WsSession::Ptr const& _session) +{ + bcos::WriteGuard l(x_configuredNode2ID); + std::string p2pNodeID = _session->nodeId(); + auto it = m_configuredNode2ID.find(_session->endPointInfo()); + if (it != m_configuredNode2ID.end()) + { + it->second = p2pNodeID; + GATEWAY_LOG(INFO) << LOG_DESC("updateNodeIDInfo: update the nodeID") + << LOG_KV("nodeid", p2pNodeID) + << LOG_KV("endpoint", _session->endPoint()); + } + else + { + GATEWAY_LOG(INFO) << LOG_DESC("updateNodeIDInfo can't find endpoint") + << LOG_KV("nodeid", p2pNodeID) + << LOG_KV("endpoint", _session->endPoint()); + } +} + +void Service::removeSessionInfo(WsSession::Ptr const& _session) +{ + RecursiveGuard l(x_nodeID2Session); + auto it = m_nodeID2Session.find(_session->nodeId()); + if (it != m_nodeID2Session.end()) + { + GATEWAY_LOG(INFO) << "onP2PDisconnectand remove from m_nodeID2Session" + << LOG_KV("p2pid", _session->nodeId()) + << LOG_KV("endpoint", _session->endPoint()); + + m_nodeID2Session.erase(it); + } +} +void Service::onP2PDisconnect(WsSession::Ptr _session) +{ + // remove the session information + removeSessionInfo(_session); + // update the session nodeID to empty + UpgradableGuard l(x_configuredNode2ID); + for (auto& it : m_configuredNode2ID) + { + if (it.second == _session->nodeId()) + { + UpgradeGuard ul(l); + it.second.clear(); + break; + } + } +} + +void Service::reconnect() +{ + // obtain the un-connected peers information + EndPointsPtr unconnectedPeers = std::make_shared>(); + { + bcos::ReadGuard l(x_configuredNode2ID); + for (auto const& it : m_configuredNode2ID) + { + if (it.second == nodeID()) + { + continue; + } + if (!it.second.empty() && isConnected(it.first)) + { + continue; + } + unconnectedPeers->insert(it.first); + } + } + setReconnectedPeers(unconnectedPeers); + WsService::reconnect(); +} + +WsSession::Ptr Service::getSessionByNodeID(std::string const& _nodeID) +{ + RecursiveGuard l(x_nodeID2Session); + auto it = m_nodeID2Session.find(_nodeID); + if (it == m_nodeID2Session.end()) + { + return nullptr; + } + return it->second; +} + +void Service::asyncSendMessageByNodeID( + std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc) +{ + auto p2pMsg = std::dynamic_pointer_cast(msg); + if (p2pMsg->header()->dstGwNode().empty()) + { + p2pMsg->header()->setDstGwNode(dstNodeID); + } + if (p2pMsg->header()->srcGwNode().empty()) + { + p2pMsg->header()->setSrcGwNode(m_nodeID); + } + return asyncSendMessageWithForward(dstNodeID, msg, options, respFunc); +} + +void Service::asyncSendMessageWithForward( + std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc) +{ + auto p2pMsg = std::dynamic_pointer_cast(msg); + // without nextHop: maybe network unreachable or with distance equal to 1 + auto nextHop = m_routerTable->getNextHop(dstNodeID); + if (nextHop.empty()) + { + return asyncSendMessage(dstNodeID, msg, options, respFunc); + } + // with nextHop, send the message to nextHop + GATEWAY_LOG(TRACE) << LOG_DESC("asyncSendMessageByNodeID") << printMessage(p2pMsg); + return asyncSendMessage(nextHop, msg, options, respFunc); +} + + +void Service::asyncSendMessage( + std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc) +{ + try + { + // ignore self + if (dstNodeID == m_nodeID) + { + return; + } + auto session = getSessionByNodeID(dstNodeID); + if (session) + { + WsSessions sessions = WsSessions(); + sessions.emplace_back(session); + return WsService::asyncSendMessage(sessions, msg, options, respFunc); + } + + if (respFunc) + { + Error::Ptr error = std::make_shared( + -1, "send message to " + dstNodeID + + " failed for no network established, msg: " + printWsMessage(msg)); + respFunc(std::move(error), nullptr, nullptr); + } + GATEWAY_LOG(WARNING) + << LOG_DESC("asyncSendMessageByNodeID failed for no network established, msg detail:") + << printWsMessage(msg); + } + catch (std::exception const& e) + { + GATEWAY_LOG(ERROR) << "asyncSendMessageByNodeID" << LOG_KV("dstNode", dstNodeID) + << LOG_KV("what", boost::diagnostic_information(e)); + if (respFunc) + { + respFunc(std::make_shared(-1, "send message to " + dstNodeID + " failed for " + + boost::diagnostic_information(e)), + nullptr, nullptr); + } + } +} + +void Service::onRecvMessage(MessageFace::Ptr _msg, std::shared_ptr _session) +{ + auto p2pMsg = std::dynamic_pointer_cast(_msg); + // find the dstNode + if (p2pMsg->header()->dstGwNode().empty() || p2pMsg->header()->dstGwNode() == m_nodeID) + { + GATEWAY_LOG(TRACE) << LOG_DESC("onRecvMessage, dispatch for find the dst node") + << printMessage(p2pMsg); + WsService::onRecvMessage(_msg, _session); + return; + } + // forward the message + if (p2pMsg->header()->ttl() >= m_routerTable->unreachableDistance()) + { + GATEWAY_LOG(WARNING) << LOG_DESC("onRecvMessage: ttl expired") << printMessage(p2pMsg); + return; + } + p2pMsg->header()->setTTL(p2pMsg->header()->ttl() + 1); + asyncSendMessageWithForward( + p2pMsg->header()->dstGwNode(), p2pMsg, bcos::boostssl::ws::Options(), nullptr); +} + + +void Service::asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, Options options) +{ + auto reachableNodes = m_routerTable->getAllReachableNode(); + try + { + for (auto const& node : reachableNodes) + { + asyncSendMessageByNodeID(node, msg, options); + } + } + catch (std::exception& e) + { + GATEWAY_LOG(WARNING) << LOG_BADGE("asyncBroadcastMessage exception") + << LOG_KV("msg", printWsMessage(msg)) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + +void Service::asyncSendMessageByP2PNodeID(uint16_t type, std::string const& dstNodeID, + std::shared_ptr payload, Options options, RespCallBack callback) +{ + auto message = m_messageFactory->buildMessage(); + message->setPacketType(type); + message->setPayload(payload); + asyncSendMessageByNodeID(dstNodeID, message, options, callback); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h new file mode 100644 index 00000000..a3c8f5ea --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Service.h + * @author: yujiechen + * @date 2024-08-26 + */ + +#pragma once +#include "../Common.h" +#include "ppc-framework/protocol/Message.h" +#include "router/RouterTableInterface.h" +#include +namespace ppc::gateway +{ +class Service : public bcos::boostssl::ws::WsService +{ +public: + using Ptr = std::shared_ptr; + Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _routerTableFactory, + int unreachableDistance, std::string _moduleName = "DEFAULT"); + + virtual void asyncSendMessageByNodeID(std::string const& dstNodeID, + bcos::boostssl::MessageFace::Ptr msg, + bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options(), + bcos::boostssl::ws::RespCallBack respFunc = bcos::boostssl::ws::RespCallBack()); + virtual void asyncSendMessageByP2PNodeID(uint16_t type, std::string const& dstNodeID, + std::shared_ptr payload, + bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options(), + bcos::boostssl::ws::RespCallBack callback = bcos::boostssl::ws::RespCallBack()); + + virtual void asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, + bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options()); + + RouterTableFactory::Ptr const& routerTableFactory() const { return m_routerTableFactory; } + RouterTableInterface::Ptr const& routerTable() const { return m_routerTable; } + + std::string const& nodeID() const { return m_nodeID; } + +protected: + void onRecvMessage(bcos::boostssl::MessageFace::Ptr _msg, + bcos::boostssl::ws::WsSession::Ptr _session) override; + + virtual void onP2PConnect(bcos::boostssl::ws::WsSession::Ptr _session); + virtual void onP2PDisconnect(bcos::boostssl::ws::WsSession::Ptr _session); + + void reconnect() override; + + void updateNodeIDInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); + void removeSessionInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); + bcos::boostssl::ws::WsSession::Ptr getSessionByNodeID(std::string const& _nodeID); + + virtual void asyncSendMessageWithForward(std::string const& dstNodeID, + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::Options options, + bcos::boostssl::ws::RespCallBack respFunc); + + virtual void asyncSendMessage(std::string const& dstNodeID, + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::Options options, + bcos::boostssl::ws::RespCallBack respFunc); + +protected: + std::string m_nodeID; + // nodeID=>session + std::unordered_map m_nodeID2Session; + bcos::RecursiveMutex x_nodeID2Session; + + RouterTableFactory::Ptr m_routerTableFactory; + RouterTableInterface::Ptr m_routerTable; + + // configuredNode=>nodeID + std::map m_configuredNode2ID; + mutable bcos::SharedMutex x_configuredNode2ID; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp new file mode 100644 index 00000000..00f3e7c7 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp @@ -0,0 +1,189 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouterManager.cpp + * @author: yujiechen + * @date 2024-08-26 + */ +#include "RouterManager.h" +#include "ppc-framework/gateway/GatewayProtocol.h" +#include "ppc-framework/protocol/Message.h" +#include + +using namespace bcos; +using namespace bcos::boostssl::ws; +using namespace bcos::boostssl; +using namespace ppc::gateway; +using namespace ppc::protocol; + +RouterManager::RouterManager(Service::Ptr service) : m_service(std::move(service)) +{ + // process router packet related logic + m_service->registerMsgHandler((uint16_t)GatewayPacketType::RouterTableSyncSeq, + boost::bind(&RouterManager::onReceiveRouterSeq, this, boost::placeholders::_1, + boost::placeholders::_2)); + + m_service->registerMsgHandler((uint16_t)GatewayPacketType::RouterTableResponse, + boost::bind(&RouterManager::onReceivePeersRouterTable, this, boost::placeholders::_1, + boost::placeholders::_2)); + + m_service->registerMsgHandler((uint16_t)GatewayPacketType::RouterTableRequest, + boost::bind(&RouterManager::onReceiveRouterTableRequest, this, boost::placeholders::_1, + boost::placeholders::_2)); + m_routerTimer = std::make_shared(3000, "routerSeqSync"); + m_routerTimer->registerTimeoutHandler([this]() { broadcastRouterSeq(); }); +} + +void RouterManager::start() +{ + if (m_routerTimer) + { + m_routerTimer->start(); + } +} + +void RouterManager::stop() +{ + if (m_routerTimer) + { + m_routerTimer->stop(); + } +} + +void RouterManager::onReceiveRouterSeq(MessageFace::Ptr msg, WsSession::Ptr session) +{ + auto statusSeq = + boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)msg->payload()->data())); + if (!tryToUpdateSeq(session->nodeId(), statusSeq)) + { + return; + } + GATEWAY_LOG(INFO) << LOG_BADGE("onReceiveRouterSeq") + << LOG_DESC("receive router seq and request router table") + << LOG_KV("peer", session->nodeId()) << LOG_KV("seq", statusSeq); + // request router table to peer + auto p2pMsg = std::dynamic_pointer_cast(msg); + auto dstP2PNodeID = (!p2pMsg->header()->srcGwNode().empty()) ? p2pMsg->header()->srcGwNode() : + session->nodeId(); + m_service->asyncSendMessageByP2PNodeID((uint16_t)GatewayPacketType::RouterTableRequest, + dstP2PNodeID, std::make_shared()); +} + +bool RouterManager::tryToUpdateSeq(std::string const& _p2pNodeID, uint32_t _seq) +{ + UpgradableGuard l(x_node2Seq); + auto it = m_node2Seq.find(_p2pNodeID); + if (it != m_node2Seq.end() && it->second >= _seq) + { + return false; + } + UpgradeGuard upgradeGuard(l); + m_node2Seq[_p2pNodeID] = _seq; + return true; +} + +// receive routerTable from peers +void RouterManager::onReceivePeersRouterTable(MessageFace::Ptr msg, WsSession::Ptr session) +{ + auto routerTable = m_service->routerTableFactory()->createRouterTable(ref(*(msg->payload()))); + + GATEWAY_LOG(INFO) << LOG_BADGE("onReceivePeersRouterTable") << LOG_KV("peer", session->nodeId()) + << LOG_KV("entrySize", routerTable->routerEntries().size()); + joinRouterTable(session->nodeId(), routerTable); +} + +// receive routerTable request from peer +void RouterManager::onReceiveRouterTableRequest(MessageFace::Ptr msg, WsSession::Ptr session) +{ + GATEWAY_LOG(INFO) << LOG_BADGE("onReceiveRouterTableRequest") + << LOG_KV("peer", session->nodeId()) + << LOG_KV("entrySize", m_service->routerTable()->routerEntries().size()); + + auto routerTableData = std::make_shared(); + m_service->routerTable()->encode(*routerTableData); + auto p2pMsg = std::dynamic_pointer_cast(msg); + auto dstP2PNodeID = (!p2pMsg->header()->srcGwNode().empty()) ? p2pMsg->header()->srcGwNode() : + session->nodeId(); + m_service->asyncSendMessageByP2PNodeID( + (uint16_t)GatewayPacketType::RouterTableResponse, dstP2PNodeID, routerTableData); +} + +void RouterManager::joinRouterTable( + std::string const& _generatedFrom, RouterTableInterface::Ptr _routerTable) +{ + std::set unreachableNodes; + bool updated = false; + auto const& entries = _routerTable->routerEntries(); + for (auto const& it : entries) + { + auto entry = it.second; + if (m_service->routerTable()->update(unreachableNodes, _generatedFrom, entry) && !updated) + { + updated = true; + } + } + + GATEWAY_LOG(INFO) << LOG_BADGE("joinRouterTable") << LOG_DESC("create router entry") + << LOG_KV("dst", _generatedFrom); + + auto entry = m_service->routerTableFactory()->createRouterEntry(); + entry->setDstNode(_generatedFrom); + entry->setDistance(0); + if (m_service->routerTable()->update(unreachableNodes, m_service->nodeID(), entry) && !updated) + { + updated = true; + } + if (!updated) + { + GATEWAY_LOG(DEBUG) << LOG_BADGE("joinRouterTable") << LOG_DESC("router table not updated") + << LOG_KV("dst", _generatedFrom); + return; + } + onP2PNodesUnreachable(unreachableNodes); + m_statusSeq++; + broadcastRouterSeq(); +} + + +// called when the nodes become unreachable +void RouterManager::onP2PNodesUnreachable(std::set const& _p2pNodeIDs) +{ + std::vector> handlers; + { + ReadGuard readGuard(x_unreachableHandlers); + handlers = m_unreachableHandlers; + } + // TODO: async here + for (auto const& node : _p2pNodeIDs) + { + for (auto const& it : m_unreachableHandlers) + { + it(node); + } + } +} + +void RouterManager::broadcastRouterSeq() +{ + m_routerTimer->restart(); + + auto seq = m_statusSeq.load(); + auto statusSeq = boost::asio::detail::socket_ops::host_to_network_long(seq); + auto message = m_service->messageFactory()->buildMessage(); + message->setPacketType((uint16_t)GatewayPacketType::RouterTableSyncSeq); + message->setPayload(std::make_shared((byte*)&statusSeq, (byte*)&statusSeq + 4)); + // the router table should only exchange between neighbor + m_service->broadcastMessage(message); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h new file mode 100644 index 00000000..f99c0182 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouterManager.h + * @author: yujiechen + * @date 2024-08-26 + */ + +#pragma once +#include "../Service.h" +#include "RouterTableInterface.h" +#include + +namespace ppc::gateway +{ +class RouterManager +{ +public: + RouterManager(Service::Ptr service); + virtual ~RouterManager() = default; + + // handlers called when the node is unreachable + void registerUnreachableHandler(std::function _handler) + { + bcos::WriteGuard l(x_unreachableHandlers); + m_unreachableHandlers.emplace_back(_handler); + } + + virtual void start(); + virtual void stop(); + +private: + void onReceiveRouterSeq( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + bool tryToUpdateSeq(std::string const& _p2pNodeID, uint32_t _seq); + void broadcastRouterSeq(); + + void onReceivePeersRouterTable( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + void onReceiveRouterTableRequest( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + + void joinRouterTable(std::string const& _generatedFrom, RouterTableInterface::Ptr _routerTable); + + void onP2PNodesUnreachable(std::set const& _p2pNodeIDs); + +private: + // for message forward + Service::Ptr m_service; + std::shared_ptr m_routerTimer; + std::atomic m_statusSeq{1}; + + // called when the given node unreachable + std::vector> m_unreachableHandlers; + mutable bcos::SharedMutex x_unreachableHandlers; + + std::map m_node2Seq; + mutable bcos::SharedMutex x_node2Seq; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp new file mode 100644 index 00000000..f8a2a22a --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2024 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouterTableImpl.cpp + * @author: yujiechen + * @date 2022-5-24 + */ +#include "RouterTableImpl.h" +#include "ppc-gateway/Common.h" + +using namespace bcos; +using namespace ppc::gateway; +using namespace ppc::protocol; + +void RouterTable::encode(bcos::bytes& _encodedData) +{ + WriteGuard writeGuard(x_routerEntries); + m_inner()->routerEntries.clear(); + // encode m_routerEntries + for (auto const& it : m_routerEntries) + { + auto entry = std::dynamic_pointer_cast(it.second); + m_inner()->routerEntries.emplace_back(entry->inner()); + } + tars::TarsOutputStream output; + m_inner()->writeTo(output); + output.getByteBuffer().swap(_encodedData); +} + +void RouterTable::decode(bcos::bytesConstRef _decodedData) +{ + tars::TarsInputStream input; + input.setBuffer((const char*)_decodedData.data(), _decodedData.size()); + WriteGuard writeGuard(x_routerEntries); + m_inner()->readFrom(input); + // decode into m_routerEntries + m_routerEntries.clear(); + for (auto& it : m_inner()->routerEntries) + { + auto entry = + std::make_shared([m_entry = it]() mutable { return &m_entry; }); + m_routerEntries.insert(std::make_pair(entry->dstNode(), entry)); + } +} + +bool RouterTable::erase(std::set& _unreachableNodes, std::string const& _p2pNodeID) +{ + bool updated = false; + WriteGuard writeGuard(x_routerEntries); + // erase router-entry of the _p2pNodeID + auto it = m_routerEntries.find(_p2pNodeID); + if (it != m_routerEntries.end()) + { + // Note: reset the distance to m_unreachableDistance, to notify that the _p2pNodeID is + // unreachable + it->second->setDistance(m_unreachableDistance); + it->second->clearNextHop(); + _unreachableNodes.insert(it->second->dstNode()); + + GATEWAY_LOG(INFO) << LOG_BADGE("erase") << LOG_DESC("make the router unreachable") + << LOG_KV("dst", _p2pNodeID) << LOG_KV("distance", it->second->distance()) + << LOG_KV("size", m_routerEntries.size()); + updated = true; + } + // update the router-entry with nextHop equal to _p2pNodeID to be unreachable + updateDistanceForAllRouterEntries(_unreachableNodes, _p2pNodeID, m_unreachableDistance); + return updated; +} + +void RouterTable::updateDistanceForAllRouterEntries( + std::set& _unreachableNodes, std::string const& _nextHop, int32_t _newDistance) +{ + for (auto& it : m_routerEntries) + { + auto entry = it.second; + if (entry->nextHop() == _nextHop) + { + auto oldDistance = entry->distance(); + entry->setDistance(_newDistance + (oldDistance - 1)); + if (entry->distance() >= m_unreachableDistance) + { + entry->clearNextHop(); + _unreachableNodes.insert(entry->dstNode()); + } + GATEWAY_LOG(INFO) << LOG_BADGE("updateDistanceForAllRouterEntries") + << LOG_DESC( + "update entry since the nextHop distance has been updated") + << LOG_KV("dst", entry->dstNode()) << LOG_KV("nextHop", _nextHop) + << LOG_KV("distance", entry->distance()) + << LOG_KV("oldDistance", oldDistance) + << LOG_KV("size", m_routerEntries.size()); + } + } +} + +bool RouterTable::update(std::set& _unreachableNodes, + std::string const& _generatedFrom, RouterTableEntryInterface::Ptr _entry) +{ + if (c_fileLogLevel <= TRACE) + [[unlikely]] + { + GATEWAY_LOG(TRACE) << LOG_BADGE("update") << LOG_DESC("receive entry") + << LOG_KV("dst", printP2PIDElegantly(_entry->dstNode())) + << LOG_KV("distance", _entry->distance()) + << LOG_KV("from", _generatedFrom); + } + auto ret = updateDstNodeEntry(_generatedFrom, _entry); + // the dst entry has not been updated + if (!ret) + { + return false; + } + + UpgradableGuard l(x_routerEntries); + auto it = m_routerEntries.find(_entry->dstNode()); + if (it == m_routerEntries.end()) + { + return false; + } + // get the latest distance + auto& currentEntry = it->second; + auto _newDistance = currentEntry->distance(); + if (_newDistance >= m_unreachableDistance) + { + currentEntry->clearNextHop(); + _unreachableNodes.insert(_entry->dstNode()); + } + // the dst entry has updated, update the distance of the router-entries with nextHop equal to + // dstNode + UpgradeGuard upgradeGuard(l); + if (_newDistance == 1) + { + currentEntry->clearNextHop(); + } + updateDistanceForAllRouterEntries(_unreachableNodes, _entry->dstNode(), _newDistance); + return true; +} + +bool RouterTable::updateDstNodeEntry( + std::string const& _generatedFrom, RouterTableEntryInterface::Ptr _entry) +{ + UpgradableGuard upgradableGuard(x_routerEntries); + // the node self + if (_entry->dstNode() == m_nodeID) + { + return false; + } + // insert new entry + auto it = m_routerEntries.find(_entry->dstNode()); + if (it == m_routerEntries.end()) + { + UpgradeGuard upgradeGuard(upgradableGuard); + _entry->incDistance(1); + if (_generatedFrom != m_nodeID) + { + _entry->setNextHop(_generatedFrom); + } + m_routerEntries.insert(std::make_pair(_entry->dstNode(), _entry)); + GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC("insert new entry into the routerTable") + << LOG_KV("distance", _entry->distance()) + << LOG_KV("dst", _entry->dstNode()) + << LOG_KV("nextHop", _entry->nextHop()) + << LOG_KV("size", m_routerEntries.size()); + return true; + } + + // discover smaller distance + auto currentEntry = it->second; + auto currentDistance = currentEntry->distance(); + auto distance = _entry->distance() + 1; + if (currentDistance > distance) + { + UpgradeGuard upgradeGuard(upgradableGuard); + if (_generatedFrom != m_nodeID) + { + currentEntry->setNextHop(_generatedFrom); + } + currentEntry->setDistance(distance); + GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC("discover smaller distance, update entry") + << LOG_KV("distance", currentEntry->distance()) + << LOG_KV("oldDistance", currentDistance) + << LOG_KV("dst", _entry->dstNode()) + << LOG_KV("nextHop", _entry->nextHop()) + << LOG_KV("size", m_routerEntries.size()); + return true; + } + // the distance information for the nextHop changed + if (currentEntry->nextHop() == _generatedFrom) + { + // distance not updated + if (currentEntry->distance() == distance) + { + return false; + } + // unreachable condition + if (currentEntry->distance() >= m_unreachableDistance && distance >= m_unreachableDistance) + { + return false; + } + currentEntry->setDistance(distance); + if (currentEntry->distance() >= m_unreachableDistance) + { + currentEntry->clearNextHop(); + } + GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC( + "distance of the nextHop entry " + "updated, update the current entry") + << LOG_KV("dst", currentEntry->dstNode()) + << LOG_KV("nextHop", currentEntry->nextHop()) + << LOG_KV("distance", currentEntry->distance()) + << LOG_KV("size", m_routerEntries.size()); + return true; + } + return false; +} + +std::string RouterTable::getNextHop(std::string const& _nodeID) +{ + std::string emptyNextHop; + ReadGuard readGuard(x_routerEntries); + auto it = m_routerEntries.find(_nodeID); + if (it == m_routerEntries.end()) + { + return emptyNextHop; + } + if (it->second->distance() >= m_unreachableDistance) + { + return emptyNextHop; + } + return it->second->nextHop(); +} + +std::set RouterTable::getAllReachableNode() +{ + std::set reachableNodes; + ReadGuard readGuard(x_routerEntries); + for (auto const& it : m_routerEntries) + { + auto entry = it.second; + if (entry->distance() < m_unreachableDistance) + { + reachableNodes.insert(entry->dstNode()); + } + } + + if (c_fileLogLevel <= LogLevel::TRACE) + [[unlikely]] + { + std::stringstream nodes; + std::for_each(reachableNodes.begin(), reachableNodes.end(), + [&](const auto& item) { nodes << printP2PIDElegantly(item) << ","; }); + GATEWAY_LOG(TRACE) << LOG_BADGE("getAllReachableNode") + << LOG_KV("nodes size", reachableNodes.size()) + << LOG_KV("nodes", nodes.str()); + } + + return reachableNodes; +} diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h new file mode 100644 index 00000000..72eda232 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2024 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouterTableImpl.h + * @author: yujiechen + * @date 2022-5-24 + */ +#pragma once +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wunused-parameter" + +#include "RouterTableInterface.h" +#include +#include +#include + +namespace ppc::gateway +{ +class RouterTableEntry : public RouterTableEntryInterface +{ +public: + using Ptr = std::shared_ptr; + RouterTableEntry() + : m_inner([m_entry = ppctars::RouterTableEntry()]() mutable { return &m_entry; }) + {} + RouterTableEntry(std::function _inner) + : m_inner(std::move(_inner)) + {} + RouterTableEntry(RouterTableEntry&&) = delete; + RouterTableEntry(const RouterTableEntry&) = delete; + RouterTableEntry& operator=(const RouterTableEntry&) = delete; + RouterTableEntry& operator=(RouterTableEntry&&) = delete; + ~RouterTableEntry() override = default; + + void setDstNode(std::string const& _dstNode) override { m_inner()->dstNode = _dstNode; } + void setNextHop(std::string const& _nextHop) override { m_inner()->nextHop = _nextHop; } + void clearNextHop() override { m_inner()->nextHop = std::string(); } + void setDistance(int32_t _distance) override { m_inner()->distance = _distance; } + void incDistance(int32_t _deltaDistance) override { m_inner()->distance += _deltaDistance; } + + std::string const& dstNode() const override { return m_inner()->dstNode; } + std::string const& nextHop() const override { return m_inner()->nextHop; } + int32_t distance() const override { return m_inner()->distance; } + + ppctars::RouterTableEntry const& inner() const { return *(m_inner()); } + +private: + std::function m_inner; +}; + +class RouterTable : public RouterTableInterface +{ +public: + using Ptr = std::shared_ptr; + RouterTable() : m_inner([m_table = ppctars::RouterTable()]() mutable { return &m_table; }) {} + RouterTable(bcos::bytesConstRef _decodedData) : RouterTable() { decode(_decodedData); } + RouterTable(RouterTable&&) = delete; + RouterTable(const RouterTable&) = delete; + RouterTable& operator=(const RouterTable&) = delete; + RouterTable& operator=(RouterTable&&) = delete; + ~RouterTable() override = default; + + void encode(bcos::bytes& _encodedData) override; + void decode(bcos::bytesConstRef _decodedData) override; + + std::map const& routerEntries() override + { + return m_routerEntries; + } + // append the unreachableNodes into param _unreachableNodes + bool update(std::set& _unreachableNodes, std::string const& _generatedFrom, + RouterTableEntryInterface::Ptr _entry) override; + // append the unreachableNodes into param _unreachableNodes + bool erase(std::set& _unreachableNodes, std::string const& _p2pNodeID) override; + + void setNodeID(std::string const& _nodeID) override { m_nodeID = _nodeID; } + std::string const& nodeID() const override { return m_nodeID; } + + void setUnreachableDistance(int _unreachableDistance) override + { + m_unreachableDistance = _unreachableDistance; + } + + int unreachableDistance() const override { return m_unreachableDistance; } + std::string getNextHop(std::string const& _nodeID) override; + std::set getAllReachableNode() override; + + bool updateDstNodeEntry( + std::string const& _generatedFrom, RouterTableEntryInterface::Ptr _entry); + void updateDistanceForAllRouterEntries(std::set& _unreachableNodes, + std::string const& _nextHop, int32_t _newDistance); + +private: + std::string m_nodeID; + std::function m_inner; + std::map m_routerEntries; + mutable bcos::SharedMutex x_routerEntries; + + int m_unreachableDistance = 10; +}; + +class RouterTableFactoryImpl : public RouterTableFactory +{ +public: + using Ptr = std::shared_ptr; + RouterTableInterface::Ptr createRouterTable() override + { + return std::make_shared(); + } + RouterTableInterface::Ptr createRouterTable(bcos::bytesConstRef _decodedData) override + { + return std::make_shared(_decodedData); + } + + RouterTableEntryInterface::Ptr createRouterEntry() override + { + return std::make_shared(); + } +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h new file mode 100644 index 00000000..e60d3096 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2024 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouterTableInterface.h + * @author: yujiechen + * @date 2022-5-24 + */ +#pragma once +#include +#include +#include +namespace ppc::gateway +{ +class RouterTableEntryInterface +{ +public: + using Ptr = std::shared_ptr; + RouterTableEntryInterface() = default; + RouterTableEntryInterface(const RouterTableEntryInterface&) = delete; + RouterTableEntryInterface(RouterTableEntryInterface&&) = delete; + RouterTableEntryInterface& operator=(RouterTableEntryInterface&&) = delete; + RouterTableEntryInterface& operator=(const RouterTableEntryInterface&) = delete; + virtual ~RouterTableEntryInterface() = default; + + virtual void setDstNode(std::string const& _dstNode) = 0; + virtual void setNextHop(std::string const& _nextHop) = 0; + virtual void clearNextHop() = 0; + virtual void setDistance(int32_t _distance) = 0; + virtual void incDistance(int32_t _deltaDistance) = 0; + + virtual std::string const& dstNode() const = 0; + virtual std::string const& nextHop() const = 0; + virtual int32_t distance() const = 0; +}; + +class RouterTableInterface +{ +public: + using Ptr = std::shared_ptr; + RouterTableInterface() = default; + RouterTableInterface(const RouterTableInterface&) = delete; + RouterTableInterface(RouterTableInterface&&) = delete; + RouterTableInterface& operator=(RouterTableInterface&&) = delete; + RouterTableInterface& operator=(const RouterTableInterface&) = delete; + virtual ~RouterTableInterface() = default; + + virtual bool update(std::set& _unreachableNodes, std::string const& _generatedFrom, + RouterTableEntryInterface::Ptr _entry) = 0; + virtual bool erase(std::set& _unreachableNodes, std::string const& _p2pNodeID) = 0; + + virtual std::map const& routerEntries() = 0; + + virtual void setNodeID(std::string const& _nodeID) = 0; + virtual std::string const& nodeID() const = 0; + virtual void setUnreachableDistance(int _unreachableDistance) = 0; + virtual int unreachableDistance() const = 0; + virtual std::string getNextHop(std::string const& _nodeID) = 0; + virtual std::set getAllReachableNode() = 0; + + virtual void encode(bcos::bytes& _encodedData) = 0; + virtual void decode(bcos::bytesConstRef _decodedData) = 0; +}; + +class RouterTableFactory +{ +public: + using Ptr = std::shared_ptr; + RouterTableFactory() = default; + RouterTableFactory(RouterTableFactory&&) = delete; + RouterTableFactory(const RouterTableFactory&) = delete; + RouterTableFactory& operator=(const RouterTableFactory&) = delete; + RouterTableFactory& operator=(RouterTableFactory&&) = delete; + virtual ~RouterTableFactory() = default; + + virtual RouterTableInterface::Ptr createRouterTable() = 0; + virtual RouterTableInterface::Ptr createRouterTable(bcos::bytesConstRef _decodedData) = 0; + virtual RouterTableEntryInterface::Ptr createRouterEntry() = 0; +}; + +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-protocol/src/JsonTaskImpl.h b/cpp/ppc-protocol/src/JsonTaskImpl.h index dea185b8..b0883b31 100644 --- a/cpp/ppc-protocol/src/JsonTaskImpl.h +++ b/cpp/ppc-protocol/src/JsonTaskImpl.h @@ -30,14 +30,19 @@ class JsonTaskImpl : public Task { public: using Ptr = std::shared_ptr; - JsonTaskImpl(std::string const& _selfPartyID, std::string const& _prePath = "data") : m_selfPartyID(_selfPartyID), m_prePath(_prePath) + JsonTaskImpl(std::string const& _selfPartyID, std::string const& _prePath = "data") + : m_selfPartyID(_selfPartyID), m_prePath(_prePath) {} - JsonTaskImpl(std::string const& _selfPartyID, std::string_view _taskData, std::string const& _prePath = "data") : JsonTaskImpl(_selfPartyID, _prePath) + JsonTaskImpl(std::string const& _selfPartyID, std::string_view _taskData, + std::string const& _prePath = "data") + : JsonTaskImpl(_selfPartyID, _prePath) { decode(_taskData); } - JsonTaskImpl(std::string const& _selfPartyID, Json::Value const& _taskJson, std::string const& _prePath = "data") : JsonTaskImpl(_selfPartyID, _prePath) + JsonTaskImpl(std::string const& _selfPartyID, Json::Value const& _taskJson, + std::string const& _prePath = "data") + : JsonTaskImpl(_selfPartyID, _prePath) { decodeJsonValue(_taskJson); } @@ -64,10 +69,7 @@ class JsonTaskImpl : public Task return m_peerParties; } - std::vector const& getReceiverLists() const override - { - return m_receiverLists; - } + std::vector const& getReceiverLists() const override { return m_receiverLists; } // params of the task, can be deserialized using json std::string const& param() const override { return m_param; } @@ -89,11 +91,8 @@ class JsonTaskImpl : public Task { m_syncResultToPeer = _syncResultToPeer; } - void setLowBandwidth(bool _lowBandwidth) override - { - m_lowBandwidth = _lowBandwidth; - } - + void setLowBandwidth(bool _lowBandwidth) override { m_lowBandwidth = _lowBandwidth; } + // decode the task void decode(std::string_view _taskData) override; virtual void decodeJsonValue(Json::Value const& root); diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp new file mode 100644 index 00000000..08ba9cdd --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp @@ -0,0 +1,146 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageHeaderImpl.cpp + * @author: yujiechen + * @date 2024-08-23 + */ +#include "MessageHeaderImpl.h" +#include "ppc-framework/Common.h" +#include "ppc-utilities/Utilities.h" +#include + +using namespace ppc::protocol; +using namespace bcos; +using namespace ppc; + +void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const +{ + // the componentType + uint16_t componentType = + boost::asio::detail::socket_ops::host_to_network_short(m_componentType); + buffer.insert(buffer.end(), (byte*)&componentType, (byte*)&componentType + 2); + // the source nodeID that send the message + uint16_t srcNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_srcNode.size()); + buffer.insert(buffer.end(), (byte*)&srcNodeLen, (byte*)&srcNodeLen + 2); + buffer.insert(buffer.end(), m_srcNode.begin(), m_srcNode.end()); + // the target nodeID that should receive the message + uint16_t dstNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstNode.size()); + buffer.insert(buffer.end(), (byte*)&dstNodeLen, (byte*)&dstNodeLen + 2); + buffer.insert(buffer.end(), m_dstNode.begin(), m_dstNode.end()); + bcos::bytes m_dstNode; + // the target agency that need receive the message + uint16_t dstInstLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstInst.size()); + buffer.insert(buffer.end(), (byte*)&dstInstLen, (byte*)&dstInstLen + 2); + buffer.insert(buffer.end(), m_dstInst.begin(), m_dstInst.end()); +} + + +int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t const _offset) +{ + auto offset = _offset; + CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, data.size()); + // the componentType + auto pointer = data.data() + offset; + m_componentType = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // srcNode + offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), (pointer - data.data())); + // dstNode + offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset); + // dstInst + offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset); + return offset; +} + +void MessageHeaderImpl::encode(bcos::bytes& buffer) const +{ + buffer.clear(); + // the version, 2Bytes + uint16_t version = boost::asio::detail::socket_ops::host_to_network_short(m_version); + buffer.insert(buffer.end(), (byte*)&version, (byte*)&version + 2); + // the packetType, 2Bytes + uint16_t packetType = boost::asio::detail::socket_ops::host_to_network_short(m_packetType); + buffer.insert(buffer.end(), (byte*)&packetType, (byte*)&packetType + 2); + // the ttl, 2Bytes + uint16_t ttl = boost::asio::detail::socket_ops::host_to_network_short(m_ttl); + buffer.insert(buffer.end(), (byte*)&ttl, (byte*)&ttl + 2); + // the ext, 2Bytes + uint16_t ext = boost::asio::detail::socket_ops::host_to_network_short(m_ext); + buffer.insert(buffer.end(), (byte*)&ext, (byte*)&ext + 2); + // the traceID, 2+Bytes + uint16_t traceIDLen = boost::asio::detail::socket_ops::host_to_network_short(m_traceID.size()); + buffer.insert(buffer.end(), (byte*)&traceIDLen, (byte*)&traceIDLen + 2); + buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end()); + // srcGwNode, 2+Bytes + uint16_t srcGwNodeLen = + boost::asio::detail::socket_ops::host_to_network_short(m_srcGwNode.size()); + buffer.insert(buffer.end(), (byte*)&srcGwNodeLen, (byte*)&srcGwNodeLen + 2); + buffer.insert(buffer.end(), m_srcGwNode.begin(), m_srcGwNode.end()); + // dstGwNode, 2+Bytes + uint16_t dstGwNodeLen = + boost::asio::detail::socket_ops::host_to_network_short(m_dstGwNode.size()); + buffer.insert(buffer.end(), (byte*)&dstGwNodeLen, (byte*)&dstGwNodeLen + 2); + buffer.insert(buffer.end(), m_dstGwNode.begin(), m_dstGwNode.end()); + if (!hasOptionalField()) + { + return; + } + // encode the optionalField + m_optionalField->encode(buffer); + m_length = buffer.size(); +} + +int64_t MessageHeaderImpl::decode(bcos::bytesConstRef data) +{ + if (data.size() < MESSAGE_MIN_LENGTH) + { + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment("Malform message for too small!")); + } + auto pointer = data.data(); + // the version + m_version = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // the pacektType + m_packetType = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // the ttl + m_ttl = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // the ext + m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // the traceID + bcos::bytes traceIDData; + auto offset = + decodeNetworkBuffer(traceIDData, data.data(), data.size(), (pointer - data.data())); + m_traceID = std::string(traceIDData.begin(), traceIDData.end()); + // srcGwNode + bcos::bytes srcGWNodeData; + offset = decodeNetworkBuffer(srcGWNodeData, data.data(), data.size(), offset); + m_srcGwNode = std::string(srcGWNodeData.begin(), srcGWNodeData.end()); + // dstGwNode + bcos::bytes dstGWNodeData; + offset = decodeNetworkBuffer(dstGWNodeData, data.data(), data.size(), offset); + m_dstGwNode = std::string(dstGWNodeData.begin(), dstGWNodeData.end()); + // optionalField + if (hasOptionalField()) + { + offset = m_optionalField->decode(data, offset); + } + m_length = offset; + return offset; +} diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h new file mode 100644 index 00000000..a8dda5a9 --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h @@ -0,0 +1,82 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageHeaderImpl.h + * @author: yujiechen + * @date 2024-08-23 + */ +#pragma once +#include "ppc-framework/gateway/GatewayProtocol.h" +#include "ppc-framework/protocol/Message.h" + +namespace ppc::protocol +{ +class MessageOptionalHeaderImpl : public MessageOptionalHeader +{ +public: + using Ptr = std::shared_ptr; + MessageOptionalHeaderImpl() = default; + MessageOptionalHeaderImpl(bcos::bytesConstRef data, uint64_t const offset) + { + decode(data, offset); + } + + ~MessageOptionalHeaderImpl() override = default; + + void encode(bcos::bytes& buffer) const override; + int64_t decode(bcos::bytesConstRef data, uint64_t const offset) override; +}; + +class MessageHeaderImpl : public MessageHeader +{ +public: + using Ptr = std::shared_ptr; + MessageHeaderImpl() { m_optionalField = std::make_shared(); } + MessageHeaderImpl(bcos::bytesConstRef data) { decode(data); } + ~MessageHeaderImpl() override {} + + void encode(bcos::bytes& buffer) const override; + int64_t decode(bcos::bytesConstRef data) override; + + virtual bool hasOptionalField() const + { + return m_packetType == (uint16_t)ppc::gateway::GatewayPacketType::P2PMessage; + } + + bool isRespPacket() const override + { + return m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::Response; + } + void setRespPacket() override { m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::Response; } + +private: + // version(2) + packetType(2) + ttl(2) + ext(2) + traceIDLen(2) + srcGwNodeLen(2) + dstGwNode(2) + const size_t MESSAGE_MIN_LENGTH = 14; +}; + +class MessageHeaderBuilderImpl : public MessageHeaderBuilder +{ +public: + using Ptr = std::shared_ptr; + MessageHeaderBuilderImpl() = default; + ~MessageHeaderBuilderImpl() {} + + MessageHeader::Ptr build(bcos::bytesConstRef data) override + { + return std::make_shared(data); + } + MessageHeader::Ptr build() override { return std::make_shared(); } +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessageImpl.cpp b/cpp/ppc-protocol/src/v1/MessageImpl.cpp new file mode 100644 index 00000000..09f4183a --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessageImpl.cpp @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageImpl.cpp + * @author: yujiechen + * @date 2024-08-23 + */ + +#include "MessageImpl.h" + +using namespace bcos; +using namespace ppc::protocol; + +bool MessageImpl::encode(bcos::bytes& _buffer) +{ + // encode the header + bcos::bytes headerData; + m_header->encode(headerData); + // encode the payload + if (m_payload) + { + headerData.insert(headerData.end(), m_payload->begin(), m_payload->end()); + } +} + +bool MessageImpl::encode(bcos::boostssl::EncodedMsg& encodedMsg) +{ + // header + m_header->encode(encodedMsg.header); + // assign the payload back + encodedMsg.payload = m_payload; +} + +int64_t MessageImpl::decode(bytesConstRef buffer) +{ + if (buffer.size() > m_maxMessageLen) + { + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "Malform message for over the size limit, max allowed size is: " + + std::to_string(m_maxMessageLen))); + } + // decode the header + m_header = m_headerBuilder->build(buffer); + // decode the payload + if (!m_payload) + { + m_payload = std::make_shared(); + } + m_payload->clear(); + m_payload->insert(m_payload->end(), buffer.data() + m_header->length(), buffer.end()); +} \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessageImpl.h b/cpp/ppc-protocol/src/v1/MessageImpl.h new file mode 100644 index 00000000..208db1b3 --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessageImpl.h @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageImpl.h + * @author: yujiechen + * @date 2024-08-23 + */ +#pragma once +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/Message.h" + +namespace ppc::protocol +{ +class MessageImpl : public Message +{ +public: + using Ptr = std::shared_ptr; + MessageImpl(MessageHeaderBuilder::Ptr headerBuilder, size_t maxMessageLen) + : m_headerBuilder(std::move(headerBuilder)), m_maxMessageLen(maxMessageLen) + {} + MessageImpl( + MessageHeaderBuilder::Ptr headerBuilder, size_t maxMessageLen, bcos::bytesConstRef buffer) + : MessageImpl(headerBuilder, maxMessageLen) + { + decode(buffer); + } + + ~MessageImpl() override = default; + + bool encode(bcos::bytes& _buffer) override; + // encode and return the {header, payload} + bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) override; + int64_t decode(bcos::bytesConstRef _buffer) override; + +private: + MessageHeaderBuilder::Ptr m_headerBuilder; + + // default max message length is 100MB + size_t m_maxMessageLen = 100 * 1024 * 1024; +}; + +class MessageBuilderImpl : public MessageBuilder +{ +public: + using Ptr = std::shared_ptr; + MessageBuilderImpl(MessageHeaderBuilder::Ptr msgHeaderBuilder) + : m_msgHeaderBuilder(std::move(msgHeaderBuilder)) + {} + + MessageBuilderImpl(MessageHeaderBuilder::Ptr msgHeaderBuilder, size_t maxMessageLen) + : MessageBuilderImpl(std::move(msgHeaderBuilder)) + { + m_maxMessageLen = maxMessageLen; + } + + ~MessageBuilderImpl() override {} + + Message::Ptr build() override + { + return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen); + } + Message::Ptr build(bcos::bytesConstRef buffer) override + { + return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen, buffer); + } + +private: + MessageHeaderBuilder::Ptr m_msgHeaderBuilder; + // default max message length is 100MB + size_t m_maxMessageLen = 100 * 1024 * 1024; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp new file mode 100644 index 00000000..59397edf --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessagePayloadImpl.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#include "MessagePayloadImpl.h" + +#include "ppc-utilities/Utilities.h" +#include + +using namespace ppc::protocol; +using namespace bcos; + +int64_t MessagePayloadImpl::encode(bcos::bytes& buffer) const +{ + // version + uint16_t version = boost::asio::detail::socket_ops::host_to_network_short(m_version); + buffer.insert(buffer.end(), (byte*)&version, (byte*)&version + 2); + // topic + uint16_t topicLen = boost::asio::detail::socket_ops::host_to_network_short(m_topic.size()); + buffer.insert(buffer.end(), (byte*)&topicLen, (byte*)&topicLen + 2); + buffer.insert(buffer.end(), m_topic.begin(), m_topic.end()); + // data + uint16_t dataLen = boost::asio::detail::socket_ops::host_to_network_short(m_data.size()); + buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 2); + buffer.insert(buffer.end(), m_data.begin(), m_data.end()); + // update the length + m_length = buffer.size(); + return m_length; +} + +int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) +{ + // check the message + if (buffer.size() < MIN_PAYLOAD_LEN) + { + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment("Malform payload for too small!")); + } + auto pointer = buffer.data(); + // the version + m_version = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // topic + bcos::bytes topicData; + auto offset = + decodeNetworkBuffer(topicData, buffer.data(), buffer.size(), (pointer - buffer.data())); + m_topic = std::string(topicData.begin(), topicData.end()); + // data + offset = decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset); + return offset; +} \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h new file mode 100644 index 00000000..f7f5a3e8 --- /dev/null +++ b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessagePayloadImpl.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/MessagePayload.h" + +namespace ppc::protocol +{ +class MessagePayloadImpl : public MessagePayload +{ +public: + using Ptr = std::shared_ptr; + MessagePayloadImpl() = default; + MessagePayloadImpl(bcos::bytesConstRef buffer) { decode(buffer); } + ~MessagePayloadImpl() override {} + + int64_t encode(bcos::bytes& buffer) const override; + int64_t decode(bcos::bytesConstRef data) override; + +private: + const unsigned int MIN_PAYLOAD_LEN = 6; +}; + + +class MessagePayloadBuilderImpl : public MessagePayloadBuilder +{ +public: + using Ptr = std::shared_ptr; + MessagePayloadBuilderImpl() = default; + ~MessagePayloadBuilderImpl() override {} + MessagePayload::Ptr build() override { return std::make_shared(); } + MessagePayload::Ptr build(bcos::bytesConstRef buffer) override + { + return std::make_shared(buffer); + } +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars new file mode 100644 index 00000000..48096cdd --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars @@ -0,0 +1,13 @@ +module ppctars +{ +struct RouterTableEntry +{ + 1 require string dstNode; + 2 optional string nextHop; + 3 require int distance; +}; +struct RouterTable +{ + 1 optional vector routerEntries; +}; +}; \ No newline at end of file diff --git a/cpp/ppc-tools/src/codec/CodecUtility.h b/cpp/ppc-tools/src/codec/CodecUtility.h index ce7a208a..d8f83201 100644 --- a/cpp/ppc-tools/src/codec/CodecUtility.h +++ b/cpp/ppc-tools/src/codec/CodecUtility.h @@ -20,7 +20,6 @@ #pragma once #include "openssl/bn.h" #include "ppc-framework/libwrapper/BigNum.h" -#include "ppc-utilities/Utilities.h" #include #include #include diff --git a/cpp/ppc-utilities/Utilities.h b/cpp/ppc-utilities/Utilities.h index b9b8e0ae..4b04ea21 100644 --- a/cpp/ppc-utilities/Utilities.h +++ b/cpp/ppc-utilities/Utilities.h @@ -20,20 +20,22 @@ #pragma once #include "ppc-framework/Common.h" +#include namespace ppc { inline uint64_t decodeNetworkBuffer( bcos::bytes& _result, bcos::byte const* buffer, unsigned int bufferLen, uint64_t const offset) { - CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, bufferLen); + uint64_t curOffset = offset; + CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); auto dataLen = - boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)buffer + offset)); - offset += 2; - CHECK_OFFSET_WITH_THROW_EXCEPTION(offset, bufferLen); - buffer.insert( - buffer.end(), (bcos::byte*)_buffer + offset, (bcos::byte*)_buffer + offset + dataLen); - offset += dataLen; - return offset; + boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)buffer + curOffset)); + curOffset += 2; + CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); + _result.insert( + _result.end(), (bcos::byte*)buffer + curOffset, (bcos::byte*)buffer + curOffset + dataLen); + curOffset += dataLen; + return curOffset; } } // namespace ppc \ No newline at end of file diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json index 5913a7c1..a6334850 100644 --- a/cpp/vcpkg-configuration.json +++ b/cpp/vcpkg-configuration.json @@ -2,8 +2,8 @@ "registries": [ { "kind": "git", - "repository": "https://github.com/FISCO-BCOS/registry", - "baseline": "070f336149afdac5cc9ace97df01de7ee31aab30", + "repository": "https://github.com/cyjseagull/registry", + "baseline": "6160f4167d4ab801f16a18bdd842a75edf7b069e", "packages": [ "openssl", "bcos-utilities", From 6dcefa067b912dd5ac1db1c83cb3dab9ef0f727f Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 27 Aug 2024 19:39:32 +0800 Subject: [PATCH 015/120] add gateway router (#11) --- cpp/ppc-framework/front/FrontConfig.h | 95 ++++++++ cpp/ppc-framework/front/IFront.h | 106 +++++++++ cpp/ppc-framework/gateway/GatewayProtocol.h | 12 + cpp/ppc-framework/gateway/IGateway.h | 70 ++++++ cpp/ppc-framework/protocol/INodeInfo.h | 65 ++++++ cpp/ppc-framework/protocol/Message.h | 42 +++- cpp/ppc-framework/protocol/MessagePayload.h | 11 +- cpp/ppc-framework/protocol/RouteType.h | 54 +++++ .../ppc-gateway/gateway/GatewayImpl.cpp | 195 +++++++++++++++++ .../ppc-gateway/gateway/GatewayImpl.h | 88 ++++++++ .../gateway/SendMessageWithRetry.cpp | 114 ++++++++++ .../gateway/SendMessageWithRetry.h | 62 ++++++ .../gateway/cache/MessageCache.cpp | 94 ++++++++ .../ppc-gateway/gateway/cache/MessageCache.h | 70 ++++++ .../gateway/router/GatewayNodeInfo.h | 74 +++++++ .../gateway/router/GatewayNodeInfoImpl.cpp | 206 ++++++++++++++++++ .../gateway/router/GatewayNodeInfoImpl.h | 100 +++++++++ .../gateway/router/LocalRouter.cpp | 122 +++++++++++ .../ppc-gateway/gateway/router/LocalRouter.h | 71 ++++++ .../gateway/router/PeerRouterTable.cpp | 138 ++++++++++++ .../gateway/router/PeerRouterTable.h | 57 +++++ cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp | 24 ++ cpp/ppc-gateway/ppc-gateway/p2p/Service.h | 10 +- .../ppc-gateway/p2p/router/RouterManager.h | 1 + cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp | 63 +++++- cpp/ppc-protocol/src/v1/MessageHeaderImpl.h | 3 + cpp/ppc-protocol/src/v1/MessageImpl.h | 28 +++ .../src/v1/MessagePayloadImpl.cpp | 19 +- cpp/ppc-protocol/src/v1/MessagePayloadImpl.h | 1 + .../ppc-tars-protocol/impl/NodeInfoImpl.cpp | 40 ++++ .../ppc-tars-protocol/impl/NodeInfoImpl.h | 88 ++++++++ .../ppc-tars-protocol/tars/NodeInfo.tars | 15 ++ 32 files changed, 2102 insertions(+), 36 deletions(-) create mode 100644 cpp/ppc-framework/front/FrontConfig.h create mode 100644 cpp/ppc-framework/front/IFront.h create mode 100644 cpp/ppc-framework/gateway/IGateway.h create mode 100644 cpp/ppc-framework/protocol/INodeInfo.h create mode 100644 cpp/ppc-framework/protocol/RouteType.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h create mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h new file mode 100644 index 00000000..e9775c79 --- /dev/null +++ b/cpp/ppc-framework/front/FrontConfig.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontConfig.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#pragma once +#include +#include +#include + +namespace ppc::front +{ +/** + * @brief the gateway endpoint + * + */ +class GatewayEndPoint +{ +public: + GatewayEndPoint() = default; + GatewayEndPoint(std::string const& host, uint16_t port) : m_host(std::move(host)), m_port(port) + {} + virtual ~GatewayEndPoint() = default; + + virtual std::string const& host() const { return m_host; } + uint16_t port() const { return m_port; } + + void setHost(std::string host) { m_host = std::move(host); } + void setPort(uint16_t port) { m_port = port; } + +private: + // the host + std::string m_host; + // the port + uint16_t m_port; +}; + +// Note: swig explosed interface +class FrontConfig +{ +public: + using Ptr = std::shared_ptr; + FrontConfig(int threadPoolSize, std::string agencyID) + : m_threadPoolSize(threadPoolSize), m_agencyID(std::move(agencyID)) + {} + virtual ~FrontConfig() = default; + + virtual int threadPoolSize() const { return m_threadPoolSize; } + virtual std::string const agencyID() const { return m_agencyID; } + virtual std::vector const& gatewayInfo() const { return m_gatewayInfo; } + virtual void setGatewayInfo(std::vector gatewayInfo) + { + m_gatewayInfo = std::move(gatewayInfo); + } + + virtual void appendGatewayInfo(GatewayEndPoint&& endpoint) + { + // TODO:check the endpoint + m_gatewayInfo.push_back(endpoint); + } + +private: + int m_threadPoolSize; + std::string m_agencyID; + std::vector m_gatewayInfo; +}; + +class FrontConfigBuilder +{ +public: + using Ptr = std::shared_ptr; + FrontConfigBuilder() = default; + virtual ~FrontConfigBuilder() = default; + + FrontConfig::Ptr build(int threadPoolSize, std::string agencyID) + { + return std::make_shared(threadPoolSize, agencyID); + } +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h new file mode 100644 index 00000000..be850b65 --- /dev/null +++ b/cpp/ppc-framework/front/IFront.h @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IFront.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include "FrontConfig.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/RouteType.h" +#include + +namespace ppc::front +{ +class IFront +{ +public: + using Ptr = std::shared_ptr; + + IFront() = default; + virtual ~IFront() = default; + + /** + * @brief start the IFront + * + * @param front the IFront to start + */ + virtual void start() const = 0; + /** + * @brief stop the IFront + * + * @param front the IFront to stop + */ + virtual void stop() const = 0; + + /** + * + * @param front the front object + * @param topic the topic + * @param callback the callback called when receive specified topic + */ + virtual void registerTopicHandler( + std::string const& topic, ppc::protocol::MessageCallback callback) = 0; + + /** + * @brief async send message + * + * @param routeType the route type + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ + virtual void asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload, int seq, long timeout, ppc::protocol::MessageCallback callback) = 0; + + // the sync interface for async_send_message + virtual ppc::protocol::Message::Ptr push(ppc::protocol::RouteType routeType, std::string topic, + std::string dstInst, std::string dstNodeID, std::string const& componentType, + bcos::bytes&& payload, int seq, long timeout) = 0; + + /** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ + virtual void onReceiveMessage( + ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) = 0; +}; + +class IFrontBuilder +{ +public: + using Ptr = std::shared_ptr; + IFrontBuilder() = default; + virtual ~IFrontBuilder() = default; + + /** + * @brief create the Front using specified config + * + * @param config the config used to build the Front + * @return IFront::Ptr he created Front + */ + virtual IFront::Ptr build(ppc::front::FrontConfig::Ptr config) const = 0; + virtual IFront::Ptr buildClient(std::string endPoint) const = 0; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/gateway/GatewayProtocol.h b/cpp/ppc-framework/gateway/GatewayProtocol.h index c3c0a94d..b91137d6 100644 --- a/cpp/ppc-framework/gateway/GatewayProtocol.h +++ b/cpp/ppc-framework/gateway/GatewayProtocol.h @@ -25,6 +25,7 @@ namespace ppc::gateway enum class GatewayPacketType : uint16_t { P2PMessage = 0x00, + BroadcastMessage = 0x01, RouterTableSyncSeq = 0x10, RouterTableResponse = 0x11, RouterTableRequest = 0x12 @@ -33,5 +34,16 @@ enum class GatewayPacketType : uint16_t enum class GatewayMsgExtFlag : uint16_t { Response = 0x1, + RouteByNodeID = 0x2, + RouteByAgency = 0x4, + RouteByComponent = 0x8, + RouteByTopic = 0x10 +}; + +enum CommonError : int32_t +{ + SUCCESS = 0, + TIMEOUT = 1000, // for gateway + NotFoundFrontServiceDispatchMsg = 1001 }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h new file mode 100644 index 00000000..9af59dba --- /dev/null +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IGateway.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "../protocol/INodeInfo.h" +#include "../protocol/Message.h" +#include "../protocol/RouteType.h" +#include + + +namespace ppc::gateway +{ +using ErrorCallbackFunc = std::function; +/** + * @brief: A list of interfaces provided by the gateway which are called by the front service. + */ +class IGateway +{ +public: + using Ptr = std::shared_ptr; + IGateway() = default; + virtual ~IGateway() {} + + /** + * @brief: start/stop service + */ + virtual void start() = 0; + virtual void stop() = 0; + + /** + * @brief send message to gateway + * + * @param routeType the route type + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ + virtual void asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) = 0; + + virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo); + virtual void unRegisterNodeInfo(bcos::bytesConstRef nodeID); + virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic); + virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic); +}; + +} // namespace ppc::gateway diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h new file mode 100644 index 00000000..859d6ac6 --- /dev/null +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file INodeInfo.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/front/IFront.h" +#include + +namespace ppc::protocol +{ +// the node information +class INodeInfo +{ +public: + using Ptr = std::shared_ptr; + INodeInfo() = default; + virtual ~INodeInfo() = default; + + virtual std::string const& endPoint() const = 0; + virtual bcos::bytesConstRef nodeID() const = 0; + + // components + virtual void setComponents(std::vector const& components) = 0; + virtual std::set const& components() const = 0; + + virtual void encode(bcos::bytes& data) const = 0; + virtual void decode(bcos::bytesConstRef data) = 0; + + virtual void setFront(ppc::front::IFront::Ptr&& front) = 0; + virtual ppc::front::IFront::Ptr const& getFront() const = 0; + + virtual bool equal(INodeInfo::Ptr const& info) + { + return (nodeID() == info->nodeID()) && (components() == info->components()); + } +}; +class INodeInfoFactory +{ +public: + using Ptr = std::shared_ptr; + INodeInfoFactory(bcos::bytes nodeID) : m_nodeID(std::move(nodeID)) {} + virtual ~INodeInfoFactory() = default; + + virtual INodeInfo::Ptr build() = 0; + virtual INodeInfo::Ptr build(std::string const& endPoint) = 0; + +protected: + bcos::bytes m_nodeID; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 40c5f614..b318c1c0 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -19,12 +19,13 @@ */ #pragma once #include "../Common.h" +#include "RouteType.h" #include #include +#include #include #include #include - namespace ppc::protocol { class MessageOptionalHeader @@ -38,8 +39,8 @@ class MessageOptionalHeader virtual int64_t decode(bcos::bytesConstRef data, uint64_t const _offset) = 0; // the componentType - virtual uint8_t componentType() const { return m_componentType; } - virtual void setComponentType(uint8_t componentType) { m_componentType = componentType; } + virtual std::string componentType() const { return m_componentType; } + virtual void setComponentType(std::string componentType) { m_componentType = componentType; } // the source nodeID that send the message virtual bcos::bytes const& srcNode() const { return m_srcNode; } @@ -50,18 +51,24 @@ class MessageOptionalHeader virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; } // the target agency that need receive the message - virtual bcos::bytes const& dstInst() const { return m_dstInst; } - virtual void setDstInst(bcos::bytes const& dstInst) { m_dstInst = dstInst; } + virtual std::string const& dstInst() const { return m_dstInst; } + virtual void setDstInst(std::string const& dstInst) { m_dstInst = dstInst; } + + // the topic + virtual std::string const& topic() const { return m_topic; } + virtual void setTopic(std::string&& topic) { m_topic = std::move(topic); } + virtual void setTopic(std::string const& topic) { m_topic = topic; } protected: + std::string m_topic; // the componentType - uint8_t m_componentType; + std::string m_componentType; // the source nodeID that send the message bcos::bytes m_srcNode; // the target nodeID that should receive the message bcos::bytes m_dstNode; // the target agency that need receive the message - bcos::bytes m_dstInst; + std::string m_dstInst; }; class MessageHeader @@ -117,9 +124,12 @@ class MessageHeader // Note: only for log std::string_view dstP2PNodeIDView() const { return printP2PIDElegantly(m_dstGwNode); } + virtual uint16_t routeType() const = 0; + virtual void setRouteType(ppc::protocol::RouteType type) = 0; + protected: // the msg version, used to support compatibility - uint8_t m_version; + uint8_t m_version = 0; // the traceID std::string m_traceID; // the srcGwNode @@ -129,7 +139,7 @@ class MessageHeader // the packetType uint16_t m_packetType; // the ttl - int16_t m_ttl; + int16_t m_ttl = 0; // the ext(contains the router policy and response flag) uint16_t m_ext; //// the optional field(used to route between components and nodes) @@ -187,14 +197,18 @@ class MessageHeaderBuilder virtual MessageHeader::Ptr build() = 0; }; -class MessageBuilder +class MessageBuilder : public bcos::boostssl::MessageFaceFactory { public: + using Ptr = std::shared_ptr; MessageBuilder() = default; - virtual ~MessageBuilder() = default; + ~MessageBuilder() override = default; virtual Message::Ptr build() = 0; virtual Message::Ptr build(bcos::bytesConstRef buffer) = 0; + virtual Message::Ptr build(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload) = 0; }; inline std::string printMessage(Message::Ptr const& _msg) @@ -218,4 +232,10 @@ inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg) return stringstream.str(); } +// function to send response +using SendResponseFunction = std::function; +using ReceiveMsgFunc = std::function; +using MessageCallback = std::function; + } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h index 4329aae2..14304a16 100644 --- a/cpp/ppc-framework/protocol/MessagePayload.h +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -36,22 +36,21 @@ class MessagePayload // the version virtual uint8_t version() const { return m_version; } virtual void setVersion(uint8_t version) { m_version = version; } - // the topic - virtual std::string const& topic() const { return m_topic; } - virtual void setTopic(std::string&& topic) { m_topic = std::move(topic); } - virtual void setTopic(std::string const& topic) { m_topic = topic; } // data virtual bcos::bytes const& data() const { return m_data; } virtual void setData(bcos::bytes&& data) { m_data = std::move(data); } virtual void setData(bcos::bytes const& data) { m_data = data; } + // the seq + virtual uint16_t seq() const { return m_seq; } + virtual void setSeq(uint16_t seq) { m_seq = seq; } // the length virtual int64_t length() const { return m_length; } protected: // the front payload version, used to support compatibility uint8_t m_version; - // the topic - std::string m_topic; + // the seq + uint16_t m_seq; bcos::bytes m_data; int64_t mutable m_length; }; diff --git a/cpp/ppc-framework/protocol/RouteType.h b/cpp/ppc-framework/protocol/RouteType.h new file mode 100644 index 00000000..1e3a715e --- /dev/null +++ b/cpp/ppc-framework/protocol/RouteType.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RouteType.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#pragma once +#include +#include + +namespace ppc::protocol +{ +enum class RouteType : uint8_t +{ + ROUTE_THROUGH_NODEID = 0x00, + ROUTE_THROUGH_COMPONENT = 0x01, + ROUTE_THROUGH_AGENCY = 0x02, + ROUTE_THROUGH_TOPIC = 0x03 +}; + +inline std::ostream& operator<<(std::ostream& _out, RouteType const& _type) +{ + switch (_type) + { + case RouteType::ROUTE_THROUGH_NODEID: + _out << "RouteThroughNodeID"; + break; + case RouteType::ROUTE_THROUGH_COMPONENT: + _out << "RouteThroughComponent"; + break; + case RouteType::ROUTE_THROUGH_AGENCY: + _out << "RouteThroughAgency"; + break; + default: + _out << "UnknownRouteType"; + break; + } + return _out; +} +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp new file mode 100644 index 00000000..7b4240c9 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayImpl.cpp + * @author: yujiechen + * @date 2024-08-26 + */ +#include "GatewayImpl.h" +#include "SendMessageWithRetry.h" +#include "cache/MessageCache.h" +#include "ppc-framework/gateway/GatewayProtocol.h" +#include "router/GatewayNodeInfoImpl.h" + +using namespace bcos; +using namespace ppc; +using namespace ppc::protocol; +using namespace ppc::gateway; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; + +GatewayImpl::GatewayImpl(Service::Ptr const& service, + ppc::front::IFrontBuilder::Ptr const& frontBuilder, + std::shared_ptr ioService, std::string const& agency) + : m_service(service), + m_msgBuilder( + std::dynamic_pointer_cast(service->messageFactory())), + m_frontBuilder(frontBuilder), + m_agency(agency), + m_p2pRouterManager(std::make_shared(service)), + m_gatewayInfoFactory(std::make_shared(service->nodeID(), agency)), + m_localRouter(std::make_shared( + m_gatewayInfoFactory, m_frontBuilder, std::make_shared(ioService))), + m_peerRouter(std::make_shared(m_service)) +{ + m_service->registerMsgHandler((uint16_t)GatewayPacketType::P2PMessage, + boost::bind(&GatewayImpl::onReceiveP2PMessage, this, boost::placeholders::_1, + boost::placeholders::_2)); + + m_service->registerMsgHandler((uint16_t)GatewayPacketType::BroadcastMessage, + boost::bind(&GatewayImpl::onReceiveBroadcastMessage, this, boost::placeholders::_1, + boost::placeholders::_2)); +} + +void GatewayImpl::start() +{ + if (m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("Gateway has already been started"); + return; + } + m_running = true; + m_service->start(); + m_p2pRouterManager->start(); + GATEWAY_LOG(INFO) << LOG_DESC("Start gateway success"); +} + +void GatewayImpl::stop() +{ + if (!m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("Gateway has already been stopped"); + return; + } + m_running = false; + m_service->stop(); + m_p2pRouterManager->stop(); + GATEWAY_LOG(INFO) << LOG_DESC("Stop gateway success"); +} + +void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, + std::string const& topic, std::string const& dstInst, std::string const& componentType, + bcos::bytes&& payload) +{ + // dispatcher to all the local front + auto p2pMessage = m_msgBuilder->build( + routeType, topic, dstInst, bcos::bytes(), componentType, std::move(payload)); + p2pMessage->setPacketType((uint16_t)GatewayPacketType::BroadcastMessage); + m_localRouter->dispatcherMessage(p2pMessage, nullptr); + // broadcast message to all peers + m_peerRouter->asyncBroadcastMessage(p2pMessage); +} + + +void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload, long timeout, ReceiveMsgFunc callback) +{ + // check the localRouter + auto p2pMessage = m_msgBuilder->build( + routeType, topic, dstInst, dstNodeID, componentType, std::move(payload)); + p2pMessage->setPacketType((uint16_t)GatewayPacketType::P2PMessage); + auto nodeList = m_localRouter->chooseReceiver(p2pMessage); + // case send to the same agency + if (!nodeList.empty()) + { + GATEWAY_LOG(TRACE) << LOG_DESC("hit the local router, dispatch message directly") + << LOG_KV("msg", printMessage(p2pMessage)); + m_localRouter->dispatcherMessage(p2pMessage, callback); + return; + } + // try to find the dstP2PNode + auto selectedP2PNodes = m_peerRouter->selectRouter(routeType, p2pMessage); + if (selectedP2PNodes.empty()) + { + GATEWAY_LOG(INFO) << LOG_DESC("can't find the gateway to send the message") + << LOG_KV("detail", printMessage(p2pMessage)); + if (callback) + { + callback(std::make_shared( + -1, "can't find the gateway to send the message, traceID: " + + p2pMessage->header()->traceID())); + } + return; + } + // send the message to gateway + auto retry = std::make_shared( + m_service, std::move(selectedP2PNodes), std::move(p2pMessage), callback, timeout); + retry->trySendMessage(); +} + +void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr session) +{ + // try to dispatcher to the front + auto p2pMessage = std::dynamic_pointer_cast(msg); + auto self = std::weak_ptr(shared_from_this()); + auto callback = [p2pMessage, session, self](Error::Ptr error) { + auto gateway = self.lock(); + if (!gateway) + { + return; + } + std::string errorCode = std::to_string(CommonError::SUCCESS); + if (error && error->errorCode() != 0) + { + GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveP2PMessage: dispatcherMessage failed") + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + errorCode = std::to_string(error->errorCode()); + } + + std::shared_ptr payload = + std::make_shared(errorCode.begin(), errorCode.end()); + gateway->m_service->sendRespMessageBySession(session, p2pMessage, std::move(payload)); + }; + + auto ret = m_localRouter->dispatcherMessage(p2pMessage, callback); + if (!ret) + { + GATEWAY_LOG(ERROR) + << LOG_DESC( + "onReceiveP2PMessage failed to find the node that can dispatch this message") + << LOG_KV("msg", printMessage(p2pMessage)); + callback(std::make_shared(CommonError::NotFoundFrontServiceDispatchMsg, + "unable to find the ndoe to dispatcher this message, message detail: " + + printMessage(p2pMessage))); + } +} + +void GatewayImpl::onReceiveBroadcastMessage(MessageFace::Ptr msg, WsSession::Ptr) +{ + auto p2pMessage = std::dynamic_pointer_cast(msg); + GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveBroadcastMessage, dispatcher") + << LOG_KV("msg", printMessage(p2pMessage)); + m_localRouter->dispatcherMessage(p2pMessage, nullptr); +} + +void GatewayImpl::registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) +{ + m_localRouter->registerNodeInfo(nodeInfo); +} + +void GatewayImpl::unRegisterNodeInfo(bcos::bytesConstRef nodeID) +{ + m_localRouter->unRegisterNode(nodeID.toBytes()); +} +void GatewayImpl::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) +{ + m_localRouter->registerTopic(nodeID, topic); +} +void GatewayImpl::unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) +{ + m_localRouter->unRegisterTopic(nodeID, topic); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h new file mode 100644 index 00000000..b71bd5b1 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayImpl.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/gateway/IGateway.h" +#include "ppc-gateway/gateway/router/GatewayNodeInfo.h" +#include "ppc-gateway/p2p/Service.h" +#include "ppc-gateway/p2p/router/RouterManager.h" +#include "router/LocalRouter.h" +#include "router/PeerRouterTable.h" + +namespace ppc::gateway +{ +class GatewayImpl : public IGateway, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + GatewayImpl(Service::Ptr const& service, ppc::front::IFrontBuilder::Ptr const& frontBuilder, + std::shared_ptr ioService, std::string const& agency); + ~GatewayImpl() override = default; + + void start() override; + void stop() override; + + /** + * @brief send message to gateway + * + * @param routeType the route type + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ + void asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) override; + + void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, std::string const& componentType, bcos::bytes&& payload); + + + void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; + void unRegisterNodeInfo(bcos::bytesConstRef nodeID) override; + void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + +protected: + virtual void onReceiveP2PMessage( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + virtual void onReceiveBroadcastMessage( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + +private: + bool m_running = false; + Service::Ptr m_service; + ppc::protocol::MessageBuilder::Ptr m_msgBuilder; + + ppc::front::IFrontBuilder::Ptr m_frontBuilder; + std::string m_agency; + + RouterManager::Ptr m_p2pRouterManager; + + GatewayNodeInfoFactory::Ptr m_gatewayInfoFactory; + LocalRouter::Ptr m_localRouter; + PeerRouterTable::Ptr m_peerRouter; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp new file mode 100644 index 00000000..19093b32 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SendMessageWithRetry.h + * @author: yujiechen + * @date 2024-08-26 + */ +#include "SendMessageWithRetry.h" +#include "ppc-framework/gateway/GatewayProtocol.h" +#include "ppc-gateway/Common.h" + +using namespace bcos; +using namespace ppc; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; +using namespace ppc::gateway; +using namespace ppc::protocol; + +// random choose one p2pID to send message +GatewayNodeInfo::Ptr SendMessageWithRetry::chooseP2pNode() +{ + RecursiveGuard lock(x_mutex); + if (!m_dstNodeList.empty()) + { + auto selectedNode = m_dstNodeList.begin(); + m_dstNodeList.erase(m_dstNodeList.begin()); + return *selectedNode; + } + return nullptr; +} + +// send the message with retry +void SendMessageWithRetry::trySendMessage() +{ + if (m_dstNodeList.empty()) + { + GATEWAY_LOG(DEBUG) << LOG_DESC("Gateway::SendMessageWithRetry") + << LOG_DESC("unable to send the message") << printMessage(m_p2pMessage); + if (m_respFunc) + { + m_respFunc(std::make_shared( + -1, "can't find the gateway to send the message, detail: " + + printMessage(m_p2pMessage))); + } + return; + } + auto choosedNode = chooseP2pNode(); + auto self = shared_from_this(); + auto startT = utcTime(); + auto callback = [self, startT]( + bcos::Error::Ptr error, MessageFace::Ptr msg, WsSession::Ptr session) { + std::ignore = session; + if (error && error->errorCode() != 0) + { + GATEWAY_LOG(DEBUG) << LOG_BADGE("trySendMessage") + << LOG_DESC("send message failed, retry again") + << LOG_KV("msg", printMessage(self->m_p2pMessage)) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()) + << LOG_KV("timeCost", (utcTime() - startT)); + // try again + self->trySendMessage(); + return; + } + // check the errorCode + try + { + auto payload = msg->payload(); + int respCode = boost::lexical_cast(std::string(payload->begin(), payload->end())); + // the peer gateway not response not ok ,it means the gateway not dispatch the + // message successfully,find another gateway and try again + if (respCode != CommonError::SUCCESS) + { + GATEWAY_LOG(DEBUG) + << LOG_BADGE("trySendMessage again") << LOG_KV("respCode", respCode) + << LOG_KV("msg", printMessage(self->m_p2pMessage)); + // try again + self->trySendMessage(); + return; + } + GATEWAY_LOG(TRACE) << LOG_BADGE("asyncSendMessageByNodeID success") + << LOG_KV("msg", printMessage(self->m_p2pMessage)); + // send message successfully + if (self->m_respFunc) + { + self->m_respFunc(nullptr); + } + return; + } + catch (const std::exception& e) + { + GATEWAY_LOG(ERROR) << LOG_BADGE("trySendMessage and receive response exception") + << LOG_KV("msg", printMessage(self->m_p2pMessage)) + << LOG_KV("error", boost::diagnostic_information(e)); + + self->trySendMessage(); + } + }; + // Note: make 10s configuarable here + m_service->asyncSendMessageByNodeID( + choosedNode->p2pNodeID(), m_p2pMessage, bcos::boostssl::ws::Options(m_timeout), callback); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h b/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h new file mode 100644 index 00000000..7df3b6f3 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file SendMessageWithRetry.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/protocol/Message.h" +#include "ppc-gateway/gateway/router/GatewayNodeInfo.h" +#include "ppc-gateway/p2p/Service.h" +#include +#include + +namespace ppc::gateway +{ +class SendMessageWithRetry : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + SendMessageWithRetry(Service::Ptr const& service, GatewayNodeInfos&& dstNodeList, + ppc::protocol::Message::Ptr&& p2pMessage, ppc::protocol::ReceiveMsgFunc respFunc, + long timeout) + : m_service(service), + m_dstNodeList(std::move(dstNodeList)), + m_p2pMessage(std::move(p2pMessage)), + m_respFunc(std::move(respFunc)), + m_timeout(timeout) + { + if (m_timeout < 0) + { + m_timeout = 10000; + } + } + // random choose one p2pID to send message + GatewayNodeInfo::Ptr chooseP2pNode(); + + // send the message with retry + void trySendMessage(); + +private: + // mutex for p2pIDs + mutable bcos::RecursiveMutex x_mutex; + GatewayNodeInfos m_dstNodeList; + ppc::protocol::Message::Ptr m_p2pMessage; + Service::Ptr m_service; + ppc::protocol::ReceiveMsgFunc m_respFunc; + long m_timeout; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp new file mode 100644 index 00000000..09c8f8e2 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageCache.cpp + * @author: yujiechen + * @date 2024-08-26 + */ + +#include "MessageCache.h" +#include "ppc-gateway/Common.h" + +using namespace ppc; +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::gateway; + +void MessageCache::insertCache( + std::string const& topic, ppc::protocol::Message::Ptr const& msg, ReceiveMsgFunc callback) +{ + // hold the message + GATEWAY_LOG(DEBUG) << LOG_BADGE("MessageCache: insertCache") << LOG_KV("topic", topic); + bcos::ReadGuard l(x_msgCache); + auto it = m_msgCache.find(topic); + if (it == m_msgCache.end()) + { + it->second->messages.emplace_back(MessageInfo{msg, callback}); + return; + } + // insert new holding-queue + auto queue = std::make_shared(); + queue->messages.emplace_back(MessageInfo{msg, callback}); + // create timer to handle timeout + queue->timer = std::make_shared( + *m_ioService, boost::posix_time::minutes(m_holdingMessageMinutes)); + queue->timer->async_wait([self = weak_from_this(), topic](boost::system::error_code _error) { + if (!_error) + { + auto cache = self.lock(); + if (cache) + { + // remove timeout message + auto msgQueue = cache->pop(topic); + if (!msgQueue) + { + return; + } + msgQueue->timer->cancel(); + cache->onTimeout(msgQueue); + } + } + }); + m_msgCache[topic] = queue; +} + +HoldingMessageQueue::Ptr MessageCache::pop(const std::string& topic) +{ + WriteGuard lock(x_msgCache); + auto it = m_msgCache.find(topic); + if (it == m_msgCache.end()) + { + return nullptr; + } + HoldingMessageQueue::Ptr ret = it->second; + m_msgCache.erase(topic); + return ret; +} + +void MessageCache::onTimeout(HoldingMessageQueue::Ptr const& queue) +{ + if (!queue) + { + return; + } + // dispatch the ack + for (auto& msgInfo : queue->messages) + { + if (msgInfo.callback) + { + msgInfo.callback(std::make_shared(-1, SEND_MESSAGE_TO_FRONT_TIMEOUT)); + } + } +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h b/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h new file mode 100644 index 00000000..65502edb --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageCache.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/Message.h" +#include "tbb/concurrent_vector.h" +#include +#include +#include + +namespace ppc::gateway +{ +struct MessageInfo +{ + ppc::protocol::Message::Ptr msg; + ppc::protocol::ReceiveMsgFunc callback; +}; +struct HoldingMessageQueue +{ + using Ptr = std::shared_ptr; + HoldingMessageQueue() = default; + + tbb::concurrent_vector messages; + std::shared_ptr timer; +}; + +class MessageCache : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + MessageCache(std::shared_ptr ioService) + : m_ioService(std::move(ioService)) + {} + virtual ~MessageCache() = default; + + void insertCache(std::string const& topic, ppc::protocol::Message::Ptr const& msg, + ppc::protocol::ReceiveMsgFunc callback); + HoldingMessageQueue::Ptr pop(std::string const& topic); + +private: + void onTimeout(HoldingMessageQueue::Ptr const& queue); + +private: + int m_holdingMessageMinutes = 30; + std::shared_ptr m_ioService; + /** + * hold the message for the situation that + * gateway receives message from the other side while the task has not been registered. + */ + mutable bcos::SharedMutex x_msgCache; + std::unordered_map m_msgCache; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h new file mode 100644 index 00000000..e25db02e --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayNodeInfo.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/protocol/INodeInfo.h" +#include +namespace ppc::gateway +{ +class GatewayNodeInfo +{ +public: + using Ptr = std::shared_ptr; + GatewayNodeInfo() = default; + virtual ~GatewayNodeInfo() = default; + + // the gateway nodeID + virtual std::string const& p2pNodeID() const = 0; + // the agency + virtual std::string const& agency() const = 0; + // get the node information by nodeID + virtual ppc::protocol::INodeInfo::Ptr nodeInfo(bcos::bytes const& nodeID) const = 0; + virtual bool tryAddNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; + virtual void removeNodeInfo(bcos::bytes const& nodeID) = 0; + + virtual std::vector chooseRouteByComponent( + bool selectAll, std::string const& component) const = 0; + virtual std::vector chooseRouterByAgency(bool selectAll) const = 0; + virtual std::vector chooseRouterByTopic( + bool selectAll, std::string const& topic) const = 0; + + virtual void encode(bcos::bytes& data) const = 0; + virtual void decode(bcos::bytesConstRef data) = 0; + + virtual void registerTopic(bcos::bytes const& nodeID, std::string const& topic) = 0; + virtual void unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) = 0; + + virtual std::map nodeList() const = 0; +}; + +class GatewayNodeInfoFactory +{ +public: + using Ptr = std::shared_ptr; + GatewayNodeInfoFactory() = default; + virtual ~GatewayNodeInfoFactory() = default; + + virtual GatewayNodeInfo::Ptr build() const = 0; +}; +struct GatewayNodeInfoCmp +{ + bool operator()(GatewayNodeInfo::Ptr const& _first, GatewayNodeInfo::Ptr const& _second) const + { + // increase order + return _first->p2pNodeID() > _second->p2pNodeID(); + } +}; +using GatewayNodeInfos = std::set; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp new file mode 100644 index 00000000..f7c9cefe --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -0,0 +1,206 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayNodeInfoImpl.cpp + * @author: yujiechen + * @date 2024-08-26 + */ +#include "GatewayNodeInfoImpl.h" +#include "ppc-tars-protocol/Common.h" +#include "ppc-tars-protocol/impl/NodeInfoImpl.h" + +using namespace ppctars; +using namespace ppc::protocol; +using namespace ppc::gateway; + + +// the gateway nodeID +std::string const& GatewayNodeInfoImpl::p2pNodeID() const +{ + return m_inner()->p2pNodeID; +} +// the agency +std::string const& GatewayNodeInfoImpl::agency() const +{ + return m_inner()->agency; +} +// get the node information by nodeID +INodeInfo::Ptr GatewayNodeInfoImpl::nodeInfo(bcos::bytes const& nodeID) const +{ + bcos::ReadGuard l(x_nodeList); + if (m_nodeList.count(nodeID)) + { + return m_nodeList.at(nodeID); + } + return nullptr; +} + +bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info) +{ + auto nodeID = info->nodeID().toBytes(); + auto existedNodeInfo = nodeInfo(nodeID); + // update the info + if (existedNodeInfo == nullptr || !existedNodeInfo->equal(info)) + { + bcos::WriteGuard l(x_nodeList); + m_nodeList[nodeID] = info; + return true; + } + return false; +} + +void GatewayNodeInfoImpl::removeNodeInfo(bcos::bytes const& nodeID) +{ + // remove the nodeInfo + { + bcos::UpgradableGuard l(x_nodeList); + auto it = m_nodeList.find(nodeID); + if (it == m_nodeList.end()) + { + return; + } + bcos::UpgradeGuard ul(l); + m_nodeList.erase(it); + } + // remove the topic info + { + bcos::UpgradableGuard l(x_topicInfo); + auto it = m_topicInfo.find(nodeID); + if (it != m_topicInfo.end()) + { + bcos::UpgradeGuard ul(l); + m_topicInfo.erase(it); + } + } +} + +std::vector GatewayNodeInfoImpl::chooseRouteByComponent( + bool selectAll, std::string const& component) const +{ + std::vector result; + bcos::ReadGuard l(x_nodeList); + for (auto const& it : m_nodeList) + { + if (it.second->components().count(component)) + { + result.emplace_back(it.second->getFront()); + } + if (!result.empty() && !selectAll) + { + break; + } + } + return result; +} + + +vector GatewayNodeInfoImpl::chooseRouterByAgency(bool selectAll) const +{ + std::vector result; + bcos::ReadGuard l(x_nodeList); + for (auto const& it : m_nodeList) + { + result.emplace_back(it.second->getFront()); + if (!result.empty() && !selectAll) + { + break; + } + } + return result; +} + +std::vector GatewayNodeInfoImpl::chooseRouterByTopic( + bool selectAll, std::string const& topic) const +{ + std::vector result; + bcos::ReadGuard l(x_topicInfo); + for (auto const& it : m_topicInfo) + { + INodeInfo::Ptr selectedNode = nullptr; + if (it.second.count(topic)) + { + selectedNode = nodeInfo(it.first); + } + if (selectedNode != nullptr) + { + result.emplace_back(selectedNode->getFront()); + } + if (!result.empty() && !selectAll) + { + break; + } + } + return result; +} +void GatewayNodeInfoImpl::registerTopic(bcos::bytes const& nodeID, std::string const& topic) +{ + bcos::UpgradableGuard l(x_topicInfo); + if (m_topicInfo.count(nodeID) && m_topicInfo.at(nodeID).count(topic)) + { + return; + } + bcos::UpgradeGuard ul(l); + if (!m_topicInfo.count(nodeID)) + { + m_topicInfo[nodeID] = std::set(); + } + m_topicInfo[nodeID].insert(topic); +} + +void GatewayNodeInfoImpl::unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) +{ + bcos::UpgradableGuard l(x_topicInfo); + if (!m_topicInfo.count(nodeID) || !m_topicInfo.at(nodeID).count(topic)) + { + return; + } + bcos::UpgradeGuard ul(l); + m_topicInfo[nodeID].erase(topic); +} + +void GatewayNodeInfoImpl::encode(bcos::bytes& data) const +{ + m_inner()->nodeList.clear(); + { + bcos::ReadGuard l(x_nodeList); + // encode nodeList + for (auto const& it : m_nodeList) + { + auto nodeInfo = std::dynamic_pointer_cast(it.second); + m_inner()->nodeList.emplace_back(nodeInfo->inner()); + } + } + tars::TarsOutputStream output; + m_inner()->writeTo(output); + output.getByteBuffer().swap(data); +} + +void GatewayNodeInfoImpl::decode(bcos::bytesConstRef data) +{ + tars::TarsInputStream input; + input.setBuffer((const char*)data.data(), data.size()); + m_inner()->readFrom(input); + { + bcos::WriteGuard l(x_nodeList); + // decode into m_nodeList + m_nodeList.clear(); + for (auto& it : m_inner()->nodeList) + { + auto nodeInfoPtr = + std::make_shared([m_entry = it]() mutable { return &m_entry; }); + m_nodeList.insert(std::make_pair(nodeInfoPtr->nodeID().toBytes(), nodeInfoPtr)); + } + } +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h new file mode 100644 index 00000000..cb0dfc10 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayNodeInfoImpl.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "GatewayNodeInfo.h" +#include "ppc-tars-protocol/tars/NodeInfo.h" +#include +#include + +namespace ppc::gateway +{ +class GatewayNodeInfoImpl : public GatewayNodeInfo +{ +public: + using Ptr = std::shared_ptr; + GatewayNodeInfoImpl(std::string const& p2pNodeID, std::string const& agency) + : m_inner([inner = ppctars::GatewayNodeInfo()]() mutable { return &inner; }) + { + m_inner()->p2pNodeID = p2pNodeID; + m_inner()->agency = agency; + } + ~GatewayNodeInfoImpl() override = default; + + // the gateway nodeID + std::string const& p2pNodeID() const override; + // the agency + std::string const& agency() const override; + // the node information + + // get the node information by nodeID + ppc::protocol::INodeInfo::Ptr nodeInfo(bcos::bytes const& nodeID) const override; + + void encode(bcos::bytes& data) const override; + void decode(bcos::bytesConstRef data) override; + + bool tryAddNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; + void removeNodeInfo(bcos::bytes const& nodeID) override; + + std::vector chooseRouteByComponent( + bool selectAll, std::string const& component) const override; + std::vector chooseRouterByAgency(bool selectAll) const override; + std::vector chooseRouterByTopic( + bool selectAll, std::string const& topic) const override; + + void registerTopic(bcos::bytes const& nodeID, std::string const& topic) override; + void unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) override; + + std::map nodeList() const override + { + bcos::WriteGuard l(x_nodeList); + return m_nodeList; + } + +private: + std::function m_inner; + // NodeID => nodeInfo + std::map m_nodeList; + mutable bcos::SharedMutex x_nodeList; + + // NodeID=>topics + using Topics = std::set; + std::map m_topicInfo; + mutable bcos::SharedMutex x_topicInfo; +}; + +class GatewayNodeInfoFactoryImpl : public GatewayNodeInfoFactory +{ +public: + using Ptr = std::shared_ptr; + GatewayNodeInfoFactoryImpl(std::string const& p2pNodeID, std::string const& agency) + : m_p2pNodeID(p2pNodeID), m_agency(agency) + {} + ~GatewayNodeInfoFactoryImpl() override = default; + + GatewayNodeInfo::Ptr build() const override + { + return std::make_shared(m_p2pNodeID, m_agency); + } + +private: + std::string m_p2pNodeID; + std::string m_agency; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp new file mode 100644 index 00000000..9df4e45f --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -0,0 +1,122 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LocalRouter.h + * @author: yujiechen + * @date 2024-08-26 + */ +#include "LocalRouter.h" +#include "ppc-framework/Common.h" +#include "ppc-framework/gateway/GatewayProtocol.h" + +using namespace bcos; +using namespace ppc::protocol; +using namespace ppc::gateway; + +void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& topic) +{ + m_routerInfo->registerTopic(_nodeID.toBytes(), topic); + // try to dispatch the cacheInfo + if (!m_cache) + { + return; + } + auto msgQueue = m_cache->pop(topic); + if (!msgQueue) + { + return; + } + if (msgQueue->timer) + { + msgQueue->timer->cancel(); + } + for (auto const& msgInfo : msgQueue->messages) + { + dispatcherMessage(msgInfo.msg, msgInfo.callback, false); + } +} + +void LocalRouter::unRegisterTopic(bcos::bytesConstRef _nodeID, std::string const& topic) +{ + m_routerInfo->unRegisterTopic(_nodeID.toBytes(), topic); +} + +bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc callback, bool holding) +{ + auto frontList = chooseReceiver(msg); + // send success + if (!frontList.empty()) + { + for (auto const& front : frontList) + { + front->onReceiveMessage(msg, callback); + } + return true; + } + if (!holding) + { + return false; + } + // no connection found, cache the topic message and dispatcher later + if (msg->header()->routeType() == (uint16_t)RouteType::ROUTE_THROUGH_TOPIC && m_cache) + { + m_cache->insertCache(msg->header()->optionalField()->topic(), msg, callback); + return true; + } + return false; +} + +std::vector LocalRouter::chooseReceiver( + ppc::protocol::Message::Ptr const& msg) +{ + std::vector receivers; + if (msg->header()->optionalField()->dstInst() != m_routerInfo->agency()) + { + return receivers; + } + bool selectAll = + (msg->header()->packetType() == (uint16_t)GatewayPacketType::BroadcastMessage ? true : + false); + switch (msg->header()->routeType()) + { + case (uint16_t)RouteType::ROUTE_THROUGH_NODEID: + { + auto gatewayInfo = m_routerInfo->nodeInfo(msg->header()->optionalField()->dstNode()); + if (gatewayInfo != nullptr) + { + receivers.emplace_back(gatewayInfo->getFront()); + } + return receivers; + } + case (uint16_t)RouteType::ROUTE_THROUGH_COMPONENT: + { + return m_routerInfo->chooseRouteByComponent( + selectAll, msg->header()->optionalField()->componentType()); + } + case (uint16_t)RouteType::ROUTE_THROUGH_AGENCY: + { + return m_routerInfo->chooseRouterByAgency(selectAll); + } + case (uint16_t)RouteType::ROUTE_THROUGH_TOPIC: + { + return m_routerInfo->chooseRouterByTopic( + selectAll, msg->header()->optionalField()->topic()); + } + default: + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "chooseReceiver failed for unknown routeType, message detail: " + + printMessage(msg))); + } +} diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h new file mode 100644 index 00000000..194ef6ab --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LocalRouter.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "../cache/MessageCache.h" +#include "GatewayNodeInfo.h" +#include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/RouteType.h" + +namespace ppc::gateway +{ +class LocalRouter +{ +public: + using Ptr = std::shared_ptr; + LocalRouter(GatewayNodeInfoFactory::Ptr nodeInfoFactory, + ppc::front::IFrontBuilder::Ptr frontBuilder, MessageCache::Ptr msgCache) + : m_routerInfo(std::move(nodeInfoFactory->build())), + m_frontBuilder(std::move(frontBuilder)), + m_cache(std::move(msgCache)) + {} + + virtual ~LocalRouter() = default; + + virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) + { + nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); + return m_routerInfo->tryAddNodeInfo(nodeInfo); + } + + virtual void unRegisterNode(bcos::bytes const& nodeID) { m_routerInfo->removeNodeInfo(nodeID); } + + virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic); + virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic); + + virtual std::vector chooseReceiver( + ppc::protocol::Message::Ptr const& msg); + + // TODO: register component + virtual bool dispatcherMessage(ppc::protocol::Message::Ptr const& msg, + ppc::protocol::ReceiveMsgFunc callback, bool holding = true); + +private: + ppc::front::IFrontBuilder::Ptr m_frontBuilder; + GatewayNodeInfo::Ptr m_routerInfo; + + // NodeID=>topics + using Topics = std::set; + std::map m_topicInfo; + mutable bcos::SharedMutex x_topicInfo; + + MessageCache::Ptr m_cache; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp new file mode 100644 index 00000000..a2bccd0a --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -0,0 +1,138 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PeerRouterTable.cpp + * @author: yujiechen + * @date 2024-08-27 + */ +#include "PeerRouterTable.h" +#include "ppc-framework/Common.h" +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::gateway; +using namespace ppc::protocol; + +void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) +{ + auto nodeList = gatewayInfo->nodeList(); + bcos::WriteGuard l(x_mutex); + for (auto const& it : nodeList) + { + // update nodeID => gatewayInfos + m_nodeID2GatewayInfos[it.first].insert(gatewayInfo); + } + // update agency => gatewayInfos + m_agency2GatewayInfos[gatewayInfo->agency()].insert(gatewayInfo); +} + +GatewayNodeInfos PeerRouterTable::selectRouter( + RouteType const& routeType, Message::Ptr const& msg) const +{ + switch (routeType) + { + case RouteType::ROUTE_THROUGH_NODEID: + return selectRouterByNodeID(msg); + case RouteType::ROUTE_THROUGH_COMPONENT: + return selectRouterByComponent(msg); + case RouteType::ROUTE_THROUGH_AGENCY: + case RouteType::ROUTE_THROUGH_TOPIC: + return selectRouterByAgency(msg); + default: + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "selectRouter failed for encounter unsupported routeType: " + + std::to_string((uint16_t)routeType))); + } +} + +GatewayNodeInfos PeerRouterTable::selectRouterByNodeID(Message::Ptr const& msg) const +{ + GatewayNodeInfos result; + bcos::ReadGuard l(x_mutex); + auto it = m_nodeID2GatewayInfos.find(msg->header()->optionalField()->dstNode()); + // no router found + if (it == m_nodeID2GatewayInfos.end()) + { + return result; + } + return it->second; +} + + +GatewayNodeInfos PeerRouterTable::selectRouterByAgency(Message::Ptr const& msg) const +{ + GatewayNodeInfos result; + bcos::ReadGuard l(x_mutex); + auto it = m_agency2GatewayInfos.find(msg->header()->optionalField()->dstInst()); + // no router found + if (it == m_agency2GatewayInfos.end()) + { + return result; + } + return it->second; +} + +GatewayNodeInfos PeerRouterTable::selectRouterByComponent(Message::Ptr const& msg) const +{ + GatewayNodeInfos result; + bcos::ReadGuard l(x_mutex); + auto it = m_agency2GatewayInfos.find(msg->header()->optionalField()->dstInst()); + // no router found + if (it == m_agency2GatewayInfos.end()) + { + return result; + } + auto const& gatewayInfos = it->second; + // foreach all gateways to find the component + for (auto const& it : gatewayInfos) + { + auto const& nodeListInfo = it->nodeList(); + for (auto const& nodeInfo : nodeListInfo) + { + if (nodeInfo.second->components().count( + msg->header()->optionalField()->componentType())) + { + result.insert(it); + break; + } + } + } + return result; +} + +void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const +{ + bcos::ReadGuard l(x_mutex); + for (auto const& it : m_agency2GatewayInfos) + { + auto selectedIndex = rand() % it.second.size(); + auto iterator = it.second.begin(); + if (selectedIndex > 0) + { + std::advance(iterator, selectedIndex); + } + auto selectedNode = *iterator; + // ignore self + if (selectedNode->p2pNodeID() == m_service->nodeID()) + { + continue; + } + GATEWAY_LOG(TRACE) << LOG_DESC("asyncBroadcastMessage") + << LOG_KV("nodeID", printP2PIDElegantly(selectedNode->p2pNodeID())) + << LOG_KV("msg", printMessage(msg)); + m_service->asyncSendMessageByNodeID(selectedNode->p2pNodeID(), msg); + } +} diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h new file mode 100644 index 00000000..7a433303 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file PeerRouterTable.h + * @author: yujiechen + * @date 2024-08-27 + */ +#pragma once +#include "GatewayNodeInfo.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/RouteType.h" +#include "ppc-gateway/p2p/Service.h" +#include +#include + +namespace ppc::gateway +{ +class PeerRouterTable +{ +public: + using Ptr = std::shared_ptr; + PeerRouterTable(Service::Ptr service) : m_service(std::move(service)) {} + virtual ~PeerRouterTable() = default; + + virtual void updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo); + virtual GatewayNodeInfos selectRouter( + ppc::protocol::RouteType const& routeType, ppc::protocol::Message::Ptr const& msg) const; + + virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const; + +private: + virtual GatewayNodeInfos selectRouterByNodeID(ppc::protocol::Message::Ptr const& msg) const; + virtual GatewayNodeInfos selectRouterByComponent(ppc::protocol::Message::Ptr const& msg) const; + virtual GatewayNodeInfos selectRouterByAgency(ppc::protocol::Message::Ptr const& msg) const; + + +private: + Service::Ptr m_service; + // nodeID => p2pNodes + std::map m_nodeID2GatewayInfos; + // agency => p2pNodes + std::map m_agency2GatewayInfos; + mutable bcos::SharedMutex x_mutex; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp index 469e5b27..fd36e94e 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -296,4 +296,28 @@ void Service::asyncSendMessageByP2PNodeID(uint16_t type, std::string const& dstN message->setPacketType(type); message->setPayload(payload); asyncSendMessageByNodeID(dstNodeID, message, options, callback); +} + +void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& session, + bcos::boostssl::MessageFace::Ptr msg, std::shared_ptr&& payload) +{ + auto respMessage = std::dynamic_pointer_cast(m_messageFactory->buildMessage()); + auto requestMsg = std::dynamic_pointer_cast(msg); + if (requestMsg->header() && requestMsg->header()->optionalField()) + { + respMessage->header()->optionalField()->setDstNode( + requestMsg->header()->optionalField()->srcNode()); + respMessage->header()->optionalField()->setSrcNode( + requestMsg->header()->optionalField()->dstNode()); + } + respMessage->header()->setTraceID(requestMsg->header()->traceID()); + respMessage->header()->setRespPacket(); + respMessage->header()->setRouteType(ppc::protocol::RouteType::ROUTE_THROUGH_NODEID); + respMessage->setPayload(std::move(payload)); + + WsSessions sessions; + sessions.emplace_back(session); + WsService::asyncSendMessage(sessions, respMessage); + GATEWAY_LOG(TRACE) << "sendRespMessageBySession" << LOG_KV("resp", printMessage(respMessage)) + << LOG_KV("payload size", payload->size()); } \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h index a3c8f5ea..e78aff23 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h +++ b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h @@ -36,7 +36,8 @@ class Service : public bcos::boostssl::ws::WsService bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options(), bcos::boostssl::ws::RespCallBack respFunc = bcos::boostssl::ws::RespCallBack()); - virtual void asyncSendMessageByP2PNodeID(uint16_t type, std::string const& dstNodeID, + + virtual void asyncSendMessageByP2PNodeID(uint16_t packetType, std::string const& dstNodeID, std::shared_ptr payload, bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options(), bcos::boostssl::ws::RespCallBack callback = bcos::boostssl::ws::RespCallBack()); @@ -44,10 +45,17 @@ class Service : public bcos::boostssl::ws::WsService virtual void asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::Options options = bcos::boostssl::ws::Options()); + virtual void sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& session, + bcos::boostssl::MessageFace::Ptr msg, std::shared_ptr&& payload); + RouterTableFactory::Ptr const& routerTableFactory() const { return m_routerTableFactory; } RouterTableInterface::Ptr const& routerTable() const { return m_routerTable; } std::string const& nodeID() const { return m_nodeID; } + bcos::boostssl::MessageFaceFactory::Ptr const& messageFactory() const + { + return m_messageFactory; + } protected: void onRecvMessage(bcos::boostssl::MessageFace::Ptr _msg, diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h index f99c0182..a1701c9e 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h @@ -28,6 +28,7 @@ namespace ppc::gateway class RouterManager { public: + using Ptr = std::shared_ptr; RouterManager(Service::Ptr service); virtual ~RouterManager() = default; diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp index 08ba9cdd..f14e1cac 100644 --- a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp +++ b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp @@ -29,9 +29,10 @@ using namespace ppc; void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const { // the componentType - uint16_t componentType = - boost::asio::detail::socket_ops::host_to_network_short(m_componentType); - buffer.insert(buffer.end(), (byte*)&componentType, (byte*)&componentType + 2); + uint16_t componentTypeLen = + boost::asio::detail::socket_ops::host_to_network_short(m_componentType.size()); + buffer.insert(buffer.end(), (byte*)&componentTypeLen, (byte*)&componentTypeLen + 2); + buffer.insert(buffer.end(), m_componentType.begin(), m_componentType.end()); // the source nodeID that send the message uint16_t srcNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_srcNode.size()); buffer.insert(buffer.end(), (byte*)&srcNodeLen, (byte*)&srcNodeLen + 2); @@ -55,13 +56,17 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con // the componentType auto pointer = data.data() + offset; m_componentType = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); - pointer += 2; + bcos::bytes componentType; + offset = decodeNetworkBuffer(componentType, data.data(), data.size(), (pointer - data.data())); + m_componentType = std::string(componentType.begin(), componentType.end()); // srcNode - offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), (pointer - data.data())); + offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), offset); // dstNode offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset); - // dstInst - offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset); + // dstInst, TODO: optimize here + bcos::bytes dstInstData; + offset = decodeNetworkBuffer(dstInstData, data.data(), data.size(), offset); + m_dstInst = std::string(dstInstData.begin(), dstInstData.end()); return offset; } @@ -144,3 +149,47 @@ int64_t MessageHeaderImpl::decode(bcos::bytesConstRef data) m_length = offset; return offset; } + +uint16_t MessageHeaderImpl::routeType() const +{ + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByComponent) + { + return (uint16_t)RouteType::ROUTE_THROUGH_COMPONENT; + } + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByAgency) + { + return (uint16_t)RouteType::ROUTE_THROUGH_AGENCY; + } + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByAgency) + { + return (uint16_t)RouteType::ROUTE_THROUGH_AGENCY; + } + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByTopic) + { + return (uint16_t)RouteType::ROUTE_THROUGH_TOPIC; + } + // default is route though nodeID + return (uint16_t)RouteType::ROUTE_THROUGH_NODEID; +} + +void MessageHeaderImpl::setRouteType(ppc::protocol::RouteType type) +{ + switch (type) + { + case RouteType::ROUTE_THROUGH_NODEID: + m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByNodeID; + break; + case RouteType::ROUTE_THROUGH_COMPONENT: + m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByComponent; + break; + case RouteType::ROUTE_THROUGH_AGENCY: + m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByAgency; + break; + case RouteType::ROUTE_THROUGH_TOPIC: + m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByTopic; + break; + default: + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "Invalid route type: " + std::to_string((uint16_t)type))); + } +} \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h index a8dda5a9..9d9c3850 100644 --- a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h +++ b/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h @@ -61,6 +61,9 @@ class MessageHeaderImpl : public MessageHeader } void setRespPacket() override { m_ext |= (uint16_t)ppc::gateway::GatewayMsgExtFlag::Response; } + uint16_t routeType() const override; + void setRouteType(ppc::protocol::RouteType type) override; + private: // version(2) + packetType(2) + ttl(2) + ext(2) + traceIDLen(2) + srcGwNodeLen(2) + dstGwNode(2) const size_t MESSAGE_MIN_LENGTH = 14; diff --git a/cpp/ppc-protocol/src/v1/MessageImpl.h b/cpp/ppc-protocol/src/v1/MessageImpl.h index 208db1b3..aec9cb3d 100644 --- a/cpp/ppc-protocol/src/v1/MessageImpl.h +++ b/cpp/ppc-protocol/src/v1/MessageImpl.h @@ -20,6 +20,9 @@ #pragma once #include "ppc-framework/Common.h" #include "ppc-framework/protocol/Message.h" +#include +#include +#include namespace ppc::protocol { @@ -75,6 +78,31 @@ class MessageBuilderImpl : public MessageBuilder { return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen, buffer); } + Message::Ptr build(ppc::protocol::RouteType routeType, std::string const& topic, + std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, + bcos::bytes&& payload) override + { + auto msg = build(); + msg->header()->setRouteType(routeType); + msg->header()->optionalField()->setDstInst(dstInst); + msg->header()->optionalField()->setDstNode(dstNodeID); + msg->header()->optionalField()->setTopic(topic); + msg->header()->optionalField()->setComponentType(componentType); + msg->setPayload(std::make_shared(std::move(payload))); + return msg; + } + + bcos::boostssl::MessageFace::Ptr buildMessage() override + { + return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen); + } + + std::string newSeq() override + { + std::string seq = boost::uuids::to_string(boost::uuids::random_generator()()); + seq.erase(std::remove(seq.begin(), seq.end(), '-'), seq.end()); + return seq; + } private: MessageHeaderBuilder::Ptr m_msgHeaderBuilder; diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp index 59397edf..360567b4 100644 --- a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp +++ b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp @@ -31,10 +31,9 @@ int64_t MessagePayloadImpl::encode(bcos::bytes& buffer) const // version uint16_t version = boost::asio::detail::socket_ops::host_to_network_short(m_version); buffer.insert(buffer.end(), (byte*)&version, (byte*)&version + 2); - // topic - uint16_t topicLen = boost::asio::detail::socket_ops::host_to_network_short(m_topic.size()); - buffer.insert(buffer.end(), (byte*)&topicLen, (byte*)&topicLen + 2); - buffer.insert(buffer.end(), m_topic.begin(), m_topic.end()); + // seq + uint16_t seq = boost::asio::detail::socket_ops::host_to_network_short(m_seq); + buffer.insert(buffer.end(), (byte*)&seq, (byte*)&seq + 2); // data uint16_t dataLen = boost::asio::detail::socket_ops::host_to_network_short(m_data.size()); buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 2); @@ -56,12 +55,10 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) // the version m_version = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); pointer += 2; - // topic - bcos::bytes topicData; - auto offset = - decodeNetworkBuffer(topicData, buffer.data(), buffer.size(), (pointer - buffer.data())); - m_topic = std::string(topicData.begin(), topicData.end()); + // the seq + CHECK_OFFSET_WITH_THROW_EXCEPTION((pointer - buffer.data()), buffer.size()); + m_seq = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; // data - offset = decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset); - return offset; + return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), (pointer - buffer.data())); } \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h index f7f5a3e8..c85c0589 100644 --- a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h +++ b/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h @@ -35,6 +35,7 @@ class MessagePayloadImpl : public MessagePayload int64_t decode(bcos::bytesConstRef data) override; private: + // version + seq + dataLen const unsigned int MIN_PAYLOAD_LEN = 6; }; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp b/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp new file mode 100644 index 00000000..e8f585d4 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file NodeInfoImpl.h + * @author: yujiechen + * @date 2024-08-26 + */ + +#include "NodeInfoImpl.h" +#include "../Common.h" + +using namespace ppctars; +using namespace ppc::protocol; + +void NodeInfoImpl::encode(bcos::bytes& data) const +{ + tars::TarsOutputStream output; + m_inner()->writeTo(output); + output.getByteBuffer().swap(data); +} +void NodeInfoImpl::decode(bcos::bytesConstRef data) +{ + tars::TarsInputStream input; + input.setBuffer((const char*)data.data(), data.size()); + m_inner()->readFrom(input); + m_components = + std::set(m_inner()->components.begin(), m_inner()->components.end()); +} \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h new file mode 100644 index 00000000..ff4e1627 --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file NodeInfoImpl.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-tars-protocol/tars/NodeInfo.h" +#include +namespace ppc::protocol +{ +// the node information +class NodeInfoImpl : public INodeInfo +{ +public: + using Ptr = std::shared_ptr; + explicit NodeInfoImpl(std::function inner) : m_inner(std::move(inner)) {} + + NodeInfoImpl(bcos::bytesConstRef const& nodeID) + : m_inner([inner = ppctars::NodeInfo()]() mutable { return &inner; }) + { + m_inner()->nodeID = std::vector(nodeID.begin(), nodeID.end()); + } + NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) + : NodeInfoImpl(nodeID) + { + m_inner()->endPoint = endPoint; + } + ~NodeInfoImpl() override = default; + + void setComponents(std::vector const& components) override + { + m_components = std::set(components.begin(), components.end()); + m_inner()->components = components; + } + std::set const& components() const override { return m_components; } + + std::string const& endPoint() const override { return m_inner()->endPoint; } + + bcos::bytesConstRef nodeID() const override + { + return {reinterpret_cast(m_inner()->nodeID.data()), + m_inner()->nodeID.size()}; + } + + void encode(bcos::bytes& data) const override; + void decode(bcos::bytesConstRef data) override; + ppctars::NodeInfo const& inner() { return *(m_inner()); } + + void setFront(ppc::front::IFront::Ptr&& front) override { m_front = std::move(front); } + ppc::front::IFront::Ptr const& getFront() const override { return m_front; } + +private: + ppc::front::IFront::Ptr m_front; + std::set m_components; + std::function m_inner; +}; + +class NodeInfoFactory : public INodeInfoFactory +{ +public: + using Ptr = std::shared_ptr; + NodeInfoFactory(bcos::bytesConstRef const& nodeID) : INodeInfoFactory(nodeID.toBytes()) {} + ~NodeInfoFactory() override {} + + INodeInfo::Ptr build() override { return std::make_shared(bcos::ref(m_nodeID)); } + + + INodeInfo::Ptr build(std::string const& endPoint) override + { + return std::make_shared(bcos::ref(m_nodeID), endPoint); + } +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars new file mode 100644 index 00000000..9a0c67ec --- /dev/null +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars @@ -0,0 +1,15 @@ +module ppctars +{ + struct NodeInfo + { + 1 require vector nodeID; + 2 require string endPoint; + 3 optional vector components; + }; + struct GatewayNodeInfo + { + 1 require string p2pNodeID; + 2 require string agency; + 3 optional vector nodeList; + }; +}; From b6cee04154585dff832b7f48d4aeb4ad9c1f46c5 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 27 Aug 2024 20:58:32 +0800 Subject: [PATCH 016/120] add gateway-router-manager (#12) --- cpp/ppc-framework/gateway/GatewayProtocol.h | 5 +- .../ppc-gateway/gateway/GatewayImpl.cpp | 4 + .../ppc-gateway/gateway/GatewayImpl.h | 2 + .../gateway/router/GatewayNodeInfo.h | 15 ++ .../gateway/router/GatewayNodeInfoImpl.cpp | 10 + .../gateway/router/GatewayNodeInfoImpl.h | 4 + .../gateway/router/GatewayRouterManager.cpp | 183 ++++++++++++++++++ .../gateway/router/GatewayRouterManager.h | 70 +++++++ .../gateway/router/LocalRouter.cpp | 2 + .../ppc-gateway/gateway/router/LocalRouter.h | 31 ++- .../gateway/router/PeerRouterTable.cpp | 8 + .../ppc-gateway/p2p/router/RouterManager.h | 3 +- .../ppc-tars-protocol/tars/NodeInfo.tars | 1 + 13 files changed, 334 insertions(+), 4 deletions(-) create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h diff --git a/cpp/ppc-framework/gateway/GatewayProtocol.h b/cpp/ppc-framework/gateway/GatewayProtocol.h index b91137d6..ebd9b7a3 100644 --- a/cpp/ppc-framework/gateway/GatewayProtocol.h +++ b/cpp/ppc-framework/gateway/GatewayProtocol.h @@ -28,7 +28,10 @@ enum class GatewayPacketType : uint16_t BroadcastMessage = 0x01, RouterTableSyncSeq = 0x10, RouterTableResponse = 0x11, - RouterTableRequest = 0x12 + RouterTableRequest = 0x12, + SyncNodeSeq = 0x20, + RequestNodeStatus = 0x21, + ResponseNodeStatus = 0x22, }; enum class GatewayMsgExtFlag : uint16_t diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 7b4240c9..6707fae9 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -51,6 +51,8 @@ GatewayImpl::GatewayImpl(Service::Ptr const& service, m_service->registerMsgHandler((uint16_t)GatewayPacketType::BroadcastMessage, boost::bind(&GatewayImpl::onReceiveBroadcastMessage, this, boost::placeholders::_1, boost::placeholders::_2)); + m_gatewayRouterManager = std::make_shared( + m_service, m_gatewayInfoFactory, m_localRouter, m_peerRouter); } void GatewayImpl::start() @@ -63,6 +65,7 @@ void GatewayImpl::start() m_running = true; m_service->start(); m_p2pRouterManager->start(); + m_gatewayRouterManager->start(); GATEWAY_LOG(INFO) << LOG_DESC("Start gateway success"); } @@ -76,6 +79,7 @@ void GatewayImpl::stop() m_running = false; m_service->stop(); m_p2pRouterManager->stop(); + m_gatewayRouterManager->stop(); GATEWAY_LOG(INFO) << LOG_DESC("Stop gateway success"); } diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index b71bd5b1..9d2ced23 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -22,6 +22,7 @@ #include "ppc-gateway/gateway/router/GatewayNodeInfo.h" #include "ppc-gateway/p2p/Service.h" #include "ppc-gateway/p2p/router/RouterManager.h" +#include "router/GatewayRouterManager.h" #include "router/LocalRouter.h" #include "router/PeerRouterTable.h" @@ -80,6 +81,7 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this #include +#include namespace ppc::gateway { class GatewayNodeInfo @@ -33,6 +35,9 @@ class GatewayNodeInfo virtual std::string const& p2pNodeID() const = 0; // the agency virtual std::string const& agency() const = 0; + virtual uint32_t statusSeq() const = 0; + virtual void setStatusSeq(uint32_t statusSeq) = 0; + // get the node information by nodeID virtual ppc::protocol::INodeInfo::Ptr nodeInfo(bcos::bytes const& nodeID) const = 0; virtual bool tryAddNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; @@ -51,6 +56,7 @@ class GatewayNodeInfo virtual void unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) = 0; virtual std::map nodeList() const = 0; + virtual uint16_t nodeSize() const = 0; }; class GatewayNodeInfoFactory @@ -71,4 +77,13 @@ struct GatewayNodeInfoCmp } }; using GatewayNodeInfos = std::set; + +inline std::string printNodeStatus(GatewayNodeInfo::Ptr const& status) +{ + std::ostringstream stringstream; + stringstream << LOG_KV("p2pNodeID", status->p2pNodeID()) << LOG_KV("agency", status->agency()) + << LOG_KV("statusSeq", status->statusSeq()) + << LOG_KV("nodeSize", status->nodeSize()); + return stringstream.str(); +} } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp index f7c9cefe..3c718fe9 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -36,6 +36,16 @@ std::string const& GatewayNodeInfoImpl::agency() const { return m_inner()->agency; } + +uint32_t GatewayNodeInfoImpl::statusSeq() const +{ + return m_inner()->statusSeq; +} +void GatewayNodeInfoImpl::setStatusSeq(uint32_t statusSeq) +{ + m_inner()->statusSeq = statusSeq; +} + // get the node information by nodeID INodeInfo::Ptr GatewayNodeInfoImpl::nodeInfo(bcos::bytes const& nodeID) const { diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index cb0dfc10..72c6d440 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -66,6 +66,10 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo bcos::WriteGuard l(x_nodeList); return m_nodeList; } + uint32_t statusSeq() const override; + void setStatusSeq(uint32_t statusSeq) override; + + virtual uint16_t nodeSize() const override { return m_nodeList.size(); } private: std::function m_inner; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp new file mode 100644 index 00000000..a26844e4 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayRouterManager.h + * @author: yujiechen + * @date 2024-08-26 + */ +#include "GatewayRouterManager.h" +#include "ppc-framework/gateway/GatewayProtocol.h" +#include + +using namespace ppc::protocol; +using namespace ppc; +using namespace bcos; +using namespace ppc::gateway; +using namespace bcos::boostssl; +using namespace bcos::boostssl::ws; + +GatewayRouterManager::GatewayRouterManager(Service::Ptr service, + GatewayNodeInfoFactory::Ptr nodeStatusFactory, LocalRouter::Ptr localRouter, + PeerRouterTable::Ptr peerRouter) + : m_service(std::move(service)), + m_nodeStatusFactory(std::move(nodeStatusFactory)), + m_localRouter(std::move(localRouter)), + m_peerRouter(std::move(peerRouter)) +{ + m_service->registerMsgHandler((uint16_t)GatewayPacketType::SyncNodeSeq, + boost::bind(&GatewayRouterManager::onReceiveNodeSeqMessage, this, boost::placeholders::_1, + boost::placeholders::_2)); + + m_service->registerMsgHandler((uint16_t)GatewayPacketType::RequestNodeStatus, + boost::bind(&GatewayRouterManager::onReceiveRequestNodeStatusMsg, this, + boost::placeholders::_1, boost::placeholders::_2)); + + m_service->registerMsgHandler((uint16_t)GatewayPacketType::ResponseNodeStatus, + boost::bind(&GatewayRouterManager::onRecvResponseNodeStatusMsg, this, + boost::placeholders::_1, boost::placeholders::_2)); + + m_timer = std::make_shared(SEQ_SYNC_PERIOD, "seqSync"); + // broadcast seq periodically + m_timer->registerTimeoutHandler([this]() { broadcastStatusSeq(); }); +} + + +void GatewayRouterManager::start() +{ + if (m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been started"); + return; + } + m_running = true; + m_timer->start(); + GATEWAY_LOG(INFO) << LOG_DESC("start GatewayRouterManager success"); +} + +void GatewayRouterManager::stop() +{ + if (!m_running) + { + GATEWAY_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been stopped"); + return; + } + m_running = false; + m_timer->stop(); + GATEWAY_LOG(INFO) << LOG_DESC("stop GatewayRouterManager success"); +} + +void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSession::Ptr session) +{ + auto statusSeq = + boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)msg->payload()->data())); + + auto p2pMessage = std::dynamic_pointer_cast(msg); + auto const& from = (p2pMessage->header()->srcGwNode().size() > 0) ? + p2pMessage->header()->srcGwNode() : + session->nodeId(); + auto statusSeqChanged = statusChanged(from, statusSeq); + if (!statusSeqChanged) + { + return; + } + // status changed, request for the nodeStatus + GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveNodeSeqMessage") << LOG_KV("from", from) + << LOG_KV("statusSeq", statusSeq); + m_service->asyncSendMessageByP2PNodeID( + (uint16_t)GatewayPacketType::RequestNodeStatus, from, std::make_shared()); +} + + +bool GatewayRouterManager::statusChanged(std::string const& p2pNodeID, uint32_t seq) +{ + bool ret = true; + ReadGuard l(x_p2pID2Seq); + auto it = m_p2pID2Seq.find(p2pNodeID); + if (it != m_p2pID2Seq.end()) + { + ret = (seq > it->second); + } + return ret; +} + +void GatewayRouterManager::broadcastStatusSeq() +{ + m_timer->restart(); + auto message = std::dynamic_pointer_cast(m_service->messageFactory()->buildMessage()); + message->setPacketType((uint16_t)GatewayPacketType::SyncNodeSeq); + auto seq = m_localRouter->statusSeq(); + auto statusSeq = boost::asio::detail::socket_ops::host_to_network_long(seq); + auto payload = std::make_shared((byte*)&statusSeq, (byte*)&statusSeq + 4); + message->setPayload(payload); + GATEWAY_LOG(TRACE) << LOG_DESC("broadcastStatusSeq") << LOG_KV("seq", seq); + m_service->asyncBroadcastMessage(message); +} + + +void GatewayRouterManager::onReceiveRequestNodeStatusMsg( + MessageFace::Ptr msg, WsSession::Ptr session) +{ + auto p2pMessage = std::dynamic_pointer_cast(msg); + auto const& from = (!p2pMessage->header()->srcGwNode().empty()) ? + p2pMessage->header()->srcGwNode() : + session->nodeId(); + + auto nodeStatusData = m_localRouter->generateNodeStatus(); + if (!nodeStatusData) + { + GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveRequestNodeStatusMsg: generate nodeInfo error") + << LOG_KV("from", from); + return; + } + GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveRequestNodeStatusMsg: response the latest nodeStatus") + << LOG_KV("from", from); + m_service->asyncSendMessageByP2PNodeID( + (uint16_t)GatewayPacketType::ResponseNodeStatus, from, nodeStatusData); +} + +void GatewayRouterManager::onRecvResponseNodeStatusMsg(MessageFace::Ptr msg, WsSession::Ptr session) +{ + auto nodeStatus = m_nodeStatusFactory->build(); + nodeStatus->decode(bytesConstRef(msg->payload()->data(), msg->payload()->size())); + + auto p2pMessage = std::dynamic_pointer_cast(msg); + auto const& from = (!p2pMessage->header()->srcGwNode().empty()) ? + p2pMessage->header()->srcGwNode() : + session->nodeId(); + + GATEWAY_LOG(INFO) << LOG_DESC("onRecvResponseNodeStatusMsg") << LOG_KV("from", from) + << LOG_KV("statusSeq", nodeStatus->statusSeq()) + << LOG_KV("agency", nodeStatus->agency()); + updatePeerNodeStatus(from, nodeStatus); +} + +void GatewayRouterManager::updatePeerNodeStatus( + std::string const& p2pID, GatewayNodeInfo::Ptr status) +{ + auto statusSeq = status->statusSeq(); + { + UpgradableGuard l(x_p2pID2Seq); + if (m_p2pID2Seq.contains(p2pID) && (m_p2pID2Seq.at(p2pID) >= statusSeq)) + { + return; + } + UpgradeGuard ul(l); + m_p2pID2Seq[p2pID] = statusSeq; + } + GATEWAY_LOG(INFO) << LOG_DESC("updatePeerNodeStatus") << LOG_KV("from", p2pID) + << LOG_KV("statusSeq", status->statusSeq()) + << LOG_KV("agency", status->agency()); + m_peerRouter->updateGatewayInfo(status); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h new file mode 100644 index 00000000..4cf9b0ba --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayRouterManager.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "ppc-gateway/gateway/router/GatewayNodeInfo.h" +#include "ppc-gateway/gateway/router/LocalRouter.h" +#include "ppc-gateway/gateway/router/PeerRouterTable.h" +#include "ppc-gateway/p2p/Service.h" +#include +#include + +namespace ppc::gateway +{ +class GatewayRouterManager +{ +public: + using Ptr = std::shared_ptr; + GatewayRouterManager(Service::Ptr service, GatewayNodeInfoFactory::Ptr nodeStatusFactory, + LocalRouter::Ptr localRouter, PeerRouterTable::Ptr peerRouter); + virtual void start(); + virtual void stop(); + +protected: + virtual void onReceiveNodeSeqMessage( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + + virtual void onReceiveRequestNodeStatusMsg( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + + virtual void onRecvResponseNodeStatusMsg( + bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); + bool statusChanged(std::string const& p2pNodeID, uint32_t seq); + void broadcastStatusSeq(); + + void updatePeerNodeStatus(std::string const& p2pID, GatewayNodeInfo::Ptr status); + +private: + Service::Ptr m_service; + GatewayNodeInfoFactory::Ptr m_nodeStatusFactory; + std::shared_ptr m_timer; + + LocalRouter::Ptr m_localRouter; + PeerRouterTable::Ptr m_peerRouter; + + bool m_running = false; + + // P2pID => statusSeq + std::map m_p2pID2Seq; + mutable bcos::SharedMutex x_p2pID2Seq; + + // TODO: make this configurable + unsigned const SEQ_SYNC_PERIOD = 3000; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 9df4e45f..7d934daa 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -25,6 +25,7 @@ using namespace bcos; using namespace ppc::protocol; using namespace ppc::gateway; +// Note: the change of the topic will not trigger router-update void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& topic) { m_routerInfo->registerTopic(_nodeID.toBytes(), topic); @@ -48,6 +49,7 @@ void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& } } +// Note: the change of the topic will not trigger router-update void LocalRouter::unRegisterTopic(bcos::bytesConstRef _nodeID, std::string const& topic) { m_routerInfo->unRegisterTopic(_nodeID.toBytes(), topic); diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index 194ef6ab..8b931c9c 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -42,10 +42,19 @@ class LocalRouter virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) { nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); - return m_routerInfo->tryAddNodeInfo(nodeInfo); + auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); + if (ret) + { + increaseSeq(); + } + return ret; } - virtual void unRegisterNode(bcos::bytes const& nodeID) { m_routerInfo->removeNodeInfo(nodeID); } + virtual void unRegisterNode(bcos::bytes const& nodeID) + { + m_routerInfo->removeNodeInfo(nodeID); + increaseSeq(); + } virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic); virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic); @@ -57,10 +66,28 @@ class LocalRouter virtual bool dispatcherMessage(ppc::protocol::Message::Ptr const& msg, ppc::protocol::ReceiveMsgFunc callback, bool holding = true); + std::shared_ptr generateNodeStatus() + { + auto data = std::make_shared(); + m_routerInfo->encode(*data); + return data; + } + uint32_t statusSeq() { return m_statusSeq; } + +private: + uint32_t increaseSeq() + { + uint32_t statusSeq = ++m_statusSeq; + return statusSeq; + } + + private: ppc::front::IFrontBuilder::Ptr m_frontBuilder; GatewayNodeInfo::Ptr m_routerInfo; + std::atomic m_statusSeq{1}; + // NodeID=>topics using Topics = std::set; std::map m_topicInfo; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index a2bccd0a..e5367eda 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -33,8 +33,16 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) for (auto const& it : nodeList) { // update nodeID => gatewayInfos + if (!m_nodeID2GatewayInfos.count(it.first)) + { + m_nodeID2GatewayInfos.insert(std::make_pair(it.first, GatewayNodeInfos())); + } m_nodeID2GatewayInfos[it.first].insert(gatewayInfo); } + if (!m_agency2GatewayInfos.count(gatewayInfo->agency())) + { + m_agency2GatewayInfos.insert(std::make_pair(gatewayInfo->agency(), GatewayNodeInfos())); + } // update agency => gatewayInfos m_agency2GatewayInfos[gatewayInfo->agency()].insert(gatewayInfo); } diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h index a1701c9e..64ebf4c8 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h @@ -61,7 +61,6 @@ class RouterManager // for message forward Service::Ptr m_service; std::shared_ptr m_routerTimer; - std::atomic m_statusSeq{1}; // called when the given node unreachable std::vector> m_unreachableHandlers; @@ -69,5 +68,7 @@ class RouterManager std::map m_node2Seq; mutable bcos::SharedMutex x_node2Seq; + + std::atomic m_statusSeq{1}; }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars index 9a0c67ec..06055197 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars +++ b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars @@ -11,5 +11,6 @@ module ppctars 1 require string p2pNodeID; 2 require string agency; 3 optional vector nodeList; + 4 optional int statusSeq; }; }; From 4d36b1233b6fbb7292da72e11073e9c2c8b88453 Mon Sep 17 00:00:00 2001 From: shareong <740310627@qq.com> Date: Thu, 29 Aug 2024 16:15:26 +0800 Subject: [PATCH 017/120] add scheduler --- python/ppc_common/db_models/computing_node.py | 10 + python/ppc_common/db_models/config.sql | 26 + .../ppc_common/db_models/job_unit_record.py | 14 - .../ppc_common/db_models/job_worker_record.py | 15 + .../async_subprocess_executor.py | 24 +- .../async_thread_executor.py | 32 +- .../thread_event_manager.py | 24 +- .../dataset_handler_initialize.py | 2 +- .../ppc_protos/generated/ppc_pb2.py | 58 +-- python/ppc_common/ppc_protos/ppc.proto | 151 +----- .../ppc_common/ppc_utils/anonymous_search.py | 420 ---------------- python/ppc_common/ppc_utils/audit_utils.py | 56 --- python/ppc_common/ppc_utils/cem_utils.py | 160 ------ python/ppc_common/ppc_utils/permission.py | 79 --- python/ppc_common/ppc_utils/utils.py | 458 +---------------- python/ppc_model/common/initializer.py | 2 - .../local_processing/psi_select.py | 2 +- python/ppc_scheduler/__init__.py | 0 python/ppc_scheduler/common/__init__.py | 0 python/ppc_scheduler/common/global_context.py | 8 + python/ppc_scheduler/common/initializer.py | 91 ++++ python/ppc_scheduler/common/log_utils.py | 71 +++ .../ppc_scheduler/conf/application-sample.yml | 20 + python/ppc_scheduler/conf/logging.conf | 40 ++ python/ppc_scheduler/database/__init__.py | 0 .../database/computing_node_mapper.py | 69 +++ .../database/job_worker_mapper.py | 32 ++ python/ppc_scheduler/endpoints/__init__.py | 0 python/ppc_scheduler/endpoints/body_schema.py | 18 + .../ppc_scheduler/endpoints/job_controller.py | 48 ++ python/ppc_scheduler/endpoints/restx.py | 35 ++ python/ppc_scheduler/job/__init__.py | 0 python/ppc_scheduler/job/job_manager.py | 123 +++++ python/ppc_scheduler/job/job_status.py | 6 + python/ppc_scheduler/job/job_type.py | 8 + .../ppc_scheduler/mpc_generator/__init__.py | 0 .../ppc_scheduler/mpc_generator/generator.py | 468 ++++++++++++++++++ .../mpc_generator/mpc_func_str.py | 64 +++ .../mpc_sample/aggr_func_only.mpc | 136 +++++ .../mpc_sample/aggr_func_with_group_by.mpc | 234 +++++++++ .../mpc_sample/basic_arith_ope.mpc | 78 +++ .../mpc_generator/test_generator.py | 101 ++++ python/ppc_scheduler/node/__init__.py | 0 .../node/computing_node_client/__init__.py | 0 .../model_node_client.py | 87 ++++ .../computing_node_client/mpc_node_client.py | 32 ++ .../computing_node_client/psi_node_client.py | 66 +++ .../node/computing_node_client/utils.py | 8 + python/ppc_scheduler/node/node_manager.py | 32 ++ python/ppc_scheduler/ppc_scheduler_app.py | 48 ++ python/ppc_scheduler/workflow/__init__.py | 0 .../ppc_scheduler/workflow/common/__init__.py | 0 python/ppc_scheduler/workflow/common/codec.py | 57 +++ .../workflow/common/default_flow_config.py | 225 +++++++++ .../workflow/common/flow_utils.py | 34 ++ .../workflow/common/job_context.py | 115 +++++ .../workflow/common/worker_status.py | 7 + .../workflow/common/worker_type.py | 17 + python/ppc_scheduler/workflow/constructor.py | 82 +++ python/ppc_scheduler/workflow/scheduler.py | 87 ++++ .../ppc_scheduler/workflow/worker/__init__.py | 0 .../workflow/worker/api_worker.py | 10 + .../workflow/worker/engine/__init__.py | 0 .../workflow/worker/engine/model_engine.py | 101 ++++ .../workflow/worker/engine/mpc_engine.py | 284 +++++++++++ .../workflow/worker/engine/psi_engine.py | 158 ++++++ .../workflow/worker/exit_worker.py | 33 ++ .../workflow/worker/model_worker.py | 19 + .../workflow/worker/mpc_worker.py | 19 + .../workflow/worker/psi_worker.py | 19 + .../workflow/worker/python_worker.py | 10 + .../workflow/worker/shell_worker.py | 10 + .../ppc_scheduler/workflow/worker/worker.py | 96 ++++ .../workflow/worker/worker_factory.py | 32 ++ python/requirements.txt | 10 +- 75 files changed, 3455 insertions(+), 1426 deletions(-) create mode 100644 python/ppc_common/db_models/computing_node.py create mode 100644 python/ppc_common/db_models/config.sql delete mode 100644 python/ppc_common/db_models/job_unit_record.py create mode 100644 python/ppc_common/db_models/job_worker_record.py delete mode 100644 python/ppc_common/ppc_utils/anonymous_search.py delete mode 100644 python/ppc_common/ppc_utils/audit_utils.py delete mode 100644 python/ppc_common/ppc_utils/cem_utils.py delete mode 100644 python/ppc_common/ppc_utils/permission.py create mode 100644 python/ppc_scheduler/__init__.py create mode 100644 python/ppc_scheduler/common/__init__.py create mode 100644 python/ppc_scheduler/common/global_context.py create mode 100644 python/ppc_scheduler/common/initializer.py create mode 100644 python/ppc_scheduler/common/log_utils.py create mode 100644 python/ppc_scheduler/conf/application-sample.yml create mode 100644 python/ppc_scheduler/conf/logging.conf create mode 100644 python/ppc_scheduler/database/__init__.py create mode 100644 python/ppc_scheduler/database/computing_node_mapper.py create mode 100644 python/ppc_scheduler/database/job_worker_mapper.py create mode 100644 python/ppc_scheduler/endpoints/__init__.py create mode 100644 python/ppc_scheduler/endpoints/body_schema.py create mode 100644 python/ppc_scheduler/endpoints/job_controller.py create mode 100644 python/ppc_scheduler/endpoints/restx.py create mode 100644 python/ppc_scheduler/job/__init__.py create mode 100644 python/ppc_scheduler/job/job_manager.py create mode 100644 python/ppc_scheduler/job/job_status.py create mode 100644 python/ppc_scheduler/job/job_type.py create mode 100644 python/ppc_scheduler/mpc_generator/__init__.py create mode 100644 python/ppc_scheduler/mpc_generator/generator.py create mode 100644 python/ppc_scheduler/mpc_generator/mpc_func_str.py create mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc create mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc create mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc create mode 100644 python/ppc_scheduler/mpc_generator/test_generator.py create mode 100644 python/ppc_scheduler/node/__init__.py create mode 100644 python/ppc_scheduler/node/computing_node_client/__init__.py create mode 100644 python/ppc_scheduler/node/computing_node_client/model_node_client.py create mode 100644 python/ppc_scheduler/node/computing_node_client/mpc_node_client.py create mode 100644 python/ppc_scheduler/node/computing_node_client/psi_node_client.py create mode 100644 python/ppc_scheduler/node/computing_node_client/utils.py create mode 100644 python/ppc_scheduler/node/node_manager.py create mode 100644 python/ppc_scheduler/ppc_scheduler_app.py create mode 100644 python/ppc_scheduler/workflow/__init__.py create mode 100644 python/ppc_scheduler/workflow/common/__init__.py create mode 100644 python/ppc_scheduler/workflow/common/codec.py create mode 100644 python/ppc_scheduler/workflow/common/default_flow_config.py create mode 100644 python/ppc_scheduler/workflow/common/flow_utils.py create mode 100644 python/ppc_scheduler/workflow/common/job_context.py create mode 100644 python/ppc_scheduler/workflow/common/worker_status.py create mode 100644 python/ppc_scheduler/workflow/common/worker_type.py create mode 100644 python/ppc_scheduler/workflow/constructor.py create mode 100644 python/ppc_scheduler/workflow/scheduler.py create mode 100644 python/ppc_scheduler/workflow/worker/__init__.py create mode 100644 python/ppc_scheduler/workflow/worker/api_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/__init__.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/model_engine.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/mpc_engine.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/psi_engine.py create mode 100644 python/ppc_scheduler/workflow/worker/exit_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/model_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/mpc_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/psi_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/python_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/shell_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/worker.py create mode 100644 python/ppc_scheduler/workflow/worker/worker_factory.py diff --git a/python/ppc_common/db_models/computing_node.py b/python/ppc_common/db_models/computing_node.py new file mode 100644 index 00000000..bc0b988d --- /dev/null +++ b/python/ppc_common/db_models/computing_node.py @@ -0,0 +1,10 @@ + +from ppc_common.db_models import db + + +class ComputingNodeRecord(db.Model): + __tablename__ = 't_computing_node' + id = db.Column(db.String(255), primary_key=True) + url = db.Column(db.String(255)) + type = db.Column(db.String(255)) + loading = db.Column(db.Integer) diff --git a/python/ppc_common/db_models/config.sql b/python/ppc_common/db_models/config.sql new file mode 100644 index 00000000..295f5820 --- /dev/null +++ b/python/ppc_common/db_models/config.sql @@ -0,0 +1,26 @@ + +CREATE TABLE t_job_worker ( + worker_id VARCHAR(100) PRIMARY KEY, + job_id VARCHAR(255) INDEX, + type VARCHAR(255), + status VARCHAR(255), + upstreams TEXT, + inputs_statement TEXT, + outputs TEXT, + create_time BIGINT, + update_time BIGINT +)ENGINE=InnoDB default charset=utf8mb4 default collate=utf8mb4_unicode_ci; + +CREATE TABLE t_computing_node ( + id VARCHAR(255) PRIMARY KEY, + url VARCHAR(255), + type VARCHAR(255), + loading INT +)ENGINE=InnoDB default charset=utf8mb4 default collate=utf8mb4_unicode_ci; + + +INSERT INTO t_computing_node (id, url, type, loading) +VALUES + ("001", '127.0.0.1:10200', 'PSI', 0), + ("002", '127.0.0.1:10201', 'MPC', 0), + ("003", '127.0.0.1:10202', 'MODEL', 0); diff --git a/python/ppc_common/db_models/job_unit_record.py b/python/ppc_common/db_models/job_unit_record.py deleted file mode 100644 index f953ecbb..00000000 --- a/python/ppc_common/db_models/job_unit_record.py +++ /dev/null @@ -1,14 +0,0 @@ - -from ppc_common.db_models import db - - -class JobUnitRecord(db.Model): - __tablename__ = 't_job_unit' - unit_id = db.Column(db.String(100), primary_key=True) - job_id = db.Column(db.String(255), index=True) - type = db.Column(db.String(255)) - status = db.Column(db.String(255), index=True) - upstream_units = db.Column(db.Text) - inputs_statement = db.Column(db.Text) - outputs = db.Column(db.Text) - update_time = db.Column(db.BigInteger) diff --git a/python/ppc_common/db_models/job_worker_record.py b/python/ppc_common/db_models/job_worker_record.py new file mode 100644 index 00000000..29f94f61 --- /dev/null +++ b/python/ppc_common/db_models/job_worker_record.py @@ -0,0 +1,15 @@ + +from ppc_common.db_models import db + + +class JobWorkerRecord(db.Model): + __tablename__ = 't_job_worker' + worker_id = db.Column(db.String(100), primary_key=True) + job_id = db.Column(db.String(255), index=True) + type = db.Column(db.String(255)) + status = db.Column(db.String(255)) + upstreams = db.Column(db.Text) + inputs_statement = db.Column(db.Text) + outputs = db.Column(db.Text) + create_time = db.Column(db.BigInteger) + update_time = db.Column(db.BigInteger) diff --git a/python/ppc_common/ppc_async_executor/async_subprocess_executor.py b/python/ppc_common/ppc_async_executor/async_subprocess_executor.py index 87083ac8..afff9950 100644 --- a/python/ppc_common/ppc_async_executor/async_subprocess_executor.py +++ b/python/ppc_common/ppc_async_executor/async_subprocess_executor.py @@ -15,30 +15,30 @@ def __init__(self, logger): self._cleanup_thread.daemon = True self._cleanup_thread.start() - def execute(self, task_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], + def execute(self, target_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], args=()): process = multiprocessing.Process(target=target, args=args) process.start() with self.lock: - self.processes[task_id] = process + self.processes[target_id] = process - def kill(self, task_id: str): + def kill(self, target_id: str): with self.lock: - if task_id not in self.processes: + if target_id not in self.processes: return False else: - process = self.processes[task_id] + process = self.processes[target_id] process.terminate() - self.logger.info(f"Task {task_id} has been terminated!") + self.logger.info(f"Target {target_id} has been terminated!") return True def kill_all(self): with self.lock: keys = self.processes.keys() - for task_id in keys: - self.kill(task_id) + for target_id in keys: + self.kill(target_id) def _loop_cleanup(self): while True: @@ -48,13 +48,13 @@ def _loop_cleanup(self): def _cleanup_finished_processes(self): with self.lock: finished_processes = [ - (task_id, proc) for task_id, proc in self.processes.items() if not proc.is_alive()] + (target_id, proc) for target_id, proc in self.processes.items() if not proc.is_alive()] - for task_id, process in finished_processes: + for target_id, process in finished_processes: with self.lock: process.join() # 确保进程资源释放 - del self.processes[task_id] - self.logger.info(f"Cleanup finished task process {task_id}") + del self.processes[target_id] + self.logger.info(f"Cleanup finished process {target_id}") def __del__(self): self.kill_all() diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py index 0bc65b91..15b1f5ca 100644 --- a/python/ppc_common/ppc_async_executor/async_thread_executor.py +++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py @@ -17,44 +17,44 @@ def __init__(self, event_manager: ThreadEventManager, logger): self._cleanup_thread.daemon = True self._cleanup_thread.start() - def execute(self, task_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], + def execute(self, target_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None], args=()): def thread_target(logger, on_finish, *args): try: target(*args) - on_finish(task_id, True) + on_finish(target_id, True) except Exception as e: logger.warn(traceback.format_exc()) - on_finish(task_id, False, e) + on_finish(target_id, False, e) thread = threading.Thread(target=thread_target, args=( self.logger, on_target_finish,) + args) thread.start() with self.lock: - self.threads[task_id] = thread + self.threads[target_id] = thread stop_event = threading.Event() - self.event_manager.add_event(task_id, stop_event) + self.event_manager.add_event(target_id, stop_event) - def kill(self, task_id: str): + def kill(self, target_id: str): with self.lock: - if task_id not in self.threads: + if target_id not in self.threads: return False else: - thread = self.threads[task_id] + thread = self.threads[target_id] - self.event_manager.set_event(task_id) + self.event_manager.set_event(target_id) thread.join() - self.logger.info(f"Task {task_id} has been stopped!") + self.logger.info(f"Target {target_id} has been stopped!") return True def kill_all(self): with self.lock: keys = self.threads.keys() - for task_id in keys: - self.kill(task_id) + for target_id in keys: + self.kill(target_id) def _loop_cleanup(self): while True: @@ -64,12 +64,12 @@ def _loop_cleanup(self): def _cleanup_finished_threads(self): with self.lock: finished_threads = [ - task_id for task_id, thread in self.threads.items() if not thread.is_alive()] + target_id for target_id, thread in self.threads.items() if not thread.is_alive()] - for task_id in finished_threads: + for target_id in finished_threads: with self.lock: - del self.threads[task_id] - self.logger.info(f"Cleanup finished task thread {task_id}") + del self.threads[target_id] + self.logger.info(f"Cleanup finished thread {target_id}") def __del__(self): self.kill_all() diff --git a/python/ppc_common/ppc_async_executor/thread_event_manager.py b/python/ppc_common/ppc_async_executor/thread_event_manager.py index f4311b69..b78a7883 100644 --- a/python/ppc_common/ppc_async_executor/thread_event_manager.py +++ b/python/ppc_common/ppc_async_executor/thread_event_manager.py @@ -10,25 +10,25 @@ def __init__(self): self.events: Dict[str, threading.Event] = {} self.rw_lock = rwlock.RWLockWrite() - def add_event(self, task_id: str, event: threading.Event) -> None: + def add_event(self, target_id: str, event: threading.Event) -> None: with self.rw_lock.gen_wlock(): - self.events[task_id] = event + self.events[target_id] = event - def remove_event(self, task_id: str): + def remove_event(self, target_id: str): with self.rw_lock.gen_wlock(): - if task_id in self.events: - del self.events[task_id] + if target_id in self.events: + del self.events[target_id] - def set_event(self, task_id: str): + def set_event(self, target_id: str): with self.rw_lock.gen_wlock(): - if task_id in self.events: - self.events[task_id].set() + if target_id in self.events: + self.events[target_id].set() else: - raise KeyError(f"Task ID {task_id} not found") + raise KeyError(f"Target id {target_id} not found") - def event_status(self, task_id: str) -> bool: + def event_status(self, target_id: str) -> bool: with self.rw_lock.gen_rlock(): - if task_id in self.events: - return self.events[task_id].is_set() + if target_id in self.events: + return self.events[target_id].is_set() else: return False diff --git a/python/ppc_common/ppc_initialize/dataset_handler_initialize.py b/python/ppc_common/ppc_initialize/dataset_handler_initialize.py index 4b2e64fe..3abe855e 100644 --- a/python/ppc_common/ppc_initialize/dataset_handler_initialize.py +++ b/python/ppc_common/ppc_initialize/dataset_handler_initialize.py @@ -10,7 +10,7 @@ class DataSetHandlerInitialize: def __init__(self, config, logger): self._config = config self._logger = logger - self._init_sql_storage() + # self._init_sql_storage() self._init_remote_storage() self._init_dataset_factory() diff --git a/python/ppc_common/ppc_protos/generated/ppc_pb2.py b/python/ppc_common/ppc_protos/generated/ppc_pb2.py index 5c8200ef..623c0434 100644 --- a/python/ppc_common/ppc_protos/generated/ppc_pb2.py +++ b/python/ppc_common/ppc_protos/generated/ppc_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: ppc.proto -# Protobuf Python Version: 4.25.1 +# Protobuf Python Version: 4.25.3 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -12,53 +12,21 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tppc.proto\x12\tppc.proto\"\xe5\x01\n\x0f\x41lgorithmDetail\x12\x1d\n\x15\x61lgorithm_description\x18\x01 \x01(\t\x12\x16\n\x0e\x61lgorithm_type\x18\x02 \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x03 \x01(\x03\x12\x13\n\x0bupdate_time\x18\x04 \x01(\x03\x12\x17\n\x0fowner_user_name\x18\x05 \x01(\t\x12\x17\n\x0fowner_agency_id\x18\x06 \x01(\t\x12\x19\n\x11owner_agency_name\x18\x07 \x01(\t\x12$\n\x08ppc_flow\x18\x08 \x01(\x0b\x32\x12.ppc.proto.PpcFlow\"\xc1\x01\n\x07PpcFlow\x12\x13\n\x0binput_count\x18\x01 \x01(\t\x12\x12\n\nsql_module\x18\x02 \x01(\t\x12\x12\n\nmpc_module\x18\x03 \x01(\t\x12\x18\n\x10mpc_model_module\x18\x04 \x01(\t\x12\x14\n\x0cmatch_module\x18\x05 \x01(\t\x12\x19\n\x11\x61lgorithm_subtype\x18\x06 \x01(\t\x12\x1a\n\x12participant_agency\x18\x07 \x01(\t\x12\x12\n\nmodel_type\x18\x08 \x01(\t\"\xee\x01\n\rDatasetDetail\x12\x15\n\rdataset_title\x18\x01 \x01(\t\x12\x1b\n\x13\x64\x61taset_description\x18\x02 \x01(\t\x12\x14\n\x0c\x64\x61taset_hash\x18\x03 \x01(\t\x12\x14\n\x0c\x63olumn_count\x18\x04 \x01(\x03\x12\x11\n\trow_count\x18\x05 \x01(\x03\x12\x13\n\x0b\x63reate_time\x18\x06 \x01(\x03\x12\x11\n\tuser_name\x18\x07 \x01(\t\x12\x17\n\x0f\x64\x61ta_field_list\x18\x08 \x01(\t\x12\x14\n\x0c\x64\x61taset_size\x18\t \x01(\x03\x12\x13\n\x0bupdate_time\x18\n \x01(\x03\"K\n\x0f\x44\x61tasetAuthData\x12\x0f\n\x07is_auth\x18\x01 \x01(\x08\x12\x13\n\x0b\x63reate_time\x18\x02 \x01(\x03\x12\x12\n\nvalid_time\x18\x03 \x01(\x03\"\xde\x01\n\x12jobDatasetProvider\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x13\n\x0b\x61gency_name\x18\x02 \x01(\t\x12\x12\n\ndataset_id\x18\x03 \x01(\t\x12\x15\n\rdataset_title\x18\x04 \x01(\t\x12\x15\n\rloading_index\x18\x05 \x01(\x03\x12\x1b\n\x13\x64\x61taset_description\x18\x06 \x01(\t\x12\x17\n\x0fowner_user_name\x18\x07 \x01(\t\x12\x14\n\x0c\x63olumns_size\x18\x08 \x01(\x03\x12\x12\n\nis_instant\x18\t \x01(\x03\"\x94\x01\n\x15\x64\x61tasourceInstantInfo\x12\r\n\x05\x64\x62_ip\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x62_name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x62_password\x18\x03 \x01(\t\x12\x0f\n\x07\x64\x62_port\x18\x04 \x01(\x03\x12\x0e\n\x06\x64\x62_sql\x18\x05 \x01(\t\x12\x14\n\x0c\x64\x62_user_name\x18\x06 \x01(\t\x12\x0f\n\x07\x64\x62_type\x18\x07 \x01(\t\"I\n\x16jobDatasetProviderList\x12/\n\x08provider\x18\x01 \x03(\x0b\x32\x1d.ppc.proto.jobDatasetProvider\"m\n\x16jobComputationProvider\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x13\n\x0b\x61gency_name\x18\x02 \x01(\t\x12\x12\n\nagency_url\x18\x03 \x01(\t\x12\x17\n\x0f\x63omputing_index\x18\x04 \x01(\x03\"Q\n\x1ajobComputationProviderList\x12\x33\n\x08provider\x18\x01 \x03(\x0b\x32!.ppc.proto.jobComputationProvider\")\n\x15jobResultReceiverList\x12\x10\n\x08receiver\x18\x01 \x03(\t\"\x8f\x05\n\x08JobEvent\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x11\n\tjob_title\x18\x02 \x01(\t\x12\x17\n\x0fjob_description\x18\x03 \x01(\t\x12\x14\n\x0cjob_priority\x18\x04 \x01(\x03\x12\x13\n\x0bjob_creator\x18\x05 \x01(\t\x12\x1b\n\x13initiator_agency_id\x18\x06 \x01(\t\x12\x1d\n\x15initiator_agency_name\x18\x07 \x01(\t\x12\x1b\n\x13initiator_signature\x18\x08 \x01(\t\x12\x18\n\x10job_algorithm_id\x18\t \x01(\t\x12\x1b\n\x13job_algorithm_title\x18\n \x01(\t\x12\x1d\n\x15job_algorithm_version\x18\x0b \x01(\t\x12\x1a\n\x12job_algorithm_type\x18\x0c \x01(\t\x12$\n\x1cjob_dataset_provider_list_pb\x18\r \x01(\t\x12#\n\x1bjob_result_receiver_list_pb\x18\x0e \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x0f \x01(\x03\x12\x13\n\x0bupdate_time\x18\x10 \x01(\x03\x12\x12\n\npsi_fields\x18\x11 \x01(\t\x12\x1d\n\x15job_algorithm_subtype\x18\x12 \x01(\t\x12\x14\n\x0cmatch_fields\x18\x13 \x01(\t\x12\x16\n\x0eis_cem_encrypt\x18\x14 \x01(\x08\x12\x17\n\x0f\x64\x61taset_id_list\x18\x15 \x03(\t\x12\x30\n\x0e\x64\x61taset_detail\x18\x16 \x01(\x0b\x32\x18.ppc.proto.DatasetDetail\x12\x1e\n\x16tag_provider_agency_id\x18\x17 \x01(\t\x12\x10\n\x08job_type\x18\x18 \x01(\t\"/\n\tAuditItem\x12\x13\n\x0b\x64\x65scription\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"5\n\tAuditData\x12(\n\naudit_item\x18\x01 \x03(\x0b\x32\x14.ppc.proto.AuditItem\"E\n\x10JobOutputPreview\x12\x0e\n\x06header\x18\x01 \x03(\t\x12!\n\x04line\x18\x02 \x03(\x0b\x32\x13.ppc.proto.DataLine\"\x19\n\x08\x44\x61taLine\x12\r\n\x05value\x18\x01 \x03(\t\"=\n\x0eInputStatement\x12\x15\n\rupstream_unit\x18\x01 \x01(\t\x12\x14\n\x0coutput_index\x18\x02 \x01(\x03\"M\n\x16JobUnitInputsStatement\x12\x33\n\x10inputs_statement\x18\x01 \x03(\x0b\x32\x19.ppc.proto.InputStatement\"!\n\x0eJobUnitOutputs\x12\x0f\n\x07outputs\x18\x01 \x03(\t\")\n\x0fJobUnitUpstream\x12\x16\n\x0eupstream_units\x18\x01 \x03(\t\"<\n\tAlgorithm\x12\x14\n\x0c\x61lgorithm_id\x18\x01 \x01(\t\x12\x19\n\x11\x61lgorithm_version\x18\x02 \x01(\t\"6\n\nAlgorithms\x12(\n\nalgorithms\x18\x01 \x03(\x0b\x32\x14.ppc.proto.Algorithmb\x06proto3') + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tppc.proto\x12\tppc.proto\"8\n\x0eInputStatement\x12\x10\n\x08upstream\x18\x01 \x01(\t\x12\x14\n\x0coutput_index\x18\x02 \x01(\x03\"O\n\x18JobWorkerInputsStatement\x12\x33\n\x10inputs_statement\x18\x01 \x03(\x0b\x32\x19.ppc.proto.InputStatement\"\'\n\x12JobWorkerUpstreams\x12\x11\n\tupstreams\x18\x01 \x03(\t\"#\n\x10JobWorkerOutputs\x12\x0f\n\x07outputs\x18\x01 \x03(\tb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ppc_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals['_ALGORITHMDETAIL']._serialized_start = 25 - _globals['_ALGORITHMDETAIL']._serialized_end = 254 - _globals['_PPCFLOW']._serialized_start = 257 - _globals['_PPCFLOW']._serialized_end = 450 - _globals['_DATASETDETAIL']._serialized_start = 453 - _globals['_DATASETDETAIL']._serialized_end = 691 - _globals['_DATASETAUTHDATA']._serialized_start = 693 - _globals['_DATASETAUTHDATA']._serialized_end = 768 - _globals['_JOBDATASETPROVIDER']._serialized_start = 771 - _globals['_JOBDATASETPROVIDER']._serialized_end = 993 - _globals['_DATASOURCEINSTANTINFO']._serialized_start = 996 - _globals['_DATASOURCEINSTANTINFO']._serialized_end = 1144 - _globals['_JOBDATASETPROVIDERLIST']._serialized_start = 1146 - _globals['_JOBDATASETPROVIDERLIST']._serialized_end = 1219 - _globals['_JOBCOMPUTATIONPROVIDER']._serialized_start = 1221 - _globals['_JOBCOMPUTATIONPROVIDER']._serialized_end = 1330 - _globals['_JOBCOMPUTATIONPROVIDERLIST']._serialized_start = 1332 - _globals['_JOBCOMPUTATIONPROVIDERLIST']._serialized_end = 1413 - _globals['_JOBRESULTRECEIVERLIST']._serialized_start = 1415 - _globals['_JOBRESULTRECEIVERLIST']._serialized_end = 1456 - _globals['_JOBEVENT']._serialized_start = 1459 - _globals['_JOBEVENT']._serialized_end = 2114 - _globals['_AUDITITEM']._serialized_start = 2116 - _globals['_AUDITITEM']._serialized_end = 2163 - _globals['_AUDITDATA']._serialized_start = 2165 - _globals['_AUDITDATA']._serialized_end = 2218 - _globals['_JOBOUTPUTPREVIEW']._serialized_start = 2220 - _globals['_JOBOUTPUTPREVIEW']._serialized_end = 2289 - _globals['_DATALINE']._serialized_start = 2291 - _globals['_DATALINE']._serialized_end = 2316 - _globals['_INPUTSTATEMENT']._serialized_start = 2318 - _globals['_INPUTSTATEMENT']._serialized_end = 2379 - _globals['_JOBUNITINPUTSSTATEMENT']._serialized_start = 2381 - _globals['_JOBUNITINPUTSSTATEMENT']._serialized_end = 2458 - _globals['_JOBUNITOUTPUTS']._serialized_start = 2460 - _globals['_JOBUNITOUTPUTS']._serialized_end = 2493 - _globals['_JOBUNITUPSTREAM']._serialized_start = 2495 - _globals['_JOBUNITUPSTREAM']._serialized_end = 2536 - _globals['_ALGORITHM']._serialized_start = 2538 - _globals['_ALGORITHM']._serialized_end = 2598 - _globals['_ALGORITHMS']._serialized_start = 2600 - _globals['_ALGORITHMS']._serialized_end = 2654 + DESCRIPTOR._options = None + _globals['_INPUTSTATEMENT']._serialized_start=24 + _globals['_INPUTSTATEMENT']._serialized_end=80 + _globals['_JOBWORKERINPUTSSTATEMENT']._serialized_start=82 + _globals['_JOBWORKERINPUTSSTATEMENT']._serialized_end=161 + _globals['_JOBWORKERUPSTREAMS']._serialized_start=163 + _globals['_JOBWORKERUPSTREAMS']._serialized_end=202 + _globals['_JOBWORKEROUTPUTS']._serialized_start=204 + _globals['_JOBWORKEROUTPUTS']._serialized_end=239 # @@protoc_insertion_point(module_scope) diff --git a/python/ppc_common/ppc_protos/ppc.proto b/python/ppc_common/ppc_protos/ppc.proto index a9571c0f..ca9a3a5b 100644 --- a/python/ppc_common/ppc_protos/ppc.proto +++ b/python/ppc_common/ppc_protos/ppc.proto @@ -2,159 +2,20 @@ syntax = "proto3"; package ppc.proto; -message AlgorithmDetail { - string algorithm_description = 1; - string algorithm_type = 2; - int64 create_time = 3; - int64 update_time = 4; - string owner_user_name = 5; - string owner_agency_id = 6; - string owner_agency_name = 7; - PpcFlow ppc_flow = 8; -} - -message PpcFlow { - // input_count need present '3+' - string input_count = 1; - string sql_module = 2; - string mpc_module = 3; - string mpc_model_module = 4; - string match_module = 5; - string algorithm_subtype = 6; - string participant_agency = 7; - string model_type = 8; -} - -message DatasetDetail { - string dataset_title = 1; - string dataset_description = 2; - string dataset_hash = 3; - int64 column_count = 4; - int64 row_count = 5; - int64 create_time = 6; - string user_name = 7; - string data_field_list = 8; - int64 dataset_size = 9; - int64 update_time = 10; -} - -message DatasetAuthData { - bool is_auth = 1; - int64 create_time = 2; - int64 valid_time = 3; -} - - -message jobDatasetProvider { - string agency_id = 1; - string agency_name = 2; - string dataset_id = 3; - string dataset_title = 4; - int64 loading_index = 5; - string dataset_description = 6; - string owner_user_name = 7; - int64 columns_size = 8; - int64 is_instant = 9; -} - -message datasourceInstantInfo { - string db_ip = 1; - string db_name = 2; - string db_password = 3; - int64 db_port = 4; - string db_sql = 5; - string db_user_name = 6; - string db_type = 7; -} - -message jobDatasetProviderList { - repeated jobDatasetProvider provider = 1; -} - -message jobComputationProvider { - string agency_id = 1; - string agency_name = 2; - string agency_url = 3; - int64 computing_index = 4; -} - -message jobComputationProviderList { - repeated jobComputationProvider provider = 1; -} - -message jobResultReceiverList { - repeated string receiver = 1; -} - - -message JobEvent { - string job_id = 1; - string job_title = 2; - string job_description = 3; - int64 job_priority = 4; - string job_creator = 5; - string initiator_agency_id = 6; - string initiator_agency_name = 7; - string initiator_signature = 8; - string job_algorithm_id = 9; - string job_algorithm_title = 10; - string job_algorithm_version = 11; - string job_algorithm_type = 12; - string job_dataset_provider_list_pb = 13; // jobDatasetProviderList - string job_result_receiver_list_pb = 14; // jobResultReceiverList - int64 create_time = 15; - int64 update_time = 16; - string psi_fields = 17; - string job_algorithm_subtype = 18; - string match_fields = 19; - bool is_cem_encrypt = 20; - repeated string dataset_id_list = 21; // cem encrypt dataset id without username prefix - DatasetDetail dataset_detail = 22; // cem encrypt dataset transfer - string tag_provider_agency_id = 23; - string job_type = 24; -} - -message AuditItem { - string description = 1; - string value = 2; -} - -message AuditData { - repeated AuditItem audit_item = 1; -} - -message JobOutputPreview { - repeated string header = 1; - repeated DataLine line = 2; -} - -message DataLine { - repeated string value = 1; -} message InputStatement { - string upstream_unit = 1; + string upstream = 1; int64 output_index = 2; } -message JobUnitInputsStatement { +message JobWorkerInputsStatement { repeated InputStatement inputs_statement = 1; } -message JobUnitOutputs { - repeated string outputs = 1; -} - -message JobUnitUpstream { - repeated string upstream_units = 1; +message JobWorkerUpstreams { + repeated string upstreams = 1; } -message Algorithm { - string algorithm_id = 1; - string algorithm_version = 2; -} - -message Algorithms { - repeated Algorithm algorithms = 1; +message JobWorkerOutputs { + repeated string outputs = 1; } - diff --git a/python/ppc_common/ppc_utils/anonymous_search.py b/python/ppc_common/ppc_utils/anonymous_search.py deleted file mode 100644 index 80c3df61..00000000 --- a/python/ppc_common/ppc_utils/anonymous_search.py +++ /dev/null @@ -1,420 +0,0 @@ -import hashlib -import json -import logging -import os -import random -import string -import unittest -import uuid -from io import BytesIO - -import pandas as pd - -from ppc_common.ppc_utils import utils, http_utils -from ppc_common.ppc_crypto import crypto_utils -from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode - -log = logging.getLogger(__name__) - - -def make_hash(data): - m = hashlib.sha3_256() - m.update(data) - return m.hexdigest() - - -def requester_gen_ot_cipher(id_list, obfuscation_order): - blinding_a = crypto_utils.get_random_int() - blinding_b = crypto_utils.get_random_int() - x_value = crypto_utils.ot_base_pown(blinding_a) - y_value = crypto_utils.ot_base_pown(blinding_b) - c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) - id_index_list = [] - send_hash_vec = [] - z_value_list = [] - for id_hash in id_list: - obs_list = id_obfuscation(obfuscation_order, None) - id_index = random.randint(0, obfuscation_order) - obs_list[id_index] = id_hash - id_index_list.append(id_index) - send_hash_vec.append(obs_list) - z_value = crypto_utils.ot_base_pown(c_blinding - id_index) - z_value_list.append(str(z_value)) - return id_index_list, blinding_b, x_value, y_value, send_hash_vec, z_value_list - - -def provider_gen_ot_cipher(x_value, y_value, send_hash_vec, z_value_list, data_map, is_contain_result=True): - """ - data_map = hashmap[id1: "message1", - id2: "message2"] - """ - if isinstance(x_value, str): - x_value = int(x_value) - if isinstance(y_value, str): - y_value = int(y_value) - if len(send_hash_vec) != len(z_value_list): - raise PpcException(PpcErrorCode.AYS_LENGTH_ERROR.get_code(), - PpcErrorCode.AYS_LENGTH_ERROR.get_msg()) - message_cipher_vec = [] - for idx, z_value in enumerate(z_value_list): - if isinstance(z_value, str): - z_value = int(z_value) - message_cipher_list = [] - message_int_len = 0 - for send_hash in send_hash_vec[idx]: - blinding_r = crypto_utils.get_random_int() - blinding_s = crypto_utils.get_random_int() - w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), - crypto_utils.ot_base_pown(blinding_r)) - key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), - crypto_utils.ot_pown(y_value, blinding_r)) - aes_key_bytes = os.urandom(16) - aes_default = utils.AESCipher(aes_key_bytes) - aes_key_base64str = utils.bytes_to_base64str(aes_key_bytes) - message_int, message_int_len = crypto_utils.ot_str_to_int( - aes_key_base64str) - - if send_hash not in data_map: - letters = string.ascii_lowercase - # random string - ot_message_str = 'message not found' - cipher = aes_default.encrypt(ot_message_str) - cipher_str = utils.bytes_to_base64str(cipher) - else: - if is_contain_result: - ot_message_str = json.dumps(data_map[send_hash]) - cipher = aes_default.encrypt(ot_message_str) - cipher_str = utils.bytes_to_base64str(cipher) - else: - ot_message_str = 'True' - cipher = aes_default.encrypt(ot_message_str) - cipher_str = utils.bytes_to_base64str(cipher) - message_cipher = key_value ^ message_int - z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) - message_cipher_list.append({ - "w": str(w_value), - "e": str(message_cipher), - "len": message_int_len, - 'aesCipher': cipher_str, - }) - message_cipher_vec.append(message_cipher_list) - return message_cipher_vec - - -def requester_ot_recover_result(id_index_list, blinding_b, message_cipher_vec): - if len(id_index_list) != len(message_cipher_vec): - raise PpcException(PpcErrorCode.AYS_LENGTH_ERROR.get_code(), - PpcErrorCode.AYS_LENGTH_ERROR.get_msg()) - result_list = [] - for idx, id_index in enumerate(id_index_list): - w_value = message_cipher_vec[idx][id_index]['w'] - e_value = message_cipher_vec[idx][id_index]['e'] - message_len = message_cipher_vec[idx][id_index]['len'] - cipher_str = message_cipher_vec[idx][id_index]['aesCipher'] - - if isinstance(w_value, str): - w_value = int(w_value) - if isinstance(e_value, str): - e_value = int(e_value) - w_1 = crypto_utils.ot_pown(w_value, blinding_b) - message_recover = w_1 ^ e_value - try: - aes_key = crypto_utils.ot_int_to_str(message_recover, message_len) - key_recover = utils.base64str_to_bytes(aes_key) - aes_recover = utils.AESCipher(key_recover) - cipher_recover = utils.base64str_to_bytes(cipher_str) - message_result = aes_recover.decrypt(cipher_recover) - result_list.append(message_result) - except Exception as be: - result_list.append(None) - return result_list - - -def id_obfuscation(obfuscation_order, rule=None): - # use rule extend different id order, such as driver card or name: - if rule is not None: - print('obfuscation is work in progress') - obs_list = [] - for i in range(0, obfuscation_order+1): - obs_list.append(make_hash(bytes(str(uuid.uuid4()), 'utf8'))) - return obs_list - - -def prepare_dataset_with_matrix(search_id_matrix, tmp_file_path, prepare_dataset_tmp_file_path): - search_reg = 'ppc-normal-prefix' - for search_list in search_id_matrix: - for search_id in search_list: - search_reg = '{}|{}'.format(search_reg, search_id) - exec_command = 'head -n 1 {} >> {}'.format( - tmp_file_path, prepare_dataset_tmp_file_path) - (status, result) = utils.getstatusoutput(exec_command) - if status != 0: - log.error( - f'[OnError]prepare_dataset_with_matrix! status is {status}, output is {result}') - else: - log.info( - f'prepare_dataset_with_matrix success! status is {status}, output is {result}') - exec_command = 'grep -E \'{}\' {} >> {}'.format( - search_reg, tmp_file_path, prepare_dataset_tmp_file_path) - (status, result) = utils.getstatusoutput(exec_command) - if status != 0: - log.error( - f'[OnError]prepare_dataset_with_matrix! status is {status}, output is {result}') - else: - log.info( - f'prepare_dataset_with_matrix success! status is {status}, output is {result}') - - -class TestOtMethods(unittest.TestCase): - - def test_choice_all_flow(self): - file_path = '/Users/asher/Downloads/test_file.csv' - # data_pd = pd.read_csv(file_path, index_col=0, header=0) - data_pd = pd.read_csv(file_path) - data_map = data_pd.set_index(data_pd.columns[0]).T.to_dict('list') - - choice = ['bob', '小鸡', '美丽'] - obs_order = 10 - _id_index_list, _blinding_b, _x_value, _y_value, _send_hash_vec, _z_value_list = requester_gen_ot_cipher( - choice, obs_order) - _message_cipher_vec = provider_gen_ot_cipher( - _x_value, _y_value, _send_hash_vec, _z_value_list, data_map) - result = requester_ot_recover_result( - _id_index_list, _blinding_b, _message_cipher_vec) - for idx, id_num in enumerate(choice): - print(f"found {id_num} value is {result[idx]}") - - # def test_choice_ot(self): - # choice_list = [1, 4] - # blinding_a = crypto_utils.get_random_int() - # blinding_b = crypto_utils.get_random_int() - # x_value = crypto_utils.ot_base_pown(blinding_a) - # y_value = crypto_utils.ot_base_pown(blinding_b) - # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) - # # c_value = crypto_utils.ot_base_pown(c_blinding) - # z_value_list = [] - # for choice in choice_list: - # z_value = crypto_utils.ot_base_pown(c_blinding - choice) - # z_value_list.append(z_value) - # # send x_value, y_value, z_value - # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] - # message_list = [] - # for message_str in message_str_list: - # message_int, message_int_len = crypto_utils.ot_str_to_int(message_str) - # message_list.append(message_int) - # # message_list = [111111, 222222, 333333, 444444, 555555] - # cipher_vec = [] - # for z_value in z_value_list: - # cipher_list = [] - # for message in message_list: - # blinding_r = crypto_utils.get_random_int() - # blinding_s = crypto_utils.get_random_int() - # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), - # crypto_utils.ot_base_pown(blinding_r)) - # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), - # crypto_utils.ot_pown(y_value, blinding_r)) - # z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) - # e_cipher = key_value ^ message - # cipher_list.append({ - # "w": w_value, - # "e": e_cipher - # }) - # cipher_vec.append(cipher_list) - # - # for cipher_each in cipher_vec: - # for cipher in cipher_each: - # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) - # message_recover = w_1 ^ cipher['e'] - # print(message_recover) - # for idx, cipher_list in enumerate(cipher_vec): - # w_1 = crypto_utils.ot_pown(cipher_list[choice_list[idx]]['w'], blinding_b) - # message_recover = w_1 ^ cipher_list[choice_list[idx]]['e'] - # s = crypto_utils.ot_int_to_str(message_recover) - # print(s) - - # def test_id_ot(self): - # print("test_id_ot") - # choice_id_list = [crypto_utils.ot_str_to_int('小明'), crypto_utils.ot_str_to_int('张三')] - # blinding_a = crypto_utils.get_random_int() - # blinding_b = crypto_utils.get_random_int() - # x_value = crypto_utils.ot_base_pown(blinding_a) - # y_value = crypto_utils.ot_base_pown(blinding_b) - # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) - # # c_value = crypto_utils.ot_base_pown(c_blinding) - # z_value_list = [] - # for choice in choice_id_list: - # z_value = crypto_utils.ot_base_pown(c_blinding - choice) - # z_value_list.append(z_value) - # # z_value = crypto_utils.ot_base_pown(c_blinding - choice_id) - # # send x_value, y_value, z_value - # id_str_list = ['小往', '小明', 'asher', 'set', '张三'] - # id_list = [] - # for id_str in id_str_list: - # id_list.append(crypto_utils.ot_str_to_int(id_str)) - # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] - # message_list = [] - # for message_str in message_str_list: - # message_list.append(crypto_utils.ot_str_to_int(message_str)) - # # message_list = [111111, 222222, 333333, 444444, 555555] - # cipher_vec = [] - # for z_value in z_value_list: - # cipher_list = [] - # for idx, message in enumerate(message_list): - # blinding_r = crypto_utils.get_random_int() - # blinding_s = crypto_utils.get_random_int() - # z_value_use = crypto_utils.ot_mul_n(z_value, crypto_utils.ot_base_pown(id_list[idx])) - # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), - # crypto_utils.ot_base_pown(blinding_r)) - # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value_use, blinding_s), - # crypto_utils.ot_pown(y_value, blinding_r)) - # e_cipher = key_value ^ message - # cipher_list.append({ - # "w": w_value, - # "e": e_cipher - # }) - # cipher_vec.append(cipher_list) - # - # for idx, cipher_list in enumerate(cipher_vec): - # for now_idx, cipher in enumerate(cipher_list): - # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) - # message_recover = w_1 ^ cipher['e'] - # # print(message_recover) - # # print(idx) - # # print(now_idx) - # if (idx == 0 and now_idx == 1) or (idx == 1 and now_idx == 4): - # s = crypto_utils.ot_int_to_str(message_recover) - # print(s) - - def test_pd_with_multi_index(self): - print(True) - df = pd.DataFrame( - [[21, 'Amol', 72, 67], - [23, 'Lini', 78, 69], - [32, 'Kiku', 74, 56], - [52, 'Ajit', 54, 76], - [53, 'Ajit', 55, 78] - ], - columns=['rollno', 'name', 'physics', 'botony']) - - print('DataFrame with default index\n', df) - # set multiple columns as index - # df_map = df.set_index('name').T.to_dict('list') - # df_map = df.set_index('name').groupby(level=0).apply(lambda x: x.to_dict('r')).to_dict() - df_map = df.set_index('name').groupby(level=0).apply( - lambda x: x.to_dict('r')).to_dict() - print(df_map) - print(json.dumps(df_map['Ajit'])) - print(type(json.dumps(df_map['Kiku']))) - - def test_prepare_dataset_with_matrix(self): - tmp_file_path = "/Users/asher/Desktop/数据集2021/8_1_100w.csv" - prepare_dataset_tmp_file_path = "/Users/asher/Desktop/数据集2021/pre-test_100.csv" - search_id_matrix = [['645515750175253924', '779808531920530393'], [ - '399352968694137676', '399352968694137676222', '399352968694137']] - prepare_dataset_with_matrix( - search_id_matrix, tmp_file_path, prepare_dataset_tmp_file_path) - - # - # def test_choice_ot_multi(self): - # choice_list = [1, 2, 4] - # blinding_a = crypto_utils.get_random_int() - # blinding_b = crypto_utils.get_random_int() - # x_value = crypto_utils.ot_base_pown(blinding_a) - # y_value = crypto_utils.ot_base_pown(blinding_b) - # c_blinding = crypto_utils.ot_mul_fi(blinding_a, blinding_b) - # # c_value = crypto_utils.ot_base_pown(c_blinding) - # choice_final = 0 - # for choice in choice_list: - # choice_final = choice_final + choice - # z_value = crypto_utils.ot_base_pown(c_blinding - choice_final) - # # send x_value, y_value, z_value - # message_str_list = ['hello', 'world', 'ot', 'cipher', 'test'] - # message_list = [] - # for message_str in message_str_list: - # message_list.append(crypto_utils.ot_str_to_int(message_str)) - # # message_list = [111111, 222222, 333333, 444444, 555555] - # cipher_list = [] - # for message in message_list: - # blinding_r = crypto_utils.get_random_int() - # blinding_s = crypto_utils.get_random_int() - # w_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(x_value, blinding_s), - # crypto_utils.ot_base_pown(blinding_r)) - # key_value = crypto_utils.ot_mul_n(crypto_utils.ot_pown(z_value, blinding_s), - # crypto_utils.ot_pown(y_value, blinding_r)) - # z_value = crypto_utils.ot_mul_n(z_value, crypto_utils.DEFAULT_G) - # e_cipher = key_value ^ message - # cipher_list.append({ - # "w": w_value, - # "e": e_cipher - # }) - # - # # for cipher in cipher_list: - # # w_1 = crypto_utils.ot_pown(cipher['w'], blinding_b) - # # message_recover = w_1 ^ cipher['e'] - # # print(message_recover) - # - # for choice in choice_list: - # w_1 = crypto_utils.ot_pown(cipher_list[choice]['w'], blinding_b) - # for item in choice_list: - # if choice == item: - # continue - # else: - # base_g = crypto_utils.ot_base_pown(-item) - # w_1 = crypto_utils.ot_mul_n(w_1, base_g) - # message_recover = w_1 ^ cipher_list[choice]['e'] - # print(message_recover) - # # s = crypto_utils.ot_int_to_str(message_recover) - # # print(s) - # - - -# if __name__ == '__main__': - - # print(True) - # df = pd.DataFrame( - # [[21, 'Amol', 72, 67], - # [23, 'Lini', 78, 69], - # [32, 'Kiku', 74, 56], - # [52, 'Ajit', 54, 76]], - # columns=['rollno', 'name', 'physics', 'botony']) - # - # print('DataFrame with default index\n', df) - # # set multiple columns as index - # df = df.set_index(['rollno', 'name']) - # - # print('\nDataFrame with MultiIndex\n', df) - # point1 = point.base() - # print(point.base(scalar1).hex()) - - # json_response = "{\"id\": {\"0\": \"67b176705b46206614219f47a05aee7ae6a3edbe850bbbe214c536b989aea4d2\", \"1\": \"b1b1bd1ed240b1496c81ccf19ceccf2af6fd24fac10ae42023628abbe2687310\"}, \"x0\": {\"0\": 10, \"1\": 20}, \"x1\": {\"0\": 11, \"1\": 22}}" - # json_dict = json.loads(json_response) - # # json_pd = pd.json_normalize(json_dict) - # print(json_dict) - - # hex_str = utils.make_hash(bytes(str(796443), 'utf8'), CryptoType.ECDSA, HashType.HEXSTR) - # print(hex_str) - # csv_path1 = '/Users/asher/Downloads/UseCase120/usecase120_party1.csv' - # csv_path2 = '/Users/asher/Downloads/UseCase120/usecase120_party2.csv' - # data = pd.read_csv(csv_path1) - # if 'id' in data.columns.values: - # duplicated_list = data.duplicated('id', False).tolist() - # if True in duplicated_list: - # log.error(f"[OnError]id duplicated, check csv file") - # raise PpcException(PpcErrorCode.DATASET_CSV_ERROR.get_code(), - # PpcErrorCode.DATASET_CSV_ERROR.get_msg()) - # start = time.time() - # get_pd_file_with_hash_requester(csv_path1, f'{csv_path1}-pre', 2) - # start2 = time.time() - # print(f"requester prepare time is {start2 - start}s") - # - # get_pd_file_with_hash_data_provider(csv_path2, f'{csv_path2}-pre') - # start3 = time.time() - # print(f"provider prepare time is {start3 - start2}s") - # provider_output_path = '/Users/asher/Downloads/UseCase120/output.csv' - # get_anonymous_data(f'{csv_path1}-pre-requester', f'{csv_path2}-pre', provider_output_path) - # start4 = time.time() - # print(f"provider compute time is {start4 - start3}s") - # result_path = '/Users/asher/Downloads/UseCase120/result.csv' - # recover_result_data(f'{csv_path1}-pre', '/Users/asher/Downloads/UseCase120/output.csv', result_path) - # end = time.time() - # print(f"requester get result time is {end - start4}s") diff --git a/python/ppc_common/ppc_utils/audit_utils.py b/python/ppc_common/ppc_utils/audit_utils.py deleted file mode 100644 index 913a3934..00000000 --- a/python/ppc_common/ppc_utils/audit_utils.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -import getopt -import sys - -from ppc_common.ppc_utils import utils -from ppc_common.ppc_utils.utils import CryptoType - -AUDIT_KEYS = ["agency_name", "input_dataset_hash", - "psi_input_hash", "psi_output_hash", - "mpc_result_hash"] - - -def parse_parameter(argv): - file_path = 0 - data_hash_value = 0 - crypto_type = None - try: - opts, args = getopt.getopt( - argv, "hf:v:c:", ["file_path=", "data_hash_value="]) - except getopt.GetoptError: - usage() - sys.exit(2) - if len(opts) == 0: - usage() - sys.exit(2) - for opt, arg in opts: - if opt == '-h': - usage() - sys.exit(0) - elif opt in ("-f", "--file_path"): - file_path = arg - elif opt in ("-v", "--data_hash_value"): - data_hash_value = arg - elif opt in ("-c", "--crypto_type"): - crypto_type = arg - else: - usage() - sys.exit(2) - return file_path, data_hash_value, crypto_type - - -def usage(): - print('audit.py -f -v -c crypto_type') - print('usage:') - print(' -f Notice:The file will be hashed to audit.') - print(' -v Notice:The dataset hash value will be compared to audit.') - print( - ' -c Notice:The crypto type[ECDSA or GM] will be used.') - - -if __name__ == '__main__': - file_path, data_hash_value, crypto_type = parse_parameter(sys.argv[1:]) - file_data_hash = utils.make_hash_from_file_path( - file_path, CryptoType[crypto_type]) - print(f'The file hash is:{file_data_hash}') - print(f'Audit result is:{file_data_hash == data_hash_value}') diff --git a/python/ppc_common/ppc_utils/cem_utils.py b/python/ppc_common/ppc_utils/cem_utils.py deleted file mode 100644 index 6e9d9d5f..00000000 --- a/python/ppc_common/ppc_utils/cem_utils.py +++ /dev/null @@ -1,160 +0,0 @@ -import json -import unittest - -LEGAL_OPERATOR_LIST = ['>', '<', '>=', '<=', '==', '!='] -LEGAL_QUOTA_LIST = ['and', 'or'] - - -def get_rule_detail(match_module_dict): - ruleset = match_module_dict['ruleset'] - operator_set = set() - count_set = set() - quota_set = set() - for ruleset_item in ruleset: - sub_rule_set = ruleset_item['set'] - for sub_rule_item in sub_rule_set: - rule = sub_rule_item['rule'] - operator_set.add(rule['operator']) - count_set.add(rule['count']) - quota_set.add(rule['quota']) - return operator_set, count_set, quota_set - - -def check_dataset_id_has_duplicated(match_module_dict): - ruleset = match_module_dict['ruleset'] - for ruleset_item in ruleset: - sub_rule_set = ruleset_item['set'] - for sub_rule_item in sub_rule_set: - dataset_id_set = set() - dataset_id_list = sub_rule_item['dataset'] - for dataset_id in dataset_id_list: - dataset_id_set.add(dataset_id) - if len(dataset_id_set) != len(dataset_id_list): - return True - return False - - -def get_dataset_id_set(match_module_dict): - dataset_id_set = set() - ruleset = match_module_dict['ruleset'] - for ruleset_item in ruleset: - sub_rule_set = ruleset_item['set'] - for sub_rule_item in sub_rule_set: - dataset_id_list = sub_rule_item['dataset'] - for dataset_id in dataset_id_list: - dataset_id_set.add(dataset_id) - return dataset_id_set - - -# get field_dataset_map: {'x1':['d1', 'd2', 'd3'], 'x2':['d1', 'd2', 'd3']} -def parse_field_dataset_map(match_module_dict): - field_dataset_map = {} - ruleset = match_module_dict['ruleset'] - for ruleset_item in ruleset: - field = ruleset_item['field'] - if field in field_dataset_map: - field_dataset_id_set = field_dataset_map[field] - else: - field_dataset_id_set = set() - field_dataset_map[field] = field_dataset_id_set - sub_rule_set = ruleset_item['set'] - for sub_rule_item in sub_rule_set: - dataset_id_list = sub_rule_item['dataset'] - for dataset_id in dataset_id_list: - field_dataset_id_set.add(dataset_id) - return field_dataset_map - - -# get dataset_field_map: {'d1':['x1', 'x2'], 'd2':['x1', 'x2'], 'd3':['x1', 'x2']} -def parse_dataset_field_map(dataset_id_set, field_dataset_map): - dataset_field_map = {} - for dataset_id in dataset_id_set: - for field, field_dataset_id_set in field_dataset_map.items(): - if dataset_id in field_dataset_id_set: - if dataset_id in dataset_field_map: - dataset_field_set = dataset_field_map[dataset_id] - else: - dataset_field_set = set() - dataset_field_map[dataset_id] = dataset_field_set - dataset_field_set.add(field) - return dataset_field_map - - -def parse_match_param(match_fields, match_module_dict): - # step1 get field_dataset_map: {'x1':['d1', 'd2', 'd3'], 'x2':['d1', 'd2', 'd3']} - field_dataset_map = parse_field_dataset_map(match_module_dict) - # step2 get all dataset_id {'d1', 'd2', 'd3'} - dataset_id_set = set() - for field, field_dataset_id_set in field_dataset_map.items(): - dataset_id_set.update(field_dataset_id_set) - # step3 get dataset_field_map: {'d1':['x1', 'x2'], 'd2':['x1', 'x2'], 'd3':['x1', 'x2']} - dataset_field_map = parse_dataset_field_map( - dataset_id_set, field_dataset_map) - # step4 get match_param_list from dataset_field_map and match_field: - # [ - # {'dataset_id':'d1', 'match_field':{'x1':'xxx, 'x2':'xxx}, - # {'dataset_id':'d2', 'match_field':{'x1':'xxx, 'x2':'xxx}, - # {'dataset_id':'d3', 'match_field':{'x1':'xxx, 'x2':'xxx}, - # ] - match_param_list = parse_match_param_list(dataset_field_map, match_fields) - return dataset_id_set, match_param_list - - -# get match_param_list from dataset_field_map and match_field: -# [ -# {'dataset_id':'d1', 'match_field':{'x1':'xxx, 'x2':'xxx}, -# {'dataset_id':'d2', 'match_field':{'x1':'xxx, 'x2':'xxx}, -# {'dataset_id':'d3', 'match_field':{'x1':'xxx, 'x2':'xxx}, -# ] -def parse_match_param_list(dataset_field_map, match_fields): - match_param_list = [] - match_fields = match_fields.replace("'", '"') - match_fields_object = json.loads(match_fields) - for dataset_id, field_set in dataset_field_map.items(): - match_param = {'dataset_id': dataset_id} - field_value_map = {} - for field in field_set: - # allow some part field match - if field in match_fields_object.keys(): - field_value_map[field] = match_fields_object[field] - match_param['match_field'] = field_value_map - match_param_list.append(match_param) - return match_param_list - - -class TestCemUtils(unittest.TestCase): - def test_cem_match_algorithm_load(self): - match_module = '{"ruleset":[' \ - '{"field":"x1",' \ - '"set":[' \ - '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ - '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ - '{"field":"x2","set":[' \ - '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d2-encrypted","d3-encrypted"]}]}]}' - match_module_dict = json.loads(match_module) - # match_module_dict = utils.json_loads(match_module) - print(match_module_dict) - - def test_check_dataset_id_has_duplicated(self): - match_module = '{"ruleset":[' \ - '{"field":"x1",' \ - '"set":[' \ - '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ - '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ - '{"field":"x2","set":[' \ - '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d2-encrypted","d3-encrypted"]}]}]}' - match_module_dict = json.loads(match_module) - has_duplicated = check_dataset_id_has_duplicated(match_module_dict) - assert has_duplicated == False - - def test_check_dataset_id_has_duplicated(self): - match_module = '{"ruleset":[' \ - '{"field":"x1",' \ - '"set":[' \ - '{"rule":{"operator":"<","count":50,"quota":"and"},"dataset":["d1-encrypted","d2-encrypted"]},' \ - '{"rule":{"operator":">","count":3,"quota":"or"},"dataset":["d3-encrypted"]}]},' \ - '{"field":"x2","set":[' \ - '{"rule":{"operator":"<","count":2,"quota":"or"},"dataset":["d1-encrypted","d1-encrypted","d3-encrypted"]}]}]}' - match_module_dict = json.loads(match_module) - has_duplicated = check_dataset_id_has_duplicated(match_module_dict) - assert has_duplicated == True diff --git a/python/ppc_common/ppc_utils/permission.py b/python/ppc_common/ppc_utils/permission.py deleted file mode 100644 index 7eb0c619..00000000 --- a/python/ppc_common/ppc_utils/permission.py +++ /dev/null @@ -1,79 +0,0 @@ -from enum import Enum - -from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode -from ppc_common.ppc_utils.utils import JobRole, JobStatus - - -def check_job_status(job_status): - if job_status in JobStatus.__members__: - return True - return False - - -def check_job_role(job_role): - if job_role in JobRole.__members__: - return True - return False - - -ADMIN_PERMISSIONS = 'ADMIN_PERMISSIONS' - - -class UserRole(Enum): - ADMIN = 1 - DATA_PROVIDER = 2 - ALGO_PROVIDER = 3 - DATA_CONSUMER = 4 - - -class PermissionGroup(Enum): - AGENCY_GROUP = 1 - DATA_GROUP = 2 - ALGO_GROUP = 3 - JOB_GROUP = 4 - AUDIT_GROUP = 5 - - -class AgencyGroup(Enum): - LIST_AGENCY = 1 - WRITE_AGENCY = 2 - - -class DataGroup(Enum): - LIST_DATA = 1 - READ_DATA_PUBLIC_INFO = 2 - READ_DATA_PRIVATE_INFO = 3 - WRITE_DATA = 4 - - -class AlgoGroup(Enum): - LIST_ALGO = 1 - READ_ALGO_PUBLIC_INFO = 2 - READ_ALGO_PRIVATE_INFO = 3 - WRITE_ALGO = 4 - - -class JobGroup(Enum): - LIST_JOB = 1 - READ_JOB_PUBLIC_INFO = 2 - READ_JOB_PRIVATE_INFO = 3 - WRITE_JOB = 4 - - -class AuditGroup(Enum): - READ_AUDIT = 1 - WRITE_AUDIT = 2 - - -# permissions formed as permission_a|permission_b|permission_a_group|... -def check_permission(permissions, needed_permission_group, *needed_permissions): - permission_list = permissions.split('|') - if ADMIN_PERMISSIONS in permission_list: - return 0 - if needed_permission_group in permission_list: - return 1 - for needed_permission in needed_permissions: - if needed_permission in permission_list: - return 1 - raise PpcException(PpcErrorCode.INSUFFICIENT_AUTHORITY.get_code( - ), PpcErrorCode.INSUFFICIENT_AUTHORITY.get_msg()) diff --git a/python/ppc_common/ppc_utils/utils.py b/python/ppc_common/ppc_utils/utils.py index d2564cce..51605969 100644 --- a/python/ppc_common/ppc_utils/utils.py +++ b/python/ppc_common/ppc_utils/utils.py @@ -1,29 +1,22 @@ import base64 -import datetime import hashlib import io import json import logging import os -import random import re import shutil -import string import subprocess import time -import uuid -from concurrent.futures.thread import ThreadPoolExecutor -from enum import Enum, IntEnum, unique +from enum import Enum, unique import jwt -import pandas as pd from ecdsa import SigningKey, SECP256k1, VerifyingKey from gmssl import func, sm2, sm3 from google.protobuf.descriptor import FieldDescriptor from jsoncomment import JsonComment from pysmx.SM3 import SM3 -from ppc_common.ppc_protos.generated.ppc_pb2 import DatasetDetail from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode log = logging.getLogger(__name__) @@ -118,80 +111,17 @@ class CryptoType(Enum): ECDSA = 1 GM = 2 - @unique class HashType(Enum): BYTES = 1 HEXSTR = 2 -class DeployMode(IntEnum): - PRIVATE_MODE = 1 - SAAS_MODE = 2 - PROXY_MODE = 3 - - -class JobRole(Enum): - ALGORITHM_PROVIDER = 1 - COMPUTATION_PROVIDER = 2 - DATASET_PROVIDER = 3 - DATA_CONSUMER = 4 - - -class JobStatus(Enum): - SUCCEED = 1 - FAILED = 2 - RUNNING = 3 - WAITING = 4 - NONE = 5 - RETRY = 6 - - class AlgorithmType(Enum): Train = "Train", Predict = "Predict" -class AlgorithmSubtype(Enum): - HeteroLR = 1 - HomoLR = 2 - HeteroNN = 3 - HomoNN = 4 - HeteroXGB = 5 - - -class DataAlgorithmType(Enum): - PIR = 'pir' - CEM = 'cem' - ALL = 'all' - - -class IdPrefixEnum(Enum): - DATASET = "d-" - ALGORITHM = "a-" - JOB = "j-" - - -class JobCemResultSuffixEnum(Enum): - JOB_ALGORITHM_AGENCY = "-1" - INITIAL_JOB_AGENCY = "-2" - - -class OriginAlgorithm(Enum): - PPC_AYS = ["a-1001", "匿踪查询", "1", "1.0"] - PPC_PSI = ["a-1002", "隐私求交", "2+", "1.0"] - - -executor = ThreadPoolExecutor(max_workers=10) - - -def async_fn(func): - def wrapper(*args, **kwargs): - executor.submit(func, *args, **kwargs) - - return wrapper - - def json_loads(json_config): try: json_comment = JsonComment(json) @@ -498,392 +428,6 @@ def load_credential_from_file(filepath): return f.read() -def getstatusoutput(cmd): - """replace commands.getstatusoutput - - Arguments: - cmd {[string]} - """ - - get_cmd = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ret = get_cmd.communicate() - out = ret[0] - err = ret[1] - output = '' - if not out is None: - output = output + out.decode('utf-8') - if not err is None: - output = output + err.decode('utf-8') - - log.debug(' cmd is %s, status is %s, output is %s', - cmd, str(get_cmd.returncode), output) - - return (get_cmd.returncode, output) - - -def make_dataset_metadata(data_field_list, dataset_description, dataset_size, row_count, column_count, - version_hash, user_name): - dataset_metadata = DatasetDetail() - dataset_metadata.data_field_list = data_field_list - dataset_metadata.dataset_description = dataset_description - dataset_metadata.dataset_size = dataset_size - dataset_metadata.row_count = row_count - dataset_metadata.column_count = column_count - dataset_metadata.dataset_hash = version_hash - dataset_metadata.user_name = user_name - - date_time = make_timestamp() - dataset_metadata.create_time = date_time - dataset_metadata.update_time = date_time - return pb_to_str(dataset_metadata) - - -def default_model_module(subtype): - defaultModelModule = [ - { - 'label': 'use_psi', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否进行隐私求交,只能为0或1' - }, - { - 'label': 'fillna', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否进行缺失值填充,只能为0或1' - }, - { - 'label': 'na_select', - 'type': 'float', - 'value': 1, - 'min_value': 0, - 'max_value': 1, - 'description': '缺失值筛选阈值,取值范围为0~1之间(0表示只要有缺失值就移除,1表示移除全为缺失值的列)' - }, - { - 'label': 'filloutlier', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否进行异常值处理,只能为0或1' - }, - { - 'label': 'normalized', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否归一化,只能为0或1' - }, - { - 'label': 'standardized', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否标准化,只能为0或1' - }, - { - 'label': 'categorical', - 'type': 'string', - 'value': '', - 'description': '标记所有分类特征字段，格式："x1,x12" (空代表无分类特征)' - }, - { - 'label': 'psi_select_col', - 'type': 'string', - 'value': '', - 'description': 'PSI稳定性筛选时间列名(空代表不进行PSI筛选)' - }, - { - 'label': 'psi_select_base', - 'type': 'string', - 'value': '', - 'description': 'PSI稳定性筛选的基期(空代表不进行PSI筛选)' - }, - { - 'label': 'psi_select_thresh', - 'type': 'float', - 'value': 0.3, - 'min_value': 0, - 'max_value': 1, - 'description': 'PSI筛选阈值,取值范围为0~1之间' - }, - { - 'label': 'psi_select_bins', - 'type': 'int', - 'value': 4, - 'min_value': 3, - 'max_value': 100, - 'description': '计算PSI时分箱数, 取值范围为3~100之间' - }, - { - 'label': 'corr_select', - 'type': 'float', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '特征相关性筛选阈值,取值范围为0~1之间(值为0时不进行相关性筛选)' - }, - { - 'label': 'use_iv', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否使用iv进行特征筛选,只能为0或1' - }, - { - 'label': 'group_num', - 'type': 'int', - 'value': 4, - 'min_value': 3, - 'max_value': 100, - 'description': 'woe计算分箱数,取值范围为3~100之间的整数' - }, - { - 'label': 'iv_thresh', - 'type': 'float', - 'value': 0.1, - 'min_value': 0.01, - 'max_value': 1, - 'description': 'iv特征筛选的阈值,取值范围为0.01~1之间' - }] - - if subtype == 'HeteroXGB': - defaultModelModule.extend([ - { - 'label': 'use_goss', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '是否采用goss抽样加速训练, 只能为0或1' - }, - { - 'label': 'test_dataset_percentage', - 'type': 'float', - 'value': 0.3, - 'min_value': 0.1, - 'max_value': 0.5, - 'description': '测试集比例, 取值范围为0.1~0.5之间' - }, - { - 'label': 'learning_rate', - 'type': 'float', - 'value': 0.1, - 'min_value': 0.01, - 'max_value': 1, - 'description': '学习率, 取值范围为0.01~1之间' - }, - { - 'label': 'num_trees', - 'type': 'int', - 'value': 6, - 'min_value': 1, - 'max_value': 300, - 'description': 'XGBoost迭代树棵树, 取值范围为1~300之间的整数' - }, - { - 'label': 'max_depth', - 'type': 'int', - 'value': 3, - 'min_value': 1, - 'max_value': 6, - 'description': 'XGBoost树深度, 取值范围为1~6之间的整数' - }, - { - 'label': 'max_bin', - 'type': 'int', - 'value': 4, - 'min_value': 3, - 'max_value': 100, - 'description': '特征分箱数, 取值范围为3~100之间的整数' - }, - { - 'label': 'silent', - 'type': 'bool', - 'value': 0, - 'min_value': 0, - 'max_value': 1, - 'description': '值为1时不打印运行信息,只能为0或1' - }, - { - 'label': 'subsample', - 'type': 'float', - 'value': 1, - 'min_value': 0.1, - 'max_value': 1, - 'description': '训练每棵树使用的样本比例,取值范围为0.1~1之间' - }, - { - 'label': 'colsample_bytree', - 'type': 'float', - 'value': 1, - 'min_value': 0.1, - 'max_value': 1, - 'description': '训练每棵树使用的特征比例,取值范围为0.1~1之间' - }, - { - 'label': 'colsample_bylevel', - 'type': 'float', - 'value': 1, - 'min_value': 0.1, - 'max_value': 1, - 'description': '训练每一层使用的特征比例,取值范围为0.1~1之间' - }, - { - 'label': 'reg_alpha', - 'type': 'float', - 'value': 0, - 'min_value': 0, - 'description': 'L1正则化项，用于控制模型复杂度,取值范围为大于等于0的数值' - }, - { - 'label': 'reg_lambda', - 'type': 'float', - 'value': 1, - 'min_value': 0, - 'description': 'L2正则化项,用于控制模型复杂度,取值范围为大于等于0的数值' - }, - { - 'label': 'gamma', - 'type': 'float', - 'value': 0, - 'min_value': 0, - 'description': '最优分割点所需的最小损失函数下降值,取值范围为大于等于0的数值' - }, - { - 'label': 'min_child_weight', - 'type': 'float', - 'value': 0, - 'min_value': 0, - 'description': '最优分割点所需的最小叶子节点权重,取值范围为大于等于0的数值' - }, - { - 'label': 'min_child_samples', - 'type': 'int', - 'value': 10, - 'min_value': 1, - 'max_value': 1000, - 'description': '最优分割点所需的最小叶子节点样本数量,取值范围为1~1000之间的整数' - }, - { - 'label': 'seed', - 'type': 'int', - 'value': 2024, - 'min_value': 0, - 'max_value': 10000, - 'description': '分割训练集/测试集时随机数种子,取值范围为0~10000之间的整数' - }, - { - 'label': 'early_stopping_rounds', - 'type': 'int', - 'value': 5, - 'min_value': 0, - 'max_value': 100, - 'description': '指定迭代多少次没有提升则停止训练, 值为0时不执行, 取值范围为0~100之间的整数' - }, - { - 'label': 'eval_metric', - 'type': 'string', - 'value': 'auc', - 'description': '早停的评估指标,支持:auc, acc, recall, precision' - }, - { - 'label': 'verbose_eval', - 'type': 'int', - 'value': 1, - 'min_value': 0, - 'max_value': 100, - 'description': '按传入的间隔输出训练过程中的评估信息,0表示不打印' - }, - { - 'label': 'eval_set_column', - 'type': 'string', - 'value': '', - 'description': '指定训练集测试集标记字段名称' - }, - { - 'label': 'train_set_value', - 'type': 'string', - 'value': '', - 'description': '指定训练集标记值' - }, - { - 'label': 'eval_set_value', - 'type': 'string', - 'value': '', - 'description': '指定测试集标记值' - }, - { - 'label': 'train_features', - 'type': 'string', - 'value': '', - 'description': '指定入模特征' - } # , - # { - # 'label': 'threads', - # 'type': 'int', - # 'value': 8, - # 'min_value': 1, - # 'max_value': 8, - # 'description': '取值范围为1~8之间的整数' - # } - ]) - else: - defaultModelModule.extend([ - { - 'label': 'test_dataset_percentage', - 'type': 'float', - 'value': 0.3, - 'min_value': 0.1, - 'max_value': 0.5, - 'description': '取值范围为0.1~0.5之间' - }, - { - 'label': 'learning_rate', - 'type': 'float', - 'value': 0.3, - 'min_value': 0.001, - 'max_value': 0.999, - 'description': '取值范围为0.001~0.999之间' - }, - { - 'label': 'epochs', - 'type': 'int', - 'value': 3, - 'min_value': 1, - 'max_value': 5, - 'description': '取值范围为1~5之间的整数' - }, - { - 'label': 'batch_size', - 'type': 'int', - 'value': 16, - 'min_value': 1, - 'max_value': 100, - 'description': '取值范围为1~100之间的整数' - }, - { - 'label': 'threads', - 'type': 'int', - 'value': 8, - 'min_value': 1, - 'max_value': 8, - 'description': '取值范围为1~8之间的整数' - } - ]) - - return defaultModelModule - - def merge_files(file_list, output_file): try: with open(output_file, 'wb') as outfile: diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py index 6dd6258f..1d22de03 100644 --- a/python/ppc_model/common/initializer.py +++ b/python/ppc_model/common/initializer.py @@ -1,11 +1,9 @@ import logging import logging.config -import os import yaml from ppc_common.deps_services import storage_loader -from ppc_common.deps_services.storage_api import StorageType from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager from ppc_common.ppc_utils import common_func from ppc_model.network.grpc.grpc_client import GrpcClient diff --git a/python/ppc_model/preprocessing/local_processing/psi_select.py b/python/ppc_model/preprocessing/local_processing/psi_select.py index 5bd1223e..3a3db8c7 100644 --- a/python/ppc_model/preprocessing/local_processing/psi_select.py +++ b/python/ppc_model/preprocessing/local_processing/psi_select.py @@ -34,7 +34,7 @@ def calculate_psi(expected, actual, buckettype='bins', buckets=10, axis=0): axis: axis by which variables are defined, 0 for vertical, 1 for horizontal Returns: - psi_values: ndarray of psi values for each variable + psi_values: ndarray of engine values for each variable Author: Matthew Burke diff --git a/python/ppc_scheduler/__init__.py b/python/ppc_scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/common/__init__.py b/python/ppc_scheduler/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/common/global_context.py b/python/ppc_scheduler/common/global_context.py new file mode 100644 index 00000000..7a090c83 --- /dev/null +++ b/python/ppc_scheduler/common/global_context.py @@ -0,0 +1,8 @@ +import os + +from ppc_scheduler.common.initializer import Initializer + +dirName, _ = os.path.split(os.path.abspath(__file__)) +config_path = "application.yml" + +components = Initializer(log_config_path='logging.conf', config_path=config_path) diff --git a/python/ppc_scheduler/common/initializer.py b/python/ppc_scheduler/common/initializer.py new file mode 100644 index 00000000..6ea92305 --- /dev/null +++ b/python/ppc_scheduler/common/initializer.py @@ -0,0 +1,91 @@ +import logging +import logging.config +import os +from contextlib import contextmanager + +import yaml +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ppc_common.deps_services import storage_loader +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize +from ppc_common.ppc_utils import common_func +from ppc_scheduler.job.job_manager import JobManager + + +class Initializer: + def __init__(self, log_config_path, config_path): + self.job_cache_dir = None + self.log_config_path = log_config_path + self.config_path = config_path + self.config_data = None + self.job_manager = None + self.thread_event_manager = None + self.sql_session = None + self.sql_engine = None + self.storage_client = None + # 只用于测试 + self.mock_logger = None + self.dataset_handler_initializer = None + + def init_all(self): + self.init_log() + self.init_cache() + self.init_config() + self.init_job_manager() + self.init_sql_client() + self.init_storage_client() + self.init_others() + + def init_log(self): + logging.config.fileConfig(self.log_config_path) + + def init_cache(self): + self.job_cache_dir = common_func.get_config_value( + "JOB_TEMP_DIR", "/tmp", self.config_data, False) + if not os.path.exists(self.job_cache_dir): + os.makedirs(self.job_cache_dir) + + def init_config(self): + with open(self.config_path, 'rb') as f: + self.config_data = yaml.safe_load(f.read()) + + def init_job_manager(self): + self.thread_event_manager = ThreadEventManager() + self.job_manager = JobManager( + logger=self.logger(), + thread_event_manager=self.thread_event_manager, + workspace=self.config_data['WORKSPACE'], + job_timeout_h=self.config_data['JOB_TIMEOUT_H'] + ) + + def init_sql_client(self): + self.sql_engine = create_engine(self.config_data['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True) + self.sql_session = sessionmaker(bind=self.sql_engine) + + @contextmanager + def create_sql_session(self): + session = self.sql_session() + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() + + def init_storage_client(self): + self.storage_client = storage_loader.load( + self.config_data, self.logger()) + + def init_others(self): + self.dataset_handler_initializer = DataSetHandlerInitialize( + self.config_data, self.logger()) + + def logger(self, name=None): + if self.mock_logger is None: + return logging.getLogger(name) + else: + return self.mock_logger diff --git a/python/ppc_scheduler/common/log_utils.py b/python/ppc_scheduler/common/log_utils.py new file mode 100644 index 00000000..5e675d2f --- /dev/null +++ b/python/ppc_scheduler/common/log_utils.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import os + +from ppc_common.ppc_utils import utils, path + + +def job_start_log_info(job_id): + return f"=====================start_{job_id}=====================" + + +def job_end_log_info(job_id): + return f"======================end_{job_id}======================" + + +def upload_job_log(storage_client, job_id, extra=None): + job_log_path = None + try: + file_path = path.get_path() + job_log_path = utils.get_log_temp_file_path(file_path, job_id) + origin_log_path = utils.get_log_file_path(file_path) + filter_job_log(job_id, origin_log_path, job_log_path) + if extra is not None: + job_log = open(job_log_path, 'a+') + job_log.write('\n' * 3) + job_log.write(extra) + job_log.close() + storage_client.upload_file(job_log_path, job_id + os.sep + utils.LOG_NAME) + finally: + os.remove(job_log_path) + + +def read_line_inverse(file): + file.seek(0, 2) + current_position = file.tell() + position = 0 + while position + current_position >= 0: + while True: + position -= 1 + try: + file.seek(position, 2) + if file.read(1) == b'\n': + break + except: + # point at file header + file.seek(0, 0) + break + line = file.readline() + yield line + + +def filter_job_log(job_id, origin_log_path, job_log_path): + origin_log_file = open(origin_log_path, 'rb') + line_list = [] + need_record = False + + # search job log + for line in read_line_inverse(origin_log_file): + if need_record: + line_list.append(line) + if not need_record and str(line).__contains__(job_end_log_info(job_id)): + need_record = True + line_list.append(line) + elif str(line).__contains__(job_start_log_info(job_id)): + break + origin_log_file.close() + + # save log lines into temp file + job_log_file = open(job_log_path, 'wb+') + job_log_file.writelines(list(reversed(line_list))) + job_log_file.close() diff --git a/python/ppc_scheduler/conf/application-sample.yml b/python/ppc_scheduler/conf/application-sample.yml new file mode 100644 index 00000000..0be7408c --- /dev/null +++ b/python/ppc_scheduler/conf/application-sample.yml @@ -0,0 +1,20 @@ +HOST: "0.0.0.0" +HTTP_PORT: 43471 + +JOB_TIMEOUT_H: 1800 + +AGENCY_ID: "1001" +PPCS_RPC_TOKEN: "ppcs_psi_apikey" + +WORKSPACE: "/data/app/files/job" +HDFS_ENDPOINT: "http://127.0.0.1:50070" + +# mysql or dm +DB_TYPE: "mysql" +SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4" +# SQLALCHEMY_DATABASE_URI: "dm+dmPython://ppcv16:ppc12345678@127.0.0.1:5236" + +SQLALCHEMY_TRACK_MODIFICATIONS: False + +MPC_NODE_DIRECT_PORT: 5899 +IS_MALICIOUS: False \ No newline at end of file diff --git a/python/ppc_scheduler/conf/logging.conf b/python/ppc_scheduler/conf/logging.conf new file mode 100644 index 00000000..bff66b16 --- /dev/null +++ b/python/ppc_scheduler/conf/logging.conf @@ -0,0 +1,40 @@ +[loggers] +keys=root,wsgi + +[logger_root] +level=INFO +handlers=consoleHandler,fileHandler + +[logger_wsgi] +level = INFO +handlers = accessHandler +qualname = wsgi +propagate = 0 + +[handlers] +keys=fileHandler,consoleHandler,accessHandler + +[handler_accessHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/wedpr/appmonitor.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_fileHandler] +class=handlers.TimedRotatingFileHandler +args=('/data/app/wedpr/scheduler.log', 'D', 1, 30, 'utf-8') +level=INFO +formatter=simpleFormatter + +[handler_consoleHandler] +class=StreamHandler +args=(sys.stdout,) +level=ERROR +formatter=simpleFormatter + +[formatters] +keys=simpleFormatter + +[formatter_simpleFormatter] +format=[%(levelname)s][%(asctime)s %(msecs)03d][%(process)d][%(filename)s:%(lineno)d] %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/python/ppc_scheduler/database/__init__.py b/python/ppc_scheduler/database/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/database/computing_node_mapper.py b/python/ppc_scheduler/database/computing_node_mapper.py new file mode 100644 index 00000000..0a95ad79 --- /dev/null +++ b/python/ppc_scheduler/database/computing_node_mapper.py @@ -0,0 +1,69 @@ +from sqlalchemy import update, and_, select, delete + +from ppc_common.db_models.computing_node import ComputingNodeRecord + + +def insert_computing_node(session, node_id: str, url: str, node_type: str, loading: int): + new_node = ComputingNodeRecord( + id=node_id, + url=url, + type=node_type, + loading=loading + ) + + session.add(new_node) + + +def delete_computing_node(session, url: str, node_type: str): + stmt = ( + delete(ComputingNodeRecord).where( + and_( + ComputingNodeRecord.url == url, + ComputingNodeRecord.type == node_type + ) + ) + ) + + result = session.execute(stmt) + + return result.rowcount > 0 + + +def get_and_update_min_loading_url(session, node_type): + subquery = ( + select([ComputingNodeRecord.id]).where( + and_( + ComputingNodeRecord.type == node_type + ) + ).order_by(ComputingNodeRecord.loading.asc()).limit(1) + ).scalar_subquery() + + stmt = ( + update(ComputingNodeRecord).where( + and_( + ComputingNodeRecord.id == subquery + ) + ).values( + loading=ComputingNodeRecord.loading + 1 + ).returning(ComputingNodeRecord.url) + ) + + result = session.execute(stmt) + return result.scalar() + + +def release_loading(session, url: str, node_type: str): + stmt = ( + update(ComputingNodeRecord).where( + and_( + ComputingNodeRecord.url == url, + ComputingNodeRecord.type == node_type, + ComputingNodeRecord.loading > 0 + ) + ).values( + loading=ComputingNodeRecord.loading - 1 + ) + ) + result = session.execute(stmt) + + return result.rowcount > 0 diff --git a/python/ppc_scheduler/database/job_worker_mapper.py b/python/ppc_scheduler/database/job_worker_mapper.py new file mode 100644 index 00000000..a4ee3699 --- /dev/null +++ b/python/ppc_scheduler/database/job_worker_mapper.py @@ -0,0 +1,32 @@ +from sqlalchemy import and_, update +from sqlalchemy.exc import NoResultFound + +from ppc_common.db_models.job_worker_record import JobWorkerRecord +from ppc_common.ppc_utils import utils +from ppc_scheduler.workflow.common import codec + + +def query_job_worker(session, job_id, worker_id): + try: + return session.query(JobWorkerRecord).filter( + and_(JobWorkerRecord.worker_id == worker_id, + JobWorkerRecord.job_id == job_id)).one() + except NoResultFound: + return None + + +def update_job_worker(session, job_id, worker_id, status, outputs): + stmt = ( + update(JobWorkerRecord).where( + and_( + JobWorkerRecord.job_id == job_id, + JobWorkerRecord.worker_id == worker_id + ) + ).values( + status=status, + outputs=codec.serialize_worker_outputs_for_db(outputs), + update_time=utils.make_timestamp() + ) + ) + result = session.execute(stmt) + return result.rowcount > 0 diff --git a/python/ppc_scheduler/endpoints/__init__.py b/python/ppc_scheduler/endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/endpoints/body_schema.py b/python/ppc_scheduler/endpoints/body_schema.py new file mode 100644 index 00000000..01ad24b8 --- /dev/null +++ b/python/ppc_scheduler/endpoints/body_schema.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import json + +from flask_restx import fields + +from ppc_model.network.http.restx import api + +response_base = api.model('Response base info', { + 'errorCode': fields.Integer(description='return code'), + 'message': fields.String(description='return message') +}) + +response_job_status = api.inherit('Task status', response_base, { + 'data': fields.Raw(description='Task status data as key-value dictionary', example={ + 'status': 'RUNNING', + 'time_costs': '30s' + }) +}) diff --git a/python/ppc_scheduler/endpoints/job_controller.py b/python/ppc_scheduler/endpoints/job_controller.py new file mode 100644 index 00000000..883fbca8 --- /dev/null +++ b/python/ppc_scheduler/endpoints/job_controller.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +from flask import request +from flask_restx import Resource + +from ppc_common.ppc_utils import utils +from ppc_scheduler.common.global_context import components +from ppc_scheduler.endpoints.body_schema import response_job_status, response_base +from ppc_scheduler.endpoints.restx import api + +ns = api.namespace('ppc-scheduler/job', + description='Operations related to run job') + + +@ns.route('/') +class JobCollection(Resource): + + @api.response(201, 'Job started successfully.', response_base) + def post(self, job_id): + """ + Run a specific job by job_id. + """ + args = request.get_json() + components.logger().info(f"run job request, job_id: {job_id}, args: {args}") + components.job_manager.run_task(job_id, (args,)) + return utils.BASE_RESPONSE + + @api.response(200, 'Job status retrieved successfully.', response_job_status) + def get(self, job_id): + """ + Get the status of a specific job by job_id. + """ + response = utils.BASE_RESPONSE + status, time_costs = components.job_manager.status(job_id) + response['data'] = { + 'status': status, + 'time_costs': time_costs + } + return response + + @api.response(200, 'Job killed successfully.', response_base) + def delete(self, job_id): + """ + Kill a specific job by job_id. + """ + components.logger().info(f"kill request, job_id: {job_id}") + components.job_manager.kill_job(job_id) + return utils.BASE_RESPONSE diff --git a/python/ppc_scheduler/endpoints/restx.py b/python/ppc_scheduler/endpoints/restx.py new file mode 100644 index 00000000..28f9829f --- /dev/null +++ b/python/ppc_scheduler/endpoints/restx.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from flask_restx import Api + +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.global_context import components + +authorizations = { + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + } +} + +api = Api(version='1.0', title='Ppc Scheduler Service', + authorizations=authorizations, security='apikey') + + +@api.errorhandler(PpcException) +def default_error_handler(e): + components.logger().exception(e) + info = e.to_dict() + response = {'errorCode': info['code'], 'message': info['message']} + components.logger().error(f"OnError: code: {info['code']}, message: {info['message']}") + return response, 500 + + +@api.errorhandler(BaseException) +def default_error_handler(e): + components.logger().exception(e) + message = 'unknown error.' + response = {'errorCode': PpcErrorCode.INTERNAL_ERROR, 'message': message} + components.logger().error(f"OnError: unknown error") + return response, 500 diff --git a/python/ppc_scheduler/job/__init__.py b/python/ppc_scheduler/job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/job/job_manager.py b/python/ppc_scheduler/job/job_manager.py new file mode 100644 index 00000000..95afa15a --- /dev/null +++ b/python/ppc_scheduler/job/job_manager.py @@ -0,0 +1,123 @@ +import datetime +import threading +import time +from typing import Union + +from readerwriterlock import rwlock + +from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_scheduler.common import log_utils +from ppc_scheduler.job.job_status import JobStatus +from ppc_scheduler.workflow.scheduler import Scheduler + + +class JobManager: + def __init__(self, logger, + thread_event_manager: ThreadEventManager, + workspace, + job_timeout_h: Union[int, float]): + self.logger = logger + self._thread_event_manager = thread_event_manager + self._workspace = workspace + self._job_timeout_s = job_timeout_h * 3600 + self._rw_lock = rwlock.RWLockWrite() + self._jobs: dict[str, list] = {} + self._async_executor = AsyncThreadExecutor( + event_manager=self._thread_event_manager, logger=logger) + self._cleanup_thread = threading.Thread(target=self._loop_cleanup) + self._cleanup_thread.daemon = True + self._cleanup_thread.start() + self.scheduler = Scheduler(self._workspace) + + def run_task(self, job_id, args=()): + """ + 发起任务 + param args: 任务参数 + """ + with self._rw_lock.gen_wlock(): + if job_id in self._jobs: + self.logger.info(f"Task already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}") + return + self._jobs[job_id] = [JobStatus.RUNNING, datetime.datetime.now(), 0] + self.logger.info(log_utils.job_start_log_info(job_id)) + self._async_executor.execute(job_id, self._run_job_flow, self._on_task_finish, args) + + def _run_job_flow(self, args): + """ + 运行任务流 + + """ + self.scheduler.schedule_job_flow(args) + + def kill_job(self, job_id: str): + with self._rw_lock.gen_rlock(): + if job_id not in self._jobs or self._jobs[job_id][0] != JobStatus.RUNNING: + return + + self.logger.info(f"Kill job, job_id: {job_id}") + self._async_executor.kill(job_id) + + with self._rw_lock.gen_wlock(): + self._jobs[job_id][0] = JobStatus.FAILURE + + def status(self, job_id: str) -> [str, float]: + """ + 返回: 任务状态, 执行耗时(s) + """ + with self._rw_lock.gen_rlock(): + if job_id not in self._jobs: + raise PpcException( + PpcErrorCode.JOB_NOT_FOUND.get_code(), + PpcErrorCode.JOB_NOT_FOUND.get_msg()) + status = self._jobs[job_id][0] + time_costs = self._jobs[job_id][2] + return status, time_costs + + def _on_task_finish(self, job_id: str, is_succeeded: bool, e: Exception = None): + with self._rw_lock.gen_wlock(): + time_costs = (datetime.datetime.now() - + self._jobs[job_id][1]).total_seconds() + self._jobs[job_id][2] = time_costs + if is_succeeded: + self._jobs[job_id][0] = JobStatus.SUCCESS + self.logger.info(f"Job {job_id} completed, time_costs: {time_costs}s") + else: + self._jobs[job_id][0] = JobStatus.FAILURE + self.logger.warn(f"Job {job_id} failed, time_costs: {time_costs}s, error: {e}") + self.logger.info(log_utils.job_end_log_info(job_id)) + + def _loop_cleanup(self): + while True: + self._terminate_timeout_jobs() + self._cleanup_finished_jobs() + time.sleep(5) + + def _terminate_timeout_jobs(self): + jobs_to_kill = [] + with self._rw_lock.gen_rlock(): + for job_id, value in self._jobs.items(): + alive_time = (datetime.datetime.now() - + value[1]).total_seconds() + if alive_time >= self._job_timeout_s and value[0] == JobStatus.RUNNING: + jobs_to_kill.append(job_id) + + for job_id in jobs_to_kill: + self.logger.warn(f"Job is timeout, job_id: {job_id}") + self.kill_job(job_id) + + def _cleanup_finished_jobs(self): + jobs_to_cleanup = [] + with self._rw_lock.gen_rlock(): + for job_id, value in self._jobs.items(): + alive_time = (datetime.datetime.now() - + value[1]).total_seconds() + if alive_time >= self._job_timeout_s + 3600: + jobs_to_cleanup.append((job_id, value[3])) + with self._rw_lock.gen_wlock(): + for job_id, job_id in jobs_to_cleanup: + if job_id in self._jobs: + del self._jobs[job_id] + self._thread_event_manager.remove_event(job_id) + self.logger.info(f"Cleanup job cache, job_id: {job_id}") diff --git a/python/ppc_scheduler/job/job_status.py b/python/ppc_scheduler/job/job_status.py new file mode 100644 index 00000000..a0a530ad --- /dev/null +++ b/python/ppc_scheduler/job/job_status.py @@ -0,0 +1,6 @@ +class JobStatus: + RUNNING = 'RUNNING' + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + TIMEOUT = 'TIMEOUT' + KILLED = 'KILLED' diff --git a/python/ppc_scheduler/job/job_type.py b/python/ppc_scheduler/job/job_type.py new file mode 100644 index 00000000..d734639f --- /dev/null +++ b/python/ppc_scheduler/job/job_type.py @@ -0,0 +1,8 @@ + +class JobType: + PSI = "PSI" + MPC = "MPC" + PREPROCESSING = 'PREPROCESSING' + FEATURE_ENGINEERING = 'FEATURE_ENGINEERING' + TRAINING = 'TRAINING' + PREDICTION = 'PREDICTION' diff --git a/python/ppc_scheduler/mpc_generator/__init__.py b/python/ppc_scheduler/mpc_generator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/mpc_generator/generator.py b/python/ppc_scheduler/mpc_generator/generator.py new file mode 100644 index 00000000..86466192 --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/generator.py @@ -0,0 +1,468 @@ +from enum import Enum + +import sqlparse +import sqlvalidator +from sqlparse.exceptions import SQLParseError +from sqlparse.sql import Comparison, Identifier, Function +from sqlparse.tokens import Punctuation, Operator, Name, Token + +from ppc_scheduler.mpc_generator import mpc_func_str +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_common.ppc_utils.utils import PPC_RESULT_FIELDS_FLAG, PPC_RESULT_VALUES_FLAG + + +class SqlPattern(Enum): + BASIC_ARITH_OPE = 1 + AGGR_FUNC_ONLY = 2 + AGGR_FUNC_WITH_GROUP_BY = 3 + + +INDENT = " " + +SUPPORTED_KEYWORDS = [ + 'SELECT', + 'FROM', + 'WHERE', + 'JOIN', + 'INNER JOIN', + 'ON', + 'AS', + 'GROUP BY', + 'COUNT', + 'SUM', + 'AVG', + 'MAX', + 'MIN', +] + +VALUE_TYPE = 'pfix' + +GROUP_BY_COLUMN_CODE = 'group_indexes_key[i]' + +DISPLAY_FIELDS_FUNC = 'set_display_field_names' + +DISPLAY_RESULT_FUNC = 'display_data' + + +class CodeGenerator: + + def __init__(self, sql_str): + self.sql_str = sql_str + + # three patterns supported + self.sql_pattern = SqlPattern.BASIC_ARITH_OPE + + # based on ID only + self.need_run_psi = False + + # record dataset sources + self.table_set = set() + + # 0: table number, 1: column index + self.group_by_column = [] + + # filter conditions + self.condition_list = [] + + self.result_fields = [] + + self.max_column_index = [] + + def sql_to_mpc_code(self): + operators = self.pre_parsing() + format_sql_str = sqlparse.format(self.sql_str, reindent=True, keyword_case='upper') + mpc_str = self.generate_common_code(format_sql_str) + mpc_str = self.generate_function_code(mpc_str) + mpc_str = self.generate_result_calculation_code(operators, mpc_str) + mpc_str = self.generate_result_print_code(mpc_str) + mpc_str = self.generate_mpc_execution_code(mpc_str) + mpc_str = self.replace_max_filed(mpc_str) + return mpc_str + + def pre_parsing(self): + try: + if not sqlvalidator.parse(self.sql_str).is_valid(): + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement") + + # format sql str + format_sql_str = sqlparse.format(self.sql_str, reindent=True, keyword_case='upper') + + tokens = sqlparse.parse(format_sql_str)[0].tokens + + # warning unsupported keywords + self.check_sql_tokens(tokens) + + # parse table aliases + aliases = self.find_table_alias(tokens, {}, False) + + # recover table aliases + new_sql_str = self.recover_table_name(tokens, aliases, '') + format_new_sql_str = sqlparse.format(new_sql_str, reindent=True, keyword_case='upper') + tokens = sqlparse.parse(format_new_sql_str)[0].tokens + + # parse filters (only 'id' based column alignment is supported currently) + self.find_table_and_condition(tokens, False) + + # check table suffix and number of participants + self.check_table() + + # ensure that all tables participate in alignment + self.check_table_alignment(self.need_run_psi, len(self.table_set)) + + self.max_column_index = [0 for _ in range(len(self.table_set))] + + self.check_sql_pattern(tokens) + + operators = self.extract_operators(format_new_sql_str) + + return operators + except SQLParseError: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement") + + def check_sql_tokens(self, tokens): + for token in tokens: + if token.is_keyword and token.value not in SUPPORTED_KEYWORDS: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), f"keyword '{token.value}' not supported") + if hasattr(token, 'tokens'): + self.check_sql_tokens(token.tokens) + + def find_table_alias(self, tokens, aliases, after_from): + end_current_round = False + for i in range(len(tokens)): + if after_from and tokens[i].value == 'AS': + # find a alias + end_current_round = True + aliases[tokens[i + 2].value] = tokens[i - 2].value + for i in range(len(tokens)): + if tokens[i].value == 'FROM': + after_from = True + if after_from and not end_current_round and hasattr(tokens[i], 'tokens'): + self.find_table_alias(tokens[i].tokens, aliases, after_from) + return aliases + + def recover_table_name(self, tokens, aliases, format_sql_str): + for i in range(len(tokens)): + if tokens[i].value == 'AS' and tokens[i + 2].value in aliases: + break + elif not tokens[i].is_group: + if tokens[i].value in aliases: + format_sql_str += aliases[tokens[i].value] + else: + format_sql_str += tokens[i].value + elif hasattr(tokens[i], 'tokens'): + format_sql_str = self.recover_table_name(tokens[i].tokens, aliases, format_sql_str) + return format_sql_str + + def find_table_and_condition(self, tokens, after_from): + for token in tokens: + if token.value == 'FROM': + after_from = True + if after_from: + if type(token) == Comparison: + self.check_equal_comparison(token.tokens) + self.condition_list.append(token.value) + if type(token) == Identifier and '.' not in token.value: + self.table_set.add(token.value) + elif hasattr(token, 'tokens'): + self.find_table_and_condition(token.tokens, after_from) + + def check_equal_comparison(self, tokens): + for i in range(len(tokens)): + if tokens[i].value == '=': + self.need_run_psi = True + elif tokens[i].value == '.' and tokens[i + 1].value != 'id': + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), + f"only 'id' based column alignment is supported currently") + elif hasattr(tokens[i], 'tokens'): + self.check_equal_comparison(tokens[i].tokens) + + def check_table_alignment(self, has_equal_comparison, table_count): + if has_equal_comparison: + column = self.condition_list[0].split('=')[0].strip() + table = column[0:column.find('.')] + + # all tables must be aligned + self.equal_comparison_dfs(table, [0 for _ in range(len(self.condition_list))]) + if len(self.table_set) != table_count: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "all tables must be aligned") + + def equal_comparison_dfs(self, table, flag_array): + for i in range(len(self.condition_list)): + if flag_array[i] == 0 and table in self.condition_list[i]: + flag_array[i] = 1 + columns = self.condition_list[i].split('=') + for column in columns: + table = column[0:column.find('.')].strip() + self.table_set.add(table) + self.equal_comparison_dfs(table, flag_array) + + def check_table(self): + table_count = len(self.table_set) + if table_count > 5 or table_count < 2: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "source must be 2 to 5 parties") + suffixes = set() + for table in self.table_set: + suffixes.add(table[-1]) + for i in range(table_count): + if str(i) not in suffixes: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "invalid suffix of table name") + + def update_max_field(self, table_number, field_number): + if field_number > self.max_column_index[table_number]: + self.max_column_index[table_number] = field_number + + def replace_max_filed(self, mpc_str): + for i in range(len(self.max_column_index)): + mpc_str = mpc_str.replace(f'$(source{i}_column_count)', str(self.max_column_index[i] + 1)) + return mpc_str + + def check_sql_pattern(self, tokens): + for i in range(len(tokens)): + if tokens[i].value == 'GROUP BY': + self.sql_pattern = SqlPattern.AGGR_FUNC_WITH_GROUP_BY + items = tokens[i + 2].value.split('.') + self.group_by_column = [int(items[0][-1]), get_column_number(items[1])] + elif type(tokens[i]) == Function: + self.sql_pattern = SqlPattern.AGGR_FUNC_ONLY + elif hasattr(tokens[i], 'tokens'): + self.check_sql_pattern(tokens[i].tokens) + + def extract_operators(self, format_sql_str): + start = format_sql_str.find("SELECT") + end = format_sql_str.find("FROM") + operators = format_sql_str[start + 6:end].split(',') + for i in range(len(operators)): + if ' AS ' in operators[i]: + index = operators[i].find(' AS ') + self.result_fields.append(operators[i][index + 4:].strip().strip('\n').strip('\'').strip('\"').strip()) + operators[i] = operators[i][:index].strip() + else: + self.result_fields.append(f"result{i}") + operators[i] = operators[i].strip() + if ' ' in self.result_fields[-1]: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "result field cannot contain space") + return operators + + def generate_common_code(self, format_sql_str): + table_count = len(self.table_set) + result_column_count = len(self.result_fields) + + if self.need_run_psi: + mpc_str = '# PSI_OPTION=True\n\n' + else: + mpc_str = '' + mpc_str += '# BIT_LENGTH = 128\n\n' + mpc_str += '# This file is generated automatically by ams\n' + mpc_str += f"'''\n{format_sql_str}\n'''\n\n" + mpc_str += "from ppc import *\n\n" + # mpc_str += "program.use_trunc_pr = True\n" + # mpc_str += "program.use_split(3)\n" + mpc_str += "n_threads = 8\n" + mpc_str += f"value_type = {VALUE_TYPE}\n\n" + if VALUE_TYPE == 'pfix': + mpc_str += f"pfix.set_precision(16, 47)\n\n" + + for i in range(table_count): + mpc_str += f"SOURCE{i} = {i}\n" + mpc_str += f"source{i}_record_count = $(source{i}_record_count)\n" + mpc_str += f"source{i}_column_count = $(source{i}_column_count)\n" + mpc_str += f"source{i}_record = Matrix(source{i}_record_count, source{i}_column_count, value_type)\n\n" + + if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE: + mpc_str += "# basic arithmetic operation means that all parties have same number of record\n" + mpc_str += "result_record = $(source0_record_count)\n" + mpc_str += f"results = Matrix(result_record, {result_column_count}, value_type)\n\n\n" + elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + mpc_str += f"results = Matrix(1, {result_column_count}, value_type)\n\n\n" + elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + mpc_str += "# group by means all parties have same number of record\n" + mpc_str += "source_record_count = $(source0_record_count)\n" + mpc_str += "result_record = cint(source_record_count)\n" + mpc_str += f"results = Matrix(source_record_count, {result_column_count}, value_type)\n\n\n" + + mpc_str += "def read_data_collection(data_collection, party_id):\n" + mpc_str += f"{INDENT}if data_collection.sizes[0] > 0:\n" + mpc_str += f"{INDENT}{INDENT}data_collection.input_from(party_id)\n\n\n" + + return mpc_str + + def generate_function_code(self, mpc_str): + if self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + mpc_str += mpc_func_str.FUNC_COMPUTE_SUM + mpc_str += mpc_func_str.FUNC_COMPUTE_COUNT + mpc_str += mpc_func_str.FUNC_COMPUTE_AVG + mpc_str += mpc_func_str.FUNC_COMPUTE_MAX + mpc_str += mpc_func_str.FUNC_COMPUTE_MIN + elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + mpc_str += mpc_func_str.GROUP_BY_GLOBAL_VARIABLE + mpc_str += mpc_func_str.FUNC_COMPUTE_GROUP_BY_INDEXES + mpc_str += mpc_func_str.FUNC_COMPUTE_SUM_WITH_GROUP_BY + mpc_str += mpc_func_str.FUNC_COMPUTE_COUNT_WITH_GROUP_BY + mpc_str += mpc_func_str.FUNC_COMPUTE_AVG_WITH_GROUP_BY + mpc_str += mpc_func_str.FUNC_COMPUTE_MAX_WITH_GROUP_BY + mpc_str += mpc_func_str.FUNC_COMPUTE_MIN_WITH_GROUP_BY + return mpc_str + + def generate_result_calculation_code(self, operators, mpc_str): + for i in range(len(operators)): + tokens = sqlparse.parse(operators[i])[0].tokens + participants_set = set() + formula_str = self.generate_formula(tokens, '', participants_set) + if len(participants_set) == 1: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "disabled query pattern") + + if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE or self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + mpc_str += f"def calculate_result_{i}():\n" \ + f"{INDENT}@for_range_opt_multithread(n_threads, result_record)\n" \ + f"{INDENT}def _(i):\n" \ + f"{INDENT}{INDENT}results[i][{i}] = {formula_str}\n\n\n" + elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + mpc_str += f"def calculate_result_{i}():\n" \ + f"{INDENT}results[0][{i}] = {formula_str}\n\n\n" + return mpc_str + + def generate_result_print_code(self, mpc_str): + field_print_str = f"{PPC_RESULT_FIELDS_FLAG} = ['{self.result_fields[0]}'" + for i in range(1, len(self.result_fields)): + field_print_str += f", '{self.result_fields[i]}'" + field_print_str += ']' + + if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE or self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + result_print_str = f"{PPC_RESULT_VALUES_FLAG} = [results[i][0].reveal()" + for i in range(1, len(self.result_fields)): + result_print_str += f", results[i][{i}].reveal()" + result_print_str += ']' + mpc_str += f"def print_results():\n" \ + f"{INDENT}{field_print_str}\n" \ + f"{INDENT}{DISPLAY_FIELDS_FUNC}({PPC_RESULT_FIELDS_FLAG})\n\n" \ + f"{INDENT}@for_range_opt(result_record)\n" \ + f"{INDENT}def _(i):\n" \ + f"{INDENT}{INDENT}{result_print_str}\n" \ + f"{INDENT}{INDENT}{DISPLAY_RESULT_FUNC}({PPC_RESULT_VALUES_FLAG})\n\n\n" + elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + result_print_str = f"{PPC_RESULT_VALUES_FLAG} = [results[0][0].reveal()" + for i in range(1, len(self.result_fields)): + result_print_str += f", results[0][{i}].reveal()" + result_print_str += ']' + mpc_str += f"def print_results():\n" \ + f"{INDENT}{field_print_str}\n" \ + f"{INDENT}{DISPLAY_FIELDS_FUNC}({PPC_RESULT_FIELDS_FLAG})\n\n" \ + f"{INDENT}{result_print_str}\n" \ + f"{INDENT}{DISPLAY_RESULT_FUNC}({PPC_RESULT_VALUES_FLAG})\n\n\n" + return mpc_str + + def generate_mpc_execution_code(self, mpc_str): + mpc_str += 'def ppc_main():\n' + for i in range(len(self.table_set)): + mpc_str += f"{INDENT}read_data_collection(source{i}_record, SOURCE{i})\n" + + if self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + mpc_str += f"\n{INDENT}compute_group_by_indexes(source{self.group_by_column[0]}_record, " \ + f"{self.group_by_column[1]})\n\n" + + for i in range(len(self.result_fields)): + mpc_str += f"{INDENT}calculate_result_{i}()\n" + + mpc_str += f"\n{INDENT}print_results()\n\n\n" + + mpc_str += "ppc_main()\n" + return mpc_str + + def generate_formula(self, tokens, formula_str, participants_set): + for token in tokens: + if token.ttype == Punctuation \ + or token.ttype == Operator \ + or token.ttype == Token.Literal.Number.Integer \ + or token.ttype == Token.Operator.Comparison: + formula_str += token.value + elif type(token) == Function: + formula_str += self.handle_function(token) + elif type(token) == Identifier and token.tokens[0].ttype == Name and len(token.tokens) >= 3: + (table_number, field_number) = self.handle_basic_identifier(token) + if self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + if table_number != self.group_by_column[0] or field_number != self.group_by_column[1]: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement") + self.update_max_field(table_number, field_number) + formula_str += GROUP_BY_COLUMN_CODE + elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "disabled query pattern") + elif self.sql_pattern == SqlPattern.BASIC_ARITH_OPE: + if token.value == token.parent.value: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), + "disabled query pattern") + self.update_max_field(table_number, field_number) + formula_str += f"source{table_number}_record[i][{field_number}]" + participants_set.add(table_number) + elif hasattr(token, 'tokens'): + formula_str = self.generate_formula(token.tokens, formula_str, participants_set) + return formula_str + + def handle_function(self, token): + tokens = token.tokens + func_name = tokens[0].value + (table_number, field_number) = self.handle_parenthesis(tokens[1]) + self.update_max_field(table_number, field_number) + return self.func_to_formula(func_name, table_number, field_number) + + def func_to_formula(self, func, table_number, field_number): + if self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY: + formula = { + 'COUNT': f"{mpc_func_str.FUNC_COMPUTE_COUNT_NAME}(source{table_number}_record_count)", + 'SUM': f"{mpc_func_str.FUNC_COMPUTE_SUM_NAME}(source{table_number}_record, " + f"source{table_number}_record_count, {field_number})", + 'AVG': f"{mpc_func_str.FUNC_COMPUTE_AVG_NAME}(source{table_number}_record, " + f"source{table_number}_record_count, {field_number})", + 'MAX': f"{mpc_func_str.FUNC_COMPUTE_MAX_NAME}(source{table_number}_record, " + f"source{table_number}_record_count, {field_number})", + 'MIN': f"{mpc_func_str.FUNC_COMPUTE_MIN_NAME}(source{table_number}_record, " + f"source{table_number}_record_count, {field_number})" + } + elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY: + formula = { + 'COUNT': f"{mpc_func_str.FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME}(i)", + 'SUM': f"{mpc_func_str.FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME}(source{table_number}_record," + f" {field_number}, i)", + 'AVG': f"{mpc_func_str.FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME}(source{table_number}_record," + f" {field_number}, i)", + 'MAX': f"{mpc_func_str.FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME}(source{table_number}_record," + f" {field_number}, i)", + 'MIN': f"{mpc_func_str.FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME}(source{table_number}_record," + f" {field_number}, i)" + } + else: + formula = {} + + return formula.get(func, '') + + def handle_parenthesis(self, token): + for token in token.tokens: + if type(token) == Identifier: + (table_number, field_number) = self.handle_basic_identifier(token) + return table_number, field_number + + def handle_basic_identifier(self, token): + tokens = token.tokens + + if tokens[0].value not in self.table_set: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "table name not matched") + if tokens[1].value != '.': + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "invalid identifier") + + field_num = get_column_number(tokens[2].value) + + return int(tokens[0].value[-1]), field_num + + +def get_column_number(field_name): + field_len = len(field_name) + field_num = 0 + for i in range(field_len, 0, -1): + try: + int(field_name[i - 1:field_len]) + except ValueError: + if i == field_len: + raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), + f"invalid field suffix of table column '{field_name}'") + field_num = int(field_name[i:field_len]) + break + return field_num diff --git a/python/ppc_scheduler/mpc_generator/mpc_func_str.py b/python/ppc_scheduler/mpc_generator/mpc_func_str.py new file mode 100644 index 00000000..87501124 --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/mpc_func_str.py @@ -0,0 +1,64 @@ +import os + +FILE_PATH = os.path.abspath(__file__) + +CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".") + +AGGR_FUNC_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_only.mpc" +GROUP_BY_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_with_group_by.mpc" + +FUNC_COMPUTE_SUM_NAME = 'compute_sum' +FUNC_COMPUTE_COUNT_NAME = 'compute_count' +FUNC_COMPUTE_AVG_NAME = 'compute_avg' +FUNC_COMPUTE_MAX_NAME = 'compute_max' +FUNC_COMPUTE_MIN_NAME = 'compute_min' + +FUNC_COMPUTE_GROUP_BY_INDEXES_NAME = 'compute_group_by_indexes' +FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME = 'compute_sum_with_group_by' +FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME = 'compute_count_with_group_by' +FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME = 'compute_avg_with_group_by' +FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME = 'compute_max_with_group_by' +FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME = 'compute_min_with_group_by' + +with open(AGGR_FUNC_SAMPLE_PATH, "r") as file: + AGGR_FUNC_SAMPLE_STR = file.read() + +with open(GROUP_BY_SAMPLE_PATH, "r") as file: + GROUP_BY_SAMPLE_STR = file.read() + + +def get_body_str_by_name(start_str, end_str, sql_pattern): + if sql_pattern == 1: + source_str = AGGR_FUNC_SAMPLE_STR + elif sql_pattern == 2: + source_str = GROUP_BY_SAMPLE_STR + else: + return '' + + start_index = source_str.find(start_str) + source_str = source_str[start_index:] + + end_index = source_str.find(end_str) + len(end_str) + return source_str[:end_index] + + +def get_func_str_by_name(func_name, sql_pattern): + start_str = f"def {func_name}" + end_str = "\n\n\n" + return get_body_str_by_name(start_str, end_str, sql_pattern) + + +FUNC_COMPUTE_SUM = get_func_str_by_name(FUNC_COMPUTE_SUM_NAME, 1) +FUNC_COMPUTE_COUNT = get_func_str_by_name(FUNC_COMPUTE_COUNT_NAME, 1) +FUNC_COMPUTE_AVG = get_func_str_by_name(FUNC_COMPUTE_AVG_NAME, 1) +FUNC_COMPUTE_MAX = get_func_str_by_name(FUNC_COMPUTE_MAX_NAME, 1) +FUNC_COMPUTE_MIN = get_func_str_by_name(FUNC_COMPUTE_MIN_NAME, 1) + +GROUP_BY_GLOBAL_VARIABLE = get_body_str_by_name("# matrix of indexes", "\n\n\n", 2) + +FUNC_COMPUTE_GROUP_BY_INDEXES = get_func_str_by_name(FUNC_COMPUTE_GROUP_BY_INDEXES_NAME, 2) +FUNC_COMPUTE_SUM_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME, 2) +FUNC_COMPUTE_COUNT_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME, 2) +FUNC_COMPUTE_AVG_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME, 2) +FUNC_COMPUTE_MAX_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME, 2) +FUNC_COMPUTE_MIN_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME, 2) diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc new file mode 100644 index 00000000..9ddb3c34 --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc @@ -0,0 +1,136 @@ +# PSI_OPTION=True + +# BIT_LENGTH = 128 + +# This file is generated automatically by ams +''' +SELECT COUNT(s1.field3) + COUNT(s2.field3) AS r0, + SUM(s1.field3) + COUNT(s0.field0) AS 'count', + (MAX(s0.field1) + MAX(s2.field1)) / 2 AS r1, + (AVG(s1.field2) + AVG(s2.field2)) / 2 AS r2, + MIN(s1.field0) - MIN(s0.field0) AS r3 +FROM (source0 AS s0 + INNER JOIN source1 AS s1 ON s0.id = s1.id) +INNER JOIN source2 AS s2 ON s0.id = s2.id; +''' + +from ppc import * + +n_threads = 8 +value_type = pfix + +pfix.set_precision(16, 47) + +SOURCE0 = 0 +source0_record_count = $(source0_record_count) +source0_column_count = 2 +source0_record = Matrix(source0_record_count, source0_column_count, value_type) + +SOURCE1 = 1 +source1_record_count = $(source1_record_count) +source1_column_count = 4 +source1_record = Matrix(source1_record_count, source1_column_count, value_type) + +SOURCE2 = 2 +source2_record_count = $(source2_record_count) +source2_column_count = 4 +source2_record = Matrix(source2_record_count, source2_column_count, value_type) + +results = Matrix(1, 5, value_type) + + +def read_data_collection(data_collection, party_id): + if data_collection.sizes[0] > 0: + data_collection.input_from(party_id) + + +def compute_sum(source, record_count, col_index): + records_sum = Array(1, value_type) + records_sum[0] = source[0][col_index] + + @for_range(1, record_count) + def _(i): + records_sum[0] = records_sum[0] + source[i][col_index] + + return records_sum[0] + + +def compute_count(record_count): + return record_count + + +def compute_avg(source, record_count, col_index): + records_sum = Array(1, value_type) + records_sum[0] = source[0][col_index] + + @for_range(1, record_count) + def _(i): + records_sum[0] = records_sum[0] + source[i][col_index] + + return records_sum[0] / record_count + + +def compute_max(source, record_count, col_index): + max_record = Array(1, value_type) + max_record[0] = source[0][col_index] + + @for_range(1, record_count) + def _(i): + max_record[0] = condition(max_record[0] < source[i][col_index], source[i][col_index], max_record[0]) + + return max_record[0] + + +def compute_min(source, record_count, col_index): + min_record = Array(1, value_type) + min_record[0] = source[0][col_index] + + @for_range(1, record_count) + def _(i): + min_record[0] = condition(min_record[0] > source[i][col_index], source[i][col_index], min_record[0]) + + return min_record[0] + + +def calculate_result_0(): + results[0][0] = compute_count(source1_record_count)+compute_count(source2_record_count) + + +def calculate_result_1(): + results[0][1] = compute_sum(source1_record, source1_record_count, 3)+compute_count(source0_record_count) + + +def calculate_result_2(): + results[0][2] = (compute_max(source0_record, source0_record_count, 1)+compute_max(source2_record, source2_record_count, 1))/2 + + +def calculate_result_3(): + results[0][3] = (compute_avg(source1_record, source1_record_count, 2)+compute_avg(source2_record, source2_record_count, 2))/2 + + +def calculate_result_4(): + results[0][4] = compute_min(source1_record, source1_record_count, 0)-compute_min(source0_record, source0_record_count, 0) + + +def print_results(): + result_fields = ['r0', 'count', 'r1', 'r2', 'r3'] + set_display_field_names(result_fields) + + result_values = [results[0][0].reveal(), results[0][1].reveal(), results[0][2].reveal(), results[0][3].reveal(), results[0][4].reveal()] + display_data(result_values) + + +def ppc_main(): + read_data_collection(source0_record, SOURCE0) + read_data_collection(source1_record, SOURCE1) + read_data_collection(source2_record, SOURCE2) + calculate_result_0() + calculate_result_1() + calculate_result_2() + calculate_result_3() + calculate_result_4() + + print_results() + + +ppc_main() diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc new file mode 100644 index 00000000..f1a01339 --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc @@ -0,0 +1,234 @@ +# PSI_OPTION=True + +# BIT_LENGTH = 128 + +# This file is generated automatically by ams +''' +SELECT 3*s1.field4 AS r0, + COUNT(s1.field4) AS 'count', + AVG(s0.field1) * 2 + s1.field4 AS r1, + (SUM(s0.field2) + SUM(s1.field2))/(COUNT(s1.field3) + 100/(MIN(s0.field1)+MIN(s1.field1))) + 10, + MAX(s1.field1), + MIN(s2.field2) +FROM (source0 AS s0 + INNER JOIN source1 AS s1 ON s0.id = s1.id) +INNER JOIN source2 AS s2 ON s0.id = s2.id +GROUP BY s1.field4; +''' + +from ppc import * + +n_threads = 8 +value_type = pfix + +pfix.set_precision(16, 47) + +SOURCE0 = 0 +source0_record_count = $(source0_record_count) +source0_column_count = 3 +source0_record = Matrix(source0_record_count, source0_column_count, value_type) + +SOURCE1 = 1 +source1_record_count = $(source1_record_count) +source1_column_count = 5 +source1_record = Matrix(source1_record_count, source1_column_count, value_type) + +SOURCE2 = 2 +source2_record_count = $(source2_record_count) +source2_column_count = 3 +source2_record = Matrix(source2_record_count, source2_column_count, value_type) + +# group by means all parties have same number of record +source_record_count = $(source0_record_count) +result_record = cint(source_record_count) +results = Matrix(source_record_count, 6, value_type) + + +def read_data_collection(data_collection, party_id): + if data_collection.sizes[0] > 0: + data_collection.input_from(party_id) + + +# matrix of indexes after group by: +# 0 1 2 +# 0 count1 start_index end_index +# 1 count2 start_index end_index +# 2 count3 start_index end_index +# ... +# source_record_count - 1 ... +group_indexes_key = Array(source_record_count, value_type) +group_indexes_matrix = Matrix(source_record_count, 3, pint) +group_column = Array(source_record_count, value_type) + + +def compute_group_by_indexes(source, col_index): + # group_count group_index group_flag + group_states = Array(3, cint) + + @for_range_opt(source_record_count) + def _(i): + group_column[i] = source[i][col_index] + group_states[1] = 0 + group_states[2] = 0 + + @for_range(group_states[0]) + def _(j): + @if_(pint(group_indexes_key[j] == source[i][col_index]).reveal()) + def _(): + group_states[1] = j + group_states[2] = 1 + + @if_e(group_states[2] == 0) + def _(): + # new item + group_indexes_key[group_states[0]] = source[i][col_index] + group_indexes_matrix[group_states[0]][0] = 1 + group_indexes_matrix[group_states[0]][1] = i + group_indexes_matrix[group_states[0]][2] = i + group_states[0] = group_states[0] + 1 + + @else_ + def _(): + group_indexes_matrix[group_states[1]][0] = group_indexes_matrix[group_states[1]][0] + 1 + group_indexes_matrix[group_states[1]][2] = i + + global result_record + result_record = group_states[0] + + +def compute_sum_with_group_by(source, col_index, group_row_index): + records_sum = Array(1, value_type) + + start_index = group_indexes_matrix[group_row_index][1].reveal() + end_index = group_indexes_matrix[group_row_index][2].reveal() + + records_sum[0] = source[start_index][col_index] + + @for_range(start_index + 1, end_index + 1) + def _(i): + @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal()) + def _(): + records_sum[0] = records_sum[0] + source[i][col_index] + + return records_sum[0] + + +def compute_count_with_group_by(group_row_index): + return group_indexes_matrix[group_row_index][0] + + +def compute_avg_with_group_by(source, col_index, group_row_index): + records_sum = Array(1, value_type) + + start_index = group_indexes_matrix[group_row_index][1].reveal() + end_index = group_indexes_matrix[group_row_index][2].reveal() + + records_sum[0] = source[start_index][col_index] + + @for_range(start_index + 1, end_index + 1) + def _(i): + @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal()) + def _(): + records_sum[0] = records_sum[0] + source[i][col_index] + + return value_type(records_sum[0] / group_indexes_matrix[group_row_index][0]) + + +def compute_max_with_group_by(source, col_index, group_row_index): + max_records = Array(1, value_type) + + start_index = group_indexes_matrix[group_row_index][1].reveal() + end_index = group_indexes_matrix[group_row_index][2].reveal() + + max_records[0] = source[start_index][col_index] + + @for_range(start_index + 1, end_index + 1) + def _(i): + @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal()) + def _(): + max_records[0] = condition(max_records[0] < source[i][col_index], source[i][col_index], max_records[0]) + + return max_records[0] + + +def compute_min_with_group_by(source, col_index, group_row_index): + min_records = Array(1, value_type) + + start_index = group_indexes_matrix[group_row_index][1].reveal() + end_index = group_indexes_matrix[group_row_index][2].reveal() + + min_records[0] = source[start_index][col_index] + + @for_range(start_index + 1, end_index + 1) + def _(i): + @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal()) + def _(): + min_records[0] = condition(min_records[0] > source[i][col_index], source[i][col_index], min_records[0]) + + return min_records[0] + + +def calculate_result_0(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][0] = 3*group_indexes_key[i] + + +def calculate_result_1(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][1] = compute_count_with_group_by(i) + + +def calculate_result_2(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][2] = compute_avg_with_group_by(source0_record, 1, i)*2+group_indexes_key[i] + + +def calculate_result_3(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][3] = (compute_sum_with_group_by(source0_record, 2, i)+compute_sum_with_group_by(source1_record, 2, i))/(compute_count_with_group_by(i)+100/(compute_min_with_group_by(source0_record, 1, i)+compute_min_with_group_by(source1_record, 1, i)))+10 + + +def calculate_result_4(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][4] = compute_max_with_group_by(source1_record, 1, i) + + +def calculate_result_5(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][5] = compute_min_with_group_by(source2_record, 2, i) + + +def print_results(): + result_fields = ['r0', 'count', 'r1', 'result3', 'result4', 'result5'] + set_display_field_names(result_fields) + + @for_range_opt(result_record) + def _(i): + result_values = [results[i][0].reveal(), results[i][1].reveal(), results[i][2].reveal(), results[i][3].reveal(), results[i][4].reveal(), results[i][5].reveal()] + display_data(result_values) + + +def ppc_main(): + read_data_collection(source0_record, SOURCE0) + read_data_collection(source1_record, SOURCE1) + read_data_collection(source2_record, SOURCE2) + + compute_group_by_indexes(source1_record, 4) + + calculate_result_0() + calculate_result_1() + calculate_result_2() + calculate_result_3() + calculate_result_4() + calculate_result_5() + + print_results() + + +ppc_main() diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc new file mode 100644 index 00000000..fa2fc97d --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc @@ -0,0 +1,78 @@ +# PSI_OPTION=True + +# BIT_LENGTH = 128 + +# This file is generated automatically by ams +''' +SELECT 3*(s1.field3 + s2.field3) - s0.field3 AS r0, + (s0.field1 + s2.field1) / 2 * s1.field1 AS r1 +FROM (source0 AS s0 + INNER JOIN source1 AS s1 ON s0.id = s1.id) +INNER JOIN source2 AS s2 ON s0.id = s2.id; +''' + +from ppc import * + +n_threads = 8 +value_type = pfix + +pfix.set_precision(16, 47) + +SOURCE0 = 0 +source0_record_count = $(source0_record_count) +source0_column_count = 4 +source0_record = Matrix(source0_record_count, source0_column_count, value_type) + +SOURCE1 = 1 +source1_record_count = $(source1_record_count) +source1_column_count = 4 +source1_record = Matrix(source1_record_count, source1_column_count, value_type) + +SOURCE2 = 2 +source2_record_count = $(source2_record_count) +source2_column_count = 4 +source2_record = Matrix(source2_record_count, source2_column_count, value_type) + +# basic arithmetic operation means that all parties have same number of record +result_record = $(source0_record_count) +results = Matrix(result_record, 2, value_type) + + +def read_data_collection(data_collection, party_id): + if data_collection.sizes[0] > 0: + data_collection.input_from(party_id) + + +def calculate_result_0(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][0] = 3*(source1_record[i][3]+source2_record[i][3])-source0_record[i][3] + + +def calculate_result_1(): + @for_range_opt_multithread(n_threads, result_record) + def _(i): + results[i][1] = (source0_record[i][1]+source2_record[i][1])/2*source1_record[i][1] + + +def print_results(): + result_fields = ['r0', 'r1'] + set_display_field_names(result_fields) + + @for_range_opt(result_record) + def _(i): + result_values = [results[i][0].reveal(), results[i][1].reveal()] + display_data(result_values) + + +def ppc_main(): + read_data_collection(source0_record, SOURCE0) + read_data_collection(source1_record, SOURCE1) + read_data_collection(source2_record, SOURCE2) + calculate_result_0() + calculate_result_1() + + print_results() + + +ppc_main() diff --git a/python/ppc_scheduler/mpc_generator/test_generator.py b/python/ppc_scheduler/mpc_generator/test_generator.py new file mode 100644 index 00000000..9a37a2e1 --- /dev/null +++ b/python/ppc_scheduler/mpc_generator/test_generator.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +import os +import unittest + +from ppc_scheduler.mpc_generator.generator import CodeGenerator +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + +FILE_PATH = os.path.abspath(__file__) + +CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".") + +BASIC_ARITH_OPE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}basic_arith_ope.mpc" +AGGR_FUNC_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_only.mpc" +GROUP_BY_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_with_group_by.mpc" + +with open(BASIC_ARITH_OPE_PATH, "r") as file: + BASIC_ARITH_OPE_STR = file.read() + +with open(AGGR_FUNC_SAMPLE_PATH, "r") as file: + AGGR_FUNC_SAMPLE_STR = file.read() + +with open(GROUP_BY_SAMPLE_PATH, "r") as file: + GROUP_BY_SAMPLE_STR = file.read() + + +class TestGenerator(unittest.TestCase): + + def test_bad_sql(self): + try: + sql = "select from a from b where c = d" + code_generator = CodeGenerator(sql) + code_generator.sql_to_mpc_code() + except PpcException as e: + self.assertEqual(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), e.code) + + sql = "select a0.f1 + b1.f1 from a0, b1 where a0.id=b1.id" + code_generator = CodeGenerator(sql) + self.assertIsNotNone(code_generator.sql_to_mpc_code()) + + def test_unsupported_keyword(self): + try: + sql = "select s0.f1 + s1.f1 from s0, s1 where s0.f1 > 1 and s0.f1 < 10" + code_generator = CodeGenerator(sql) + code_generator.sql_to_mpc_code() + except PpcException as e: + self.assertEqual("keyword 'AND' not supported", e.message) + + def test_disabled_query_pattern(self): + try: + sql = "select s0.f1, 3 + s1.f1 from s0, s1" + code_generator = CodeGenerator(sql) + code_generator.sql_to_mpc_code() + except PpcException as e: + self.assertEqual("disabled query pattern", e.message) + + try: + sql = "select s0.f1, s1.f1 + s1.f1 from s0, s1" + code_generator = CodeGenerator(sql) + code_generator.sql_to_mpc_code() + except PpcException as e: + self.assertEqual("disabled query pattern", e.message) + + def test_basic_pattern(self): + sql = "SELECT 3*(s1.field3 + s2.field3) - s0.field3 AS r0, \ + (s0.field1 + s2.field1) / 2 * s1.field1 AS r1\ + FROM (source0 AS s0\ + INNER JOIN source1 AS s1 ON s0.id = s1.id)\ + INNER JOIN source2 AS s2 ON s0.id = s2.id;" + code_generator = CodeGenerator(sql) + self.assertEqual(BASIC_ARITH_OPE_STR, code_generator.sql_to_mpc_code()) + + def test_single_aggre_pattern(self): + sql = "SELECT COUNT(s1.field3) + COUNT(s2.field3) AS r0,\ + SUM(s1.field3) + COUNT(s0.field0) AS 'count',\ + (MAX(s0.field1) + MAX(s2.field1)) / 2 AS r1,\ + (AVG(s1.field2) + AVG(s2.field2)) / 2 AS r2,\ + MIN(s1.field0) - MIN(s0.field0) AS r3\ + FROM (source0 AS s0\ + INNER JOIN source1 AS s1 ON s0.id = s1.id)\ + INNER JOIN source2 AS s2 ON s0.id = s2.id;" + code_generator = CodeGenerator(sql) + self.assertEqual(AGGR_FUNC_SAMPLE_STR, code_generator.sql_to_mpc_code()) + + def test_group_by_pattern(self): + sql = "SELECT 3*s1.field4 AS r0,\ + COUNT(s1.field4) AS 'count', \ + AVG(s0.field1) * 2 + s1.field4 AS r1,\ + (SUM(s0.field2) + SUM(s1.field2))/(COUNT(s1.field3) + 100/(MIN(s0.field1)+MIN(s1.field1))) + 10,\ + MAX(s1.field1),\ + MIN(s2.field2)\ + FROM (source0 AS s0\ + INNER JOIN source1 AS s1 ON s0.id = s1.id)\ + INNER JOIN source2 AS s2 ON s0.id = s2.id\ + GROUP BY s1.field4;" + code_generator = CodeGenerator(sql) + self.assertEqual(GROUP_BY_SAMPLE_STR, code_generator.sql_to_mpc_code()) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/python/ppc_scheduler/node/__init__.py b/python/ppc_scheduler/node/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/node/computing_node_client/__init__.py b/python/ppc_scheduler/node/computing_node_client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/node/computing_node_client/model_node_client.py b/python/ppc_scheduler/node/computing_node_client/model_node_client.py new file mode 100644 index 00000000..03456c9b --- /dev/null +++ b/python/ppc_scheduler/node/computing_node_client/model_node_client.py @@ -0,0 +1,87 @@ +import time + +from ppc_common.ppc_utils import http_utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + +RUN_MODEL_API_PREFIX = "/api/ppc-model/pml/run-model-task/" +GET_MODEL_LOG_API_PREFIX = "/api/ppc-model/pml/record-model-log/" + + +class ModelClient: + def __init__(self, log, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): + self.log = log + self.endpoint = endpoint + self.polling_interval_s = polling_interval_s + self.max_retries = max_retries + self.retry_delay_s = retry_delay_s + self._completed_status = 'COMPLETED' + self._failed_status = 'FAILED' + + def run(self, args): + task_id = args['task_id'] + try: + self.log.info(f"ModelApi: begin to run model task {task_id}") + response = self._send_request_with_retry(http_utils.send_post_request, + endpoint=self.endpoint, + uri=RUN_MODEL_API_PREFIX + task_id, + params=args) + check_response(response) + return self._poll_task_status(task_id) + except Exception as e: + self.log.error(f"ModelApi: run model task error, task: {task_id}, error: {e}") + raise e + + def kill(self, job_id): + try: + self.log.info(f"ModelApi: begin to kill model task {job_id}") + response = self._send_request_with_retry(http_utils.send_delete_request, + endpoint=self.endpoint, + uri=RUN_MODEL_API_PREFIX + job_id) + check_response(response) + self.log.info(f"ModelApi: model task {job_id} was killed") + return response + except Exception as e: + self.log.warn(f"ModelApi: kill model task {job_id} failed, error: {e}") + raise e + + def _poll_task_status(self, task_id): + while True: + response = self._send_request_with_retry(http_utils.send_get_request, + endpoint=self.endpoint, + uri=RUN_MODEL_API_PREFIX + task_id) + check_response(response) + if response['data']['status'] == self._completed_status: + self.log.info(f"task {task_id} completed, response: {response['data']}") + return response + elif response['data']['status'] == self._failed_status: + self.log.warn(f"task {task_id} failed, response: {response['data']}") + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) + else: + time.sleep(self.polling_interval_s) + + def get_remote_log(self, job_id): + response = self._send_request_with_retry(http_utils.send_get_request, + endpoint=self.endpoint, + uri=GET_MODEL_LOG_API_PREFIX + job_id) + check_response(response) + return response['data'] + + def _send_request_with_retry(self, request_func, *args, **kwargs): + attempt = 0 + while attempt < self.max_retries: + try: + response = request_func(*args, **kwargs) + return response + except Exception as e: + self.log.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") + attempt += 1 + if attempt < self.max_retries: + time.sleep(self.retry_delay_s) + else: + self.log.warn(f"Request failed after {self.max_retries} attempts") + raise e + + +def check_response(response): + if response['errorCode'] != 0: + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['message']) diff --git a/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py b/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py new file mode 100644 index 00000000..f64ece20 --- /dev/null +++ b/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py @@ -0,0 +1,32 @@ +import random + +from ppc_common.ppc_utils import http_utils, utils +from ppc_scheduler.node.computing_node_client.utils import check_privacy_service_response + + +class MpcClient: + def __init__(self, endpoint): + self.endpoint = endpoint + + def run(self, job_info, token): + params = { + 'jsonrpc': '2', + 'method': 'run', + 'token': token, + 'id': random.randint(1, 65535), + 'params': job_info + } + response = http_utils.send_post_request(self.endpoint, None, params) + check_privacy_service_response(response) + return response['result'] + + def kill(self, job_id, token): + params = { + 'jsonrpc': '2', + 'method': 'kill', + 'token': token, + 'id': random.randint(1, 65535), + 'params': {'jobId': job_id} + } + http_utils.send_post_request(self.endpoint, None, params) + return utils.make_response(0, "success", None) diff --git a/python/ppc_scheduler/node/computing_node_client/psi_node_client.py b/python/ppc_scheduler/node/computing_node_client/psi_node_client.py new file mode 100644 index 00000000..a4506a4c --- /dev/null +++ b/python/ppc_scheduler/node/computing_node_client/psi_node_client.py @@ -0,0 +1,66 @@ +import random +import time + +from ppc_common.ppc_utils import http_utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_scheduler.node.computing_node_client.utils import check_privacy_service_response + + +class PsiClient: + def __init__(self, log, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): + self.log = log + self.endpoint = endpoint + self.polling_interval_s = polling_interval_s + self.max_retries = max_retries + self.retry_delay_s = retry_delay_s + self._async_run_task_method = 'asyncRunTask' + self._get_task_status_method = 'getTaskStatus' + self._completed_status = 'COMPLETED' + self._failed_status = 'FAILED' + + def run(self, job_info, token): + params = { + 'jsonrpc': '1', + 'method': self._async_run_task_method, + 'token': token, + 'id': random.randint(1, 65535), + 'params': job_info + } + response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + check_privacy_service_response(response) + return self._poll_task_status(job_info['taskID'], token) + + def _poll_task_status(self, task_id, token): + while True: + params = { + 'jsonrpc': '1', + 'method': self._get_task_status_method, + 'token': token, + 'id': random.randint(1, 65535), + 'params': { + 'taskID': task_id, + } + } + response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + check_privacy_service_response(response) + if response['result']['status'] == self._completed_status: + return response['result'] + elif response['result']['status'] == self._failed_status: + self.log.warn(f"task {task_id} failed, response: {response['data']}") + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) + time.sleep(self.polling_interval_s) + + def _send_request_with_retry(self, request_func, *args, **kwargs): + attempt = 0 + while attempt < self.max_retries: + try: + response = request_func(*args, **kwargs) + return response + except Exception as e: + self.log.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") + attempt += 1 + if attempt < self.max_retries: + time.sleep(self.retry_delay_s) + else: + self.log.warn(f"Request failed after {self.max_retries} attempts") + raise e diff --git a/python/ppc_scheduler/node/computing_node_client/utils.py b/python/ppc_scheduler/node/computing_node_client/utils.py new file mode 100644 index 00000000..3d3527c8 --- /dev/null +++ b/python/ppc_scheduler/node/computing_node_client/utils.py @@ -0,0 +1,8 @@ +from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException + + +def check_privacy_service_response(response): + if 'result' not in response.keys(): + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), "http request error") + elif 0 != response['result']['code'] or response['result']['status'] == 'FAILED': + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['result']['message']) diff --git a/python/ppc_scheduler/node/node_manager.py b/python/ppc_scheduler/node/node_manager.py new file mode 100644 index 00000000..fa2b2247 --- /dev/null +++ b/python/ppc_scheduler/node/node_manager.py @@ -0,0 +1,32 @@ +from ppc_scheduler.database import computing_node_mapper +from ppc_scheduler.workflow.common.worker_type import WorkerType + + +class ComputingNodeManager: + type_map = { + WorkerType.T_PSI: 'PSI', + WorkerType.T_MPC: 'MPC', + WorkerType.T_PREPROCESSING: 'MODEL', + WorkerType.T_FEATURE_ENGINEERING: 'MODEL', + WorkerType.T_TRAINING: 'MODEL', + WorkerType.T_PREDICTION: 'MODEL', + } + + def __init__(self, components): + self.components = components + + def add_node(self, node_id: str, url: str, worker_type: str): + with self.components.create_sql_session() as session: + computing_node_mapper.insert_computing_node(session, node_id, url, self.type_map[worker_type], 0) + + def remove_node(self, url: str, worker_type: str): + with self.components.create_sql_session() as session: + computing_node_mapper.delete_computing_node(session, url, self.type_map[worker_type]) + + def get_node(self, worker_type: str): + with self.components.create_sql_session() as session: + return computing_node_mapper.get_and_update_min_loading_url(session, self.type_map[worker_type]) + + def release_node(self, url: str, worker_type: str): + with self.components.create_sql_session() as session: + return computing_node_mapper.release_loading(session, url, self.type_map[worker_type]) diff --git a/python/ppc_scheduler/ppc_scheduler_app.py b/python/ppc_scheduler/ppc_scheduler_app.py new file mode 100644 index 00000000..fb44f888 --- /dev/null +++ b/python/ppc_scheduler/ppc_scheduler_app.py @@ -0,0 +1,48 @@ +# Note: here can't be refactored by autopep + +from ppc_scheduler.endpoints.restx import api +from ppc_scheduler.endpoints.job_controller import ns as job_namespace +from ppc_scheduler.common.global_context import components +from paste.translogger import TransLogger +from flask import Flask, Blueprint +from cheroot.wsgi import Server as WSGIServer +from cheroot.ssl.builtin import BuiltinSSLAdapter +import os +import sys +sys.path.append("../") + + +app = Flask(__name__) + + +def initialize_app(app): + # 初始化应用功能组件 + components.init_all() + + app.config.update(components.config_data) + blueprint = Blueprint('api', __name__, url_prefix='/api') + api.init_app(blueprint) + api.add_namespace(job_namespace) + app.register_blueprint(blueprint) + + +if __name__ == '__main__': + initialize_app(app) + + app.config['SECRET_KEY'] = os.urandom(24) + server = WSGIServer((app.config['HOST'], app.config['HTTP_PORT']), + TransLogger(app, setup_console_handler=False), numthreads=2) + + ssl_switch = app.config['SSL_SWITCH'] + protocol = 'http' + if ssl_switch == 1: + protocol = 'https' + server.ssl_adapter = BuiltinSSLAdapter( + certificate=app.config['SSL_CRT'], + private_key=app.config['SSL_KEY'], + certificate_chain=app.config['CA_CRT']) + + message = f"Starting ppc scheduler server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']}" + print(message) + components.logger().info(message) + server.start() diff --git a/python/ppc_scheduler/workflow/__init__.py b/python/ppc_scheduler/workflow/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/common/__init__.py b/python/ppc_scheduler/workflow/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/common/codec.py b/python/ppc_scheduler/workflow/common/codec.py new file mode 100644 index 00000000..3fde280a --- /dev/null +++ b/python/ppc_scheduler/workflow/common/codec.py @@ -0,0 +1,57 @@ +from ppc_common.ppc_protos.generated.ppc_pb2 import JobWorkerOutputs, JobWorkerUpstreams, \ + InputStatement, JobWorkerInputsStatement +from ppc_common.ppc_utils import utils + + +def deserialize_worker_outputs(outputs_str): + outputs = [] + outputs_pb = JobWorkerOutputs() + utils.str_to_pb(outputs_pb, outputs_str) + for output in outputs_pb.outputs: + outputs.append(output) + return outputs + + +def serialize_worker_outputs_for_db(outputs): + outputs_pb = JobWorkerOutputs() + for output in outputs: + outputs_pb.outputs.append(output) + return utils.pb_to_str(outputs_pb) + + +def deserialize_upstreams(upstreams_str): + upstreams = [] + upstream_pb = JobWorkerUpstreams() + utils.str_to_pb(upstream_pb, upstreams_str) + for upstream in upstream_pb.upstreams: + upstreams.append(upstream) + return upstreams + + +def serialize_upstreams_for_db(upstreams): + upstreams_pb = JobWorkerUpstreams() + for upstream in upstreams: + upstreams_pb.upstreams.append(upstream) + return utils.pb_to_str(upstreams_pb) + + +def deserialize_inputs_statement(inputs_statement_str): + inputs_statement = [] + inputs_statement_pb = JobWorkerInputsStatement() + utils.str_to_pb(inputs_statement_pb, inputs_statement_str) + for input_statement_pb in inputs_statement_pb.inputs_statement: + inputs_statement.append({ + 'upstream': input_statement_pb.upstream, + 'output_index': input_statement_pb.output_index + }) + return inputs_statement + + +def serialize_inputs_statement_for_db(inputs_statement): + inputs_statement_pb = JobWorkerInputsStatement() + for input_statement in inputs_statement: + input_statement_pb = InputStatement() + input_statement_pb.upstream = input_statement['upstream'] + input_statement_pb.output_index = input_statement['output_index'] + inputs_statement_pb.inputs_statement.append(input_statement_pb) + return utils.pb_to_str(inputs_statement_pb) diff --git a/python/ppc_scheduler/workflow/common/default_flow_config.py b/python/ppc_scheduler/workflow/common/default_flow_config.py new file mode 100644 index 00000000..6797bbc4 --- /dev/null +++ b/python/ppc_scheduler/workflow/common/default_flow_config.py @@ -0,0 +1,225 @@ +flow_dict = { + "PSI": [ + { + "index": 1, + "type": "T_PSI", + "isParamsProvided": False, + "params": { + "type": 0, + "algorithm": 0, + "syncResult": True, + "parties": [ + { + "index": "", + "partyIndex": 1 + }, + { + "index": "", + "partyIndex": 0, + "data": { + "index": "", + "input": { + "type": 2, + "path": "" + }, + "output": { + "type": 2, + "path": "" + } + } + } + ] + } + } + ], + + "MPC": [ + { + "index": 1, + "type": "T_MPC" + } + ], + + "PSI_MPC": [ + { + "index": 1, + "type": "T_PSI" + }, + { + "index": 2, + "type": "T_MPC", + "upstreams": [ + { + "index": 1, + "output_input_map": [ + "0:0" + ] + } + ] + } + ], + + "PREPROCESSING": [ + { + "index": 1, + "type": "T_PREPROCESSING" + } + ], + + "FEATURE_ENGINEERING": [ + { + "index": 1, + "type": "T_PREPROCESSING" + }, + { + "index": 2, + "type": "T_FEATURE_ENGINEERING", + "upstreams": [ + { + "index": 1 + } + ] + } + ], + + "TRAINING": [ + { + "index": 1, + "type": "T_PREPROCESSING" + }, + { + "index": 2, + "type": "T_TRAINING", + "upstreams": [ + { + "index": 1 + } + ] + } + ], + + "PREDICTION": [ + { + "index": 1, + "type": "T_PREPROCESSING" + }, + { + "index": 2, + "type": "T_PREDICTION", + "upstreams": [ + { + "index": 1 + } + ] + } + ], + + "FEATURE_ENGINEERING_TRAINING": [ + { + "index": 1, + "type": "T_PREPROCESSING" + }, + { + "index": 2, + "type": "T_FEATURE_ENGINEERING", + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "T_TRAINING", + "upstreams": [ + { + "index": 2 + } + ] + } + ], + + "PSI_FEATURE_ENGINEERING": [ + { + "index": 1, + "type": "T_PSI" + }, + { + "index": 2, + "type": "T_PREPROCESSING", + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "T_FEATURE_ENGINEERING", + "upstreams": [ + { + "index": 2 + } + ] + } + ], + + "PSI_TRAINING": [ + { + "index": 1, + "type": "T_PSI" + }, + { + "index": 2, + "type": "T_PREPROCESSING", + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "T_TRAINING", + "upstreams": [ + { + "index": 2 + } + ] + } + ], + + "PSI_FEATURE_ENGINEERING_TRAINING": [ + { + "index": 1, + "type": "T_PSI" + }, + { + "index": 2, + "type": "T_PREPROCESSING", + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "T_FEATURE_ENGINEERING", + "upstreams": [ + { + "index": 2 + } + ] + }, + { + "index": 4, + "type": "T_TRAINING", + "upstreams": [ + { + "index": 3 + } + ] + } + ] +} diff --git a/python/ppc_scheduler/workflow/common/flow_utils.py b/python/ppc_scheduler/workflow/common/flow_utils.py new file mode 100644 index 00000000..3d05e11e --- /dev/null +++ b/python/ppc_scheduler/workflow/common/flow_utils.py @@ -0,0 +1,34 @@ +from ppc_scheduler.workflow.common.worker_type import WorkerType + + +def cat_worker_id(job_id, index, worker_type): + return f"{job_id}_{index}_{worker_type}" + + +def success_id(job_id): + return cat_worker_id(job_id, 0, WorkerType.T_ON_SUCCESS) + + +def failure_id(job_id): + return cat_worker_id(job_id, 0, WorkerType.T_ON_FAILURE) + + +def to_origin_inputs(worker_inputs): + inputs = [] + for each in worker_inputs: + output_index = each['output_index'] + upstream_outputs = each['upstream_outputs'] + inputs.append(upstream_outputs[output_index]) + return inputs + + +def to_worker_inputs(job_workers, inputs_statement): + worker_inputs = [] + for each in inputs_statement: + output_index = each['output_index'] + upstream_unit = each['upstream_unit'] + worker_inputs.append({ + 'output_index': output_index, + 'upstream_outputs': job_workers[upstream_unit] + }) + return worker_inputs diff --git a/python/ppc_scheduler/workflow/common/job_context.py b/python/ppc_scheduler/workflow/common/job_context.py new file mode 100644 index 00000000..0a6c6feb --- /dev/null +++ b/python/ppc_scheduler/workflow/common/job_context.py @@ -0,0 +1,115 @@ +import os + +from ppc_scheduler.job.job_type import JobType +from ppc_scheduler.mpc_generator.generator import CodeGenerator +from ppc_scheduler.workflow.common.default_flow_config import flow_dict +from ppc_scheduler.common.global_context import components + + +class JobContext: + PSI_PREPARE_FILE = "psi_inputs" + PSI_RESULT_INDEX_FILE = "psi_result_index" + PSI_RESULT_FILE = "psi_result" + MPC_PREPARE_FILE = "mpc_prepare.csv" + MPC_RESULT_FILE = "mpc_result.csv" + MPC_OUTPUT_FILE = "mpc_output.txt" + HDFS_STORAGE_PATH = "/user/ppc/" + + def __init__(self, args, workspace): + self.args = args + #todo: 确保java服务给过来的任务信息包含如下字段，如果是建模相关的任务，还需要job_context.model_config_dict = args['model_config'] + # 如果是mpc任务，还需要有args['sql']，或者args['mpc_content'] + self.job_id: str = args['job_id'] + self.user_name: str = args['user_name'] + self.dataset_id: str = args['dataset_id'] + + self.psi_fields: str = args['psi_fields'] + + self.result_receiver_list: list = args['result_receiver_list'] + self.participant_id_list: list = args['participant_id_list'] + self.job_type = args['job_type'] + self.dataset_record_count = args['dataset_record_count'] + + self.my_index = None + self.need_run_psi = False + self.need_run_fe = False + self.mpc_content = None + + self.model_config_dict: dict = {} + self.tag_provider_agency_id = None + self.job_subtype = None + self.predict_algorithm = None + + self.worker_configs: list = [] + self.workflow_view_path = 'workflow_view' + + self.workspace = workspace + self.job_cache_dir = "{}{}{}".format(self.workspace, os.sep, self.job_id) + self.dataset_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.dataset_id) + self.psi_prepare_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_PREPARE_FILE) + self.psi_result_index_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_INDEX_FILE) + self.psi_result_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_FILE) + self.mpc_file_name = "{}.mpc".format(self.job_id) + self.mpc_model_module_name = "{}.json".format(self.job_id) + self.mpc_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.mpc_file_name) + self.mpc_prepare_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_PREPARE_FILE) + self.mpc_result_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_RESULT_FILE) + self.mpc_output_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_OUTPUT_FILE) + + @staticmethod + def load_from_args(args, workspace): + job_context = JobContext(args, workspace) + job_context.my_index = job_context.participant_id_list.index(components.config_data['AGENCY_ID']) + + if job_context.job_type == JobType.PREPROCESSING or \ + job_context.job_type == JobType.TRAINING or \ + job_context.job_type == JobType.PREDICTION or \ + job_context.job_type == JobType.FEATURE_ENGINEERING: + + job_context.model_config_dict = args['model_config'] + job_context.tag_provider_agency_id = job_context.participant_id_list[0] + if 'job_subtype' in job_context.model_config_dict: + job_context.job_subtype = job_context.model_config_dict['job_subtype'] + if 'predict_algorithm' in job_context.model_config_dict: + job_context.predict_algorithm = job_context.model_config_dict['predict_algorithm'] + if 'use_psi' in job_context.model_config_dict: + job_context.need_run_psi = job_context.model_config_dict['use_psi'] == 1 + if 'use_iv' in job_context.model_config_dict: + job_context.need_run_fe = job_context.model_config_dict['use_iv'] == 1 + + if job_context.job_type == JobType.PSI: + job_context.worker_configs = flow_dict['PSI'] + elif job_context.job_type == JobType.MPC: + if 'sql' in job_context.args: + job_context.sql = args['sql'] + job_context.mpc_content = CodeGenerator(job_context.sql) + else: + job_context.mpc_content = job_context.args['mpc_content'] + if "PSI_OPTION=True" in job_context.mpc_content: + job_context.need_run_psi = True + job_context.worker_configs = flow_dict['PSI_MPC'] + else: + job_context.worker_configs = flow_dict['MPC'] + elif job_context.job_type == JobType.PREPROCESSING: + job_context.worker_configs = flow_dict['PREPROCESSING'] + elif job_context.job_type == JobType.FEATURE_ENGINEERING: + if job_context.need_run_psi: + job_context.worker_configs = flow_dict['PSI_FEATURE_ENGINEERING'] + else: + job_context.worker_configs = flow_dict['FEATURE_ENGINEERING'] + elif job_context.job_type == JobType.TRAINING: + if job_context.need_run_psi: + if job_context.need_run_fe: + job_context.worker_configs = flow_dict['PSI_FEATURE_ENGINEERING_TRAINING'] + else: + job_context.worker_configs = flow_dict['PSI_TRAINING'] + else: + if job_context.need_run_fe: + job_context.worker_configs = flow_dict['FEATURE_ENGINEERING_TRAINING'] + else: + job_context.worker_configs = flow_dict['TRAINING'] + elif job_context.job_type == JobType.PREDICTION: + job_context.worker_configs = flow_dict['PREDICTION'] + else: + raise Exception("Unsupported job type {}".format(job_context.job_type)) + return job_context diff --git a/python/ppc_scheduler/workflow/common/worker_status.py b/python/ppc_scheduler/workflow/common/worker_status.py new file mode 100644 index 00000000..f915f473 --- /dev/null +++ b/python/ppc_scheduler/workflow/common/worker_status.py @@ -0,0 +1,7 @@ +class WorkerStatus: + PENDING = 'PENDING' + RUNNING = 'RUNNING' + SUCCESS = 'SUCCESS' + FAILURE = 'FAILURE' + TIMEOUT = 'TIMEOUT' + KILLED = 'KILLED' diff --git a/python/ppc_scheduler/workflow/common/worker_type.py b/python/ppc_scheduler/workflow/common/worker_type.py new file mode 100644 index 00000000..d10f1f8f --- /dev/null +++ b/python/ppc_scheduler/workflow/common/worker_type.py @@ -0,0 +1,17 @@ +class WorkerType: + # generic job worker + T_API = 'T_API' + T_PYTHON = 'T_PYTHON' + T_SHELL = 'T_SHELL' + + # specific job worker + T_PSI = 'T_PSI' + T_MPC = 'T_MPC' + T_PREPROCESSING = 'T_PREPROCESSING' + T_FEATURE_ENGINEERING = 'T_FEATURE_ENGINEERING' + T_TRAINING = 'T_TRAINING' + T_PREDICTION = 'T_PREDICTION' + + # finish job + T_ON_SUCCESS = 'T_ON_SUCCESS' + T_ON_FAILURE = 'T_ON_FAILURE' diff --git a/python/ppc_scheduler/workflow/constructor.py b/python/ppc_scheduler/workflow/constructor.py new file mode 100644 index 00000000..ca6cf22b --- /dev/null +++ b/python/ppc_scheduler/workflow/constructor.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +from ppc_common.db_models.job_worker_record import JobWorkerRecord +from ppc_common.ppc_utils import utils +from ppc_scheduler.common.global_context import components +from ppc_scheduler.database import job_worker_mapper +from ppc_scheduler.workflow.common import codec, flow_utils +from ppc_scheduler.workflow.common.job_context import JobContext +from ppc_scheduler.workflow.common.worker_status import WorkerStatus + + +class Constructor: + def __init__(self): + self.log = components.logger() + + def build_flow_context(self, job_context: JobContext): + self.log.info(f"start build_flow_context, job_id: {job_context.job_id}") + job_id = job_context.job_id + flow_context = {} + index_type_map = {} + for worker_config in job_context.worker_configs: + index_type_map[worker_config['index']] = worker_config['type'] + + for worker_config in job_context.worker_configs: + worker_type = worker_config['type'] + worker_id = flow_utils.cat_worker_id(job_id, worker_config['index'], worker_type) + upstreams = [] + inputs_statement = [] + inputs_statement_tuple = [] + if 'upstream' in worker_config: + for upstream_config in worker_config["upstreams"]: + index = upstream_config['index'] + upstream_id = flow_utils.cat_worker_id(job_id, index, index_type_map[index]) + upstreams.append(upstream_id), + if 'output_input_map' in upstream_config: + for mapping in upstream_config.get("output_input_map", []): + output_index, input_index = mapping.split(":") + inputs_statement_tuple.append((upstream_id, int(output_index), int(input_index))) + + inputs_statement_tuple.sort(key=lambda x: x[2]) + for upstream_id, output_index, _ in inputs_statement_tuple: + inputs_statement.append( + { + 'output_index': output_index, + 'upstream': upstream_id + } + ) + worker_context = self._construct_context(job_context, worker_id, worker_type, + upstreams, inputs_statement) + flow_context[worker_id] = worker_context + self.log.info(f"end build_flow_context, flow_context:\n{flow_context}") + + def _construct_context(self, job_context, worker_id, worker_type, upstreams, inputs_statement): + context = { + 'type': worker_type, + 'status': WorkerStatus.PENDING, + 'upstreams': upstreams, + 'inputs_statement': inputs_statement + } + + with components.create_sql_session() as session: + worker_record = job_worker_mapper.query_job_worker(components.create_sql_session, + job_context.job_id, worker_id) + if worker_record is None: + worker_record = JobWorkerRecord( + worker_id=worker_id, + job_id=job_context.job_id, + type=worker_type, + status=WorkerStatus.PENDING, + upstreams=codec.serialize_upstreams_for_db(upstreams), + inputs_statement=codec.serialize_inputs_statement_for_db(inputs_statement), + create_time=utils.make_timestamp(), + update_time=utils.make_timestamp() + ) + session.add(worker_record) + session.commit() + else: + context['status'] = worker_record.status + context['upstreams'] = codec.deserialize_upstreams(worker_record.upstreams) + context['inputs_statement'] = codec.deserialize_inputs_statement(worker_record.inputs_statement) + + self.log.debug(f"Load worker_context successfully, worker_id: {worker_id}, context:\n{context}") + return context diff --git a/python/ppc_scheduler/workflow/scheduler.py b/python/ppc_scheduler/workflow/scheduler.py new file mode 100644 index 00000000..ed2e1fc4 --- /dev/null +++ b/python/ppc_scheduler/workflow/scheduler.py @@ -0,0 +1,87 @@ +from prefect import Flow +from prefect.executors import LocalDaskExecutor +from prefect.triggers import all_successful, any_failed + +from ppc_scheduler.workflow.common import flow_utils +from ppc_scheduler.workflow.common.job_context import JobContext +from ppc_scheduler.workflow.common.worker_status import WorkerStatus +from ppc_scheduler.workflow.common.worker_type import WorkerType +from ppc_scheduler.workflow.constructor import Constructor +from ppc_scheduler.workflow.worker.worker_factory import WorkerFactory + + +class Scheduler: + def __init__(self, workspace): + self.workspace = workspace + self.constructor = Constructor() + + def schedule_job_flow(self, args): + job_context = JobContext.load_from_args(args, self.workspace) + flow_context = self.constructor.build_flow_context(job_context) + self._run(job_context, flow_context) + + @staticmethod + def _run(job_context, flow_context): + job_workers = {} + job_id = job_context.job_id + job_flow = Flow(f"job_flow_{job_id}") + + # create a final job worker to handle success + finish_job_on_success = WorkerFactory.build_worker( + job_context, + flow_utils.success_id(job_id), + WorkerType.T_ON_SUCCESS) + + finish_job_on_success.trigger = all_successful + finish_job_on_success.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow) + job_flow.add_task(finish_job_on_success) + + # set reference task to bind job flow status + job_flow.set_reference_tasks([finish_job_on_success]) + + # create a final job worker to handle failure + finish_job_on_failure = WorkerFactory.build_worker( + job_context, + flow_utils.failure_id(job_id), + WorkerType.T_ON_FAILURE) + + # do finish_job_on_failure while any job worker failed + finish_job_on_failure.trigger = any_failed + finish_job_on_failure.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow) + job_flow.add_task(finish_job_on_failure) + + # create main job workers + for worker_id in flow_context: + worker_type = flow_context[worker_id]['type'] + job_worker = WorkerFactory.build_worker(job_context, worker_id, worker_type) + job_flow.add_task(job_worker) + job_workers[worker_id] = job_worker + + # set upstream for final job + finish_job_on_success.set_upstream(job_worker, flow=job_flow) + finish_job_on_failure.set_upstream(job_worker, flow=job_flow) + + # customize main job workers + for worker_id in flow_context: + # set upstream + upstreams = flow_context[worker_id]['upstreams'] + for upstream in upstreams: + if upstream not in job_workers: + raise Exception(-1, f"upstream job worker not found: {upstream}, " + f"job_id: {job_context.job_id}") + job_workers[worker_id].set_upstream(job_workers[upstream], flow=job_flow) + + # bind worker inputs + inputs_statement = flow_context[worker_id]['inputs_statement'] + worker_inputs = flow_utils.to_worker_inputs(job_workers, inputs_statement) + job_workers[worker_id].bind(worker_status=flow_context[worker_id]['status'], + worker_inputs=worker_inputs, flow=job_flow) + + # enable parallel execution + job_flow.executor = LocalDaskExecutor() + + # run dag workflow + job_flow_state = job_flow.run() + + # save workflow view as file + job_flow.visualize(job_flow_state, job_context.workflow_view_path, 'svg') diff --git a/python/ppc_scheduler/workflow/worker/__init__.py b/python/ppc_scheduler/workflow/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/worker/api_worker.py b/python/ppc_scheduler/workflow/worker/api_worker.py new file mode 100644 index 00000000..2698133b --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/api_worker.py @@ -0,0 +1,10 @@ +from ppc_scheduler.workflow.worker.worker import Worker + + +class ApiWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs): + ... diff --git a/python/ppc_scheduler/workflow/worker/engine/__init__.py b/python/ppc_scheduler/workflow/worker/engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/worker/engine/model_engine.py b/python/ppc_scheduler/workflow/worker/engine/model_engine.py new file mode 100644 index 00000000..2542c611 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/engine/model_engine.py @@ -0,0 +1,101 @@ +import os +import time + +from ppc_scheduler.workflow.common.job_context import JobContext +from ppc_scheduler.workflow.common.worker_type import WorkerType + + +class ModelWorkerEngine: + def __init__(self, model_client, worker_type, components, job_context: JobContext): + self.model_client = model_client + self.worker_type = worker_type + self.components = components + self.job_context = job_context + self.log = self.components.logger() + + def run(self) -> list: + if self.worker_type == WorkerType.T_PREPROCESSING: + self._run_preprocessing() + elif self.worker_type == WorkerType.T_FEATURE_ENGINEERING: + self._run_feature_engineering() + elif self.worker_type == WorkerType.T_TRAINING: + self._run_training() + elif self.worker_type == WorkerType.T_PREDICTION: + self._run_prediction() + else: + raise ValueError(f"Unsupported worker type: {self.worker_type}") + return [] + + def _run_preprocessing(self): + start = time.time() + job_id = self.job_context.job_id + task_id = job_id + '_d' + user_name = self.job_context.user_name + dataset_storage_path = os.path.join(user_name, self.job_context.dataset_id) + args = { + 'job_id': job_id, + 'task_id': task_id, + 'task_type': 'PREPROCESSING', + 'dataset_id': self.job_context.dataset_id, + 'dataset_storage_path': dataset_storage_path, + 'job_algorithm_type': self.job_context.job_type, + 'need_run_psi': self.job_context.need_run_psi, + 'model_dict': self.job_context.model_config_dict + } + self.log.info(f"start prepare_xgb, job_id: {job_id}, task_id: {task_id}, args: {args}") + self.model_client.run(args) + self.log.info( + f"call compute_xgb_job service success, job: {job_id}, " + f"task_id: {task_id}, timecost: {time.time() - start}") + + def _run_feature_engineering(self): + start = time.time() + job_id = self.job_context.job_id + task_id = job_id + '_f' + args = { + 'job_id': job_id, + 'task_id': task_id, + 'task_type': 'FEATURE_ENGINEERING', + 'is_label_holder': self.job_context.tag_provider_agency_id == self.components.config_data['AGENCY_ID'], + 'result_receiver_id_list': self.job_context.result_receiver_list, + 'participant_id_list': self.job_context.participant_id_list, + 'model_dict': self.job_context.model_config_dict + } + self.log.info(f"start feature_engineering, job_id: {job_id}, task_id: {task_id}, args: {args}") + self.model_client.run(args) + self.log.info( + f"call compute_xgb_job service success, job: {job_id}, " + f"task_id: {task_id}, timecost: {time.time() - start}") + + def _run_training(self): + # todo 支持LR + task_id = self.job_context.job_id + '_t' + task_type = 'XGB_TRAINING' + xgb_predict_algorithm = '' + self._run_model(task_id, task_type, xgb_predict_algorithm) + + def _run_prediction(self): + # todo 支持LR + task_id = self.job_context.job_id + '_p' + task_type = 'XGB_PREDICTING' + xgb_predict_algorithm = self.job_context.predict_algorithm + self._run_model(task_id, task_type, xgb_predict_algorithm) + + def _run_model(self, task_id, task_type, model_algorithm): + job_id = self.job_context.job_id + args = { + "job_id": job_id, + 'task_id': task_id, + 'task_type': task_type, + 'is_label_holder': self.job_context.tag_provider_agency_id == self.components.config_data['AGENCY_ID'], + 'result_receiver_id_list': self.job_context.result_receiver_list, + 'participant_id_list': self.job_context.participant_id_list, + 'model_predict_algorithm': model_algorithm, + "algorithm_type": self.job_context.job_type, + "algorithm_subtype": self.job_context.job_subtype, + "model_dict": self.job_context.model_config_dict + } + self.log.info(f"start run xgb task, job_id, job: {job_id}, " + f"task_id: {task_id}, task_type: {task_type}, args: {args}") + self.model_client.run(args) + self.log.info(f"call compute_xgb_job service success, job: {job_id}") diff --git a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py new file mode 100644 index 00000000..b159c7b2 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- +import os + +import pandas as pd + +from ppc_common.ppc_dataset import dataset_helper_factory +from ppc_common.ppc_utils import utils +from ppc_scheduler.mpc_generator.generator import CodeGenerator +from ppc_scheduler.workflow.common.job_context import JobContext + + +class MpcWorkerEngine: + def __init__(self, mpc_client, worker_type, components, job_context: JobContext): + self.mpc_client = mpc_client + self.worker_type = worker_type + self.components = components + self.job_context = job_context + self.log = self.components.logger() + + def run(self) -> list: + bit_length = self._prepare_mpc_file() + if self.job_context.need_run_psi: + self._prepare_mpc_after_psi() + else: + self._prepare_mpc_without_psi() + self._run_mpc_job(bit_length) + self._finish_mpc_job() + return [self.job_context.mpc_output_path] + + def _prepare_mpc_file(self): + # if self.job_context.sql is not None: + # # compile sql to mpc content + # mpc_content = CodeGenerator(self.job_context.sql) + # else: + # mpc_content = self.job_context.mpc_content + utils.write_content_to_file(self.job_context.mpc_content, self.job_context.mpc_file_path) + + self.components.storage_client.upload_file(self.job_context.mpc_file_path, + self.job_context.job_id + os.sep + self.job_context.mpc_file_name) + return self._get_share_bytes_length(self.job_context.mpc_content) + + def _run_mpc_job(self, bit_length): + job_id = self.job_context.job_id + mpc_record = 0 + self.log.info(f"start compute_mpc_job, job id: {job_id}") + if self.job_context.mpc_prepare_path: + mpc_record = sum(1 for _ in open(self.job_context.mpc_prepare_path)) + self.log.info(f"compute_mpc_job, mpc record: {mpc_record}") + utils.replace(self.job_context.mpc_file_path, mpc_record) + self._replace_mpc_field_holder() + self.components.storage_client.upload_file(self.job_context.mpc_file_path, + job_id + os.sep + self.job_context.mpc_file_name) + job_info = { + "jobId": job_id, + "mpcNodeUseGateway": False, + "receiverNodeIp": "", + "mpcNodeDirectPort": self.components.config_data["MPC_NODE_DIRECT_PORT"], + "participantCount": len(self.job_context.participant_id_list), + "selfIndex": self.job_context.my_index, + "isMalicious": self.components.config_data["IS_MALICIOUS"], + "bitLength": bit_length, + "inputFileName": "{}-P{}-0".format(JobContext.MPC_PREPARE_FILE, self.job_context.my_index), + "outputFileName": JobContext.MPC_OUTPUT_FILE + } + + self.log.info(f"call compute_mpc_job service, model run, params: {job_info}") + self.mpc_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) + self.components.storage_client.download_file(job_id + os.sep + JobContext.MPC_OUTPUT_FILE, + self.job_context.mpc_output_path) + self.log.info(f"call compute_mpc_job service success") + + def _finish_mpc_job(self): + job_id = self.job_context.job_id + index_file = None + if self.components.config_data['AGENCY_ID'] in self.job_context.result_receiver_list: + if self.job_context.need_run_psi: + if not utils.file_exists(self.job_context.psi_result_path): + self.log.info( + f"download finish_mpc_job psi_result_path, job_id={job_id}, " + f"download {job_id + os.sep + JobContext.PSI_RESULT_FILE}") + self.components.storage_client.download_file(job_id + os.sep + JobContext.PSI_RESULT_FILE, + self.job_context.psi_result_path) + self._order_psi_csv() + index_file = self.job_context.psi_result_path + if not utils.file_exists(self.job_context.mpc_output_path): + self.log.info( + f"download finish_mpc_job mpc_output_path, job_id={job_id}, " + f"download {job_id + os.sep + JobContext.MPC_OUTPUT_FILE}") + self.components.storage_client.download_file(job_id + os.sep + JobContext.MPC_OUTPUT_FILE, + self.job_context.mpc_output_path) + + self.log.info( + f"finish_mpc_job mpc_output_path, job_id={job_id}") + self._parse_and_write_final_mpc_result(self.job_context.mpc_output_path, index_file) + self.components.storage_client.upload_file(self.job_context.mpc_result_path, + job_id + os.sep + JobContext.MPC_RESULT_FILE) + self.log.info(f"finish_mpc_job success, job_id={job_id}") + + def _get_share_bytes_length(self, algorithm_content): + target = '# BIT_LENGTH = ' + if target in algorithm_content: + start = algorithm_content.find(target) + end = algorithm_content.find('\n', start + len(target)) + bit_length = int(algorithm_content[start + len(target): end].strip()) + self.log.info(f"OUTPUT_BIT_LENGTH = {bit_length}") + return bit_length + else: + self.log.info(f"OUTPUT_BIT_LENGTH = 64") + return 64 + + def _get_dataset_column_count(self): + with open(self.job_context.mpc_file_path, "r") as file: + mpc_str = file.read() + + lines = mpc_str.split('\n') + for line in lines: + if f"source{self.job_context.my_index}_column_count =" in line or \ + f"source{self.job_context.my_index}_column_count=" in line: + index = line.find('=') + return int(line[index + 1:].strip('\n').strip()) + + def _make_dataset_to_mpc_data_plus_psi_data(self, my_dataset_number): + chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, + chunksize=self.components.dataset_handler_initializer.file_chunk_config.read_chunk_size) + psi_data = pd.read_csv(self.job_context.psi_result_path, delimiter=utils.CSV_SEP) + for chunk in chunk_list: + self._make_dataset_field_normalized(chunk) + mpc_data_df = pd.merge(chunk, psi_data, on=['id']).sort_values( + by='id', ascending=True) + self._save_selected_column_data( + my_dataset_number, mpc_data_df, self.job_context.mpc_prepare_path) + + @staticmethod + def _make_dataset_field_normalized(dataset_df): + data_field = dataset_df.columns.values + if 'id' in data_field: + data_field_normalized_names = ['id'] + size = len(data_field) + for i in range(size - 1): + data_field_normalized_names.append( + utils.NORMALIZED_NAMES.format(i)) + else: + data_field_normalized_names = [] + size = len(data_field) + for i in range(size): + data_field_normalized_names.append( + utils.NORMALIZED_NAMES.format(i)) + dataset_df.columns = data_field_normalized_names + + @staticmethod + def _save_selected_column_data(my_dataset_number, data_df, mpc_prepare_path): + column_list = [] + for i in range(0, int(my_dataset_number)): + column_list.append(utils.NORMALIZED_NAMES.format(i)) + result_new = pd.DataFrame(data_df, columns=column_list) + # sep must be space (ppc-mpc inputs) + result_new.to_csv(mpc_prepare_path, sep=' ', mode='a', header=False, index=None) + + def _prepare_mpc_after_psi(self): + job_id = self.job_context.job_id + self.log.info(f"start prepare_mpc_after_psi, job_id={job_id}") + my_dataset_number = self._get_dataset_column_count() + + dataset_helper_factory.download_dataset( + dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, + dataset_user=self.job_context.user_name, + dataset_id=self.job_context.dataset_id, + dataset_local_path=self.job_context.dataset_file_path, + log_keyword="prepare_mpc_after_psi", + logger=self.log) + + if not utils.file_exists(self.job_context.psi_result_path): + self.log.info( + f"prepare_mpc_after_psi, download psi_result_path ,job_id={job_id}, " + f"download {job_id + os.sep + JobContext.PSI_RESULT_FILE}") + + self.components.storage_client.download_file(job_id + os.sep + JobContext.PSI_RESULT_FILE, + self.job_context.psi_result_path) + + self._make_dataset_to_mpc_data_plus_psi_data(my_dataset_number) + + hdfs_mpc_prepare_path = "{}-P{}-0".format(job_id + os.sep + JobContext.MPC_PREPARE_FILE, + self.job_context.my_index) + self.components.storage_client.upload_file(self.job_context.mpc_prepare_path, hdfs_mpc_prepare_path) + self.log.info(f"call prepare_mpc_after_psi success: job_id={job_id}") + + def _make_dataset_to_mpc_data_direct(self, my_dataset_number): + chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, + chunksize=self.components.dataset_handler_initializer.file_chunk_config.read_chunk_size) + for chunk in chunk_list: + self._make_dataset_field_normalized(chunk) + self._save_selected_column_data(my_dataset_number, chunk, self.job_context.mpc_prepare_path) + + def _prepare_mpc_without_psi(self): + job_id = self.job_context.job_id + self.log.info(f"start prepare_mpc_without_psi, job_id={job_id}") + my_dataset_number = self._get_dataset_column_count() + dataset_helper_factory.download_dataset( + dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, + dataset_user=self.job_context.user_name, + dataset_id=self.job_context.dataset_id, + dataset_local_path=self.job_context.dataset_file_path, + log_keyword="prepare_mpc_without_psi", + logger=self.log) + + self._make_dataset_to_mpc_data_direct(my_dataset_number) + + hdfs_mpc_prepare_path = "{}-P{}-0".format(job_id + os.sep + JobContext.MPC_PREPARE_FILE, + self.job_context.my_index) + self.components.storage_client.upload_file(self.job_context.mpc_prepare_path, hdfs_mpc_prepare_path) + self.log.info(f"call prepare_mpc_without_psi success: job_id={job_id}") + + def _order_psi_csv(self): + """ + order_psi_csv + """ + data = pd.read_csv(self.job_context.psi_result_path, delimiter=utils.CSV_SEP) + data.sort_values(by="id").to_csv(self.job_context.psi_result_path, + sep=utils.CSV_SEP, header=True, index=None) + + def _parse_and_write_final_mpc_result(self, mpc_output_path, index_file): + self.log.info("run parse_and_write_final_mpc_result") + final_result_fields = 'id' + need_add_fields = True + column_count = 0 + for row_data in open(mpc_output_path): + if row_data.__contains__(utils.PPC_RESULT_FIELDS_FLAG): + need_add_fields = False + final_result_fields += ',' + \ + row_data[row_data.find( + '=') + 1:].strip().replace(utils.BLANK_SEP, utils.CSV_SEP) + elif row_data.__contains__(utils.PPC_RESULT_VALUES_FLAG): + column_count = len(row_data.split( + '=')[1].strip().split(utils.BLANK_SEP)) + break + + if need_add_fields: + for i in range(column_count): + final_result_fields += ',' + 'result' + str(i) + + id_list = [] + if self.job_context.need_run_psi: + df = pd.read_csv(index_file, delimiter=utils.CSV_SEP) + for result_id in df["id"]: + id_list.append(result_id) + + with open(self.job_context.mpc_result_path, "w", encoding='utf-8') as file: + file.write(final_result_fields + '\n') + row_count = 0 + for row_data in open(mpc_output_path): + if row_data.__contains__(utils.PPC_RESULT_VALUES_FLAG): + values = row_data.split('=')[1].strip().split(utils.BLANK_SEP) + if self.job_context.need_run_psi: + if row_count >= len(id_list): + row = str(id_list[-1] + row_count - len(id_list) + 1) + else: + row = str(id_list[row_count]) + else: + row = str(row_count) + + for value in values: + try: + row += (',' + value) + except: + row += (',%s' % value) + file.write(row + '\n') + row_count += 1 + file.close() + + self.log.info("finish parse_and_write_final_mpc_result") + + def _replace_mpc_field_holder(self): + party_count = len(self.job_context.participant_id_list) + if self.job_context.need_run_psi: + dataset_record_count = 0 + if self.job_context.mpc_prepare_path: + dataset_record_count = sum(1 for _ in open(self.job_context.mpc_prepare_path)) + for i in range(party_count): + utils.replace(self.job_context.mpc_file_path, + dataset_record_count, f'$(source{i}_record_count)') + else: + for i in range(party_count): + utils.replace(self.job_context.mpc_file_path, + self.job_context.dataset_record_count, f'$(source{i}_record_count)') diff --git a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py new file mode 100644 index 00000000..acc5ed29 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +import codecs +import os +import time + +from ppc_common.ppc_dataset import dataset_helper_factory +from ppc_common.ppc_utils import utils, common_func +from ppc_scheduler.workflow.common.job_context import JobContext + + +class PsiWorkerEngine: + def __init__(self, psi_client, worker_type, components, job_context: JobContext): + self.psi_client = psi_client + self.worker_type = worker_type + self.components = components + self.job_context = job_context + self.log = self.components.logger() + + def run(self) -> list: + job_id = self.job_context.job_id + start_time = time.time() + self.origin_dataset_to_psi_inputs() + + self.log.info(f"compute two party psi, job_id={job_id}") + if len(self.job_context.participant_id_list) == 2: + self._run_two_party_psi() + else: + self._run_multi_party_psi() + time_costs = time.time() - start_time + self.log.info(f"computing psi finished, job_id={job_id}, timecost: {time_costs}s") + return [JobContext.HDFS_STORAGE_PATH + job_id + os.sep + self.job_context.PSI_RESULT_INDEX_FILE] + + def _run_two_party_psi(self): + job_id = self.job_context.job_id + agency_id = self.components.config_data['AGENCY_ID'] + job_info = { + "taskID": job_id, + "type": 0, + "algorithm": 0, + "syncResult": True, + "lowBandwidth": False, + "parties": [ + { + "id": self.job_context.participant_id_list[1 - self.job_context.my_index], + "partyIndex": 1 - self.job_context.my_index + }, + { + "id": agency_id, + "partyIndex": self.job_context.my_index, + "data": + { + "id": self.job_context.job_id, + "input": { + "type": 2, + "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_PREPARE_FILE + }, + "output": { + "type": 2, + "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_RESULT_FILE + } + } + } + ] + } + psi_result = self.psi_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) + self.log.info(f"call psi service successfully, job_id={job_id}, result: {psi_result}") + + def _run_multi_party_psi(self): + job_id = self.job_context.job_id + participant_number = len(self.job_context.participant_id_list) + participant_list = [] + # parties_index = [0, 1 ··· 1, 2] + parties_index = participant_number * [1] # role: partner + parties_index[0] = 0 # role: calculator + parties_index[-1] = 2 # role: master + for index, agency_id in enumerate(self.job_context.participant_id_list): + party_map = {} + if self.job_context.my_index == index: + party_map["id"] = agency_id + party_map["partyIndex"] = parties_index[index] + party_map["data"] = { + "id": self.job_context.job_id, + "input": { + "type": 2, + "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_PREPARE_FILE + }, + "output": { + "type": 2, + "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_RESULT_FILE + } + } + else: + party_map["id"] = agency_id + party_map["partyIndex"] = parties_index[index] + participant_list.append(party_map) + + job_info = { + "taskID": job_id, + "type": 0, + "algorithm": 4, + "syncResult": True, + "receiverList": self.job_context.result_receiver_list, + "parties": participant_list + } + psi_result = self.psi_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) + self.log.info(f"call psi service successfully, job_id={job_id}, result: {psi_result}") + + def origin_dataset_to_psi_inputs(self): + dataset_helper_factory.download_dataset( + dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, + dataset_user=self.job_context.user_name, + dataset_id=self.job_context.dataset_id, + dataset_local_path=self.job_context.dataset_file_path, + log_keyword="prepare_dataset", + logger=self.log) + + field = (self.job_context.psi_fields.split(utils.CSV_SEP)[self.job_context.my_index]).lower() + if field == '': + field = 'id' + prepare_file = open(self.job_context.psi_prepare_path, 'w') + psi_split_reg = "===" + file_encoding = common_func.get_file_encoding(self.job_context.dataset_file_path) + if psi_split_reg in field: + field_multi = field.split(psi_split_reg) + with codecs.open(self.job_context.dataset_file_path, "r", file_encoding) as dataset: + fields = next(dataset).lower() + fields_list = fields.strip().split(utils.CSV_SEP) + id_idx_list = [] + for filed_idx in field_multi: + id_idx_list.append(fields_list.index(filed_idx)) + for line in dataset: + if line.strip() == "": + continue + final_str = "" + for id_idx_multi in id_idx_list: + if len(line.strip().split(utils.CSV_SEP, id_idx_multi + 1)) < id_idx_multi + 1: + continue + final_str = "{}-{}".format(final_str, line.strip().split( + utils.CSV_SEP, id_idx_multi + 1)[id_idx_multi]).strip("\r\n") + print(final_str, file=prepare_file) + prepare_file.close() + else: + with codecs.open(self.job_context.dataset_file_path, "r", file_encoding) as dataset: + # ignore lower/upper case + fields = next(dataset).lower() + fields_list = fields.strip().split(utils.CSV_SEP) + id_idx = fields_list.index(field) + for line in dataset: + if line.strip() == "": + continue + if len(line.strip().split(utils.CSV_SEP, id_idx + 1)) < id_idx + 1: + continue + print(line.strip().split(utils.CSV_SEP, id_idx + 1) + [id_idx], file=prepare_file) + prepare_file.close() + self.components.storage_client.upload_file(self.job_context.psi_prepare_path, + self.job_context.job_id + os.sep + JobContext.PSI_PREPARE_FILE) + utils.delete_file(self.job_context.psi_prepare_path) diff --git a/python/ppc_scheduler/workflow/worker/exit_worker.py b/python/ppc_scheduler/workflow/worker/exit_worker.py new file mode 100644 index 00000000..9cff596b --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/exit_worker.py @@ -0,0 +1,33 @@ +import os +import time + +from ppc_common.ppc_utils import utils +from ppc_scheduler.common import log_utils +from ppc_scheduler.workflow.common.worker_type import WorkerType +from ppc_scheduler.workflow.worker.worker import Worker + + +class ExitWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs): + log_utils.upload_job_log(self.components.storage_client, self.job_context.job_id) + self._save_workflow_view_file() + if self.worker_type == WorkerType.T_ON_FAILURE: + # notice job manager that this job has failed + raise Exception() + + def _save_workflow_view_file(self): + file = f"{self.job_context.workflow_view_path}.svg" + try_count = 10 + while try_count > 0: + if utils.file_exists(file): + break + time.sleep(1) + try_count -= 1 + + self.components.storage_client.upload_file(file, + self.job_context.job_id + os.sep + + self.job_context.workflow_view_path) diff --git a/python/ppc_scheduler/workflow/worker/model_worker.py b/python/ppc_scheduler/workflow/worker/model_worker.py new file mode 100644 index 00000000..e843bd08 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/model_worker.py @@ -0,0 +1,19 @@ +from ppc_scheduler.node.computing_node_client import ModelClient +from ppc_scheduler.workflow.worker.engine.model_engine import ModelWorkerEngine +from ppc_scheduler.workflow.worker.worker import Worker + + +class ModelWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs): + node_endpoint = self.node_manager.get_node(self.worker_type) + model_client = ModelClient(self.components.logger(), node_endpoint) + model_engine = ModelWorkerEngine(model_client, self.worker_type, self.components, self.job_context) + try: + outputs = model_engine.run() + return outputs + finally: + self.node_manager.release_node(node_endpoint, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/mpc_worker.py b/python/ppc_scheduler/workflow/worker/mpc_worker.py new file mode 100644 index 00000000..c5a98c5f --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/mpc_worker.py @@ -0,0 +1,19 @@ +from ppc_scheduler.node.computing_node_client import MpcClient +from ppc_scheduler.workflow.worker.engine.mpc_engine import MpcWorkerEngine +from ppc_scheduler.workflow.worker.worker import Worker + + +class MpcWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs) -> list: + node_endpoint = self.node_manager.get_node(self.worker_type) + mpc_client = MpcClient(node_endpoint) + mpc_engine = MpcWorkerEngine(mpc_client, self.worker_type, self.components, self.job_context) + try: + outputs = mpc_engine.run() + return outputs + finally: + self.node_manager.release_node(node_endpoint, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/psi_worker.py b/python/ppc_scheduler/workflow/worker/psi_worker.py new file mode 100644 index 00000000..e8f18cca --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/psi_worker.py @@ -0,0 +1,19 @@ +from ppc_scheduler.node.computing_node_client.psi_node_client import PsiClient +from ppc_scheduler.workflow.worker.engine.psi_engine import PsiWorkerEngine +from ppc_scheduler.workflow.worker.worker import Worker + + +class PsiWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs) -> list: + node_endpoint = self.node_manager.get_node(self.worker_type) + psi_client = PsiClient(self.components.logger(), node_endpoint) + psi_engine = PsiWorkerEngine(psi_client, self.worker_type, self.components, self.job_context) + try: + outputs = psi_engine.run() + return outputs + finally: + self.node_manager.release_node(node_endpoint, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/python_worker.py b/python/ppc_scheduler/workflow/worker/python_worker.py new file mode 100644 index 00000000..a6a36013 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/python_worker.py @@ -0,0 +1,10 @@ +from ppc_scheduler.workflow.worker.worker import Worker + + +class PythonWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs): + ... diff --git a/python/ppc_scheduler/workflow/worker/shell_worker.py b/python/ppc_scheduler/workflow/worker/shell_worker.py new file mode 100644 index 00000000..832b8794 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/shell_worker.py @@ -0,0 +1,10 @@ +from ppc_scheduler.workflow.worker.worker import Worker + + +class ShellWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + + def engine_run(self, worker_inputs): + ... diff --git a/python/ppc_scheduler/workflow/worker/worker.py b/python/ppc_scheduler/workflow/worker/worker.py new file mode 100644 index 00000000..b9823ce2 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/worker.py @@ -0,0 +1,96 @@ +import time + +from func_timeout import FunctionTimedOut +from prefect import Task +from prefect.engine import signals + +from ppc_scheduler.database import job_worker_mapper +from ppc_scheduler.node.node_manager import ComputingNodeManager +from ppc_scheduler.workflow.common import codec, flow_utils +from ppc_scheduler.workflow.common.worker_status import WorkerStatus +from ppc_scheduler.workflow.common.worker_type import WorkerType + + +class Worker(Task): + def __init__(self, components, job_context, worker_id, worker_type, retries=0, retry_delay_s=0, *args, **kwargs): + super().__init__(*args, **kwargs) + self.components = components + self.node_manager = ComputingNodeManager(components) + self.log = components.logger() + self.job_context = job_context + self.worker_id = worker_id + self.worker_type = worker_type + self.retries = retries + self.retry_delay_s = retry_delay_s + + def engine_run(self, worker_inputs) -> list: + # this func should be implemented by subclass + ... + + def run(self, worker_status, worker_inputs): + try: + # job is killed + if self.components.thread_event_manager.event_status(self.job_context.job_id): + self._write_failed_status(WorkerStatus.KILLED) + self.log.warn( + f"worker was killed, job_id: {self.job_context.job_id}, worker: {self.worker_id}") + raise signals.FAIL(message='killed!') + + if worker_status == WorkerStatus.SUCCESS: + # return outputs saved in db directly while current worker has been finished + return self._load_output_from_db() + + inputs = [] + if self.worker_type != WorkerType.T_ON_SUCCESS \ + and self.worker_type != WorkerType.T_ON_FAILURE: + inputs = flow_utils.to_origin_inputs(worker_inputs) + + outputs = self._try_run_task(inputs) + self._save_worker_result(outputs) + except FunctionTimedOut: + self._write_failed_status(WorkerStatus.TIMEOUT) + self.log.error( + f"worker was timeout, job_id: {self.job_context.job_id}, worker: {self.worker_id}") + raise signals.FAIL(message='timeout!') + except BaseException as be: + self._write_failed_status(WorkerStatus.FAILURE) + self.log.error(f"[OnError]job worker failed, job_id: {self.job_context.job_id}, worker: {self.worker_id}") + self.log.exception(be) + raise signals.FAIL(message='failed!') + + def _try_run_task(self, inputs): + self.log.info(f"job_id: {self.job_context.job_id}, worker: {self.worker_id}, inputs: {inputs}") + # parse inputs for worker + if self.retries: + attempt = 0 + while attempt <= self.retries: + try: + outputs = self.engine_run(inputs) + return outputs + except Exception as e: + attempt += 1 + if attempt > self.retries: + self.log.warn( + f"worker failed after {self.retries} attempts, " + f"job_id: {self.job_context.job_id}, worker: {self.worker_id}") + raise e + else: + time.sleep(self.retry_delay_s) + else: + outputs = self.engine_run(inputs) + self.log.info(f"job_id: {self.job_context.job_id}, worker: {self.worker_id}, outputs: {outputs}") + return outputs + + def _load_output_from_db(self): + with self.components.create_sql_session() as session: + worker_record = job_worker_mapper.query_job_worker(session, self.job_context.job_id, self.worker_id) + return codec.deserialize_worker_outputs(worker_record.outputs) + + def _save_worker_result(self, outputs): + with self.components.create_sql_session() as session: + job_worker_mapper.update_job_worker(session, self.job_context.job_id, self.worker_id, + WorkerStatus.SUCCESS, outputs) + + def _write_failed_status(self, status): + with self.components.create_sql_session() as session: + job_worker_mapper.update_job_worker(session, self.job_context.job_id, self.worker_id, status, []) diff --git a/python/ppc_scheduler/workflow/worker/worker_factory.py b/python/ppc_scheduler/workflow/worker/worker_factory.py new file mode 100644 index 00000000..016ce9b4 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/worker_factory.py @@ -0,0 +1,32 @@ +from ppc_scheduler.common.global_context import components +from ppc_scheduler.workflow.common.worker_type import WorkerType +from ppc_scheduler.workflow.worker.api_worker import ApiWorker +from ppc_scheduler.workflow.worker.exit_worker import ExitWorker +from ppc_scheduler.workflow.worker.model_worker import ModelWorker +from ppc_scheduler.workflow.worker.mpc_worker import MpcWorker +from ppc_scheduler.workflow.worker.psi_worker import PsiWorker +from ppc_scheduler.workflow.worker.python_worker import PythonWorker +from ppc_scheduler.workflow.worker.shell_worker import ShellWorker + + +class WorkerFactory: + + @staticmethod + def build_worker(job_context, worker_id, worker_type): + if worker_type == WorkerType.T_API: + return ApiWorker(components, job_context, worker_id, worker_type) + elif worker_type == WorkerType.T_PYTHON: + return PythonWorker(components, job_context, worker_id, worker_type) + elif worker_type == WorkerType.T_SHELL: + return ShellWorker(components, job_context, worker_id, worker_type) + elif worker_type == WorkerType.T_PSI: + return PsiWorker(components, job_context, worker_id, worker_type) + elif worker_type == WorkerType.T_MPC: + return MpcWorker(components, job_context, worker_id, worker_type) + elif worker_type == WorkerType.T_PREPROCESSING or \ + worker_type == WorkerType.T_FEATURE_ENGINEERING or \ + worker_type == WorkerType.T_TRAINING or \ + worker_type == WorkerType.T_PREDICTION: + return ModelWorker(components, job_context, worker_id, worker_type) + else: + return ExitWorker(components, job_context, worker_id, worker_type) diff --git a/python/requirements.txt b/python/requirements.txt index 14e6b826..e464227f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -24,9 +24,8 @@ promise~=2.3 # protobuf>=4.21.6,<5.0dev protobuf>=5.27.1 pycryptodome==3.9.9 -PyJWT==2.4.0 -PyYAML==5.4.1 -# sha3==0.2.1 +pyjwt +pyyaml mysqlclient==2.1.0 waitress==2.1.2 sqlparse~=0.4.1 @@ -38,7 +37,7 @@ google~=3.0.0 paste~=3.5.0 func_timeout==4.3.0 cheroot==8.5.2 -prefect==0.14.15 +prefect==1.4.0 gmssl~=3.2.1 readerwriterlock~=1.0.4 jsoncomment~=0.2.3 @@ -54,12 +53,11 @@ networkx pydot snowland-smx numpy==1.23.1 -graphviz~=0.20.1 +graphviz grpcio==1.62.1 grpcio-tools==1.62.1 xlrd~=1.0.0 MarkupSafe>=2.1.1 -Werkzeug==2.3.8 urllib3==1.26.18 phe chardet From 55da76108d732c939674a3a2c90591666b008165 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 3 Sep 2024 19:23:39 +0800 Subject: [PATCH 018/120] add front implementation (#14) * add front implementation * use grpc protocol between front and gateway * add grpc --- cpp/CMakeLists.txt | 22 +- cpp/cmake/Dependencies.cmake | 1 + cpp/cmake/IncludeDirectories.cmake | 3 +- cpp/cmake/TargetSettings.cmake | 11 +- cpp/libinitializer/Initializer.cpp | 2 +- cpp/libinitializer/ProtocolInitializer.h | 2 +- cpp/ppc-framework/front/Channel.h | 2 +- cpp/ppc-framework/front/FrontInterface.h | 18 +- cpp/ppc-framework/front/IFront.h | 80 +- cpp/ppc-framework/gateway/GatewayInterface.h | 81 -- cpp/ppc-framework/gateway/IGateway.h | 16 +- cpp/ppc-framework/protocol/INodeInfo.h | 32 +- cpp/ppc-framework/protocol/Message.h | 66 +- cpp/ppc-framework/protocol/MessagePayload.h | 18 + cpp/ppc-framework/protocol/PPCMessageFace.h | 4 +- cpp/ppc-front/ppc-front/CallbackManager.cpp | 158 ++++ cpp/ppc-front/ppc-front/CallbackManager.h | 113 +++ cpp/ppc-front/ppc-front/Front.cpp | 339 +------- cpp/ppc-front/ppc-front/Front.h | 164 +--- cpp/ppc-front/ppc-front/FrontImpl.cpp | 256 ++++++ cpp/ppc-front/ppc-front/FrontImpl.h | 183 +++++ cpp/ppc-front/ppc-front/PPCChannel.h | 4 +- cpp/ppc-front/ppc-front/PPCChannelManager.cpp | 2 - .../test/unittests/PPCChannelTest.cpp | 2 +- cpp/ppc-gateway/CMakeLists.txt | 4 +- cpp/ppc-gateway/ppc-gateway/Common.h | 13 +- .../ppc-gateway/FrontNodeManager.cpp | 62 -- .../ppc-gateway/FrontNodeManager.h | 54 -- cpp/ppc-gateway/ppc-gateway/Gateway.cpp | 774 ------------------ cpp/ppc-gateway/ppc-gateway/Gateway.h | 210 ----- .../ppc-gateway/GatewayConfigContext.cpp | 13 +- .../ppc-gateway/GatewayConfigContext.h | 6 +- .../ppc-gateway/GatewayConfigLoader.cpp | 150 ++++ .../ppc-gateway/GatewayConfigLoader.h | 52 ++ .../ppc-gateway/GatewayFactory.cpp | 77 ++ cpp/ppc-gateway/ppc-gateway/GatewayFactory.h | 54 ++ cpp/ppc-gateway/ppc-gateway/ProTaskManager.h | 96 --- cpp/ppc-gateway/ppc-gateway/TaskManager.cpp | 113 --- cpp/ppc-gateway/ppc-gateway/TaskManager.h | 69 -- .../ppc-gateway/WebSocketService.cpp | 387 --------- .../ppc-gateway/WebSocketService.h | 147 ---- .../ppc-gateway/gateway/GatewayImpl.cpp | 21 +- .../ppc-gateway/gateway/GatewayImpl.h | 10 +- .../gateway/cache/MessageCache.cpp | 2 +- .../gateway/router/GatewayNodeInfo.h | 7 +- .../gateway/router/GatewayNodeInfoImpl.cpp | 44 +- .../gateway/router/GatewayNodeInfoImpl.h | 26 +- .../gateway/router/LocalRouter.cpp | 4 +- .../ppc-gateway/gateway/router/LocalRouter.h | 3 +- cpp/ppc-gateway/ppc-gateway/p2p/Service.h | 13 + .../ppc-gateway/p2p/router/RouterTableImpl.h | 4 +- cpp/ppc-gateway/test/demo/gateway_demo.cpp | 7 +- .../test/unittests/GatewayTest.cpp | 4 +- cpp/ppc-pir/src/OtPIRConfig.h | 4 +- cpp/ppc-pir/src/OtPIRImpl.cpp | 182 ++-- cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h | 1 - .../src/bs-ecdh-psi/BsEcdhPSIInterface.h | 1 - cpp/ppc-psi/src/bs-ecdh-psi/Common.h | 37 +- .../src/bs-ecdh-psi/protocol/Message.h | 3 +- cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h | 2 +- cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp | 4 +- cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h | 2 +- cpp/ppc-psi/src/cm2020-psi/Common.h | 1 - .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 2 +- .../src/cm2020-psi/core/CM2020PSIReceiver.h | 1 - .../src/cm2020-psi/core/CM2020PSISender.cpp | 4 +- .../src/ecdh-conn-psi/EcdhConnPSIConfig.h | 2 +- .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp | 2 +- .../src/ecdh-conn-psi/EcdhConnPSIImpl.h | 2 +- .../src/ecdh-multi-psi/EcdhMultiPSIConfig.h | 5 +- .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp | 2 +- .../src/ecdh-multi-psi/EcdhMultiPSIImpl.h | 4 +- .../EcdhMultiPSIMessageFactory.h | 2 +- cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp | 2 +- cpp/ppc-psi/src/labeled-psi/Common.h | 1 - .../src/labeled-psi/LabeledPSIConfig.h | 2 +- .../src/labeled-psi/LabeledPSIImpl.cpp | 6 +- cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h | 4 +- .../src/labeled-psi/core/LabeledPSIParams.h | 1 - .../labeled-psi/core/LabeledPSIReceiver.cpp | 2 +- .../src/labeled-psi/core/LabeledPSIReceiver.h | 2 +- .../src/labeled-psi/core/LabeledPSISender.cpp | 2 +- .../src/labeled-psi/core/LabeledPSISender.h | 2 +- .../src/labeled-psi/core/ResultPackage.h | 1 - cpp/ppc-psi/src/labeled-psi/core/SenderDB.h | 2 +- cpp/ppc-psi/src/psi-framework/TaskGuarder.h | 2 +- .../src/psi-framework/protocol/PSIMessage.cpp | 2 +- cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp | 4 +- .../src/ra2018-psi/protocol/RA2018Message.h | 2 +- cpp/ppc-rpc/src/Rpc.h | 2 +- cpp/ppc-rpc/src/RpcMemory.cpp | 16 - cpp/ppc-rpc/src/RpcMemory.h | 9 +- .../client/FrontServiceClient.h | 139 ---- .../client/GatewayServiceClient.h | 298 ------- .../ppc-tars-protocol/tars/Error.tars | 6 - .../ppc-tars-protocol/tars/FrontService.tars | 7 - .../tars/GatewayService.tars | 19 - .../ppc-tars-protocol/tars/NodeInfo.tars | 16 - .../ppc-tars-protocol/tars/TaskInfo.tars | 7 - cpp/ppc-tars-protocol/test/CMakeLists.txt | 13 - cpp/ppc-tars-protocol/test/SerializeTest.cpp | 70 -- cpp/ppc-tars-protocol/test/main.cpp | 2 - cpp/ppc-tools/src/config/PPCConfig.cpp | 72 +- cpp/ppc-tools/src/config/PPCConfig.h | 10 +- cpp/ppc-utilities/Utilities.h | 9 + cpp/test-utils/FakeFront.h | 2 +- cpp/test-utils/FakePPCMessage.h | 2 - cpp/vcpkg.json | 8 + cpp/wedpr-protocol/CMakeLists.txt | 4 + cpp/wedpr-protocol/grpc/CMakeLists.txt | 32 + cpp/wedpr-protocol/grpc/client/CMakeLists.txt | 3 + cpp/wedpr-protocol/grpc/client/Common.h | 23 + .../grpc/client/FrontClient.cpp | 51 ++ cpp/wedpr-protocol/grpc/client/FrontClient.h | 42 + .../grpc/client/GatewayClient.cpp | 41 + .../grpc/client/GatewayClient.h | 61 ++ cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 56 ++ cpp/wedpr-protocol/grpc/client/GrpcClient.h | 65 ++ cpp/wedpr-protocol/proto/pb/NodeInfo.proto | 19 + cpp/wedpr-protocol/proto/pb/Service.proto | 43 + .../proto}/tars/RouterTable.tars | 0 cpp/wedpr-protocol/protobuf/CMakeLists.txt | 27 + cpp/wedpr-protocol/protobuf/Common.h | 50 ++ .../protobuf}/NodeInfoImpl.cpp | 18 +- .../protobuf}/NodeInfoImpl.h | 35 +- .../protocol}/CMakeLists.txt | 0 .../protocol}/src/CMakeLists.txt | 0 .../protocol}/src/JsonTaskImpl.cpp | 0 .../protocol}/src/JsonTaskImpl.h | 0 .../protocol}/src/PPCMessage.cpp | 64 -- .../protocol}/src/PPCMessage.h | 36 +- .../protocol}/src/v1/MessageHeaderImpl.cpp | 8 + .../protocol}/src/v1/MessageHeaderImpl.h | 30 +- .../protocol}/src/v1/MessageImpl.cpp | 0 .../protocol}/src/v1/MessageImpl.h | 19 +- .../protocol}/src/v1/MessagePayloadImpl.cpp | 18 +- .../protocol}/src/v1/MessagePayloadImpl.h | 0 .../protocol}/tests/CMakeLists.txt | 0 .../protocol}/tests/PPCMessageTest.cpp | 0 .../protocol}/tests/TestTaskImpl.cpp | 0 .../protocol}/tests/main.cpp | 0 .../tars}/CMakeLists.txt | 17 +- .../tars}/Common.h | 95 --- .../tars}/TarsSerialize.h | 3 +- .../tars}/TarsServantProxyCallback.h | 0 .../tars/TarsStruct.cpp} | 9 +- .../tars}/TarsStruct.h | 9 +- 147 files changed, 2259 insertions(+), 3631 deletions(-) delete mode 100644 cpp/ppc-framework/gateway/GatewayInterface.h create mode 100644 cpp/ppc-front/ppc-front/CallbackManager.cpp create mode 100644 cpp/ppc-front/ppc-front/CallbackManager.h create mode 100644 cpp/ppc-front/ppc-front/FrontImpl.cpp create mode 100644 cpp/ppc-front/ppc-front/FrontImpl.h delete mode 100644 cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp delete mode 100644 cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h delete mode 100644 cpp/ppc-gateway/ppc-gateway/Gateway.cpp delete mode 100644 cpp/ppc-gateway/ppc-gateway/Gateway.h create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp create mode 100644 cpp/ppc-gateway/ppc-gateway/GatewayFactory.h delete mode 100644 cpp/ppc-gateway/ppc-gateway/ProTaskManager.h delete mode 100644 cpp/ppc-gateway/ppc-gateway/TaskManager.cpp delete mode 100644 cpp/ppc-gateway/ppc-gateway/TaskManager.h delete mode 100644 cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp delete mode 100644 cpp/ppc-gateway/ppc-gateway/WebSocketService.h delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars delete mode 100644 cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars delete mode 100644 cpp/ppc-tars-protocol/test/CMakeLists.txt delete mode 100644 cpp/ppc-tars-protocol/test/SerializeTest.cpp delete mode 100644 cpp/ppc-tars-protocol/test/main.cpp create mode 100644 cpp/wedpr-protocol/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/grpc/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/grpc/client/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/grpc/client/Common.h create mode 100644 cpp/wedpr-protocol/grpc/client/FrontClient.cpp create mode 100644 cpp/wedpr-protocol/grpc/client/FrontClient.h create mode 100644 cpp/wedpr-protocol/grpc/client/GatewayClient.cpp create mode 100644 cpp/wedpr-protocol/grpc/client/GatewayClient.h create mode 100644 cpp/wedpr-protocol/grpc/client/GrpcClient.cpp create mode 100644 cpp/wedpr-protocol/grpc/client/GrpcClient.h create mode 100644 cpp/wedpr-protocol/proto/pb/NodeInfo.proto create mode 100644 cpp/wedpr-protocol/proto/pb/Service.proto rename cpp/{ppc-tars-protocol/ppc-tars-protocol => wedpr-protocol/proto}/tars/RouterTable.tars (100%) create mode 100644 cpp/wedpr-protocol/protobuf/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/protobuf/Common.h rename cpp/{ppc-tars-protocol/ppc-tars-protocol/impl => wedpr-protocol/protobuf}/NodeInfoImpl.cpp (67%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol/impl => wedpr-protocol/protobuf}/NodeInfoImpl.h (69%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/CMakeLists.txt (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/CMakeLists.txt (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/JsonTaskImpl.cpp (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/JsonTaskImpl.h (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/PPCMessage.cpp (62%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/PPCMessage.h (82%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessageHeaderImpl.cpp (94%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessageHeaderImpl.h (72%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessageImpl.cpp (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessageImpl.h (81%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessagePayloadImpl.cpp (72%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/src/v1/MessagePayloadImpl.h (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/tests/CMakeLists.txt (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/tests/PPCMessageTest.cpp (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/tests/TestTaskImpl.cpp (100%) rename cpp/{ppc-protocol => wedpr-protocol/protocol}/tests/main.cpp (100%) rename cpp/{ppc-tars-protocol => wedpr-protocol/tars}/CMakeLists.txt (67%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol => wedpr-protocol/tars}/Common.h (79%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol => wedpr-protocol/tars}/TarsSerialize.h (97%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol => wedpr-protocol/tars}/TarsServantProxyCallback.h (100%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp => wedpr-protocol/tars/TarsStruct.cpp} (73%) rename cpp/{ppc-tars-protocol/ppc-tars-protocol => wedpr-protocol/tars}/TarsStruct.h (89%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 75586dc4..ca01df86 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -55,6 +55,7 @@ project(WeDPR-Component VERSION ${VERSION}) include(CompilerSettings) include(BuildInfoGenerator) +set(PROTO_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/generated/pb) include(IncludeDirectories) # the target settings @@ -67,17 +68,24 @@ set(JNI_SOURCE_PATH wedpr-component-sdk/bindings/java/src/main/c) set(SDK_SOURCE_LIST ppc-homo ppc-crypto-core wedpr-component-sdk ${JNI_SOURCE_PATH}) # Note: the udf depends on mysql, not enabled in the full node mode set(UDF_SOURCE_LIST ${SDK_SOURCE_LIST} ppc-udf) -set(ALL_SOURCE_LIST - ${SDK_SOURCE_LIST} ppc-crypto - libhelper libinitializer ppc-io ppc-protocol - ppc-gateway ppc-front ppc-tars-protocol - ppc-tools ppc-storage ppc-psi ppc-rpc - ppc-http ppc-mpc ppc-pir - ${CEM_SOURCE} ppc-main) set(CEM_SOURCE "") if(BUILD_CEM) set(CEM_SOURCE "ppc-cem") endif() +#set(ALL_SOURCE_LIST +# ${SDK_SOURCE_LIST} ppc-crypto +# libhelper libinitializer ppc-io ppc-protocol +# ppc-gateway ppc-front ppc-tars-protocol +# ppc-tools ppc-storage ppc-psi ppc-rpc +# ppc-http ppc-mpc ppc-pir +# ${CEM_SOURCE} ppc-main) + +set(ALL_SOURCE_LIST + ${SDK_SOURCE_LIST} ppc-crypto + libhelper ppc-io wedpr-protocol + ppc-gateway ppc-front + ppc-tools ppc-storage ppc-psi ppc-rpc + ppc-http ppc-mpc ppc-pir ${CEM_SOURCE}) if(BUILD_WEDPR_TOOLKIT) # fetch the python dependencies diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake index ff1000b4..4a2183ca 100644 --- a/cpp/cmake/Dependencies.cmake +++ b/cpp/cmake/Dependencies.cmake @@ -31,6 +31,7 @@ if(BUILD_ALL) find_package(SEAL REQUIRED) find_package(Kuku REQUIRED) + find_package(gRPC REQUIRED) # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 include(ProjectAPSI) diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index d6be1b25..b2f9e8c8 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -1,10 +1,11 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_BINARY_DIR}/generated/) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-protocol) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-sdk) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-tars-protocol) +include_directories(${PROTO_OUTPUT_PATH}) set(VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include") include_directories(${VCPKG_INCLUDE_PATH}) \ No newline at end of file diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 7f29bb81..73882793 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -17,8 +17,15 @@ set(TOOLS_TARGET "ppc-tools") # ppc-protocol set(PROTOCOL_TARGET "ppc-protocol") -# ppc-tars-protocol -set(TARS_PROTOCOL_TARGET "ppc-protocol-tars") +# wedpr-protocol/tars +set(TARS_PROTOCOL_TARGET "wedpr-tars-protocol") + +# wedpr-protocol/protobuf +set(PB_PROTOCOL_TARGET "wedpr-pb-protocol") + +# wedpr-protocol/grpc-client +set(SERVICE_CLIENT_TARGET "service-client") +set(SERVICE_CLIENT_PB_TARGET "service-client-pb") # ppc-front SET(FRONT_TARGET "ppc-front") diff --git a/cpp/libinitializer/Initializer.cpp b/cpp/libinitializer/Initializer.cpp index cff5c1a3..99465a3a 100644 --- a/cpp/libinitializer/Initializer.cpp +++ b/cpp/libinitializer/Initializer.cpp @@ -25,9 +25,9 @@ #include "ppc-framework/protocol/Protocol.h" #include "ppc-pir/src/OtPIRFactory.h" #include "ppc-pir/src/OtPIRImpl.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h" #include "ppc-psi/src/cm2020-psi/CM2020PSIFactory.h" +#include "protocol/src/PPCMessage.h" #if 0 //TODO: optimize here #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h" diff --git a/cpp/libinitializer/ProtocolInitializer.h b/cpp/libinitializer/ProtocolInitializer.h index 4d9546d4..56200e5a 100644 --- a/cpp/libinitializer/ProtocolInitializer.h +++ b/cpp/libinitializer/ProtocolInitializer.h @@ -28,10 +28,10 @@ #include "ppc-framework/protocol/GlobalConfig.h" #include "ppc-framework/protocol/PPCMessageFace.h" #include "ppc-io/src/DataResourceLoaderImpl.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-storage/src/FileStorageFactoryImpl.h" #include "ppc-storage/src/SQLStorageFactoryImpl.h" #include "ppc-tools/src/config/PPCConfig.h" +#include "protocol/src/PPCMessage.h" #include #include diff --git a/cpp/ppc-framework/front/Channel.h b/cpp/ppc-framework/front/Channel.h index 784a06fa..c344578e 100644 --- a/cpp/ppc-framework/front/Channel.h +++ b/cpp/ppc-framework/front/Channel.h @@ -46,7 +46,7 @@ class Channel * @brief notice task info to gateway by front * @param _taskInfo the latest task information */ - virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; + virtual bcos::Error::Ptr notifyTaskInfo(std::string const& taskID) = 0; /** * @brief: send message diff --git a/cpp/ppc-framework/front/FrontInterface.h b/cpp/ppc-framework/front/FrontInterface.h index 41e491d7..e7648725 100644 --- a/cpp/ppc-framework/front/FrontInterface.h +++ b/cpp/ppc-framework/front/FrontInterface.h @@ -44,20 +44,6 @@ class FrontInterface using Ptr = std::shared_ptr; FrontInterface() = default; virtual ~FrontInterface() {} - /** - * @brief: start/stop service - */ - virtual void start() = 0; - virtual void stop() = 0; - - /** - * @brief: receive message from gateway, call by gateway - * @param _message: received ppc message - * @return void - */ - virtual void onReceiveMessage( - front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) = 0; - /** * @brief: send message to other party by gateway @@ -78,11 +64,11 @@ class FrontInterface * @brief notice task info to gateway * @param _taskInfo the latest task information */ - virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; + virtual bcos::Error::Ptr notifyTaskInfo(std::string const& taskID) = 0; // erase the task-info when task finished virtual bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) = 0; - + // get the agencyList from the gateway virtual void asyncGetAgencyList(GetAgencyListCallback _callback) = 0; diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index be850b65..5de205f3 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -19,32 +19,47 @@ */ #pragma once #include "FrontConfig.h" +#include "ppc-framework/protocol/INodeInfo.h" #include "ppc-framework/protocol/Message.h" #include "ppc-framework/protocol/RouteType.h" #include namespace ppc::front { -class IFront +class IFrontClient +{ +public: + using Ptr = std::shared_ptr; + IFrontClient() = default; + virtual ~IFrontClient() = default; + /** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ + virtual void onReceiveMessage( + ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) = 0; +}; +class IFront : public virtual IFrontClient { public: using Ptr = std::shared_ptr; IFront() = default; - virtual ~IFront() = default; + ~IFront() override = default; /** * @brief start the IFront * * @param front the IFront to start */ - virtual void start() const = 0; + virtual void start() = 0; /** * @brief stop the IFront * * @param front the IFront to stop */ - virtual void stop() const = 0; + virtual void stop() = 0; /** * @@ -53,38 +68,61 @@ class IFront * @param callback the callback called when receive specified topic */ virtual void registerTopicHandler( - std::string const& topic, ppc::protocol::MessageCallback callback) = 0; + std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) = 0; /** * @brief async send message * * @param routeType the route type - * @param topic the topic - * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * @param routeInfo the route info, include + * - topic the topic + * - dstInst the dst agency(must set when 'route by agency' and 'route by * component') - * @param dstNodeID the dst nodeID(must set when 'route by nodeID') - * @param componentType the componentType(must set when 'route by component') + * - dstNodeID the dst nodeID(must set when 'route by nodeID') + * - componentType the componentType(must set when 'route by component') * @param payload the payload to send * @param seq the message seq * @param timeout timeout * @param callback callback */ - virtual void asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, - std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, - bcos::bytes&& payload, int seq, long timeout, ppc::protocol::MessageCallback callback) = 0; + virtual void asyncSendMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, + long timeout, ppc::protocol::ReceiveMsgFunc errorCallback, + ppc::protocol::MessageCallback callback) = 0; // the sync interface for async_send_message - virtual ppc::protocol::Message::Ptr push(ppc::protocol::RouteType routeType, std::string topic, - std::string dstInst, std::string dstNodeID, std::string const& componentType, - bcos::bytes&& payload, int seq, long timeout) = 0; + virtual bcos::Error::Ptr push(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, + long timeout) = 0; + + virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; + virtual ppc::protocol::Message::Ptr peek(std::string const& topic); + /** - * @brief: receive message from gateway, call by gateway - * @param _message: received ppc message - * @return void + * @brief register the nodeInfo to the gateway + * @param nodeInfo the nodeInfo */ - virtual void onReceiveMessage( - ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) = 0; + virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; + + /** + * @brief unRegister the nodeInfo to the gateway + */ + virtual void unRegisterNodeInfo() = 0; + + /** + * @brief register the topic + * + * @param topic the topic to register + */ + virtual void registerTopic(std::string const& topic) = 0; + + /** + * @brief unRegister the topic + * + * @param topic the topic to unregister + */ + virtual void unRegisterTopic(std::string const& topic) = 0; }; class IFrontBuilder @@ -101,6 +139,6 @@ class IFrontBuilder * @return IFront::Ptr he created Front */ virtual IFront::Ptr build(ppc::front::FrontConfig::Ptr config) const = 0; - virtual IFront::Ptr buildClient(std::string endPoint) const = 0; + virtual IFrontClient::Ptr buildClient(std::string endPoint) const = 0; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/gateway/GatewayInterface.h b/cpp/ppc-framework/gateway/GatewayInterface.h deleted file mode 100644 index 52ceda88..00000000 --- a/cpp/ppc-framework/gateway/GatewayInterface.h +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @brief interface for Gateway module - * @file GatewayInterface.h - * @author: shawnhe - * @date 2022-10-19 - */ -#pragma once -#include "ppc-framework/front/FrontInterface.h" -#include "ppc-framework/protocol/PPCMessageFace.h" -#include "ppc-framework/protocol/Task.h" -#include - -namespace ppc -{ -namespace gateway -{ -using ErrorCallbackFunc = std::function; - -/** - * @brief: A list of interfaces provided by the gateway which are called by the front service. - */ -class GatewayInterface -{ -public: - using Ptr = std::shared_ptr; - GatewayInterface() = default; - virtual ~GatewayInterface() {} - - /** - * @brief: start/stop service - */ - virtual void start() = 0; - virtual void stop() = 0; - -public: - /** - * @brief: send message to gateway - * @param _agencyID: agency ID of receiver - * @param _message: ppc message data - * @param _callback: callback - * @return void - */ - virtual void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, - ErrorCallbackFunc _callback) = 0; - - - /** - * @brief notice task info to gateway - * @param _taskInfo the latest task information - */ - virtual bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) = 0; - - // erase the task-info when task finished - virtual bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) = 0; - - // register the gateway info of other parties - virtual bcos::Error::Ptr registerGateway( - const std::vector& _gatewayList) = 0; - - virtual void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front) = 0; - virtual void unregisterFront(std::string const&) {} - - virtual void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) = 0; -}; - -} // namespace gateway -} // namespace ppc diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index 9af59dba..d65171c0 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -57,14 +57,16 @@ class IGateway * @param timeout timeout * @param callback callback */ - virtual void asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, - std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, - bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) = 0; + virtual void asyncSendMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, + long timeout, ppc::protocol::ReceiveMsgFunc callback) = 0; - virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo); - virtual void unRegisterNodeInfo(bcos::bytesConstRef nodeID); - virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic); - virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic); + virtual void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) = 0; + virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; + virtual void unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; + virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) = 0; + virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) = 0; }; } // namespace ppc::gateway diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 859d6ac6..2d4f8e19 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -18,9 +18,16 @@ * @date 2024-08-26 */ #pragma once -#include "ppc-framework/front/IFront.h" +#include #include +#include +#include +#include +namespace ppc::front +{ +class IFrontClient; +} namespace ppc::protocol { // the node information @@ -35,14 +42,14 @@ class INodeInfo virtual bcos::bytesConstRef nodeID() const = 0; // components - virtual void setComponents(std::vector const& components) = 0; + virtual void setComponents(std::set const& components) = 0; virtual std::set const& components() const = 0; virtual void encode(bcos::bytes& data) const = 0; virtual void decode(bcos::bytesConstRef data) = 0; - virtual void setFront(ppc::front::IFront::Ptr&& front) = 0; - virtual ppc::front::IFront::Ptr const& getFront() const = 0; + virtual void setFront(std::shared_ptr&& front) = 0; + virtual std::shared_ptr const& getFront() const = 0; virtual bool equal(INodeInfo::Ptr const& info) { @@ -62,4 +69,21 @@ class INodeInfoFactory protected: bcos::bytes m_nodeID; }; + +inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo) +{ + if (!nodeInfo) + { + return "nullptr"; + } + std::ostringstream stringstream; + stringstream << LOG_KV("endPoint", nodeInfo->endPoint()); + std::string components = ""; + for (auto const& it : nodeInfo->components()) + { + components = components + it + ","; + } + stringstream << LOG_KV("components", components); + return stringstream.str(); +} } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index b318c1c0..95f7225e 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -19,9 +19,11 @@ */ #pragma once #include "../Common.h" +#include "MessagePayload.h" #include "RouteType.h" #include #include +#include #include #include #include @@ -59,12 +61,17 @@ class MessageOptionalHeader virtual void setTopic(std::string&& topic) { m_topic = std::move(topic); } virtual void setTopic(std::string const& topic) { m_topic = topic; } + virtual std::string srcInst() const { return m_srcInst; } + virtual void setSrcInst(std::string const& srcInst) { m_srcInst = srcInst; } + protected: std::string m_topic; // the componentType std::string m_componentType; // the source nodeID that send the message bcos::bytes m_srcNode; + // the source agency + std::string m_srcInst; // the target nodeID that should receive the message bcos::bytes m_dstNode; // the target agency that need receive the message @@ -126,6 +133,7 @@ class MessageHeader virtual uint16_t routeType() const = 0; virtual void setRouteType(ppc::protocol::RouteType type) = 0; + virtual bool hasOptionalField() const = 0; protected: // the msg version, used to support compatibility @@ -181,9 +189,18 @@ class Message : virtual public bcos::boostssl::MessageFace m_payload = std::move(_payload); } + void setFrontMessage(MessagePayload::Ptr frontMessage) + { + m_frontMessage = std::move(frontMessage); + } + + MessagePayload::Ptr const& frontMessage() const { return m_frontMessage; } + + protected: MessageHeader::Ptr m_header; std::shared_ptr m_payload; + MessagePayload::Ptr m_frontMessage; }; class MessageHeaderBuilder @@ -195,6 +212,7 @@ class MessageHeaderBuilder virtual MessageHeader::Ptr build(bcos::bytesConstRef _data) = 0; virtual MessageHeader::Ptr build() = 0; + virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) = 0; }; class MessageBuilder : public bcos::boostssl::MessageFaceFactory @@ -206,13 +224,43 @@ class MessageBuilder : public bcos::boostssl::MessageFaceFactory virtual Message::Ptr build() = 0; virtual Message::Ptr build(bcos::bytesConstRef buffer) = 0; - virtual Message::Ptr build(ppc::protocol::RouteType routeType, std::string const& topic, - std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, - bcos::bytes&& payload) = 0; + virtual Message::Ptr build(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) = 0; }; +class MessageOptionalHeaderBuilder +{ +public: + using Ptr = std::shared_ptr; + MessageOptionalHeaderBuilder() = default; + virtual ~MessageOptionalHeaderBuilder() = default; + + virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) = 0; + virtual MessageOptionalHeader::Ptr build() = 0; +}; + +inline std::string printOptionalField(MessageOptionalHeader::Ptr optionalHeader) +{ + if (!optionalHeader) + { + return "nullptr"; + } + std::ostringstream stringstream; + stringstream << LOG_KV("topic", optionalHeader->topic()) + << LOG_KV("componentType", optionalHeader->componentType()) + << LOG_KV("srcNode", *(bcos::toHexString(optionalHeader->srcNode()))) + << LOG_KV("dstNode", *(bcos::toHexString(optionalHeader->dstNode()))) + << LOG_KV("dstInst", optionalHeader->dstInst()); + + return stringstream.str(); +} + inline std::string printMessage(Message::Ptr const& _msg) { + if (!_msg) + { + return "nullptr"; + } std::ostringstream stringstream; stringstream << LOG_KV("from", _msg->header()->srcP2PNodeIDView()) << LOG_KV("to", _msg->header()->dstP2PNodeIDView()) @@ -221,11 +269,19 @@ inline std::string printMessage(Message::Ptr const& _msg) << LOG_KV("traceID", _msg->header()->traceID()) << LOG_KV("packetType", _msg->header()->packetType()) << LOG_KV("length", _msg->length()); + if (_msg->header()->hasOptionalField()) + { + stringstream << printOptionalField(_msg->header()->optionalField()); + } return stringstream.str(); } inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg) { + if (!_msg) + { + return "nullptr"; + } std::ostringstream stringstream; stringstream << LOG_KV("rsp", _msg->isRespPacket()) << LOG_KV("traceID", _msg->seq()) << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length()); @@ -233,9 +289,9 @@ inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg) } // function to send response -using SendResponseFunction = std::function; +using SendResponseFunction = std::function&& payload)>; using ReceiveMsgFunc = std::function; using MessageCallback = std::function; - +using MessageDispatcherCallback = std::function; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h index 14304a16..f5fbf9e3 100644 --- a/cpp/ppc-framework/protocol/MessagePayload.h +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -23,6 +23,10 @@ namespace ppc::protocol { +enum class FrontMsgExtFlag : uint16_t +{ + Response = 0x1 +}; class MessagePayload { public: @@ -46,12 +50,26 @@ class MessagePayload // the length virtual int64_t length() const { return m_length; } + // the traceID + virtual std::string const& traceID() const { return m_traceID; } + virtual void setTraceID(std::string const& traceID) { m_traceID = traceID; } + + virtual uint16_t ext() const { return m_ext; } + virtual void setExt(uint16_t ext) { m_ext = ext; } + + virtual void setRespPacket() { m_ext |= (uint16_t)FrontMsgExtFlag::Response; } + + virtual bool isRespPacket() { return m_ext &= (uint16_t)FrontMsgExtFlag::Response; } + protected: // the front payload version, used to support compatibility uint8_t m_version; // the seq uint16_t m_seq; + // the traceID + std::string m_traceID; bcos::bytes m_data; + uint16_t m_ext; int64_t mutable m_length; }; diff --git a/cpp/ppc-framework/protocol/PPCMessageFace.h b/cpp/ppc-framework/protocol/PPCMessageFace.h index 275a473d..f6de1791 100644 --- a/cpp/ppc-framework/protocol/PPCMessageFace.h +++ b/cpp/ppc-framework/protocol/PPCMessageFace.h @@ -21,6 +21,7 @@ #pragma once #include "Protocol.h" +#include "ppc-framework/protocol/Message.h" #include #include #include @@ -58,8 +59,6 @@ class PPCMessageFace virtual void setTaskID(std::string const&) = 0; virtual std::string const& sender() const = 0; virtual void setSender(std::string const&) = 0; - virtual uint16_t ext() const = 0; - virtual void setExt(uint16_t) = 0; virtual std::shared_ptr data() const = 0; virtual void setData(std::shared_ptr) = 0; virtual std::map header() = 0; @@ -87,6 +86,7 @@ class PPCMessageFaceFactory public: virtual ~PPCMessageFaceFactory() {} virtual PPCMessageFace::Ptr buildPPCMessage() = 0; + virtual PPCMessageFace::Ptr buildPPCMessage(ppc::protocol::Message::Ptr msg) = 0; virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesConstRef _data) = 0; virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesPointer _buffer) = 0; virtual PPCMessageFace::Ptr buildPPCMessage(uint8_t _taskType, uint8_t _algorithmType, diff --git a/cpp/ppc-front/ppc-front/CallbackManager.cpp b/cpp/ppc-front/ppc-front/CallbackManager.cpp new file mode 100644 index 00000000..51031f5b --- /dev/null +++ b/cpp/ppc-front/ppc-front/CallbackManager.cpp @@ -0,0 +1,158 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CallbackManager.cpp + * @author: yujiechen + * @date 2024-08-30 + */ +#include "CallbackManager.h" +#include "Common.h" +#include "ppc-framework/protocol/Protocol.h" +#include + +using namespace bcos; +using namespace ppc; +using namespace ppc::front; +using namespace ppc::protocol; + +void CallbackManager::addCallback( + std::string const& traceID, long timeout, ppc::protocol::MessageCallback msgCallback) +{ + if (!msgCallback) + { + return; + } + auto callback = std::make_shared(msgCallback); + // set the timeout handler + if (timeout > 0) + { + callback->timeoutHandler = std::make_shared( + *m_ioService, boost::posix_time::milliseconds(timeout)); + auto self = weak_from_this(); + callback->timeoutHandler->async_wait([self, traceID](const boost::system::error_code& e) { + auto front = self.lock(); + if (front) + { + front->onMessageTimeout(e, traceID); + } + }); + } + // insert the callback into m_traceID2Callback + WriteGuard l(x_traceID2Callback); + m_traceID2Callback.insert(std::make_pair(traceID, callback)); +} + +Callback::Ptr CallbackManager::pop(std::string const& traceID) +{ + bcos::UpgradableGuard l(x_traceID2Callback); + auto it = m_traceID2Callback.find(traceID); + if (it == m_traceID2Callback.end()) + { + return nullptr; + } + auto callback = it->second; + m_traceID2Callback.erase(it); + return callback; +} + +void CallbackManager::onMessageTimeout( + const boost::system::error_code& e, std::string const& traceID) +{ + // the timer has been canceled + if (e) + { + return; + } + try + { + auto callback = pop(traceID); + if (!callback) + { + return; + } + if (callback->timeoutHandler) + { + callback->timeoutHandler->cancel(); + } + auto errorMsg = "send message with traceID=" + traceID + " timeout"; + auto error = std::make_shared(PPCRetCode::TIMEOUT, errorMsg); + m_threadPool->enqueue( + [callback, error]() { callback->msgCallback(error, nullptr, nullptr); }); + FRONT_LOG(WARNING) << LOG_BADGE("onMessageTimeout") << LOG_KV("traceID", traceID); + } + catch (std::exception& e) + { + FRONT_LOG(ERROR) << "onMessageTimeout" << LOG_KV("traceID", traceID) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + + +void CallbackManager::handleCallback(bcos::Error::Ptr const& error, std::string const& traceID, + Message::Ptr message, SendResponseFunction resFunc) +{ + auto callback = pop(traceID); + if (!callback) + { + return; + } + if (callback->timeoutHandler) + { + callback->timeoutHandler->cancel(); + } + if (!message) + { + return; + } + m_threadPool->enqueue( + [error, callback, message, resFunc] { callback->msgCallback(error, message, resFunc); }); +} + + +void CallbackManager::registerTopicHandler( + std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) +{ + bcos::WriteGuard l(x_topicHandlers); + m_topicHandlers.insert(std::make_pair(topic, callback)); +} + +void CallbackManager::onReceiveMessage(std::string const& topic, Message::Ptr msg) +{ + MessageDispatcherCallback callback = nullptr; + { + bcos::ReadGuard l(x_topicHandlers); + auto it = m_topicHandlers.find(topic); + if (it == m_topicHandlers.end()) + { + FRONT_LOG(DEBUG) << LOG_DESC( + "onReceiveMessage: not find the handler, put into the buffer") + << LOG_KV("topic", topic); + addMsgCache(topic, msg); + return; + } + callback = it->second; + } + m_threadPool->enqueue([callback, msg]() { + try + { + callback(std::move(msg)); + } + catch (Exception e) + { + FRONT_LOG(WARNING) << LOG_DESC("onReceiveMessage: dispatcher exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/CallbackManager.h b/cpp/ppc-front/ppc-front/CallbackManager.h new file mode 100644 index 00000000..95894d52 --- /dev/null +++ b/cpp/ppc-front/ppc-front/CallbackManager.h @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file CallbackManager.h + * @author: yujiechen + * @date 2024-08-30 + */ +#pragma once +#include "ppc-framework/protocol/Message.h" +#include +#include +#include +#include +#include +#include +#define TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS 1 +#include +#include + +namespace ppc::front +{ +class Callback +{ +public: + using Ptr = std::shared_ptr; + Callback(ppc::protocol::MessageCallback callback) : msgCallback(std::move(callback)) {} + + ppc::protocol::MessageCallback msgCallback; + std::shared_ptr timeoutHandler; +}; +class CallbackManager : public std::enable_shared_from_this +{ +public: + using MsgQueueType = bcos::ConcurrentQueue; + using Ptr = std::shared_ptr; + CallbackManager( + bcos::ThreadPool::Ptr threadPool, std::shared_ptr ioService) + : m_threadPool(std::move(threadPool)), m_ioService(std::move(ioService)) + {} + virtual ~CallbackManager() = default; + + virtual void addCallback( + std::string const& traceID, long timeout, ppc::protocol::MessageCallback msgCallback); + + virtual Callback::Ptr pop(std::string const& traceID); + + virtual void handleCallback(bcos::Error::Ptr const& error, std::string const& traceID, + ppc::protocol::Message::Ptr message, ppc::protocol::SendResponseFunction resFunc); + + virtual void onReceiveMessage(std::string const& topic, ppc::protocol::Message::Ptr msg); + + virtual void registerTopicHandler( + std::string const& topic, ppc::protocol::MessageDispatcherCallback callback); + + virtual ppc::protocol::Message::Ptr pop(std::string const& topic, int timeoutMs) + { + auto it = m_msgCache.find(topic); + if (it == m_msgCache.end()) + { + return nullptr; + } + auto msgQueue = it->second; + if (msgQueue->empty()) + { + return nullptr; + } + auto result = msgQueue->tryPop(timeoutMs); + return result.second; + } + +private: + void onMessageTimeout(const boost::system::error_code& e, std::string const& traceID); + void addMsgCache(std::string const& topic, ppc::protocol::Message::Ptr msg) + { + auto it = m_msgCache.find(topic); + if (it == m_msgCache.end()) + { + m_msgCache.insert(std::make_pair(topic, std::make_shared())); + } + auto msgQueue = m_msgCache[topic]; + // push + msgQueue->push(std::move(msg)); + } + +private: + bcos::ThreadPool::Ptr m_threadPool; + std::shared_ptr m_ioService; + // traceID => callback + std::unordered_map m_traceID2Callback; + mutable bcos::SharedMutex x_traceID2Callback; + + // topic => messageDispatcherCallback + std::map m_topicHandlers; + mutable bcos::SharedMutex x_topicHandlers; + + // the messageCache for the message with no topic handler defined + uint64_t m_maxMsgCacheSize = 10000; + // TODO: check the queueSize + tbb::concurrent_unordered_map> m_msgCache; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/Front.cpp b/cpp/ppc-front/ppc-front/Front.cpp index 4fa711c0..23b379e0 100644 --- a/cpp/ppc-front/ppc-front/Front.cpp +++ b/cpp/ppc-front/ppc-front/Front.cpp @@ -17,349 +17,76 @@ * @author: shawnhe * @date 2022-10-20 */ - #include "Front.h" -#include -#include -#include -#include +using namespace ppc; using namespace bcos; -using namespace ppc::front; using namespace ppc::protocol; - -void Front::start() -{ - if (m_running) - { - FRONT_LOG(INFO) << LOG_DESC("Front has already been started"); - return; - } - m_running = true; - FRONT_LOG(INFO) << LOG_DESC("start the Front"); - m_thread = std::make_shared([&] { - bcos::pthread_setThreadName("front_io_service"); - while (m_running) - { - try - { - m_ioService->run(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if (m_running && m_ioService->stopped()) - { - m_ioService->restart(); - } - } - catch (std::exception& e) - { - FRONT_LOG(WARNING) - << LOG_DESC("Exception in Front Thread:") << boost::diagnostic_information(e); - } - } - FRONT_LOG(INFO) << "Front exit"; - }); -} - - -/** - * @brief: receive message from gateway, call by gateway - * @param _message: received ppc message - * @return void - */ -void Front::onReceiveMessage(PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) -{ - if (_callback) - { - if (m_threadPool) - { - m_threadPool->enqueue([_callback] { _callback(nullptr); }); - } - else - { - _callback(nullptr); - } - } - - FRONT_LOG(TRACE) << LOG_BADGE("onReceiveMessage") << printPPCMsg(_message); - // response package - if (_message->response()) - { - handleCallback(nullptr, _message->uuid(), _message, _message->sender()); - return; - } - - uint16_t type = ((uint16_t)_message->taskType() << 8) | _message->algorithmType(); - auto it = m_handlers.find(type); - if (it != m_handlers.end()) - { - if (m_threadPool) - { - auto handler = it->second; - m_threadPool->enqueue([handler, _message] { handler(_message); }); - } - else - { - it->second(std::move(_message)); - } - } - else - { - FRONT_LOG(WARNING) << LOG_BADGE("onReceiveMessage") << LOG_DESC("message handler not found") - << LOG_KV("taskType", unsigned(_message->taskType())) - << LOG_KV("algorithmType", unsigned(_message->algorithmType())) - << LOG_KV("messageType", unsigned(_message->messageType())) - << LOG_KV("seq", _message->seq()) << LOG_KV("taskID", _message->taskID()) - << LOG_KV("sender", _message->sender()); - } -} - - +using namespace ppc::front; /** * @brief: send message to other party by gateway * @param _agencyID: agency ID of receiver * @param _message: ppc message data + * @param _callback: callback called when the message sent successfully + * @param _respCallback: callback called when receive the response from peer * @return void */ -void Front::asyncSendMessage(const std::string& _agencyID, PPCMessageFace::Ptr _message, +void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) { - // generate uuid for the message - static thread_local auto uuid_gen = boost::uuids::basic_random_generator(); - std::string uuid = boost::uuids::to_string(uuid_gen()); - _message->setUuid(uuid); - // call gateway interface to send the message - _message->setSender(m_selfAgencyId); - auto taskID = _message->taskID(); - FRONT_LOG(TRACE) << LOG_BADGE("asyncSendMessage") - << LOG_KV("taskType", unsigned(_message->taskType())) - << LOG_KV("algorithmType", unsigned(_message->algorithmType())) - << LOG_KV("messageType", unsigned(_message->messageType())) - << LOG_KV("seq", _message->seq()) << LOG_KV("taskID", _message->taskID()) - << LOG_KV("receiver", _agencyID) << LOG_KV("uud", _message->uuid()); - // timeout logic - if (_respCallback) - { - auto callback = std::make_shared(_respCallback); - addCallback(uuid, callback); - if (_timeout > 0) - { - auto timeoutHandler = std::make_shared( - *m_ioService, boost::posix_time::milliseconds(_timeout)); - callback->timeoutHandler = timeoutHandler; - auto self = weak_from_this(); - timeoutHandler->async_wait( - [self, _agencyID, taskID, uuid](const boost::system::error_code& e) { - auto front = self.lock(); - if (front) - { - front->onMessageTimeout(e, _agencyID, taskID, uuid); - } - }); - } - } + auto routeInfo = m_front->routerInfoBuilder()->build(); + routeInfo->setDstInst(_agencyID); + routeInfo->setTopic(_message->taskID()); + bcos::bytes data; + _message->encode(data); auto self = weak_from_this(); - sendMessageToGateway(_agencyID, std::move(_message), uuid, false, - [self, uuid, _agencyID, taskID, _callback](const bcos::Error::Ptr& _error) { + m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_AGENCY, routeInfo, std::move(data), + _message->seq(), _timeout, _callback, + [self, _agencyID, _respCallback]( + Error::Ptr error, Message::Ptr msg, SendResponseFunction resFunc) { auto front = self.lock(); if (!front) { return; } - // send message to gateway error, try to handleCallback - if (_error && (_error->errorCode() != 0)) - { - front->handleCallback(_error, uuid, nullptr, _agencyID); - } - if (_callback) - { - _callback(_error); - } - }); -} - -void Front::handleCallback(bcos::Error::Ptr const& _error, std::string const& _uuid, - PPCMessageFace::Ptr _message, std::string const& _agencyID) -{ - auto callback = getAndEraseCallback(_uuid); - if (!callback) - { - return; - } - // cancel the timer - if (callback->timeoutHandler) - { - callback->timeoutHandler->cancel(); - } - if (!_message) - { - return; - } - auto self = weak_from_this(); - auto respFunc = [self, _agencyID, _uuid](PPCMessageFace::Ptr _resp) { - auto front = self.lock(); - if (!front) - { - return; - } - front->sendMessageToGateway(front->m_selfAgencyId, std::move(_resp), _uuid, true, - [_agencyID](const bcos::Error::Ptr& _error) { - if (!_error) + auto responseCallback = [resFunc](PPCMessageFace::Ptr msg) { + if (!msg) { return; } - FRONT_LOG(WARNING) - << LOG_DESC("asyncSendResponse message error") << LOG_KV("agency", _agencyID) - << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); - }); - }; - if (m_threadPool) - { - m_threadPool->enqueue([_error, callback, _message, _agencyID, respFunc] { - callback->callback(_error, _agencyID, _message, respFunc); + std::shared_ptr payload = std::make_shared(); + msg->encode(*payload); + resFunc(std::move(payload)); + }; + if (msg == nullptr) + { + _respCallback(error, _agencyID, nullptr, responseCallback); + } + // get the agencyID + _respCallback(error, msg->header()->optionalField()->srcInst(), + front->m_messageFactory->buildPPCMessage(msg), responseCallback); }); - } - else - { - callback->callback(_error, _agencyID, std::move(_message), std::move(respFunc)); - } -} - -void Front::sendMessageToGateway(std::string const& _agencyID, PPCMessageFace::Ptr _message, - std::string const& _uuid, bool _response, ErrorCallbackFunc _callback) -{ - _message->setSender(m_selfAgencyId); - _message->setUuid(_uuid); - if (_response) - { - _message->setResponse(); - } - m_gatewayInterface->asyncSendMessage(_agencyID, std::move(_message), std::move(_callback)); } -// send response +// send response when receiving message from given agencyID void Front::asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) -{ - FRONT_LOG(TRACE) << LOG_DESC("asyncSendResponse") << printPPCMsg(_message); - sendMessageToGateway(_agencyID, std::move(_message), _uuid, true, std::move(_callback)); -} - -void Front::onMessageTimeout(const boost::system::error_code& e, std::string const& _agencyID, - std::string const& _taskID, std::string const& _uuid) -{ - // the timer has been canceled - if (e) - { - return; - } - - try - { - auto callback = getAndEraseCallback(_uuid); - if (!callback) - { - return; - } - if (callback->timeoutHandler) - { - callback->timeoutHandler->cancel(); - } - auto errorMsg = "send message with uuid=" + _uuid + ", agency = " + _agencyID + - ", task = " + _taskID + " timeout"; - auto error = std::make_shared(PPCRetCode::TIMEOUT, errorMsg); - if (m_threadPool) - { - m_threadPool->enqueue([callback, _agencyID, error]() { - callback->callback(error, _agencyID, nullptr, nullptr); - }); - } - else - { - callback->callback(std::move(error), _agencyID, nullptr, nullptr); - } - FRONT_LOG(WARNING) << LOG_BADGE("onMessageTimeout") << LOG_KV("uuid", _uuid) - << LOG_KV("agency", _agencyID) << LOG_KV("task", _taskID); - } - catch (std::exception& e) - { - FRONT_LOG(ERROR) << "onMessageTimeout" << LOG_KV("uuid", _uuid) - << LOG_KV("error", boost::diagnostic_information(e)); - } -} - +{} /** * @brief notice task info to gateway * @param _taskInfo the latest task information */ -bcos::Error::Ptr Front::notifyTaskInfo(GatewayTaskInfo::Ptr _taskInfo) +bcos::Error::Ptr Front::notifyTaskInfo(std::string const& taskID) { - auto startT = bcos::utcSteadyTime(); - if (_taskInfo->serviceEndpoint.empty()) - { - _taskInfo->serviceEndpoint = m_selfEndPoint; - } - auto ret = m_gatewayInterface->notifyTaskInfo(_taskInfo); - FRONT_LOG(INFO) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", _taskInfo->taskID) - << LOG_KV("serviceEndpoint", _taskInfo->serviceEndpoint) - << LOG_KV("timecost", bcos::utcSteadyTime() - startT); - return ret; + m_front->registerTopic(taskID); } // erase the task-info when task finished bcos::Error::Ptr Front::eraseTaskInfo(std::string const& _taskID) { - auto startT = bcos::utcSteadyTime(); - auto ret = m_gatewayInterface->eraseTaskInfo(_taskID); - FRONT_LOG(INFO) << LOG_BADGE("eraseTaskInfo") << LOG_KV("taskID", _taskID) - << LOG_KV("timecost", bcos::utcSteadyTime() - startT); - return ret; + m_front->unRegisterTopic(_taskID); } // get the agencyList from the gateway -void Front::asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) -{ - FRONT_LOG(TRACE) << LOG_BADGE("asyncGetAgencyList"); - if (!m_gatewayInterface) - { - std::vector emptyAgencies; - _callback(std::make_shared( - -1, "asyncGetAgencyList failed for the gateway not been inited into front!"), - std::move(emptyAgencies)); - return; - } - m_gatewayInterface->asyncGetAgencyList(std::move(_callback)); -} - -void Front::stop() -{ - if (!m_running) - { - FRONT_LOG(INFO) << LOG_DESC("Front has already been stopped"); - return; - } - m_running = false; - if (m_ioService) - { - m_ioService->stop(); - } - if (m_thread) - { - // stop the thread - if (m_thread->get_id() != std::this_thread::get_id()) - { - m_thread->join(); - } - else - { - m_thread->detach(); - } - } -} - - -Front::Ptr FrontFactory::buildFront(std::shared_ptr _ioService) -{ - FRONT_LOG(INFO) << LOG_BADGE("buildFront") << LOG_KV("agencyID", m_selfAgencyId); - return std::make_shared(std::move(_ioService), m_selfAgencyId, m_threadPool); -} \ No newline at end of file +void Front::asyncGetAgencyList(GetAgencyListCallback _callback) {} diff --git a/cpp/ppc-front/ppc-front/Front.h b/cpp/ppc-front/ppc-front/Front.h index 0796b195..6394df73 100644 --- a/cpp/ppc-front/ppc-front/Front.h +++ b/cpp/ppc-front/ppc-front/Front.h @@ -19,73 +19,31 @@ */ #pragma once - -#include "Common.h" -#include "FrontService.h" +#include "FrontImpl.h" #include "ppc-framework/front/FrontInterface.h" -#include "ppc-framework/gateway/GatewayInterface.h" -#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" -#include "ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h" -#include -#include - - -#include +#include "ppc-framework/protocol/PPCMessageFace.h" -namespace ppc +namespace ppc::front { -namespace front -{ -struct Callback -{ - using Ptr = std::shared_ptr; - Callback(CallbackFunc _callback) : callback(std::move(_callback)) {} - std::shared_ptr timeoutHandler; - CallbackFunc callback; -}; -class Front : public front::FrontInterface, public std::enable_shared_from_this +class Front : public FrontInterface, public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - - Front(std::shared_ptr _ioService, std::string const& _selfAgencyId, - bcos::ThreadPool::Ptr _threadPool, std::string const& _selfEndPoint = "localhost") - : m_ioService(std::move(_ioService)), - m_selfAgencyId(_selfAgencyId), - m_threadPool(std::move(_threadPool)) - { - m_selfEndPoint = _selfEndPoint; - } - - Front(const Front&) = delete; - Front(Front&&) = delete; - - Front& operator=(const Front&) = delete; - Front& operator=(Front&&) = delete; - - virtual ~Front() override = default; - - void start() override; - void stop() override; - - /** - * @brief: receive message from gateway, call by gateway - * @param _message: received ppc message - * @return void - */ - void onReceiveMessage( - front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override; - + Front(FrontImpl::Ptr front) : m_front(std::move(front)) {} + ~Front() override {} /** * @brief: send message to other party by gateway * @param _agencyID: agency ID of receiver * @param _message: ppc message data + * @param _callback: callback called when the message sent successfully + * @param _respCallback: callback called when receive the response from peer * @return void */ - void asyncSendMessage(const std::string& _agencyID, PPCMessageFace::Ptr _message, + void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) override; + // send response when receiving message from given agencyID void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override; @@ -93,98 +51,38 @@ class Front : public front::FrontInterface, public std::enable_shared_from_this< * @brief notice task info to gateway * @param _taskInfo the latest task information */ - bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) override; + bcos::Error::Ptr notifyTaskInfo(std::string const& taskID) override; + // erase the task-info when task finished bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override; // get the agencyList from the gateway - void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override; + void asyncGetAgencyList(GetAgencyListCallback _callback) override; // register message handler for algorithm void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, std::function _handler) { uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; - m_handlers[type] = std::move(_handler); - } - - ppc::gateway::GatewayInterface::Ptr gatewayInterface() { return m_gatewayInterface; } - void setGatewayInterface(ppc::gateway::GatewayInterface::Ptr _gatewayInterface) - { - m_gatewayInterface = std::move(_gatewayInterface); + auto self = weak_from_this(); + m_front->registerTopicHandler( + std::to_string(type), [self, _handler](ppc::protocol::Message::Ptr msg) { + auto front = self.lock(); + if (!front) + { + return; + } + if (msg == nullptr) + { + _handler(nullptr); + return; + } + _handler(front->m_messageFactory->buildPPCMessage(msg)); + }); } - std::shared_ptr threadPool() const { return m_threadPool; } - const std::string& selfAgencyId() const { return m_selfAgencyId; } - // Note: the selfEndPoint must be setted before start the front - virtual void setSelfEndPoint(std::string const& _selfEndPoint) - { - FRONT_LOG(INFO) << LOG_DESC("setSelfEndPoint: ") << _selfEndPoint; - m_selfEndPoint = _selfEndPoint; - } - -private: - void addCallback(std::string const& _uuid, Callback::Ptr _callback) - { - bcos::WriteGuard l(x_uuidToCallback); - m_uuidToCallback[_uuid] = std::move(_callback); - } - - Callback::Ptr getAndEraseCallback(std::string const& _uuid) - { - bcos::UpgradableGuard l(x_uuidToCallback); - auto it = m_uuidToCallback.find(_uuid); - if (it != m_uuidToCallback.end()) - { - auto callback = it->second; - bcos::UpgradeGuard ul(l); - m_uuidToCallback.erase(it); - return callback; - } - return nullptr; - } - - void onMessageTimeout(const boost::system::error_code& e, std::string const& _agencyID, - std::string const& _taskID, std::string const& _uuid); - void handleCallback(bcos::Error::Ptr const& _error, std::string const& _uuid, - PPCMessageFace::Ptr _message, std::string const& _agencyID); - void sendMessageToGateway(std::string const& _agencyID, PPCMessageFace::Ptr _msg, - std::string const& _uuid, bool _response, ErrorCallbackFunc _callback); - private: - std::shared_ptr m_ioService; - std::string m_selfAgencyId; - bcos::ThreadPool::Ptr m_threadPool; - - // gatewayInterface - ppc::gateway::GatewayInterface::Ptr m_gatewayInterface; - std::unordered_map> m_handlers; - - // uuid->callback - std::unordered_map m_uuidToCallback; - bcos::SharedMutex x_uuidToCallback; - - bool m_running = false; - // the thread to run ioservice - std::shared_ptr m_thread; + FrontImpl::Ptr m_front; + ppc::front::PPCMessageFaceFactory::Ptr m_messageFactory; }; - -class FrontFactory -{ -public: - using Ptr = std::shared_ptr; - FrontFactory(std::string _selfAgencyId, std::shared_ptr _threadPool) - : m_selfAgencyId(std::move(_selfAgencyId)), m_threadPool(std::move(_threadPool)) - {} - -public: - Front::Ptr buildFront(std::shared_ptr _ioService); - -private: - std::string m_selfAgencyId; - // thread pool - std::shared_ptr m_threadPool; -}; - -} // namespace front -} // namespace ppc +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/FrontImpl.cpp b/cpp/ppc-front/ppc-front/FrontImpl.cpp new file mode 100644 index 00000000..467e4bf6 --- /dev/null +++ b/cpp/ppc-front/ppc-front/FrontImpl.cpp @@ -0,0 +1,256 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontImpl.cpp + * @author: yujiechen + * @date 2024-08-30 + */ +#include "FrontImpl.h" +#include "ppc-utilities/Utilities.h" + +using namespace bcos; +using namespace ppc; +using namespace ppc::front; +using namespace ppc::protocol; + +FrontImpl::FrontImpl(std::shared_ptr threadPool, + ppc::protocol::INodeInfo::Ptr nodeInfo, MessagePayloadBuilder::Ptr messageFactory, + ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, + ppc::gateway::IGateway::Ptr const& gateway, std::shared_ptr ioService) + : m_threadPool(std::move(threadPool)), + m_nodeInfo(std::move(nodeInfo)), + m_messageFactory(std::move(messageFactory)), + m_routerInfoBuilder(std::move(routerInfoBuilder)), + m_ioService(std::move(ioService)), + m_gatewayClient(gateway) +{ + m_nodeID = m_nodeInfo->nodeID().toBytes(); + m_callbackManager = std::make_shared(m_threadPool, ioService); +} + +/** + * @brief start the IFront + * + * @param front the IFront to start + */ +void FrontImpl::start() +{ + if (m_running) + { + FRONT_LOG(INFO) << LOG_DESC("The front has already been started"); + return; + } + m_running = true; + m_thread = std::make_shared([&] { + bcos::pthread_setThreadName("front_io_service"); + while (m_running) + { + try + { + m_ioService->run(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + if (m_running && m_ioService->stopped()) + { + m_ioService->restart(); + } + } + catch (std::exception& e) + { + FRONT_LOG(WARNING) + << LOG_DESC("Exception in Front Thread:") << boost::diagnostic_information(e); + } + } + FRONT_LOG(INFO) << "Front exit"; + }); +} +/** + * @brief stop the IFront + * + * @param front the IFront to stop + */ +void FrontImpl::stop() +{ + if (!m_running) + { + FRONT_LOG(INFO) << LOG_DESC("The front has already been stopped"); + return; + } + m_running = false; + if (m_ioService) + { + m_ioService->stop(); + } + if (m_thread) + { + // stop the thread + if (m_thread->get_id() != std::this_thread::get_id()) + { + m_thread->join(); + } + else + { + m_thread->detach(); + } + } +} + +/** + * @brief async send message + * + * @param routeType the route type + * @param routeInfo the route info, include + * - topic the topic + * - dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * - dstNodeID the dst nodeID(must set when 'route by nodeID') + * - componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ +void FrontImpl::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, + bcos::bytes&& payload, int seq, long timeout, ReceiveMsgFunc errorCallback, + MessageCallback callback) +{ + // generate the frontMessage + MessagePayload::Ptr frontMessage = m_messageFactory->build(); + auto traceID = ppc::generateUUID(); + frontMessage->setTraceID(traceID); + frontMessage->setSeq(seq); + frontMessage->setData(std::move(payload)); + m_callbackManager->addCallback(traceID, timeout, callback); + auto self = weak_from_this(); + // send the message to the gateway + asyncSendMessageToGateway(false, std::move(frontMessage), routeType, routeInfo, timeout, + [self, traceID, routeInfo, errorCallback](bcos::Error::Ptr error) { + auto front = self.lock(); + if (!front) + { + return; + } + // send success + if (error && error->errorCode() != 0) + { + // send failed + FRONT_LOG(ERROR) << LOG_DESC("asyncSendMessage failed") + << LOG_KV("routeInfo", printOptionalField(routeInfo)) + << LOG_KV("traceID", traceID) << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + // try to trigger the callback + front->handleCallback(error, traceID, nullptr); + } + // Note: be careful block here when use push + if (errorCallback) + { + errorCallback(error); + } + }); +} + +void FrontImpl::handleCallback( + bcos::Error::Ptr const& error, std::string const& traceID, Message::Ptr message) +{ + auto self = weak_from_this(); + m_callbackManager->handleCallback(error, traceID, std::move(message), + [self, message](std::shared_ptr&& payload) { + auto front = self.lock(); + if (!front) + { + return; + } + auto frontMessage = front->m_messageFactory->build(); + // set the traceID + frontMessage->setTraceID(message->header()->traceID()); + ///// populate the route info + auto routerInfo = front->m_routerInfoBuilder->build(message->header()->optionalField()); + // set the dstNodeID + routerInfo->setDstNode(message->header()->optionalField()->srcNode()); + // set the srcNodeID + routerInfo->setSrcNode(message->header()->optionalField()->dstNode()); + front->asyncSendMessageToGateway(true, std::move(frontMessage), + RouteType::ROUTE_THROUGH_NODEID, routerInfo, 0, + [routerInfo](bcos::Error::Ptr error) { + if (!error || error->errorCode() == 0) + { + return; + } + FRONT_LOG(WARNING) << LOG_DESC("send response message error") + << LOG_KV("routeInfo", printOptionalField(routerInfo)) + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + }); + }); +} + +void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::Ptr&& frontMessage, + RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, long timeout, + ReceiveMsgFunc callback) +{ + if (responsePacket) + { + frontMessage->setRespPacket(); + } + routeInfo->setSrcNode(m_nodeID); + auto payload = std::make_shared(); + frontMessage->encode(*payload); + m_gatewayClient->asyncSendMessage(routeType, routeInfo, std::move(*payload), timeout, callback); +} + + +/** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ +void FrontImpl::onReceiveMessage(Message::Ptr const& msg, ReceiveMsgFunc callback) +{ + try + { + // response to the gateway + if (callback) + { + m_threadPool->enqueue([callback] { callback(nullptr); }); + } + FRONT_LOG(TRACE) << LOG_BADGE("onReceiveMessage") << LOG_KV("msg", printMessage(msg)); + auto frontMessage = m_messageFactory->build(bcos::ref(*(msg->payload()))); + msg->setFrontMessage(frontMessage); + // the response packet, dispatcher by callback + if (frontMessage->isRespPacket()) + { + handleCallback(nullptr, msg->header()->traceID(), msg); + return; + } + // dispatcher by topic + m_callbackManager->onReceiveMessage(msg->header()->optionalField()->topic(), msg); + } + catch (Exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") + << LOG_KV("msg", printMessage(msg)) + << LOG_KV("error", boost::diagnostic_information(e)); + } +} + +// the sync interface for asyncSendMessage +bcos::Error::Ptr FrontImpl::push(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, + bcos::bytes&& payload, int seq, long timeout) +{ + auto promise = std::make_shared>(); + asyncSendMessage( + routeType, routeInfo, std::move(payload), seq, timeout, + [promise](bcos::Error::Ptr error) { promise->set_value(error); }, nullptr); + return promise->get_future().get(); +} \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/FrontImpl.h b/cpp/ppc-front/ppc-front/FrontImpl.h new file mode 100644 index 00000000..3156b6c7 --- /dev/null +++ b/cpp/ppc-front/ppc-front/FrontImpl.h @@ -0,0 +1,183 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontImpl.h + * @author: yujiechen + * @date 2024-08-30 + */ +#pragma once +#include "CallbackManager.h" +#include "Common.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/gateway/IGateway.h" +#include +#include +#include + +namespace ppc::front +{ +class FrontImpl : public IFront, public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + FrontImpl(std::shared_ptr threadPool, ppc::protocol::INodeInfo::Ptr nodeInfo, + ppc::protocol::MessagePayloadBuilder::Ptr messageFactory, + ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, + ppc::gateway::IGateway::Ptr const& gateway, + std::shared_ptr ioService); + ~FrontImpl() override = default; + + /** + * @brief start the IFront + * + * @param front the IFront to start + */ + void start() override; + /** + * @brief stop the IFront + * + * @param front the IFront to stop + */ + void stop() override; + + bcos::Error::Ptr push(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, + long timeout) override; + /** + * @brief async send message + * + * @param routeType the route type + * @param routeInfo the route info, include + * - topic the topic + * - dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * - dstNodeID the dst nodeID(must set when 'route by nodeID') + * - componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ + void asyncSendMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, + long timeout, ppc::protocol::ReceiveMsgFunc errorCallback, + ppc::protocol::MessageCallback callback) override; + + /** + * @brief: receive message from gateway, call by gateway + * @param _message: received ppc message + * @return void + */ + void onReceiveMessage( + ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) override; + + ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) override + { + return m_callbackManager->pop(topic, timeoutMs); + } + + ppc::protocol::Message::Ptr peek(std::string const& topic) override + { + return m_callbackManager->pop(topic, 0); + } + /** + * + * @param front the front object + * @param topic the topic + * @param callback the callback called when receive specified topic + */ + void registerTopicHandler( + std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) override + { + m_callbackManager->registerTopicHandler(topic, callback); + } + + /** + * @brief register the nodeInfo to the gateway + * @param nodeInfo the nodeInfo + */ + void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override + { + FRONT_LOG(INFO) << LOG_DESC("registerNodeInfo") + << LOG_KV("nodeInfo", printNodeInfo(m_nodeInfo)); + m_gatewayClient->registerNodeInfo(m_nodeInfo); + } + + /** + * @brief unRegister the nodeInfo to the gateway + */ + void unRegisterNodeInfo() override + { + FRONT_LOG(INFO) << LOG_DESC("unRegisterNodeInfo"); + m_gatewayClient->unRegisterNodeInfo(bcos::ref(m_nodeID)); + } + + /** + * @brief register the topic + * + * @param topic the topic to register + */ + void registerTopic(std::string const& topic) override + { + FRONT_LOG(INFO) << LOG_DESC("register topic: ") << topic; + m_gatewayClient->registerTopic(bcos::ref(m_nodeID), topic); + } + + /** + * @brief unRegister the topic + * + * @param topic the topic to unregister + */ + void unRegisterTopic(std::string const& topic) override + { + FRONT_LOG(INFO) << LOG_DESC("unregister topic: ") << topic; + m_gatewayClient->unRegisterTopic(bcos::ref(m_nodeID), topic); + } + + ppc::protocol::MessageOptionalHeaderBuilder::Ptr const routerInfoBuilder() const + { + return m_routerInfoBuilder; + } + ppc::protocol::MessagePayloadBuilder::Ptr const payloadFactory() const + { + return m_messageFactory; + } + +private: + void asyncSendMessageToGateway(bool responsePacket, + ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, long timeout, + ppc::protocol::ReceiveMsgFunc callback); + + void handleCallback(bcos::Error::Ptr const& error, std::string const& traceID, + ppc::protocol::Message::Ptr message); + +private: + bcos::bytes m_nodeID; + std::shared_ptr m_threadPool; + ppc::protocol::INodeInfo::Ptr m_nodeInfo; + ppc::protocol::MessagePayloadBuilder::Ptr m_messageFactory; + ppc::protocol::MessageOptionalHeaderBuilder::Ptr m_routerInfoBuilder; + + ppc::gateway::IGateway::Ptr m_gatewayClient; + std::shared_ptr m_ioService; + + CallbackManager::Ptr m_callbackManager; + + bool m_running = false; + // the thread to run ioservice + std::shared_ptr m_thread; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-front/ppc-front/PPCChannel.h b/cpp/ppc-front/ppc-front/PPCChannel.h index 8b8ec506..f881a915 100644 --- a/cpp/ppc-front/ppc-front/PPCChannel.h +++ b/cpp/ppc-front/ppc-front/PPCChannel.h @@ -52,9 +52,9 @@ class PPCChannel : public Channel, public std::enable_shared_from_thisnotifyTaskInfo(std::move(_taskInfo)); + return m_front->notifyTaskInfo(std::move(taskID)); }; /** diff --git a/cpp/ppc-front/ppc-front/PPCChannelManager.cpp b/cpp/ppc-front/ppc-front/PPCChannelManager.cpp index 897e902d..c42b88d3 100644 --- a/cpp/ppc-front/ppc-front/PPCChannelManager.cpp +++ b/cpp/ppc-front/ppc-front/PPCChannelManager.cpp @@ -41,7 +41,6 @@ void PPCChannelManager::registerMsgHandlerForChannel(uint8_t _taskType, uint8_t }); } - Channel::Ptr PPCChannelManager::buildChannelForTask(const std::string& _taskID) { FRONT_LOG(INFO) << LOG_BADGE("buildChannelForTask") << LOG_KV("taskID", _taskID); @@ -130,7 +129,6 @@ void PPCChannelManager::onMessageArrived(PPCMessageFace::Ptr _message) } } - void PPCChannelManager::removeHoldingMessages(const std::string& _taskID) { WriteGuard lock(x_message_channel); diff --git a/cpp/ppc-front/test/unittests/PPCChannelTest.cpp b/cpp/ppc-front/test/unittests/PPCChannelTest.cpp index b301bcfb..d0d2d6cf 100644 --- a/cpp/ppc-front/test/unittests/PPCChannelTest.cpp +++ b/cpp/ppc-front/test/unittests/PPCChannelTest.cpp @@ -20,7 +20,7 @@ #include "ppc-front/ppc-front/PPCChannel.h" #include "ppc-front/ppc-front/PPCChannelManager.h" -#include "ppc-protocol/src/PPCMessage.h" +#include "protocol/src/PPCMessage.h" #include #include #include diff --git a/cpp/ppc-gateway/CMakeLists.txt b/cpp/ppc-gateway/CMakeLists.txt index 730bbda6..05c438e2 100644 --- a/cpp/ppc-gateway/CMakeLists.txt +++ b/cpp/ppc-gateway/CMakeLists.txt @@ -10,7 +10,9 @@ file(GLOB_RECURSE SRCS ppc-gateway/*.cpp) find_package(tarscpp REQUIRED) add_library(${GATEWAY_TARGET} ${SRCS}) -target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} ${HTTP_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} tarscpp::tarsservant tarscpp::tarsutil TBB::tbb) +target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} + ${HTTP_TARGET} ${PROTOCOL_TARGET} + ${TARS_PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET} tarscpp::tarsservant tarscpp::tarsutil TBB::tbb) if (APPLE) # target_compile_options(${GATEWAY_TARGET} PRIVATE -faligned-allocation) diff --git a/cpp/ppc-gateway/ppc-gateway/Common.h b/cpp/ppc-gateway/ppc-gateway/Common.h index 02f1f2de..c6dc211c 100644 --- a/cpp/ppc-gateway/ppc-gateway/Common.h +++ b/cpp/ppc-gateway/ppc-gateway/Common.h @@ -32,18 +32,7 @@ namespace ppc::gateway { #define GATEWAY_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY]" -#define GATEWAY_WS_CLIENT_MODULE "m_gateway_websocket_client" -#define GATEWAY_WS_SERVER_MODULE "m_gateway_websocket_server" -#define GATEWAY_THREAD_POOL_MODULE "t_gateway" - -#define SEND_MESSAGE_TO_FRONT_SUCCESS "success" -#define SEND_MESSAGE_TO_FRONT_ERROR "error" -#define SEND_MESSAGE_TO_FRONT_TIMEOUT "timeout" - -#define SEND_MESSAGE_TO_FRONT_SUCCESS_CODE "E0000000000" -#define SEND_MESSAGE_TO_FRONT_ERROR_CODE "-1" - -//HTTP HEADER DEFINE +// HTTP HEADER DEFINE #define HEAD_TASK_ID "x-ptp-session-id" #define HEAD_ALGO_TYPE "x-ptp-algorithm-type" #define HEAD_TASK_TYPE "x-ptp-task-type" diff --git a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp deleted file mode 100644 index b4c2e5a9..00000000 --- a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontNodeManager.cpp - * @author: shawnhe - * @date 2022-10-23 - */ - -#include "FrontNodeManager.h" - -using namespace bcos; -using namespace ppc::gateway; -using namespace ppc::front; - -FrontInterface::Ptr FrontNodeManager::getFront(const std::string& _serviceEndpoint) -{ - bcos::ReadGuard lock(x_frontNodes); - auto it = m_frontNodes.find(_serviceEndpoint); - if (it != m_frontNodes.end()) - { - return it->second; - } - return nullptr; -} - -void FrontNodeManager::registerFront( - std::string const& _endPoint, front::FrontInterface::Ptr _front) -{ - bcos::UpgradableGuard l; - if (m_frontNodes.count(_endPoint)) - { - return; - } - bcos::UpgradeGuard ul(l); - m_frontNodes[_endPoint] = _front; - GATEWAY_LOG(INFO) << LOG_DESC("registerFront success") << LOG_KV("endPoint", _endPoint); -} - -void FrontNodeManager::unregisterFront(std::string const& _endPoint) -{ - bcos::UpgradableGuard l; - auto it = m_frontNodes.find(_endPoint); - if (it == m_frontNodes.end()) - { - return; - } - bcos::UpgradeGuard ul(l); - m_frontNodes.erase(it); - GATEWAY_LOG(INFO) << LOG_DESC("unregisterFront success") << LOG_KV("endPoint", _endPoint); -} diff --git a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h b/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h deleted file mode 100644 index d5bac0f1..00000000 --- a/cpp/ppc-gateway/ppc-gateway/FrontNodeManager.h +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontNodeManager.h - * @author: shawnhe - * @date 2022-10-23 - */ - -#pragma once -#include "Common.h" -#include "ppc-framework/front/FrontInterface.h" -#include "ppc-front/ppc-front/Front.h" -#include -#include - -namespace ppc::gateway -{ -class FrontNodeManager : public std::enable_shared_from_this -{ -public: - using Ptr = std::shared_ptr; - FrontNodeManager() = default; - virtual ~FrontNodeManager() = default; - - front::FrontInterface::Ptr getFront(const std::string& _serviceEndpoint); - virtual void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front); - - virtual void unregisterFront(std::string const& _endPoint); - - virtual std::unordered_map getAllFront() const - { - bcos::ReadGuard l(x_frontNodes); - return m_frontNodes; - } - -private: - // key: serviceEndpoint, value: FrontInterface - std::unordered_map m_frontNodes; - mutable bcos::SharedMutex x_frontNodes; -}; - -} // namespace ppc::gateway diff --git a/cpp/ppc-gateway/ppc-gateway/Gateway.cpp b/cpp/ppc-gateway/ppc-gateway/Gateway.cpp deleted file mode 100644 index f85f80f1..00000000 --- a/cpp/ppc-gateway/ppc-gateway/Gateway.cpp +++ /dev/null @@ -1,774 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file Gateway.cpp - * @author: shawnhe - * @date 2022-10-20 - */ - -#include "Gateway.h" -#include "ProTaskManager.h" -#include -#include -#include - -using namespace bcos; -using namespace bcos::boostssl; -using namespace bcos::boostssl::ws; -using namespace bcos::boostssl::context; -using namespace ppc; -using namespace ppc::gateway; -using namespace ppc::protocol; -using namespace ppc::front; - - -void Gateway::start() -{ - if (m_running) - { - GATEWAY_LOG(INFO) << LOG_DESC("Gateway already started"); - return; - } - m_running = true; - GATEWAY_LOG(INFO) << LOG_DESC("start the Gateway"); - // register handler when receiving message from other agencies - if (m_protocol == m_webSocketService->gatewayConfig() - ->config() - ->gatewayConfig() - .networkConfig.PROTOCOL_WEBSOCKET) - { - registerWebSocketMsgHandler(); - } - else if (m_protocol == m_webSocketService->gatewayConfig() - ->config() - ->gatewayConfig() - .networkConfig.PROTOCOL_HTTP) - { -#if 0 - // TODO: optimize here - registerUrlMsgHandler(); -#endif - } - m_webSocketService->start(); - m_gatewayThread = std::make_shared([&] { - bcos::pthread_setThreadName("gw_io_service"); - while (m_running.load()) - { - try - { - m_ioContext->run(); - m_ioService->run(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if (m_running) - { - if (m_ioService->stopped()) - { - m_ioService->restart(); - } - if (m_ioContext->stopped()) - { - m_ioContext->restart(); - } - } - } - catch (std::exception& e) - { - GATEWAY_LOG(WARNING) - << LOG_DESC("Exception in Gateway Thread:") << boost::diagnostic_information(e); - } - } - GATEWAY_LOG(INFO) << "Gateway exit"; - }); - GATEWAY_LOG(INFO) << LOG_BADGE("start the gateway end"); -} - - -void Gateway::stop() -{ - if (!m_running) - { - GATEWAY_LOG(INFO) << LOG_DESC("Gateway already stopped"); - return; - } - m_running = false; - GATEWAY_LOG(INFO) << LOG_DESC("stop the Gateway"); - if (m_webSocketService) - { - m_webSocketService->stop(); - } - if (m_ioService) - { - m_ioService->stop(); - } - if (m_ioContext) - { - m_ioContext->stop(); - } - // stop the gateway-thread - if (m_gatewayThread) - { - if (m_gatewayThread->get_id() != std::this_thread::get_id()) - { - m_gatewayThread->join(); - } - else - { - m_gatewayThread->detach(); - } - } - GATEWAY_LOG(INFO) << LOG_BADGE("stop the gateway end"); -} - -#if 0 -// TODO: optimize here -void Gateway::registerUrlMsgHandler() -{ - auto _url = m_webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.url; - m_webSocketService->httpServer()->registerUrlHandler( - _url, [self = weak_from_this()](bcos::boostssl::http::HttpRequest const& _httpReq, - ppc::http::RespUrlFunc _handler) { - auto gateway = self.lock(); - if (!gateway) - { - return; - } - std::string code = SEND_MESSAGE_TO_FRONT_SUCCESS_CODE; - std::string _resMsg = SEND_MESSAGE_TO_FRONT_SUCCESS; - try - { - // handle message - auto senderStr = _httpReq[HEAD_SENDER_ID].to_string(); - auto taskIdStr = _httpReq[HEAD_TASK_ID].to_string(); - auto algoTypeStr = _httpReq[HEAD_ALGO_TYPE].to_string(); - auto taskTypeStr = _httpReq[HEAD_TASK_TYPE].to_string(); - auto messageTypeStr = _httpReq[HEAD_MESSAGE_TYPE].to_string(); - auto seqStr = _httpReq[HEAD_SEQ].to_string(); - auto uuidStr = _httpReq[HEAD_UUID].to_string(); - bool isResponse = _httpReq[HEAD_IS_RESPONSE].to_string() == "1"; - auto reqBodyBytes = bcos::bytes(_httpReq.body().begin(), _httpReq.body().end()); - - GATEWAY_LOG(TRACE) << LOG_BADGE("registerUrlMsgHandler") - << LOG_KV("req size: ", reqBodyBytes.size()) - << LOG_KV("taskId: ", taskIdStr) - << LOG_KV("algorithmType: ", algoTypeStr) - << LOG_KV("taskType: ", taskTypeStr) - << LOG_KV("messageType: ", messageTypeStr) - << LOG_KV("UUID: ", uuidStr) << LOG_KV("seq: ", seqStr) - << LOG_KV("isResponse: ", isResponse); - - auto message = gateway->messageFactory()->buildPPCMessage(); - message->setData(std::make_shared(reqBodyBytes)); - message->setTaskID(taskIdStr); -#ifdef ENABLE_CONN - message->setAlgorithmType(5); - message->setTaskType(0); -#else - message->setAlgorithmType(0x00 + atoi(algoTypeStr.c_str())); - message->setTaskType(0x00 + atoi(taskTypeStr.c_str())); -#endif - message->setMessageType(0x00 + atoi(messageTypeStr.c_str())); - message->setSeq(0x00 + atoi(seqStr.c_str())); - message->setSender(senderStr); - message->setUuid(uuidStr); - if (isResponse) - { - message->setResponse(); - } - gateway->onMessageArrived(std::move(message)); - } - catch (std::exception& e) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") - << LOG_KV("exception", boost::diagnostic_information(e)); - code = SEND_MESSAGE_TO_FRONT_ERROR_CODE; - _resMsg = SEND_MESSAGE_TO_FRONT_ERROR; - } - - org::interconnection::link::TransportOutbound transportOutbound; - transportOutbound.set_allocated_message(new std::string(_resMsg)); - transportOutbound.set_allocated_code(new std::string(code)); - std::string responseStr; - transportOutbound.SerializeToString(&responseStr); - _handler(nullptr, std::move(bcos::bytes(responseStr.begin(), responseStr.end()))); - }); -} -#endif - -void Gateway::registerWebSocketMsgHandler() -{ - GATEWAY_LOG(INFO) << LOG_BADGE("registerWebSocketMsgHandler"); - m_webSocketService->webSocketServer()->registerMsgHandler( - 0, [self = weak_from_this()](const std::shared_ptr& _wsMessage, - const std::shared_ptr& _session) { - auto gateway = self.lock(); - if (!gateway) - { - return; - } - try - { - // prepare ack - auto ack = gateway->wsMessageFactory()->buildMessage(); - ack->setRespPacket(); - ack->setPacketType(_wsMessage->packetType()); - ack->setSeq(_wsMessage->seq()); - gateway->addAckCallback(_wsMessage->seq(), ack, _session); - - // handle message - auto payload = _wsMessage->payload(); - auto message = gateway->messageFactory()->buildPPCMessage(payload); - if (!message) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") - << LOG_DESC("decode ppc message error"); - gateway->sendAck(_wsMessage->seq(), SEND_MESSAGE_TO_FRONT_ERROR); - return; - } - - GATEWAY_LOG(TRACE) - << LOG_BADGE("onReceiveMsgFromOtherGateway") - << LOG_KV("taskType", unsigned(message->taskType())) - << LOG_KV("algorithmType", unsigned(message->algorithmType())) - << LOG_KV("messageType", unsigned(message->messageType())) - << LOG_KV("seq", message->seq()) << LOG_KV("taskID", message->taskID()) - << LOG_KV("sender", message->sender()); - - gateway->onMessageArrived(std::move(message)); - } - catch (std::exception& e) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("onReceiveMsgFromOtherGateway") - << LOG_KV("exception", boost::diagnostic_information(e)); - gateway->sendAck(_wsMessage->seq(), SEND_MESSAGE_TO_FRONT_ERROR); - } - }); -} - - -/** - * @brief: send message to other agency - * @param _agencyID: agency ID of receiver - * @param _message: ppc message data - * @return void - */ -void Gateway::asyncSendMessage( - const std::string& _agencyID, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) -{ - auto taskID = _message->taskID(); - GATEWAY_LOG(TRACE) << LOG_BADGE("asyncSendMessage") << printPPCMsg(_message); - try - { - if (m_protocol == m_webSocketService->gatewayConfig() - ->config() - ->gatewayConfig() - .networkConfig.PROTOCOL_WEBSOCKET) - { - auto wsClient = m_webSocketService->webSocketClient(_agencyID); - if (!wsClient) - { - onError("asyncSendMessage", taskID, - BCOS_ERROR_PTR((int)PPCRetCode::NETWORK_ERROR, - "WebSocket client not found for " + _agencyID), - _callback); - return; - } - - auto payload = std::make_shared(); - _message->encode(*payload); - auto wsMessage = m_wsMessageFactory->buildMessage(0, payload); - wsMessage->setSeq(_message->uuid()); - - // forward to other agency - auto self = weak_from_this(); - wsClient->asyncSendMessage(wsMessage, Options(m_holdingMessageMinutes * 60 * 1000), - [self, wsMessage, wsClient, taskID, _callback](const Error::Ptr& _error, - const std::shared_ptr& _msg, - const std::shared_ptr& _session) { - Error::Ptr error; - // send success - if (!_error || _error->errorCode() == 0) - { - // check ack - auto payload = _msg->payload(); - if (payload) - { - std::string status = std::string(payload->begin(), payload->end()); - if (SEND_MESSAGE_TO_FRONT_ERROR == status || - SEND_MESSAGE_TO_FRONT_TIMEOUT == status) - { - error = std::make_shared(PPCRetCode::NETWORK_ERROR, - "send message to target front error, status = " + status); - } - } - if (!error) - { - GATEWAY_LOG(TRACE) - << LOG_DESC("asyncSendMessage success") << LOG_KV("task", taskID); - // response to the client in-case of tars-error - _callback(nullptr); - return; - } - } - - if (!error) - { - error = _error; - } - - auto gateway = self.lock(); - if (!gateway) - { - return; - } - gateway->onError("asyncSendMessage", taskID, error, _callback); - }); - } -#if 0 - else if (m_protocol == m_webSocketService->gatewayConfig() - ->config() - ->gatewayConfig() - .networkConfig.PROTOCOL_HTTP) - { - Error::Ptr error; - auto httpClient = m_webSocketService->httpClient(_agencyID); - auto _url = - m_webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.url; - auto header = _message->header(); - appendHeader(header, _message); - auto body = _message->data(); - auto response = httpClient->post(_url, header, *body); - // parse response to TransportOutbound - std::string reponseStr(response.begin(), response.end()); - org::interconnection::link::TransportOutbound transportOutbound; - transportOutbound.ParseFromString(reponseStr); - if (transportOutbound.code() != SEND_MESSAGE_TO_FRONT_SUCCESS_CODE) - { - error = std::make_shared(PPCRetCode::NETWORK_ERROR, - "send message to target front error, code = " + transportOutbound.code()); - } - if (!error) - { - GATEWAY_LOG(TRACE) - << LOG_DESC("asyncSendMessage success") << LOG_KV("task", taskID); - _callback(nullptr); - return; - } - onError("asyncSendMessage", taskID, error, _callback); - } -#endif - } - catch (std::exception& e) - { - onError("asyncSendMessage", taskID, - BCOS_ERROR_PTR( - (int)PPCRetCode::EXCEPTION, std::string(boost::diagnostic_information(e))), - std::move(_callback)); - return; - } -} - -void Gateway::onError(std::string const& _desc, std::string const& _taskID, bcos::Error::Ptr _error, - ErrorCallbackFunc _callback) -{ - if (!_error || _error->errorCode() == 0) - { - return; - } - - if (_error->errorCode() == WsError::TimeOut) - { - // Lower the log level because the caller will type out the error message - GATEWAY_LOG(INFO) << LOG_BADGE(_desc) << LOG_KV("taskID", _taskID) - << LOG_KV("code", _error->errorCode()) - << LOG_KV("msg", _error->errorMessage()); - } - else - { - GATEWAY_LOG(ERROR) << LOG_BADGE(_desc) << LOG_KV("taskID", _taskID) - << LOG_KV("code", _error->errorCode()) - << LOG_KV("msg", _error->errorMessage()); - } - - if (!_callback) - { - return; - } - - if (m_threadPool) - { - m_threadPool->enqueue([_callback, _error]() { _callback(_error); }); - } - else - { - _callback(std::move(_error)); - } -} - - -/** - * @brief notice task info to gateway - * @param _taskInfo the latest task information - */ -bcos::Error::Ptr Gateway::notifyTaskInfo(GatewayTaskInfo::Ptr _taskInfo) -{ - auto startT = bcos::utcSteadyTime(); - auto error = std::make_shared(); - auto taskID = _taskInfo->taskID; - auto serviceEndpoint = _taskInfo->serviceEndpoint; - try - { - m_taskManager->registerTaskInfo(taskID, serviceEndpoint); - // check to see if any message has arrived - handleHoldingMessageQueue(std::move(_taskInfo)); - } - catch (std::exception& e) - { - error->setErrorCode(PPCRetCode::EXCEPTION); - error->setErrorMessage(boost::diagnostic_information(e)); - } - - if (error->errorCode()) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", taskID) - << LOG_DESC(error->errorMessage()); - } - GATEWAY_LOG(INFO) << LOG_BADGE("notifyTaskInfo") << LOG_KV("taskID", taskID) - << LOG_KV("serviceEndpoint", serviceEndpoint) - << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); - return error; -} - -bcos::Error::Ptr Gateway::eraseTaskInfo(std::string const& _taskID) -{ - auto startT = bcos::utcSteadyTime(); - try - { - // release held message - getAndRemoveHoldingMessages(_taskID); - m_taskManager->removeTaskInfo(_taskID); - GATEWAY_LOG(INFO) << LOG_BADGE("eraseTaskInfo") << LOG_KV("taskID", _taskID) - << LOG_KV("timecost", bcos::utcSteadyTime() - startT); - return nullptr; - } - catch (std::exception const& e) - { - GATEWAY_LOG(ERROR) << LOG_DESC("eraseTaskInfo error") - << LOG_KV("exception", boost::diagnostic_information(e)) - << LOG_KV("timecost", bcos::utcSteadyTime() - startT); - return BCOS_ERROR_PTR( - PPCRetCode::EXCEPTION, "eraseTaskInfo error: " + boost::diagnostic_information(e)); - } -} - -// register gateway url for other parties -bcos::Error::Ptr Gateway::registerGateway( - const std::vector& _gatewayList) -{ - try - { - for (const auto& gateway : _gatewayList) - { - m_webSocketService->registerGatewayUrl(gateway.agencyID, gateway.endpoint); - } - - return nullptr; - } - catch (std::exception const& e) - { - GATEWAY_LOG(ERROR) << LOG_DESC("registerGateway error") - << LOG_KV("exception", boost::diagnostic_information(e)); - return BCOS_ERROR_PTR( - PPCRetCode::EXCEPTION, "registerGateway error: " + boost::diagnostic_information(e)); - } -} - -void Gateway::asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) -{ - GATEWAY_LOG(TRACE) << LOG_BADGE("asyncGetAgencyList"); - auto const& agencies = m_webSocketService->gatewayConfig()->config()->gatewayConfig().agencies; - std::vector agencyList; - for (auto const& it : agencies) - { - agencyList.emplace_back(it.first); - } - if (!_callback) - { - return; - } - _callback(nullptr, std::move(agencyList)); -} - -void Gateway::handleHoldingMessageQueue(protocol::GatewayTaskInfo::Ptr _taskInfo) -{ - HoldingMessageQueue::Ptr queue; - { - WriteGuard l(x_holdingMessageQueue); - auto it = m_holdingMessageQueue.find(_taskInfo->taskID); - // not find the holding-queue related to the task-info - if (it == m_holdingMessageQueue.end()) - { - return; - } - queue = it->second; - // erase the queue - m_holdingMessageQueue.erase(it); - } - // cancel the timer - if (queue->timer) - { - queue->timer->cancel(); - } - - auto frontInterface = m_frontNodeManager->getFront(_taskInfo->serviceEndpoint); - if (!frontInterface) - { - GATEWAY_LOG(WARNING) << LOG_DESC( - "handleHoldingMessageQueue error for not find the corresponding front"); - return; - } - // dispatch the message - for (auto& msg : queue->messages) - { - if (!frontInterface) - { - GATEWAY_LOG(WARNING) << LOG_DESC("send message error for the target front not found"); - sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); - continue; - } - // forward to self node - frontInterface->onReceiveMessage(msg, [self = weak_from_this(), msg]( - const bcos::Error::Ptr& _error) { - auto gateway = self.lock(); - if (!gateway) - { - return; - } - - if (_error && _error->errorCode() != 0) - { - GATEWAY_LOG(WARNING) - << LOG_DESC("handleHoldingMessageQueue: dispatch the message error") - << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage()); - gateway->sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); - return; - } - - gateway->sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_SUCCESS); - }); - } -} - -// broadcast the message to all front when the task-id is not specified -void Gateway::broadcastMsgToAllFront(ppc::front::PPCMessageFace::Ptr const& _message) -{ - auto frontList = m_frontNodeManager->getAllFront(); - GATEWAY_LOG(TRACE) << LOG_DESC("broadcastMsgToAllFront") - << LOG_KV("frontSize", frontList.size()); - for (auto const& it : frontList) - { - auto const& front = it.second; - auto const& serviceEndpoint = it.first; - dispatchMessageToFront(front, _message, serviceEndpoint); - } -} - -void Gateway::dispatchMessageToFront(ppc::front::FrontInterface::Ptr const& _front, - ppc::front::PPCMessageFace::Ptr const& _message, std::string const& _serviceEndpoint) -{ - auto taskID = _message->taskID(); - // dispatch message to the given front - auto startT = utcSteadyTime(); - _front->onReceiveMessage(_message, [self = weak_from_this(), taskID, _serviceEndpoint, _message, - startT](const bcos::Error::Ptr& _error) { - auto gateway = self.lock(); - if (!gateway) - { - return; - } - - if (_error && _error->errorCode() != 0) - { - GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveMessage: dispatch message to front error") - << printPPCMsg(_message) << LOG_KV("task", taskID) - << LOG_KV("front", _serviceEndpoint) - << LOG_KV("code", _error->errorCode()) - << LOG_KV("msg", _error->errorMessage()) - << LOG_KV("timecost", (utcSteadyTime() - startT)); - gateway->sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); - return; - } - GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveMessage success") << printPPCMsg(_message) - << LOG_KV("timecost", (utcSteadyTime() - startT)); - gateway->sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_SUCCESS); - }); -} - -void Gateway::onMessageArrived(PPCMessageFace::Ptr _message) -{ - GATEWAY_LOG(TRACE) << LOG_DESC("onMessageArrived") << printPPCMsg(_message); - auto taskID = _message->taskID(); - // broadcast the message to all front when the task-id is not specified - if (taskID.empty()) - { - broadcastMsgToAllFront(_message); - return; - } - - bcos::UpgradableGuard l(x_holdingMessageQueue); - auto serviceEndpoint = m_taskManager->getServiceEndpoint(taskID); - if (!serviceEndpoint.empty()) - { - auto frontInterface = m_frontNodeManager->getFront(serviceEndpoint); - if (!frontInterface) - { - GATEWAY_LOG(WARNING) - << LOG_DESC( - "onMessageArrived: can't find the front to dispatch the receive message") - << printPPCMsg(_message) << LOG_KV("task", taskID) - << LOG_KV("frontEndPoint", serviceEndpoint); - sendAck(_message->uuid(), SEND_MESSAGE_TO_FRONT_ERROR); - return; - } - dispatchMessageToFront(frontInterface, _message, serviceEndpoint); - return; - } - // hold the message - GATEWAY_LOG(INFO) << LOG_BADGE("holdMessage") << LOG_KV("taskID", taskID); - - bcos::UpgradeGuard ul(l); - auto it = m_holdingMessageQueue.find(taskID); - if (it != m_holdingMessageQueue.end()) - { - it->second->messages.emplace_back(_message); - return; - } - // insert new holding-queue - auto queue = std::make_shared(); - queue->messages.emplace_back(_message); - // create timer to handle timeout - queue->timer = std::make_shared( - *m_ioService, boost::posix_time::minutes(m_holdingMessageMinutes)); - queue->timer->async_wait([self = weak_from_this(), taskID](boost::system::error_code _error) { - if (!_error) - { - auto gateway = self.lock(); - if (gateway) - { - // remove timeout message - auto msgQueue = gateway->getAndRemoveHoldingMessages(taskID); - gateway->handleTimeoutHoldingMessage(msgQueue); - } - } - }); - m_holdingMessageQueue[taskID] = queue; -} - -HoldingMessageQueue::Ptr Gateway::getAndRemoveHoldingMessages(const std::string& _taskID) -{ - WriteGuard lock(x_holdingMessageQueue); - auto it = m_holdingMessageQueue.find(_taskID); - if (it == m_holdingMessageQueue.end()) - { - return nullptr; - } - - HoldingMessageQueue::Ptr ret = it->second; - m_holdingMessageQueue.erase(_taskID); - return ret; -} - -void Gateway::handleTimeoutHoldingMessage(HoldingMessageQueue::Ptr _queue) -{ - if (!_queue) - { - return; - } - // dispatch the ack - for (auto& msg : _queue->messages) - { - sendAck(msg->uuid(), SEND_MESSAGE_TO_FRONT_TIMEOUT); - } -} - -void Gateway::addAckCallback( - std::string const& _uuid, MessageFace::Ptr _msg, boostssl::ws::WsSession::Ptr _session) -{ - WriteGuard lock(x_ackCallbacks); - m_ackCallbacks[_uuid] = {std::move(_msg), std::move(_session)}; -} - - -void Gateway::sendAck(std::string const& _uuid, std::string const& _status) -{ - WriteGuard lock(x_ackCallbacks); - auto it = m_ackCallbacks.find(_uuid); - if (it == m_ackCallbacks.end()) - { - return; - } - - auto payload = std::make_shared(_status.begin(), _status.end()); - auto& msg = it->second.first; - auto& session = it->second.second; - msg->setPayload(payload); - session->asyncSendMessage(msg, Options(), nullptr); - - m_ackCallbacks.erase(it); -} - -void Gateway::appendHeader( - std::map& origin_header, front::PPCMessageFace::Ptr _message) -{ - origin_header["Content-Type"] = "application/octet-stream;charset=utf-8"; - origin_header["has_uri"] = std::to_string(true); - origin_header[HEAD_TASK_ID] = _message->taskID(); - origin_header[HEAD_ALGO_TYPE] = std::to_string(_message->algorithmType()); - origin_header[HEAD_TASK_TYPE] = std::to_string(_message->taskType()); - origin_header[HEAD_SENDER_ID] = _message->sender(); - origin_header[HEAD_MESSAGE_TYPE] = std::to_string(_message->messageType()); - origin_header[HEAD_SEQ] = std::to_string(_message->seq()); - origin_header[HEAD_IS_RESPONSE] = std::to_string(_message->response()); - origin_header[HEAD_UUID] = _message->uuid(); -} - -Gateway::Ptr GatewayFactory::buildGateway(NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, - storage::CacheStorage::Ptr _cache, front::PPCMessageFaceFactory::Ptr _messageFactory, - std::shared_ptr _threadPool) -{ - auto wsMessageFactory = std::make_shared(); - auto webSocketServiceFactory = std::make_shared(); - auto ioService = std::make_shared(); - auto ioContext = std::make_shared(); - - auto webSocketService = webSocketServiceFactory->buildWebSocketService(_config, ioContext); - - TaskManager::Ptr taskManager = nullptr; - if (_arch == NodeArch::AIR || _cache == nullptr) - { - GATEWAY_LOG(INFO) << LOG_BADGE("buildGateway without cache"); - taskManager = std::make_shared(ioService); - } - else - { - GATEWAY_LOG(INFO) << LOG_BADGE("buildGateway with cache"); - taskManager = std::make_shared(_cache, ioService); - } - auto frontNodeManager = std::make_shared(); - - return std::make_shared(std::move(webSocketService), std::move(wsMessageFactory), - std::move(ioService), std::move(_messageFactory), std::move(frontNodeManager), - std::move(taskManager), std::move(_threadPool), _config->holdingMessageMinutes(), - std::move(ioContext)); -} diff --git a/cpp/ppc-gateway/ppc-gateway/Gateway.h b/cpp/ppc-gateway/ppc-gateway/Gateway.h deleted file mode 100644 index 4a0a8a77..00000000 --- a/cpp/ppc-gateway/ppc-gateway/Gateway.h +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file Gateway.h - * @author: shawnhe - * @date 2022-10-20 - */ - -#pragma once - -#include "Common.h" -#include "FrontNodeManager.h" -#include "GatewayService.h" -#include "TaskManager.h" -#include "WebSocketService.h" -#include "ppc-framework/gateway/GatewayInterface.h" -#include "ppc-protocol/src/PPCMessage.h" -#if 0 -//TODO: optimize here -#include "ppc-protocol/src/protobuf/transport.pb.h" -#endif - -#include "tbb/concurrent_vector.h" -#include -#include - -#include - -namespace ppc::gateway -{ -struct HoldingMessageQueue -{ - using Ptr = std::shared_ptr; - HoldingMessageQueue() = default; - - tbb::concurrent_vector messages; - std::shared_ptr timer; -}; - -class Gateway : public GatewayInterface, public std::enable_shared_from_this -{ -public: - using Ptr = std::shared_ptr; - - Gateway(WebSocketService::Ptr _webSocketService, - bcos::boostssl::ws::WsMessageFactory::Ptr _wsMessageFactory, - std::shared_ptr _ioService, - front::PPCMessageFaceFactory::Ptr _messageFactory, FrontNodeManager::Ptr _frontNodeManager, - TaskManager::Ptr _taskManager, std::shared_ptr _threadPool, - int _holdingMessageMinutes = 10, - std::shared_ptr _ioContext = nullptr) - : m_holdingMessageMinutes(_holdingMessageMinutes), - m_webSocketService(std::move(_webSocketService)), - m_wsMessageFactory(std::move(_wsMessageFactory)), - m_ioService(std::move(_ioService)), - m_messageFactory(std::move(_messageFactory)), - m_frontNodeManager(std::move(_frontNodeManager)), - m_taskManager(std::move(_taskManager)), - m_threadPool(std::move(_threadPool)), - m_ioContext(_ioContext), - m_protocol( - _webSocketService->gatewayConfig()->config()->gatewayConfig().networkConfig.protocol) - { - GATEWAY_LOG(INFO) << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); - } - - Gateway(const Gateway&) = delete; - Gateway(Gateway&&) = delete; - - Gateway& operator=(const Gateway&) = delete; - Gateway& operator=(Gateway&&) = delete; - - virtual ~Gateway() override { stop(); } - - void start() override; - void stop() override; - - void registerWebSocketMsgHandler(); - -#if 0 - // TODO: optimize here - void registerUrlMsgHandler(); -#endif - - /** - * @brief: send message to other agency - * @param _agencyID: agency ID of receiver - * @param _message: ppc message data - * @return void - */ - void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, - ErrorCallbackFunc _callback) override; - - /** - * @brief notice task info to gateway - * @param _taskInfo the latest task information - */ - bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr _taskInfo) override; - - // erase the task info - bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override; - - // register gateway url for other parties - bcos::Error::Ptr registerGateway( - const std::vector& _gatewayList) override; - - // get the agency-list - void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override; - - FrontNodeManager::Ptr frontNodeManager() { return m_frontNodeManager; } - TaskManager::Ptr taskManager() { return m_taskManager; } - void setTaskManager(TaskManager::Ptr _taskManager) { m_taskManager = std::move(_taskManager); } - - WebSocketService::Ptr webSocketService() { return m_webSocketService; } - - front::PPCMessageFaceFactory::Ptr messageFactory() { return m_messageFactory; } - bcos::boostssl::ws::WsMessageFactory::Ptr wsMessageFactory() { return m_wsMessageFactory; } - std::shared_ptr threadPool() { return m_threadPool; } - - void addAckCallback(std::string const& _uuid, bcos::boostssl::MessageFace::Ptr _msg, - bcos::boostssl::ws::WsSession::Ptr _session); - - void sendAck(std::string const& _uuid, std::string const& _status); - - // Note: since the front will periodically register the status, no need to response message to - // the front - void registerFront(std::string const& _endPoint, front::FrontInterface::Ptr _front) override - { - m_frontNodeManager->registerFront(_endPoint, _front); - } - - void unregisterFront(std::string const& _endPoint) override - { - m_frontNodeManager->unregisterFront(_endPoint); - } - -protected: - virtual void handleHoldingMessageQueue(protocol::GatewayTaskInfo::Ptr _taskInfo); - virtual void onMessageArrived(front::PPCMessageFace::Ptr _message); - virtual HoldingMessageQueue::Ptr getAndRemoveHoldingMessages(const std::string& _taskID); - virtual void handleTimeoutHoldingMessage(HoldingMessageQueue::Ptr _queue); - virtual void onError(std::string const& _desc, std::string const& _taskID, - bcos::Error::Ptr _error, ErrorCallbackFunc _callback); - - void broadcastMsgToAllFront(ppc::front::PPCMessageFace::Ptr const& _message); - void dispatchMessageToFront(ppc::front::FrontInterface::Ptr const& _front, - ppc::front::PPCMessageFace::Ptr const& _message, std::string const& _serviceEndpoint); - void appendHeader( - std::map& origin_header, front::PPCMessageFace::Ptr _message); - -private: - int m_holdingMessageMinutes = 30; - int m_protocol; - WebSocketService::Ptr m_webSocketService; - bcos::boostssl::ws::WsMessageFactory::Ptr m_wsMessageFactory; - std::shared_ptr m_ioService; - std::shared_ptr m_ioContext; - front::PPCMessageFaceFactory::Ptr m_messageFactory; - - FrontNodeManager::Ptr m_frontNodeManager; - TaskManager::Ptr m_taskManager; - - std::shared_ptr m_threadPool; - - // the thread to make ioservice run - std::shared_ptr m_gatewayThread; - - /** - * hold the message for the situation that - * gateway receives message from the other side while the task has not been registered. - */ - mutable boost::shared_mutex x_holdingMessageQueue; - std::unordered_map m_holdingMessageQueue; - - std::atomic_bool m_running = {false}; - - mutable boost::shared_mutex x_ackCallbacks; - std::unordered_map > - m_ackCallbacks; -}; - - -class GatewayFactory -{ -public: - using Ptr = std::shared_ptr; - -public: - GatewayFactory() = default; - ~GatewayFactory() = default; - - Gateway::Ptr buildGateway(ppc::protocol::NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, - storage::CacheStorage::Ptr _cache, front::PPCMessageFaceFactory::Ptr _messageFactory, - std::shared_ptr _threadPool); -}; - -} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp index 9a1a37ba..8243d133 100644 --- a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp @@ -22,23 +22,27 @@ using namespace bcos; using namespace ppc::gateway; +using namespace bcos::boostssl::context; + void GatewayConfigContext::initContextConfig() { - m_contextConfig = std::make_shared(); + m_contextConfig = std::make_shared(); auto const& gatewayConfig = m_config->gatewayConfig().networkConfig; // non-sm-ssl if (!gatewayConfig.enableSM) { - boostssl::context::ContextConfig::CertConfig certConfig; + ContextConfig::CertConfig certConfig; certConfig.caCert = gatewayConfig.caCertPath; certConfig.nodeCert = gatewayConfig.sslCertPath; certConfig.nodeKey = gatewayConfig.sslKeyPath; m_contextConfig->setCertConfig(certConfig); m_contextConfig->setSslType("ssl"); + // parse the nodeID + NodeInfoTools::initCert2PubHexHandler()(certConfig.nodeCert, m_nodeID); GATEWAY_LOG(INFO) << LOG_DESC("initConfig: rpc work in non-sm-ssl model") << LOG_KV("caCert", certConfig.caCert) << LOG_KV("nodeCert", certConfig.nodeCert) - << LOG_KV("nodeKey", certConfig.nodeKey); + << LOG_KV("nodeKey", certConfig.nodeKey) << LOG_KV("nodeID", m_nodeID); GATEWAY_LOG(INFO) << LOG_DESC("initContextConfig: non-sm-ssl"); return; } @@ -51,5 +55,6 @@ void GatewayConfigContext::initContextConfig() certConfig.enNodeKey = gatewayConfig.smEnSslKeyPath; m_contextConfig->setSmCertConfig(certConfig); m_contextConfig->setSslType("sm_ssl"); - GATEWAY_LOG(INFO) << LOG_DESC("initContextConfig: sm-ssl"); + NodeInfoTools::initCert2PubHexHandler()(certConfig.enNodeCert, m_nodeID); + GATEWAY_LOG(INFO) << LOG_DESC("initContextConfig: sm-ssl") << LOG_KV("nodeID", m_nodeID); } diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h index 41320ebc..259ceb71 100644 --- a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h @@ -21,8 +21,7 @@ #pragma once #include "Common.h" -#include "ppc-framework/storage/CacheStorage.h" -#include "ppc-storage/src/redis/RedisStorage.h" +#include "bcos-boostssl/context/NodeInfoTools.h" #include #include #include @@ -51,12 +50,15 @@ class GatewayConfigContext } ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } + std::string const& nodeID() const { return m_nodeID; } + private: void initContextConfig(); private: ppc::tools::PPCConfig::Ptr m_config; std::shared_ptr m_contextConfig; + std::string m_nodeID; }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp b/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp new file mode 100644 index 00000000..8d87f282 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayConfigLoader.cpp + * @author: yujiechen + * @date 2024-08-26 + */ +#include "GatewayConfigLoader.h" +#include "Common.h" +#include "bcos-utilities/FileUtility.h" +#include + +using namespace ppc; +using namespace bcos; +using namespace ppc::tools; +using namespace ppc::gateway; +using namespace bcos::boostssl; + +// load p2p connected peers +void GatewayConfigLoader::loadP2pConnectedNodes() +{ + std::string nodeFilePath = + m_config->gatewayConfig().nodePath + "/" + m_config->gatewayConfig().nodeFileName; + // load p2p connected nodes + auto jsonContent = readContentsToString(boost::filesystem::path(nodeFilePath)); + if (!jsonContent || jsonContent->empty()) + { + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment( + "loadP2pConnectedNodes: unable to read nodes json file, path=" + nodeFilePath)); + } + + parseConnectedJson(*jsonContent.get(), *m_nodeIPEndpointSet); + GATEWAY_LOG(INFO) << LOG_DESC("loadP2pConnectedNodes success!") + << LOG_KV("nodePath", m_config->gatewayConfig().nodePath) + << LOG_KV("nodeFileName", m_config->gatewayConfig().nodeFileName) + << LOG_KV("nodes", m_nodeIPEndpointSet->size()); +} + +void GatewayConfigLoader::parseConnectedJson( + const std::string& _json, EndPointSet& _nodeIPEndpointSet) +{ + /* + {"nodes":["127.0.0.1:30355","127.0.0.1:30356"}]} + */ + Json::Value root; + Json::Reader jsonReader; + try + { + if (!jsonReader.parse(_json, root)) + { + GATEWAY_LOG(ERROR) << "unable to parse connected nodes json" << LOG_KV("json:", _json); + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment("GatewayConfig: unable to parse p2p " + "connected nodes json")); + } + Json::Value jNodes = root["nodes"]; + if (jNodes.isArray()) + { + unsigned int jNodesSize = jNodes.size(); + for (unsigned int i = 0; i < jNodesSize; i++) + { + std::string host = jNodes[i].asString(); + + NodeIPEndpoint endpoint; + hostAndPort2Endpoint(host, endpoint); + _nodeIPEndpointSet.insert(endpoint); + + GATEWAY_LOG(INFO) << LOG_DESC("add one connected node") << LOG_KV("host", host); + } + } + } + catch (const std::exception& e) + { + GATEWAY_LOG(ERROR) << LOG_KV( + "parseConnectedJson error: ", boost::diagnostic_information(e)); + BOOST_THROW_EXCEPTION(e); + } +} + +bool GatewayConfigLoader::isValidPort(int port) +{ + if (port <= 1024 || port > 65535) + return false; + return true; +} + +void GatewayConfigLoader::hostAndPort2Endpoint(const std::string& _host, NodeIPEndpoint& _endpoint) +{ + std::string ip; + uint16_t port; + + std::vector s; + boost::split(s, _host, boost::is_any_of("]"), boost::token_compress_on); + if (s.size() == 2) + { // ipv6 + ip = s[0].data() + 1; + port = boost::lexical_cast(s[1].data() + 1); + } + else if (s.size() == 1) + { // ipv4 + std::vector v; + boost::split(v, _host, boost::is_any_of(":"), boost::token_compress_on); + if (v.size() < 2) + { + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment("GatewayConfig: invalid host , host=" + _host)); + } + ip = v[0]; + port = boost::lexical_cast(v[1]); + } + else + { + GATEWAY_LOG(ERROR) << LOG_DESC("not valid host value") << LOG_KV("host", _host); + BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( + "GatewayConfig: the host is invalid, host=" + _host)); + } + + if (!isValidPort(port)) + { + GATEWAY_LOG(ERROR) << LOG_DESC("the port is not valid") << LOG_KV("port", port); + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment( + "GatewayConfig: the port is invalid, port=" + std::to_string(port))); + } + + boost::system::error_code ec; + boost::asio::ip::address ip_address = boost::asio::ip::make_address(ip, ec); + if (ec.value() != 0) + { + GATEWAY_LOG(ERROR) << LOG_DESC("the host is invalid, make_address error") + << LOG_KV("host", _host); + BOOST_THROW_EXCEPTION( + WeDPRException() << errinfo_comment( + "GatewayConfig: the host is invalid make_address error, host=" + _host)); + } + _endpoint = NodeIPEndpoint{ip_address, port}; +} diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h b/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h new file mode 100644 index 00000000..1c6b418e --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayConfigLoader.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "bcos-boostssl/interfaces/NodeInfoDef.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::gateway +{ +class GatewayConfigLoader +{ +public: + using EndPointSet = std::set; + using Ptr = std::shared_ptr; + GatewayConfigLoader(ppc::tools::PPCConfig::Ptr config) + : m_config(std::move(config)), m_nodeIPEndpointSet(std::make_shared()) + { + loadP2pConnectedNodes(); + } + virtual ~GatewayConfigLoader() = default; + + EndPointSet const& nodeIPEndpointSet() const { return *m_nodeIPEndpointSet; } + + std::shared_ptr const& nodeIPEndpointSetPtr() const { return m_nodeIPEndpointSet; } + +protected: + void parseConnectedJson(const std::string& _json, EndPointSet& nodeIPEndpointSet); + void loadP2pConnectedNodes(); + void hostAndPort2Endpoint(const std::string& _host, bcos::boostssl::NodeIPEndpoint& _endpoint); + bool isValidPort(int port); + +private: + ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_nodeIPEndpointSet; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp new file mode 100644 index 00000000..51f047f3 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayFactory.cpp + * @author: yujiechen + * @date 2024-08-26 + */ +#include "GatewayFactory.h" +#include "Common.h" +#include "bcos-boostssl/websocket/WsInitializer.h" +#include "ppc-gateway/p2p/Service.h" +#include "ppc-gateway/p2p/router/RouterTableImpl.h" +#include "protocol/src/v1/MessageHeaderImpl.h" +#include "protocol/src/v1/MessageImpl.h" + +using namespace ppc; +using namespace bcos; +using namespace ppc::tools; +using namespace ppc::protocol; +using namespace ppc::gateway; +using namespace bcos::boostssl::ws; +using namespace bcos::boostssl; + +WsConfig::Ptr GatewayFactory::createServiceConfig(GatewayConfig const& config) const +{ + auto wsConfig = std::make_shared(); + wsConfig->setModel(WsModel::Mixed); + wsConfig->setListenIP(config.networkConfig.listenIp); + wsConfig->setListenPort(config.networkConfig.listenPort); + wsConfig->setSmSSL(config.networkConfig.enableSM); + wsConfig->setMaxMsgSize(config.maxAllowedMsgSize); + wsConfig->setReconnectPeriod(config.reconnectTime); + // TODO: setHeartbeatPeriod, setSendMsgTimeout + wsConfig->setThreadPoolSize(config.networkConfig.threadPoolSize); + // connected peers + wsConfig->setConnectPeers(m_gatewayConfig->nodeIPEndpointSetPtr()); + wsConfig->setDisableSsl(config.networkConfig.disableSsl); + wsConfig->setContextConfig(m_contextConfig->contextConfig()); + return wsConfig; +} + +Service::Ptr GatewayFactory::buildService() const +{ + auto wsConfig = createServiceConfig(m_config->gatewayConfig()); + auto wsInitializer = std::make_shared(); + // set the messageFactory + wsInitializer->setMessageFactory( + std::make_shared(std::make_shared())); + // set the config + wsInitializer->setConfig(wsConfig); + auto p2pService = std::make_shared(m_contextConfig->nodeID(), + std::make_shared(), m_config->gatewayConfig().unreachableDistance, + "Service"); + p2pService->setNodeEndpoints(m_gatewayConfig->nodeIPEndpointSet()); + + wsInitializer->initWsService(p2pService); + return p2pService; +} + +IGateway::Ptr GatewayFactory::build(ppc::front::IFrontBuilder::Ptr const& frontBuilder) const +{ + auto service = buildService(); + return std::make_shared( + service, frontBuilder, std::make_shared(), m_config->agencyID()); +} \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayFactory.h b/cpp/ppc-gateway/ppc-gateway/GatewayFactory.h new file mode 100644 index 00000000..2393f240 --- /dev/null +++ b/cpp/ppc-gateway/ppc-gateway/GatewayFactory.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayFactory.h + * @author: yujiechen + * @date 2024-08-26 + */ +#pragma once +#include "GatewayConfigContext.h" +#include "GatewayConfigLoader.h" +#include "bcos-boostssl/websocket/WsConfig.h" +#include "gateway/GatewayImpl.h" +#include "ppc-gateway/p2p/Service.h" +#include "ppc-tools/src/config/PPCConfig.h" + +namespace ppc::gateway +{ +class GatewayFactory +{ +public: + using Ptr = std::shared_ptr; + GatewayFactory(ppc::tools::PPCConfig::Ptr config) : m_config(std::move(config)) + { + m_contextConfig = std::make_shared(m_config); + m_gatewayConfig = std::make_shared(m_config); + } + virtual ~GatewayFactory() = default; + + IGateway::Ptr build(ppc::front::IFrontBuilder::Ptr const& frontBuilder) const; + +protected: + Service::Ptr buildService() const; + + bcos::boostssl::ws::WsConfig::Ptr createServiceConfig( + ppc::tools::GatewayConfig const& config) const; + +private: + ppc::tools::PPCConfig::Ptr m_config; + GatewayConfigContext::Ptr m_contextConfig; + GatewayConfigLoader::Ptr m_gatewayConfig; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h b/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h deleted file mode 100644 index fc8b8066..00000000 --- a/cpp/ppc-gateway/ppc-gateway/ProTaskManager.h +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file ProTaskManager.h - * @author: shawnhe - * @date 2022-10-23 - */ -#pragma once -#include "TaskManager.h" - -#include "ppc-framework/storage/CacheStorage.h" -#include -namespace ppc::gateway -{ -class ProTaskManager : public TaskManager -{ -public: - using Ptr = std::shared_ptr; - ProTaskManager( - storage::CacheStorage::Ptr _cache, std::shared_ptr _ioService) - : TaskManager(std::move(_ioService)), m_cache(std::move(_cache)) - {} - ~ProTaskManager() override = default; - - void registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint) override - { - // throw exception if taskID existed - TaskManager::registerTaskInfo(_taskID, _serviceEndpoint); - try - { - // add task info to cache server - m_cache->setValue(_taskID, _serviceEndpoint, TASK_TIMEOUT_M * 60); - } - catch (std::exception const& e) - { - GATEWAY_LOG(WARNING) << LOG_DESC( - "set value failed: " + std::string(boost::diagnostic_information(e))); - } - } - - void removeTaskInfo(const std::string& _taskID) override - { - // Note: remove the memory-task-info in-case-of the redis exception - TaskManager::removeTaskInfo(_taskID); - m_cache->deleteKey(_taskID); - } - - std::string getServiceEndpoint(const std::string& _taskID) override - { - // find task info in memory first - try - { - auto endPoint = TaskManager::getServiceEndpoint(_taskID); - if (!endPoint.empty()) - { - return endPoint; - } - // Note: different node should not share the cache with same database - // find task info in cache service - auto serviceEndpoint = m_cache->getValue(_taskID); - if (serviceEndpoint == std::nullopt) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("keyNotFoundInCache") << LOG_KV("key", _taskID); - return ""; - } - GATEWAY_LOG(TRACE) << LOG_DESC("getServiceEndpoint: find the task from redis cache") - << LOG_KV("task", _taskID); - // add task info to memory - auto taskInfo = prepareTaskInfo(_taskID, *serviceEndpoint); - addTaskInfo(_taskID, taskInfo); - return *serviceEndpoint; - } - catch (std::exception const& e) - { - GATEWAY_LOG(ERROR) << LOG_DESC("getServiceEndpoint error") - << LOG_KV("exception", boost::diagnostic_information(e)); - return ""; - } - } - -private: - storage::CacheStorage::Ptr m_cache; -}; -} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp b/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp deleted file mode 100644 index 51ad5f15..00000000 --- a/cpp/ppc-gateway/ppc-gateway/TaskManager.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file TaskManager.cpp - * @author: shawnhe - * @date 2022-10-23 - */ - -#include "TaskManager.h" -#include "ppc-framework/protocol/Protocol.h" - -using namespace bcos; -using namespace ppc::gateway; -using namespace ppc::storage; - -void TaskManager::registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("registerTaskInfo") << LOG_KV("taskID", _taskID) - << LOG_KV("serviceEndpoint", _serviceEndpoint); - if (getTaskInfo(_taskID)) - { - BOOST_THROW_EXCEPTION( - BCOS_ERROR(protocol::PPCRetCode::EXCEPTION, "task id already exists")); - } - - // add task info to memory - auto taskInfo = prepareTaskInfo(_taskID, _serviceEndpoint); - addTaskInfo(_taskID, taskInfo); -} - - -std::string TaskManager::getServiceEndpoint(const std::string& _taskID) -{ - // find task info in memory first - auto taskInfo = getTaskInfo(_taskID); - if (taskInfo) - { - return taskInfo->serviceEndpoint; - } - return ""; -} - - -TaskManager::TaskInfo::Ptr TaskManager::prepareTaskInfo( - const std::string& _taskID, const std::string& _serviceEndpoint) -{ - auto taskInfo = std::make_shared(); - taskInfo->serviceEndpoint = _serviceEndpoint; - - // create timer to handle timeout - taskInfo->timer = std::make_shared( - *m_ioService, boost::posix_time::minutes(TASK_TIMEOUT_M)); - - taskInfo->timer->async_wait( - [self = weak_from_this(), _taskID](boost::system::error_code _error) { - if (!_error) - { - auto taskManager = self.lock(); - if (taskManager) - { - // remove timeout event - taskManager->removeTaskInfo(_taskID); - } - } - }); - - return taskInfo; -} - - -TaskManager::TaskInfo::Ptr TaskManager::getTaskInfo(const std::string& _taskID) -{ - ReadGuard lock(x_tasks); - auto it = m_tasks.find(_taskID); - if (it != m_tasks.end()) - { - return it->second; - } - else - { - return nullptr; - } -} - - -void TaskManager::addTaskInfo( - const std::string& _taskID, const TaskManager::TaskInfo::Ptr& _taskInfo) -{ - WriteGuard lock(x_tasks); - GATEWAY_LOG(INFO) << LOG_BADGE("addTaskInfo") << LOG_KV("taskID", _taskID) - << LOG_KV("serviceEndpoint", _taskInfo->serviceEndpoint); - m_tasks.emplace(_taskID, _taskInfo); -} - - -void TaskManager::removeTaskInfo(const std::string& _taskID) -{ - WriteGuard lock(x_tasks); - GATEWAY_LOG(INFO) << LOG_BADGE("removeTaskInfo") << LOG_KV("taskID", _taskID); - m_tasks.erase(_taskID); -} diff --git a/cpp/ppc-gateway/ppc-gateway/TaskManager.h b/cpp/ppc-gateway/ppc-gateway/TaskManager.h deleted file mode 100644 index 214888a0..00000000 --- a/cpp/ppc-gateway/ppc-gateway/TaskManager.h +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file TaskManager.h - * @author: shawnhe - * @date 2022-10-23 - */ - -#pragma once - -#include "Common.h" -#include "GatewayConfigContext.h" -#include -#include -#include -#include -#include - -namespace ppc::gateway -{ -class TaskManager : public std::enable_shared_from_this -{ -public: - using Ptr = std::shared_ptr; - TaskManager(std::shared_ptr _ioService) - : m_ioService(std::move(_ioService)) - {} - virtual ~TaskManager() = default; - - virtual void registerTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint); - - virtual std::string getServiceEndpoint(const std::string& _taskID); - - virtual void removeTaskInfo(const std::string& _taskID); - -protected: - struct TaskInfo - { - using Ptr = std::shared_ptr; - std::string serviceEndpoint; - // timeout of the task - std::shared_ptr timer; - }; - - TaskInfo::Ptr prepareTaskInfo(const std::string& _taskID, const std::string& _serviceEndpoint); - TaskInfo::Ptr getTaskInfo(const std::string& _taskID); - void addTaskInfo(const std::string& _taskID, const TaskInfo::Ptr& _taskInfo); - -protected: - std::shared_ptr m_ioService; - // key: taskID, value: TaskInfo - std::unordered_map m_tasks; - mutable bcos::SharedMutex x_tasks; - - constexpr static uint32_t TASK_TIMEOUT_M = 24 * 60; // minutes -}; -} // namespace ppc::gateway diff --git a/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp b/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp deleted file mode 100644 index 1c644fa8..00000000 --- a/cpp/ppc-gateway/ppc-gateway/WebSocketService.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file WebSocketService.cpp - * @author: shawnhe - * @date 2022-10-23 - */ - -#include "WebSocketService.h" -#include "ppc-tools/src/config/ParamChecker.h" -#include - -using namespace bcos; -using namespace bcos::boostssl; -using namespace bcos::boostssl::ws; -using namespace ppc::gateway; -using namespace ppc::tools; -using namespace ppc::http; - -void WebSocketService::start() -{ - if (m_timer) - { - m_timer->registerTimeoutHandler( - boost::bind(&WebSocketService::reconnectUnconnectedClient, this)); - } - if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - m_wsServer->start(); - GATEWAY_LOG(INFO) << LOG_BADGE("start the WebSocketService end"); - } - else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - m_httpServer->start(); - GATEWAY_LOG(INFO) << LOG_BADGE("start the HttpService end"); - } - startConnect(); - m_timer->start(); -} - -void WebSocketService::stop() -{ - GATEWAY_LOG(INFO) << LOG_BADGE("stop the WebSocketService"); - if (m_timer) - { - m_timer->stop(); - } - ReadGuard l(x_agencyClients); - if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - for (auto& client : m_agencyClients) - { - auto it = client.second; - if (it) - { - it->stop(); - } - } - if (m_wsServer) - { - m_wsServer->stop(); - } - GATEWAY_LOG(INFO) << LOG_BADGE("stop the WebSocketService success"); - } - else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - if (m_httpServer) - { - m_httpServer->stop(); - } - GATEWAY_LOG(INFO) << LOG_BADGE("stop the HttpService success"); - } -} - - -void WebSocketService::registerGatewayUrl( - const std::string& _agencyID, const std::string& _agencyUrl) -{ - if (!insertAgency(_agencyID, _agencyUrl)) - { - return; - } - GATEWAY_LOG(INFO) << LOG_BADGE("registerGatewayUrl") << LOG_KV("agencyID", _agencyID) - << LOG_KV("agencyUrl", _agencyUrl); - try - { - if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - auto client = buildWebSocketClient(_agencyID); - insertIntoMap(_agencyID, client, x_agencyClients, m_agencyClients); - } - else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - auto client = buildHttpClient(_agencyID, m_ioContext); - insertIntoMap(_agencyID, client, x_agencyClients, m_agencyHttpClients); - } - } - catch (std::exception const& e) - { - GATEWAY_LOG(WARNING) << LOG_DESC("connect to the agency failed") - << LOG_KV("agency", _agencyID) << LOG_KV("url", _agencyUrl); - { - bcos::WriteGuard ucl(x_unConnectedAgencies); - m_unConnectedAgencies.insert(_agencyID); - } - } -} - -bool WebSocketService::insertAgency(const std::string& _agencyID, const std::string& _agencyUrl) -{ - std::vector endpoints; - boost::split(endpoints, _agencyUrl, boost::is_any_of(",")); - - WriteGuard l(x_agencies); - - if (m_urls.find(_agencyID) != m_urls.end()) - { - if (m_urls[_agencyID] == _agencyUrl) - { - // no need update - return false; - } - } - - m_urls[_agencyID] = _agencyUrl; - m_agencies[_agencyID] = endpoints; - return true; -} - -void WebSocketService::reconnectUnconnectedClient() -{ - { - // print connected clients - bcos::ReadGuard rl(x_agencyClients); - if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - GATEWAY_LOG(INFO) << LOG_DESC("connectedWebsocketClient") - << LOG_KV("size", m_agencyClients.size()); - } - else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - GATEWAY_LOG(INFO) << LOG_DESC("connectedHttpClient") - << LOG_KV("size", m_agencyHttpClients.size()); - } - } - - // start reconnecting - bcos::UpgradableGuard l(x_unConnectedAgencies); - GATEWAY_LOG(INFO) << LOG_DESC("reconnectUnconnectedClient") - << LOG_KV("size", m_unConnectedAgencies.size()); - for (auto it = m_unConnectedAgencies.begin(); it != m_unConnectedAgencies.end();) - { - try - { - if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") - << LOG_DESC("reconnectUnconnectedClient: connect to peer") - << LOG_KV("agency", *it); - auto client = buildWebSocketClient(*it); - GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") - << LOG_DESC("reconnectUnconnectedClient: connect to peer success") - << LOG_KV("agency", *it); - // insert the successfully started client into the m_agencyClients - insertIntoMap(*it, client, x_agencyClients, m_agencyClients); - // erase the connected client from the m_unConnectedAgencies - bcos::UpgradeGuard ul(l); - it = m_unConnectedAgencies.erase(it); - } - else if (m_protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - GATEWAY_LOG(INFO) << LOG_BADGE("HttpService") - << LOG_DESC("reconnectUnconnectedClient: connect to peer") - << LOG_KV("agency", *it); - auto client = buildHttpClient(*it, m_ioContext); - GATEWAY_LOG(INFO) << LOG_BADGE("HttpService") - << LOG_DESC("reconnectUnconnectedClient: connect to peer success") - << LOG_KV("agency", *it); - // insert the successfully started client into the m_agencyHttpClients - insertIntoMap(*it, client, x_agencyClients, m_agencyHttpClients); - // erase the connected client from the m_unConnectedAgencies - bcos::UpgradeGuard ul(l); - it = m_unConnectedAgencies.erase(it); - } - } - catch (std::exception const& e) - { - it++; - GATEWAY_LOG(INFO) << LOG_BADGE("reconnectUnconnectedClient failed"); - } - } - m_timer->restart(); -} - -WsService::Ptr WebSocketService::webSocketClient(const std::string& _agencyID) -{ - return getValueFromMap( - _agencyID, x_agencyClients, m_agencyClients); -} - -HttpClient::Ptr WebSocketService::httpClient(const std::string& _agencyID) -{ - return getValueFromMap( - _agencyID, x_agencyClients, m_agencyHttpClients); -} - -void WebSocketService::startConnect() -{ - GATEWAY_LOG(INFO) << LOG_DESC("WebSocketService: startConnect"); - auto const& agencyConfig = m_config->config()->gatewayConfig().agencies; - auto protocol = m_config->config()->gatewayConfig().networkConfig.protocol; - for (auto const& it : agencyConfig) - { - try - { - if (protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - auto client = buildWebSocketClient(it.first); - insertIntoMap(it.first, client, x_agencyClients, m_agencyClients); - } - else if (protocol == m_config->config()->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - auto client = buildHttpClient(it.first, m_ioContext); - insertIntoMap(it.first, client, x_agencyClients, m_agencyHttpClients); - } - } - catch (std::exception const& e) - { - { - bcos::WriteGuard l(x_unConnectedAgencies); - m_unConnectedAgencies.insert(it.first); - } - GATEWAY_LOG(WARNING) << LOG_BADGE("startConnect") - << LOG_DESC("connect to the agency failed") - << LOG_KV("agency", it.first) - << LOG_KV("exception", boost::diagnostic_information(e)); - } - } - GATEWAY_LOG(INFO) << LOG_DESC("WebSocketService: startConnect success"); -} - -HttpClient::Ptr WebSocketService::buildHttpClient( - const std::string& _agencyID, std::shared_ptr _ioContext) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService: buildHttpClient") - << LOG_KV("agency", _agencyID); - { - bcos::ReadGuard l(x_agencies); - // one agencyID => one httpClient - for (const auto& endpoint : m_agencies.at(_agencyID)) - { - if (!checkEndpoint(endpoint)) - { - BOOST_THROW_EXCEPTION( - InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); - } - std::vector url; - boost::split(url, endpoint, boost::is_any_of(":"), boost::token_compress_on); - auto client = std::make_shared(*_ioContext, url[0], std::stoi(url[1])); - return client; - } - } -} - -WsService::Ptr WebSocketService::buildWebSocketClient(std::string const& _agencyID) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketClient") << LOG_KV("agency", _agencyID); - auto peers = std::make_shared(); - { - bcos::ReadGuard l(x_agencies); - for (const auto& endpoint : m_agencies.at(_agencyID)) - { - if (!checkEndpoint(endpoint)) - { - BOOST_THROW_EXCEPTION( - InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); - } - - std::vector url; - boost::split(url, endpoint, boost::is_any_of(":"), boost::token_compress_on); - NodeIPEndpoint nodeIpEndpoint = NodeIPEndpoint(url[0], std::stoi(url[1])); - peers->insert(nodeIpEndpoint); - } - } - - auto const& gatewayConfig = m_config->config()->gatewayConfig(); - auto wsConfig = std::make_shared(); - wsConfig->setModel(WsModel::Client); - wsConfig->setConnectPeers(peers); - wsConfig->setThreadPoolSize(gatewayConfig.networkConfig.threadPoolSize); - wsConfig->setDisableSsl(gatewayConfig.networkConfig.disableSsl); - wsConfig->setMaxMsgSize(gatewayConfig.maxAllowedMsgSize); - if (!wsConfig->disableSsl()) - { - wsConfig->setContextConfig(m_config->contextConfig()); - } - - auto wsInitializer = std::make_shared(); - wsInitializer->setConfig(wsConfig); - auto wsClient = std::make_shared(GATEWAY_WS_CLIENT_MODULE); - wsClient->setTimerFactory(std::make_shared()); - wsInitializer->initWsService(wsClient); - - wsClient->start(); - GATEWAY_LOG(INFO) << LOG_BADGE("WebSocketService") << LOG_DESC("connect to peer success") - << LOG_KV("agency", _agencyID); - return wsClient; -} - -WsService::Ptr WebSocketServiceFactory::buildWebSocketServer( - const GatewayConfigContext::Ptr& _config) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketServer"); - auto wsConfig = std::make_shared(); - wsConfig->setModel(WsModel::Server); - - auto const& gatewayConfig = _config->config()->gatewayConfig(); - wsConfig->setListenIP(gatewayConfig.networkConfig.listenIp); - wsConfig->setListenPort(gatewayConfig.networkConfig.listenPort); - wsConfig->setThreadPoolSize(gatewayConfig.networkConfig.threadPoolSize); - wsConfig->setDisableSsl(gatewayConfig.networkConfig.disableSsl); - if (!wsConfig->disableSsl()) - { - wsConfig->setContextConfig(_config->contextConfig()); - } - wsConfig->setMaxMsgSize(gatewayConfig.maxAllowedMsgSize); - auto wsInitializer = std::make_shared(); - wsInitializer->setConfig(wsConfig); - auto wsService = std::make_shared(GATEWAY_WS_SERVER_MODULE); - wsService->setTimerFactory(std::make_shared()); - wsInitializer->initWsService(wsService); - - return wsService; -} - -Http::Ptr WebSocketServiceFactory::buildHttpServer(const GatewayConfigContext::Ptr& _config) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("buildHttpServer"); - auto ppcConfig = _config->config(); - auto httpFactory = std::make_shared(ppcConfig->agencyID()); - return httpFactory->buildHttp(ppcConfig); -} - -WebSocketService::Ptr WebSocketServiceFactory::buildWebSocketService( - ppc::tools::PPCConfig::Ptr const& _config, std::shared_ptr _ioContext) -{ - try - { - auto gatewayConfig = std::make_shared(_config); - auto _protocol = _config->gatewayConfig().networkConfig.protocol; - // init websocket service - if (_protocol == _config->gatewayConfig().networkConfig.PROTOCOL_WEBSOCKET) - { - GATEWAY_LOG(INFO) << LOG_BADGE("buildWebSocketService"); - auto wsServer = buildWebSocketServer(gatewayConfig); - auto webSocketService = std::make_shared(gatewayConfig, wsServer); - return webSocketService; - } - else if (_protocol == _config->gatewayConfig().networkConfig.PROTOCOL_HTTP) - { - GATEWAY_LOG(INFO) << LOG_BADGE("buildHttpService"); - auto httpServer = buildHttpServer(gatewayConfig); - auto webSocketService = - std::make_shared(gatewayConfig, httpServer, _ioContext); - return webSocketService; - } - } - catch (std::exception const& e) - { - GATEWAY_LOG(ERROR) << LOG_BADGE("buildWebSocketService") - << LOG_DESC("init gateway websocket service failed, error: " + - boost::diagnostic_information(e)); - throw e; - } -} diff --git a/cpp/ppc-gateway/ppc-gateway/WebSocketService.h b/cpp/ppc-gateway/ppc-gateway/WebSocketService.h deleted file mode 100644 index 73c03ad5..00000000 --- a/cpp/ppc-gateway/ppc-gateway/WebSocketService.h +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file WebSocketService.h - * @author: shawnhe - * @date 2022-10-23 - */ - -#pragma once - -#include "Common.h" -#include "GatewayConfigContext.h" -#include "ppc-http/src/Http.h" -#include "ppc-http/src/HttpClient.h" -#include "ppc-http/src/HttpFactory.h" -#include "ppc-tools/src/config/PPCConfig.h" -#include -#include -#include -#include -#include -#include - -namespace ppc::gateway -{ -class WebSocketService -{ -public: - using Ptr = std::shared_ptr; - using WebSocketClientMap = std::unordered_map; - using HttpClientMap = std::unordered_map; - WebSocketService( - GatewayConfigContext::Ptr _config, bcos::boostssl::ws::WsService::Ptr _wsServer) - : m_config(_config), - m_agencies(_config->config()->gatewayConfig().agencies), - m_wsServer(std::move(_wsServer)), - m_protocol(_config->config()->gatewayConfig().networkConfig.protocol), - m_timer(std::make_shared( - _config->config()->gatewayConfig().reconnectTime, "connectTimer")) - {} - - WebSocketService(GatewayConfigContext::Ptr _config, ppc::http::Http::Ptr _httpServer, - std::shared_ptr _ioContext) - : m_config(_config), - m_agencies(_config->config()->gatewayConfig().agencies), - m_httpServer(std::move(_httpServer)), - m_protocol(_config->config()->gatewayConfig().networkConfig.protocol), - m_ioContext(_ioContext), - m_timer(std::make_shared( - _config->config()->gatewayConfig().reconnectTime, "connectTimer")) - {} - - virtual ~WebSocketService() = default; - - void start(); - void stop(); - - void registerGatewayUrl(const std::string& _agencyID, const std::string& _agencyUrl); - bool insertAgency(const std::string& _agencyID, const std::string& _agencyUrl); - bcos::boostssl::ws::WsService::Ptr const& webSocketServer() const { return m_wsServer; } - ppc::http::Http::Ptr httpServer() const { return m_httpServer; } - bcos::boostssl::ws::WsService::Ptr webSocketClient(const std::string& _agencyID); - ppc::http::HttpClient::Ptr httpClient(const std::string& _agencyID); - GatewayConfigContext::Ptr const& gatewayConfig() const { return m_config; } - - -protected: - virtual bcos::boostssl::ws::WsService::Ptr buildWebSocketClient(std::string const& _agencyID); - virtual ppc::http::HttpClient::Ptr buildHttpClient( - const std::string& _agencyID, std::shared_ptr _ioContext); - virtual void startConnect(); - virtual void reconnectUnconnectedClient(); - - template - void insertIntoMap(std::string const& _key, T const& _value, bcos::SharedMutex& lock, S& _map) - { - bcos::WriteGuard l(lock); - _map[_key] = _value; - } - - template - T getValueFromMap(std::string const& _key, bcos::SharedMutex& lock, S const& _map) - { - bcos::ReadGuard l(lock); - auto it = _map.find(_key); - if (it != _map.end()) - { - return it->second; - } - return nullptr; - } - -private: - int m_protocol; - GatewayConfigContext::Ptr m_config; - bcos::SharedMutex x_agencies; - std::map> m_agencies; - std::map m_urls; - - bcos::boostssl::ws::WsService::Ptr m_wsServer; - ppc::http::Http::Ptr m_httpServer; - // the timer used to try connecting to the un-connected-clients - std::shared_ptr m_timer; - - std::shared_ptr m_ioContext; - // key: agencyID, value: WebSocketClient - WebSocketClientMap m_agencyClients; - HttpClientMap m_agencyHttpClients; - bcos::SharedMutex x_agencyClients; - // connect failed for all the agency-nodes are offline - std::set m_unConnectedAgencies; - mutable bcos::SharedMutex x_unConnectedAgencies; -}; - - -class WebSocketServiceFactory -{ -public: - using Ptr = std::shared_ptr; - -public: - WebSocketServiceFactory() = default; - ~WebSocketServiceFactory() = default; - - WebSocketService::Ptr buildWebSocketService(const ppc::tools::PPCConfig::Ptr& _config, - std::shared_ptr _ioContext = nullptr); - -private: - bcos::boostssl::ws::WsService::Ptr buildWebSocketServer( - const GatewayConfigContext::Ptr& _config); - - ppc::http::Http::Ptr buildHttpServer(const GatewayConfigContext::Ptr& _config); -}; - -} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 6707fae9..de92b96b 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -84,12 +84,14 @@ void GatewayImpl::stop() } void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, - std::string const& topic, std::string const& dstInst, std::string const& componentType, - bcos::bytes&& payload) + MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) { // dispatcher to all the local front - auto p2pMessage = m_msgBuilder->build( - routeType, topic, dstInst, bcos::bytes(), componentType, std::move(payload)); + routeInfo->setDstNode(bcos::bytes()); + routeInfo->setSrcInst(m_agency); + + auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload)); + p2pMessage->setPacketType((uint16_t)GatewayPacketType::BroadcastMessage); m_localRouter->dispatcherMessage(p2pMessage, nullptr); // broadcast message to all peers @@ -97,13 +99,14 @@ void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, } -void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType, std::string const& topic, - std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, - bcos::bytes&& payload, long timeout, ReceiveMsgFunc callback) +void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout, + ReceiveMsgFunc callback) { + routeInfo->setSrcInst(m_agency); // check the localRouter - auto p2pMessage = m_msgBuilder->build( - routeType, topic, dstInst, dstNodeID, componentType, std::move(payload)); + auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload)); + p2pMessage->setPacketType((uint16_t)GatewayPacketType::P2PMessage); auto nodeList = m_localRouter->chooseReceiver(p2pMessage); // case send to the same agency diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 9d2ced23..40f97fa7 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -53,12 +53,12 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this(-1, SEND_MESSAGE_TO_FRONT_TIMEOUT)); + msgInfo.callback(std::make_shared(-1, "timeout")); } } } \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index e73f2f1d..9338f07c 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -43,10 +43,11 @@ class GatewayNodeInfo virtual bool tryAddNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; virtual void removeNodeInfo(bcos::bytes const& nodeID) = 0; - virtual std::vector chooseRouteByComponent( + virtual std::vector> chooseRouteByComponent( bool selectAll, std::string const& component) const = 0; - virtual std::vector chooseRouterByAgency(bool selectAll) const = 0; - virtual std::vector chooseRouterByTopic( + virtual std::vector> chooseRouterByAgency( + bool selectAll) const = 0; + virtual std::vector> chooseRouterByTopic( bool selectAll, std::string const& topic) const = 0; virtual void encode(bcos::bytes& data) const = 0; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp index 3c718fe9..690ded72 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -18,8 +18,9 @@ * @date 2024-08-26 */ #include "GatewayNodeInfoImpl.h" -#include "ppc-tars-protocol/Common.h" -#include "ppc-tars-protocol/impl/NodeInfoImpl.h" +#include "wedpr-protocol/protobuf/Common.h" +#include "wedpr-protocol/protobuf/NodeInfoImpl.h" +#include "wedpr-protocol/tars/Common.h" using namespace ppctars; using namespace ppc::protocol; @@ -29,21 +30,21 @@ using namespace ppc::gateway; // the gateway nodeID std::string const& GatewayNodeInfoImpl::p2pNodeID() const { - return m_inner()->p2pNodeID; + return m_inner()->p2pnodeid(); } // the agency std::string const& GatewayNodeInfoImpl::agency() const { - return m_inner()->agency; + return m_inner()->agency(); } uint32_t GatewayNodeInfoImpl::statusSeq() const { - return m_inner()->statusSeq; + return m_inner()->statusseq(); } void GatewayNodeInfoImpl::setStatusSeq(uint32_t statusSeq) { - m_inner()->statusSeq = statusSeq; + m_inner()->set_statusseq(statusSeq); } // get the node information by nodeID @@ -96,10 +97,10 @@ void GatewayNodeInfoImpl::removeNodeInfo(bcos::bytes const& nodeID) } } -std::vector GatewayNodeInfoImpl::chooseRouteByComponent( +std::vector> GatewayNodeInfoImpl::chooseRouteByComponent( bool selectAll, std::string const& component) const { - std::vector result; + std::vector> result; bcos::ReadGuard l(x_nodeList); for (auto const& it : m_nodeList) { @@ -116,9 +117,10 @@ std::vector GatewayNodeInfoImpl::chooseRouteByComponent } -vector GatewayNodeInfoImpl::chooseRouterByAgency(bool selectAll) const +std::vector> GatewayNodeInfoImpl::chooseRouterByAgency( + bool selectAll) const { - std::vector result; + std::vector> result; bcos::ReadGuard l(x_nodeList); for (auto const& it : m_nodeList) { @@ -131,10 +133,10 @@ vector GatewayNodeInfoImpl::chooseRouterByAgency(bool s return result; } -std::vector GatewayNodeInfoImpl::chooseRouterByTopic( +std::vector> GatewayNodeInfoImpl::chooseRouterByTopic( bool selectAll, std::string const& topic) const { - std::vector result; + std::vector> result; bcos::ReadGuard l(x_topicInfo); for (auto const& it : m_topicInfo) { @@ -182,34 +184,30 @@ void GatewayNodeInfoImpl::unRegisterTopic(bcos::bytes const& nodeID, std::string void GatewayNodeInfoImpl::encode(bcos::bytes& data) const { - m_inner()->nodeList.clear(); + m_inner()->clear_nodelist(); { bcos::ReadGuard l(x_nodeList); // encode nodeList for (auto const& it : m_nodeList) { auto nodeInfo = std::dynamic_pointer_cast(it.second); - m_inner()->nodeList.emplace_back(nodeInfo->inner()); + m_inner()->mutable_nodelist()->UnsafeArenaAddAllocated(nodeInfo->innerFunc()()); } } - tars::TarsOutputStream output; - m_inner()->writeTo(output); - output.getByteBuffer().swap(data); + encodePBObject(data, m_inner()); } void GatewayNodeInfoImpl::decode(bcos::bytesConstRef data) { - tars::TarsInputStream input; - input.setBuffer((const char*)data.data(), data.size()); - m_inner()->readFrom(input); + decodePBObject(m_inner(), data); { bcos::WriteGuard l(x_nodeList); // decode into m_nodeList m_nodeList.clear(); - for (auto& it : m_inner()->nodeList) + for (int i = 0; i < m_inner()->nodelist_size(); i++) { - auto nodeInfoPtr = - std::make_shared([m_entry = it]() mutable { return &m_entry; }); + auto nodeInfoPtr = std::make_shared( + [m_entry = m_inner()->nodelist(i)]() mutable { return &m_entry; }); m_nodeList.insert(std::make_pair(nodeInfoPtr->nodeID().toBytes(), nodeInfoPtr)); } } diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index 72c6d440..4e3ba366 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -19,7 +19,7 @@ */ #pragma once #include "GatewayNodeInfo.h" -#include "ppc-tars-protocol/tars/NodeInfo.h" +#include "NodeInfo.pb.h" #include #include @@ -30,12 +30,19 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo public: using Ptr = std::shared_ptr; GatewayNodeInfoImpl(std::string const& p2pNodeID, std::string const& agency) - : m_inner([inner = ppctars::GatewayNodeInfo()]() mutable { return &inner; }) + : m_inner([inner = ppc::proto::GatewayNodeInfo()]() mutable { return &inner; }) { - m_inner()->p2pNodeID = p2pNodeID; - m_inner()->agency = agency; + m_inner()->set_p2pnodeid(p2pNodeID); + m_inner()->set_agency(agency); + } + ~GatewayNodeInfoImpl() override + { + auto allocatedNodeListSize = m_inner()->nodelist_size(); + for (int i = 0; i < allocatedNodeListSize; i++) + { + m_inner()->mutable_nodelist()->UnsafeArenaReleaseLast(); + } } - ~GatewayNodeInfoImpl() override = default; // the gateway nodeID std::string const& p2pNodeID() const override; @@ -52,10 +59,11 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo bool tryAddNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; void removeNodeInfo(bcos::bytes const& nodeID) override; - std::vector chooseRouteByComponent( + std::vector> chooseRouteByComponent( bool selectAll, std::string const& component) const override; - std::vector chooseRouterByAgency(bool selectAll) const override; - std::vector chooseRouterByTopic( + std::vector> chooseRouterByAgency( + bool selectAll) const override; + std::vector> chooseRouterByTopic( bool selectAll, std::string const& topic) const override; void registerTopic(bcos::bytes const& nodeID, std::string const& topic) override; @@ -72,7 +80,7 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo virtual uint16_t nodeSize() const override { return m_nodeList.size(); } private: - std::function m_inner; + std::function m_inner; // NodeID => nodeInfo std::map m_nodeList; mutable bcos::SharedMutex x_nodeList; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 7d934daa..2db66e76 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -80,10 +80,10 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call return false; } -std::vector LocalRouter::chooseReceiver( +std::vector LocalRouter::chooseReceiver( ppc::protocol::Message::Ptr const& msg) { - std::vector receivers; + std::vector receivers; if (msg->header()->optionalField()->dstInst() != m_routerInfo->agency()) { return receivers; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index 8b931c9c..2bd18e7f 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -59,7 +59,7 @@ class LocalRouter virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic); virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic); - virtual std::vector chooseReceiver( + virtual std::vector chooseReceiver( ppc::protocol::Message::Ptr const& msg); // TODO: register component @@ -81,7 +81,6 @@ class LocalRouter return statusSeq; } - private: ppc::front::IFrontBuilder::Ptr m_frontBuilder; GatewayNodeInfo::Ptr m_routerInfo; diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h index e78aff23..3be13906 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h +++ b/cpp/ppc-gateway/ppc-gateway/p2p/Service.h @@ -57,6 +57,19 @@ class Service : public bcos::boostssl::ws::WsService return m_messageFactory; } + void setNodeEndpoints(std::set const& endPointList) + { + bcos::WriteGuard l(x_configuredNode2ID); + for (auto const& it : endPointList) + { + if (m_configuredNode2ID.count(it)) + { + continue; + } + m_configuredNode2ID.insert(std::make_pair(it, "")); + } + } + protected: void onRecvMessage(bcos::boostssl::MessageFace::Ptr _msg, bcos::boostssl::ws::WsSession::Ptr _session) override; diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h index 72eda232..55ae8d8b 100644 --- a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h @@ -22,8 +22,8 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #include "RouterTableInterface.h" -#include -#include +#include "tars/RouterTable.h" +#include #include namespace ppc::gateway diff --git a/cpp/ppc-gateway/test/demo/gateway_demo.cpp b/cpp/ppc-gateway/test/demo/gateway_demo.cpp index b5d6f6f1..aa28ad9b 100644 --- a/cpp/ppc-gateway/test/demo/gateway_demo.cpp +++ b/cpp/ppc-gateway/test/demo/gateway_demo.cpp @@ -17,7 +17,7 @@ * @author: shawnhe * @date 2022-10-28 */ - +#if 0 #include "ppc-gateway/ppc-gateway/Gateway.h" #include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" #include "ppc-tools/src/config/PPCConfig.h" @@ -98,7 +98,7 @@ int main(int argc, char* argv[]) auto info = std::make_shared(); info->taskID = taskID; info->serviceEndpoint = "endpoint1001"; - front->notifyTaskInfo(info); + front->notifyTaskInfo(taskID); auto message = buildMessage(taskID, 0); std::cout << "send message\n" @@ -127,10 +127,11 @@ int main(int argc, char* argv[]) auto info1 = std::make_shared(); info->taskID = taskID1; info->serviceEndpoint = "endpoint1001"; - front->notifyTaskInfo(info); + front->notifyTaskInfo(taskID1); while (flag != 3) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } +#endif diff --git a/cpp/ppc-gateway/test/unittests/GatewayTest.cpp b/cpp/ppc-gateway/test/unittests/GatewayTest.cpp index 50ea67c3..49d9da28 100644 --- a/cpp/ppc-gateway/test/unittests/GatewayTest.cpp +++ b/cpp/ppc-gateway/test/unittests/GatewayTest.cpp @@ -18,10 +18,11 @@ * @date 2022-10-28 */ +#if 0 #include "ppc-gateway/ppc-gateway/Gateway.h" #include "MockCache.h" #include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" -#include "ppc-gateway/ppc-gateway/TaskManager.h" +#include "ppc-gateway/ppc-gateway/" #include "ppc-tools/src/config/PPCConfig.h" #include #include @@ -93,3 +94,4 @@ BOOST_AUTO_TEST_CASE(test_frontNodeManager) } BOOST_AUTO_TEST_SUITE_END() +#endif diff --git a/cpp/ppc-pir/src/OtPIRConfig.h b/cpp/ppc-pir/src/OtPIRConfig.h index dd4d6bda..9f6deb6f 100644 --- a/cpp/ppc-pir/src/OtPIRConfig.h +++ b/cpp/ppc-pir/src/OtPIRConfig.h @@ -28,8 +28,8 @@ #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/crypto/Oprf.h" #include "ppc-framework/protocol/Protocol.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/PSIConfig.h" +#include "protocol/src/PPCMessage.h" using namespace ppc::psi; @@ -67,4 +67,4 @@ class OtPIRConfig : public PSIConfig uint16_t m_parallelism; }; -} // namespace ppc::psi \ No newline at end of file +} // namespace ppc::pir \ No newline at end of file diff --git a/cpp/ppc-pir/src/OtPIRImpl.cpp b/cpp/ppc-pir/src/OtPIRImpl.cpp index 3ee59301..cd39535b 100644 --- a/cpp/ppc-pir/src/OtPIRImpl.cpp +++ b/cpp/ppc-pir/src/OtPIRImpl.cpp @@ -21,10 +21,10 @@ #include "OtPIRImpl.h" #include "BaseOT.h" #include "Common.h" +#include "OtPIR.h" #include "ppc-framework/protocol/Protocol.h" #include "ppc-front/Common.h" -#include "OtPIR.h" -#include "ppc-tars-protocol/TarsSerialize.h" +#include "wedpr-protocol/tars/TarsSerialize.h" #include @@ -43,8 +43,8 @@ OtPIRImpl::OtPIRImpl(const OtPIRConfig::Ptr& _config, unsigned _idleTimeMs) m_ioService(std::make_shared()), m_parallelism(m_config->parallelism()) { - m_senderThreadPool = std::make_shared( - "OT-PIR-Sender", std::thread::hardware_concurrency()); + m_senderThreadPool = + std::make_shared("OT-PIR-Sender", std::thread::hardware_concurrency()); m_ot = std::make_shared(m_config->eccCrypto(), m_config->hash()); } @@ -76,8 +76,8 @@ void OtPIRImpl::start() } catch (std::exception& e) { - FRONT_LOG(WARNING) << LOG_DESC("Exception in OT-PIR Thread:") - << boost::diagnostic_information(e); + FRONT_LOG(WARNING) + << LOG_DESC("Exception in OT-PIR Thread:") << boost::diagnostic_information(e); } } PIR_LOG(INFO) << "OT-PIR exit"; @@ -143,7 +143,7 @@ void OtPIRImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) catch (std::exception const& e) { PIR_LOG(WARNING) << LOG_DESC("onReceiveMessage exception") << printPPCMsg(_msg) - << LOG_KV("error", boost::diagnostic_information(e)); + << LOG_KV("error", boost::diagnostic_information(e)); } } @@ -159,12 +159,11 @@ void OtPIRImpl::onReceivedErrorNotification(const std::string& _taskID) } } -void OtPIRImpl::onSelfError( - const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) +void OtPIRImpl::onSelfError(const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) { PIR_LOG(WARNING) << LOG_DESC("onSelfError") << LOG_KV("task", _taskID) - << LOG_KV("error", _error->errorMessage()) - << LOG_KV("noticePeer", _noticePeer); + << LOG_KV("error", _error->errorMessage()) + << LOG_KV("noticePeer", _noticePeer); auto taskState = findPendingTask(_taskID); if (!taskState) @@ -261,23 +260,23 @@ void OtPIRImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _me } default: { - PIR_LOG(WARNING) - << LOG_DESC("unsupported messageType ") << unsigned(_message->messageType()); + PIR_LOG(WARNING) << LOG_DESC("unsupported messageType ") + << unsigned(_message->messageType()); break; } } } catch (std::exception const& e) { - PIR_LOG(WARNING) - << LOG_DESC("handleReceivedMessage exception") - << LOG_KV("type", unsigned(_message->messageType())) << printPPCMsg(_message) - << LOG_KV("error", boost::diagnostic_information(e)); + PIR_LOG(WARNING) << LOG_DESC("handleReceivedMessage exception") + << LOG_KV("type", unsigned(_message->messageType())) + << printPPCMsg(_message) + << LOG_KV("error", boost::diagnostic_information(e)); } }); } -void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) +void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) { // 接收方不需要记录taskID // if (m_taskState->taskDone()) @@ -285,21 +284,27 @@ void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) // return; // } PIR_LOG(DEBUG) << LOG_BADGE("onHelloReceiver") << LOG_KV("taskID", _message->taskID()) - << LOG_KV("seq", _message->seq()) << LOG_KV("length", _message->length()); + << LOG_KV("seq", _message->seq()) << LOG_KV("length", _message->length()); ppctars::SenderMessageParams senderMessageParams; ppctars::serialize::decode(*_message->data(), senderMessageParams); // crypto::SenderMessage senderMessage; // TODO: how to find my dataset // m_taskState->setReader(io::LineReader::Ptr _reader, int64_t _readerParam) - try { + try + { // auto writer = m_taskState->reader(); auto receiver = findReceiver(_message->taskID()); auto path = receiver.path; - PIR_LOG(INFO) << LOG_BADGE("onHelloReceiver") << LOG_KV("taskID", _message->taskID()) << LOG_KV("requestAgencyDataset", path) << LOG_KV("sendObfuscatedHash", std::string(senderMessageParams.sendObfuscatedHash.begin(), senderMessageParams.sendObfuscatedHash.end())); + PIR_LOG(INFO) << LOG_BADGE("onHelloReceiver") << LOG_KV("taskID", _message->taskID()) + << LOG_KV("requestAgencyDataset", path) + << LOG_KV("sendObfuscatedHash", + std::string(senderMessageParams.sendObfuscatedHash.begin(), + senderMessageParams.sendObfuscatedHash.end())); auto messageKeypair = m_ot->prepareDataset(senderMessageParams.sendObfuscatedHash, path); - auto receiverMessage = m_ot->receiverGenerateMessage(senderMessageParams.pointX, senderMessageParams.pointY, messageKeypair, senderMessageParams.pointZ); + auto receiverMessage = m_ot->receiverGenerateMessage(senderMessageParams.pointX, + senderMessageParams.pointY, messageKeypair, senderMessageParams.pointZ); ppctars::ReceiverMessageParams receiverMessageParams; receiverMessageParams.encryptMessagePair = receiverMessage.encryptMessagePair; receiverMessageParams.encryptCipher = receiverMessage.encryptCipher; @@ -307,60 +312,65 @@ void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message) // PIR_LOG(INFO) << LOG_BADGE("buildPPCMessage"); auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR), - uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, - std::make_shared()); + uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, + std::make_shared()); message->setMessageType(uint8_t(OTPIRMessageType::RESULTS)); ppctars::serialize::encode(receiverMessageParams, *message->data()); // PIR_LOG(INFO) << LOG_BADGE("asyncSendMessage"); m_config->front()->asyncSendMessage( - m_taskState->peerID(), message, m_config->networkTimeout(), - [self = weak_from_this()](bcos::Error::Ptr _error) { - auto receiver = self.lock(); - if (!receiver) - { - return; - } - if (_error && _error->errorCode()) - { - receiver->onReceiverTaskDone(std::move(_error)); - } - }, - nullptr); + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); auto endTask = std::make_shared(m_taskState->task()->id()); m_taskState->onTaskFinished(endTask, true); - } - catch (bcos::Error const& e) { - PIR_LOG(WARNING) << LOG_DESC("onHelloReceiver exception") - << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); - onSelfError( - m_taskID, std::make_shared(e.errorCode(), e.errorMessage()), true); + } + catch (bcos::Error const& e) + { + PIR_LOG(WARNING) << LOG_DESC("onHelloReceiver exception") << LOG_KV("code", e.errorCode()) + << LOG_KV("msg", e.errorMessage()); + onSelfError(m_taskID, std::make_shared(e.errorCode(), e.errorMessage()), true); } } void OtPIRImpl::onSnederResults(ppc::front::PPCMessageFace::Ptr _message) { PIR_LOG(DEBUG) << LOG_BADGE("onSnederResults") << LOG_KV("taskID", _message->taskID()) - << LOG_KV("seq", _message->seq()); + << LOG_KV("seq", _message->seq()); ppctars::ReceiverMessageParams receiverMessageParams; ppctars::serialize::decode(*_message->data(), receiverMessageParams); crypto::SenderMessage senderMessage = findSender(_message->taskID()); - PIR_LOG(DEBUG) << LOG_BADGE("onSnederResults") << LOG_KV("scalarBlidingB", toHex(senderMessage.scalarBlidingB)) - << LOG_KV("pointWList Size", receiverMessageParams.pointWList.size()) - << LOG_KV("encryptCipher Size", receiverMessageParams.encryptCipher.size()) - << LOG_KV("encryptMessagePair Size", receiverMessageParams.encryptMessagePair.size()); - bcos::bytes result = m_ot->finishSender(senderMessage.scalarBlidingB, receiverMessageParams.pointWList, receiverMessageParams.encryptMessagePair, receiverMessageParams.encryptCipher); + PIR_LOG(DEBUG) << LOG_BADGE("onSnederResults") + << LOG_KV("scalarBlidingB", toHex(senderMessage.scalarBlidingB)) + << LOG_KV("pointWList Size", receiverMessageParams.pointWList.size()) + << LOG_KV("encryptCipher Size", receiverMessageParams.encryptCipher.size()) + << LOG_KV("encryptMessagePair Size", + receiverMessageParams.encryptMessagePair.size()); + bcos::bytes result = + m_ot->finishSender(senderMessage.scalarBlidingB, receiverMessageParams.pointWList, + receiverMessageParams.encryptMessagePair, receiverMessageParams.encryptCipher); saveResults(result); auto endTask = std::make_shared(m_taskState->task()->id()); m_taskState->onTaskFinished(endTask, true); } -void OtPIRImpl::asyncRunTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) +void OtPIRImpl::asyncRunTask( + ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) { - //TODO + // TODO PIR_LOG(INFO) << LOG_DESC("receive a task") << LOG_KV("taskID", _task->id()); - m_taskID = _task->id(); + m_taskID = _task->id(); addTask(_task, [self = weak_from_this(), taskID = _task->id(), _onTaskFinished]( ppc::protocol::TaskResult::Ptr&& _result) { auto result = std::move(_result); @@ -429,14 +439,17 @@ void OtPIRImpl::asyncRunTask() PIR_LOG(TRACE) << LOG_DESC("originData") << LOG_KV("originData", originData); PirTaskMessage taskMessage = parseJson(originData); - PIR_LOG(TRACE) << LOG_DESC("taskMessage") << LOG_KV("requestAgencyDataset", taskMessage.requestAgencyDataset) << LOG_KV("prefixLength", taskMessage.prefixLength) << LOG_KV("searchId", taskMessage.searchId); + PIR_LOG(TRACE) << LOG_DESC("taskMessage") + << LOG_KV("requestAgencyDataset", taskMessage.requestAgencyDataset) + << LOG_KV("prefixLength", taskMessage.prefixLength) + << LOG_KV("searchId", taskMessage.searchId); auto writer = loadWriter(task->id(), dataResource, m_enableOutputExists); m_taskState->setWriter(writer); runSenderGenerateCipher(taskMessage); } else if (role == uint16_t(PartyType::Server)) { - // server接受任务请求，初始化reader + // server接受任务请求，初始化reader PIR_LOG(TRACE) << LOG_DESC("Server init"); crypto::ReceiverMessage receiverMessage; receiverMessage.path = dataResource->desc()->path(); @@ -444,7 +457,6 @@ void OtPIRImpl::asyncRunTask() // m_resource = dataResource; // auto reader = loadReader(task->id(), dataResource, DataSchema::Bytes); // m_taskState->setReader(reader, -1); - } else { @@ -458,14 +470,14 @@ void OtPIRImpl::asyncRunTask() catch (bcos::Error const& e) { PIR_LOG(WARNING) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) - << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); + << LOG_KV("code", e.errorCode()) << LOG_KV("msg", e.errorMessage()); onSelfError( task->id(), std::make_shared(e.errorCode(), e.errorMessage()), true); } catch (const std::exception& e) { PIR_LOG(WARNING) << LOG_DESC("asyncRunTask exception") << printTaskInfo(task) - << LOG_KV("error", boost::diagnostic_information(e)); + << LOG_KV("error", boost::diagnostic_information(e)); onSelfError(task->id(), std::make_shared((int)OTPIRRetCode::ON_EXCEPTION, "exception caught while running task: " + boost::diagnostic_information(e)), @@ -473,7 +485,7 @@ void OtPIRImpl::asyncRunTask() } // notify the taskInfo to the front - error = m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + error = m_config->front()->notifyTaskInfo(task->id()); if (error && error->errorCode()) { onSelfError(task->id(), error, true); @@ -486,7 +498,9 @@ void OtPIRImpl::runSenderGenerateCipher(PirTaskMessage taskMessage) { return; } - crypto::SenderMessage senderMessage = m_ot->senderGenerateCipher(bcos::bytes(taskMessage.searchId.begin(), taskMessage.searchId.end()), taskMessage.prefixLength); + crypto::SenderMessage senderMessage = m_ot->senderGenerateCipher( + bcos::bytes(taskMessage.searchId.begin(), taskMessage.searchId.end()), + taskMessage.prefixLength); ppctars::SenderMessageParams senderMessageParams; senderMessageParams.pointX = senderMessage.pointX; senderMessageParams.pointY = senderMessage.pointY; @@ -494,28 +508,31 @@ void OtPIRImpl::runSenderGenerateCipher(PirTaskMessage taskMessage) // senderMessageParams.requestAgencyDataset = taskMessage.requestAgencyDataset; senderMessageParams.sendObfuscatedHash = senderMessage.sendObfuscatedHash; auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR), - uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, - std::make_shared()); + uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, std::make_shared()); message->setMessageType(uint8_t(OTPIRMessageType::HELLO_RECEIVER)); ppctars::serialize::encode(senderMessageParams, *message->data()); addSender(senderMessage); - // PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) << LOG_KV("requestAgencyDataset", senderMessageParams.requestAgencyDataset); - PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) << LOG_KV("sendObfuscatedHash", std::string(senderMessageParams.sendObfuscatedHash.begin(), senderMessageParams.sendObfuscatedHash.end())); + // PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) << + // LOG_KV("requestAgencyDataset", senderMessageParams.requestAgencyDataset); + PIR_LOG(INFO) << LOG_BADGE("runSenderGenerateCipher") << LOG_KV("taskID", m_taskID) + << LOG_KV("sendObfuscatedHash", + std::string(senderMessageParams.sendObfuscatedHash.begin(), + senderMessageParams.sendObfuscatedHash.end())); // senderMessageParams.taskId = m_taskID; m_config->front()->asyncSendMessage( - m_taskState->peerID(), message, m_config->networkTimeout(), - [self = weak_from_this()](bcos::Error::Ptr _error) { - auto receiver = self.lock(); - if (!receiver) - { - return; - } - if (_error && _error->errorCode()) - { - receiver->onReceiverTaskDone(std::move(_error)); - } - }, - nullptr); + m_taskState->peerID(), message, m_config->networkTimeout(), + [self = weak_from_this()](bcos::Error::Ptr _error) { + auto receiver = self.lock(); + if (!receiver) + { + return; + } + if (_error && _error->errorCode()) + { + receiver->onReceiverTaskDone(std::move(_error)); + } + }, + nullptr); } @@ -537,7 +554,7 @@ void OtPIRImpl::onReceiverTaskDone(bcos::Error::Ptr _error) m_taskState->onTaskFinished(m_taskResult, true); PIR_LOG(INFO) << LOG_BADGE("receiverTaskDone") << LOG_KV("taskID", m_taskID) - << LOG_KV("detail", message); + << LOG_KV("detail", message); } @@ -546,17 +563,16 @@ void OtPIRImpl::saveResults(bcos::bytes result) PIR_LOG(INFO) << LOG_BADGE("saveResults") LOG_KV("taskID", m_taskID); try { - DataBatch::Ptr finalResults = std::make_shared(); - finalResults->append(result); - m_taskState->writeLines(finalResults, DataSchema::Bytes); + DataBatch::Ptr finalResults = std::make_shared(); + finalResults->append(result); + m_taskState->writeLines(finalResults, DataSchema::Bytes); } catch (const std::exception& e) { PIR_LOG(WARNING) << LOG_KV("taskID", m_taskID) - << LOG_KV("error", boost::diagnostic_information(e)); + << LOG_KV("error", boost::diagnostic_information(e)); auto error = std::make_shared( - (int)OTPIRRetCode::ON_EXCEPTION, boost::diagnostic_information(e)); + (int)OTPIRRetCode::ON_EXCEPTION, boost::diagnostic_information(e)); onReceiverTaskDone(error); } } - diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h index 55c7f671..2b8de4ad 100644 --- a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h @@ -35,7 +35,6 @@ namespace ppc::psi { - class BsEcdhPSIImpl : public BsEcdhPSIInterface, public std::enable_shared_from_this { public: diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h index 7c21bd95..6a3534fb 100644 --- a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h +++ b/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h @@ -27,7 +27,6 @@ namespace ppc::psi { - class BsEcdhPSIInterface { public: diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/Common.h b/cpp/ppc-psi/src/bs-ecdh-psi/Common.h index 4e45004e..3e241b5b 100644 --- a/cpp/ppc-psi/src/bs-ecdh-psi/Common.h +++ b/cpp/ppc-psi/src/bs-ecdh-psi/Common.h @@ -1,22 +1,22 @@ /** -* Copyright (C) 2023 WeDPR. -* SPDX-License-Identifier: Apache-2.0 -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* @file Common.h -* @author: shawnhe -* @date 2023-09-22 -*/ + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2023-09-22 + */ #pragma once @@ -29,7 +29,6 @@ namespace ppc::psi { - DERIVE_PPC_EXCEPTION(BsEcdhException); #define BS_ECDH_PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: BS-ECDH-PSI") diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h index e01179be..397c837e 100644 --- a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h +++ b/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h @@ -22,7 +22,7 @@ #include "ppc-framework/protocol/Protocol.h" #include "ppc-framework/rpc/RpcTypeDef.h" -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include #include #include @@ -31,7 +31,6 @@ namespace ppc::psi { - struct GetTaskStatusRequest { using Ptr = std::shared_ptr; diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h index 59a74e84..2f07e2cc 100644 --- a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h @@ -28,8 +28,8 @@ #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/crypto/Oprf.h" #include "ppc-framework/protocol/Protocol.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/PSIConfig.h" +#include "protocol/src/PPCMessage.h" namespace ppc::psi { diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp index a1aa8a62..197670f9 100644 --- a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp @@ -137,7 +137,7 @@ void CM2020PSIImpl::asyncRunTask() taskPair.second(std::move(result)); // mark this taskID as occupied - m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + m_config->front()->notifyTaskInfo(task->id()); return; } @@ -210,7 +210,7 @@ void CM2020PSIImpl::asyncRunTask() } // notify the taskInfo to the front - error = m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + error = m_config->front()->notifyTaskInfo(task->id()); if (error && error->errorCode()) { onSelfError(task->id(), error, true); diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h index 06012237..7cd416e0 100644 --- a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h +++ b/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h @@ -35,8 +35,8 @@ #include "ppc-framework/protocol/Task.h" #include "ppc-framework/task/TaskFrameworkInterface.h" #include "ppc-front/ppc-front/PPCChannel.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-tools/src/common/TransTools.h" +#include "protocol/src/PPCMessage.h" namespace ppc::psi { diff --git a/cpp/ppc-psi/src/cm2020-psi/Common.h b/cpp/ppc-psi/src/cm2020-psi/Common.h index 52de3fb2..237f1a65 100644 --- a/cpp/ppc-psi/src/cm2020-psi/Common.h +++ b/cpp/ppc-psi/src/cm2020-psi/Common.h @@ -28,7 +28,6 @@ namespace ppc::psi { - DERIVE_PPC_EXCEPTION(CM2020Exception); #define CM2020_PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI: CM2020-PSI") diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp index f8b5d3dd..2e4f46b0 100644 --- a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp @@ -22,8 +22,8 @@ #include "CM2020PSI.h" #include "openssl/rand.h" #include "ppc-crypto/src/prng/AESPRNG.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" #include "ppc-tools/src/common/TransTools.h" +#include "wedpr-protocol/tars/TarsSerialize.h" #include using namespace ppc::psi; diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h index 8bd33468..7c3184fd 100644 --- a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h @@ -31,7 +31,6 @@ namespace ppc::psi { - class CM2020PSIReceiver : public std::enable_shared_from_this { public: diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp index 7fa95bb1..860f8bfd 100644 --- a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp +++ b/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp @@ -22,8 +22,8 @@ #include "CM2020PSI.h" #include "openssl/rand.h" #include "ppc-crypto/src/prng/AESPRNG.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" #include "ppc-tools/src/common/TransTools.h" +#include "wedpr-protocol/tars/TarsSerialize.h" #include using namespace ppc::psi; @@ -499,7 +499,7 @@ void CM2020PSISender::onMatrixColumnReceived(PPCMessageFace::Ptr _message) return; } CM2020_PSI_LOG(INFO) << LOG_BADGE("onMatrixColumnReceived") << LOG_KV("taskID", m_taskID) - << LOG_KV("seq", _message->seq()); + << LOG_KV("seq", _message->seq()); try { m_channel->onMessageArrived(uint8_t(CM2020PSIMessageType::MATRIX), _message); diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h index 63d09b3b..d42c8bf8 100644 --- a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h @@ -23,8 +23,8 @@ #include "ppc-framework/crypto/ECDHCrypto.h" #include "ppc-framework/io/DataResourceLoader.h" #include "ppc-framework/protocol/Protocol.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-tools/src/config/PPCConfig.h" +#include "protocol/src/PPCMessage.h" #include #include diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp index 70bfa486..55e22be3 100644 --- a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp @@ -50,7 +50,7 @@ void EcdhConnPSIImpl::asyncRunTask( // init process triggleProcess((uint8_t)EcdhConnProcess::HandShakeProcess, -1); // notify the taskInfo to the front - m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + m_config->front()->notifyTaskInfo(_task->id()); if (role == uint16_t(PartyType::Client)) { ECDH_CONN_LOG(INFO) << LOG_DESC("Client do the Task") << LOG_KV("taskID", _task->id()); diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h index 3806491e..53d7cd4a 100644 --- a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h +++ b/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h @@ -27,9 +27,9 @@ #include "core/EcdhConnPSIServer.h" #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/task/TaskFrameworkInterface.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-rpc/src/RpcFactory.h" #include "ppc-tools/src/common/ConcurrentPool.h" +#include "protocol/src/PPCMessage.h" #include #include #include diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h index 3f5b1bd7..9b256a72 100644 --- a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h @@ -2,8 +2,8 @@ #include "EcdhMultiPSIMessageFactory.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/protocol/Protocol.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/PSIConfig.h" +#include "protocol/src/PPCMessage.h" #include #include #include @@ -28,7 +28,8 @@ class EcdhMultiPSIConfig : public PSIConfig m_dataBatchSize(_dataBatchSize) {} - virtual ~EcdhMultiPSIConfig() { + virtual ~EcdhMultiPSIConfig() + { if (m_threadPool) { m_threadPool->stop(); diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp index d881405d..24fa7194 100644 --- a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp @@ -168,7 +168,7 @@ void EcdhMultiPSIImpl::asyncRunTask( } // notify the taskInfo to the front - m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + m_config->front()->notifyTaskInfo(_task->id()); } catch (bcos::Error const& e) { diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h index 91f898a5..b2d8e968 100644 --- a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h @@ -1,12 +1,12 @@ #pragma once -#include "ppc-psi/src/psi-framework/TaskGuarder.h" #include "Common.h" #include "EcdhMultiPSIConfig.h" #include "core/EcdhMultiPSICalculator.h" #include "core/EcdhMultiPSIMaster.h" #include "core/EcdhMultiPSIPartner.h" #include "ppc-framework/task/TaskFrameworkInterface.h" -#include "ppc-protocol/src/PPCMessage.h" +#include "ppc-psi/src/psi-framework/TaskGuarder.h" +#include "protocol/src/PPCMessage.h" #include #include #include diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h index a51bd0b3..d4a8bde8 100644 --- a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h +++ b/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h @@ -1,6 +1,6 @@ #pragma once -#include "ppc-psi/src/psi-framework/protocol/PSIMessage.h" #include "Common.h" +#include "ppc-psi/src/psi-framework/protocol/PSIMessage.h" namespace ppc::psi { class EcdhMultiPSIMessageFactory : public PSIMessageFactoryImpl diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp index 4f79196b..051ee101 100644 --- a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp +++ b/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp @@ -123,7 +123,7 @@ void EcdhPSIImpl::runPSI(TaskState::Ptr const& _taskState) { ECDH_LOG(INFO) << LOG_DESC("runPSI") << printTaskInfo(_taskState->task()); // notify the taskInfo to the front - m_config->front()->notifyTaskInfo(std::make_shared(_taskState->task()->id())); + m_config->front()->notifyTaskInfo(_taskState->task()->id()); // the psi-client send handshake request to the server if (_taskState->task()->selfParty()->partyIndex() == (int)PartyType::Client) { diff --git a/cpp/ppc-psi/src/labeled-psi/Common.h b/cpp/ppc-psi/src/labeled-psi/Common.h index 78ab15e3..95fe31f1 100644 --- a/cpp/ppc-psi/src/labeled-psi/Common.h +++ b/cpp/ppc-psi/src/labeled-psi/Common.h @@ -28,7 +28,6 @@ namespace ppc::psi { - DERIVE_PPC_EXCEPTION(ConfigPowersDagException); DERIVE_PPC_EXCEPTION(TooManyItemsException); DERIVE_PPC_EXCEPTION(ResultPackageException); diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h b/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h index fc2524bc..01b7ebdd 100644 --- a/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h @@ -24,8 +24,8 @@ #include "ppc-crypto/src/oprf/EcdhOprf.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/crypto/Oprf.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/PSIConfig.h" +#include "protocol/src/PPCMessage.h" #include #include diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp index bac2fa70..c9726b47 100644 --- a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp @@ -25,7 +25,7 @@ #include "core/SenderDB.h" #include "core/TaskCommand.h" #include "ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "wedpr-protocol/tars/TarsSerialize.h" using namespace ppc::psi; using namespace ppc::protocol; @@ -122,7 +122,7 @@ void LabeledPSIImpl::asyncRunTask( addReceiver(receiver); // notify the taskInfo to the front - error = m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + error = m_config->front()->notifyTaskInfo(_task->id()); if (error && error->errorCode()) { onSelfError(_task->id(), error, true); @@ -347,7 +347,7 @@ void LabeledPSIImpl::asyncRunSenderTask( addPendingTask(taskState); // notify the taskInfo to the front - error = m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + error = m_config->front()->notifyTaskInfo(_task->id()); if (error && error->errorCode()) { onSelfError(_task->id(), error, true); diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h index bafa00c1..5765187b 100644 --- a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h +++ b/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h @@ -33,8 +33,8 @@ #include "ppc-framework/protocol/Task.h" #include "ppc-framework/task/TaskFrameworkInterface.h" #include "ppc-front/ppc-front/PPCChannel.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-tools/src/common/TransTools.h" +#include "protocol/src/PPCMessage.h" namespace ppc::psi { @@ -119,7 +119,7 @@ class LabeledPSIImpl : public bcos::Worker, protected: // allow the output-path exists, for ut bool m_enableOutputExists = false; - + private: void waitSignal() { diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h index 9cbf3b97..70b21e82 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h @@ -35,7 +35,6 @@ namespace ppc::psi { - struct BaseParams { size_t maxItemsPerBin; diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp index f9b3ebe7..37221243 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp @@ -30,7 +30,7 @@ #include "bcos-utilities/DataConvertUtility.h" #include "ppc-psi/src/labeled-psi/Common.h" #include "ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "wedpr-protocol/tars/TarsSerialize.h" using namespace ppc::psi; using namespace ppc::protocol; diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h index 7e392d19..cfb54d5d 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h @@ -24,10 +24,10 @@ #include "ppc-framework/protocol/Task.h" #include "ppc-framework/task/TaskFrameworkInterface.h" #include "ppc-front/ppc-front/PPCChannel.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/labeled-psi/LabeledPSIConfig.h" #include "ppc-psi/src/labeled-psi/protocol/Protocol.h" #include "ppc-tools/src/common/Progress.h" +#include "protocol/src/PPCMessage.h" #include #include diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp index 6cf086f5..419140cf 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp @@ -25,8 +25,8 @@ #include "LabeledPSIParams.h" #include "LabeledPSISender.h" #include "QueryPackage.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" #include "ppc-tools/src/common/TransTools.h" +#include "wedpr-protocol/tars/TarsSerialize.h" using namespace ppc::psi; using namespace ppc::protocol; diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h index 32497157..bddd01c6 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h +++ b/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h @@ -33,9 +33,9 @@ #include "ppc-framework/crypto/Oprf.h" #include "ppc-framework/protocol/Task.h" #include "ppc-front/ppc-front/PPCChannel.h" -#include "ppc-protocol/src/PPCMessage.h" #include "ppc-psi/src/labeled-psi/LabeledPSIConfig.h" #include "ppc-psi/src/labeled-psi/protocol/Protocol.h" +#include "protocol/src/PPCMessage.h" namespace ppc::psi { diff --git a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h index 7e0d32f0..b6a55ad2 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h +++ b/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h @@ -29,7 +29,6 @@ namespace ppc::psi { - /** Stores a decrypted and decoded PSI response and optionally a labeled PSI response. */ diff --git a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h index c9580b59..a84c6935 100644 --- a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h +++ b/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h @@ -55,7 +55,7 @@ #include "../protocol/Protocol.h" #include "BinBundle.h" #include "SenderCache.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" +#include "wedpr-protocol/tars/TarsSerialize.h" namespace ppc::psi { diff --git a/cpp/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/ppc-psi/src/psi-framework/TaskGuarder.h index a51c48f1..aea7e3bf 100644 --- a/cpp/ppc-psi/src/psi-framework/TaskGuarder.h +++ b/cpp/ppc-psi/src/psi-framework/TaskGuarder.h @@ -24,7 +24,7 @@ #include "../PSIConfig.h" #include "TaskState.h" #include "ppc-framework/protocol/Protocol.h" -#include "ppc-protocol/src/PPCMessage.h" +#include "protocol/src/PPCMessage.h" #include namespace ppc::psi diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp index 1a93d668..376a9f91 100644 --- a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp +++ b/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp @@ -18,7 +18,7 @@ * @date 2022-11-9 */ #include "PSIMessage.h" -#include "ppc-tars-protocol/Common.h" +#include "wedpr-protocol/tars/Common.h" using namespace ppc::psi; using namespace bcos; diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp index 362857c7..7decb401 100644 --- a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp +++ b/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp @@ -72,7 +72,7 @@ void RA2018PSIImpl::asyncRunTask( { if (m_disabled) { - m_config->front()->notifyTaskInfo(std::make_shared(_task->id())); + m_config->front()->notifyTaskInfo(_task->id()); auto taskResult = std::make_shared(_task->id()); auto error = BCOS_ERROR_PTR( (int)RA2018PSIDisabled, "The ra2018-psi has been disabled by this node!"); @@ -189,7 +189,7 @@ void RA2018PSIImpl::runPSI(TaskState::Ptr const& _taskState) { auto task = _taskState->task(); // notify the taskInfo to the front - m_config->front()->notifyTaskInfo(std::make_shared(task->id())); + m_config->front()->notifyTaskInfo(task->id()); switch (task->selfParty()->partyIndex()) { case (int)ppc::protocol::PartyType::Client: diff --git a/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h b/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h index c14d7c81..e6a49166 100644 --- a/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h +++ b/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h @@ -21,7 +21,7 @@ #include "../../psi-framework/protocol/PSIMessage.h" #include "../Common.h" #include "../core/CuckooFilterInfo.h" -#include "ppc-tars-protocol/Common.h" +#include "wedpr-protocol/tars/Common.h" namespace ppc::psi { diff --git a/cpp/ppc-rpc/src/Rpc.h b/cpp/ppc-rpc/src/Rpc.h index a48a9b2b..59eadce5 100644 --- a/cpp/ppc-rpc/src/Rpc.h +++ b/cpp/ppc-rpc/src/Rpc.h @@ -21,7 +21,7 @@ #include "ppc-framework/front/FrontInterface.h" #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/rpc/RpcStatusInterface.h" -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include #include #include diff --git a/cpp/ppc-rpc/src/RpcMemory.cpp b/cpp/ppc-rpc/src/RpcMemory.cpp index c44fd9c7..86074c47 100644 --- a/cpp/ppc-rpc/src/RpcMemory.cpp +++ b/cpp/ppc-rpc/src/RpcMemory.cpp @@ -109,22 +109,6 @@ TaskResult::Ptr RpcMemory::getTaskStatus(const std::string& _taskID) return m_tasks[_taskID].second; } -bcos::Error::Ptr RpcMemory::insertGateway( - const std::string& _agencyID, const std::string& _endpoint) -{ - try - { - std::vector gatewayList; - gatewayList.push_back({_agencyID, _endpoint}); - m_gateway->registerGateway(gatewayList); - return nullptr; - } - catch (std::exception const& e) - { - return std::make_shared( - PPCRetCode::EXCEPTION, "insertGateway error: " + boost::diagnostic_information(e)); - } -} bcos::Error::Ptr RpcMemory::deleteGateway(const std::string& _agencyID) { diff --git a/cpp/ppc-rpc/src/RpcMemory.h b/cpp/ppc-rpc/src/RpcMemory.h index e275b3ea..30430027 100644 --- a/cpp/ppc-rpc/src/RpcMemory.h +++ b/cpp/ppc-rpc/src/RpcMemory.h @@ -20,23 +20,18 @@ */ #pragma once -#include "ppc-framework/gateway/GatewayInterface.h" #include "ppc-framework/rpc/RpcStatusInterface.h" #include #include namespace ppc::rpc { - class RpcMemory : public RpcStatusInterface { public: using Ptr = std::shared_ptr; - RpcMemory(ppc::gateway::GatewayInterface::Ptr _gateway) - : m_gateway(std::move(_gateway)), - m_taskCleaner(std::make_shared(60 * 60 * 1000, "taskCleaner")) - {} + RpcMemory() : m_taskCleaner(std::make_shared(60 * 60 * 1000, "taskCleaner")) {} ~RpcMemory() override = default; void start() override; @@ -54,8 +49,6 @@ class RpcMemory : public RpcStatusInterface void cleanTask(); private: - ppc::gateway::GatewayInterface::Ptr m_gateway; - mutable bcos::SharedMutex x_tasks; std::unordered_map> m_tasks; std::shared_ptr m_taskCleaner; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h deleted file mode 100644 index 2209f62f..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/FrontServiceClient.h +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @brief client for front service - * @file FrontServiceClient.h - * @author: shawnhe - * @date 2022-10-20 - */ - -#pragma once - -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" - -#include "FrontService.h" -#include "ppc-framework/front/FrontInterface.h" -#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" -#include -#include - -#include - -#define FRONTCLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("FrontServiceClient") - -namespace ppctars -{ -class FrontServiceClient : public ppc::front::FrontInterface -{ -public: - void start() override {} - void stop() override {} - - explicit FrontServiceClient(const ppctars::FrontServicePrx& proxy) : m_proxy(proxy) {} - - void onReceiveMessage( - ppc::front::PPCMessageFace::Ptr _message, ppc::front::ErrorCallbackFunc _callback) override - { - class Callback : public FrontServicePrxCallback - { - public: - explicit Callback(ppc::front::ErrorCallbackFunc callback) - : m_callback(std::move(callback)) - {} - - void callback_onReceiveMessage(const ppctars::Error& ret) override - { - if (!m_callback) - { - return; - } - m_callback(toBcosError(ret)); - } - - void callback_onReceiveMessage_exception(tars::Int32 ret) override - { - if (!m_callback) - { - return; - } - m_callback(toBcosError(ret)); - } - - private: - ppc::front::ErrorCallbackFunc m_callback; - }; - - auto startT = bcos::utcSteadyTime(); - // encode message to bytes - bcos::bytes buffer; - _message->encode(buffer); - - FRONTCLIENT_LOG(TRACE) << LOG_DESC("after decode") - << LOG_KV("taskType", unsigned(_message->taskType())) - << LOG_KV("algorithmType", unsigned(_message->algorithmType())) - << LOG_KV("messageType", unsigned(_message->messageType())) - << LOG_KV("seq", _message->seq()) - << LOG_KV("taskID", _message->taskID()); - - m_proxy->tars_set_timeout(c_networkTimeout) - ->async_onReceiveMessage( - new Callback(_callback), std::vector(buffer.begin(), buffer.end())); - BCOS_LOG(TRACE) << LOG_DESC("call front onReceiveMessage") - << LOG_KV("msgSize", buffer.size()) - << LOG_KV("timecost", bcos::utcSteadyTime() - startT); - } - - // Note: since ppc-front is integrated with the node, no-need to implement this method - bcos::Error::Ptr notifyTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr) override - { - throw std::runtime_error("notifyTaskInfo: unimplemented interface!"); - } - - // Note: since ppc-front is integrated with the node, no-need to implement this method - // erase the task-info when task finished - bcos::Error::Ptr eraseTaskInfo(std::string const&) override - { - throw std::runtime_error("eraseTaskInfo: unimplemented interface!"); - } - - // Note: since ppc-front is integrated with the node, no-need to implement this method - void asyncSendMessage(const std::string&, ppc::front::PPCMessageFace::Ptr, uint32_t _timeout, - ppc::front::ErrorCallbackFunc _callback, ppc::front::CallbackFunc _respCallback) override - { - throw std::runtime_error("asyncSendMessage: unimplemented interface!"); - } - - // Note: since ppc-front is integrated with the node, no-need to implement this method - // send response when receiving message from given agencyID - void asyncSendResponse(const std::string&, std::string const&, ppc::front::PPCMessageFace::Ptr, - ppc::front::ErrorCallbackFunc) override - { - throw std::runtime_error("asyncSendResponse: unimplemented interface!"); - } - - // Note: since ppc-front is integrated with the node, no-need to implement this method - void asyncGetAgencyList(ppc::front::GetAgencyListCallback) override - { - throw std::runtime_error("asyncGetAgencyList: unimplemented interface!"); - } - -private: - // 1800s - const int c_networkTimeout = 1800000; - - ppctars::FrontServicePrx m_proxy; -}; -} // namespace ppctars diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h b/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h deleted file mode 100644 index 52db1dcc..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.h +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @brief client for gateway service - * @file GatewayServiceClient.h - * @author: shawnhe - * @date 2022-10-20 - */ - -#pragma once - -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" - -#include "GatewayService.h" -#include "ppc-framework/Common.h" -#include "ppc-framework/gateway/GatewayInterface.h" -#include "ppc-framework/protocol/Protocol.h" -#include "ppc-tars-protocol/Common.h" -#include "ppc-tars-protocol/TarsServantProxyCallback.h" -#include -#include -#include - -#include - -#define GATEWAYCLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("GatewayServiceClient") - -namespace ppctars -{ -class GatewayServiceClient : public ppc::gateway::GatewayInterface -{ -public: - void start() override {} - void stop() override {} - - explicit GatewayServiceClient(std::string const& _gatewayServiceName, - const ppctars::GatewayServicePrx& _prx, int _holdingMessageMinutes) - : m_gatewayServiceName(_gatewayServiceName), - m_prx(_prx), - m_networkTimeout(_holdingMessageMinutes * 60 * 1000) // convert to ms - { - BCOS_LOG(INFO) << LOG_DESC("GatewayServiceClient") - << LOG_KV("networkTimeout", m_networkTimeout); - } - - - void asyncSendMessage(const std::string& _agencyID, ppc::front::PPCMessageFace::Ptr _message, - ppc::gateway::ErrorCallbackFunc _callback) override - { - class Callback : public GatewayServicePrxCallback - { - public: - explicit Callback(ppc::gateway::ErrorCallbackFunc callback) - : m_callback(std::move(callback)) - {} - - void callback_asyncSendMessage(const ppctars::Error& ret) override - { - s_tarsTimeoutCount.store(0); - if (!m_callback) - { - return; - } - m_callback(toBcosError(ret)); - } - - void callback_asyncSendMessage_exception(tars::Int32 ret) override - { - s_tarsTimeoutCount++; - if (!m_callback) - { - return; - } - m_callback(toBcosError(ret)); - } - - private: - ppc::gateway::ErrorCallbackFunc m_callback; - }; - - // encode message to bytes - bcos::bytes buffer; - _message->encode(buffer); - - GATEWAYCLIENT_LOG(TRACE) << LOG_DESC("send message to gateway by client") - << LOG_KV("taskType", unsigned(_message->taskType())) - << LOG_KV("algorithmType", unsigned(_message->algorithmType())) - << LOG_KV("messageType", unsigned(_message->messageType())) - << LOG_KV("seq", _message->seq()) - << LOG_KV("taskID", _message->taskID()) - << LOG_KV("receiver", _agencyID); - - m_prx->tars_set_timeout(m_networkTimeout) - ->async_asyncSendMessage(new Callback(_callback), _agencyID, - std::vector(buffer.begin(), buffer.end())); - } - - bcos::Error::Ptr notifyTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr _taskInfo) override - { - auto tarsTaskInfo = toTarsTaskInfo(_taskInfo); - auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); - // broadcast to all gateways - uint errorCount = 0; - std::string lastErrorMsg; - for (auto& endPoint : activeEndPoints) - { - auto prx = - ppctars::createServantProxy(m_gatewayServiceName, endPoint); - auto error = prx->tars_set_timeout(m_networkTimeout)->notifyTaskInfo(tarsTaskInfo); - if (error.errorCode) - { - ++errorCount; - lastErrorMsg = error.errorMessage; - } - } - if (errorCount) - { - return std::make_shared( - ppc::protocol::PPCRetCode::NOTIFY_TASK_ERROR, lastErrorMsg); - } - - return nullptr; - } - - bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override - { - auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); - // broadcast to all gateways - uint errorCount = 0; - for (auto& endPoint : activeEndPoints) - { - try - { - auto prx = - ppctars::createServantProxy(m_gatewayServiceName, endPoint); - auto error = prx->tars_set_timeout(m_networkTimeout)->eraseTaskInfo(_taskID); - if (error.errorCode) - { - ++errorCount; - } - } - catch (std::exception const& e) - { - ++errorCount; - BCOS_LOG(INFO) << LOG_DESC("eraseTaskInfo exception") - << LOG_KV("exception", boost::diagnostic_information(e)); - } - } - if (errorCount) - { - return std::make_shared( - -1, "eraseTaskInfo: error count: " + std::to_string(errorCount)); - } - - return nullptr; - } - - bcos::Error::Ptr registerGateway( - const std::vector& _gatewayList) override - { - std::vector gatewayList; - for (const auto& gateway : _gatewayList) - { - ppctars::GatewayInfo tarsGate; - tarsGate.agencyID = gateway.agencyID; - tarsGate.endpoint = gateway.endpoint; - gatewayList.push_back(tarsGate); - } - auto activeEndPoints = tarsProxyActiveEndPoints(m_prx); - // broadcast to all gateways - uint errorCount = 0; - std::string lastErrorMsg; - for (auto& endPoint : activeEndPoints) - { - auto prx = - ppctars::createServantProxy(m_gatewayServiceName, endPoint); - auto error = prx->tars_set_timeout(m_networkTimeout)->registerGateway(gatewayList); - if (error.errorCode) - { - ++errorCount; - lastErrorMsg = error.errorMessage; - } - } - if (errorCount) - { - return std::make_shared( - ppc::protocol::PPCRetCode::REGISTER_GATEWAY_URL_ERROR, lastErrorMsg); - } - - return nullptr; - } - - - void registerFront(std::string const& _endPoint, ppc::front::FrontInterface::Ptr) override - { - // Error registerFront(string endPoint); - class Callback : public GatewayServicePrxCallback - { - public: - explicit Callback(std::function _callback) - : GatewayServicePrxCallback(), m_callback(_callback) - {} - ~Callback() override {} - - void callback_asyncRegisterFront(const ppctars::Error& ret) override - { - m_callback(toBcosError(ret)); - } - void callback_asyncRegisterFront_exception(tars::Int32 ret) override - { - m_callback(toBcosError(ret)); - } - - private: - std::function m_callback; - }; - auto startT = bcos::utcSteadyTime(); - auto callback = [_endPoint, startT](bcos::Error::Ptr _error) { - if (!_error || _error->errorCode() == 0) - { - GATEWAYCLIENT_LOG(TRACE) - << LOG_DESC("registerFront success") << LOG_KV("endPoint", _endPoint) - << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); - return; - } - GATEWAYCLIENT_LOG(INFO) - << LOG_DESC("registerFront failed") << LOG_KV("code", _error->errorCode()) - << LOG_KV("endPoint", _endPoint) - << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); - }; - m_prx->tars_set_timeout(m_networkTimeout) - ->async_asyncRegisterFront(new Callback(callback), _endPoint); - } - - void asyncGetAgencyList(ppc::front::GetAgencyListCallback _callback) override - { - class Callback : public GatewayServicePrxCallback - { - public: - explicit Callback(ppc::front::GetAgencyListCallback _callback) - : GatewayServicePrxCallback(), m_callback(_callback) - {} - ~Callback() override {} - - void callback_asyncGetAgencyList( - const ppctars::Error& ret, std::vector const& _agencyList) override - { - auto tmpAgencyList = _agencyList; - m_callback(toBcosError(ret), std::move(tmpAgencyList)); - } - void callback_asyncGetAgencyList_exception(tars::Int32 ret) override - { - std::vector emptyAgencyList; - m_callback(toBcosError(ret), std::move(emptyAgencyList)); - } - - private: - ppc::front::GetAgencyListCallback m_callback; - }; - m_prx->async_asyncGetAgencyList(new Callback(_callback)); - } - - // Note: unregisterFront is the function of the front-inner, no need to implement the client - void unregisterFront(std::string const&) override - { - throw std::runtime_error("unregisterFront: unimplemented interface!"); - } - - -protected: - static bool shouldStopCall() { return (s_tarsTimeoutCount >= c_maxTarsTimeoutCount); } - - -private: - std::string m_gatewayServiceName; - ppctars::GatewayServicePrx m_prx; - - // 1800s - int m_networkTimeout = 1800000; - std::string const c_moduleName = "GatewayServiceClient"; - - static std::atomic s_tarsTimeoutCount; - static const int64_t c_maxTarsTimeoutCount; -}; -} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars deleted file mode 100644 index 5fa26bd7..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/Error.tars +++ /dev/null @@ -1,6 +0,0 @@ -module ppctars { - struct Error { - 1 optional int errorCode; - 2 optional string errorMessage; - }; -}; \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars deleted file mode 100644 index 763bf91f..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/FrontService.tars +++ /dev/null @@ -1,7 +0,0 @@ -#include "Error.tars" -module ppctars { - interface FrontService { - // avoid using unsigned char - Error onReceiveMessage(vector message); - }; -}; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars deleted file mode 100644 index 017ca607..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/GatewayService.tars +++ /dev/null @@ -1,19 +0,0 @@ -#include "TaskInfo.tars" -#include "Error.tars" -module ppctars -{ - struct GatewayInfo { - 1 optional string agencyID; - 2 optional string endpoint; - }; - interface GatewayService - { - Error asyncSendMessage(string agencyID, vector message); - - Error notifyTaskInfo(TaskInfo taskInfo); - Error eraseTaskInfo(string taskID); - Error registerGateway(vector gatewayList); - Error asyncRegisterFront(string endPoint); - Error asyncGetAgencyList(out vector _agencyList); - }; -}; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars deleted file mode 100644 index 06055197..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/NodeInfo.tars +++ /dev/null @@ -1,16 +0,0 @@ -module ppctars -{ - struct NodeInfo - { - 1 require vector nodeID; - 2 require string endPoint; - 3 optional vector components; - }; - struct GatewayNodeInfo - { - 1 require string p2pNodeID; - 2 require string agency; - 3 optional vector nodeList; - 4 optional int statusSeq; - }; -}; diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars b/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars deleted file mode 100644 index 1d6af6f4..00000000 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/TaskInfo.tars +++ /dev/null @@ -1,7 +0,0 @@ -module ppctars { -struct TaskInfo -{ - 1 require string taskID; - 2 require string serviceEndpoint; -}; -}; diff --git a/cpp/ppc-tars-protocol/test/CMakeLists.txt b/cpp/ppc-tars-protocol/test/CMakeLists.txt deleted file mode 100644 index 21935b94..00000000 --- a/cpp/ppc-tars-protocol/test/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -file(GLOB_RECURSE SOURCES "*.cpp") - -# cmake settings -set(TARS_PROTOCOL_TEST_BINARY_NAME test-ppc-tars-protocol) - -add_executable(${TARS_PROTOCOL_TEST_BINARY_NAME} ${SOURCES}) -target_include_directories(${TARS_PROTOCOL_TEST_BINARY_NAME} PRIVATE .) - -target_compile_options(${TARS_PROTOCOL_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unused-variable) - -target_link_libraries(${TARS_PROTOCOL_TEST_BINARY_NAME} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) - -add_test(NAME test-tars-protocol WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TARS_PROTOCOL_TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/test/SerializeTest.cpp b/cpp/ppc-tars-protocol/test/SerializeTest.cpp deleted file mode 100644 index b6f33f9f..00000000 --- a/cpp/ppc-tars-protocol/test/SerializeTest.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayTest.cpp - * @author: shawnhe - * @date 2022-10-28 - */ - -#include "TaskInfo.h" -#include "ppc-protocol/src/PPCMessage.h" -#include "ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h" -#include -#include -#include -#include - -using namespace ppc; -using namespace ppctars; -using namespace ppctars::serialize; -using namespace ppc::protocol; -using namespace ppc::front; - -using namespace bcos; -using namespace bcos::test; - -BOOST_FIXTURE_TEST_SUITE(TarsProtocolTest, TestPromptFixture) - - -BOOST_AUTO_TEST_CASE(test_labeledPSIDataStructure) -{ - TaskInfo taskInfo; - taskInfo.taskID = "123fdsa456"; - taskInfo.serviceEndpoint = "234f567dfaj"; - - auto encodedData = std::make_shared(); - serialize::encode(taskInfo, *encodedData); - - TaskInfo taskInfoD; - serialize::decode(*encodedData, taskInfoD); - BOOST_CHECK(taskInfoD == taskInfo); - - auto messageFactory = std::make_shared(); - auto queryMessage = messageFactory->buildPPCMessage( - uint8_t(1), uint8_t(2), "345", std::make_shared()); - queryMessage->setMessageType(uint8_t(6)); - ppctars::serialize::encode(taskInfo, *queryMessage->data()); - auto buffer = std::make_shared(); - queryMessage->encode(*buffer); - - auto newMessage = messageFactory->buildPPCMessage(); - newMessage->decode(buffer); - - TaskInfo taskInfoN; - serialize::decode(*newMessage->data(), taskInfoN); - BOOST_CHECK(taskInfoN == taskInfo); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/cpp/ppc-tars-protocol/test/main.cpp b/cpp/ppc-tars-protocol/test/main.cpp deleted file mode 100644 index 8034bf67..00000000 --- a/cpp/ppc-tars-protocol/test/main.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define BOOST_TEST_MAIN -#include diff --git a/cpp/ppc-tools/src/config/PPCConfig.cpp b/cpp/ppc-tools/src/config/PPCConfig.cpp index 07a773c6..ace6cee6 100644 --- a/cpp/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/ppc-tools/src/config/PPCConfig.cpp @@ -49,14 +49,17 @@ void PPCConfig::loadGatewayConfig( PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the redis config success"); } - // load the agencies config - PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the agency config"); - auto agencyInfo = parseAgencyConfig(_pt, "agency", "agency"); - m_gatewayConfig.agencies = std::move(agencyInfo); - PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the agency config sucess") - << LOG_KV("agencySize", m_gatewayConfig.agencies.size()); + m_gatewayConfig.nodePath = _pt.get("gateway.nodes_path", "./"); + m_gatewayConfig.nodeFileName = _pt.get("gateway.nodes_file", "nodes.json"); m_gatewayConfig.reconnectTime = _pt.get("gateway.reconnect_time", 10000); + m_gatewayConfig.unreachableDistance = _pt.get("gateway.unreachable_distance", 10); + if (m_gatewayConfig.unreachableDistance < GatewayConfig::MinUnreachableDistance) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << bcos::errinfo_comment( + "Invalid unreachable_distance, must no smaller than " + + std::to_string(GatewayConfig::MinUnreachableDistance))); + } // load the maxAllowedMsgSize, in MBytes m_gatewayConfig.maxAllowedMsgSize = _pt.get("gateway.max_allow_msg_size", GatewayConfig::DefaultMaxAllowedMsgSize) * @@ -78,7 +81,6 @@ void PPCConfig::loadGatewayConfig( << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); } - int PPCConfig::loadHoldingMessageMinutes( const boost::property_tree::ptree& _pt, std::string const& _section) { @@ -127,62 +129,6 @@ void PPCConfig::initRedisConfigForGateway( << LOG_KV("socketTimeout", _redisConfig.socketTimeout); } -std::map> PPCConfig::parseAgencyConfig( - const boost::property_tree::ptree& _pt, std::string const& _sectionName, - std::string const& _subSectionName) -{ - PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") << LOG_KV("section", _sectionName) - << LOG_KV("subSection", _subSectionName); - std::map> agencyInfo; - // without the config - if (!_pt.get_child_optional(_sectionName)) - { - PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig return for empty config") - << LOG_KV("section", _sectionName) - << LOG_KV("subSection", _subSectionName); - return agencyInfo; - } - // tranverse the child_section to parse the agencyInfo - for (auto const& it : _pt.get_child(_sectionName)) - { - if (it.first.find(_subSectionName) != 0) - { - continue; - } - // find and parse the agencyInfo - auto key = it.first.data(); - std::vector agencyIDInfo; - boost::split(agencyIDInfo, key, boost::is_any_of(".")); - // invalid agencyID - if (agencyIDInfo.size() < 2) - { - BOOST_THROW_EXCEPTION( - InvalidConfig() << bcos::errinfo_comment("Invalid agency key " + it.first + - ", the key must be in format of " + - _sectionName + ".${agencyID}")); - } - std::vector endPointInfos; - auto value = it.second.data(); - boost::split(endPointInfos, value, boost::is_any_of(",")); - for (auto& endpoint : endPointInfos) - { - if (!checkEndpoint(endpoint)) - { - BOOST_THROW_EXCEPTION( - InvalidConfig() << bcos::errinfo_comment("Invalid agency endpoint" + endpoint)); - } - } - auto& currentEndPoints = agencyInfo[agencyIDInfo.at(1)]; - currentEndPoints.reserve(currentEndPoints.size() + endPointInfos.size()); - std::move(std::begin(endPointInfos), std::end(endPointInfos), - std::back_inserter(currentEndPoints)); - PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") - << LOG_KV("agencyID", agencyIDInfo.at(1)) - << LOG_KV("endPointSize", currentEndPoints.size()); - } - PPCConfig_LOG(INFO) << LOG_DESC("parseAgencyConfig") << LOG_KV("agencySize", agencyInfo.size()); - return agencyInfo; -} void PPCConfig::loadNetworkConfig(NetworkConfig& _config, const char* _certPath, boost::property_tree::ptree const& _pt, std::string const& _sectionName, int _defaultListenPort, diff --git a/cpp/ppc-tools/src/config/PPCConfig.h b/cpp/ppc-tools/src/config/PPCConfig.h index 6fd10a14..3bd29cfa 100644 --- a/cpp/ppc-tools/src/config/PPCConfig.h +++ b/cpp/ppc-tools/src/config/PPCConfig.h @@ -94,14 +94,17 @@ struct GatewayConfig constexpr static uint64_t DefaultMaxAllowedMsgSize = 100; constexpr static uint64_t MinMsgSize = 10 * 1024 * 1024; constexpr static uint64_t MaxMsgSize = 1024 * 1024 * 1024; + constexpr static int MinUnreachableDistance = 2; bool disableCache; NetworkConfig networkConfig; ppc::storage::CacheStorageConfig cacheStorageConfig; - // agencyID => endPointList - std::map> agencies; + std::string nodeFileName; + std::string nodePath; uint64_t maxAllowedMsgSize = DefaultMaxAllowedMsgSize; int reconnectTime = 10000; + // the unreachable distance + int unreachableDistance = 10; }; // the ecdh-psi config @@ -294,9 +297,6 @@ class PPCConfig void initRedisConfigForGateway( ppc::storage::CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt); - std::map> parseAgencyConfig( - const boost::property_tree::ptree& _pt, std::string const& _sectionName, - std::string const& _subSectionName); // load the tars-config for the given service, e.g: /* diff --git a/cpp/ppc-utilities/Utilities.h b/cpp/ppc-utilities/Utilities.h index 4b04ea21..13739719 100644 --- a/cpp/ppc-utilities/Utilities.h +++ b/cpp/ppc-utilities/Utilities.h @@ -21,6 +21,9 @@ #include "ppc-framework/Common.h" #include +#include +#include +#include namespace ppc { @@ -38,4 +41,10 @@ inline uint64_t decodeNetworkBuffer( curOffset += dataLen; return curOffset; } + +inline std::string generateUUID() +{ + static thread_local auto uuid_gen = boost::uuids::basic_random_generator(); + return boost::uuids::to_string(uuid_gen()); +} } // namespace ppc \ No newline at end of file diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h index c5f36374..b7cdc97d 100644 --- a/cpp/test-utils/FakeFront.h +++ b/cpp/test-utils/FakeFront.h @@ -182,7 +182,7 @@ class FakeFront : public FrontInterface } } - bcos::Error::Ptr notifyTaskInfo(protocol::GatewayTaskInfo::Ptr) override { return nullptr; } + bcos::Error::Ptr notifyTaskInfo(std::string const&) override { return nullptr; } // erase the task-info when task finished bcos::Error::Ptr eraseTaskInfo(std::string const&) override { return nullptr; } diff --git a/cpp/test-utils/FakePPCMessage.h b/cpp/test-utils/FakePPCMessage.h index 279cb1cf..a9070eae 100644 --- a/cpp/test-utils/FakePPCMessage.h +++ b/cpp/test-utils/FakePPCMessage.h @@ -49,8 +49,6 @@ class FakePPCMessage : public PPCMessageFace std::string const& sender() const override { return m_sender; } void setSender(std::string const& _sender) override { m_sender = _sender; } - uint16_t ext() const override { return m_ext; } - void setExt(uint16_t _ext) override { m_ext = _ext; } std::shared_ptr data() const override { return m_data; } void setData(std::shared_ptr _data) override { m_data = _data; } // Note: we don't fake the encode-decode here diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 635e6990..a56cfe04 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -51,6 +51,10 @@ { "name": "libsodium", "version": "1.0.18#9" + }, + { + "name": "grpc", + "version": "1.51.1#1" } ], "features": { @@ -90,6 +94,10 @@ "name": "mysql-connector-cpp", "version>=": "8.0.32" }, + { + "name": "grpc", + "version>=": "1.51.1" + }, "libhdfs3", "tarscpp", "tbb", diff --git a/cpp/wedpr-protocol/CMakeLists.txt b/cpp/wedpr-protocol/CMakeLists.txt new file mode 100644 index 00000000..33136407 --- /dev/null +++ b/cpp/wedpr-protocol/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(protocol) +add_subdirectory(protobuf) +add_subdirectory(tars) +add_subdirectory(grpc) diff --git a/cpp/wedpr-protocol/grpc/CMakeLists.txt b/cpp/wedpr-protocol/grpc/CMakeLists.txt new file mode 100644 index 00000000..e9a9279b --- /dev/null +++ b/cpp/wedpr-protocol/grpc/CMakeLists.txt @@ -0,0 +1,32 @@ +# proto generation +set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/pb) + +file(GLOB_RECURSE MESSAGES_PROTOS "${PROTO_INPUT_PATH}/Service*.proto") + +# create PROTO_OUTPUT_PATH +file(MAKE_DIRECTORY ${PROTO_OUTPUT_PATH}) + +find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin REQUIRED) +find_program(PROTOC_BINARY protoc REQUIRED) + +foreach(proto_file ${MESSAGES_PROTOS}) + get_filename_component(basename ${proto_file} NAME_WE) + set(generated_file ${PROTO_OUTPUT_PATH}/${basename}.grpc.pb.cc) + + list(APPEND GRPC_MESSAGES_SRCS ${generated_file}) + message("Command: ${PROTOC_BINARY} --grpc_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${proto_file}") + add_custom_command( + OUTPUT ${generated_file} + COMMAND ${PROTOC_BINARY} + ARGS --grpc_out ${PROTO_OUTPUT_PATH} + -I ${PROTO_INPUT_PATH} + --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}" + ${proto_file} DEPENDS ${proto_file} + COMMENT "Generating ${generated_file} from ${proto_file}" + ) +endforeach() + +add_library(${SERVICE_CLIENT_PB_TARGET} ${GRPC_MESSAGES_SRCS}) +target_link_libraries(${SERVICE_CLIENT_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure) + +add_subdirectory(client) \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/CMakeLists.txt b/cpp/wedpr-protocol/grpc/client/CMakeLists.txt new file mode 100644 index 00000000..c7f6c983 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${SERVICE_CLIENT_TARGET} ${SRCS}) +target_link_libraries(${SERVICE_CLIENT_TARGET} PUBLIC ${SERVICE_CLIENT_PB_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/Common.h b/cpp/wedpr-protocol/grpc/client/Common.h new file mode 100644 index 00000000..68deebe7 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/Common.h @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2021-04-12 + */ +#pragma once +#include "ppc-framework/Common.h" + +#define GRPC_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GRPC][CLIENT]" \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp new file mode 100644 index 00000000..4f351dc8 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontClient.cpp + * @author: yujiechen + * @date 2024-09-02 + */ +#include "FrontClient.h" +#include "wedpr-protocol/protobuf/Common.h" + + +using namespace ppc::protocol; +using namespace ppc::proto; +using grpc::Channel; +using grpc::ClientAsyncResponseReader; +using grpc::ClientContext; +using grpc::CompletionQueue; +using grpc::Status; + +void FrontClient::onReceiveMessage(ppc::protocol::Message::Ptr const& msg, ReceiveMsgFunc callback) +{ + // TODO: optimize here + ReceivedMessage receivedMsg; + bcos::bytes encodedData; + msg->encode(encodedData); + receivedMsg.set_data(encodedData.data(), encodedData.size()); + + auto grpcCallback = [callback](ClientContext const&, Status const& status, Error&& response) { + auto error = std::make_shared(response.errorcode(), response.errormessage()); + callback(error); + }; + + auto call = std::make_shared(grpcCallback); + call->responseReader = + m_stub->PrepareAsynconReceiveMessage(&call->context, receivedMsg, &m_client->queue()); + call->responseReader->StartCall(); + // send request, upon completion of the RPC, "reply" be updated with the server's response + call->responseReader->Finish(&call->reply, &call->status, (void*)call.get()); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.h b/cpp/wedpr-protocol/grpc/client/FrontClient.h new file mode 100644 index 00000000..61205f7c --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontClient.h + * @author: yujiechen + * @date 2024-09-02 + */ +#pragma once +#include "GrpcClient.h" +#include "ppc-framework/front/IFront.h" + +namespace ppc::protocol +{ +class FrontClient : public ppc::front::IFrontClient +{ +public: + using Ptr = std::shared_ptr; + FrontClient(GrpcClient::Ptr client) + : m_client(std::move(client)), m_stub(ppc::proto::Front::NewStub(m_client->channel())) + {} + + ~FrontClient() override = default; + void onReceiveMessage( + ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) override; + +private: + std::unique_ptr m_stub; + GrpcClient::Ptr m_client; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp new file mode 100644 index 00000000..bdcb43b9 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayClient.h + * @author: yujiechen + * @date 2024-09-02 + */ +#include "GatewayClient.h" + +using namespace ppc; +using namespace ppc::gateway; +using namespace ppc::protocol; + + +void GatewayClient::start() {} +void GatewayClient::stop() {} + +void GatewayClient::asyncSendMessage(RouteType routeType, + MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout, + ReceiveMsgFunc callback) +{} + +void GatewayClient::asyncSendbroadcastMessage( + RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) +{} +void GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) {} +void GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) {} +void GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) {} +void GatewayClient::unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) {} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h new file mode 100644 index 00000000..f1e9a012 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayClient.h + * @author: yujiechen + * @date 2024-09-02 + */ +#pragma once +#include "ppc-framework/gateway/IGateway.h" + +namespace ppc::protocol +{ +class GatewayClient : public ppc::gateway::IGateway +{ +public: + using Ptr = std::shared_ptr; + GatewayClient() = default; + ~GatewayClient() override = default; + + + void start() override; + void stop() override; + + /** + * @brief send message to gateway + * + * @param routeType the route type + * @param topic the topic + * @param dstInst the dst agency(must set when 'route by agency' and 'route by + * component') + * @param dstNodeID the dst nodeID(must set when 'route by nodeID') + * @param componentType the componentType(must set when 'route by component') + * @param payload the payload to send + * @param seq the message seq + * @param timeout timeout + * @param callback callback + */ + void asyncSendMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, + long timeout, ppc::protocol::ReceiveMsgFunc callback) override; + + void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override; + void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; + void unRegisterNodeInfo(bcos::bytesConstRef nodeID) override; + void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp new file mode 100644 index 00000000..019e5c10 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcClient.cpp + * @author: yujiechen + * @date 2024-09-02 + */ +#include "GrpcClient.h" +#include "Common.h" + + +using namespace ppc::protocol; +using namespace grpc; + +void GrpcClient::handleRpcResponse() +{ + void* callback; + bool ok = false; + // Block until the next result is available in the completion queue "m_queue". + while (m_queue.Next(&callback, &ok)) + { + try + { + // The tag in this example is the memory location of the call object + // Note: the should been managed by shared_ptr + AsyncClientCall* call = static_cast(callback); + + // Verify that the request was completed successfully. Note that "ok" + // corresponds solely to the request for updates introduced by Finish(). + if (!ok) + { + GRPC_CLIENT_LOG(WARNING) + << LOG_DESC("handleRpcResponse: receive response with unormal status"); + return; + } + call->callback(call->context, call->status, std::move(call->reply)); + } + catch (std::exception const& e) + { + GRPC_CLIENT_LOG(WARNING) << LOG_DESC("handleRpcResponse exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + } +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.h b/cpp/wedpr-protocol/grpc/client/GrpcClient.h new file mode 100644 index 00000000..b00437ac --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcClient.h + * @author: yujiechen + * @date 2024-09-02 + */ +#pragma once +#include "Service.grpc.pb.h" +#include + +namespace ppc::protocol +{ +// struct for keeping state and data information +class AsyncClientCall +{ +public: + using CallbackDef = + std::function; + AsyncClientCall(CallbackDef _callback) : callback(std::move(_callback)) {} + + CallbackDef callback; + // Container for the data we expect from the server. + ppc::proto::Error reply; + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + grpc::ClientContext context; + // Storage for the status of the RPC upon completion. + grpc::Status status; + std::unique_ptr> responseReader; +}; + +class GrpcClient +{ +public: + using Ptr = std::shared_ptr; + GrpcClient(std::shared_ptr channel) : m_channel(std::move(channel)) {} + + virtual ~GrpcClient() = default; + + std::shared_ptr const& channel() { return m_channel; } + grpc::CompletionQueue& queue() { return m_queue; } + + void handleRpcResponse(); + +private: + std::shared_ptr m_channel; + // The producer-consumer queue we use to communicate asynchronously with the + // gRPC runtime. + // TODO: check threadsafe + grpc::CompletionQueue m_queue; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/proto/pb/NodeInfo.proto b/cpp/wedpr-protocol/proto/pb/NodeInfo.proto new file mode 100644 index 00000000..f238ee1e --- /dev/null +++ b/cpp/wedpr-protocol/proto/pb/NodeInfo.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package ppc.proto; + +message NodeInfo{ + // the nodeID + bytes nodeID = 1; + // the endPoint + string endPoint = 2; + // the components + repeated string components = 3; + string topic = 4; +}; + +message GatewayNodeInfo{ + string p2pNodeID = 1; + string agency = 2; + repeated NodeInfo nodeList = 3; + int32 statusSeq = 4; +}; \ No newline at end of file diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto new file mode 100644 index 00000000..b41fd3c4 --- /dev/null +++ b/cpp/wedpr-protocol/proto/pb/Service.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; +import "NodeInfo.proto"; +option java_package = "ppc.proto.grpc"; + +package ppc.proto; + +message Error{ + // the errorCode + int64 errorCode = 1; + // the errorMessage + string errorMessage = 2; +}; + +message ReceivedMessage{ + bytes data = 1; +}; + +service Front { + rpc onReceiveMessage (ReceivedMessage) returns (Error) {} +} + +message RouteInfo{ + string topic = 1; + string componentType = 2; + bytes srcNode = 3; + bytes dstNode = 4; + bytes dstInst = 5; +}; + +message SendedMessageRequest{ + int32 routeType = 1; + RouteInfo routeInfo = 2; + bytes payload = 3; + int64 timeout = 4; +}; + +service Gateway{ + rpc asyncSendMessage(SendedMessageRequest) returns(Error){} + rpc registerNodeInfo(NodeInfo) returns(Error){} + rpc unRegisterNodeInfo(NodeInfo)returns(Error){} + rpc registerTopic(NodeInfo) returns(Error){} + rpc unRegisterTopic(NodeInfo) returns(Error){} +}; \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars b/cpp/wedpr-protocol/proto/tars/RouterTable.tars similarity index 100% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/tars/RouterTable.tars rename to cpp/wedpr-protocol/proto/tars/RouterTable.tars diff --git a/cpp/wedpr-protocol/protobuf/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/CMakeLists.txt new file mode 100644 index 00000000..6dd7bc81 --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/CMakeLists.txt @@ -0,0 +1,27 @@ +# proto generation +set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/pb) + +file(GLOB_RECURSE MESSAGES_PROTOS "${PROTO_INPUT_PATH}/*.proto") + +find_program(PROTOC_BINARY protoc REQUIRED) + +# create PROTO_OUTPUT_PATH +file(MAKE_DIRECTORY ${PROTO_OUTPUT_PATH}) +foreach(proto_file ${MESSAGES_PROTOS}) + get_filename_component(basename ${proto_file} NAME_WE) + set(generated_file ${PROTO_OUTPUT_PATH}/${basename}.pb.cc) + + list(APPEND MESSAGES_SRCS ${generated_file}) + + message("Command: protoc --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file}") + add_custom_command( + OUTPUT ${generated_file} + COMMAND ${PROTOC_BINARY} --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file} + COMMENT "Generating ${generated_file} from ${proto_file}" + VERBATIM + ) +endforeach() + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${PB_PROTOCOL_TARGET} ${SRCS} ${MESSAGES_SRCS}) +target_link_libraries(${PB_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/Common.h b/cpp/wedpr-protocol/protobuf/Common.h new file mode 100644 index 00000000..09796003 --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/Common.h @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2021-04-12 + */ +#pragma once +#include "bcos-utilities/Common.h" +#include "ppc-framework/Common.h" + +namespace ppc::protocol +{ +DERIVE_PPC_EXCEPTION(PBObjectEncodeException); +DERIVE_PPC_EXCEPTION(PBObjectDecodeException); + +template +void encodePBObject(bcos::bytes& _encodedData, T _pbObject) +{ + auto encodedData = std::make_shared(); + _encodedData.resize(_pbObject->ByteSizeLong()); + if (!_pbObject->SerializeToArray(_encodedData.data(), _encodedData.size())) + { + BOOST_THROW_EXCEPTION(PBObjectEncodeException() + << bcos::errinfo_comment("encode PBObject into bytes data failed")); + } +} + +template +void decodePBObject(T _pbObject, bcos::bytesConstRef _data) +{ + if (!_pbObject->ParseFromArray(_data.data(), _data.size())) + { + BOOST_THROW_EXCEPTION(PBObjectDecodeException() + << bcos::errinfo_comment("decode bytes data into PBObject failed")); + } +} +} // namespace ppc::protocol diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp similarity index 67% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp rename to cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp index e8f585d4..742d53a8 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.cpp +++ b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp @@ -19,22 +19,22 @@ */ #include "NodeInfoImpl.h" -#include "../Common.h" +#include "Common.h" -using namespace ppctars; using namespace ppc::protocol; void NodeInfoImpl::encode(bcos::bytes& data) const { - tars::TarsOutputStream output; - m_inner()->writeTo(output); - output.getByteBuffer().swap(data); + // set the components + for (auto const& component : m_components) + { + m_inner()->add_components(component); + } + encodePBObject(data, m_inner()); } void NodeInfoImpl::decode(bcos::bytesConstRef data) { - tars::TarsInputStream input; - input.setBuffer((const char*)data.data(), data.size()); - m_inner()->readFrom(input); + decodePBObject(m_inner(), data); m_components = - std::set(m_inner()->components.begin(), m_inner()->components.end()); + std::set(m_inner()->components().begin(), m_inner()->components().end()); } \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h similarity index 69% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h rename to cpp/wedpr-protocol/protobuf/NodeInfoImpl.h index ff4e1627..1a5a4000 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/impl/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h @@ -18,8 +18,8 @@ * @date 2024-08-26 */ #pragma once +#include "NodeInfo.pb.h" #include "ppc-framework/protocol/INodeInfo.h" -#include "ppc-tars-protocol/tars/NodeInfo.h" #include namespace ppc::protocol { @@ -28,46 +28,49 @@ class NodeInfoImpl : public INodeInfo { public: using Ptr = std::shared_ptr; - explicit NodeInfoImpl(std::function inner) : m_inner(std::move(inner)) {} + explicit NodeInfoImpl(std::function inner) : m_inner(std::move(inner)) + {} NodeInfoImpl(bcos::bytesConstRef const& nodeID) - : m_inner([inner = ppctars::NodeInfo()]() mutable { return &inner; }) + : m_inner([inner = ppc::proto::NodeInfo()]() mutable { return &inner; }) { - m_inner()->nodeID = std::vector(nodeID.begin(), nodeID.end()); + m_inner()->set_nodeid(nodeID.data(), nodeID.size()); } NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) : NodeInfoImpl(nodeID) { - m_inner()->endPoint = endPoint; + m_inner()->set_endpoint(endPoint); } ~NodeInfoImpl() override = default; - void setComponents(std::vector const& components) override + void setComponents(std::set const& components) override { - m_components = std::set(components.begin(), components.end()); - m_inner()->components = components; + m_components = components; } std::set const& components() const override { return m_components; } - std::string const& endPoint() const override { return m_inner()->endPoint; } + std::string const& endPoint() const override { return m_inner()->endpoint(); } bcos::bytesConstRef nodeID() const override { - return {reinterpret_cast(m_inner()->nodeID.data()), - m_inner()->nodeID.size()}; + return {reinterpret_cast(m_inner()->nodeid().data()), + m_inner()->nodeid().size()}; } void encode(bcos::bytes& data) const override; void decode(bcos::bytesConstRef data) override; - ppctars::NodeInfo const& inner() { return *(m_inner()); } + std::function innerFunc() { return m_inner; } - void setFront(ppc::front::IFront::Ptr&& front) override { m_front = std::move(front); } - ppc::front::IFront::Ptr const& getFront() const override { return m_front; } + void setFront(std::shared_ptr&& front) override + { + m_front = std::move(front); + } + std::shared_ptr const& getFront() const override { return m_front; } private: - ppc::front::IFront::Ptr m_front; + std::shared_ptr m_front; std::set m_components; - std::function m_inner; + std::function m_inner; }; class NodeInfoFactory : public INodeInfoFactory diff --git a/cpp/ppc-protocol/CMakeLists.txt b/cpp/wedpr-protocol/protocol/CMakeLists.txt similarity index 100% rename from cpp/ppc-protocol/CMakeLists.txt rename to cpp/wedpr-protocol/protocol/CMakeLists.txt diff --git a/cpp/ppc-protocol/src/CMakeLists.txt b/cpp/wedpr-protocol/protocol/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-protocol/src/CMakeLists.txt rename to cpp/wedpr-protocol/protocol/src/CMakeLists.txt diff --git a/cpp/ppc-protocol/src/JsonTaskImpl.cpp b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp similarity index 100% rename from cpp/ppc-protocol/src/JsonTaskImpl.cpp rename to cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp diff --git a/cpp/ppc-protocol/src/JsonTaskImpl.h b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.h similarity index 100% rename from cpp/ppc-protocol/src/JsonTaskImpl.h rename to cpp/wedpr-protocol/protocol/src/JsonTaskImpl.h diff --git a/cpp/ppc-protocol/src/PPCMessage.cpp b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp similarity index 62% rename from cpp/ppc-protocol/src/PPCMessage.cpp rename to cpp/wedpr-protocol/protocol/src/PPCMessage.cpp index 3f724e87..6c2a0fa0 100644 --- a/cpp/ppc-protocol/src/PPCMessage.cpp +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp @@ -29,29 +29,12 @@ void PPCMessage::encode(bytes& _buffer) { _buffer.clear(); - uint32_t seq = boost::asio::detail::socket_ops::host_to_network_long(m_seq); - uint16_t taskIDLength = boost::asio::detail::socket_ops::host_to_network_short(m_taskID.size()); - uint16_t senderLength = boost::asio::detail::socket_ops::host_to_network_short(m_sender.size()); uint32_t dataLength = boost::asio::detail::socket_ops::host_to_network_long(m_data->size()); - uint16_t ext = boost::asio::detail::socket_ops::host_to_network_short(m_ext); _buffer.insert(_buffer.end(), (byte*)&m_version, (byte*)&m_version + 1); _buffer.insert(_buffer.end(), (byte*)&m_taskType, (byte*)&m_taskType + 1); _buffer.insert(_buffer.end(), (byte*)&m_algorithmType, (byte*)&m_algorithmType + 1); _buffer.insert(_buffer.end(), (byte*)&m_messageType, (byte*)&m_messageType + 1); - _buffer.insert(_buffer.end(), (byte*)&seq, (byte*)&seq + 4); - _buffer.insert(_buffer.end(), (byte*)&taskIDLength, (byte*)&taskIDLength + 2); - _buffer.insert(_buffer.end(), m_taskID.begin(), m_taskID.end()); - _buffer.insert(_buffer.end(), (byte*)&senderLength, (byte*)&senderLength + 2); - _buffer.insert(_buffer.end(), m_sender.begin(), m_sender.end()); - _buffer.insert(_buffer.end(), (byte*)&ext, (byte*)&ext + 2); - // encode the uuid: uuidLen, uuidData - auto uuidLen = m_uuid.size(); - _buffer.insert(_buffer.end(), (byte*)&uuidLen, (byte*)&uuidLen + 1); - if (uuidLen > 0) - { - _buffer.insert(_buffer.end(), m_uuid.begin(), m_uuid.end()); - } // encode the data: dataLen, dataData _buffer.insert(_buffer.end(), (byte*)&dataLength, (byte*)&dataLength + 4); if (dataLength > 0) @@ -99,53 +82,6 @@ int64_t PPCMessage::decode(uint32_t _length, bcos::byte* _data) m_messageType = *((uint8_t*)p); p += 1; - // seq field - m_seq = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)p)); - p += 4; - - // taskIDLength - uint16_t taskIDLength = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); - p += 2; - minLen += taskIDLength; - if (_length < minLen) - { - return -1; - } - - // taskID field - m_taskID.insert(m_taskID.begin(), p, p + taskIDLength); - p += taskIDLength; - - // senderLength - uint16_t senderLength = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); - p += 2; - minLen += senderLength; - if (_length < minLen) - { - return -1; - } - // sender field - m_sender.insert(m_sender.begin(), p, p + senderLength); - p += senderLength; - - // ext field - m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)p)); - p += 2; - - // decode the uuid - auto uuidLen = *((byte*)p); - p += 1; - minLen += uuidLen; - if (_length < minLen) - { - return -1; - } - if (uuidLen > 0) - { - m_uuid.assign(p, p + uuidLen); - p += uuidLen; - } - // dataLength uint32_t dataLength = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)p)); p += 4; diff --git a/cpp/ppc-protocol/src/PPCMessage.h b/cpp/wedpr-protocol/protocol/src/PPCMessage.h similarity index 82% rename from cpp/ppc-protocol/src/PPCMessage.h rename to cpp/wedpr-protocol/protocol/src/PPCMessage.h index ca34657b..fc6983f6 100644 --- a/cpp/ppc-protocol/src/PPCMessage.h +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.h @@ -36,10 +36,10 @@ namespace front class PPCMessage : public PPCMessageFace { public: - // version(1) + taskType(1) + algorithmType(1) + messageType(1) + seq(4) - // + taskIDLength(2) + senderLength(2) + ext(2) + uuidLen(1) + dataLen(4) + data(N) + // version(1) + taskType(1) + algorithmType(1) + messageType(1) + // + dataLen(4) + data(N) // + header(M) - const static size_t MESSAGE_MIN_LENGTH = 15; + const static size_t MESSAGE_MIN_LENGTH = 8; using Ptr = std::shared_ptr; PPCMessage() { m_data = std::make_shared(); } @@ -59,8 +59,6 @@ class PPCMessage : public PPCMessageFace void setTaskID(std::string const& _taskID) override { m_taskID = _taskID; } std::string const& sender() const override { return m_sender; } void setSender(std::string const& _sender) override { m_sender = _sender; } - virtual uint16_t ext() const override { return m_ext; } - virtual void setExt(uint16_t _ext) override { m_ext = _ext; } std::shared_ptr data() const override { return m_data; } // Note: here directly use passed-in _data, make-sure _data not changed before send the message void setData(std::shared_ptr _data) override { m_data = _data; } @@ -81,9 +79,9 @@ class PPCMessage : public PPCMessageFace uint32_t length() const override { return m_length; } // determine the message is response or not - bool response() const override { return m_ext & MessageExtFlag::ResponseFlag; } + bool response() const override { return m_isResponse; } // set the message to be response - void setResponse() override { m_ext |= MessageExtFlag::ResponseFlag; } + void setResponse() override { m_isResponse = true; } protected: std::string encodeMap(const std::map& _map); @@ -97,7 +95,7 @@ class PPCMessage : public PPCMessageFace uint32_t m_seq = 0; std::string m_taskID; std::string m_sender; - uint16_t m_ext = 0; + bool m_isResponse; // the uuid used to find the response-callback std::string m_uuid; std::shared_ptr m_data; @@ -149,6 +147,28 @@ class PPCMessageFactory : public PPCMessageFaceFactory } return msg; } + + PPCMessageFace::Ptr buildPPCMessage(ppc::protocol::Message::Ptr msg) override + { + auto ppcMsg = buildPPCMessage(); + auto frontMsg = msg->frontMessage(); + if (frontMsg) + { + ppcMsg->setSeq(frontMsg->seq()); + ppcMsg->setUuid(frontMsg->traceID()); + if (frontMsg->isRespPacket()) + { + ppcMsg->setResponse(); + } + } + if (msg->header() && msg->header()->optionalField()) + { + auto const& routeInfo = msg->header()->optionalField(); + ppcMsg->setTaskID(routeInfo->topic()); + ppcMsg->setSender(routeInfo->srcInst()); + } + return ppcMsg; + } }; } // namespace front diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp similarity index 94% rename from cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp rename to cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp index f14e1cac..117f304c 100644 --- a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp @@ -37,6 +37,10 @@ void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const uint16_t srcNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_srcNode.size()); buffer.insert(buffer.end(), (byte*)&srcNodeLen, (byte*)&srcNodeLen + 2); buffer.insert(buffer.end(), m_srcNode.begin(), m_srcNode.end()); + // the source agency + uint16_t srcInstLen = boost::asio::detail::socket_ops::host_to_network_short(m_srcInst.size()); + buffer.insert(buffer.end(), (byte*)&srcInstLen, (byte*)&srcInstLen + 2); + buffer.insert(buffer.end(), m_srcInst.begin(), m_srcInst.end()); // the target nodeID that should receive the message uint16_t dstNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstNode.size()); buffer.insert(buffer.end(), (byte*)&dstNodeLen, (byte*)&dstNodeLen + 2); @@ -61,6 +65,10 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con m_componentType = std::string(componentType.begin(), componentType.end()); // srcNode offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), offset); + // source inst + bcos::bytes sourceInst; + offset = decodeNetworkBuffer(sourceInst, data.data(), data.size(), offset); + m_srcInst = std::string(sourceInst.begin(), sourceInst.end()); // dstNode offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset); // dstInst, TODO: optimize here diff --git a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h similarity index 72% rename from cpp/ppc-protocol/src/v1/MessageHeaderImpl.h rename to cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h index 9d9c3850..17f4b99d 100644 --- a/cpp/ppc-protocol/src/v1/MessageHeaderImpl.h +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h @@ -28,6 +28,18 @@ class MessageOptionalHeaderImpl : public MessageOptionalHeader public: using Ptr = std::shared_ptr; MessageOptionalHeaderImpl() = default; + MessageOptionalHeaderImpl(MessageOptionalHeader::Ptr const& optionalHeader) + { + if (!optionalHeader) + { + return; + } + setTopic(optionalHeader->topic()); + setComponentType(optionalHeader->componentType()); + setSrcNode(optionalHeader->srcNode()); + setDstNode(optionalHeader->dstNode()); + setDstInst(optionalHeader->dstInst()); + } MessageOptionalHeaderImpl(bcos::bytesConstRef data, uint64_t const offset) { decode(data, offset); @@ -50,7 +62,7 @@ class MessageHeaderImpl : public MessageHeader void encode(bcos::bytes& buffer) const override; int64_t decode(bcos::bytesConstRef data) override; - virtual bool hasOptionalField() const + bool hasOptionalField() const override { return m_packetType == (uint16_t)ppc::gateway::GatewayPacketType::P2PMessage; } @@ -81,5 +93,21 @@ class MessageHeaderBuilderImpl : public MessageHeaderBuilder return std::make_shared(data); } MessageHeader::Ptr build() override { return std::make_shared(); } + MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) override + { + return std::make_shared(optionalHeader); + } +}; +class MessageOptionalHeaderBuilderImpl : public MessageOptionalHeaderBuilder +{ +public: + using Ptr = std::shared_ptr; + MessageOptionalHeaderBuilderImpl() = default; + ~MessageOptionalHeaderBuilderImpl() override = default; + + MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) override + { + return std::make_shared(optionalHeader); + } }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessageImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp similarity index 100% rename from cpp/ppc-protocol/src/v1/MessageImpl.cpp rename to cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp diff --git a/cpp/ppc-protocol/src/v1/MessageImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h similarity index 81% rename from cpp/ppc-protocol/src/v1/MessageImpl.h rename to cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h index aec9cb3d..e261a7a2 100644 --- a/cpp/ppc-protocol/src/v1/MessageImpl.h +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h @@ -20,6 +20,7 @@ #pragma once #include "ppc-framework/Common.h" #include "ppc-framework/protocol/Message.h" +#include "ppc-utilities/Utilities.h" #include #include #include @@ -78,16 +79,13 @@ class MessageBuilderImpl : public MessageBuilder { return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen, buffer); } - Message::Ptr build(ppc::protocol::RouteType routeType, std::string const& topic, - std::string const& dstInst, bcos::bytes const& dstNodeID, std::string const& componentType, - bcos::bytes&& payload) override + + Message::Ptr build(ppc::protocol::RouteType routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override { auto msg = build(); msg->header()->setRouteType(routeType); - msg->header()->optionalField()->setDstInst(dstInst); - msg->header()->optionalField()->setDstNode(dstNodeID); - msg->header()->optionalField()->setTopic(topic); - msg->header()->optionalField()->setComponentType(componentType); + msg->header()->setOptionalField(routeInfo); msg->setPayload(std::make_shared(std::move(payload))); return msg; } @@ -97,12 +95,11 @@ class MessageBuilderImpl : public MessageBuilder return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen); } - std::string newSeq() override + virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) { - std::string seq = boost::uuids::to_string(boost::uuids::random_generator()()); - seq.erase(std::remove(seq.begin(), seq.end(), '-'), seq.end()); - return seq; + return m_msgHeaderBuilder->build(optionalHeader); } + std::string newSeq() override { return generateUUID(); } private: MessageHeaderBuilder::Ptr m_msgHeaderBuilder; diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp similarity index 72% rename from cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp rename to cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp index 360567b4..7f1f676a 100644 --- a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp @@ -34,6 +34,13 @@ int64_t MessagePayloadImpl::encode(bcos::bytes& buffer) const // seq uint16_t seq = boost::asio::detail::socket_ops::host_to_network_short(m_seq); buffer.insert(buffer.end(), (byte*)&seq, (byte*)&seq + 2); + // ext field + uint16_t ext = boost::asio::detail::socket_ops::host_to_network_short(m_ext); + buffer.insert(buffer.end(), (byte*)&ext, (byte*)&ext + 2); + // traceID + uint16_t traceIDLen = boost::asio::detail::socket_ops::host_to_network_short(m_traceID.size()); + buffer.insert(buffer.end(), (byte*)&traceIDLen, (byte*)&traceIDLen + 2); + buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end()); // data uint16_t dataLen = boost::asio::detail::socket_ops::host_to_network_short(m_data.size()); buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 2); @@ -59,6 +66,15 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) CHECK_OFFSET_WITH_THROW_EXCEPTION((pointer - buffer.data()), buffer.size()); m_seq = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); pointer += 2; + // the ext + CHECK_OFFSET_WITH_THROW_EXCEPTION((pointer - buffer.data()), buffer.size()); + m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); + pointer += 2; + // the traceID + bcos::bytes traceID; + auto offset = + decodeNetworkBuffer(traceID, buffer.data(), buffer.size(), (pointer - buffer.data())); + m_traceID = std::string(traceID.begin(), traceID.end()); // data - return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), (pointer - buffer.data())); + return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset); } \ No newline at end of file diff --git a/cpp/ppc-protocol/src/v1/MessagePayloadImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.h similarity index 100% rename from cpp/ppc-protocol/src/v1/MessagePayloadImpl.h rename to cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.h diff --git a/cpp/ppc-protocol/tests/CMakeLists.txt b/cpp/wedpr-protocol/protocol/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-protocol/tests/CMakeLists.txt rename to cpp/wedpr-protocol/protocol/tests/CMakeLists.txt diff --git a/cpp/ppc-protocol/tests/PPCMessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp similarity index 100% rename from cpp/ppc-protocol/tests/PPCMessageTest.cpp rename to cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp diff --git a/cpp/ppc-protocol/tests/TestTaskImpl.cpp b/cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp similarity index 100% rename from cpp/ppc-protocol/tests/TestTaskImpl.cpp rename to cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp diff --git a/cpp/ppc-protocol/tests/main.cpp b/cpp/wedpr-protocol/protocol/tests/main.cpp similarity index 100% rename from cpp/ppc-protocol/tests/main.cpp rename to cpp/wedpr-protocol/protocol/tests/main.cpp diff --git a/cpp/ppc-tars-protocol/CMakeLists.txt b/cpp/wedpr-protocol/tars/CMakeLists.txt similarity index 67% rename from cpp/ppc-tars-protocol/CMakeLists.txt rename to cpp/wedpr-protocol/tars/CMakeLists.txt index 7c684a1a..d016016e 100644 --- a/cpp/ppc-tars-protocol/CMakeLists.txt +++ b/cpp/wedpr-protocol/tars/CMakeLists.txt @@ -1,13 +1,14 @@ cmake_minimum_required(VERSION 3.14) include(Version) -project(ppc-tars-protocol VERSION ${VERSION}) +project(wedpr-tars-protocol VERSION ${VERSION}) # for tars generator -set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/ppc-tars-protocol/tars) +set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/tars) find_program(TARS_TARS2CPP tars2cpp REQUIRED) -file(GLOB_RECURSE TARS_INPUT "*.tars") +set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/tars) +file(GLOB_RECURSE TARS_INPUT "${PROTO_INPUT_PATH}/*.tars") # generate tars if (TARS_INPUT) @@ -28,7 +29,7 @@ endif () set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${OUT_TARS_H_LIST}") -file(GLOB_RECURSE SRC_LIST "ppc-tars-protocol/*.cpp" "ppc-tars-protocol/*.h") +file(GLOB_RECURSE SRC_LIST *.cpp) find_package(tarscpp REQUIRED) add_library(${TARS_PROTOCOL_TARGET} ${SRC_LIST} ${OUT_TARS_H_LIST}) @@ -36,7 +37,7 @@ target_include_directories(${TARS_PROTOCOL_TARGET} PUBLIC $ $ $ - $) + $) target_link_libraries(${TARS_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} tarscpp::tarsservant tarscpp::tarsutil) # ut @@ -45,9 +46,3 @@ if (TESTS) set(CTEST_OUTPUT_ON_FAILURE TRUE) add_subdirectory(test) endif () - - -include(GNUInstallDirs) -#install(TARGETS ${TARS_PROTOCOL_TARGET} EXPORT ppcTargets ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") -install(DIRECTORY "ppc-tars-protocol" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" FILES_MATCHING PATTERN "*.h") -install(DIRECTORY "${CMAKE_BINARY_DIR}/generated/ppc-tars-protocol" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" FILES_MATCHING PATTERN "*.h") \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h b/cpp/wedpr-protocol/tars/Common.h similarity index 79% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h rename to cpp/wedpr-protocol/tars/Common.h index 576e3ec6..90f62d33 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/Common.h +++ b/cpp/wedpr-protocol/tars/Common.h @@ -20,14 +20,10 @@ #pragma once -#include "Error.h" #include "TarsServantProxyCallback.h" -#include "TaskInfo.h" -#include "ppc-framework/gateway/GatewayInterface.h" #include "ppc-framework/protocol/Task.h" #include "ppc-tools/src/config/ParamChecker.h" #include -#include #include #include #include @@ -118,97 +114,6 @@ using BufferWriterStdByteVector = BufferWriter>; using BufferWriterString = BufferWriter; } // namespace serialize - -template -bool checkConnection(std::string const& _module, std::string const& _func, const T& prx, - std::function _errorCallback, bool _callsErrorCallback = true) -{ - auto cb = prx->tars_get_push_callback(); - assert(cb); - auto* tarsServantProxyCallback = (TarsServantProxyCallback*)cb.get(); - - if (tarsServantProxyCallback->available()) - { - return true; - } - - if (_errorCallback && _callsErrorCallback) - { - std::string errorMessage = - _module + " calls interface " + _func + " failed for empty connection"; - _errorCallback(std::make_shared(-1, errorMessage)); - } - return false; -} - - -inline ppctars::TaskInfo toTarsTaskInfo(ppc::protocol::GatewayTaskInfo::Ptr _taskInfo) -{ - ppctars::TaskInfo tarsTaskInfo; - if (!_taskInfo) - { - return tarsTaskInfo; - } - - tarsTaskInfo.taskID = _taskInfo->taskID; - tarsTaskInfo.serviceEndpoint = _taskInfo->serviceEndpoint; - - return tarsTaskInfo; -} - -inline ppc::protocol::GatewayTaskInfo::Ptr toGatewayTaskInfo(ppctars::TaskInfo _taskInfo) -{ - auto gatewayTaskInfo = std::make_shared(); - - gatewayTaskInfo->taskID = _taskInfo.taskID; - gatewayTaskInfo->serviceEndpoint = _taskInfo.serviceEndpoint; - - return gatewayTaskInfo; -} - -inline ppctars::Error toTarsError(const bcos::Error& error) -{ - ppctars::Error tarsError; - tarsError.errorCode = error.errorCode(); - tarsError.errorMessage = error.errorMessage(); - - return tarsError; -} - -template -inline ppctars::Error toTarsError(const T& error) -{ - ppctars::Error tarsError; - - if (error) - { - tarsError.errorCode = error->errorCode(); - tarsError.errorMessage = error->errorMessage(); - } - - return tarsError; -} - -inline bcos::Error::Ptr toBcosError(const ppctars::Error& error) -{ - if (error.errorCode == 0) - { - return nullptr; - } - - return std::make_shared(error.errorCode, error.errorMessage); -} - -inline bcos::Error::Ptr toBcosError(tars::Int32 ret) -{ - if (ret == 0) - { - return nullptr; - } - - return std::make_shared(ret, "TARS error!"); -} - inline std::string getProxyDesc(std::string const& _servantName) { std::string desc = diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h b/cpp/wedpr-protocol/tars/TarsSerialize.h similarity index 97% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h rename to cpp/wedpr-protocol/tars/TarsSerialize.h index b15ea3f4..51bb4591 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsSerialize.h +++ b/cpp/wedpr-protocol/tars/TarsSerialize.h @@ -25,7 +25,6 @@ namespace ppctars::serialize { - void encode(TarsStruct auto const& object, bcos::bytes& out) { tars::TarsOutputStream output; @@ -40,4 +39,4 @@ void decode(const bcos::bytes& in, TarsStruct auto& out) out.readFrom(input); } -} // namespace ppctars +} // namespace ppctars::serialize diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h b/cpp/wedpr-protocol/tars/TarsServantProxyCallback.h similarity index 100% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/TarsServantProxyCallback.h rename to cpp/wedpr-protocol/tars/TarsServantProxyCallback.h diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp b/cpp/wedpr-protocol/tars/TarsStruct.cpp similarity index 73% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp rename to cpp/wedpr-protocol/tars/TarsStruct.cpp index 534a993f..064378c9 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/client/GatewayServiceClient.cpp +++ b/cpp/wedpr-protocol/tars/TarsStruct.cpp @@ -13,12 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * @file GatewayServiceClient.cpp + * @file TarsStruct.cpp * @author: shawnhe - * @date 2022-10-20 + * @date 2022-11-4 */ -#include "GatewayServiceClient.h" - -std::atomic ppctars::GatewayServiceClient::s_tarsTimeoutCount = {0}; -const int64_t ppctars::GatewayServiceClient::c_maxTarsTimeoutCount = 500; +#include "TarsStruct.h" \ No newline at end of file diff --git a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h b/cpp/wedpr-protocol/tars/TarsStruct.h similarity index 89% rename from cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h rename to cpp/wedpr-protocol/tars/TarsStruct.h index 7d775a45..395725cd 100644 --- a/cpp/ppc-tars-protocol/ppc-tars-protocol/TarsStruct.h +++ b/cpp/wedpr-protocol/tars/TarsStruct.h @@ -24,16 +24,17 @@ namespace ppctars::serialize { - template concept TarsStruct = requires(TarsStructType tarsStruct) { { tarsStruct.className() - } -> std::same_as; + } + ->std::same_as; { tarsStruct.MD5() - } -> std::same_as; + } + ->std::same_as; tarsStruct.resetDefautlt(); }; -} +} // namespace ppctars::serialize From 58a61acfa620ec566d67527ab6e3a4def37d6761 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 4 Sep 2024 10:36:15 +0800 Subject: [PATCH 019/120] add grpc server implementation (#15) * implement GatewayClient * update client implement * add grpc server implementation --- cpp/cmake/TargetSettings.cmake | 9 +- cpp/ppc-framework/gateway/IGateway.h | 10 +- cpp/ppc-framework/protocol/INodeInfo.h | 2 + .../ppc-gateway/gateway/GatewayImpl.cpp | 14 +- .../ppc-gateway/gateway/GatewayImpl.h | 8 +- cpp/wedpr-protocol/grpc/CMakeLists.txt | 7 +- cpp/wedpr-protocol/grpc/client/CMakeLists.txt | 2 +- cpp/wedpr-protocol/grpc/client/Common.h | 3 +- .../grpc/client/FrontClient.cpp | 22 +-- cpp/wedpr-protocol/grpc/client/FrontClient.h | 7 +- .../grpc/client/GatewayClient.cpp | 59 +++++-- .../grpc/client/GatewayClient.h | 26 ++-- cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 56 ------- cpp/wedpr-protocol/grpc/client/GrpcClient.h | 29 +--- cpp/wedpr-protocol/grpc/server/CMakeLists.txt | 3 + cpp/wedpr-protocol/grpc/server/Common.h | 25 +++ .../grpc/server/FrontServer.cpp | 52 +++++++ cpp/wedpr-protocol/grpc/server/FrontServer.h | 44 ++++++ .../grpc/server/GatewayServer.cpp | 145 ++++++++++++++++++ .../grpc/server/GatewayServer.h | 59 +++++++ cpp/wedpr-protocol/grpc/server/GrpcServer.cpp | 63 ++++++++ cpp/wedpr-protocol/grpc/server/GrpcServer.h | 57 +++++++ cpp/wedpr-protocol/proto/pb/Service.proto | 8 +- cpp/wedpr-protocol/protobuf/NodeInfoImpl.h | 6 + .../protobuf/RequestConverter.h | 134 ++++++++++++++++ 25 files changed, 700 insertions(+), 150 deletions(-) delete mode 100644 cpp/wedpr-protocol/grpc/client/GrpcClient.cpp create mode 100644 cpp/wedpr-protocol/grpc/server/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/grpc/server/Common.h create mode 100644 cpp/wedpr-protocol/grpc/server/FrontServer.cpp create mode 100644 cpp/wedpr-protocol/grpc/server/FrontServer.h create mode 100644 cpp/wedpr-protocol/grpc/server/GatewayServer.cpp create mode 100644 cpp/wedpr-protocol/grpc/server/GatewayServer.h create mode 100644 cpp/wedpr-protocol/grpc/server/GrpcServer.cpp create mode 100644 cpp/wedpr-protocol/grpc/server/GrpcServer.h create mode 100644 cpp/wedpr-protocol/protobuf/RequestConverter.h diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 73882793..645cf931 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -23,9 +23,12 @@ set(TARS_PROTOCOL_TARGET "wedpr-tars-protocol") # wedpr-protocol/protobuf set(PB_PROTOCOL_TARGET "wedpr-pb-protocol") -# wedpr-protocol/grpc-client -set(SERVICE_CLIENT_TARGET "service-client") -set(SERVICE_CLIENT_PB_TARGET "service-client-pb") +# wedpr-protocol/grpc/client +set(SERVICE_CLIENT_TARGET "wedpr-client") +# wedpr-protocol/grpc/server +set(SERVICE_SERVER_TARGET "wedpr-server") +# wedpr-protocol/proto generated file +set(SERVICE_PB_TARGET "service-pb") # ppc-front SET(FRONT_TARGET "ppc-front") diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index d65171c0..28b2fcd8 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -63,10 +63,12 @@ class IGateway virtual void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) = 0; - virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; - virtual void unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; - virtual void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) = 0; - virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) = 0; + virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; + virtual bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; + virtual bcos::Error::Ptr registerTopic( + bcos::bytesConstRef nodeID, std::string const& topic) = 0; + virtual bcos::Error::Ptr unRegisterTopic( + bcos::bytesConstRef nodeID, std::string const& topic) = 0; }; } // namespace ppc::gateway diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 2d4f8e19..76f460b2 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -40,6 +40,8 @@ class INodeInfo virtual std::string const& endPoint() const = 0; virtual bcos::bytesConstRef nodeID() const = 0; + virtual void setNodeID(bcos::bytesConstRef nodeID) = 0; + virtual void setEndPoint(std::string const& endPoint) = 0; // components virtual void setComponents(std::set const& components) = 0; diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index de92b96b..4abad7a4 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -183,20 +183,26 @@ void GatewayImpl::onReceiveBroadcastMessage(MessageFace::Ptr msg, WsSession::Ptr m_localRouter->dispatcherMessage(p2pMessage, nullptr); } -void GatewayImpl::registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) +bcos::Error::Ptr GatewayImpl::registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) { m_localRouter->registerNodeInfo(nodeInfo); + return nullptr; } -void GatewayImpl::unRegisterNodeInfo(bcos::bytesConstRef nodeID) +bcos::Error::Ptr GatewayImpl::unRegisterNodeInfo(bcos::bytesConstRef nodeID) { m_localRouter->unRegisterNode(nodeID.toBytes()); + return nullptr; } -void GatewayImpl::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) + +bcos::Error::Ptr GatewayImpl::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) { m_localRouter->registerTopic(nodeID, topic); + return nullptr; } -void GatewayImpl::unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) + +bcos::Error::Ptr GatewayImpl::unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) { m_localRouter->unRegisterTopic(nodeID, topic); + return nullptr; } \ No newline at end of file diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 40f97fa7..66d692bf 100644 --- a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -61,10 +61,10 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_thisencode(encodedData); receivedMsg.set_data(encodedData.data(), encodedData.size()); - auto grpcCallback = [callback](ClientContext const&, Status const& status, Error&& response) { - auto error = std::make_shared(response.errorcode(), response.errormessage()); - callback(error); - }; - - auto call = std::make_shared(grpcCallback); - call->responseReader = - m_stub->PrepareAsynconReceiveMessage(&call->context, receivedMsg, &m_client->queue()); - call->responseReader->StartCall(); - // send request, upon completion of the RPC, "reply" be updated with the server's response - call->responseReader->Finish(&call->reply, &call->status, (void*)call.get()); + ClientContext context; + auto response = std::make_shared(); + m_stub->async()->onReceiveMessage(&context, &receivedMsg, response.get(), + [response, callback](Status status) { callback(toError(status, std::move(*response))); }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.h b/cpp/wedpr-protocol/grpc/client/FrontClient.h index 61205f7c..4f40b4bb 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.h +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.h @@ -23,13 +23,11 @@ namespace ppc::protocol { -class FrontClient : public ppc::front::IFrontClient +class FrontClient : public ppc::front::IFrontClient, public GrpcClient { public: using Ptr = std::shared_ptr; - FrontClient(GrpcClient::Ptr client) - : m_client(std::move(client)), m_stub(ppc::proto::Front::NewStub(m_client->channel())) - {} + FrontClient(std::shared_ptr channel) : GrpcClient(std::move(channel)) {} ~FrontClient() override = default; void onReceiveMessage( @@ -37,6 +35,5 @@ class FrontClient : public ppc::front::IFrontClient private: std::unique_ptr m_stub; - GrpcClient::Ptr m_client; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index bdcb43b9..50a445ae 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -18,24 +18,59 @@ * @date 2024-09-02 */ #include "GatewayClient.h" +#include "Common.h" +#include "protobuf/RequestConverter.h" using namespace ppc; +using namespace ppc::proto; +using namespace grpc; using namespace ppc::gateway; using namespace ppc::protocol; - -void GatewayClient::start() {} -void GatewayClient::stop() {} - void GatewayClient::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout, ReceiveMsgFunc callback) -{} +{ + auto request = generateRequest(routeType, routeInfo, std::move(payload), timeout); + ClientContext context; + auto response = std::make_shared(); + m_stub->async()->asyncSendMessage(&context, request.get(), response.get(), + [callback, response](Status status) { callback(toError(status, std::move(*response))); }); +} + + +bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) +{ + auto request = toNodeInfoRequest(nodeInfo); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = m_stub->registerNodeInfo(&context, *request, response.get()); + return toError(status, std::move(*response)); +} + +bcos::Error::Ptr GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) +{ + auto request = toNodeInfoRequest(nodeID, ""); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = m_stub->unRegisterNodeInfo(&context, *request, response.get()); + return toError(status, std::move(*response)); +} +bcos::Error::Ptr GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) +{ + auto request = toNodeInfoRequest(nodeID, topic); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = m_stub->registerTopic(&context, *request, response.get()); + return toError(status, std::move(*response)); +} -void GatewayClient::asyncSendbroadcastMessage( - RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) -{} -void GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) {} -void GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) {} -void GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) {} -void GatewayClient::unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) {} \ No newline at end of file +bcos::Error::Ptr GatewayClient::unRegisterTopic( + bcos::bytesConstRef nodeID, std::string const& topic) +{ + auto request = toNodeInfoRequest(nodeID, topic); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = m_stub->unRegisterTopic(&context, *request, response.get()); + return toError(status, std::move(*response)); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index f1e9a012..a6635ccc 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -18,20 +18,24 @@ * @date 2024-09-02 */ #pragma once +#include "GrpcClient.h" #include "ppc-framework/gateway/IGateway.h" namespace ppc::protocol { -class GatewayClient : public ppc::gateway::IGateway +class GatewayClient : public ppc::gateway::IGateway, public GrpcClient { public: using Ptr = std::shared_ptr; - GatewayClient() = default; + GatewayClient(std::shared_ptr channel) + : GrpcClient(std::move(channel)), m_stub(ppc::proto::Gateway::NewStub(m_channel)) + {} + ~GatewayClient() override = default; - void start() override; - void stop() override; + void start() override {} + void stop() override {} /** * @brief send message to gateway @@ -52,10 +56,14 @@ class GatewayClient : public ppc::gateway::IGateway long timeout, ppc::protocol::ReceiveMsgFunc callback) override; void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override; - void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; - void unRegisterNodeInfo(bcos::bytesConstRef nodeID) override; - void registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; - void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override + {} + bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; + bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) override; + bcos::Error::Ptr registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + bcos::Error::Ptr unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) override; + +private: + std::unique_ptr m_stub; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp deleted file mode 100644 index 019e5c10..00000000 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright (C) 2023 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GrpcClient.cpp - * @author: yujiechen - * @date 2024-09-02 - */ -#include "GrpcClient.h" -#include "Common.h" - - -using namespace ppc::protocol; -using namespace grpc; - -void GrpcClient::handleRpcResponse() -{ - void* callback; - bool ok = false; - // Block until the next result is available in the completion queue "m_queue". - while (m_queue.Next(&callback, &ok)) - { - try - { - // The tag in this example is the memory location of the call object - // Note: the should been managed by shared_ptr - AsyncClientCall* call = static_cast(callback); - - // Verify that the request was completed successfully. Note that "ok" - // corresponds solely to the request for updates introduced by Finish(). - if (!ok) - { - GRPC_CLIENT_LOG(WARNING) - << LOG_DESC("handleRpcResponse: receive response with unormal status"); - return; - } - call->callback(call->context, call->status, std::move(call->reply)); - } - catch (std::exception const& e) - { - GRPC_CLIENT_LOG(WARNING) << LOG_DESC("handleRpcResponse exception") - << LOG_KV("error", boost::diagnostic_information(e)); - } - } -} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.h b/cpp/wedpr-protocol/grpc/client/GrpcClient.h index b00437ac..028718a9 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.h +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.h @@ -23,25 +23,7 @@ namespace ppc::protocol { -// struct for keeping state and data information -class AsyncClientCall -{ -public: - using CallbackDef = - std::function; - AsyncClientCall(CallbackDef _callback) : callback(std::move(_callback)) {} - - CallbackDef callback; - // Container for the data we expect from the server. - ppc::proto::Error reply; - // Context for the client. It could be used to convey extra information to - // the server and/or tweak certain RPC behaviors. - grpc::ClientContext context; - // Storage for the status of the RPC upon completion. - grpc::Status status; - std::unique_ptr> responseReader; -}; - +// refer to: https://grpc.io/docs/languages/cpp/callback/ class GrpcClient { public: @@ -51,15 +33,8 @@ class GrpcClient virtual ~GrpcClient() = default; std::shared_ptr const& channel() { return m_channel; } - grpc::CompletionQueue& queue() { return m_queue; } - - void handleRpcResponse(); -private: +protected: std::shared_ptr m_channel; - // The producer-consumer queue we use to communicate asynchronously with the - // gRPC runtime. - // TODO: check threadsafe - grpc::CompletionQueue m_queue; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/CMakeLists.txt b/cpp/wedpr-protocol/grpc/server/CMakeLists.txt new file mode 100644 index 00000000..1b795210 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/CMakeLists.txt @@ -0,0 +1,3 @@ +file(GLOB_RECURSE SRCS *.cpp) +add_library(${SERVICE_SERVER_TARGET} ${SRCS}) +target_link_libraries(${SERVICE_SERVER_TARGET} PUBLIC ${SERVICE_PB_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/Common.h b/cpp/wedpr-protocol/grpc/server/Common.h new file mode 100644 index 00000000..58fa79e2 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/Common.h @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2021-04-12 + */ +#pragma once +#include "ppc-framework/Common.h" + +#define GRPC_SERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GRPC][SERVER]" +#define FRONT_SERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[FRONT][SERVER]" +#define GATEWAY_SERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][SERVER]" \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp new file mode 100644 index 00000000..ee040e6f --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontServer.cpp + * @author: yujiechen + * @date 2024-09-03 + */ +#include "FrontServer.h" +#include "Common.h" +#include "protobuf/RequestConverter.h" +#include + +using namespace ppc::proto; +using namespace ppc::protocol; +using namespace grpc; + +ServerUnaryReactor* FrontServer::onReceiveMessage( + CallbackServerContext* context, const ReceivedMessage* receivedMsg, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + // decode the request + auto msg = m_msgBuilder->build(bcos::bytesConstRef( + (bcos::byte*)receivedMsg->data().data(), receivedMsg->data().size())); + m_front->onReceiveMessage(msg, [reactor, reply](bcos::Error::Ptr error) { + toSerializedError(reply, error); + reactor->Finish(Status::OK); + }); + } + catch (std::exception const& e) + { + FRONT_SERVER_LOG(ERROR) << LOG_DESC("onReceiveMessage exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared(-1, std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.h b/cpp/wedpr-protocol/grpc/server/FrontServer.h new file mode 100644 index 00000000..988c093c --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/FrontServer.h @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontServer.h + * @author: yujiechen + * @date 2024-09-03 + */ +#pragma once +#include "Service.grpc.pb.h" +#include +#include +#include + +namespace ppc::protocol +{ +class FrontServer : public ppc::proto::Front::CallbackService +{ +public: + using Ptr = std::shared_ptr; + FrontServer(ppc::protocol::MessageBuilder::Ptr msgBuilder, ppc::front::IFront::Ptr front) + : m_msgBuilder(std::move(msgBuilder)), m_front(std::move(front)) + {} + ~FrontServer() override = default; + + grpc::ServerUnaryReactor* onReceiveMessage(grpc::CallbackServerContext* context, + const ppc::proto::ReceivedMessage* receivedMsg, ppc::proto::Error* reply) override; + +private: + ppc::front::IFront::Ptr m_front; + ppc::protocol::MessageBuilder::Ptr m_msgBuilder; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp new file mode 100644 index 00000000..91f690c4 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp @@ -0,0 +1,145 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServer.cpp + * @author: yujiechen + * @date 2024-09-03 + */ +#include "GatewayServer.h" +#include "Common.h" +#include "protobuf/RequestConverter.h" +using namespace ppc::protocol; +using namespace grpc; + +ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* context, + const ppc::proto::SendedMessageRequest* sendedMsg, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + // TODO: optimize here + bcos::bytes payloadData(sendedMsg->payload().begin(), sendedMsg->payload().end()); + auto routeInfo = generateRouteInfo(m_routeInfoBuilder, sendedMsg->routeinfo()); + m_gateway->asyncSendMessage((ppc::protocol::RouteType)sendedMsg->routetype(), routeInfo, + std::move(payloadData), sendedMsg->timeout(), [reactor, reply](bcos::Error::Ptr error) { + toSerializedError(reply, error); + reactor->Finish(Status::OK); + }); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("asyncSendMessage exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared(-1, + "handle message failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + +ServerUnaryReactor* GatewayServer::registerNodeInfo(CallbackServerContext* context, + const ppc::proto::NodeInfo* serializedNodeInfo, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + auto nodeInfo = toNodeInfo(m_nodeInfoFactory, *serializedNodeInfo); + auto result = m_gateway->registerNodeInfo(nodeInfo); + toSerializedError(reply, result); + reactor->Finish(Status::OK); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("registerNodeInfo exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared(-1, + "registerNodeInfo failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + +ServerUnaryReactor* GatewayServer::unRegisterNodeInfo( + CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + auto result = m_gateway->unRegisterNodeInfo( + bcos::bytesConstRef((bcos::byte*)nodeInfo->nodeid().data(), nodeInfo->nodeid().size())); + toSerializedError(reply, result); + reactor->Finish(Status::OK); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("unRegisterNodeInfo exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared(-1, "unRegisterNodeInfo failed for : " + + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + +ServerUnaryReactor* GatewayServer::registerTopic( + CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + auto result = m_gateway->registerTopic( + bcos::bytesConstRef((bcos::byte*)nodeInfo->nodeid().data(), nodeInfo->nodeid().size()), + nodeInfo->topic()); + toSerializedError(reply, result); + reactor->Finish(Status::OK); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("unRegisterNodeInfo exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared( + -1, "registerTopic failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + +ServerUnaryReactor* GatewayServer::unRegisterTopic( + CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + auto result = m_gateway->unRegisterTopic( + bcos::bytesConstRef((bcos::byte*)nodeInfo->nodeid().data(), nodeInfo->nodeid().size()), + nodeInfo->topic()); + toSerializedError(reply, result); + reactor->Finish(Status::OK); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("unRegisterTopic exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply, + std::make_shared(-1, + "unRegisterTopic failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.h b/cpp/wedpr-protocol/grpc/server/GatewayServer.h new file mode 100644 index 00000000..d287a6c8 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayServer.h + * @author: yujiechen + * @date 2024-09-03 + */ +#pragma once +#include "Service.grpc.pb.h" +#include "ppc-framework/gateway/IGateway.h" +#include + +namespace ppc::protocol +{ +class GatewayServer : public ppc::proto::Gateway::CallbackService +{ +public: + using Ptr = std::shared_ptr; + GatewayServer(ppc::gateway::IGateway::Ptr gateway, + MessageOptionalHeaderBuilder::Ptr routeInfoBuilder, INodeInfoFactory::Ptr nodeInfoFactory) + : m_gateway(std::move(gateway)), + m_routeInfoBuilder(std::move(routeInfoBuilder)), + m_nodeInfoFactory(std::move(nodeInfoFactory)) + {} + virtual ~GatewayServer() = default; + + grpc::ServerUnaryReactor* asyncSendMessage(grpc::CallbackServerContext* context, + const ppc::proto::SendedMessageRequest* sendedMsg, ppc::proto::Error* reply) override; + + grpc::ServerUnaryReactor* registerNodeInfo(grpc::CallbackServerContext* context, + const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override; + + grpc::ServerUnaryReactor* unRegisterNodeInfo(grpc::CallbackServerContext* context, + const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override; + + grpc::ServerUnaryReactor* registerTopic(grpc::CallbackServerContext* context, + const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override; + + grpc::ServerUnaryReactor* unRegisterTopic(grpc::CallbackServerContext* context, + const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override; + +private: + ppc::gateway::IGateway::Ptr m_gateway; + MessageOptionalHeaderBuilder::Ptr m_routeInfoBuilder; + INodeInfoFactory::Ptr m_nodeInfoFactory; +}; +}; // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp new file mode 100644 index 00000000..9d612dfa --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcServer.cpp + * @author: yujiechen + * @date 2024-09-03 + */ +#include "GrpcServer.h" +#include "Common.h" +#include + +using namespace ppc::protocol; +using namespace grpc; + +void GrpcServer::start() +{ + if (m_running) + { + GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer has already been started!") + << LOG_KV("endPoint", m_config.endPoint()); + return; + } + m_running = true; + grpc::reflection::InitProtoReflectionServerBuilderPlugin(); + grpc::ServerBuilder builder; + // without authentication + builder.AddListeningPort(m_config.endPoint(), grpc::InsecureServerCredentials()); + // register the service + for (auto const& service : m_bindingServices) + { + builder.RegisterService(service.get()); + } + m_server = std::unique_ptr(builder.BuildAndStart()); + GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer start success!") + << LOG_KV("endPoint", m_config.endPoint()); +} + +void GrpcServer::stop() +{ + if (!m_running) + { + GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer has already been stopped!"); + return; + } + m_running = false; + if (m_server) + { + m_server->Shutdown(); + } + GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer stop success!"); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.h b/cpp/wedpr-protocol/grpc/server/GrpcServer.h new file mode 100644 index 00000000..869dcc88 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.h @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcServer.h + * @author: yujiechen + * @date 2024-09-03 + */ +#pragma once +#include +#include +#include + +namespace ppc::protocol +{ +// refer to: https://grpc.io/docs/languages/cpp/callback/ +struct GrpcServerConfig +{ + std::string listenIp; + int listenPort; + + std::string endPoint() const { return listenIp + ":" + std::to_string(listenPort); } +}; +class GrpcServer +{ +public: + using Ptr = std::shared_ptr; + GrpcServer(GrpcServerConfig const& config) : m_config(config) {} + virtual ~GrpcServer() = default; + + virtual void start(); + virtual void stop(); + + virtual void registerService(std::shared_ptr service) + { + m_bindingServices.emplace_back(std::move(service)); + } + +private: + bool m_running = false; + GrpcServerConfig m_config; + + std::unique_ptr m_server; + std::vector> m_bindingServices; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto index b41fd3c4..eb241af8 100644 --- a/cpp/wedpr-protocol/proto/pb/Service.proto +++ b/cpp/wedpr-protocol/proto/pb/Service.proto @@ -15,10 +15,6 @@ message ReceivedMessage{ bytes data = 1; }; -service Front { - rpc onReceiveMessage (ReceivedMessage) returns (Error) {} -} - message RouteInfo{ string topic = 1; string componentType = 2; @@ -33,7 +29,9 @@ message SendedMessageRequest{ bytes payload = 3; int64 timeout = 4; }; - +service Front { + rpc onReceiveMessage (ReceivedMessage) returns (Error) {} +} service Gateway{ rpc asyncSendMessage(SendedMessageRequest) returns(Error){} rpc registerNodeInfo(NodeInfo) returns(Error){} diff --git a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h index 1a5a4000..b0164a58 100644 --- a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h @@ -43,6 +43,12 @@ class NodeInfoImpl : public INodeInfo } ~NodeInfoImpl() override = default; + void setNodeID(bcos::bytesConstRef nodeID) override + { + m_inner()->set_nodeid(nodeID.data(), nodeID.size()); + } + void setEndPoint(std::string const& endPoint) override { m_inner()->set_endpoint(endPoint); } + void setComponents(std::set const& components) override { m_components = components; diff --git a/cpp/wedpr-protocol/protobuf/RequestConverter.h b/cpp/wedpr-protocol/protobuf/RequestConverter.h new file mode 100644 index 00000000..faefc75d --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/RequestConverter.h @@ -0,0 +1,134 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2021-04-12 + */ +#pragma once +#include "Service.pb.h" +#include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/Protocol.h" +#include +#include +#include + +namespace ppc::protocol +{ +inline MessageOptionalHeader::Ptr generateRouteInfo( + MessageOptionalHeaderBuilder::Ptr const& routeInfoBuilder, + ppc::proto::RouteInfo const& serializedRouteInfo) +{ + auto routeInfo = routeInfoBuilder->build(); + routeInfo->setComponentType(serializedRouteInfo.componenttype()); + routeInfo->setSrcNode( + bcos::bytes(serializedRouteInfo.srcnode().begin(), serializedRouteInfo.srcnode().end())); + routeInfo->setDstNode( + bcos::bytes(serializedRouteInfo.dstnode().begin(), serializedRouteInfo.dstnode().end())); + routeInfo->setDstInst(serializedRouteInfo.dstinst()); + return routeInfo; +} + +inline std::shared_ptr generateRequest(RouteType routeType, + MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout) +{ + auto request = std::make_shared(); + request->set_routetype(uint16_t(routeType)); + // set the route information + request->mutable_routeinfo()->set_topic(routeInfo->topic()); + request->mutable_routeinfo()->set_componenttype(routeInfo->componentType()); + request->mutable_routeinfo()->set_srcnode( + routeInfo->srcNode().data(), routeInfo->srcNode().size()); + request->mutable_routeinfo()->set_dstnode( + routeInfo->dstNode().data(), routeInfo->dstNode().size()); + request->mutable_routeinfo()->set_dstinst( + routeInfo->dstInst().data(), routeInfo->dstInst().size()); + // set the payload(TODO: optimize here) + request->set_payload(payload.data(), payload.size()); + request->set_timeout(timeout); + return request; +} + +inline std::shared_ptr toNodeInfoRequest( + bcos::bytesConstRef const& nodeID, std::string const& topic) +{ + auto request = std::make_shared(); + request->set_nodeid(nodeID.data(), nodeID.size()); + request->set_topic(topic); + return request; +} + +inline std::shared_ptr toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo) +{ + auto request = std::make_shared(); + if (!nodeInfo) + { + return request; + }; + request->set_nodeid(nodeInfo->nodeID().data(), nodeInfo->nodeID().size()); + request->set_endpoint(nodeInfo->endPoint()); + auto const& components = nodeInfo->components(); + for (auto const& component : components) + { + request->add_components(component); + } + return request; +} + +inline INodeInfo::Ptr toNodeInfo( + INodeInfoFactory::Ptr const& nodeInfoFactory, ppc::proto::NodeInfo const& serializedNodeInfo) +{ + auto nodeInfo = nodeInfoFactory->build(); + nodeInfo->setNodeID(bcos::bytesConstRef( + (bcos::byte*)serializedNodeInfo.nodeid().data(), serializedNodeInfo.nodeid().size())); + nodeInfo->setEndPoint(serializedNodeInfo.endpoint()); + std::set componentTypeList; + for (int i = 0; i < serializedNodeInfo.components_size(); i++) + { + componentTypeList.insert(serializedNodeInfo.components(i)); + } + nodeInfo->setComponents(componentTypeList); + return nodeInfo; +} + +inline bcos::Error::Ptr toError(grpc::Status const& status, ppc::proto::Error&& error) +{ + if (!status.ok()) + { + return std::make_shared((int32_t)status.error_code(), status.error_message()); + } + if (error.errorcode() == 0) + { + return nullptr; + } + return std::make_shared(error.errorcode(), error.errormessage()); +} + +inline void toSerializedError(ppc::proto::Error* serializedError, bcos::Error::Ptr error) +{ + if (!serializedError) + { + return; + } + if (!error) + { + serializedError->set_errorcode(PPCRetCode::SUCCESS); + return; + } + serializedError->set_errorcode(error->errorCode()); + serializedError->set_errormessage(error->errorMessage()); +} +}; // namespace ppc::protocol \ No newline at end of file From 0f9b0f6baf629e020193b57fce25a7b2e9332254 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 4 Sep 2024 11:30:43 +0800 Subject: [PATCH 020/120] move ppc-front and ppc-gateway to wedpr-transport (#16) --- cpp/CMakeLists.txt | 23 ++++++++++-------- cpp/cmake/IncludeDirectories.cmake | 11 ++++++--- cpp/cmake/TargetSettings.cmake | 4 +-- cpp/ppc-pir/tests/data/AysPreDataset.csv | 3 --- .../ppc-cem/CMakeLists.txt | 0 .../ppc-cem/src/CEMService.cpp | 0 .../ppc-cem/src/CEMService.h | 0 .../ppc-cem/src/CMakeLists.txt | 0 .../ppc-cem/src/Common.h | 0 .../ppc-cem/src/WedprUtilities.h | 0 .../ppc-cem/src/wedpr_ffi_c_equality.h | 0 .../ppc-cem/tests/CMakeLists.txt | 0 .../ppc-cem/tests/TestCEMService.cpp | 0 .../ppc-cem/tests/data/config.ini | 0 .../ppc-cem/tests/data/d1 | 0 .../ppc-cem/tests/data/d2 | 0 .../ppc-cem/tests/data/d3 | 0 .../ppc-cem/tests/main.cpp | 0 .../ppc-mpc/CMakeLists.txt | 0 .../ppc-mpc/src/CMakeLists.txt | 0 .../ppc-mpc/src/Common.h | 0 .../ppc-mpc/src/MPCService.cpp | 0 .../ppc-mpc/src/MPCService.h | 0 .../ppc-mpc/tests/CMakeLists.txt | 0 .../ppc-mpc/tests/TestMPCService.cpp | 0 .../ppc-mpc/tests/data/config.ini | 0 .../ppc-mpc}/tests/main.cpp | 0 .../ppc-pir/CMakeLists.txt | 0 .../ppc-pir/src/BaseOT.cpp | 0 .../ppc-pir/src/BaseOT.h | 0 .../ppc-pir/src/CMakeLists.txt | 0 .../ppc-pir/src/Common.h | 0 .../ppc-pir/src/OtPIRConfig.h | 0 .../ppc-pir/src/OtPIRFactory.h | 0 .../ppc-pir/src/OtPIRImpl.cpp | 0 .../ppc-pir/src/OtPIRImpl.h | 0 .../ppc-pir/src/protocol/OtPIR.tars | 0 .../ppc-pir/tests/CMakeLists.txt | 0 .../ppc-pir/tests/FakeOtPIRFactory.h | 0 .../ppc-pir/tests/TestBaseOT.cpp | 0 .../ppc-pir}/tests/main.cpp | 0 .../ppc-pir/tests/test.json | 0 .../ppc-psi/CMakeLists.txt | 0 .../ppc-psi/demo/CMakeLists.txt | 0 .../ppc-psi/demo/hash_collision_bench.cpp | 0 .../ppc-psi/src/Common.h | 0 .../ppc-psi/src/PSIConfig.h | 0 .../src/bs-ecdh-psi/BsEcdhPSIFactory.h | 0 .../ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp | 0 .../ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h | 0 .../src/bs-ecdh-psi/BsEcdhPSIInterface.h | 0 .../ppc-psi/src/bs-ecdh-psi/CMakeLists.txt | 0 .../ppc-psi/src/bs-ecdh-psi/Common.h | 0 .../src/bs-ecdh-psi/core/BsEcdhCache.cpp | 0 .../src/bs-ecdh-psi/core/BsEcdhCache.h | 0 .../src/bs-ecdh-psi/core/BsEcdhIoHandler.h | 0 .../src/bs-ecdh-psi/core/BsEcdhTaskState.h | 0 .../bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h | 0 .../src/bs-ecdh-psi/ffi/wedpr_utilities.h | 0 .../src/bs-ecdh-psi/protocol/BsEcdhResult.h | 0 .../src/bs-ecdh-psi/protocol/Message.h | 0 .../ppc-psi/src/cm2020-psi/CM2020PSIConfig.h | 0 .../ppc-psi/src/cm2020-psi/CM2020PSIFactory.h | 0 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp | 0 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.h | 0 .../ppc-psi/src/cm2020-psi/CMakeLists.txt | 0 .../ppc-psi/src/cm2020-psi/Common.h | 0 .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 0 .../src/cm2020-psi/core/CM2020PSIReceiver.h | 0 .../src/cm2020-psi/core/CM2020PSISender.cpp | 0 .../src/cm2020-psi/core/CM2020PSISender.h | 0 .../ppc-psi/src/cm2020-psi/core/TaskParams.h | 0 .../src/cm2020-psi/protocol/CM2020PSI.tars | 0 .../src/cm2020-psi/protocol/CM2020PSIResult.h | 0 .../ppc-psi/src/ecdh-conn-psi/CMakeLists.txt | 0 .../ppc-psi/src/ecdh-conn-psi/Common.h | 0 .../src/ecdh-conn-psi/EcdhConnPSIConfig.h | 0 .../src/ecdh-conn-psi/EcdhConnPSIFactory.h | 0 .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp | 0 .../src/ecdh-conn-psi/EcdhConnPSIImpl.h | 0 .../ecdh-conn-psi/EcdhConnPSIMessageFactory.h | 0 .../src/ecdh-conn-psi/EcdhConnTaskState.h | 0 .../ecdh-conn-psi/core/EcdhConnPSIClient.cpp | 0 .../ecdh-conn-psi/core/EcdhConnPSIClient.h | 0 .../ecdh-conn-psi/core/EcdhConnPSIServer.cpp | 0 .../ecdh-conn-psi/core/EcdhConnPSIServer.h | 0 .../ecdh-conn-psi/protocol/PSIConnMessage.h | 0 .../protocol/PSIConnMessageFactory.cpp | 0 .../protocol/PSIConnMessageFactory.h | 0 .../ppc-psi/src/ecdh-multi-psi/CMakeLists.txt | 0 .../ppc-psi/src/ecdh-multi-psi/Common.h | 0 .../src/ecdh-multi-psi/EcdhMultiCache.h | 0 .../src/ecdh-multi-psi/EcdhMultiPSIConfig.h | 0 .../src/ecdh-multi-psi/EcdhMultiPSIFactory.h | 0 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp | 0 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.h | 0 .../EcdhMultiPSIMessageFactory.h | 0 .../core/EcdhMultiPSICalculator.cpp | 0 .../core/EcdhMultiPSICalculator.h | 0 .../core/EcdhMultiPSIMaster.cpp | 0 .../ecdh-multi-psi/core/EcdhMultiPSIMaster.h | 0 .../core/EcdhMultiPSIPartner.cpp | 0 .../ecdh-multi-psi/core/EcdhMultiPSIPartner.h | 0 .../ppc-psi/src/ecdh-psi/CMakeLists.txt | 0 .../ppc-psi/src/ecdh-psi/Common.h | 0 .../ppc-psi/src/ecdh-psi/EcdhCache.cpp | 0 .../ppc-psi/src/ecdh-psi/EcdhCache.h | 0 .../ppc-psi/src/ecdh-psi/EcdhPSIConfig.h | 0 .../ppc-psi/src/ecdh-psi/EcdhPSIFactory.h | 0 .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp | 0 .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.h | 0 .../src/ecdh-psi/EcdhPSIMessageFactory.h | 0 .../ppc-psi/src/ecdh-psi/EcdhTaskState.h | 0 .../ppc-psi/src/labeled-psi/CMakeLists.txt | 0 .../ppc-psi/src/labeled-psi/Common.h | 0 .../src/labeled-psi/LabeledPSIConfig.h | 0 .../src/labeled-psi/LabeledPSIFactory.h | 0 .../src/labeled-psi/LabeledPSIImpl.cpp | 0 .../ppc-psi/src/labeled-psi/LabeledPSIImpl.h | 0 .../src/labeled-psi/core/BinBundle.cpp | 0 .../ppc-psi/src/labeled-psi/core/BinBundle.h | 0 .../src/labeled-psi/core/LabeledPSIParams.h | 0 .../labeled-psi/core/LabeledPSIReceiver.cpp | 0 .../src/labeled-psi/core/LabeledPSIReceiver.h | 0 .../src/labeled-psi/core/LabeledPSISender.cpp | 0 .../src/labeled-psi/core/LabeledPSISender.h | 0 .../src/labeled-psi/core/QueryPackage.cpp | 0 .../src/labeled-psi/core/QueryPackage.h | 0 .../src/labeled-psi/core/ResultPackage.cpp | 0 .../src/labeled-psi/core/ResultPackage.h | 0 .../ppc-psi/src/labeled-psi/core/SenderDB.cpp | 0 .../ppc-psi/src/labeled-psi/core/SenderDB.h | 0 .../src/labeled-psi/core/TaskCommand.h | 0 .../src/labeled-psi/protocol/LabeledPSI.tars | 0 .../labeled-psi/protocol/LabeledPSIResult.h | 0 .../src/labeled-psi/protocol/Protocol.h | 0 .../src/labeled-psi/protocol/SenderCache.tars | 0 .../ppc-psi/src/psi-framework/CMakeLists.txt | 0 .../src/psi-framework/PSIFramework.cpp | 0 .../ppc-psi/src/psi-framework/PSIFramework.h | 0 .../ppc-psi/src/psi-framework/TaskGuarder.h | 0 .../ppc-psi/src/psi-framework/TaskState.h | 0 .../interfaces/PSIMessageFactory.h | 0 .../interfaces/PSIMessageInterface.h | 0 .../src/psi-framework/protocol/PSI.tars | 0 .../src/psi-framework/protocol/PSIMessage.cpp | 0 .../src/psi-framework/protocol/PSIMessage.h | 0 .../ppc-psi/src/ra2018-psi/CMakeLists.txt | 0 .../ppc-psi/src/ra2018-psi/Common.h | 0 .../ppc-psi/src/ra2018-psi/RA2018PSIConfig.h | 0 .../ppc-psi/src/ra2018-psi/RA2018PSIFactory.h | 0 .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp | 0 .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.h | 0 .../src/ra2018-psi/core/CuckooFilterInfo.h | 0 .../src/ra2018-psi/core/CuckooFilterState.h | 0 .../ra2018-psi/core/CuckooFiltersCache.cpp | 0 .../src/ra2018-psi/core/CuckooFiltersCache.h | 0 .../ra2018-psi/core/CuckoofilterAllocator.cpp | 0 .../ra2018-psi/core/CuckoofilterAllocator.h | 0 .../src/ra2018-psi/core/RA2018PSIClient.cpp | 0 .../src/ra2018-psi/core/RA2018PSIClient.h | 0 .../ra2018-psi/core/RA2018PSIClientCache.cpp | 0 .../ra2018-psi/core/RA2018PSIClientCache.h | 0 .../src/ra2018-psi/core/RA2018PSIServer.cpp | 0 .../src/ra2018-psi/core/RA2018PSIServer.h | 0 .../src/ra2018-psi/core/Ra2018TaskParam.h | 0 .../src/ra2018-psi/protocol/RA2018Message.h | 0 .../ra2018-psi/storage/RA2018PSIStorage.cpp | 0 .../src/ra2018-psi/storage/RA2018PSIStorage.h | 0 .../ppc-psi/tests/CMakeLists.txt | 0 .../tests/bs-ecdh-psi/TestBsEcdhCache.cpp | 0 .../tests/cm2020-psi/FakeCM2020PSIFactory.h | 0 .../tests/cm2020-psi/TestCM2020Impl.cpp | 0 .../ppc-psi/tests/labeled-psi/DataTools.h | 0 .../tests/labeled-psi/FakeLabeledPSIFactory.h | 0 .../tests/labeled-psi/TestLabeledPSIImpl.cpp | 0 .../tests/labeled-psi/TestSenderDB.cpp | 0 .../ppc-psi/tests}/main.cpp | 0 .../tests/ra2018-psi/TestEcdhPSIImpl.cpp | 0 .../tests/ra2018-psi/TestRA2018Impl.cpp | 0 .../tests/ra2018-psi/TestRA2018Message.cpp | 0 .../mock-data/append-full-evaluate.txt | 0 .../ra2018-psi/mock-data/fullevaluate.txt | 0 .../tests/ra2018-psi/mock-data/psi.txt | 0 .../tests/ra2018-psi/mock-data/psi2.txt | 0 .../ppc-psi/tests/ra2018-psi/mock/Common.h | 0 .../tests/ra2018-psi/mock/EcdhPSIFixture.h | 0 .../ra2018-psi/mock/FakeRA2018PSIStorage.h | 0 .../ra2018-psi/mock/RA2018MessageFixture.h | 0 .../tests/ra2018-psi/mock/RA2018PSIFixture.h | 0 .../ppc-crypto-core/CMakeLists.txt | 0 .../ppc-crypto-core/src/CMakeLists.txt | 0 .../ppc-crypto-core/src/Common.h | 0 .../ppc-crypto-core/src/hash/BLAKE2bHash.h | 0 .../src/hash/BitMixMurmurHash.h | 0 .../src/hash/HashFactoryImpl.h | 0 .../ppc-crypto-core/src/hash/MD5Hash.h | 0 .../ppc-crypto-core/src/hash/OpenSSLHash.h | 0 .../ppc-crypto-core/src/hash/SM3Hash.h | 0 .../ppc-crypto-core/src/hash/Sha256Hash.h | 0 .../ppc-crypto-core/src/hash/Sha512Hash.h | 0 .../ppc-crypto-core/src/ore/FastOre.cpp | 0 .../ppc-crypto-core/src/ore/FastOre.h | 0 .../src/sym-crypto/OpenSSL3DES.cpp | 0 .../src/sym-crypto/OpenSSL3DES.h | 0 .../src/sym-crypto/OpenSSLAES.cpp | 0 .../src/sym-crypto/OpenSSLAES.h | 0 .../src/sym-crypto/OpenSSLCipher.h | 0 .../src/sym-crypto/OpenSSLSM4.cpp | 0 .../src/sym-crypto/OpenSSLSM4.h | 0 .../ppc-crypto-core/src/tools/BitVector.cpp | 0 .../ppc-crypto-core/src/tools/BitVector.h | 0 .../ppc-crypto-core/tests/CMakeLists.txt | 0 .../ppc-crypto-core/tests/TestBitVector.cpp | 0 .../ppc-crypto-core/tests/TestHash.cpp | 0 .../ppc-crypto-core/tests/TestOre4Float.cpp | 0 .../ppc-crypto-core/tests/TestOre4Integer.cpp | 0 .../ppc-crypto-core/tests/TestOre4String.cpp | 0 .../ppc-crypto-core/tests/TestSymCrypto.cpp | 0 .../ppc-crypto-core/tests}/main.cpp | 0 .../ppc-crypto/CMakeLists.txt | 0 .../ppc-crypto/demo/CMakeLists.txt | 0 .../ppc-crypto/demo/crypto_bench.cpp | 0 .../ppc-crypto/src/CMakeLists.txt | 0 .../ppc-crypto/src/Common.h | 0 .../src/ecc/ECDHCryptoFactoryImpl.h | 0 .../ppc-crypto/src/ecc/ECDHCryptoImpl.h | 0 .../ppc-crypto/src/ecc/EccCryptoFactoryImpl.h | 0 .../ppc-crypto/src/ecc/Ed25519EccCrypto.cpp | 0 .../ppc-crypto/src/ecc/Ed25519EccCrypto.h | 0 .../ppc-crypto/src/ecc/IppECDHCryptoImpl.h | 0 .../ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp | 0 .../ppc-crypto/src/ecc/OpenSSLEccCrypto.h | 0 .../ppc-crypto/src/ecc/core/EcPoint.h | 0 .../ppc-crypto/src/oprf/EcdhOprf.cpp | 0 .../ppc-crypto/src/oprf/EcdhOprf.h | 0 .../ppc-crypto/src/oprf/RA2018Oprf.cpp | 0 .../ppc-crypto/src/oprf/RA2018Oprf.h | 0 .../ppc-crypto/src/prng/AESPRNG.cpp | 0 .../ppc-crypto/src/prng/AESPRNG.h | 0 .../ppc-crypto/src/prng/BLAKE2bPRNG.cpp | 0 .../ppc-crypto/src/prng/BLAKE2bPRNG.h | 0 .../ppc-crypto/src/randomot/SimplestOT.cpp | 0 .../ppc-crypto/src/randomot/SimplestOT.h | 0 .../ppc-crypto/tests/CMakeLists.txt | 0 .../ppc-crypto/tests/TestEccCrypto.cpp | 0 .../ppc-crypto/tests/TestOprf.cpp | 0 .../ppc-crypto/tests/TestPRNG.cpp | 0 .../ppc-crypto/tests/TestRandomot.cpp | 0 .../ppc-crypto}/tests/main.cpp | 0 .../ppc-homo/CMakeLists.txt | 0 .../ppc-homo/codec/CMakeLists.txt | 0 .../ppc-homo/codec/Common.h | 0 .../ppc-homo/codec/FloatingPointCipher.h | 0 .../ppc-homo/codec/FloatingPointCodec.cpp | 0 .../ppc-homo/codec/FloatingPointCodec.h | 0 .../ppc-homo/codec/SignedNumberCodec.h | 0 .../ppc-homo/demo/CMakeLists.txt | 0 .../ppc-homo/demo/homo_perf_demo.cpp | 0 .../ppc-homo/fahe/CMakeLists.txt | 0 cpp/{ => wedpr-crypto}/ppc-homo/fahe/Fahe.cpp | 0 cpp/{ => wedpr-crypto}/ppc-homo/fahe/Fahe.h | 0 .../ppc-homo/ihc/CMakeLists.txt | 0 .../ppc-homo/ihc/FloatingPointIhc.cpp | 0 .../ppc-homo/ihc/FloatingPointIhc.h | 0 .../ppc-homo/ihc/IhcImpl.cpp | 0 cpp/{ => wedpr-crypto}/ppc-homo/ihc/IhcImpl.h | 0 .../ppc-homo/paillier/CMakeLists.txt | 0 .../ppc-homo/paillier/Common.h | 0 .../paillier/FloatingPointPaillier.cpp | 0 .../ppc-homo/paillier/FloatingPointPaillier.h | 0 .../ppc-homo/paillier/OpenSSLPaillier.cpp | 0 .../ppc-homo/paillier/OpenSSLPaillier.h | 0 .../paillier/OpenSSLPaillierKeyPair.cpp | 0 .../paillier/OpenSSLPaillierKeyPair.h | 0 .../ppc-homo/tests/CMakeLists.txt | 0 .../ppc-homo/tests/TestFahe.cpp | 0 .../ppc-homo/tests/TestFloatingPointCodec.cpp | 0 .../tests/TestFloatingPointPaillier.cpp | 0 .../ppc-homo/tests/TestIhc.cpp | 0 .../ppc-homo/tests/TestIhcFloating.cpp | 0 .../ppc-homo/tests/TestPaillier.cpp | 0 .../ppc-homo}/tests/main.cpp | 0 cpp/{ => wedpr-crypto}/ppc-udf/CMakeLists.txt | 0 .../ppc-udf/hive/build.gradle | 0 .../hive/gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 cpp/{ => wedpr-crypto}/ppc-udf/hive/gradlew | 0 .../ppc-udf/hive/gradlew.bat | 0 .../ppc-udf/hive/settings.gradle | 0 .../webank/wedpr/hive/udf/config/Config.java | 0 .../wedpr/hive/udf/config/ConfigLoader.java | 0 .../hive/udf/config/model/ConfigProperty.java | 0 .../hive/udf/config/model/KeyConfig.java | 0 .../hive/udf/demo/GeneratePaillierMain.java | 0 .../hive/udf/exceptions/ConfigException.java | 0 .../udf/exceptions/DecryptionException.java | 0 .../udf/exceptions/EncryptionException.java | 0 .../hive/udf/exceptions/KeyException.java | 0 .../hive/udf/impl/ihc/IhcDecryptionUDF.java | 0 .../hive/udf/impl/ihc/IhcEncryptionUDF.java | 0 .../wedpr/hive/udf/impl/ihc/IhcSumUDAF.java | 0 .../hive/udf/impl/ore/OreDecryptionUDF.java | 0 .../hive/udf/impl/ore/OreEncryptionUDF.java | 0 .../udf/impl/ore/OreNumberDecryptionUDF.java | 0 .../udf/impl/ore/OreNumberEncryptionUDF.java | 0 .../impl/paillier/PaillierDecryptionUDF.java | 0 .../impl/paillier/PaillierEncryptionUDF.java | 0 .../udf/impl/paillier/PaillierSumUDAF.java | 0 .../udf/impl/symmetric/SymDecryptionUDF.java | 0 .../udf/impl/symmetric/SymEncryptionUDF.java | 0 .../hive/src/main/resources/config.toml | 0 .../hive/udf/impl/IhcEncAndDecUDFTest.java | 0 .../wedpr/hive/udf/impl/IhcSumUDAFTest.java | 0 .../wedpr/hive/udf/impl/OreDecUDFTest.java | 0 .../hive/udf/impl/PaillierDecUDFTest.java | 0 .../hive/udf/impl/PaillierEncUDFTest.java | 0 .../hive/udf/impl/PaillierSumUDAFTest.java | 0 .../wedpr/hive/udf/impl/SymDecUDFTest.java | 0 .../hive/udf/impl/SymEncryptionUDFTest.java | 0 .../hive/src/test/resources/config.toml | 0 .../ppc-udf/mysql/ore_dec.cpp | 0 .../ppc-udf/mysql/ore_dec.h | 0 .../ppc-udf/mysql/ore_enc.cpp | 0 .../ppc-udf/mysql/ore_enc.h | 0 .../ppc-udf/mysql/paillier_dec.cpp | 0 .../ppc-udf/mysql/paillier_dec.h | 0 .../ppc-udf/mysql/paillier_enc.cpp | 0 .../ppc-udf/mysql/paillier_enc.h | 0 .../ppc-udf/mysql/paillier_sum.cpp | 0 .../ppc-udf/mysql/paillier_sum.h | 0 .../ppc-udf/mysql/sym_dec.cpp | 0 .../ppc-udf/mysql/sym_dec.h | 0 .../ppc-udf/mysql/sym_enc.cpp | 0 .../ppc-udf/mysql/sym_enc.h | 0 .../ppc-udf/tests/CMakeLists.txt | 0 .../ppc-udf/tests/OreDecUDFTest.cpp | 0 .../ppc-udf/tests/OreEncUDFTest.cpp | 0 .../ppc-udf/tests/PaillierDecUDFTest.cpp | 0 .../ppc-udf/tests/PaillierEncUDFTest.cpp | 0 .../ppc-udf/tests/PaillierSumUDFTest.cpp | 0 .../ppc-udf/tests/SymDecUDFTest.cpp | 0 .../ppc-udf/tests/SymEncUDFTest.cpp | 0 .../ppc-udf}/tests/main.cpp | 0 .../sdk}/CMakeLists.txt | 0 .../sdk}/bindings/java/build.gradle | 0 .../java/gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../sdk}/bindings/java/gradlew | 0 .../sdk}/bindings/java/gradlew.bat | 0 .../sdk}/bindings/java/settings.gradle | 0 .../wedpr/sdk/jni/jmh/BenchmarkMain.java | 0 .../wedpr/sdk/jni/jmh/PaillierBenchmark.java | 0 .../bindings/java/src/main/c/CMakeLists.txt | 0 .../bindings/java/src/main/c/jni/Common.cpp | 0 .../bindings/java/src/main/c/jni/Common.h | 0 .../java/src/main/c/jni/JNIException.cpp | 0 .../java/src/main/c/jni/JNIException.h | 0 ...om_webank_wedpr_sdk_jni_crypto_FastOre.cpp | 0 .../com_webank_wedpr_sdk_jni_crypto_FastOre.h | 0 ...dpr_sdk_jni_crypto_SymmetricEncryption.cpp | 0 ...wedpr_sdk_jni_crypto_SymmetricEncryption.h | 0 ...k_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp | 0 ...ank_wedpr_sdk_jni_homo_NativeFloatingIhc.h | 0 ...k_jni_homo_NativeFloatingPointPaillier.cpp | 0 ...sdk_jni_homo_NativeFloatingPointPaillier.h | 0 .../com_webank_wedpr_sdk_jni_homo_NativeIhc.h | 0 ...bank_wedpr_sdk_jni_homo_NativePaillier.cpp | 0 ...webank_wedpr_sdk_jni_homo_NativePaillier.h | 0 .../webank/wedpr/sdk/demo/GenerateKeys.java | 0 .../sdk/jni/codec/FloatingPointNumber.java | 0 .../wedpr/sdk/jni/codec/NumberCodec.java | 0 .../sdk/jni/codec/NumberCodecException.java | 0 .../wedpr/sdk/jni/codec/NumberCodecImpl.java | 0 .../wedpr/sdk/jni/common/JniException.java | 0 .../wedpr/sdk/jni/common/JniLibLoader.java | 0 .../wedpr/sdk/jni/common/Utilities.java | 0 .../webank/wedpr/sdk/jni/crypto/FastOre.java | 0 .../sdk/jni/crypto/SymmetricEncryption.java | 0 .../wedpr/sdk/jni/homo/NativeFloatingIhc.java | 0 .../jni/homo/NativeFloatingPointPaillier.java | 0 .../wedpr/sdk/jni/homo/NativePaillier.java | 0 .../resources/META-INF/native/win/file.list | 0 .../sdk/jni/test/NativeFloatingIhcTest.java | 0 .../test/NativeFloatingPointPaillierTest.java | 0 .../sdk/jni/test/NativePaillierTest.java | 0 .../wedpr/sdk/jni/test/NumberCodecTest.java | 0 .../sdk/jni/test/SymmetricEncryptionTest.java | 0 .../sdk}/ppc-crypto-c-sdk/CMakeLists.txt | 0 .../sdk}/ppc-crypto-c-sdk/fast_ore.cpp | 0 .../sdk}/ppc-crypto-c-sdk/fast_ore.h | 0 .../ppc-crypto-c-sdk/floating_point_ihc.cpp | 0 .../ppc-crypto-c-sdk/floating_point_ihc.h | 0 .../floating_point_paillier.cpp | 0 .../floating_point_paillier.h | 0 .../sdk}/ppc-crypto-c-sdk/homo_ihc.cpp | 0 .../sdk}/ppc-crypto-c-sdk/homo_ihc.h | 0 .../sdk}/ppc-crypto-c-sdk/homo_paillier.cpp | 0 .../sdk}/ppc-crypto-c-sdk/homo_paillier.h | 0 .../ppc-crypto-c-sdk/symmetric_encryption.cpp | 0 .../ppc-crypto-c-sdk/symmetric_encryption.h | 0 .../sdk}/ppc-crypto-c-sdk/utils/error.cpp | 0 .../sdk}/ppc-crypto-c-sdk/utils/error.h | 0 .../sdk}/ppc-crypto-c-sdk/utils/utilities.cpp | 0 .../sdk}/ppc-crypto-c-sdk/utils/utilities.h | 0 .../sdk}/tests/CMakeLists.txt | 0 .../sdk}/tests/TestFastOre.cpp | 0 .../sdk}/tests/TestHomoPaillier.cpp | 0 .../sdk}/tests/main.cpp | 0 .../libhelper/CMakeLists.txt | 0 .../libhelper/CommandHelper.cpp | 0 .../libhelper/CommandHelper.h | 0 .../libhelper/ExitHandler.h | 0 .../ppc-tools/CMakeLists.txt | 0 .../ppc-tools/src/codec/CodecUtility.h | 0 .../ppc-tools/src/common/ConcurrentPool.h | 0 .../ppc-tools/src/common/MemInfo.h | 0 .../ppc-tools/src/common/Progress.h | 0 .../ppc-tools/src/common/TransTools.h | 0 .../ppc-tools/src/config/Common.h | 0 .../ppc-tools/src/config/NetworkConfig.h | 0 .../ppc-tools/src/config/PPCConfig.cpp | 0 .../ppc-tools/src/config/PPCConfig.h | 0 .../ppc-tools/src/config/ParamChecker.h | 0 .../ppc-tools/src/cuckoo/Common.h | 0 .../ppc-tools/src/cuckoo/Cuckoofilter.h | 0 .../ppc-tools/src/cuckoo/HashTable.cpp | 0 .../ppc-tools/src/cuckoo/HashTable.h | 0 .../ppc-tools/tests/CMakeLists.txt | 0 .../ppc-tools/tests/TestCuckoofilter.cpp | 0 .../ppc-tools/tests/TestTransTools.cpp | 0 .../ppc-tools}/tests/main.cpp | 0 .../ppc-utilities/Utilities.h | 0 cpp/{ => wedpr-storage}/ppc-io/CMakeLists.txt | 0 .../ppc-io/demo/CMakeLists.txt | 0 .../ppc-io/demo/io_demo.cpp | 0 .../ppc-io/src/BaseFileLineReader.cpp | 0 .../ppc-io/src/BaseFileLineReader.h | 0 .../ppc-io/src/CMakeLists.txt | 0 cpp/{ => wedpr-storage}/ppc-io/src/Common.h | 0 .../ppc-io/src/DataResourceLoaderImpl.cpp | 0 .../ppc-io/src/DataResourceLoaderImpl.h | 0 .../ppc-io/src/FileLineReader.cpp | 0 .../ppc-io/src/FileLineReader.h | 0 .../ppc-io/src/FileLineWriter.cpp | 0 .../ppc-io/src/FileLineWriter.h | 0 .../ppc-io/src/HDFSReader.cpp | 0 .../ppc-io/src/HDFSReader.h | 0 .../ppc-io/src/HDFSWriter.cpp | 0 .../ppc-io/src/HDFSWriter.h | 0 .../ppc-io/src/SQLResultReader.h | 0 .../ppc-io/src/parser/BaseBufferParser.h | 0 .../ppc-io/src/parser/BufferParser.h | 0 .../ppc-io/src/parser/MmapParser.h | 0 .../ppc-io/tests/CMakeLists.txt | 0 .../ppc-io/tests/TestLineReader.cpp | 0 .../ppc-io/tests/TestLineWriter.cpp | 0 .../ppc-io/tests/data/linux_file.txt | 0 .../ppc-io/tests/data/testData | 0 .../ppc-io/tests/data/windows_file.txt | 0 .../ppc-io}/tests/main.cpp | 0 .../ppc-storage/CMakeLists.txt | 0 .../ppc-storage/demo/CMakeLists.txt | 0 .../ppc-storage/demo/storage_demo.cpp | 0 .../ppc-storage/src/CMakeLists.txt | 0 .../ppc-storage/src/CacheStorageFactoryImpl.h | 0 .../ppc-storage/src/Common.h | 0 .../ppc-storage/src/FileStorageFactoryImpl.h | 0 .../ppc-storage/src/SQLStorageFactoryImpl.h | 0 .../ppc-storage/src/hdfs/Common.h | 0 .../ppc-storage/src/hdfs/HDFSStorage.cpp | 0 .../ppc-storage/src/hdfs/HDFSStorage.h | 0 .../ppc-storage/src/mysql/Common.h | 0 .../ppc-storage/src/mysql/MySQLStorage.cpp | 0 .../ppc-storage/src/mysql/MySQLStorage.h | 0 .../ppc-storage/src/redis/RedisStorage.cpp | 0 .../ppc-storage/src/redis/RedisStorage.h | 0 .../ppc-front/CMakeLists.txt | 0 .../ppc-front/ppc-front/CallbackManager.cpp | 0 .../ppc-front/ppc-front/CallbackManager.h | 0 .../ppc-front/ppc-front/Common.h | 0 .../ppc-front/ppc-front/Front.cpp | 0 .../ppc-front/ppc-front/Front.h | 0 .../ppc-front/ppc-front/FrontImpl.cpp | 0 .../ppc-front/ppc-front/FrontImpl.h | 0 .../ppc-front/ppc-front/PPCChannel.cpp | 0 .../ppc-front/ppc-front/PPCChannel.h | 0 .../ppc-front/ppc-front/PPCChannelManager.cpp | 0 .../ppc-front/ppc-front/PPCChannelManager.h | 0 .../ppc-front/test/CMakeLists.txt | 0 .../test/unittests/PPCChannelTest.cpp | 0 .../ppc-front/test/unittests}/main.cpp | 0 .../ppc-gateway/CMakeLists.txt | 0 .../ppc-gateway/ppc-gateway/Common.h | 0 .../ppc-gateway/GatewayConfigContext.cpp | 0 .../ppc-gateway/GatewayConfigContext.h | 0 .../ppc-gateway/GatewayConfigLoader.cpp | 0 .../ppc-gateway/GatewayConfigLoader.h | 0 .../ppc-gateway/GatewayFactory.cpp | 0 .../ppc-gateway/ppc-gateway/GatewayFactory.h | 0 .../ppc-gateway/gateway/GatewayImpl.cpp | 0 .../ppc-gateway/gateway/GatewayImpl.h | 0 .../gateway/SendMessageWithRetry.cpp | 0 .../gateway/SendMessageWithRetry.h | 0 .../gateway/cache/MessageCache.cpp | 0 .../ppc-gateway/gateway/cache/MessageCache.h | 0 .../gateway/router/GatewayNodeInfo.h | 0 .../gateway/router/GatewayNodeInfoImpl.cpp | 0 .../gateway/router/GatewayNodeInfoImpl.h | 0 .../gateway/router/GatewayRouterManager.cpp | 0 .../gateway/router/GatewayRouterManager.h | 0 .../gateway/router/LocalRouter.cpp | 0 .../ppc-gateway/gateway/router/LocalRouter.h | 0 .../gateway/router/PeerRouterTable.cpp | 0 .../gateway/router/PeerRouterTable.h | 0 .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 0 .../ppc-gateway/ppc-gateway/p2p/Service.h | 0 .../ppc-gateway/p2p/router/RouterManager.cpp | 0 .../ppc-gateway/p2p/router/RouterManager.h | 0 .../p2p/router/RouterTableImpl.cpp | 0 .../ppc-gateway/p2p/router/RouterTableImpl.h | 0 .../p2p/router/RouterTableInterface.h | 0 .../ppc-gateway/test/CMakeLists.txt | 0 .../ppc-gateway/test/data/config0.ini | 0 .../ppc-gateway/test/demo/CMakeLists.txt | 0 .../ppc-gateway/test/demo/gateway_demo.cpp | 0 .../test/unittests/GatewayTest.cpp | 0 .../ppc-gateway/test/unittests/MockCache.h | 0 .../ppc-gateway/test/unittests}/main.cpp | 0 .../ppc-http/CMakeLists.txt | 0 .../ppc-http/demo/CMakeLists.txt | 0 .../ppc-http/demo/config.ini | 0 .../ppc-http/demo/http_client_demo.cpp | 0 .../ppc-http/demo/http_demo.cpp | 0 .../ppc-http/src/CMakeLists.txt | 0 .../ppc-http/src/Http.cpp | 0 cpp/{ => wedpr-transport}/ppc-http/src/Http.h | 0 .../ppc-http/src/HttpClient.cpp | 0 .../ppc-http/src/HttpClient.h | 0 .../ppc-http/src/HttpFactory.cpp | 0 .../ppc-http/src/HttpFactory.h | 0 .../ppc-rpc/CMakeLists.txt | 0 .../ppc-rpc/demo/CMakeLists.txt | 0 .../ppc-rpc/demo/config.ini | 0 .../ppc-rpc/demo/rpc_demo.cpp | 0 .../ppc-rpc/src/CMakeLists.txt | 0 .../ppc-rpc/src/JsonRequest.h | 0 .../ppc-rpc/src/JsonResponse.h | 0 cpp/{ => wedpr-transport}/ppc-rpc/src/Rpc.cpp | 0 cpp/{ => wedpr-transport}/ppc-rpc/src/Rpc.h | 0 .../ppc-rpc/src/RpcFactory.cpp | 0 .../ppc-rpc/src/RpcFactory.h | 0 .../ppc-rpc/src/RpcMemory.cpp | 0 .../ppc-rpc/src/RpcMemory.h | 0 .../ppc-rpc/tests/CMakeLists.txt | 0 .../ppc-rpc/tests/TestRpcRequestResponse.cpp | 0 .../ppc-rpc}/tests/main.cpp | 0 557 files changed, 23 insertions(+), 18 deletions(-) delete mode 100644 cpp/ppc-pir/tests/data/AysPreDataset.csv rename cpp/{ => wedpr-computing}/ppc-cem/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/CEMService.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/CEMService.h (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/WedprUtilities.h (100%) rename cpp/{ => wedpr-computing}/ppc-cem/src/wedpr_ffi_c_equality.h (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/TestCEMService.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/data/config.ini (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/data/d1 (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/data/d2 (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/data/d3 (100%) rename cpp/{ => wedpr-computing}/ppc-cem/tests/main.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/src/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/src/MPCService.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/src/MPCService.h (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/tests/TestMPCService.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-mpc/tests/data/config.ini (100%) rename cpp/{ppc-crypto-core => wedpr-computing/ppc-mpc}/tests/main.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-pir/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/BaseOT.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/BaseOT.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/OtPIRConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/OtPIRFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/OtPIRImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/OtPIRImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/src/protocol/OtPIR.tars (100%) rename cpp/{ => wedpr-computing}/ppc-pir/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-pir/tests/FakeOtPIRFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-pir/tests/TestBaseOT.cpp (100%) rename cpp/{ppc-crypto => wedpr-computing/ppc-pir}/tests/main.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-pir/tests/test.json (100%) rename cpp/{ => wedpr-computing}/ppc-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/demo/hash_collision_bench.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/PSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/bs-ecdh-psi/protocol/Message.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/core/TaskParams.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhCache.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhCache.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ecdh-psi/EcdhTaskState.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/LabeledPSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/LabeledPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/LabeledPSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/BinBundle.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/BinBundle.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/LabeledPSISender.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/QueryPackage.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/QueryPackage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/ResultPackage.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/ResultPackage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/SenderDB.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/SenderDB.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/core/TaskCommand.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/protocol/Protocol.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/labeled-psi/protocol/SenderCache.tars (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/PSIFramework.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/PSIFramework.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/TaskGuarder.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/TaskState.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/protocol/PSI.tars (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/psi-framework/protocol/PSIMessage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/labeled-psi/DataTools.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/labeled-psi/TestSenderDB.cpp (100%) rename cpp/{ppc-front/test/unittests => wedpr-computing/ppc-psi/tests}/main.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock-data/psi.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock/Common.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h (100%) rename cpp/{ => wedpr-computing}/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/Common.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/BLAKE2bHash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/BitMixMurmurHash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/HashFactoryImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/MD5Hash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/OpenSSLHash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/SM3Hash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/Sha256Hash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/hash/Sha512Hash.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/ore/FastOre.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/ore/FastOre.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/tools/BitVector.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/src/tools/BitVector.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestBitVector.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestHash.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestOre4Float.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestOre4Integer.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestOre4String.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto-core/tests/TestSymCrypto.cpp (100%) rename cpp/{ppc-gateway/test/unittests => wedpr-crypto/ppc-crypto-core/tests}/main.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/demo/crypto_bench.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/Common.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/ECDHCryptoImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/Ed25519EccCrypto.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/IppECDHCryptoImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/OpenSSLEccCrypto.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/ecc/core/EcPoint.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/oprf/EcdhOprf.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/oprf/EcdhOprf.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/oprf/RA2018Oprf.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/oprf/RA2018Oprf.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/prng/AESPRNG.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/prng/AESPRNG.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/prng/BLAKE2bPRNG.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/prng/BLAKE2bPRNG.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/randomot/SimplestOT.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/src/randomot/SimplestOT.h (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/tests/TestEccCrypto.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/tests/TestOprf.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/tests/TestPRNG.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-crypto/tests/TestRandomot.cpp (100%) rename cpp/{ppc-homo => wedpr-crypto/ppc-crypto}/tests/main.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/Common.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/FloatingPointCipher.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/FloatingPointCodec.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/FloatingPointCodec.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/codec/SignedNumberCodec.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/demo/homo_perf_demo.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/fahe/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/fahe/Fahe.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/fahe/Fahe.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/ihc/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/ihc/FloatingPointIhc.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/ihc/FloatingPointIhc.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/ihc/IhcImpl.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/ihc/IhcImpl.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/Common.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/FloatingPointPaillier.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/FloatingPointPaillier.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/OpenSSLPaillier.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/OpenSSLPaillier.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/paillier/OpenSSLPaillierKeyPair.h (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestFahe.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestFloatingPointCodec.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestFloatingPointPaillier.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestIhc.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestIhcFloating.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-homo/tests/TestPaillier.cpp (100%) rename cpp/{ppc-io => wedpr-crypto/ppc-homo}/tests/main.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/build.gradle (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/gradlew (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/gradlew.bat (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/settings.gradle (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/main/resources/config.toml (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/hive/src/test/resources/config.toml (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/ore_dec.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/ore_dec.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/ore_enc.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/ore_enc.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_dec.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_dec.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_enc.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_enc.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_sum.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/paillier_sum.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/sym_dec.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/sym_dec.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/sym_enc.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/mysql/sym_enc.h (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/OreDecUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/OreEncUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/PaillierDecUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/PaillierEncUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/PaillierSumUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/SymDecUDFTest.cpp (100%) rename cpp/{ => wedpr-crypto}/ppc-udf/tests/SymEncUDFTest.cpp (100%) rename cpp/{ppc-mpc => wedpr-crypto/ppc-udf}/tests/main.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/CMakeLists.txt (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/build.gradle (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/gradle/wrapper/gradle-wrapper.jar (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/gradle/wrapper/gradle-wrapper.properties (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/gradlew (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/gradlew.bat (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/settings.gradle (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/CMakeLists.txt (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/Common.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/Common.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/JNIException.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/JNIException.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/main/resources/META-INF/native/win/file.list (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/CMakeLists.txt (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/fast_ore.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/fast_ore.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/floating_point_ihc.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/floating_point_ihc.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/floating_point_paillier.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/floating_point_paillier.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/homo_ihc.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/homo_ihc.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/homo_paillier.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/homo_paillier.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/symmetric_encryption.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/symmetric_encryption.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/utils/error.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/utils/error.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/utils/utilities.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/ppc-crypto-c-sdk/utils/utilities.h (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/tests/CMakeLists.txt (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/tests/TestFastOre.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-crypto/sdk}/tests/TestHomoPaillier.cpp (100%) rename cpp/{ppc-pir => wedpr-crypto/sdk}/tests/main.cpp (100%) rename cpp/{ => wedpr-helper}/libhelper/CMakeLists.txt (100%) rename cpp/{ => wedpr-helper}/libhelper/CommandHelper.cpp (100%) rename cpp/{ => wedpr-helper}/libhelper/CommandHelper.h (100%) rename cpp/{ => wedpr-helper}/libhelper/ExitHandler.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/CMakeLists.txt (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/codec/CodecUtility.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/common/ConcurrentPool.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/common/MemInfo.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/common/Progress.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/common/TransTools.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/config/Common.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/config/NetworkConfig.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/config/PPCConfig.cpp (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/config/PPCConfig.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/config/ParamChecker.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/cuckoo/Common.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/cuckoo/Cuckoofilter.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/cuckoo/HashTable.cpp (100%) rename cpp/{ => wedpr-helper}/ppc-tools/src/cuckoo/HashTable.h (100%) rename cpp/{ => wedpr-helper}/ppc-tools/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-helper}/ppc-tools/tests/TestCuckoofilter.cpp (100%) rename cpp/{ => wedpr-helper}/ppc-tools/tests/TestTransTools.cpp (100%) rename cpp/{ppc-psi => wedpr-helper/ppc-tools}/tests/main.cpp (100%) rename cpp/{ => wedpr-helper}/ppc-utilities/Utilities.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-io/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-io/demo/io_demo.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/BaseFileLineReader.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/BaseFileLineReader.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/Common.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/DataResourceLoaderImpl.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/DataResourceLoaderImpl.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/FileLineReader.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/FileLineReader.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/FileLineWriter.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/FileLineWriter.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/HDFSReader.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/HDFSReader.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/HDFSWriter.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/HDFSWriter.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/SQLResultReader.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/parser/BaseBufferParser.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/parser/BufferParser.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/src/parser/MmapParser.h (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/TestLineReader.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/TestLineWriter.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/data/linux_file.txt (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/data/testData (100%) rename cpp/{ => wedpr-storage}/ppc-io/tests/data/windows_file.txt (100%) rename cpp/{ppc-rpc => wedpr-storage/ppc-io}/tests/main.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-storage/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-storage/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-storage/demo/storage_demo.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/CacheStorageFactoryImpl.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/Common.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/FileStorageFactoryImpl.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/SQLStorageFactoryImpl.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/hdfs/Common.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/hdfs/HDFSStorage.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/hdfs/HDFSStorage.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/mysql/Common.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/mysql/MySQLStorage.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/mysql/MySQLStorage.h (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/redis/RedisStorage.cpp (100%) rename cpp/{ => wedpr-storage}/ppc-storage/src/redis/RedisStorage.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/CallbackManager.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/CallbackManager.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/Common.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/Front.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/Front.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/FrontImpl.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/FrontImpl.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/PPCChannel.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/PPCChannel.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/PPCChannelManager.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-front/ppc-front/PPCChannelManager.h (100%) rename cpp/{ => wedpr-transport}/ppc-front/test/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-front/test/unittests/PPCChannelTest.cpp (100%) rename cpp/{ppc-tools/tests => wedpr-transport/ppc-front/test/unittests}/main.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/Common.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayConfigContext.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayConfigLoader.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayFactory.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/GatewayFactory.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/Service.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/Service.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/data/config0.ini (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/demo/gateway_demo.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/unittests/GatewayTest.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-gateway/test/unittests/MockCache.h (100%) rename cpp/{ppc-udf/tests => wedpr-transport/ppc-gateway/test/unittests}/main.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-http/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-http/demo/config.ini (100%) rename cpp/{ => wedpr-transport}/ppc-http/demo/http_client_demo.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/demo/http_demo.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/Http.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/Http.h (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/HttpClient.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/HttpClient.h (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/HttpFactory.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-http/src/HttpFactory.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/demo/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/demo/config.ini (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/demo/rpc_demo.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/JsonRequest.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/JsonResponse.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/Rpc.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/Rpc.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/RpcFactory.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/RpcFactory.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/RpcMemory.cpp (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/src/RpcMemory.h (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/tests/CMakeLists.txt (100%) rename cpp/{ => wedpr-transport}/ppc-rpc/tests/TestRpcRequestResponse.cpp (100%) rename cpp/{wedpr-component-sdk => wedpr-transport/ppc-rpc}/tests/main.cpp (100%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ca01df86..9054a717 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -64,28 +64,31 @@ include(TargetSettings) include(Dependencies) ########### set the sources ########### -set(JNI_SOURCE_PATH wedpr-component-sdk/bindings/java/src/main/c) -set(SDK_SOURCE_LIST ppc-homo ppc-crypto-core wedpr-component-sdk ${JNI_SOURCE_PATH}) +set(JNI_SOURCE_PATH wedpr-crypto/sdk/bindings/java/src/main/c) +set(SDK_SOURCE_LIST wedpr-crypto/ppc-homo wedpr-crypto/ppc-crypto-core wedpr-crypto/sdk ${JNI_SOURCE_PATH}) # Note: the udf depends on mysql, not enabled in the full node mode -set(UDF_SOURCE_LIST ${SDK_SOURCE_LIST} ppc-udf) +set(UDF_SOURCE_LIST ${SDK_SOURCE_LIST} wedpr-crypto/ppc-udf) set(CEM_SOURCE "") if(BUILD_CEM) - set(CEM_SOURCE "ppc-cem") + set(CEM_SOURCE "wedpr-computing/ppc-cem") endif() #set(ALL_SOURCE_LIST # ${SDK_SOURCE_LIST} ppc-crypto # libhelper libinitializer ppc-io ppc-protocol # ppc-gateway ppc-front ppc-tars-protocol -# ppc-tools ppc-storage ppc-psi ppc-rpc +# wedpr-helper/ppc-tools ppc-storage ppc-psi ppc-rpc # ppc-http ppc-mpc ppc-pir # ${CEM_SOURCE} ppc-main) set(ALL_SOURCE_LIST - ${SDK_SOURCE_LIST} ppc-crypto - libhelper ppc-io wedpr-protocol - ppc-gateway ppc-front - ppc-tools ppc-storage ppc-psi ppc-rpc - ppc-http ppc-mpc ppc-pir ${CEM_SOURCE}) + ${SDK_SOURCE_LIST} + wedpr-crypto/ppc-crypto + wedpr-protocol + wedpr-helper/libhelper wedpr-helper/ppc-tools + wedpr-storage/ppc-io wedpr-storage/ppc-storage + wedpr-transport/ppc-gateway wedpr-transport/ppc-front + wedpr-transport/ppc-http wedpr-transport/ppc-rpc + wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE}) if(BUILD_WEDPR_TOOLKIT) # fetch the python dependencies diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index b2f9e8c8..5d1643be 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -2,9 +2,14 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_BINARY_DIR}/generated/) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-protocol) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-front) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ppc-gateway) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-component-sdk) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-computing) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-crypto) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-storage) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-helper) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-crypto/sdk) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/ppc-front) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/ppc-gateway) include_directories(${PROTO_OUTPUT_PATH}) set(VCPKG_INCLUDE_PATH "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include") diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 645cf931..ffe57295 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -95,7 +95,7 @@ if (ENABLE_CPU_FEATURES) set(CPU_FEATURES_LIB CpuFeatures::cpu_features) endif () -#====== wedpr-component-sdk =========== +#====== wedpr-crypto-sdk =========== set(PPC_CRYPTO_C_SDK_STATIC_TARGET ppc-crypto-c-sdk-static) set(PPC_CRYPTO_C_SDK_TARGET ppc-crypto-c-sdk) @@ -107,7 +107,7 @@ if(ARCH_NATIVE) set(PPC_CRYPTO_C_SDK_STATIC_TARGET "ppc-crypto-c-sdk-aarch64") set(PPC_CRYPTO_C_SDK_TARGET "ppc-crypto-c-sdk-static-aarch64") endif() -#====== wedpr-component-sdk =========== +#====== wedpr-crypto-sdk =========== #====== ppc-crypto-sdk-jni =========== diff --git a/cpp/ppc-pir/tests/data/AysPreDataset.csv b/cpp/ppc-pir/tests/data/AysPreDataset.csv deleted file mode 100644 index 590b61f2..00000000 --- a/cpp/ppc-pir/tests/data/AysPreDataset.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,x1 -1,test -2,test2 diff --git a/cpp/ppc-cem/CMakeLists.txt b/cpp/wedpr-computing/ppc-cem/CMakeLists.txt similarity index 100% rename from cpp/ppc-cem/CMakeLists.txt rename to cpp/wedpr-computing/ppc-cem/CMakeLists.txt diff --git a/cpp/ppc-cem/src/CEMService.cpp b/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp similarity index 100% rename from cpp/ppc-cem/src/CEMService.cpp rename to cpp/wedpr-computing/ppc-cem/src/CEMService.cpp diff --git a/cpp/ppc-cem/src/CEMService.h b/cpp/wedpr-computing/ppc-cem/src/CEMService.h similarity index 100% rename from cpp/ppc-cem/src/CEMService.h rename to cpp/wedpr-computing/ppc-cem/src/CEMService.h diff --git a/cpp/ppc-cem/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-cem/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-cem/src/CMakeLists.txt rename to cpp/wedpr-computing/ppc-cem/src/CMakeLists.txt diff --git a/cpp/ppc-cem/src/Common.h b/cpp/wedpr-computing/ppc-cem/src/Common.h similarity index 100% rename from cpp/ppc-cem/src/Common.h rename to cpp/wedpr-computing/ppc-cem/src/Common.h diff --git a/cpp/ppc-cem/src/WedprUtilities.h b/cpp/wedpr-computing/ppc-cem/src/WedprUtilities.h similarity index 100% rename from cpp/ppc-cem/src/WedprUtilities.h rename to cpp/wedpr-computing/ppc-cem/src/WedprUtilities.h diff --git a/cpp/ppc-cem/src/wedpr_ffi_c_equality.h b/cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h similarity index 100% rename from cpp/ppc-cem/src/wedpr_ffi_c_equality.h rename to cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h diff --git a/cpp/ppc-cem/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-cem/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-cem/tests/CMakeLists.txt rename to cpp/wedpr-computing/ppc-cem/tests/CMakeLists.txt diff --git a/cpp/ppc-cem/tests/TestCEMService.cpp b/cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp similarity index 100% rename from cpp/ppc-cem/tests/TestCEMService.cpp rename to cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp diff --git a/cpp/ppc-cem/tests/data/config.ini b/cpp/wedpr-computing/ppc-cem/tests/data/config.ini similarity index 100% rename from cpp/ppc-cem/tests/data/config.ini rename to cpp/wedpr-computing/ppc-cem/tests/data/config.ini diff --git a/cpp/ppc-cem/tests/data/d1 b/cpp/wedpr-computing/ppc-cem/tests/data/d1 similarity index 100% rename from cpp/ppc-cem/tests/data/d1 rename to cpp/wedpr-computing/ppc-cem/tests/data/d1 diff --git a/cpp/ppc-cem/tests/data/d2 b/cpp/wedpr-computing/ppc-cem/tests/data/d2 similarity index 100% rename from cpp/ppc-cem/tests/data/d2 rename to cpp/wedpr-computing/ppc-cem/tests/data/d2 diff --git a/cpp/ppc-cem/tests/data/d3 b/cpp/wedpr-computing/ppc-cem/tests/data/d3 similarity index 100% rename from cpp/ppc-cem/tests/data/d3 rename to cpp/wedpr-computing/ppc-cem/tests/data/d3 diff --git a/cpp/ppc-cem/tests/main.cpp b/cpp/wedpr-computing/ppc-cem/tests/main.cpp similarity index 100% rename from cpp/ppc-cem/tests/main.cpp rename to cpp/wedpr-computing/ppc-cem/tests/main.cpp diff --git a/cpp/ppc-mpc/CMakeLists.txt b/cpp/wedpr-computing/ppc-mpc/CMakeLists.txt similarity index 100% rename from cpp/ppc-mpc/CMakeLists.txt rename to cpp/wedpr-computing/ppc-mpc/CMakeLists.txt diff --git a/cpp/ppc-mpc/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-mpc/src/CMakeLists.txt rename to cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt diff --git a/cpp/ppc-mpc/src/Common.h b/cpp/wedpr-computing/ppc-mpc/src/Common.h similarity index 100% rename from cpp/ppc-mpc/src/Common.h rename to cpp/wedpr-computing/ppc-mpc/src/Common.h diff --git a/cpp/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp similarity index 100% rename from cpp/ppc-mpc/src/MPCService.cpp rename to cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp diff --git a/cpp/ppc-mpc/src/MPCService.h b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h similarity index 100% rename from cpp/ppc-mpc/src/MPCService.h rename to cpp/wedpr-computing/ppc-mpc/src/MPCService.h diff --git a/cpp/ppc-mpc/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-mpc/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-mpc/tests/CMakeLists.txt rename to cpp/wedpr-computing/ppc-mpc/tests/CMakeLists.txt diff --git a/cpp/ppc-mpc/tests/TestMPCService.cpp b/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp similarity index 100% rename from cpp/ppc-mpc/tests/TestMPCService.cpp rename to cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp diff --git a/cpp/ppc-mpc/tests/data/config.ini b/cpp/wedpr-computing/ppc-mpc/tests/data/config.ini similarity index 100% rename from cpp/ppc-mpc/tests/data/config.ini rename to cpp/wedpr-computing/ppc-mpc/tests/data/config.ini diff --git a/cpp/ppc-crypto-core/tests/main.cpp b/cpp/wedpr-computing/ppc-mpc/tests/main.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/main.cpp rename to cpp/wedpr-computing/ppc-mpc/tests/main.cpp diff --git a/cpp/ppc-pir/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/CMakeLists.txt similarity index 100% rename from cpp/ppc-pir/CMakeLists.txt rename to cpp/wedpr-computing/ppc-pir/CMakeLists.txt diff --git a/cpp/ppc-pir/src/BaseOT.cpp b/cpp/wedpr-computing/ppc-pir/src/BaseOT.cpp similarity index 100% rename from cpp/ppc-pir/src/BaseOT.cpp rename to cpp/wedpr-computing/ppc-pir/src/BaseOT.cpp diff --git a/cpp/ppc-pir/src/BaseOT.h b/cpp/wedpr-computing/ppc-pir/src/BaseOT.h similarity index 100% rename from cpp/ppc-pir/src/BaseOT.h rename to cpp/wedpr-computing/ppc-pir/src/BaseOT.h diff --git a/cpp/ppc-pir/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-pir/src/CMakeLists.txt rename to cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt diff --git a/cpp/ppc-pir/src/Common.h b/cpp/wedpr-computing/ppc-pir/src/Common.h similarity index 100% rename from cpp/ppc-pir/src/Common.h rename to cpp/wedpr-computing/ppc-pir/src/Common.h diff --git a/cpp/ppc-pir/src/OtPIRConfig.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h similarity index 100% rename from cpp/ppc-pir/src/OtPIRConfig.h rename to cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h diff --git a/cpp/ppc-pir/src/OtPIRFactory.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h similarity index 100% rename from cpp/ppc-pir/src/OtPIRFactory.h rename to cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h diff --git a/cpp/ppc-pir/src/OtPIRImpl.cpp b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp similarity index 100% rename from cpp/ppc-pir/src/OtPIRImpl.cpp rename to cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp diff --git a/cpp/ppc-pir/src/OtPIRImpl.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h similarity index 100% rename from cpp/ppc-pir/src/OtPIRImpl.h rename to cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h diff --git a/cpp/ppc-pir/src/protocol/OtPIR.tars b/cpp/wedpr-computing/ppc-pir/src/protocol/OtPIR.tars similarity index 100% rename from cpp/ppc-pir/src/protocol/OtPIR.tars rename to cpp/wedpr-computing/ppc-pir/src/protocol/OtPIR.tars diff --git a/cpp/ppc-pir/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-pir/tests/CMakeLists.txt rename to cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt diff --git a/cpp/ppc-pir/tests/FakeOtPIRFactory.h b/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h similarity index 100% rename from cpp/ppc-pir/tests/FakeOtPIRFactory.h rename to cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h diff --git a/cpp/ppc-pir/tests/TestBaseOT.cpp b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp similarity index 100% rename from cpp/ppc-pir/tests/TestBaseOT.cpp rename to cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp diff --git a/cpp/ppc-crypto/tests/main.cpp b/cpp/wedpr-computing/ppc-pir/tests/main.cpp similarity index 100% rename from cpp/ppc-crypto/tests/main.cpp rename to cpp/wedpr-computing/ppc-pir/tests/main.cpp diff --git a/cpp/ppc-pir/tests/test.json b/cpp/wedpr-computing/ppc-pir/tests/test.json similarity index 100% rename from cpp/ppc-pir/tests/test.json rename to cpp/wedpr-computing/ppc-pir/tests/test.json diff --git a/cpp/ppc-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/demo/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/demo/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/demo/CMakeLists.txt diff --git a/cpp/ppc-psi/demo/hash_collision_bench.cpp b/cpp/wedpr-computing/ppc-psi/demo/hash_collision_bench.cpp similarity index 100% rename from cpp/ppc-psi/demo/hash_collision_bench.cpp rename to cpp/wedpr-computing/ppc-psi/demo/hash_collision_bench.cpp diff --git a/cpp/ppc-psi/src/Common.h b/cpp/wedpr-computing/ppc-psi/src/Common.h similarity index 100% rename from cpp/ppc-psi/src/Common.h rename to cpp/wedpr-computing/ppc-psi/src/Common.h diff --git a/cpp/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/PSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/PSIConfig.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIInterface.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/Common.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.cpp diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhIoHandler.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h diff --git a/cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/Message.h similarity index 100% rename from cpp/ppc-psi/src/bs-ecdh-psi/protocol/Message.h rename to cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/Message.h diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp diff --git a/cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h diff --git a/cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/cm2020-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.h diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp diff --git a/cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.h diff --git a/cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/TaskParams.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/core/TaskParams.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/TaskParams.h diff --git a/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSI.tars diff --git a/cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h similarity index 100% rename from cpp/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h rename to cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/Common.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIMessageFactory.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.cpp diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIClient.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessage.h diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp diff --git a/cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIMessageFactory.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp diff --git a/cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h diff --git a/cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/ecdh-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/Common.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhCache.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhCache.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhCache.cpp diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhCache.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhCache.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhCache.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhCache.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIMessageFactory.h diff --git a/cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhTaskState.h similarity index 100% rename from cpp/ppc-psi/src/ecdh-psi/EcdhTaskState.h rename to cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhTaskState.h diff --git a/cpp/ppc-psi/src/labeled-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/labeled-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/Common.h diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/LabeledPSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/LabeledPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/LabeledPSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/BinBundle.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/BinBundle.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/BinBundle.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/BinBundle.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/BinBundle.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/BinBundle.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/BinBundle.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIParams.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/LabeledPSISender.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/QueryPackage.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/QueryPackage.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/QueryPackage.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/QueryPackage.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/QueryPackage.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/QueryPackage.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/ResultPackage.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/ResultPackage.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/ResultPackage.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/ResultPackage.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/ResultPackage.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/ResultPackage.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/SenderDB.cpp similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/SenderDB.cpp rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/SenderDB.cpp diff --git a/cpp/ppc-psi/src/labeled-psi/core/SenderDB.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/SenderDB.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/SenderDB.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/SenderDB.h diff --git a/cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/TaskCommand.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/core/TaskCommand.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/TaskCommand.h diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSI.tars diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/Protocol.h similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/protocol/Protocol.h rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/Protocol.h diff --git a/cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/SenderCache.tars similarity index 100% rename from cpp/ppc-psi/src/labeled-psi/protocol/SenderCache.tars rename to cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/SenderCache.tars diff --git a/cpp/ppc-psi/src/psi-framework/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/psi-framework/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt diff --git a/cpp/ppc-psi/src/psi-framework/PSIFramework.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp similarity index 100% rename from cpp/ppc-psi/src/psi-framework/PSIFramework.cpp rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp diff --git a/cpp/ppc-psi/src/psi-framework/PSIFramework.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/PSIFramework.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h diff --git a/cpp/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/TaskGuarder.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h diff --git a/cpp/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/TaskState.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h diff --git a/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageFactory.h diff --git a/cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSI.tars b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars similarity index 100% rename from cpp/ppc-psi/src/psi-framework/protocol/PSI.tars rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp similarity index 100% rename from cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp diff --git a/cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h similarity index 100% rename from cpp/ppc-psi/src/psi-framework/protocol/PSIMessage.h rename to cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h diff --git a/cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt diff --git a/cpp/ppc-psi/src/ra2018-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/Common.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/Common.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/Common.h diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFilterInfo.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFilterState.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckooFiltersCache.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/CuckoofilterAllocator.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClient.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIClientCache.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/RA2018PSIServer.h diff --git a/cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/core/Ra2018TaskParam.h diff --git a/cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/protocol/RA2018Message.h diff --git a/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.cpp diff --git a/cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h similarity index 100% rename from cpp/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h rename to cpp/wedpr-computing/ppc-psi/src/ra2018-psi/storage/RA2018PSIStorage.h diff --git a/cpp/ppc-psi/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-psi/tests/CMakeLists.txt rename to cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt diff --git a/cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp b/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp similarity index 100% rename from cpp/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp rename to cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp diff --git a/cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h similarity index 100% rename from cpp/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h rename to cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h diff --git a/cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp similarity index 100% rename from cpp/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp rename to cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp diff --git a/cpp/ppc-psi/tests/labeled-psi/DataTools.h b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h similarity index 100% rename from cpp/ppc-psi/tests/labeled-psi/DataTools.h rename to cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h diff --git a/cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h similarity index 100% rename from cpp/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h rename to cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h diff --git a/cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp diff --git a/cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp similarity index 100% rename from cpp/ppc-psi/tests/labeled-psi/TestSenderDB.cpp rename to cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp diff --git a/cpp/ppc-front/test/unittests/main.cpp b/cpp/wedpr-computing/ppc-psi/tests/main.cpp similarity index 100% rename from cpp/ppc-front/test/unittests/main.cpp rename to cpp/wedpr-computing/ppc-psi/tests/main.cpp diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp diff --git a/cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock-data/psi.txt rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/Common.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock/Common.h rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/FakeRA2018PSIStorage.h diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018MessageFixture.h diff --git a/cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h similarity index 100% rename from cpp/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h rename to cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h diff --git a/cpp/ppc-crypto-core/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto-core/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt diff --git a/cpp/ppc-crypto-core/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto-core/src/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt diff --git a/cpp/ppc-crypto-core/src/Common.h b/cpp/wedpr-crypto/ppc-crypto-core/src/Common.h similarity index 100% rename from cpp/ppc-crypto-core/src/Common.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/Common.h diff --git a/cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/BLAKE2bHash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/BLAKE2bHash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/BLAKE2bHash.h diff --git a/cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/BitMixMurmurHash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/BitMixMurmurHash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/BitMixMurmurHash.h diff --git a/cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/HashFactoryImpl.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/HashFactoryImpl.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/HashFactoryImpl.h diff --git a/cpp/ppc-crypto-core/src/hash/MD5Hash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/MD5Hash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/MD5Hash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/MD5Hash.h diff --git a/cpp/ppc-crypto-core/src/hash/OpenSSLHash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/OpenSSLHash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/OpenSSLHash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/OpenSSLHash.h diff --git a/cpp/ppc-crypto-core/src/hash/SM3Hash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/SM3Hash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/SM3Hash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/SM3Hash.h diff --git a/cpp/ppc-crypto-core/src/hash/Sha256Hash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/Sha256Hash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/Sha256Hash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/Sha256Hash.h diff --git a/cpp/ppc-crypto-core/src/hash/Sha512Hash.h b/cpp/wedpr-crypto/ppc-crypto-core/src/hash/Sha512Hash.h similarity index 100% rename from cpp/ppc-crypto-core/src/hash/Sha512Hash.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/hash/Sha512Hash.h diff --git a/cpp/ppc-crypto-core/src/ore/FastOre.cpp b/cpp/wedpr-crypto/ppc-crypto-core/src/ore/FastOre.cpp similarity index 100% rename from cpp/ppc-crypto-core/src/ore/FastOre.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/src/ore/FastOre.cpp diff --git a/cpp/ppc-crypto-core/src/ore/FastOre.h b/cpp/wedpr-crypto/ppc-crypto-core/src/ore/FastOre.h similarity index 100% rename from cpp/ppc-crypto-core/src/ore/FastOre.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/ore/FastOre.h diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.cpp diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSL3DES.h diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLAES.cpp diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLAES.h diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLCipher.h diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.cpp diff --git a/cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h b/cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h similarity index 100% rename from cpp/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/sym-crypto/OpenSSLSM4.h diff --git a/cpp/ppc-crypto-core/src/tools/BitVector.cpp b/cpp/wedpr-crypto/ppc-crypto-core/src/tools/BitVector.cpp similarity index 100% rename from cpp/ppc-crypto-core/src/tools/BitVector.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/src/tools/BitVector.cpp diff --git a/cpp/ppc-crypto-core/src/tools/BitVector.h b/cpp/wedpr-crypto/ppc-crypto-core/src/tools/BitVector.h similarity index 100% rename from cpp/ppc-crypto-core/src/tools/BitVector.h rename to cpp/wedpr-crypto/ppc-crypto-core/src/tools/BitVector.h diff --git a/cpp/ppc-crypto-core/tests/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto-core/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto-core/tests/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto-core/tests/CMakeLists.txt diff --git a/cpp/ppc-crypto-core/tests/TestBitVector.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestBitVector.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestBitVector.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestBitVector.cpp diff --git a/cpp/ppc-crypto-core/tests/TestHash.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestHash.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestHash.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestHash.cpp diff --git a/cpp/ppc-crypto-core/tests/TestOre4Float.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4Float.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestOre4Float.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4Float.cpp diff --git a/cpp/ppc-crypto-core/tests/TestOre4Integer.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4Integer.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestOre4Integer.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4Integer.cpp diff --git a/cpp/ppc-crypto-core/tests/TestOre4String.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4String.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestOre4String.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestOre4String.cpp diff --git a/cpp/ppc-crypto-core/tests/TestSymCrypto.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/TestSymCrypto.cpp similarity index 100% rename from cpp/ppc-crypto-core/tests/TestSymCrypto.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/TestSymCrypto.cpp diff --git a/cpp/ppc-gateway/test/unittests/main.cpp b/cpp/wedpr-crypto/ppc-crypto-core/tests/main.cpp similarity index 100% rename from cpp/ppc-gateway/test/unittests/main.cpp rename to cpp/wedpr-crypto/ppc-crypto-core/tests/main.cpp diff --git a/cpp/ppc-crypto/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt diff --git a/cpp/ppc-crypto/demo/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto/demo/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto/demo/CMakeLists.txt diff --git a/cpp/ppc-crypto/demo/crypto_bench.cpp b/cpp/wedpr-crypto/ppc-crypto/demo/crypto_bench.cpp similarity index 100% rename from cpp/ppc-crypto/demo/crypto_bench.cpp rename to cpp/wedpr-crypto/ppc-crypto/demo/crypto_bench.cpp diff --git a/cpp/ppc-crypto/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto/src/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt diff --git a/cpp/ppc-crypto/src/Common.h b/cpp/wedpr-crypto/ppc-crypto/src/Common.h similarity index 100% rename from cpp/ppc-crypto/src/Common.h rename to cpp/wedpr-crypto/ppc-crypto/src/Common.h diff --git a/cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h diff --git a/cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/ECDHCryptoImpl.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/ECDHCryptoImpl.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/ECDHCryptoImpl.h diff --git a/cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/EccCryptoFactoryImpl.h diff --git a/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp b/cpp/wedpr-crypto/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp similarity index 100% rename from cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/Ed25519EccCrypto.cpp diff --git a/cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/Ed25519EccCrypto.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/Ed25519EccCrypto.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/Ed25519EccCrypto.h diff --git a/cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/IppECDHCryptoImpl.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/IppECDHCryptoImpl.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/IppECDHCryptoImpl.h diff --git a/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp b/cpp/wedpr-crypto/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp similarity index 100% rename from cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/OpenSSLEccCrypto.cpp diff --git a/cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/OpenSSLEccCrypto.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/OpenSSLEccCrypto.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/OpenSSLEccCrypto.h diff --git a/cpp/ppc-crypto/src/ecc/core/EcPoint.h b/cpp/wedpr-crypto/ppc-crypto/src/ecc/core/EcPoint.h similarity index 100% rename from cpp/ppc-crypto/src/ecc/core/EcPoint.h rename to cpp/wedpr-crypto/ppc-crypto/src/ecc/core/EcPoint.h diff --git a/cpp/ppc-crypto/src/oprf/EcdhOprf.cpp b/cpp/wedpr-crypto/ppc-crypto/src/oprf/EcdhOprf.cpp similarity index 100% rename from cpp/ppc-crypto/src/oprf/EcdhOprf.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/oprf/EcdhOprf.cpp diff --git a/cpp/ppc-crypto/src/oprf/EcdhOprf.h b/cpp/wedpr-crypto/ppc-crypto/src/oprf/EcdhOprf.h similarity index 100% rename from cpp/ppc-crypto/src/oprf/EcdhOprf.h rename to cpp/wedpr-crypto/ppc-crypto/src/oprf/EcdhOprf.h diff --git a/cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp b/cpp/wedpr-crypto/ppc-crypto/src/oprf/RA2018Oprf.cpp similarity index 100% rename from cpp/ppc-crypto/src/oprf/RA2018Oprf.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/oprf/RA2018Oprf.cpp diff --git a/cpp/ppc-crypto/src/oprf/RA2018Oprf.h b/cpp/wedpr-crypto/ppc-crypto/src/oprf/RA2018Oprf.h similarity index 100% rename from cpp/ppc-crypto/src/oprf/RA2018Oprf.h rename to cpp/wedpr-crypto/ppc-crypto/src/oprf/RA2018Oprf.h diff --git a/cpp/ppc-crypto/src/prng/AESPRNG.cpp b/cpp/wedpr-crypto/ppc-crypto/src/prng/AESPRNG.cpp similarity index 100% rename from cpp/ppc-crypto/src/prng/AESPRNG.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/prng/AESPRNG.cpp diff --git a/cpp/ppc-crypto/src/prng/AESPRNG.h b/cpp/wedpr-crypto/ppc-crypto/src/prng/AESPRNG.h similarity index 100% rename from cpp/ppc-crypto/src/prng/AESPRNG.h rename to cpp/wedpr-crypto/ppc-crypto/src/prng/AESPRNG.h diff --git a/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp b/cpp/wedpr-crypto/ppc-crypto/src/prng/BLAKE2bPRNG.cpp similarity index 100% rename from cpp/ppc-crypto/src/prng/BLAKE2bPRNG.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/prng/BLAKE2bPRNG.cpp diff --git a/cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h b/cpp/wedpr-crypto/ppc-crypto/src/prng/BLAKE2bPRNG.h similarity index 100% rename from cpp/ppc-crypto/src/prng/BLAKE2bPRNG.h rename to cpp/wedpr-crypto/ppc-crypto/src/prng/BLAKE2bPRNG.h diff --git a/cpp/ppc-crypto/src/randomot/SimplestOT.cpp b/cpp/wedpr-crypto/ppc-crypto/src/randomot/SimplestOT.cpp similarity index 100% rename from cpp/ppc-crypto/src/randomot/SimplestOT.cpp rename to cpp/wedpr-crypto/ppc-crypto/src/randomot/SimplestOT.cpp diff --git a/cpp/ppc-crypto/src/randomot/SimplestOT.h b/cpp/wedpr-crypto/ppc-crypto/src/randomot/SimplestOT.h similarity index 100% rename from cpp/ppc-crypto/src/randomot/SimplestOT.h rename to cpp/wedpr-crypto/ppc-crypto/src/randomot/SimplestOT.h diff --git a/cpp/ppc-crypto/tests/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-crypto/tests/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-crypto/tests/CMakeLists.txt diff --git a/cpp/ppc-crypto/tests/TestEccCrypto.cpp b/cpp/wedpr-crypto/ppc-crypto/tests/TestEccCrypto.cpp similarity index 100% rename from cpp/ppc-crypto/tests/TestEccCrypto.cpp rename to cpp/wedpr-crypto/ppc-crypto/tests/TestEccCrypto.cpp diff --git a/cpp/ppc-crypto/tests/TestOprf.cpp b/cpp/wedpr-crypto/ppc-crypto/tests/TestOprf.cpp similarity index 100% rename from cpp/ppc-crypto/tests/TestOprf.cpp rename to cpp/wedpr-crypto/ppc-crypto/tests/TestOprf.cpp diff --git a/cpp/ppc-crypto/tests/TestPRNG.cpp b/cpp/wedpr-crypto/ppc-crypto/tests/TestPRNG.cpp similarity index 100% rename from cpp/ppc-crypto/tests/TestPRNG.cpp rename to cpp/wedpr-crypto/ppc-crypto/tests/TestPRNG.cpp diff --git a/cpp/ppc-crypto/tests/TestRandomot.cpp b/cpp/wedpr-crypto/ppc-crypto/tests/TestRandomot.cpp similarity index 100% rename from cpp/ppc-crypto/tests/TestRandomot.cpp rename to cpp/wedpr-crypto/ppc-crypto/tests/TestRandomot.cpp diff --git a/cpp/ppc-homo/tests/main.cpp b/cpp/wedpr-crypto/ppc-crypto/tests/main.cpp similarity index 100% rename from cpp/ppc-homo/tests/main.cpp rename to cpp/wedpr-crypto/ppc-crypto/tests/main.cpp diff --git a/cpp/ppc-homo/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/CMakeLists.txt diff --git a/cpp/ppc-homo/codec/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/codec/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/codec/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/codec/CMakeLists.txt diff --git a/cpp/ppc-homo/codec/Common.h b/cpp/wedpr-crypto/ppc-homo/codec/Common.h similarity index 100% rename from cpp/ppc-homo/codec/Common.h rename to cpp/wedpr-crypto/ppc-homo/codec/Common.h diff --git a/cpp/ppc-homo/codec/FloatingPointCipher.h b/cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCipher.h similarity index 100% rename from cpp/ppc-homo/codec/FloatingPointCipher.h rename to cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCipher.h diff --git a/cpp/ppc-homo/codec/FloatingPointCodec.cpp b/cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCodec.cpp similarity index 100% rename from cpp/ppc-homo/codec/FloatingPointCodec.cpp rename to cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCodec.cpp diff --git a/cpp/ppc-homo/codec/FloatingPointCodec.h b/cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCodec.h similarity index 100% rename from cpp/ppc-homo/codec/FloatingPointCodec.h rename to cpp/wedpr-crypto/ppc-homo/codec/FloatingPointCodec.h diff --git a/cpp/ppc-homo/codec/SignedNumberCodec.h b/cpp/wedpr-crypto/ppc-homo/codec/SignedNumberCodec.h similarity index 100% rename from cpp/ppc-homo/codec/SignedNumberCodec.h rename to cpp/wedpr-crypto/ppc-homo/codec/SignedNumberCodec.h diff --git a/cpp/ppc-homo/demo/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/demo/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/demo/CMakeLists.txt diff --git a/cpp/ppc-homo/demo/homo_perf_demo.cpp b/cpp/wedpr-crypto/ppc-homo/demo/homo_perf_demo.cpp similarity index 100% rename from cpp/ppc-homo/demo/homo_perf_demo.cpp rename to cpp/wedpr-crypto/ppc-homo/demo/homo_perf_demo.cpp diff --git a/cpp/ppc-homo/fahe/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/fahe/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/fahe/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/fahe/CMakeLists.txt diff --git a/cpp/ppc-homo/fahe/Fahe.cpp b/cpp/wedpr-crypto/ppc-homo/fahe/Fahe.cpp similarity index 100% rename from cpp/ppc-homo/fahe/Fahe.cpp rename to cpp/wedpr-crypto/ppc-homo/fahe/Fahe.cpp diff --git a/cpp/ppc-homo/fahe/Fahe.h b/cpp/wedpr-crypto/ppc-homo/fahe/Fahe.h similarity index 100% rename from cpp/ppc-homo/fahe/Fahe.h rename to cpp/wedpr-crypto/ppc-homo/fahe/Fahe.h diff --git a/cpp/ppc-homo/ihc/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/ihc/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/ihc/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/ihc/CMakeLists.txt diff --git a/cpp/ppc-homo/ihc/FloatingPointIhc.cpp b/cpp/wedpr-crypto/ppc-homo/ihc/FloatingPointIhc.cpp similarity index 100% rename from cpp/ppc-homo/ihc/FloatingPointIhc.cpp rename to cpp/wedpr-crypto/ppc-homo/ihc/FloatingPointIhc.cpp diff --git a/cpp/ppc-homo/ihc/FloatingPointIhc.h b/cpp/wedpr-crypto/ppc-homo/ihc/FloatingPointIhc.h similarity index 100% rename from cpp/ppc-homo/ihc/FloatingPointIhc.h rename to cpp/wedpr-crypto/ppc-homo/ihc/FloatingPointIhc.h diff --git a/cpp/ppc-homo/ihc/IhcImpl.cpp b/cpp/wedpr-crypto/ppc-homo/ihc/IhcImpl.cpp similarity index 100% rename from cpp/ppc-homo/ihc/IhcImpl.cpp rename to cpp/wedpr-crypto/ppc-homo/ihc/IhcImpl.cpp diff --git a/cpp/ppc-homo/ihc/IhcImpl.h b/cpp/wedpr-crypto/ppc-homo/ihc/IhcImpl.h similarity index 100% rename from cpp/ppc-homo/ihc/IhcImpl.h rename to cpp/wedpr-crypto/ppc-homo/ihc/IhcImpl.h diff --git a/cpp/ppc-homo/paillier/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/paillier/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/paillier/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/paillier/CMakeLists.txt diff --git a/cpp/ppc-homo/paillier/Common.h b/cpp/wedpr-crypto/ppc-homo/paillier/Common.h similarity index 100% rename from cpp/ppc-homo/paillier/Common.h rename to cpp/wedpr-crypto/ppc-homo/paillier/Common.h diff --git a/cpp/ppc-homo/paillier/FloatingPointPaillier.cpp b/cpp/wedpr-crypto/ppc-homo/paillier/FloatingPointPaillier.cpp similarity index 100% rename from cpp/ppc-homo/paillier/FloatingPointPaillier.cpp rename to cpp/wedpr-crypto/ppc-homo/paillier/FloatingPointPaillier.cpp diff --git a/cpp/ppc-homo/paillier/FloatingPointPaillier.h b/cpp/wedpr-crypto/ppc-homo/paillier/FloatingPointPaillier.h similarity index 100% rename from cpp/ppc-homo/paillier/FloatingPointPaillier.h rename to cpp/wedpr-crypto/ppc-homo/paillier/FloatingPointPaillier.h diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillier.cpp b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp similarity index 100% rename from cpp/ppc-homo/paillier/OpenSSLPaillier.cpp rename to cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillier.h b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.h similarity index 100% rename from cpp/ppc-homo/paillier/OpenSSLPaillier.h rename to cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.h diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp similarity index 100% rename from cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp rename to cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillierKeyPair.cpp diff --git a/cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillierKeyPair.h similarity index 100% rename from cpp/ppc-homo/paillier/OpenSSLPaillierKeyPair.h rename to cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillierKeyPair.h diff --git a/cpp/ppc-homo/tests/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-homo/tests/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-homo/tests/CMakeLists.txt diff --git a/cpp/ppc-homo/tests/TestFahe.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestFahe.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestFahe.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestFahe.cpp diff --git a/cpp/ppc-homo/tests/TestFloatingPointCodec.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestFloatingPointCodec.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestFloatingPointCodec.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestFloatingPointCodec.cpp diff --git a/cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestFloatingPointPaillier.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestFloatingPointPaillier.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestFloatingPointPaillier.cpp diff --git a/cpp/ppc-homo/tests/TestIhc.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestIhc.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestIhc.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestIhc.cpp diff --git a/cpp/ppc-homo/tests/TestIhcFloating.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestIhcFloating.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestIhcFloating.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestIhcFloating.cpp diff --git a/cpp/ppc-homo/tests/TestPaillier.cpp b/cpp/wedpr-crypto/ppc-homo/tests/TestPaillier.cpp similarity index 100% rename from cpp/ppc-homo/tests/TestPaillier.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/TestPaillier.cpp diff --git a/cpp/ppc-io/tests/main.cpp b/cpp/wedpr-crypto/ppc-homo/tests/main.cpp similarity index 100% rename from cpp/ppc-io/tests/main.cpp rename to cpp/wedpr-crypto/ppc-homo/tests/main.cpp diff --git a/cpp/ppc-udf/CMakeLists.txt b/cpp/wedpr-crypto/ppc-udf/CMakeLists.txt similarity index 100% rename from cpp/ppc-udf/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-udf/CMakeLists.txt diff --git a/cpp/ppc-udf/hive/build.gradle b/cpp/wedpr-crypto/ppc-udf/hive/build.gradle similarity index 100% rename from cpp/ppc-udf/hive/build.gradle rename to cpp/wedpr-crypto/ppc-udf/hive/build.gradle diff --git a/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar b/cpp/wedpr-crypto/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar rename to cpp/wedpr-crypto/ppc-udf/hive/gradle/wrapper/gradle-wrapper.jar diff --git a/cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties b/cpp/wedpr-crypto/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from cpp/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties rename to cpp/wedpr-crypto/ppc-udf/hive/gradle/wrapper/gradle-wrapper.properties diff --git a/cpp/ppc-udf/hive/gradlew b/cpp/wedpr-crypto/ppc-udf/hive/gradlew similarity index 100% rename from cpp/ppc-udf/hive/gradlew rename to cpp/wedpr-crypto/ppc-udf/hive/gradlew diff --git a/cpp/ppc-udf/hive/gradlew.bat b/cpp/wedpr-crypto/ppc-udf/hive/gradlew.bat similarity index 100% rename from cpp/ppc-udf/hive/gradlew.bat rename to cpp/wedpr-crypto/ppc-udf/hive/gradlew.bat diff --git a/cpp/ppc-udf/hive/settings.gradle b/cpp/wedpr-crypto/ppc-udf/hive/settings.gradle similarity index 100% rename from cpp/ppc-udf/hive/settings.gradle rename to cpp/wedpr-crypto/ppc-udf/hive/settings.gradle diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/Config.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/ConfigLoader.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/ConfigProperty.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/demo/GeneratePaillierMain.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/ConfigException.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/DecryptionException.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/EncryptionException.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/exceptions/KeyException.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcDecryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcEncryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ihc/IhcSumUDAF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreDecryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreEncryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberDecryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/ore/OreNumberEncryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierDecryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierEncryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/paillier/PaillierSumUDAF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymDecryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java similarity index 100% rename from cpp/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/impl/symmetric/SymEncryptionUDF.java diff --git a/cpp/ppc-udf/hive/src/main/resources/config.toml b/cpp/wedpr-crypto/ppc-udf/hive/src/main/resources/config.toml similarity index 100% rename from cpp/ppc-udf/hive/src/main/resources/config.toml rename to cpp/wedpr-crypto/ppc-udf/hive/src/main/resources/config.toml diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcEncAndDecUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/IhcSumUDAFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/OreDecUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierDecUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierEncUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/PaillierSumUDAFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymDecUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java b/cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java similarity index 100% rename from cpp/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/java/com/webank/wedpr/hive/udf/impl/SymEncryptionUDFTest.java diff --git a/cpp/ppc-udf/hive/src/test/resources/config.toml b/cpp/wedpr-crypto/ppc-udf/hive/src/test/resources/config.toml similarity index 100% rename from cpp/ppc-udf/hive/src/test/resources/config.toml rename to cpp/wedpr-crypto/ppc-udf/hive/src/test/resources/config.toml diff --git a/cpp/ppc-udf/mysql/ore_dec.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/ore_dec.cpp similarity index 100% rename from cpp/ppc-udf/mysql/ore_dec.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/ore_dec.cpp diff --git a/cpp/ppc-udf/mysql/ore_dec.h b/cpp/wedpr-crypto/ppc-udf/mysql/ore_dec.h similarity index 100% rename from cpp/ppc-udf/mysql/ore_dec.h rename to cpp/wedpr-crypto/ppc-udf/mysql/ore_dec.h diff --git a/cpp/ppc-udf/mysql/ore_enc.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/ore_enc.cpp similarity index 100% rename from cpp/ppc-udf/mysql/ore_enc.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/ore_enc.cpp diff --git a/cpp/ppc-udf/mysql/ore_enc.h b/cpp/wedpr-crypto/ppc-udf/mysql/ore_enc.h similarity index 100% rename from cpp/ppc-udf/mysql/ore_enc.h rename to cpp/wedpr-crypto/ppc-udf/mysql/ore_enc.h diff --git a/cpp/ppc-udf/mysql/paillier_dec.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_dec.cpp similarity index 100% rename from cpp/ppc-udf/mysql/paillier_dec.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_dec.cpp diff --git a/cpp/ppc-udf/mysql/paillier_dec.h b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_dec.h similarity index 100% rename from cpp/ppc-udf/mysql/paillier_dec.h rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_dec.h diff --git a/cpp/ppc-udf/mysql/paillier_enc.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_enc.cpp similarity index 100% rename from cpp/ppc-udf/mysql/paillier_enc.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_enc.cpp diff --git a/cpp/ppc-udf/mysql/paillier_enc.h b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_enc.h similarity index 100% rename from cpp/ppc-udf/mysql/paillier_enc.h rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_enc.h diff --git a/cpp/ppc-udf/mysql/paillier_sum.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_sum.cpp similarity index 100% rename from cpp/ppc-udf/mysql/paillier_sum.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_sum.cpp diff --git a/cpp/ppc-udf/mysql/paillier_sum.h b/cpp/wedpr-crypto/ppc-udf/mysql/paillier_sum.h similarity index 100% rename from cpp/ppc-udf/mysql/paillier_sum.h rename to cpp/wedpr-crypto/ppc-udf/mysql/paillier_sum.h diff --git a/cpp/ppc-udf/mysql/sym_dec.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/sym_dec.cpp similarity index 100% rename from cpp/ppc-udf/mysql/sym_dec.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/sym_dec.cpp diff --git a/cpp/ppc-udf/mysql/sym_dec.h b/cpp/wedpr-crypto/ppc-udf/mysql/sym_dec.h similarity index 100% rename from cpp/ppc-udf/mysql/sym_dec.h rename to cpp/wedpr-crypto/ppc-udf/mysql/sym_dec.h diff --git a/cpp/ppc-udf/mysql/sym_enc.cpp b/cpp/wedpr-crypto/ppc-udf/mysql/sym_enc.cpp similarity index 100% rename from cpp/ppc-udf/mysql/sym_enc.cpp rename to cpp/wedpr-crypto/ppc-udf/mysql/sym_enc.cpp diff --git a/cpp/ppc-udf/mysql/sym_enc.h b/cpp/wedpr-crypto/ppc-udf/mysql/sym_enc.h similarity index 100% rename from cpp/ppc-udf/mysql/sym_enc.h rename to cpp/wedpr-crypto/ppc-udf/mysql/sym_enc.h diff --git a/cpp/ppc-udf/tests/CMakeLists.txt b/cpp/wedpr-crypto/ppc-udf/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-udf/tests/CMakeLists.txt rename to cpp/wedpr-crypto/ppc-udf/tests/CMakeLists.txt diff --git a/cpp/ppc-udf/tests/OreDecUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/OreDecUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/OreDecUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/OreDecUDFTest.cpp diff --git a/cpp/ppc-udf/tests/OreEncUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/OreEncUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/OreEncUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/OreEncUDFTest.cpp diff --git a/cpp/ppc-udf/tests/PaillierDecUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/PaillierDecUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/PaillierDecUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/PaillierDecUDFTest.cpp diff --git a/cpp/ppc-udf/tests/PaillierEncUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/PaillierEncUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/PaillierEncUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/PaillierEncUDFTest.cpp diff --git a/cpp/ppc-udf/tests/PaillierSumUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/PaillierSumUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/PaillierSumUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/PaillierSumUDFTest.cpp diff --git a/cpp/ppc-udf/tests/SymDecUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/SymDecUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/SymDecUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/SymDecUDFTest.cpp diff --git a/cpp/ppc-udf/tests/SymEncUDFTest.cpp b/cpp/wedpr-crypto/ppc-udf/tests/SymEncUDFTest.cpp similarity index 100% rename from cpp/ppc-udf/tests/SymEncUDFTest.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/SymEncUDFTest.cpp diff --git a/cpp/ppc-mpc/tests/main.cpp b/cpp/wedpr-crypto/ppc-udf/tests/main.cpp similarity index 100% rename from cpp/ppc-mpc/tests/main.cpp rename to cpp/wedpr-crypto/ppc-udf/tests/main.cpp diff --git a/cpp/wedpr-component-sdk/CMakeLists.txt b/cpp/wedpr-crypto/sdk/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-sdk/CMakeLists.txt rename to cpp/wedpr-crypto/sdk/CMakeLists.txt diff --git a/cpp/wedpr-component-sdk/bindings/java/build.gradle b/cpp/wedpr-crypto/sdk/bindings/java/build.gradle similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/build.gradle rename to cpp/wedpr-crypto/sdk/bindings/java/build.gradle diff --git a/cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar b/cpp/wedpr-crypto/sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar rename to cpp/wedpr-crypto/sdk/bindings/java/gradle/wrapper/gradle-wrapper.jar diff --git a/cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties b/cpp/wedpr-crypto/sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties rename to cpp/wedpr-crypto/sdk/bindings/java/gradle/wrapper/gradle-wrapper.properties diff --git a/cpp/wedpr-component-sdk/bindings/java/gradlew b/cpp/wedpr-crypto/sdk/bindings/java/gradlew similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/gradlew rename to cpp/wedpr-crypto/sdk/bindings/java/gradlew diff --git a/cpp/wedpr-component-sdk/bindings/java/gradlew.bat b/cpp/wedpr-crypto/sdk/bindings/java/gradlew.bat similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/gradlew.bat rename to cpp/wedpr-crypto/sdk/bindings/java/gradlew.bat diff --git a/cpp/wedpr-component-sdk/bindings/java/settings.gradle b/cpp/wedpr-crypto/sdk/bindings/java/settings.gradle similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/settings.gradle rename to cpp/wedpr-crypto/sdk/bindings/java/settings.gradle diff --git a/cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java b/cpp/wedpr-crypto/sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/BenchmarkMain.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java b/cpp/wedpr-crypto/sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/jmh/java/com/webank/wedpr/sdk/jni/jmh/PaillierBenchmark.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/CMakeLists.txt b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/CMakeLists.txt rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/CMakeLists.txt diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/Common.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/Common.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/Common.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/Common.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/Common.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/JNIException.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/JNIException.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/JNIException.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/JNIException.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/JNIException.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_FastOre.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_crypto_SymmetricEncryption.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingIhc.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeFloatingPointPaillier.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativeIhc.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.cpp diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h b/cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/c/jni/com_webank_wedpr_sdk_jni_homo_NativePaillier.h diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/demo/GenerateKeys.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/FloatingPointNumber.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodec.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecException.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/codec/NumberCodecImpl.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniException.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/common/Utilities.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/FastOre.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/crypto/SymmetricEncryption.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingIhc.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativeFloatingPointPaillier.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java b/cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/java/com/webank/wedpr/sdk/jni/homo/NativePaillier.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list b/cpp/wedpr-crypto/sdk/bindings/java/src/main/resources/META-INF/native/win/file.list similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/win/file.list rename to cpp/wedpr-crypto/sdk/bindings/java/src/main/resources/META-INF/native/win/file.list diff --git a/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java b/cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingIhcTest.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java b/cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativeFloatingPointPaillierTest.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java b/cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NativePaillierTest.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java b/cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/NumberCodecTest.java diff --git a/cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java b/cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java similarity index 100% rename from cpp/wedpr-component-sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java rename to cpp/wedpr-crypto/sdk/bindings/java/src/test/java/com/webank/wedpr/sdk/jni/test/SymmetricEncryptionTest.java diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/CMakeLists.txt b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/CMakeLists.txt rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/CMakeLists.txt diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/fast_ore.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/fast_ore.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/fast_ore.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/fast_ore.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/fast_ore.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_ihc.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_ihc.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_ihc.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_ihc.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_paillier.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_paillier.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/floating_point_paillier.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/floating_point_paillier.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_ihc.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_ihc.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_ihc.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_ihc.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_ihc.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_paillier.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_paillier.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_paillier.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/homo_paillier.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/homo_paillier.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/symmetric_encryption.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/symmetric_encryption.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/symmetric_encryption.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/symmetric_encryption.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/error.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/error.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/error.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/error.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/error.h diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.cpp b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/utilities.cpp similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.cpp rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/utilities.cpp diff --git a/cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.h b/cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/utilities.h similarity index 100% rename from cpp/wedpr-component-sdk/ppc-crypto-c-sdk/utils/utilities.h rename to cpp/wedpr-crypto/sdk/ppc-crypto-c-sdk/utils/utilities.h diff --git a/cpp/wedpr-component-sdk/tests/CMakeLists.txt b/cpp/wedpr-crypto/sdk/tests/CMakeLists.txt similarity index 100% rename from cpp/wedpr-component-sdk/tests/CMakeLists.txt rename to cpp/wedpr-crypto/sdk/tests/CMakeLists.txt diff --git a/cpp/wedpr-component-sdk/tests/TestFastOre.cpp b/cpp/wedpr-crypto/sdk/tests/TestFastOre.cpp similarity index 100% rename from cpp/wedpr-component-sdk/tests/TestFastOre.cpp rename to cpp/wedpr-crypto/sdk/tests/TestFastOre.cpp diff --git a/cpp/wedpr-component-sdk/tests/TestHomoPaillier.cpp b/cpp/wedpr-crypto/sdk/tests/TestHomoPaillier.cpp similarity index 100% rename from cpp/wedpr-component-sdk/tests/TestHomoPaillier.cpp rename to cpp/wedpr-crypto/sdk/tests/TestHomoPaillier.cpp diff --git a/cpp/ppc-pir/tests/main.cpp b/cpp/wedpr-crypto/sdk/tests/main.cpp similarity index 100% rename from cpp/ppc-pir/tests/main.cpp rename to cpp/wedpr-crypto/sdk/tests/main.cpp diff --git a/cpp/libhelper/CMakeLists.txt b/cpp/wedpr-helper/libhelper/CMakeLists.txt similarity index 100% rename from cpp/libhelper/CMakeLists.txt rename to cpp/wedpr-helper/libhelper/CMakeLists.txt diff --git a/cpp/libhelper/CommandHelper.cpp b/cpp/wedpr-helper/libhelper/CommandHelper.cpp similarity index 100% rename from cpp/libhelper/CommandHelper.cpp rename to cpp/wedpr-helper/libhelper/CommandHelper.cpp diff --git a/cpp/libhelper/CommandHelper.h b/cpp/wedpr-helper/libhelper/CommandHelper.h similarity index 100% rename from cpp/libhelper/CommandHelper.h rename to cpp/wedpr-helper/libhelper/CommandHelper.h diff --git a/cpp/libhelper/ExitHandler.h b/cpp/wedpr-helper/libhelper/ExitHandler.h similarity index 100% rename from cpp/libhelper/ExitHandler.h rename to cpp/wedpr-helper/libhelper/ExitHandler.h diff --git a/cpp/ppc-tools/CMakeLists.txt b/cpp/wedpr-helper/ppc-tools/CMakeLists.txt similarity index 100% rename from cpp/ppc-tools/CMakeLists.txt rename to cpp/wedpr-helper/ppc-tools/CMakeLists.txt diff --git a/cpp/ppc-tools/src/codec/CodecUtility.h b/cpp/wedpr-helper/ppc-tools/src/codec/CodecUtility.h similarity index 100% rename from cpp/ppc-tools/src/codec/CodecUtility.h rename to cpp/wedpr-helper/ppc-tools/src/codec/CodecUtility.h diff --git a/cpp/ppc-tools/src/common/ConcurrentPool.h b/cpp/wedpr-helper/ppc-tools/src/common/ConcurrentPool.h similarity index 100% rename from cpp/ppc-tools/src/common/ConcurrentPool.h rename to cpp/wedpr-helper/ppc-tools/src/common/ConcurrentPool.h diff --git a/cpp/ppc-tools/src/common/MemInfo.h b/cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h similarity index 100% rename from cpp/ppc-tools/src/common/MemInfo.h rename to cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h diff --git a/cpp/ppc-tools/src/common/Progress.h b/cpp/wedpr-helper/ppc-tools/src/common/Progress.h similarity index 100% rename from cpp/ppc-tools/src/common/Progress.h rename to cpp/wedpr-helper/ppc-tools/src/common/Progress.h diff --git a/cpp/ppc-tools/src/common/TransTools.h b/cpp/wedpr-helper/ppc-tools/src/common/TransTools.h similarity index 100% rename from cpp/ppc-tools/src/common/TransTools.h rename to cpp/wedpr-helper/ppc-tools/src/common/TransTools.h diff --git a/cpp/ppc-tools/src/config/Common.h b/cpp/wedpr-helper/ppc-tools/src/config/Common.h similarity index 100% rename from cpp/ppc-tools/src/config/Common.h rename to cpp/wedpr-helper/ppc-tools/src/config/Common.h diff --git a/cpp/ppc-tools/src/config/NetworkConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h similarity index 100% rename from cpp/ppc-tools/src/config/NetworkConfig.h rename to cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h diff --git a/cpp/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp similarity index 100% rename from cpp/ppc-tools/src/config/PPCConfig.cpp rename to cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp diff --git a/cpp/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h similarity index 100% rename from cpp/ppc-tools/src/config/PPCConfig.h rename to cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h diff --git a/cpp/ppc-tools/src/config/ParamChecker.h b/cpp/wedpr-helper/ppc-tools/src/config/ParamChecker.h similarity index 100% rename from cpp/ppc-tools/src/config/ParamChecker.h rename to cpp/wedpr-helper/ppc-tools/src/config/ParamChecker.h diff --git a/cpp/ppc-tools/src/cuckoo/Common.h b/cpp/wedpr-helper/ppc-tools/src/cuckoo/Common.h similarity index 100% rename from cpp/ppc-tools/src/cuckoo/Common.h rename to cpp/wedpr-helper/ppc-tools/src/cuckoo/Common.h diff --git a/cpp/ppc-tools/src/cuckoo/Cuckoofilter.h b/cpp/wedpr-helper/ppc-tools/src/cuckoo/Cuckoofilter.h similarity index 100% rename from cpp/ppc-tools/src/cuckoo/Cuckoofilter.h rename to cpp/wedpr-helper/ppc-tools/src/cuckoo/Cuckoofilter.h diff --git a/cpp/ppc-tools/src/cuckoo/HashTable.cpp b/cpp/wedpr-helper/ppc-tools/src/cuckoo/HashTable.cpp similarity index 100% rename from cpp/ppc-tools/src/cuckoo/HashTable.cpp rename to cpp/wedpr-helper/ppc-tools/src/cuckoo/HashTable.cpp diff --git a/cpp/ppc-tools/src/cuckoo/HashTable.h b/cpp/wedpr-helper/ppc-tools/src/cuckoo/HashTable.h similarity index 100% rename from cpp/ppc-tools/src/cuckoo/HashTable.h rename to cpp/wedpr-helper/ppc-tools/src/cuckoo/HashTable.h diff --git a/cpp/ppc-tools/tests/CMakeLists.txt b/cpp/wedpr-helper/ppc-tools/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-tools/tests/CMakeLists.txt rename to cpp/wedpr-helper/ppc-tools/tests/CMakeLists.txt diff --git a/cpp/ppc-tools/tests/TestCuckoofilter.cpp b/cpp/wedpr-helper/ppc-tools/tests/TestCuckoofilter.cpp similarity index 100% rename from cpp/ppc-tools/tests/TestCuckoofilter.cpp rename to cpp/wedpr-helper/ppc-tools/tests/TestCuckoofilter.cpp diff --git a/cpp/ppc-tools/tests/TestTransTools.cpp b/cpp/wedpr-helper/ppc-tools/tests/TestTransTools.cpp similarity index 100% rename from cpp/ppc-tools/tests/TestTransTools.cpp rename to cpp/wedpr-helper/ppc-tools/tests/TestTransTools.cpp diff --git a/cpp/ppc-psi/tests/main.cpp b/cpp/wedpr-helper/ppc-tools/tests/main.cpp similarity index 100% rename from cpp/ppc-psi/tests/main.cpp rename to cpp/wedpr-helper/ppc-tools/tests/main.cpp diff --git a/cpp/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h similarity index 100% rename from cpp/ppc-utilities/Utilities.h rename to cpp/wedpr-helper/ppc-utilities/Utilities.h diff --git a/cpp/ppc-io/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/CMakeLists.txt similarity index 100% rename from cpp/ppc-io/CMakeLists.txt rename to cpp/wedpr-storage/ppc-io/CMakeLists.txt diff --git a/cpp/ppc-io/demo/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-io/demo/CMakeLists.txt rename to cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt diff --git a/cpp/ppc-io/demo/io_demo.cpp b/cpp/wedpr-storage/ppc-io/demo/io_demo.cpp similarity index 100% rename from cpp/ppc-io/demo/io_demo.cpp rename to cpp/wedpr-storage/ppc-io/demo/io_demo.cpp diff --git a/cpp/ppc-io/src/BaseFileLineReader.cpp b/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.cpp similarity index 100% rename from cpp/ppc-io/src/BaseFileLineReader.cpp rename to cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.cpp diff --git a/cpp/ppc-io/src/BaseFileLineReader.h b/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h similarity index 100% rename from cpp/ppc-io/src/BaseFileLineReader.h rename to cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h diff --git a/cpp/ppc-io/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-io/src/CMakeLists.txt rename to cpp/wedpr-storage/ppc-io/src/CMakeLists.txt diff --git a/cpp/ppc-io/src/Common.h b/cpp/wedpr-storage/ppc-io/src/Common.h similarity index 100% rename from cpp/ppc-io/src/Common.h rename to cpp/wedpr-storage/ppc-io/src/Common.h diff --git a/cpp/ppc-io/src/DataResourceLoaderImpl.cpp b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp similarity index 100% rename from cpp/ppc-io/src/DataResourceLoaderImpl.cpp rename to cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp diff --git a/cpp/ppc-io/src/DataResourceLoaderImpl.h b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h similarity index 100% rename from cpp/ppc-io/src/DataResourceLoaderImpl.h rename to cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h diff --git a/cpp/ppc-io/src/FileLineReader.cpp b/cpp/wedpr-storage/ppc-io/src/FileLineReader.cpp similarity index 100% rename from cpp/ppc-io/src/FileLineReader.cpp rename to cpp/wedpr-storage/ppc-io/src/FileLineReader.cpp diff --git a/cpp/ppc-io/src/FileLineReader.h b/cpp/wedpr-storage/ppc-io/src/FileLineReader.h similarity index 100% rename from cpp/ppc-io/src/FileLineReader.h rename to cpp/wedpr-storage/ppc-io/src/FileLineReader.h diff --git a/cpp/ppc-io/src/FileLineWriter.cpp b/cpp/wedpr-storage/ppc-io/src/FileLineWriter.cpp similarity index 100% rename from cpp/ppc-io/src/FileLineWriter.cpp rename to cpp/wedpr-storage/ppc-io/src/FileLineWriter.cpp diff --git a/cpp/ppc-io/src/FileLineWriter.h b/cpp/wedpr-storage/ppc-io/src/FileLineWriter.h similarity index 100% rename from cpp/ppc-io/src/FileLineWriter.h rename to cpp/wedpr-storage/ppc-io/src/FileLineWriter.h diff --git a/cpp/ppc-io/src/HDFSReader.cpp b/cpp/wedpr-storage/ppc-io/src/HDFSReader.cpp similarity index 100% rename from cpp/ppc-io/src/HDFSReader.cpp rename to cpp/wedpr-storage/ppc-io/src/HDFSReader.cpp diff --git a/cpp/ppc-io/src/HDFSReader.h b/cpp/wedpr-storage/ppc-io/src/HDFSReader.h similarity index 100% rename from cpp/ppc-io/src/HDFSReader.h rename to cpp/wedpr-storage/ppc-io/src/HDFSReader.h diff --git a/cpp/ppc-io/src/HDFSWriter.cpp b/cpp/wedpr-storage/ppc-io/src/HDFSWriter.cpp similarity index 100% rename from cpp/ppc-io/src/HDFSWriter.cpp rename to cpp/wedpr-storage/ppc-io/src/HDFSWriter.cpp diff --git a/cpp/ppc-io/src/HDFSWriter.h b/cpp/wedpr-storage/ppc-io/src/HDFSWriter.h similarity index 100% rename from cpp/ppc-io/src/HDFSWriter.h rename to cpp/wedpr-storage/ppc-io/src/HDFSWriter.h diff --git a/cpp/ppc-io/src/SQLResultReader.h b/cpp/wedpr-storage/ppc-io/src/SQLResultReader.h similarity index 100% rename from cpp/ppc-io/src/SQLResultReader.h rename to cpp/wedpr-storage/ppc-io/src/SQLResultReader.h diff --git a/cpp/ppc-io/src/parser/BaseBufferParser.h b/cpp/wedpr-storage/ppc-io/src/parser/BaseBufferParser.h similarity index 100% rename from cpp/ppc-io/src/parser/BaseBufferParser.h rename to cpp/wedpr-storage/ppc-io/src/parser/BaseBufferParser.h diff --git a/cpp/ppc-io/src/parser/BufferParser.h b/cpp/wedpr-storage/ppc-io/src/parser/BufferParser.h similarity index 100% rename from cpp/ppc-io/src/parser/BufferParser.h rename to cpp/wedpr-storage/ppc-io/src/parser/BufferParser.h diff --git a/cpp/ppc-io/src/parser/MmapParser.h b/cpp/wedpr-storage/ppc-io/src/parser/MmapParser.h similarity index 100% rename from cpp/ppc-io/src/parser/MmapParser.h rename to cpp/wedpr-storage/ppc-io/src/parser/MmapParser.h diff --git a/cpp/ppc-io/tests/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-io/tests/CMakeLists.txt rename to cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt diff --git a/cpp/ppc-io/tests/TestLineReader.cpp b/cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp similarity index 100% rename from cpp/ppc-io/tests/TestLineReader.cpp rename to cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp diff --git a/cpp/ppc-io/tests/TestLineWriter.cpp b/cpp/wedpr-storage/ppc-io/tests/TestLineWriter.cpp similarity index 100% rename from cpp/ppc-io/tests/TestLineWriter.cpp rename to cpp/wedpr-storage/ppc-io/tests/TestLineWriter.cpp diff --git a/cpp/ppc-io/tests/data/linux_file.txt b/cpp/wedpr-storage/ppc-io/tests/data/linux_file.txt similarity index 100% rename from cpp/ppc-io/tests/data/linux_file.txt rename to cpp/wedpr-storage/ppc-io/tests/data/linux_file.txt diff --git a/cpp/ppc-io/tests/data/testData b/cpp/wedpr-storage/ppc-io/tests/data/testData similarity index 100% rename from cpp/ppc-io/tests/data/testData rename to cpp/wedpr-storage/ppc-io/tests/data/testData diff --git a/cpp/ppc-io/tests/data/windows_file.txt b/cpp/wedpr-storage/ppc-io/tests/data/windows_file.txt similarity index 100% rename from cpp/ppc-io/tests/data/windows_file.txt rename to cpp/wedpr-storage/ppc-io/tests/data/windows_file.txt diff --git a/cpp/ppc-rpc/tests/main.cpp b/cpp/wedpr-storage/ppc-io/tests/main.cpp similarity index 100% rename from cpp/ppc-rpc/tests/main.cpp rename to cpp/wedpr-storage/ppc-io/tests/main.cpp diff --git a/cpp/ppc-storage/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/CMakeLists.txt similarity index 100% rename from cpp/ppc-storage/CMakeLists.txt rename to cpp/wedpr-storage/ppc-storage/CMakeLists.txt diff --git a/cpp/ppc-storage/demo/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-storage/demo/CMakeLists.txt rename to cpp/wedpr-storage/ppc-storage/demo/CMakeLists.txt diff --git a/cpp/ppc-storage/demo/storage_demo.cpp b/cpp/wedpr-storage/ppc-storage/demo/storage_demo.cpp similarity index 100% rename from cpp/ppc-storage/demo/storage_demo.cpp rename to cpp/wedpr-storage/ppc-storage/demo/storage_demo.cpp diff --git a/cpp/ppc-storage/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-storage/src/CMakeLists.txt rename to cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt diff --git a/cpp/ppc-storage/src/CacheStorageFactoryImpl.h b/cpp/wedpr-storage/ppc-storage/src/CacheStorageFactoryImpl.h similarity index 100% rename from cpp/ppc-storage/src/CacheStorageFactoryImpl.h rename to cpp/wedpr-storage/ppc-storage/src/CacheStorageFactoryImpl.h diff --git a/cpp/ppc-storage/src/Common.h b/cpp/wedpr-storage/ppc-storage/src/Common.h similarity index 100% rename from cpp/ppc-storage/src/Common.h rename to cpp/wedpr-storage/ppc-storage/src/Common.h diff --git a/cpp/ppc-storage/src/FileStorageFactoryImpl.h b/cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h similarity index 100% rename from cpp/ppc-storage/src/FileStorageFactoryImpl.h rename to cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h diff --git a/cpp/ppc-storage/src/SQLStorageFactoryImpl.h b/cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h similarity index 100% rename from cpp/ppc-storage/src/SQLStorageFactoryImpl.h rename to cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h diff --git a/cpp/ppc-storage/src/hdfs/Common.h b/cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h similarity index 100% rename from cpp/ppc-storage/src/hdfs/Common.h rename to cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h diff --git a/cpp/ppc-storage/src/hdfs/HDFSStorage.cpp b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp similarity index 100% rename from cpp/ppc-storage/src/hdfs/HDFSStorage.cpp rename to cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp diff --git a/cpp/ppc-storage/src/hdfs/HDFSStorage.h b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.h similarity index 100% rename from cpp/ppc-storage/src/hdfs/HDFSStorage.h rename to cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.h diff --git a/cpp/ppc-storage/src/mysql/Common.h b/cpp/wedpr-storage/ppc-storage/src/mysql/Common.h similarity index 100% rename from cpp/ppc-storage/src/mysql/Common.h rename to cpp/wedpr-storage/ppc-storage/src/mysql/Common.h diff --git a/cpp/ppc-storage/src/mysql/MySQLStorage.cpp b/cpp/wedpr-storage/ppc-storage/src/mysql/MySQLStorage.cpp similarity index 100% rename from cpp/ppc-storage/src/mysql/MySQLStorage.cpp rename to cpp/wedpr-storage/ppc-storage/src/mysql/MySQLStorage.cpp diff --git a/cpp/ppc-storage/src/mysql/MySQLStorage.h b/cpp/wedpr-storage/ppc-storage/src/mysql/MySQLStorage.h similarity index 100% rename from cpp/ppc-storage/src/mysql/MySQLStorage.h rename to cpp/wedpr-storage/ppc-storage/src/mysql/MySQLStorage.h diff --git a/cpp/ppc-storage/src/redis/RedisStorage.cpp b/cpp/wedpr-storage/ppc-storage/src/redis/RedisStorage.cpp similarity index 100% rename from cpp/ppc-storage/src/redis/RedisStorage.cpp rename to cpp/wedpr-storage/ppc-storage/src/redis/RedisStorage.cpp diff --git a/cpp/ppc-storage/src/redis/RedisStorage.h b/cpp/wedpr-storage/ppc-storage/src/redis/RedisStorage.h similarity index 100% rename from cpp/ppc-storage/src/redis/RedisStorage.h rename to cpp/wedpr-storage/ppc-storage/src/redis/RedisStorage.h diff --git a/cpp/ppc-front/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/CMakeLists.txt similarity index 100% rename from cpp/ppc-front/CMakeLists.txt rename to cpp/wedpr-transport/ppc-front/CMakeLists.txt diff --git a/cpp/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp similarity index 100% rename from cpp/ppc-front/ppc-front/CallbackManager.cpp rename to cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp diff --git a/cpp/ppc-front/ppc-front/CallbackManager.h b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h similarity index 100% rename from cpp/ppc-front/ppc-front/CallbackManager.h rename to cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h diff --git a/cpp/ppc-front/ppc-front/Common.h b/cpp/wedpr-transport/ppc-front/ppc-front/Common.h similarity index 100% rename from cpp/ppc-front/ppc-front/Common.h rename to cpp/wedpr-transport/ppc-front/ppc-front/Common.h diff --git a/cpp/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp similarity index 100% rename from cpp/ppc-front/ppc-front/Front.cpp rename to cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp diff --git a/cpp/ppc-front/ppc-front/Front.h b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h similarity index 100% rename from cpp/ppc-front/ppc-front/Front.h rename to cpp/wedpr-transport/ppc-front/ppc-front/Front.h diff --git a/cpp/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp similarity index 100% rename from cpp/ppc-front/ppc-front/FrontImpl.cpp rename to cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp diff --git a/cpp/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h similarity index 100% rename from cpp/ppc-front/ppc-front/FrontImpl.h rename to cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h diff --git a/cpp/ppc-front/ppc-front/PPCChannel.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.cpp similarity index 100% rename from cpp/ppc-front/ppc-front/PPCChannel.cpp rename to cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.cpp diff --git a/cpp/ppc-front/ppc-front/PPCChannel.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h similarity index 100% rename from cpp/ppc-front/ppc-front/PPCChannel.h rename to cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h diff --git a/cpp/ppc-front/ppc-front/PPCChannelManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.cpp similarity index 100% rename from cpp/ppc-front/ppc-front/PPCChannelManager.cpp rename to cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.cpp diff --git a/cpp/ppc-front/ppc-front/PPCChannelManager.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h similarity index 100% rename from cpp/ppc-front/ppc-front/PPCChannelManager.h rename to cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h diff --git a/cpp/ppc-front/test/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/test/CMakeLists.txt similarity index 100% rename from cpp/ppc-front/test/CMakeLists.txt rename to cpp/wedpr-transport/ppc-front/test/CMakeLists.txt diff --git a/cpp/ppc-front/test/unittests/PPCChannelTest.cpp b/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp similarity index 100% rename from cpp/ppc-front/test/unittests/PPCChannelTest.cpp rename to cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp diff --git a/cpp/ppc-tools/tests/main.cpp b/cpp/wedpr-transport/ppc-front/test/unittests/main.cpp similarity index 100% rename from cpp/ppc-tools/tests/main.cpp rename to cpp/wedpr-transport/ppc-front/test/unittests/main.cpp diff --git a/cpp/ppc-gateway/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt similarity index 100% rename from cpp/ppc-gateway/CMakeLists.txt rename to cpp/wedpr-transport/ppc-gateway/CMakeLists.txt diff --git a/cpp/ppc-gateway/ppc-gateway/Common.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/Common.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayConfigContext.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayConfigLoader.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayFactory.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/GatewayFactory.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/GatewayFactory.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/Service.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/Service.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.h diff --git a/cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h similarity index 100% rename from cpp/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h rename to cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableInterface.h diff --git a/cpp/ppc-gateway/test/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt similarity index 100% rename from cpp/ppc-gateway/test/CMakeLists.txt rename to cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt diff --git a/cpp/ppc-gateway/test/data/config0.ini b/cpp/wedpr-transport/ppc-gateway/test/data/config0.ini similarity index 100% rename from cpp/ppc-gateway/test/data/config0.ini rename to cpp/wedpr-transport/ppc-gateway/test/data/config0.ini diff --git a/cpp/ppc-gateway/test/demo/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/test/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-gateway/test/demo/CMakeLists.txt rename to cpp/wedpr-transport/ppc-gateway/test/demo/CMakeLists.txt diff --git a/cpp/ppc-gateway/test/demo/gateway_demo.cpp b/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp similarity index 100% rename from cpp/ppc-gateway/test/demo/gateway_demo.cpp rename to cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp diff --git a/cpp/ppc-gateway/test/unittests/GatewayTest.cpp b/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp similarity index 100% rename from cpp/ppc-gateway/test/unittests/GatewayTest.cpp rename to cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp diff --git a/cpp/ppc-gateway/test/unittests/MockCache.h b/cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h similarity index 100% rename from cpp/ppc-gateway/test/unittests/MockCache.h rename to cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h diff --git a/cpp/ppc-udf/tests/main.cpp b/cpp/wedpr-transport/ppc-gateway/test/unittests/main.cpp similarity index 100% rename from cpp/ppc-udf/tests/main.cpp rename to cpp/wedpr-transport/ppc-gateway/test/unittests/main.cpp diff --git a/cpp/ppc-http/CMakeLists.txt b/cpp/wedpr-transport/ppc-http/CMakeLists.txt similarity index 100% rename from cpp/ppc-http/CMakeLists.txt rename to cpp/wedpr-transport/ppc-http/CMakeLists.txt diff --git a/cpp/ppc-http/demo/CMakeLists.txt b/cpp/wedpr-transport/ppc-http/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-http/demo/CMakeLists.txt rename to cpp/wedpr-transport/ppc-http/demo/CMakeLists.txt diff --git a/cpp/ppc-http/demo/config.ini b/cpp/wedpr-transport/ppc-http/demo/config.ini similarity index 100% rename from cpp/ppc-http/demo/config.ini rename to cpp/wedpr-transport/ppc-http/demo/config.ini diff --git a/cpp/ppc-http/demo/http_client_demo.cpp b/cpp/wedpr-transport/ppc-http/demo/http_client_demo.cpp similarity index 100% rename from cpp/ppc-http/demo/http_client_demo.cpp rename to cpp/wedpr-transport/ppc-http/demo/http_client_demo.cpp diff --git a/cpp/ppc-http/demo/http_demo.cpp b/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp similarity index 100% rename from cpp/ppc-http/demo/http_demo.cpp rename to cpp/wedpr-transport/ppc-http/demo/http_demo.cpp diff --git a/cpp/ppc-http/src/CMakeLists.txt b/cpp/wedpr-transport/ppc-http/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-http/src/CMakeLists.txt rename to cpp/wedpr-transport/ppc-http/src/CMakeLists.txt diff --git a/cpp/ppc-http/src/Http.cpp b/cpp/wedpr-transport/ppc-http/src/Http.cpp similarity index 100% rename from cpp/ppc-http/src/Http.cpp rename to cpp/wedpr-transport/ppc-http/src/Http.cpp diff --git a/cpp/ppc-http/src/Http.h b/cpp/wedpr-transport/ppc-http/src/Http.h similarity index 100% rename from cpp/ppc-http/src/Http.h rename to cpp/wedpr-transport/ppc-http/src/Http.h diff --git a/cpp/ppc-http/src/HttpClient.cpp b/cpp/wedpr-transport/ppc-http/src/HttpClient.cpp similarity index 100% rename from cpp/ppc-http/src/HttpClient.cpp rename to cpp/wedpr-transport/ppc-http/src/HttpClient.cpp diff --git a/cpp/ppc-http/src/HttpClient.h b/cpp/wedpr-transport/ppc-http/src/HttpClient.h similarity index 100% rename from cpp/ppc-http/src/HttpClient.h rename to cpp/wedpr-transport/ppc-http/src/HttpClient.h diff --git a/cpp/ppc-http/src/HttpFactory.cpp b/cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp similarity index 100% rename from cpp/ppc-http/src/HttpFactory.cpp rename to cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp diff --git a/cpp/ppc-http/src/HttpFactory.h b/cpp/wedpr-transport/ppc-http/src/HttpFactory.h similarity index 100% rename from cpp/ppc-http/src/HttpFactory.h rename to cpp/wedpr-transport/ppc-http/src/HttpFactory.h diff --git a/cpp/ppc-rpc/CMakeLists.txt b/cpp/wedpr-transport/ppc-rpc/CMakeLists.txt similarity index 100% rename from cpp/ppc-rpc/CMakeLists.txt rename to cpp/wedpr-transport/ppc-rpc/CMakeLists.txt diff --git a/cpp/ppc-rpc/demo/CMakeLists.txt b/cpp/wedpr-transport/ppc-rpc/demo/CMakeLists.txt similarity index 100% rename from cpp/ppc-rpc/demo/CMakeLists.txt rename to cpp/wedpr-transport/ppc-rpc/demo/CMakeLists.txt diff --git a/cpp/ppc-rpc/demo/config.ini b/cpp/wedpr-transport/ppc-rpc/demo/config.ini similarity index 100% rename from cpp/ppc-rpc/demo/config.ini rename to cpp/wedpr-transport/ppc-rpc/demo/config.ini diff --git a/cpp/ppc-rpc/demo/rpc_demo.cpp b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp similarity index 100% rename from cpp/ppc-rpc/demo/rpc_demo.cpp rename to cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp diff --git a/cpp/ppc-rpc/src/CMakeLists.txt b/cpp/wedpr-transport/ppc-rpc/src/CMakeLists.txt similarity index 100% rename from cpp/ppc-rpc/src/CMakeLists.txt rename to cpp/wedpr-transport/ppc-rpc/src/CMakeLists.txt diff --git a/cpp/ppc-rpc/src/JsonRequest.h b/cpp/wedpr-transport/ppc-rpc/src/JsonRequest.h similarity index 100% rename from cpp/ppc-rpc/src/JsonRequest.h rename to cpp/wedpr-transport/ppc-rpc/src/JsonRequest.h diff --git a/cpp/ppc-rpc/src/JsonResponse.h b/cpp/wedpr-transport/ppc-rpc/src/JsonResponse.h similarity index 100% rename from cpp/ppc-rpc/src/JsonResponse.h rename to cpp/wedpr-transport/ppc-rpc/src/JsonResponse.h diff --git a/cpp/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp similarity index 100% rename from cpp/ppc-rpc/src/Rpc.cpp rename to cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp diff --git a/cpp/ppc-rpc/src/Rpc.h b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h similarity index 100% rename from cpp/ppc-rpc/src/Rpc.h rename to cpp/wedpr-transport/ppc-rpc/src/Rpc.h diff --git a/cpp/ppc-rpc/src/RpcFactory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp similarity index 100% rename from cpp/ppc-rpc/src/RpcFactory.cpp rename to cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp diff --git a/cpp/ppc-rpc/src/RpcFactory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h similarity index 100% rename from cpp/ppc-rpc/src/RpcFactory.h rename to cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h diff --git a/cpp/ppc-rpc/src/RpcMemory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp similarity index 100% rename from cpp/ppc-rpc/src/RpcMemory.cpp rename to cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp diff --git a/cpp/ppc-rpc/src/RpcMemory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h similarity index 100% rename from cpp/ppc-rpc/src/RpcMemory.h rename to cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h diff --git a/cpp/ppc-rpc/tests/CMakeLists.txt b/cpp/wedpr-transport/ppc-rpc/tests/CMakeLists.txt similarity index 100% rename from cpp/ppc-rpc/tests/CMakeLists.txt rename to cpp/wedpr-transport/ppc-rpc/tests/CMakeLists.txt diff --git a/cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp b/cpp/wedpr-transport/ppc-rpc/tests/TestRpcRequestResponse.cpp similarity index 100% rename from cpp/ppc-rpc/tests/TestRpcRequestResponse.cpp rename to cpp/wedpr-transport/ppc-rpc/tests/TestRpcRequestResponse.cpp diff --git a/cpp/wedpr-component-sdk/tests/main.cpp b/cpp/wedpr-transport/ppc-rpc/tests/main.cpp similarity index 100% rename from cpp/wedpr-component-sdk/tests/main.cpp rename to cpp/wedpr-transport/ppc-rpc/tests/main.cpp From 3c14d5b52481b306d4866820e06878b2804b3334 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 4 Sep 2024 15:32:49 +0800 Subject: [PATCH 021/120] add sdk to wedpr-transport (#17) * remove ppc-tars-service * add sdk to wedpr-transport --- cpp/cmake/TargetSettings.cmake | 3 + cpp/libinitializer/Initializer.cpp | 2 - cpp/ppc-framework/front/FrontConfig.h | 63 ++++---- cpp/ppc-framework/front/IFrontBuilder.h | 41 +++++ cpp/ppc-framework/protocol/EndPoint.h | 59 +++++++ cpp/ppc-framework/protocol/INodeInfo.h | 7 +- cpp/ppc-framework/protocol/RouteType.h | 4 +- cpp/ppc-main/gateway/GatewayServiceApp.cpp | 133 ---------------- cpp/ppc-main/gateway/GatewayServiceApp.h | 54 ------- cpp/ppc-main/pro-node/ProNodeServiceApp.cpp | 147 ------------------ cpp/ppc-main/pro-node/ProNodeServiceApp.h | 52 ------- .../FrontService/FrontServiceServer.cpp | 55 ------- .../FrontService/FrontServiceServer.h | 53 ------- .../GatewayService/GatewayInitializer.cpp | 56 ------- .../GatewayService/GatewayInitializer.h | 75 --------- .../GatewayService/GatewayServiceServer.cpp | 141 ----------------- .../GatewayService/GatewayServiceServer.h | 74 --------- .../ppc-tools/src/config/PPCConfig.cpp | 81 ---------- .../ppc-tools/src/config/PPCConfig.h | 47 ------ cpp/wedpr-protocol/grpc/client/FrontClient.h | 4 +- .../grpc/client/GatewayClient.h | 4 +- cpp/wedpr-protocol/grpc/client/GrpcClient.h | 5 +- cpp/wedpr-protocol/grpc/server/GrpcServer.cpp | 6 +- cpp/wedpr-protocol/grpc/server/GrpcServer.h | 6 +- cpp/wedpr-protocol/protobuf/NodeInfoImpl.h | 13 +- .../ppc-front/ppc-front/FrontFactory.cpp | 37 +++++ .../ppc-front/ppc-front/FrontFactory.h | 42 +++++ .../ppc-gateway/CMakeLists.txt | 6 +- cpp/wedpr-transport/sdk/CMakeLists.txt | 8 + cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 43 +++++ cpp/wedpr-transport/sdk/ProTransportImpl.h | 48 ++++++ cpp/wedpr-transport/sdk/Transport.h | 38 +++++ cpp/wedpr-transport/sdk/TransportBuilder.h | 57 +++++++ cpp/wedpr-transport/sdk/TransportImpl.h | 46 ++++++ 34 files changed, 477 insertions(+), 1033 deletions(-) create mode 100644 cpp/ppc-framework/front/IFrontBuilder.h create mode 100644 cpp/ppc-framework/protocol/EndPoint.h delete mode 100644 cpp/ppc-main/gateway/GatewayServiceApp.cpp delete mode 100644 cpp/ppc-main/gateway/GatewayServiceApp.h delete mode 100644 cpp/ppc-main/pro-node/ProNodeServiceApp.cpp delete mode 100644 cpp/ppc-main/pro-node/ProNodeServiceApp.h delete mode 100644 cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp delete mode 100644 cpp/ppc-tars-service/FrontService/FrontServiceServer.h delete mode 100644 cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp delete mode 100644 cpp/ppc-tars-service/GatewayService/GatewayInitializer.h delete mode 100644 cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp delete mode 100644 cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h create mode 100644 cpp/wedpr-transport/sdk/CMakeLists.txt create mode 100644 cpp/wedpr-transport/sdk/ProTransportImpl.cpp create mode 100644 cpp/wedpr-transport/sdk/ProTransportImpl.h create mode 100644 cpp/wedpr-transport/sdk/Transport.h create mode 100644 cpp/wedpr-transport/sdk/TransportBuilder.h create mode 100644 cpp/wedpr-transport/sdk/TransportImpl.h diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index ffe57295..46735456 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -133,6 +133,9 @@ endif() set(BOOST_UNIT_TEST Boost::unit_test_framework) +# ==== the transport sdk=== +set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk) + # ==== the swig wrapper ===== set(WEDPR_PYTHON_TOOLKIT "wedpr_python_toolkit") set(WEDPR_PYTHON_TOOLKIT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TOOLKIT}) \ No newline at end of file diff --git a/cpp/libinitializer/Initializer.cpp b/cpp/libinitializer/Initializer.cpp index 99465a3a..cb5d68f7 100644 --- a/cpp/libinitializer/Initializer.cpp +++ b/cpp/libinitializer/Initializer.cpp @@ -89,8 +89,6 @@ void Initializer::init(ppc::protocol::NodeArch _arch) } init->fetchAgencyListPeriodically(); }); - // load the tars config - m_config->loadTarsConfig(m_configPath); } INIT_LOG(INFO) << LOG_DESC("init the frontService success") << LOG_KV("agency", m_config->agencyID()) << LOG_KV("arch", _arch); diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h index e9775c79..6cee462a 100644 --- a/cpp/ppc-framework/front/FrontConfig.h +++ b/cpp/ppc-framework/front/FrontConfig.h @@ -19,65 +19,58 @@ */ #pragma once +#include "ppc-framework/protocol/EndPoint.h" #include #include #include namespace ppc::front { -/** - * @brief the gateway endpoint - * - */ -class GatewayEndPoint -{ -public: - GatewayEndPoint() = default; - GatewayEndPoint(std::string const& host, uint16_t port) : m_host(std::move(host)), m_port(port) - {} - virtual ~GatewayEndPoint() = default; - - virtual std::string const& host() const { return m_host; } - uint16_t port() const { return m_port; } - - void setHost(std::string host) { m_host = std::move(host); } - void setPort(uint16_t port) { m_port = port; } - -private: - // the host - std::string m_host; - // the port - uint16_t m_port; -}; - // Note: swig explosed interface class FrontConfig { public: using Ptr = std::shared_ptr; - FrontConfig(int threadPoolSize, std::string agencyID) - : m_threadPoolSize(threadPoolSize), m_agencyID(std::move(agencyID)) + FrontConfig(int threadPoolSize, std::string nodeID) + : m_threadPoolSize(threadPoolSize), m_nodeID(std::move(nodeID)) {} virtual ~FrontConfig() = default; virtual int threadPoolSize() const { return m_threadPoolSize; } - virtual std::string const agencyID() const { return m_agencyID; } - virtual std::vector const& gatewayInfo() const { return m_gatewayInfo; } - virtual void setGatewayInfo(std::vector gatewayInfo) + virtual std::string const& nodeID() const { return m_nodeID; } + virtual std::vector const& gatewayInfo() const + { + return m_gatewayInfo; + } + virtual void setGatewayInfo(std::vector gatewayInfo) { m_gatewayInfo = std::move(gatewayInfo); } - virtual void appendGatewayInfo(GatewayEndPoint&& endpoint) + virtual void appendGatewayInfo(ppc::protocol::EndPoint&& endpoint) { // TODO:check the endpoint m_gatewayInfo.push_back(endpoint); } + ppc::protocol::EndPoint const& selfEndPoint() const { return m_selfEndPoint; } + void setSelfEndPoint(ppc::protocol::EndPoint const& endPoint) { m_selfEndPoint = endPoint; } + + // TODO here + std::string gatewayEndPoints() { return ""; } + + std::string const& loadBalancePolicy() const { return m_loadBanlancePolicy; } + void setLoadBalancePolicy(std::string const& loadBanlancePolicy) + { + m_loadBanlancePolicy = loadBanlancePolicy; + } + private: + std::string m_loadBanlancePolicy = "round_robin"; + ppc::protocol::EndPoint m_selfEndPoint; int m_threadPoolSize; - std::string m_agencyID; - std::vector m_gatewayInfo; + std::string m_nodeID; + std::vector m_gatewayInfo; }; class FrontConfigBuilder @@ -87,9 +80,9 @@ class FrontConfigBuilder FrontConfigBuilder() = default; virtual ~FrontConfigBuilder() = default; - FrontConfig::Ptr build(int threadPoolSize, std::string agencyID) + FrontConfig::Ptr build(int threadPoolSize, std::string nodeID) { - return std::make_shared(threadPoolSize, agencyID); + return std::make_shared(threadPoolSize, nodeID); } }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/front/IFrontBuilder.h b/cpp/ppc-framework/front/IFrontBuilder.h new file mode 100644 index 00000000..50423645 --- /dev/null +++ b/cpp/ppc-framework/front/IFrontBuilder.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file IFront.h + * @author: yujiechen + * @date 2024-08-22 + */ +#pragma once +#include "IFront.h" + +namespace ppc::front +{ +class IFrontBuilder +{ +public: + using Ptr = std::shared_ptr; + IFrontBuilder() = default; + virtual ~IFrontBuilder() = default; + + /** + * @brief create the Front using specified config + * + * @param config the config used to build the Front + * @return IFront::Ptr he created Front + */ + virtual IFront::Ptr build(ppc::front::FrontConfig::Ptr config) const = 0; + virtual IFrontClient::Ptr buildClient(std::string endPoint) const = 0; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/EndPoint.h b/cpp/ppc-framework/protocol/EndPoint.h new file mode 100644 index 00000000..099b7de5 --- /dev/null +++ b/cpp/ppc-framework/protocol/EndPoint.h @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EndPoint.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#pragma once +#include +#include +#include + +namespace ppc::protocol +{ +/** + * @brief the endpoint + * + */ +class EndPoint +{ +public: + EndPoint() = default; + EndPoint(std::string const& host, uint16_t port) : m_host(std::move(host)), m_port(port) {} + virtual ~EndPoint() = default; + + virtual std::string const& host() const { return m_host; } + uint16_t port() const { return m_port; } + + void setHost(std::string host) { m_host = std::move(host); } + void setPort(uint16_t port) { m_port = port; } + + std::string entryPoint() const { return m_host + ":" + std::to_string(m_port); } + + std::string listenEndPoint() const { return m_listenIp + ":" + std::to_string(m_port); } + + std::string const& listenIp() const { return m_listenIp; } + +private: + // the listenIp + std::string m_listenIp = "0.0.0.0"; + // the host + std::string m_host; + // the port + uint16_t m_port; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 76f460b2..680f0df9 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -62,14 +62,11 @@ class INodeInfoFactory { public: using Ptr = std::shared_ptr; - INodeInfoFactory(bcos::bytes nodeID) : m_nodeID(std::move(nodeID)) {} + INodeInfoFactory() = default; virtual ~INodeInfoFactory() = default; virtual INodeInfo::Ptr build() = 0; - virtual INodeInfo::Ptr build(std::string const& endPoint) = 0; - -protected: - bcos::bytes m_nodeID; + virtual INodeInfo::Ptr build(bcos::bytesConstRef nodeID, std::string const& endPoint) = 0; }; inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo) diff --git a/cpp/ppc-framework/protocol/RouteType.h b/cpp/ppc-framework/protocol/RouteType.h index 1e3a715e..cb7feb96 100644 --- a/cpp/ppc-framework/protocol/RouteType.h +++ b/cpp/ppc-framework/protocol/RouteType.h @@ -19,8 +19,8 @@ */ #pragma once -#include #include +#include namespace ppc::protocol { @@ -51,4 +51,4 @@ inline std::ostream& operator<<(std::ostream& _out, RouteType const& _type) } return _out; } -} // namespace ppc::front \ No newline at end of file +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-main/gateway/GatewayServiceApp.cpp b/cpp/ppc-main/gateway/GatewayServiceApp.cpp deleted file mode 100644 index 70dbf15f..00000000 --- a/cpp/ppc-main/gateway/GatewayServiceApp.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayServiceApp.cpp - * @author: shawnhe - * @date 2022-10-21 - */ -#include "GatewayServiceApp.h" -#include "ppc-storage/src/CacheStorageFactoryImpl.h" -#include "ppc-tars-protocol/Common.h" -#include "ppc-tars-service/GatewayService/GatewayInitializer.h" -#include "ppc-tars-service/GatewayService/GatewayServiceServer.h" -#include "ppc-tools/src/config/NetworkConfig.h" - -using namespace ppctars; -using namespace ppc; -using namespace ppc::tools; -using namespace ppc::gateway; -using namespace ppc::storage; -using namespace ppc::protocol; - -using namespace bcos; -using namespace bcos::boostssl; -using namespace bcos::boostssl::ws; - -void GatewayServiceApp::initialize() -{ - try - { - m_configPath = tars::ServerConfig::BasePath + "/config.ini"; - addConfig("config.ini"); - - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService") << LOG_KV("config", m_configPath); - initService(m_configPath); - - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService success") << LOG_KV("config", m_configPath); - - GatewayServiceParam param; - param.gateway = m_gatewayInitializer->gateway(); - param.ppcMsgFactory = m_gatewayInitializer->messageFactory(); - addServantWithParams( - getProxyDesc(GATEWAY_SERVANT_NAME), param); - } - catch (std::exception const& e) - { - std::cout << "init GatewayServiceApp failed, error: " << boost::diagnostic_information(e) - << std::endl; - throw e; - } -} - -void GatewayServiceApp::initService(std::string const& _configPath) -{ - boost::property_tree::ptree pt; - boost::property_tree::read_ini(_configPath, pt); - auto ppcConfig = std::make_shared(); - // init the log - m_logInitializer = std::make_shared(); - m_logInitializer->initLog(pt); - - // load the gatewayConfig - auto config = std::make_shared(); - config->loadGatewayConfig( - ppc::protocol::NodeArch::PRO, tars::ServerConfig::BasePath.c_str(), pt); - // add the config - auto const& networkConfig = config->gatewayConfig().networkConfig; - - if (!networkConfig.disableSsl) - { - GATEWAYAPP_LOG(INFO) << LOG_DESC("addConfig") << LOG_KV("enableSM", networkConfig.enableSM); - if (!networkConfig.enableSM) - { - addConfig(std::string(NetworkConfig::CA_CERT_NAME)); - addConfig(std::string(NetworkConfig::SSL_CERT_NAME)); - addConfig(std::string(NetworkConfig::SSL_KEY_NAME)); - } - else - { - addConfig(std::string(NetworkConfig::SM_CA_CERT_NAME)); - addConfig(std::string(NetworkConfig::SM_SSL_CERT_NAME)); - addConfig(std::string(NetworkConfig::SM_SSL_KEY_NAME)); - addConfig(std::string(NetworkConfig::SM_SSL_EN_KEY_NAME)); - addConfig(std::string(NetworkConfig::SM_SSL_EN_CERT_NAME)); - } - GATEWAYAPP_LOG(INFO) << LOG_DESC("addConfig success") - << LOG_KV("enableSM", networkConfig.enableSM); - } - - // redis cache - storage::CacheStorage::Ptr cache; - if (!config->gatewayConfig().disableCache) - { - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: buildRedisStorage") - << config->gatewayConfig().cacheStorageConfig.desc(); - auto cacheStorageFactory = std::make_shared(); - cache = cacheStorageFactory->createCacheStorage(config->gatewayConfig().cacheStorageConfig); - try - { - cache->exists("check_cache"); - } - catch (std::exception& e) - { - BOOST_THROW_EXCEPTION( - InvalidConfig() << errinfo_comment( - "init cache error:" + std::string(boost::diagnostic_information(e)))); - } - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: buildRedisStorage success"); - } - - // message factory - auto messageFactory = std::make_shared(); - // global thread pool - auto threadPoolSize = config->gatewayConfig().networkConfig.threadPoolSize; - auto threadPool = std::make_shared(GATEWAY_THREAD_POOL_MODULE, threadPoolSize); - - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: build and start gateway"); - m_gatewayInitializer = std::make_shared( - NodeArch::PRO, config, cache, messageFactory, threadPool); - m_gatewayInitializer->start(); - GATEWAYAPP_LOG(INFO) << LOG_DESC("initService: build and start gateway success"); -} \ No newline at end of file diff --git a/cpp/ppc-main/gateway/GatewayServiceApp.h b/cpp/ppc-main/gateway/GatewayServiceApp.h deleted file mode 100644 index 7c462f2e..00000000 --- a/cpp/ppc-main/gateway/GatewayServiceApp.h +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayServiceApp.h - * @author: shawnhe - * @date 2022-10-21 - */ -#pragma once -#include "ppc-framework/protocol/Protocol.h" -#include "ppc-tars-protocol/Common.h" -#include "ppc-tars-service/GatewayService/GatewayInitializer.h" -#include -#include - -#define GATEWAYAPP_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GatewayServiceApp]" - -namespace ppctars -{ -class GatewayServiceApp : public tars::Application -{ -public: - GatewayServiceApp() {} - ~GatewayServiceApp() override{}; - - void destroyApp() override - { - if (m_gatewayInitializer) - { - m_gatewayInitializer->stop(); - } - } - void initialize() override; - -protected: - virtual void initService(std::string const& _configPath); - -private: - std::string m_configPath; - bcos::BoostLogInitializer::Ptr m_logInitializer; - ppctars::GatewayInitializer::Ptr m_gatewayInitializer; -}; -} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp b/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp deleted file mode 100644 index 4199d163..00000000 --- a/cpp/ppc-main/pro-node/ProNodeServiceApp.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file ProNodeServiceApp.cpp - * @author: yujiechen - * @date 2022-11-25 - */ -#include "ProNodeServiceApp.h" -#include "GatewayService.h" -#include "ppc-rpc/src/RpcFactory.h" -#include "ppc-rpc/src/RpcMemory.h" -#include "ppc-tars-protocol/Common.h" -#include "ppc-tars-protocol/client/GatewayServiceClient.h" -#include "ppc-tars-service/FrontService/FrontServiceServer.h" - -using namespace ppctars; -using namespace ppc::initializer; -using namespace ppc::rpc; -using namespace ppc::tools; -using namespace ppc::protocol; - -void ProNodeServiceApp::destroyApp() -{ - NodeService_LOG(INFO) << LOG_DESC("destroyApp"); - if (m_rpc) - { - m_rpc->stop(); - } - if (m_nodeInitializer) - { - m_nodeInitializer->stop(); - } - NodeService_LOG(INFO) << LOG_DESC("destroyApp success"); -} - -void ProNodeServiceApp::initialize() -{ - try - { - m_configPath = tars::ServerConfig::BasePath + "/config.ini"; - addConfig("config.ini"); - - NodeService_LOG(INFO) << LOG_DESC("initService") << LOG_KV("config", m_configPath); - initService(m_configPath); - NodeService_LOG(INFO) << LOG_DESC("initService success") << LOG_KV("config", m_configPath); - } - catch (std::exception const& e) - { - // since the tars will not print the detailed information when exceptioned, we print here - std::cout << "init NodeServiceApp failed, error: " << boost::diagnostic_information(e) - << std::endl; - throw e; - } -} - -void ProNodeServiceApp::initService(std::string const& _configPath) -{ - // init the log - boost::property_tree::ptree pt; - boost::property_tree::read_ini(_configPath, pt); - auto ppcConfig = std::make_shared(); - ppcConfig->loadSelfTarsEndpoint(pt); - // init the log - m_logInitializer = std::make_shared(); - m_logInitializer->initLog(pt); - - // init the nodeInitializer - addConfig(std::string(NODE_PEM_NAME)); - NodeService_LOG(INFO) << LOG_DESC("initService: init the node"); - m_nodeInitializer = std::make_shared(_configPath); - auto privateKeyPath = tars::ServerConfig::BasePath + "/" + std::string(NODE_PEM_NAME); - NodeService_LOG(INFO) << LOG_DESC("generate the node private key path: ") << privateKeyPath; - m_nodeInitializer->config()->setPrivateKeyPath(privateKeyPath); - - m_nodeInitializer->init(ppc::protocol::NodeArch::PRO); - NodeService_LOG(INFO) << LOG_DESC("initService: init the node success"); - - auto config = m_nodeInitializer->config(); - - // init the gateway - auto gatewayServiceName = config->gatewayServiceName(); - auto endPoints = config->getServiceEndPointsByName(gatewayServiceName); - NodeService_LOG(INFO) << LOG_DESC("initGateway") << LOG_KV("serviceName", gatewayServiceName) - << LOG_KV("endPointSize", endPoints.size()); - auto gatewayPrx = - createServantProxy(true, gatewayServiceName, toTarsEndPoints(endPoints)); - auto gateway = std::make_shared( - gatewayServiceName, gatewayPrx, config->holdingMessageMinutes()); - NodeService_LOG(INFO) << LOG_DESC("initGateway success"); - - // addservant for the front-service - auto frontInitializer = m_nodeInitializer->frontInitializer(); - FrontServiceParam frontParam{frontInitializer}; - addServantWithParams( - getProxyDesc(ppc::protocol::FRONT_SERVANT_NAME), frontParam); - // get the endpoint for front-service-object - auto ret = - getEndPointDescByAdapter(this, ppc::protocol::FRONT_SERVANT_NAME, ppcConfig->endpoint()); - if (!ret.first) - { - throw std::runtime_error("get load endpoint for front-service-object information failed"); - } - - std::string selfEndPoint = ret.second; - NodeService_LOG(INFO) << LOG_DESC("get local-endpoint for the front-service-object success") - << LOG_KV("endPoint", selfEndPoint); - frontInitializer->front()->setSelfEndPoint(selfEndPoint); - // set the gateway into front - m_nodeInitializer->frontInitializer()->front()->setGatewayInterface(gateway); - - // init the rpc - NodeService_LOG(INFO) << LOG_DESC("init the rpc"); - // load the rpc config - // not specify the certPath in air-mode - config->loadRpcConfig(nullptr, pt); - // init RpcStatusInterface - RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(gateway); - m_nodeInitializer->frontInitializer()->setRpcStatus(rpcStatusInterface); - auto rpcFactory = std::make_shared(config->agencyID()); - m_rpc = rpcFactory->buildRpc(config); - m_rpc->setRpcStorage(rpcStatusInterface); - m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); - - m_nodeInitializer->registerRpcHandler(m_rpc); - NodeService_LOG(INFO) << LOG_DESC("init the rpc success"); - - // start the node - NodeService_LOG(INFO) << LOG_DESC("start the node"); - m_nodeInitializer->start(); - NodeService_LOG(INFO) << LOG_DESC("start the node success"); - - NodeService_LOG(INFO) << LOG_DESC("start the rpc"); - m_rpc->start(); - NodeService_LOG(INFO) << LOG_DESC("start the rpc success"); -} \ No newline at end of file diff --git a/cpp/ppc-main/pro-node/ProNodeServiceApp.h b/cpp/ppc-main/pro-node/ProNodeServiceApp.h deleted file mode 100644 index 13883c62..00000000 --- a/cpp/ppc-main/pro-node/ProNodeServiceApp.h +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file ProNodeServiceApp.h - * @author: yujiechen - * @date 2022-11-25 - */ -#pragma once -#include "libinitializer/Initializer.h" -#include -#include - -#define NodeService_LOG(LEVEL) BCOS_LOG(LEVEL) << "[NodeServiceApp]" - -namespace ppc::rpc -{ -class Rpc; -} -namespace ppctars -{ -class ProNodeServiceApp : public tars::Application -{ -public: - ProNodeServiceApp() {} - ~ProNodeServiceApp() override{}; - - void destroyApp() override; - void initialize() override; - -protected: - virtual void initService(std::string const& _configPath); - -private: - std::string m_configPath; - bcos::BoostLogInitializer::Ptr m_logInitializer; - ppc::initializer::Initializer::Ptr m_nodeInitializer; - // TODO: rpc support pro-mode - std::shared_ptr m_rpc; -}; -} // namespace ppctars \ No newline at end of file diff --git a/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp b/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp deleted file mode 100644 index 7ca25f65..00000000 --- a/cpp/ppc-tars-service/FrontService/FrontServiceServer.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontServiceServer.cpp - * @author: shawnhe - * @date 2022-10-20 - */ - -#include "FrontServiceServer.h" -#include "ppc-framework/Common.h" -#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" - -#define FRONTSERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("FrontServer") - -using namespace ppctars; - -ppctars::Error FrontServiceServer::onReceiveMessage( - const vector& _message, tars::TarsCurrentPtr _current) -{ - FRONTSERVER_LOG(TRACE) << LOG_DESC("receive message from gateway"); - - auto startT = bcos::utcSteadyTime(); - _current->setResponse(false); - - auto ppcMessage = m_frontInitializer->messageFactory()->buildPPCMessage( - bcos::bytesConstRef((bcos::byte const*)_message.data(), _message.size())); - if (!ppcMessage) - { - auto error = ppctars::Error(); - error.errorCode = ppc::protocol::PPCRetCode::DECODE_PPC_MESSAGE_ERROR; - error.errorMessage = "decode ppc message error"; - return error; - } - auto msgSize = _message.size(); - m_frontInitializer->front()->onReceiveMessage( - ppcMessage, [_current, startT, msgSize](bcos::Error::Ptr error) { - BCOS_LOG(TRACE) << LOG_DESC("onReceiveMessage") << LOG_KV("msgSize", msgSize) - << LOG_KV("timecost", (bcos::utcSteadyTime() - startT)); - async_response_onReceiveMessage(_current, toTarsError(error)); - }); - - return {}; -} \ No newline at end of file diff --git a/cpp/ppc-tars-service/FrontService/FrontServiceServer.h b/cpp/ppc-tars-service/FrontService/FrontServiceServer.h deleted file mode 100644 index c0065f21..00000000 --- a/cpp/ppc-tars-service/FrontService/FrontServiceServer.h +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontServiceServer.h - * @author: shawnhe - * @date 2022-10-20 - */ - -#pragma once - -#include "FrontService.h" -#include "libinitializer/FrontInitializer.h" -#include "ppc-protocol/src/PPCMessage.h" -#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" - -namespace ppctars -{ -struct FrontServiceParam -{ - ppc::initializer::FrontInitializer::Ptr frontInitializer; -}; - -class FrontServiceServer : public FrontService -{ -public: - FrontServiceServer(FrontServiceParam const& _param) - : m_frontInitializer(_param.frontInitializer) - {} - - ~FrontServiceServer() override {} - void initialize() override {} - void destroy() override {} - -public: - ppctars::Error onReceiveMessage( - const vector& _message, tars::TarsCurrentPtr _current) override; - -private: - ppc::initializer::FrontInitializer::Ptr m_frontInitializer; -}; -} // namespace ppctars diff --git a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp deleted file mode 100644 index 75ed9814..00000000 --- a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayInitializer.cpp - * @author: shawnhe - * @date 2022-10-21 - */ - -#include "GatewayInitializer.h" - -using namespace ppctars; -using namespace ppc; -using namespace ppc::front; -using namespace ppc::gateway; -using namespace ppc::storage; -using namespace ppc::protocol; - -using namespace bcos; -using namespace bcos::boostssl; -using namespace bcos::boostssl::ws; - -void GatewayInitializer::start() -{ - if (m_gateway) - { - m_gateway->start(); - } -} - -void GatewayInitializer::stop() -{ - if (m_gateway) - { - m_gateway->stop(); - } -} - -void GatewayInitializer::init(ppc::protocol::NodeArch _arch) -{ - GATEWAY_LOG(INFO) << LOG_BADGE("init gateway") << LOG_KV("arch", _arch); - m_gateway = m_gatewayFactory->buildGateway( - _arch, m_gatewayConfig, m_cache, m_messageFactory, m_threadPool); - GATEWAY_LOG(INFO) << LOG_BADGE("init gateway success"); -} \ No newline at end of file diff --git a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h b/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h deleted file mode 100644 index 00285359..00000000 --- a/cpp/ppc-tars-service/GatewayService/GatewayInitializer.h +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayInitializer.h - * @author: shawnhe - * @date 2022-10-21 - */ - -#pragma once - -#include -#include - -namespace ppctars -{ -class GatewayInitializer -{ -public: - using Ptr = std::shared_ptr; - - explicit GatewayInitializer(ppc::protocol::NodeArch _arch, ppc::tools::PPCConfig::Ptr _config, - ppc::storage::CacheStorage::Ptr _cache, ppc::front::PPCMessageFactory::Ptr _messageFactory, - std::shared_ptr _threadPool) - : m_gatewayConfig(std::move(_config)), - m_cache(std::move(_cache)), - m_messageFactory(std::move(_messageFactory)), - m_threadPool(std::move(_threadPool)) - - { - init(_arch); - } - - virtual ~GatewayInitializer() { stop(); } - - void start(); - void stop(); - -public: - ppc::storage::CacheStorage::Ptr cache() { return m_cache; } - - ppc::front::PPCMessageFactory::Ptr messageFactory() { return m_messageFactory; } - std::shared_ptr threadPool() { return m_threadPool; } - - ppc::gateway::GatewayInterface::Ptr gateway() { return m_gateway; } - ppc::gateway::GatewayFactory::Ptr gatewayFactory() { return m_gatewayFactory; } - -protected: - void init(ppc::protocol::NodeArch _arch); - -private: - ppc::tools::PPCConfig::Ptr m_gatewayConfig; - ppc::storage::CacheStorage::Ptr m_cache; - - ppc::front::PPCMessageFactory::Ptr m_messageFactory; - - std::shared_ptr m_threadPool; - - ppc::gateway::GatewayInterface::Ptr m_gateway; - ppc::gateway::GatewayFactory::Ptr m_gatewayFactory; - - std::atomic_bool m_running = {false}; -}; -} // namespace ppctars diff --git a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp deleted file mode 100644 index ed7b217b..00000000 --- a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayServiceServer.cpp - * @author: shawnhe - * @date 2022-10-21 - */ -#include "GatewayServiceServer.h" -#include "FrontService.h" -#include "ppc-tars-protocol/client/FrontServiceClient.h" -#include "ppc-tars-protocol/ppc-tars-protocol/Common.h" - -using namespace ppctars; -using namespace ppc::gateway; - -ppctars::Error GatewayServiceServer::asyncSendMessage( - const std::string& _agencyID, const vector& _message, tars::TarsCurrentPtr _current) - -{ - GATEWAYSERVER_LOG(TRACE) << LOG_DESC("receive message from front"); - - _current->setResponse(false); - - auto ppcMessage = m_ppcMsgFactory->buildPPCMessage( - bcos::bytesConstRef((bcos::byte const*)_message.data(), _message.size())); - if (!ppcMessage) - { - auto error = ppctars::Error(); - error.errorCode = ppc::protocol::PPCRetCode::DECODE_PPC_MESSAGE_ERROR; - error.errorMessage = "decode ppc message error"; - async_response_asyncSendMessage(_current, error); - return error; - } - - m_gateway->asyncSendMessage(_agencyID, ppcMessage, [_current](const bcos::Error::Ptr& error) { - async_response_asyncSendMessage(_current, toTarsError(error)); - }); - - return ppctars::Error(); -} - -ppctars::Error GatewayServiceServer::notifyTaskInfo( - const ppctars::TaskInfo& _taskInfo, tars::TarsCurrentPtr _current) -{ - _current->setResponse(true); - - auto error = m_gateway->notifyTaskInfo(toGatewayTaskInfo(_taskInfo)); - - return toTarsError(error); -} - -ppctars::Error GatewayServiceServer::eraseTaskInfo( - const std::string& _taskID, tars::TarsCurrentPtr _current) -{ - _current->setResponse(true); - - auto error = m_gateway->eraseTaskInfo(_taskID); - - return toTarsError(error); -} - -ppctars::Error GatewayServiceServer::registerGateway( - const std::vector& _gatewayList, tars::TarsCurrentPtr _current) -{ - _current->setResponse(true); - - std::vector gatewayList; - for (const auto& tarsGate : _gatewayList) - { - ppc::protocol::GatewayInfo gateway; - gateway.agencyID = tarsGate.agencyID; - gateway.endpoint = tarsGate.endpoint; - gatewayList.push_back(gateway); - } - - auto error = m_gateway->registerGateway(gatewayList); - - return toTarsError(error); -} - -ppctars::Error GatewayServiceServer::asyncRegisterFront( - std::string const& _endPoint, tars::TarsCurrentPtr _current) -{ - _current->setResponse(false); - try - { - // obtain the frontServiceClient specified by _endPoint - auto self = weak_from_this(); - auto prx = createServantProxy(tars::Application::getCommunicator().get(), - _endPoint, [self, _endPoint](const tars::TC_Endpoint& ep) { - auto gatewayServer = self.lock(); - if (!gatewayServer) - { - return; - } - GATEWAYSERVER_LOG(INFO) - << LOG_DESC("UnRegisterFront for disconnect") << LOG_KV("endPoint", _endPoint); - // unregister when disconnected - gatewayServer->m_gateway->unregisterFront(_endPoint); - }); - m_gateway->registerFront(_endPoint, std::make_shared(prx)); - // register success - async_response_asyncRegisterFront( - _current, ppctars::toTarsError(nullptr)); - } - catch (std::exception const& e) - { - GATEWAYSERVER_LOG(WARNING) - << LOG_DESC("asyncRegisterFront exception") << LOG_KV("endPoint", _endPoint) - << LOG_KV("exception", boost::diagnostic_information(e)); - auto bcosError = std::make_shared(-1, - "asyncRegisterFront exception for " + std::string(boost::diagnostic_information(e))); - async_response_asyncRegisterFront(_current, ppctars::toTarsError(bcosError)); - } - return ppctars::Error(); -} - -ppctars::Error GatewayServiceServer::asyncGetAgencyList( - std::vector& _agencyList, tars::TarsCurrentPtr _current) -{ - _current->setResponse(false); - m_gateway->asyncGetAgencyList( - [_current](bcos::Error::Ptr _error, std::vector&& _agencyList) { - // response the agency-list - async_response_asyncGetAgencyList( - _current, ppctars::toTarsError(_error), std::move(_agencyList)); - }); - return ppctars::Error(); -} diff --git a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h b/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h deleted file mode 100644 index ecc1e6a4..00000000 --- a/cpp/ppc-tars-service/GatewayService/GatewayServiceServer.h +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file GatewayServiceServer.h - * @author: shawnhe - * @date 2022-10-21 - */ -#pragma once - -#include "GatewayInitializer.h" -#include "GatewayService.h" -#include "ppc-framework/Common.h" -#include "ppc-framework/gateway/GatewayInterface.h" -#include "ppc-framework/protocol/PPCMessageFace.h" -#include - -#define GATEWAYSERVER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("GatewayServiceServer") - -namespace ppctars -{ -struct GatewayServiceParam -{ - ppc::gateway::GatewayInterface::Ptr gateway; - ppc::front::PPCMessageFaceFactory::Ptr ppcMsgFactory; -}; - -class GatewayServiceServer : public ppctars::GatewayService, - public std::enable_shared_from_this -{ -public: - using Ptr = std::shared_ptr; - - explicit GatewayServiceServer(GatewayServiceParam const& _param) - : m_gateway(_param.gateway), m_ppcMsgFactory(_param.ppcMsgFactory) - {} - void initialize() override {} - void destroy() override {} - -public: - ppctars::Error asyncSendMessage(const std::string& _agencyID, - const vector& _message, tars::TarsCurrentPtr _current) override; - - ppctars::Error notifyTaskInfo( - const ppctars::TaskInfo& _taskInfo, tars::TarsCurrentPtr _current) override; - - ppctars::Error eraseTaskInfo( - const std::string& _taskID, tars::TarsCurrentPtr _current) override; - - ppctars::Error registerGateway(const std::vector& _gatewayList, - tars::TarsCurrentPtr _current) override; - - ppctars::Error asyncRegisterFront( - std::string const& _endPoint, tars::TarsCurrentPtr _current) override; - - ppctars::Error asyncGetAgencyList( - std::vector& _agencyList, tars::TarsCurrentPtr _current) override; - -private: - ppc::gateway::GatewayInterface::Ptr m_gateway; - ppc::front::PPCMessageFaceFactory::Ptr m_ppcMsgFactory; -}; -} // namespace ppctars diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index ace6cee6..f644bd6d 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -527,84 +527,3 @@ void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt) m_mpcConfig.readPerBatchLines = _pt.get("mpc.read_per_batch_lines", 100000); loadHDFSConfig(_pt); } - -//////// the tars config -void PPCConfig::loadTarsConfig(boost::property_tree::ptree const& _pt) -{ - PPCConfig_LOG(INFO) << LOG_DESC("loadTarsConfig"); - // load the service name - std::string gatewayServiceConfigSection = "tars_gateway"; - m_gatewayServiceName = - getServiceName(_pt, gatewayServiceConfigSection + ".name", GATEWAY_SERVANT_NAME); - // load the holding_msg_minutes - m_holdingMessageMinutes = - loadHoldingMessageMinutes(_pt, gatewayServiceConfigSection + ".holding_msg_minutes"); - // config the gateway information - loadServiceTarsConfig(_pt, m_gatewayServiceName, gatewayServiceConfigSection); - - PPCConfig_LOG(INFO) << LOG_DESC("loadTarsConfig success") - << LOG_KV("gatewayService", m_gatewayServiceName) - << LOG_KV("endPoints", m_serviceToEndPoints.at(m_gatewayServiceName).size()) - << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); -} - -void PPCConfig::loadServiceTarsConfig(boost::property_tree::ptree const& _pt, - std::string const& _serviceName, std::string const& _sectionName) -{ - PPCConfig_LOG(INFO) << LOG_DESC("loadServiceTarsConfig for ") << _sectionName - << LOG_KV("service", _serviceName); - if (!_pt.get_child_optional(_sectionName)) - { - BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment("Must config " + _serviceName)); - } - for (auto const& it : _pt.get_child(_sectionName)) - { - if (it.first.find("proxy.") != 0) - { - continue; - } - std::string endpoint = it.second.data(); - PPCConfig_LOG(INFO) << LOG_BADGE("loadServiceTarsConfig") - << LOG_KV("serviceName", _serviceName) << LOG_KV("endpoint", endpoint); - if (!checkEndpoint(endpoint)) - { - BOOST_THROW_EXCEPTION( - InvalidConfig() << errinfo_comment("Invalid endpoint: " + endpoint)); - } - m_serviceToEndPoints[_serviceName].emplace_back(std::move(endpoint)); - } -} - -std::string PPCConfig::getServiceName(boost::property_tree::ptree const& _pt, - std::string const& _configSection, std::string const& _objName) -{ - auto serviceName = _pt.get(_configSection, ""); - checkService(_configSection, serviceName); - return getPrxDesc(serviceName, _objName); -} - -void PPCConfig::checkService(std::string const& _serviceType, std::string const& _serviceName) -{ - if (_serviceName.empty()) - { - BOOST_THROW_EXCEPTION( - InvalidConfig() << errinfo_comment("Must set service name for " + _serviceType + "!")); - } - std::vector serviceNameList; - boost::split(serviceNameList, _serviceName, boost::is_any_of(".")); - std::string errorMsg = - "Must set service name in format of application_name.server_name with only include letters " - "and numbers for " + - _serviceType + ", invalid config now is:" + _serviceName; - if (serviceNameList.size() != 2) - { - BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment(errorMsg)); - } - for (const auto& serviceName : serviceNameList) - { - if (!isalNumStr(serviceName)) - { - BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment(errorMsg)); - } - } -} diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h index 3bd29cfa..76aac8b0 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h @@ -178,13 +178,6 @@ class PPCConfig loadGatewayConfig(_arch, _certPath, iniConfig); } - virtual void loadTarsConfig(std::string const& _configPath) - { - boost::property_tree::ptree iniConfig; - boost::property_tree::read_ini(_configPath, iniConfig); - loadTarsConfig(iniConfig); - } - virtual void loadRpcConfig(const char* _certPath, boost::property_tree::ptree const& _pt) { // rpc default disable-ssl @@ -192,11 +185,6 @@ class PPCConfig m_rpcConfig, _certPath, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true); } - virtual void loadSelfTarsEndpoint(boost::property_tree::ptree const& _pt) - { - m_endpoint = _pt.get("agency.endpoint", ""); - } - virtual void loadGatewayConfig(ppc::protocol::NodeArch _arch, const char* _certPath, boost::property_tree::ptree const& _pt); @@ -208,8 +196,6 @@ class PPCConfig virtual void loadMPCConfig(boost::property_tree::ptree const& _pt); - virtual void loadTarsConfig(boost::property_tree::ptree const& _pt); - NetworkConfig const& rpcConfig() const { return m_rpcConfig; } // the gateway-config GatewayConfig const& gatewayConfig() const { return m_gatewayConfig; } @@ -229,19 +215,6 @@ class PPCConfig uint32_t const& taskTimeoutMinutes() const { return m_taskTimeoutMinutes; } uint32_t const& threadPoolSize() const { return m_threadPoolSize; } - std::string const& gatewayServiceName() const { return m_gatewayServiceName; } - std::vector getServiceEndPointsByName(std::string const& _service) - { - // Note: not ensure thread-safe, since only use when init - std::vector endPoints; - auto it = m_serviceToEndPoints.find(_service); - if (it != m_serviceToEndPoints.end()) - { - return it->second; - } - return endPoints; - } - EcdhPSIParam const& ecdhPSIConfig() const { return m_ecdhPSIConfig; } EcdhPSIParam& mutableEcdhPSIConfig() { return m_ecdhPSIConfig; } @@ -297,22 +270,6 @@ class PPCConfig void initRedisConfigForGateway( ppc::storage::CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt); - - // load the tars-config for the given service, e.g: - /* - [tars_gateway] - name = agencyAGateway - proxy.0 = "192.168.0.1:3000" - proxy.0 = "192.168.0.2:3002" - proxy.0 = "192.168.0.3:3003" - */ - virtual void loadServiceTarsConfig(boost::property_tree::ptree const& _pt, - std::string const& _serviceName, std::string const& _sectionName); - - std::string getServiceName(boost::property_tree::ptree const& _pt, - std::string const& _configSection, std::string const& _objName); - void checkService(std::string const& _serviceType, std::string const& _serviceName); - int64_t getDataBatchSize(std::string const& _section, int64_t _dataBatchSize); int loadHoldingMessageMinutes( @@ -352,10 +309,6 @@ class PPCConfig bcos::bytes m_privateKey; std::string m_privateKeyPath; - // the tars config - std::map> m_serviceToEndPoints; - std::string m_gatewayServiceName; - bool m_disableRA2018 = false; std::string m_endpoint; diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.h b/cpp/wedpr-protocol/grpc/client/FrontClient.h index 4f40b4bb..29e1a21a 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.h +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.h @@ -27,7 +27,9 @@ class FrontClient : public ppc::front::IFrontClient, public GrpcClient { public: using Ptr = std::shared_ptr; - FrontClient(std::shared_ptr channel) : GrpcClient(std::move(channel)) {} + FrontClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) + : GrpcClient(channelConfig, endPoints) + {} ~FrontClient() override = default; void onReceiveMessage( diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index a6635ccc..9a75a52b 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -27,8 +27,8 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient { public: using Ptr = std::shared_ptr; - GatewayClient(std::shared_ptr channel) - : GrpcClient(std::move(channel)), m_stub(ppc::proto::Gateway::NewStub(m_channel)) + GatewayClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) + : GrpcClient(channelConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel)) {} ~GatewayClient() override = default; diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.h b/cpp/wedpr-protocol/grpc/client/GrpcClient.h index 028718a9..86e44fb5 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.h +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.h @@ -28,7 +28,10 @@ class GrpcClient { public: using Ptr = std::shared_ptr; - GrpcClient(std::shared_ptr channel) : m_channel(std::move(channel)) {} + GrpcClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) + : m_channel( + grpc::CreateCustomChannel(endPoints, grpc::InsecureChannelCredentials(), channelConfig)) + {} virtual ~GrpcClient() = default; diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp index 9d612dfa..8ad659b9 100644 --- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp @@ -29,14 +29,14 @@ void GrpcServer::start() if (m_running) { GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer has already been started!") - << LOG_KV("endPoint", m_config.endPoint()); + << LOG_KV("listenEndPoint", m_config.listenEndPoint()); return; } m_running = true; grpc::reflection::InitProtoReflectionServerBuilderPlugin(); grpc::ServerBuilder builder; // without authentication - builder.AddListeningPort(m_config.endPoint(), grpc::InsecureServerCredentials()); + builder.AddListeningPort(m_config.listenEndPoint(), grpc::InsecureServerCredentials()); // register the service for (auto const& service : m_bindingServices) { @@ -44,7 +44,7 @@ void GrpcServer::start() } m_server = std::unique_ptr(builder.BuildAndStart()); GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer start success!") - << LOG_KV("endPoint", m_config.endPoint()); + << LOG_KV("listenEndPoint", m_config.listenEndPoint()); } void GrpcServer::stop() diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.h b/cpp/wedpr-protocol/grpc/server/GrpcServer.h index 869dcc88..f5ac678e 100644 --- a/cpp/wedpr-protocol/grpc/server/GrpcServer.h +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.h @@ -18,6 +18,7 @@ * @date 2024-09-03 */ #pragma once +#include "ppc-framework/protocol/EndPoint.h" #include #include #include @@ -27,10 +28,9 @@ namespace ppc::protocol // refer to: https://grpc.io/docs/languages/cpp/callback/ struct GrpcServerConfig { - std::string listenIp; - int listenPort; + ppc::protocol::EndPoint endPoint; - std::string endPoint() const { return listenIp + ":" + std::to_string(listenPort); } + std::string listenEndPoint() const { return endPoint.listenEndPoint(); } }; class GrpcServer { diff --git a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h index b0164a58..f5733ade 100644 --- a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h @@ -30,12 +30,13 @@ class NodeInfoImpl : public INodeInfo using Ptr = std::shared_ptr; explicit NodeInfoImpl(std::function inner) : m_inner(std::move(inner)) {} + NodeInfoImpl() : m_inner([inner = ppc::proto::NodeInfo()]() mutable { return &inner; }) {} - NodeInfoImpl(bcos::bytesConstRef const& nodeID) - : m_inner([inner = ppc::proto::NodeInfo()]() mutable { return &inner; }) + NodeInfoImpl(bcos::bytesConstRef const& nodeID) : NodeInfoImpl() { m_inner()->set_nodeid(nodeID.data(), nodeID.size()); } + NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) : NodeInfoImpl(nodeID) { @@ -83,15 +84,15 @@ class NodeInfoFactory : public INodeInfoFactory { public: using Ptr = std::shared_ptr; - NodeInfoFactory(bcos::bytesConstRef const& nodeID) : INodeInfoFactory(nodeID.toBytes()) {} + NodeInfoFactory() {} ~NodeInfoFactory() override {} - INodeInfo::Ptr build() override { return std::make_shared(bcos::ref(m_nodeID)); } + INodeInfo::Ptr build() override { return std::make_shared(); } - INodeInfo::Ptr build(std::string const& endPoint) override + INodeInfo::Ptr build(bcos::bytesConstRef nodeID, std::string const& endPoint) override { - return std::make_shared(bcos::ref(m_nodeID), endPoint); + return std::make_shared(nodeID, endPoint); } }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp new file mode 100644 index 00000000..23b121d5 --- /dev/null +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontFactory.cpp + * @author: yujiechen + * @date 2024-9-04 + */ +#include "FrontFactory.h" +#include "FrontImpl.h" + +using namespace ppc::front; +using namespace ppc::tools; + +IFront::Ptr build(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, + ppc::protocol::MessagePayloadBuilder::Ptr messageFactory, + ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, + ppc::gateway::IGateway::Ptr const& gateway, FrontConfig::Ptr config) +{ + auto threadPool = std::make_shared("front", config->threadPoolSize()); + auto nodeInfo = nodeInfoFactory->build( + bcos::bytesConstRef((bcos::byte*)config->nodeID().data(), config->nodeID().size()), + config->selfEndPoint().entryPoint()); + return std::make_shared(threadPool, nodeInfo, messageFactory, routerInfoBuilder, + gateway, std::make_shared()); +} \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h new file mode 100644 index 00000000..e383725c --- /dev/null +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontFactory.h + * @author: yujiechen + * @date 2024-9-04 + */ +#pragma once +#include "ppc-framework/front/IFront.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-framework/gateway/IGateway.h" + +namespace ppc::front +{ +class FrontFactory +{ +public: + using Ptr = std::shared_ptr; + FrontFactory() = default; + virtual ~FrontFactory() = default; + + IFront::Ptr build(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, + ppc::protocol::MessagePayloadBuilder::Ptr messageFactory, + ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, + ppc::gateway::IGateway::Ptr const& gateway, + FrontConfig::Ptr config); +}; +} \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt index 05c438e2..2d4b7364 100644 --- a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt @@ -7,16 +7,12 @@ find_package(Boost COMPONENTS filesystem) file(GLOB_RECURSE SRCS ppc-gateway/*.cpp) -find_package(tarscpp REQUIRED) add_library(${GATEWAY_TARGET} ${SRCS}) target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} ${HTTP_TARGET} ${PROTOCOL_TARGET} - ${TARS_PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET} tarscpp::tarsservant tarscpp::tarsutil TBB::tbb) + ${TARS_PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET} TBB::tbb) -if (APPLE) - # target_compile_options(${GATEWAY_TARGET} PRIVATE -faligned-allocation) -endif () # ut if (TESTS) diff --git a/cpp/wedpr-transport/sdk/CMakeLists.txt b/cpp/wedpr-transport/sdk/CMakeLists.txt new file mode 100644 index 00000000..1211d63a --- /dev/null +++ b/cpp/wedpr-transport/sdk/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.14) +project(ppc-transport-sdk VERSION ${VERSION}) + +file(GLOB_RECURSE SRCS *.cpp) + +add_library(${WEDPR_TRANSPORT_SDK_TARGET} ${SRCS}) +target_link_libraries(${WEDPR_TRANSPORT_SDK_TARGET} PUBLIC + ${FRONT_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp new file mode 100644 index 00000000..223d4c69 --- /dev/null +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProTransportImpl.cpp + * @author: yujiechen + * @date 2024-09-04 + */ +#include "ProTransportImpl.h" +#include "protobuf/src/v1/MessageImpl.h" +#include "wedpr-protocol/grpc/server/FrontServer.h" + +using namespace ppc::front; +using namespace ppc::protocol; +ProTransportImpl::ProTransportImpl(ppc::Front::FrontConfig::Ptr config) + : m_config(std::move(config)) +{ + GrpcServerConfig{config->selfEndPoint()}; + m_server = std::make_shared(GrpcServerConfig); + + FrontFactory frontFactory; + grpc::ChannelArguments channelConfig; + channelConfig.SetLoadBalancingPolicyName(m_config->loadBalancePolicy()); + auto gateway = std::make_shared(channelConfig, m_config->gatewayEndPoints()); + m_front = frontFactory.build(std::make_shared(), + std::make_shared(), + std::make_shared(), gateway, config); + auto frontService = std::make_shared(msgBuilder, m_front); + + // register the frontService + m_server->registerService(frontService); +} \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h new file mode 100644 index 00000000..628fed53 --- /dev/null +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProTransportImpl.h + * @author: yujiechen + * @date 2024-09-04 + */ +#pragma once + +#include "TransportImpl.h" +#include "wedpr-protocol/grpc/client/GatewayClient.h" +#include "wedpr-protocol/grpc/server/GrpcServer.h" + +namespace ppc::sdk +{ +class ProTransportImpl : public Transport +{ +public: + using Ptr = std::shared_ptr; + ProTransportImpl(ppc::Front::FrontConfig::Ptr config); + + void start() override + { + m_server->start(); + m_front->start(); + } + void stop() override + { + m_server->stop(); + m_front->stop(); + } + +protected: + ppc::protocol::GrpcServer::Ptr m_server; +}; +} // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/Transport.h b/cpp/wedpr-transport/sdk/Transport.h new file mode 100644 index 00000000..eddb414f --- /dev/null +++ b/cpp/wedpr-transport/sdk/Transport.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Transport.h + * @author: yujiechen + * @date 2024-09-04 + */ +#pragma once +#include "ppc-framework/front/IFront.h" +namespace ppc::sdk +{ +class Transport +{ +public: + Transport() = default; + virtual ~Transport() = default; + + virtual void start() { m_front->start(); } + virtual void stop() { m_front->stop(); } + + virtual ppc::front::IFront::Ptr const& getFront() { return m_front; } + +protected: + ppc::front::IFront::Ptr m_front; +}; +} // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/TransportBuilder.h new file mode 100644 index 00000000..a113310e --- /dev/null +++ b/cpp/wedpr-transport/sdk/TransportBuilder.h @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TransportBuilder.h + * @author: yujiechen + * @date 2024-09-04 + */ +#pragma once +#include "ProTransportImpl.h" +#include "Transport.h" +#include "TransportImpl.h" +#include +namespace ppc::sdk +{ +enum class SDKMode : uint8_t +{ + AIR = 0x00, + PRO = 0x01, +}; +class TransportBuilder +{ +public: + using Ptr = std::shared_ptr; + TransportBuilder() = default; + virtual ~TransportBuilder() = default; + + Transport::Ptr build(SDKMode mode, ppc::Front::FrontConfig::Ptr config, + ppc::gateway::IGateway::Ptr const& gateway) + { + switch (mode) + { + case SDKMode::AIR: + { + return std::make_shared(config, gateway); + } + case SDKMode::PRO: + { + return std::make_shared(config); + } + default: + throw std::exception("Unsupported sdk mode, only support AIR/PRO mode!"); + } + } +}; +} // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/TransportImpl.h new file mode 100644 index 00000000..47222e4b --- /dev/null +++ b/cpp/wedpr-transport/sdk/TransportImpl.h @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TransportImpl.h + * @author: yujiechen + * @date 2024-09-04 + */ +#pragma once +#include "Transport.h" +#include "ppc-framework/gateway/IGateway.h" +#include "ppc-front/FrontFactory.h" +#include "protobuf/NodeInfoImpl.h" +#include "protocol/src/v1/MessageHeaderImpl.h" +#include "protocol/src/v1/MessagePayloadImpl.h" + +namespace ppc::sdk +{ +class TransportImpl : public Transport +{ +public: + TransportImpl(ppc::Front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) + : m_config(std::move(config)) + { + FrontFactory frontFactory; + m_front = frontFactory.build(std::make_shared(), + std::make_shared(), + std::make_shared(), gateway, m_config); + } + ~TransportImpl() override = default; + +protected: + ppc::Front::FrontConfig::Ptr m_config; +}; +} // namespace ppc::sdk From 0a7d99f7c4011a3755bb62a510879440b77abde9 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 4 Sep 2024 16:58:17 +0800 Subject: [PATCH 022/120] add sdk-wrapper for transport (#18) --- cpp/CMakeLists.txt | 4 +- cpp/cmake/Options.cmake | 1 + cpp/cmake/TargetSettings.cmake | 4 +- cpp/cmake/python.cmake | 104 ++++++++++++++++++ .../protocol/src/v1/MessageHeaderImpl.h | 5 + .../sdk-wrapper/CMakeLists.txt | 10 ++ .../sdk-wrapper/python/CMakeLists.txt | 54 +++++++++ .../sdk-wrapper/python/setup.py.in | 35 ++++++ .../python/swig/wedpr_python_transport.i | 49 +++++++++ cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 14 ++- cpp/wedpr-transport/sdk/ProTransportImpl.h | 3 +- cpp/wedpr-transport/sdk/Transport.h | 1 + cpp/wedpr-transport/sdk/TransportBuilder.h | 4 +- cpp/wedpr-transport/sdk/TransportImpl.h | 12 +- 14 files changed, 283 insertions(+), 17 deletions(-) create mode 100644 cpp/cmake/python.cmake create mode 100644 cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/setup.py.in create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9054a717..4ac588da 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -87,7 +87,7 @@ set(ALL_SOURCE_LIST wedpr-helper/libhelper wedpr-helper/ppc-tools wedpr-storage/ppc-io wedpr-storage/ppc-storage wedpr-transport/ppc-gateway wedpr-transport/ppc-front - wedpr-transport/ppc-http wedpr-transport/ppc-rpc + wedpr-transport/ppc-http wedpr-transport/ppc-rpc wedpr-transport/sdk wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE}) if(BUILD_WEDPR_TOOLKIT) @@ -100,7 +100,7 @@ if(BUILD_WEDPR_TOOLKIT) include(swig) message(STATUS "Getting SWIG for Windows: ...DONE") endif() - add_subdirectory(wedpr-toolkit-wrapper) + add_subdirectory(wedpr-transport/sdk-wrapper) endif() if(BUILD_ALL) diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index bf631af3..9b092e36 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -169,6 +169,7 @@ macro(print_config NAME) message("-- DEMO Enable DEMO ${ENABLE_DEMO}") message("-- BUILD_SDK BUILD SDK ${BUILD_SDK}") message("-- BUILD_UDF BUILD UDF ${BUILD_UDF}") + message("-- BUILD_WEDPR_TOOLKIT BUILD_WEDPR_TOOLKIT ${BUILD_WEDPR_TOOLKIT}") message("-- DEBUG ${DEBUG}") message("------------------------------------------------------------------------") message("") diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 46735456..519f4d55 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -137,5 +137,5 @@ set(BOOST_UNIT_TEST Boost::unit_test_framework) set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk) # ==== the swig wrapper ===== -set(WEDPR_PYTHON_TOOLKIT "wedpr_python_toolkit") -set(WEDPR_PYTHON_TOOLKIT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TOOLKIT}) \ No newline at end of file +set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport") +set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT}) \ No newline at end of file diff --git a/cpp/cmake/python.cmake b/cpp/cmake/python.cmake new file mode 100644 index 00000000..8adbf35d --- /dev/null +++ b/cpp/cmake/python.cmake @@ -0,0 +1,104 @@ +if(NOT BUILD_PYTHON) + return() +endif() + +# Use latest UseSWIG module (3.14) and Python3 module (3.18) +cmake_minimum_required(VERSION 3.18) + +# Will need swig +set(CMAKE_SWIG_FLAGS) +find_package(SWIG REQUIRED) +include(UseSWIG) + +if(${SWIG_VERSION} VERSION_GREATER_EQUAL 4) + list(APPEND CMAKE_SWIG_FLAGS "-doxygen") +endif() + +if(UNIX AND NOT APPLE) + list(APPEND CMAKE_SWIG_FLAGS "-DSWIGWORDSIZE64") +endif() + +# Find Python 3 +find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) +list(APPEND CMAKE_SWIG_FLAGS "-py3" "-DPY3") + +function(search_python_module) + set(options NO_VERSION) + set(oneValueArgs NAME PACKAGE) + set(multiValueArgs "") + cmake_parse_arguments(MODULE + "${options}" + "${oneValueArgs}" + "${multiValueArgs}" + ${ARGN} + ) + message(STATUS "Searching python module: \"${MODULE_NAME}\"") + if(${MODULE_NO_VERSION}) + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import ${MODULE_NAME}" + RESULT_VARIABLE _RESULT + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(MODULE_VERSION "unknown") + else() + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import ${MODULE_NAME}; print(${MODULE_NAME}.__version__)" + RESULT_VARIABLE _RESULT + OUTPUT_VARIABLE MODULE_VERSION + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endif() + if(${_RESULT} STREQUAL "0") + message(STATUS "Found python module: \"${MODULE_NAME}\" (found version \"${MODULE_VERSION}\")") + else() + if(FETCH_PYTHON_DEPS) + message(WARNING "Can't find python module: \"${MODULE_NAME}\", install it using pip...") + execute_process( + COMMAND ${Python3_EXECUTABLE} -m pip install --user ${MODULE_PACKAGE} + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + else() + message(FATAL_ERROR "Can't find python module: \"${MODULE_NAME}\", please install it using your system package manager.") + endif() + endif() +endfunction() + +# Find if a python builtin module is available. +# e.g +# search_python_internal_module( +# NAME +# mypy_protobuf +# ) +function(search_python_internal_module) + set(options "") + set(oneValueArgs NAME) + set(multiValueArgs "") + cmake_parse_arguments(MODULE + "${options}" + "${oneValueArgs}" + "${multiValueArgs}" + ${ARGN} + ) + message(STATUS "Searching python module: \"${MODULE_NAME}\"") + execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import ${MODULE_NAME}" + RESULT_VARIABLE _RESULT + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(${_RESULT} STREQUAL "0") + message(STATUS "Found python internal module: \"${MODULE_NAME}\"") + else() + message(FATAL_ERROR "Can't find python internal module \"${MODULE_NAME}\", please install it using your system package manager.") + endif() +endfunction() + +# Look for python module wheel +search_python_module( + NAME setuptools + PACKAGE setuptools) +search_python_module( + NAME wheel + PACKAGE wheel) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h index 17f4b99d..b48caec7 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h @@ -109,5 +109,10 @@ class MessageOptionalHeaderBuilderImpl : public MessageOptionalHeaderBuilder { return std::make_shared(optionalHeader); } + + MessageOptionalHeader::Ptr build() override + { + return std::make_shared(); + } }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt new file mode 100644 index 00000000..a56f6cf1 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt @@ -0,0 +1,10 @@ +project(wedpr-transport-swig VERSION ${VERSION}) + +find_package(SWIG REQUIRED) +include(${SWIG_USE_FILE}) + +# Add subdirectories for each language if desired +option(BUILD_PYTHON "Build Python SWIG module" ON) +if(BUILD_PYTHON) + add_subdirectory(python) +endif() diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt new file mode 100644 index 00000000..1eaab1a7 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt @@ -0,0 +1,54 @@ +file(GLOB_RECURSE SRCS *.i) +set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) + +swig_add_library( + ${WEDPR_PYTHON_TRANSPORT} + TYPE MODULE + LANGUAGE python + OUTPUT_DIR ${WEDPR_PYTHON_TRANSPORT_DIR} + SOURCES ${SRCS} +) + +message("#### Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") +target_include_directories( + ${WEDPR_PYTHON_TRANSPORT} + PRIVATE + ../include + ${Python3_INCLUDE_DIRS} + ) +set_property(TARGET ${WEDPR_PYTHON_TRANSPORT} PROPERTY SWIG_USE_TARGET_INCLUDE_DIRECTORIES ON) + +# note: macOS is APPLE and also UNIX ! +if(APPLE) + set_property(TARGET ${WEDPR_PYTHON_TRANSPORT} APPEND PROPERTY + LINK_FLAGS "-flat_namespace -undefined suppress" + ) + set_target_properties(${WEDPR_PYTHON_TRANSPORT} PROPERTIES + SUFFIX ".so" + INSTALL_RPATH "@loader_path;@loader_path/../../${WEDPR_PYTHON_TRANSPORT}/.libs" + ) +elseif(UNIX) + set_target_properties(${WEDPR_PYTHON_TRANSPORT} PROPERTIES + INSTALL_RPATH "$ORIGIN:$ORIGIN/../../${WEDPR_PYTHON_TRANSPORT}/.libs" + ) +endif() +target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${FRONT_TARGET}) + +SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_DIR}/) + +# Variable PYTHON_LIBRARIES can contains keyword `optimized` +# which won't be interpreted inside a generator expression. +# i.e. we can't use: $<$:${PYTHON_LIBRARIES}> +# see: https://cmake.org/cmake/help/git-stage/command/target_link_libraries.html#command:target_link_libraries +if(MSVC) + target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${Python3_LIBRARIES}) +endif() + +# Configure setup.py and copy to output directory +file(GENERATE OUTPUT ${WEDPR_PYTHON_TRANSPORT_DIR}/__init__.py CONTENT "__version__ = \"${PYTHON_TOOLKIT_VERSION}\"\n") +set(SETUP_PY_IN ${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in) +set(SETUP_PY_OUT ${WEDPR_PYTHON_TRANSPORT_DIR}/setup.py) +configure_file(${SETUP_PY_IN} ${SETUP_PY_OUT}) + +message(STATUS "CMAKE_INSTALL_INCLUDEDIR => ${CMAKE_INSTALL_INCLUDEDIR}") +message(STATUS "CMAKE_CURRENT_SOURCE_DIR => ${CMAKE_CURRENT_SOURCE_DIR}") \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in b/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in new file mode 100644 index 00000000..b2ec003f --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in @@ -0,0 +1,35 @@ +import setuptools.command.install +import shutil +from distutils.sysconfig import get_python_lib + + +class CompiledLibInstall(setuptools.command.install.install): + """ + Specialized install to install to python libs + """ + + def run(self): + """ + Run method called by setup + :return: + """ + # Get filenames from CMake variable + filenames = '${PYTHON_INSTALL_FILES}'.split(';') + + # Directory to install to + install_dir = get_python_lib() + + # Install files + [shutil.copy(filename, install_dir) for filename in filenames] + + +if __name__ == '__main__': + setuptools.setup( + name='wedpr-python-transport', + version='1.0.0-rc1', + packages=['wedpr-python-transport'], + license='Apache License 2.0', + author='wedpr', + author_email='wedpr@webank.com', + cmdclass={'install': CompiledLibInstall} + ) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i new file mode 100644 index 00000000..d65250d7 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i @@ -0,0 +1,49 @@ +%module wedpr_python_toolkit + +%include +%include +%include +%include +%include + + +%shared_ptr(ppc::front::FrontConfig); +%shared_ptr(bcos::Error); +%shared_ptr(ppc::protocol::Message); +%shared_ptr(ppc::protocol::MessageOptionalHeader) +%shared_ptr(ppc::sdk::Transport); + +%{ +#define SWIG_FILE_WITH_INIT +#include +#include "wedpr-transport/sdk/TransportBuilder.h" +#include "wedpr-transport/sdk/Transport.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/RouteType.h" +#include "ppc-framework/front/FrontConfig.h" +#include +%} + +namespace ppc::front{ + class FrontConfig; + class IFront; + class FrontImpl; + class FrontBuilderImpl; + class RouteType; + class GatewayEndPoint; +} + +namespace ppc::sdk{ + class Transport; + class TransportBuilder; +} + +%template(SharedFrontConfig) std::shared_ptr; +%template(SharedBcosError) std::shared_ptr; +%template(SharedMessage) std::shared_ptr; +%template(SharedRouteInfo) std::shared_ptr; +%template(SharedTransport) std::shared_ptr; + +%include "wedpr-transport/sdk/TransportBuilder.h" +%include "wedpr-transport/sdk/Transport.h" +%include "ppc-framework/front/IFront.h" \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp index 223d4c69..1573c0fa 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -18,16 +18,19 @@ * @date 2024-09-04 */ #include "ProTransportImpl.h" -#include "protobuf/src/v1/MessageImpl.h" +#include "protocol/src/v1/MessageImpl.h" #include "wedpr-protocol/grpc/server/FrontServer.h" using namespace ppc::front; using namespace ppc::protocol; -ProTransportImpl::ProTransportImpl(ppc::Front::FrontConfig::Ptr config) +using namespace ppc::sdk; + + +ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config) : m_config(std::move(config)) { - GrpcServerConfig{config->selfEndPoint()}; - m_server = std::make_shared(GrpcServerConfig); + GrpcServerConfig grpcServerConfig{config->selfEndPoint()}; + m_server = std::make_shared(grpcServerConfig); FrontFactory frontFactory; grpc::ChannelArguments channelConfig; @@ -36,6 +39,9 @@ ProTransportImpl::ProTransportImpl(ppc::Front::FrontConfig::Ptr config) m_front = frontFactory.build(std::make_shared(), std::make_shared(), std::make_shared(), gateway, config); + + auto msgBuilder = + std::make_shared(std::make_shared()); auto frontService = std::make_shared(msgBuilder, m_front); // register the frontService diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index 628fed53..d32ba9e5 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -29,7 +29,7 @@ class ProTransportImpl : public Transport { public: using Ptr = std::shared_ptr; - ProTransportImpl(ppc::Front::FrontConfig::Ptr config); + ProTransportImpl(ppc::front::FrontConfig::Ptr config); void start() override { @@ -43,6 +43,7 @@ class ProTransportImpl : public Transport } protected: + ppc::front::FrontConfig::Ptr m_config; ppc::protocol::GrpcServer::Ptr m_server; }; } // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/Transport.h b/cpp/wedpr-transport/sdk/Transport.h index eddb414f..cf9230fb 100644 --- a/cpp/wedpr-transport/sdk/Transport.h +++ b/cpp/wedpr-transport/sdk/Transport.h @@ -24,6 +24,7 @@ namespace ppc::sdk class Transport { public: + using Ptr = std::shared_ptr; Transport() = default; virtual ~Transport() = default; diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/TransportBuilder.h index a113310e..f0492822 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.h +++ b/cpp/wedpr-transport/sdk/TransportBuilder.h @@ -36,7 +36,7 @@ class TransportBuilder TransportBuilder() = default; virtual ~TransportBuilder() = default; - Transport::Ptr build(SDKMode mode, ppc::Front::FrontConfig::Ptr config, + Transport::Ptr build(SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) { switch (mode) @@ -50,7 +50,7 @@ class TransportBuilder return std::make_shared(config); } default: - throw std::exception("Unsupported sdk mode, only support AIR/PRO mode!"); + throw std::runtime_error("Unsupported sdk mode, only support AIR/PRO mode!"); } } }; diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/TransportImpl.h index 47222e4b..c32e20d3 100644 --- a/cpp/wedpr-transport/sdk/TransportImpl.h +++ b/cpp/wedpr-transport/sdk/TransportImpl.h @@ -30,17 +30,17 @@ namespace ppc::sdk class TransportImpl : public Transport { public: - TransportImpl(ppc::Front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) + TransportImpl(ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) : m_config(std::move(config)) { - FrontFactory frontFactory; - m_front = frontFactory.build(std::make_shared(), - std::make_shared(), - std::make_shared(), gateway, m_config); + ppc::front::FrontFactory frontFactory; + m_front = frontFactory.build(std::make_shared(), + std::make_shared(), + std::make_shared(), gateway, m_config); } ~TransportImpl() override = default; protected: - ppc::Front::FrontConfig::Ptr m_config; + ppc::front::FrontConfig::Ptr m_config; }; } // namespace ppc::sdk From 0fba05a81e3f11a294122303c537b2fd2fc8201b Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 4 Sep 2024 19:04:12 +0800 Subject: [PATCH 023/120] add java swig (#19) * add java swig * add java binding package --- cpp/CMakeLists.txt | 1 - cpp/cmake/TargetSettings.cmake | 5 +- cpp/cmake/java.cmake | 16 + cpp/ppc-framework/front/IFront.h | 2 +- cpp/wedpr-protocol/grpc/CMakeLists.txt | 2 +- cpp/wedpr-protocol/grpc/client/CMakeLists.txt | 2 +- cpp/wedpr-protocol/grpc/server/CMakeLists.txt | 2 +- cpp/wedpr-protocol/protobuf/NodeInfoImpl.h | 1 + cpp/wedpr-protocol/tars/CMakeLists.txt | 2 +- cpp/wedpr-transport/ppc-front/CMakeLists.txt | 2 +- .../ppc-front/ppc-front/FrontFactory.cpp | 7 +- .../ppc-front/ppc-front/FrontFactory.h | 14 +- .../sdk-wrapper/CMakeLists.txt | 8 + .../sdk-wrapper/java/CMakeLists.txt | 24 ++ .../sdk-wrapper/java/bindings/build.gradle | 199 +++++++++++++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + .../sdk-wrapper/java/bindings/gradlew | 185 ++++++++++++ .../sdk-wrapper/java/bindings/gradlew.bat | 89 ++++++ .../sdk-wrapper/java/bindings/settings.gradle | 2 + .../wedpr/sdk/jni/common/JniLibLoader.java | 278 ++++++++++++++++++ .../java/swig/wedpr_java_transport.i | 74 +++++ .../sdk-wrapper/python/CMakeLists.txt | 2 +- .../python/swig/wedpr_python_transport.i | 2 +- cpp/wedpr-transport/sdk/CMakeLists.txt | 2 +- 25 files changed, 904 insertions(+), 23 deletions(-) create mode 100644 cpp/cmake/java.cmake create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.jar create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.properties create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew.bat create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4ac588da..c02bf0aa 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -94,7 +94,6 @@ if(BUILD_WEDPR_TOOLKIT) # fetch the python dependencies option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON) message(STATUS "Python fetch dependencies: ${FETCH_PYTHON_DEPS}") - include(python) if(WIN32) message(STATUS "Getting SWIG for Windows: ...") include(swig) diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 519f4d55..b72cec18 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -138,4 +138,7 @@ set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk) # ==== the swig wrapper ===== set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport") -set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT}) \ No newline at end of file +set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT}) + +set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport") +set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/${WEDPR_JAVA_TRANSPORT}) \ No newline at end of file diff --git a/cpp/cmake/java.cmake b/cpp/cmake/java.cmake new file mode 100644 index 00000000..53e15d19 --- /dev/null +++ b/cpp/cmake/java.cmake @@ -0,0 +1,16 @@ +if(NOT BUILD_JAVA) + return() +endif() + +# Will need swig +set(CMAKE_SWIG_FLAGS) +find_package(SWIG REQUIRED) +include(UseSWIG) + +if(${SWIG_VERSION} VERSION_GREATER_EQUAL 4) + list(APPEND CMAKE_SWIG_FLAGS "-doxygen") +endif() + +if(UNIX AND NOT APPLE) + list(APPEND CMAKE_SWIG_FLAGS "-DSWIGWORDSIZE64") +endif() \ No newline at end of file diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 5de205f3..b5f94a2d 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -96,7 +96,7 @@ class IFront : public virtual IFrontClient long timeout) = 0; virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; - virtual ppc::protocol::Message::Ptr peek(std::string const& topic); + virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; /** diff --git a/cpp/wedpr-protocol/grpc/CMakeLists.txt b/cpp/wedpr-protocol/grpc/CMakeLists.txt index 01865a49..a45b773d 100644 --- a/cpp/wedpr-protocol/grpc/CMakeLists.txt +++ b/cpp/wedpr-protocol/grpc/CMakeLists.txt @@ -27,7 +27,7 @@ foreach(proto_file ${MESSAGES_PROTOS}) endforeach() add_library(${SERVICE_PB_TARGET} ${GRPC_MESSAGES_SRCS}) -target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure) +target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure gRPC::grpc++_reflection) add_subdirectory(client) add_subdirectory(server) diff --git a/cpp/wedpr-protocol/grpc/client/CMakeLists.txt b/cpp/wedpr-protocol/grpc/client/CMakeLists.txt index d6c9d625..f7926ed3 100644 --- a/cpp/wedpr-protocol/grpc/client/CMakeLists.txt +++ b/cpp/wedpr-protocol/grpc/client/CMakeLists.txt @@ -1,3 +1,3 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${SERVICE_CLIENT_TARGET} ${SRCS}) -target_link_libraries(${SERVICE_CLIENT_TARGET} PUBLIC ${SERVICE_PB_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file +target_link_libraries(${SERVICE_CLIENT_TARGET} PUBLIC ${SERVICE_PB_TARGET} ${PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/CMakeLists.txt b/cpp/wedpr-protocol/grpc/server/CMakeLists.txt index 1b795210..aaa6c243 100644 --- a/cpp/wedpr-protocol/grpc/server/CMakeLists.txt +++ b/cpp/wedpr-protocol/grpc/server/CMakeLists.txt @@ -1,3 +1,3 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${SERVICE_SERVER_TARGET} ${SRCS}) -target_link_libraries(${SERVICE_SERVER_TARGET} PUBLIC ${SERVICE_PB_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file +target_link_libraries(${SERVICE_SERVER_TARGET} PUBLIC ${SERVICE_PB_TARGET} ${PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h index f5733ade..a81ad5c0 100644 --- a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h @@ -21,6 +21,7 @@ #include "NodeInfo.pb.h" #include "ppc-framework/protocol/INodeInfo.h" #include + namespace ppc::protocol { // the node information diff --git a/cpp/wedpr-protocol/tars/CMakeLists.txt b/cpp/wedpr-protocol/tars/CMakeLists.txt index d016016e..7d0e2f8e 100644 --- a/cpp/wedpr-protocol/tars/CMakeLists.txt +++ b/cpp/wedpr-protocol/tars/CMakeLists.txt @@ -38,7 +38,7 @@ target_include_directories(${TARS_PROTOCOL_TARGET} PUBLIC $ $ $) -target_link_libraries(${TARS_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} tarscpp::tarsservant tarscpp::tarsutil) +target_link_libraries(${TARS_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} tarscpp::tarsutil) # ut if (TESTS) diff --git a/cpp/wedpr-transport/ppc-front/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/CMakeLists.txt index 71e64d23..ae55348b 100644 --- a/cpp/wedpr-transport/ppc-front/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-front/CMakeLists.txt @@ -4,7 +4,7 @@ aux_source_directory(ppc-front SRCS) add_library(${FRONT_TARGET} ${SRCS}) -target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TCMalloc) +target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) if (TESTS) enable_testing() diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp index 23b121d5..40b56ef1 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp @@ -21,11 +21,10 @@ #include "FrontImpl.h" using namespace ppc::front; -using namespace ppc::tools; +using namespace ppc::protocol; -IFront::Ptr build(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, - ppc::protocol::MessagePayloadBuilder::Ptr messageFactory, - ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, +IFront::Ptr FrontFactory::build(INodeInfoFactory::Ptr nodeInfoFactory, + MessagePayloadBuilder::Ptr messageFactory, MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, ppc::gateway::IGateway::Ptr const& gateway, FrontConfig::Ptr config) { auto threadPool = std::make_shared("front", config->threadPoolSize()); diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h index e383725c..1d29ca90 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.h @@ -18,11 +18,10 @@ * @date 2024-9-04 */ #pragma once -#include "ppc-framework/front/IFront.h" -#include "ppc-tools/src/config/PPCConfig.h" #include "ppc-framework/front/FrontConfig.h" -#include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-framework/front/IFront.h" #include "ppc-framework/gateway/IGateway.h" +#include "ppc-framework/protocol/INodeInfo.h" namespace ppc::front { @@ -33,10 +32,9 @@ class FrontFactory FrontFactory() = default; virtual ~FrontFactory() = default; - IFront::Ptr build(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, + IFront::Ptr build(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, ppc::protocol::MessagePayloadBuilder::Ptr messageFactory, - ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, - ppc::gateway::IGateway::Ptr const& gateway, - FrontConfig::Ptr config); + ppc::protocol::MessageOptionalHeaderBuilder::Ptr routerInfoBuilder, + ppc::gateway::IGateway::Ptr const& gateway, FrontConfig::Ptr config); }; -} \ No newline at end of file +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt index a56f6cf1..08c97539 100644 --- a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt @@ -6,5 +6,13 @@ include(${SWIG_USE_FILE}) # Add subdirectories for each language if desired option(BUILD_PYTHON "Build Python SWIG module" ON) if(BUILD_PYTHON) + include(python) add_subdirectory(python) endif() + +# Add subdirectories for each language if desired +option(BUILD_JAVA "Build JAVA SWIG module" ON) +if(BUILD_JAVA) + include(java) + add_subdirectory(java) +endif() diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt new file mode 100644 index 00000000..d4d36ffd --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt @@ -0,0 +1,24 @@ +file(GLOB_RECURSE SRCS *.i) +set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) + +set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.transport") + +set_property(SOURCE swig/wedpr_java_transport.i PROPERTY COMPILE_OPTIONS + -package ${WEDPR_TRANSPORT_PACKAGE}) + +swig_add_library(${WEDPR_JAVA_TRANSPORT} + TYPE MODULE + LANGUAGE java + OUTPUT_DIR ${WEDPR_JAVA_TRANSPORT_DIR} + SOURCES ${SRCS}) + +SET(LIBRARY_OUTPUT_PATH ${WEDPR_JAVA_TRANSPORT_DIR}) +message("LIBRARY_OUTPUT_PATH: ${WEDPR_JAVA_TRANSPORT_DIR}") + +set_target_properties(${WEDPR_JAVA_TRANSPORT} PROPERTIES + SWIG_USE_TARGET_INCLUDE_DIRECTORIES ON + POSITION_INDEPENDENT_CODE ON) + +target_include_directories(${WEDPR_JAVA_TRANSPORT} PRIVATE ${JNI_INCLUDE_DIRS}) + +target_link_libraries(${WEDPR_JAVA_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${JAVA_LIBRARIES}) \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle new file mode 100644 index 00000000..c455b8ad --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -0,0 +1,199 @@ +plugins { + id 'java' + id 'idea' + id 'java-library' + id 'maven-publish' + //id 'org.ajoberstar.grgit' version '4.1.1' + //id "de.undercouch.download" version "4.1.2" +// id 'com.github.sherter.google-java-format' version '0.9' +} + +println("Notice: current gradle version is " + gradle.gradleVersion) +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +[compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8' + +repositories { + maven { url "https://maven.aliyun.com/nexus/content/groups/public/" } + maven { url "https://oss.sonatype.org/service/local/staging/deploy/maven2"} + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } + maven {url "https://plugins.gradle.org/m2/"} + mavenCentral() + mavenLocal() +} + +// googleJavaFormat { +// toolVersion = '1.7' +// options style: 'AOSP' +// source = sourceSets*.allJava +// include '**/*.java' +// exclude '**/*Test.java' +// exclude '**/Test*.java' +// exclude '**/Mock*.java' +// } + +sourceSets { + main { + java { + srcDir 'src/main/java' + } + resources { + srcDir 'src/main/resources' + } + } + + jmh { + java { + srcDir 'src/jmh/java' + } + resources { + srcDir 'src/jmh/resources' + } + + runtimeClasspath += sourceSets.main.runtimeClasspath + compileClasspath += sourceSets.main.compileClasspath + + compileClasspath += main.output + runtimeClasspath += main.output + } +} +dependencies { + api ('org.slf4j:slf4j-api:1.7.36') + + jmhImplementation 'org.openjdk.jmh:jmh-core:1.36' + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.36' + + testImplementation ('org.slf4j:slf4j-log4j12:1.7.36') + testImplementation ('junit:junit:4.13.2') +} + +archivesBaseName = 'wedpr-java-transport-jni' +group = 'com.webank.wedpr' +version = '1.0.0' + +// Additional attribute definition +ext { + if (!project.hasProperty("ossrhUsername")) { + ossrhUsername="xxx" + } + + if (!project.hasProperty("ossrhPassword")) { + ossrhPassword="xxx" + } +} + +jar { + exclude '**/*.xml' + exclude '**/*.properties' + /*manifest { + try { + def repo = grgit.open(currentDir: project.rootDir) + if (repo != null) { + + def date = new Date().format("yyyy-MM-dd'T'HH:mm:ssZ") + def branch = repo.branch.getCurrent().getName() + def commit = repo.head().getAbbreviatedId(40) + + attributes(["Implementation-Timestamp": date, + "Git-Branch" : branch, + "Git-Commit" : commit]) + } + } catch (Exception e) { + e.printStackTrace() + } + } from sourceSets.main.output*/ + + doLast { + copy { + from destinationDirectory + into 'dist/apps' + } + + copy { + from configurations.runtimeClasspath + into 'dist/lib' + } + } +} + +javadoc { + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') +} + +task sourcesJar(type: Jar) { + from sourceSets.main.allJava + archiveClassifier = 'sources' + duplicatesStrategy 'exclude' +} +tasks.withType(Copy).all { + duplicatesStrategy 'exclude' +} + + +task javadocJar(type: Jar) { + from javadoc + archiveClassifier = 'javadoc' +} + +task jmh(type: JavaExec, dependsOn: jmhClasses) { + main = 'com.webank.wedpr.sdk.jni.jmh.BenchmarkMain' + classpath = sourceSets.jmh.compileClasspath + sourceSets.jmh.runtimeClasspath +} + +publishing { + publications { + mavenJava(MavenPublication) { + + artifactId project.name + groupId project.group + version project.version + + from components.java + artifact sourcesJar + artifact javadocJar + + pom { + name = 'wedpr' + description = 'wedpr java transport jni' + url = 'http://www.fisco-bcos.org' + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + scm { + connection = 'http://github.com/WeBankBlockchain/wedpr-components.git' + url = 'http://github.com/WeBankBlockchain/wedpr-components.git' + } + + developers { + developer { + id = 'zhangsan' + name = 'zhangsan' + email = 'zhangsan@example.com' + } + } + } + } + } + repositories { + maven { + def releasesRepoURL = "https://oss.sonatype.org/service/local/staging/deploy/maven2" + def snapshotsRepoURL = "https://oss.sonatype.org/content/repositories/snapshots" + allowInsecureProtocol = true + url = !version.endsWith("SNAPSHOT") ? releasesRepoURL : snapshotsRepoURL + + credentials { + username 'deployment' + password 'deployment123' + } + } + } + +} + diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.jar b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q

Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.properties b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a33e7343 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +# distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip +distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew new file mode 100644 index 00000000..4f906e0c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew.bat b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew.bat new file mode 100644 index 00000000..ac1b06f9 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle new file mode 100644 index 00000000..5c90d6b7 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'wedpr-java-transport-jni' + diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java new file mode 100644 index 00000000..9af0301e --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java @@ -0,0 +1,278 @@ +/** + * Copyright 2023 [wedpr] + * + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.webank.wedpr.sdk.jni.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.Objects; +import java.util.Scanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper class to load wedpr-java-transportjni resources. + * + * @author octopus + */ +public final class JniLibLoader { + private static final Logger logger = LoggerFactory.getLogger(JniLibLoader.class); + + public static final String LINUX = "linux"; + public static final String MAC = "mac"; + public static final String WIN = "win"; + + public static final String NATIVE_RESOURCE_LIB_NAME = "wedpr-java-transport-jni"; + public static final String NATIVE_RESOURCE_HOME = "/META-INF/native"; + public static final String NATIVE_WIN_DEPS_DIR = NATIVE_RESOURCE_HOME + "/win/"; + public static final String WIN_DEPS_FILE_LIST = "file.list"; + + public static final String OS_NAME = getOs(); + public static final String ARCH_NAME = getArch(); + + private static File WORKDIR; + + static { + // 1. initialize workdir + + // -Dcom.webank.wedpr.workdir + String workdir = System.getProperty("com.webank.wedpr.workdir"); + if (workdir != null) { + try { + File f = new File(workdir); + f.mkdirs(); + WORKDIR = f; + logger.info("initialize workdir, -Dcom.webank.wedpr.workdir: {}", WORKDIR); + } catch (Exception e) { + logger.debug("initialize workdir, cannot mkdir workdir: {}, e: ", workdir, e); + workdir = null; + } + } + + // default -Djava.io.tmpdir + if (workdir == null) { + workdir = System.getProperty("java.io.tmpdir"); + File f = new File(workdir); + f.mkdirs(); + WORKDIR = f; + + logger.info("initialize workdir, -Djava.io.tmpdirs: {}", WORKDIR); + } + } + + public static String getArch() { + String archName = System.getProperty("os.arch", ""); + if (archName.contains("aarch64") || archName.contains("arm64")) { + return "arm"; + } else { + return ""; + } + } + + public static String getLibName(String baseName) { + String osName = OS_NAME; + String archName = ARCH_NAME; + + logger.debug("osName: {}, archName: {}", osName, archName); + + if (osName.contains(WIN)) { + return baseName + ".dll"; + } else if (osName.contains(LINUX)) { + String arch = getArch(); + if ("arm".equals(arch)) { + return "lib" + baseName + "-aarch64" + ".so"; + } + return "lib" + baseName + ".so"; + } else if (osName.contains(MAC)) { + String arch = getArch(); + if ("arm".equals(arch)) { + return "lib" + baseName + "-aarch64" + ".dylib"; + } + return "lib" + baseName + ".dylib"; + } else { + throw new RuntimeException("unrecognized OS: " + osName); + } + } + + public static String getOs() { + String osName = System.getProperty("os.name").toLowerCase(); + if (osName.contains(WIN)) { + return WIN; + } else if (osName.contains(LINUX)) { + return LINUX; + } else if (osName.contains(MAC)) { + return MAC; + } else { + throw new RuntimeException("unrecognized OS: " + osName); + } + } + + public static void loadJniLibrary() { + String libPath = null; + boolean loadLibFromFsOk = false; + try { + // 1. try to load library from filesystem + // -Dcom.webank.wedpr.jni.library.path + libPath = System.getProperty("com.webank.wedpr.jni.library.path"); + if (libPath != null) { + logger.info( + "try to load library -Dcom.webank.wedpr.jni.library.path, libPath: {}", + libPath); + // try to load library from filesystem first + loadLibraryFromFs( + libPath, NATIVE_RESOURCE_LIB_NAME, new File(libPath).isAbsolute()); + loadLibFromFsOk = true; + } + } catch (Exception e0) { + logger.error("unable to load library from fs, libDir: {}, e: ", libPath, e0); + } + + // 2. try to load library from jar + if (!loadLibFromFsOk) { + try { + logger.info("try to load library from jar"); + if (Objects.equals(getOs(), WIN)) { + loadWinDepsLibraryFromJar(); + } + loadLibraryFromJar(NATIVE_RESOURCE_LIB_NAME); + } catch (Exception e1) { + logger.error("unable to load library from fs, e: ", e1); + throw new UnsatisfiedLinkError(e1.toString()); + } + } + } + + public static void loadLibrary(String lib, boolean absolute) { + if (absolute) { + System.load(lib); + } else { + System.loadLibrary(lib); + } + } + + public static void addLibraryDir(String libraryPath) + throws NoSuchFieldException, IllegalAccessException { + Field userPathsField = ClassLoader.class.getDeclaredField("usr_paths"); + userPathsField.setAccessible(true); + String[] paths = (String[]) userPathsField.get(null); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < paths.length; i++) { + if (libraryPath.equals(paths[i])) { + continue; + } + sb.append(paths[i]).append(File.pathSeparator); + } + sb.append(libraryPath); + System.setProperty("java.library.path", sb.toString()); + logger.info("modify, java.library.path: " + sb.toString()); + final Field sysPathsField = ClassLoader.class.getDeclaredField("sys_paths"); + sysPathsField.setAccessible(true); + sysPathsField.set(null, null); + } + + public static void loadLibraryFromFs(String dir, String libName, boolean absolute) { + String resource = dir + "/" + getLibName(libName); + loadLibrary(resource, absolute); + } + + public static void loadLibraryFromJar(String baseName) + throws IOException, NoSuchFieldException, IllegalAccessException { + String libName = getLibName(baseName); + File tempDir = new File(WORKDIR, String.valueOf(System.nanoTime())); + // create temp dir + tempDir.mkdirs(); + File tempFile = new File(tempDir, libName); + logger.info("tempDir: {}, tempFile: {}", tempDir, tempFile); + + String resource = NATIVE_RESOURCE_HOME + "/" + libName; + try (InputStream is = JniLibLoader.class.getResourceAsStream(resource)) { + Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + try { + tempFile.delete(); + } catch (Exception e0) { + logger.debug("delete temp file error, e: ", e0); + } + logger.error("copy dynamic lib from jar failed, resource: {}, e: ", resource, e); + throw e; + } catch (NullPointerException e) { + throw new FileNotFoundException("Cannot found " + resource + " inside the JAR."); + } + + try { + loadLibrary(tempFile.getAbsolutePath(), true); + } catch (Exception e) { + logger.error("loadLibrary error, resource: {}, e: ", resource, e); + throw e; + } finally { + try { + tempFile.delete(); + tempDir.delete(); + logger.debug("remove temp dir and temp file, dir: {}, file: {}", tempDir, tempFile); + } catch (Exception ignored) { + } + } + } + + public static void loadWinDepsLibraryFromJar() throws IOException, NoSuchFieldException { + File tempDir = new File(WORKDIR, String.valueOf(System.nanoTime())); + tempDir.mkdirs(); + + try (InputStream is = + JniLibLoader.class.getResourceAsStream(NATIVE_WIN_DEPS_DIR + WIN_DEPS_FILE_LIST)) { + if (is == null) { + throw new FileNotFoundException( + "Cannot found " + WIN_DEPS_FILE_LIST + " inside the JAR."); + } + try (Scanner scanner = new Scanner(is)) { + while (scanner.hasNext()) { + String lib = scanner.next(); + File targetFile = new File(tempDir, lib); + logger.info( + "loadWinDepsLibraryFromJar tempDir: {}, targetFile: {}", + tempDir, + targetFile); + try (InputStream fis = + JniLibLoader.class.getResourceAsStream(NATIVE_WIN_DEPS_DIR + lib)) { + if (fis == null) { + throw new FileNotFoundException( + "Cannot found " + lib + " inside the JAR."); + } + Files.copy(fis, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + loadLibrary(targetFile.getAbsolutePath(), true); + } catch (IOException e) { + targetFile.delete(); + } catch (Exception e) { + logger.error( + "loadLibrary error, resource: {}, e: ", + targetFile.getAbsolutePath(), + e); + throw e; + } finally { + try { + targetFile.delete(); + tempDir.delete(); + logger.debug("remove temp dir and temp file, {}", targetFile); + } catch (Exception ignored) { + } + } + } + } + } + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i new file mode 100644 index 00000000..05a504c8 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -0,0 +1,74 @@ +%module wedpr_java_transport + +%include "stdint.i" +%include "typemaps.i" + +#if defined(SWIGJAVA) +#if defined(SWIGWORDSIZE64) +// By default SWIG map C++ long int (i.e. int64_t) to C# int +// But we want to map it to C# long so we reuse the typemap for C++ long long. +// ref: https://github.com/swig/swig/blob/master/Lib/java/typemaps.i +// ref: https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/lang/Long.html +%define PRIMITIVE_TYPEMAP(NEW_TYPE, TYPE) +%clear NEW_TYPE; +%clear NEW_TYPE *; +%clear NEW_TYPE &; +%clear const NEW_TYPE &; +%apply TYPE { NEW_TYPE }; +%apply TYPE * { NEW_TYPE * }; +%apply TYPE & { NEW_TYPE & }; +%apply const TYPE & { const NEW_TYPE & }; +%enddef // PRIMITIVE_TYPEMAP +PRIMITIVE_TYPEMAP(long int, long long); +PRIMITIVE_TYPEMAP(unsigned long int, long long); +#undef PRIMITIVE_TYPEMAP +#endif // defined(SWIGWORDSIZE64) +#endif // defined(SWIGJAVA) + +%include +%include +%include +%include +%include + + +%shared_ptr(ppc::front::FrontConfig); +%shared_ptr(bcos::Error); +%shared_ptr(ppc::protocol::Message); +%shared_ptr(ppc::protocol::MessageOptionalHeader) +%shared_ptr(ppc::sdk::Transport); + +%{ +#define SWIG_FILE_WITH_INIT +#include +#include "wedpr-transport/sdk/TransportBuilder.h" +#include "wedpr-transport/sdk/Transport.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/RouteType.h" +#include "ppc-framework/front/FrontConfig.h" +#include +%} + +namespace ppc::front{ + class FrontConfig; + class IFront; + class FrontImpl; + class FrontBuilderImpl; + class RouteType; + class GatewayEndPoint; +} + +namespace ppc::sdk{ + class Transport; + class TransportBuilder; +} + +%template(SharedFrontConfig) std::shared_ptr; +%template(SharedBcosError) std::shared_ptr; +%template(SharedMessage) std::shared_ptr; +%template(SharedRouteInfo) std::shared_ptr; +%template(SharedTransport) std::shared_ptr; + +%include "wedpr-transport/sdk/TransportBuilder.h" +%include "wedpr-transport/sdk/Transport.h" +%include "ppc-framework/front/IFront.h" \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt index 1eaab1a7..f925ec3a 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt @@ -32,7 +32,7 @@ elseif(UNIX) INSTALL_RPATH "$ORIGIN:$ORIGIN/../../${WEDPR_PYTHON_TRANSPORT}/.libs" ) endif() -target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${FRONT_TARGET}) +target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET}) SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_DIR}/) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i index d65250d7..ef073d01 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i @@ -1,4 +1,4 @@ -%module wedpr_python_toolkit +%module wedpr_python_transport %include %include diff --git a/cpp/wedpr-transport/sdk/CMakeLists.txt b/cpp/wedpr-transport/sdk/CMakeLists.txt index 1211d63a..1231ca68 100644 --- a/cpp/wedpr-transport/sdk/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk/CMakeLists.txt @@ -5,4 +5,4 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${WEDPR_TRANSPORT_SDK_TARGET} ${SRCS}) target_link_libraries(${WEDPR_TRANSPORT_SDK_TARGET} PUBLIC - ${FRONT_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET}) \ No newline at end of file + ${FRONT_TARGET} ${PB_PROTOCOL_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${CPU_FEATURES_LIB}) \ No newline at end of file From cddb3eeba5910217d9c3870bbab0c88411d4a2b6 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 5 Sep 2024 09:37:51 +0800 Subject: [PATCH 024/120] adapt to intializer (#20) * rename ppc-main to wedpr-main * adapt to intializer * add gateway initializer --- cpp/CMakeLists.txt | 3 +- cpp/cmake/TargetSettings.cmake | 4 +- cpp/libinitializer/CMakeLists.txt | 4 - cpp/libinitializer/FrontInitializer.cpp | 74 ---------- cpp/libinitializer/FrontInitializer.h | 104 ------------- cpp/libinitializer/ProFrontInitializer.h | 79 ---------- cpp/ppc-framework/front/FrontConfig.h | 55 ++++--- cpp/ppc-framework/front/FrontInterface.h | 5 +- cpp/ppc-framework/front/IFront.h | 9 +- cpp/ppc-framework/protocol/GrpcConfig.h | 49 +++++++ cpp/ppc-framework/rpc/RpcStatusInterface.h | 3 - cpp/ppc-framework/rpc/RpcTypeDef.h | 1 - cpp/ppc-main/CMakeLists.txt | 34 ----- cpp/test-utils/FakeFront.h | 1 - cpp/wedpr-computing/ppc-psi/src/PSIConfig.h | 11 +- .../ppc-tools/src/config/PPCConfig.h | 12 ++ cpp/wedpr-initializer/CMakeLists.txt | 7 + .../Common.h | 0 .../Initializer.cpp | 137 +++++------------- .../Initializer.h | 21 +-- .../ProtocolInitializer.h | 0 cpp/wedpr-main/CMakeLists.txt | 13 ++ .../air-node/AirNodeInitializer.cpp | 36 +++-- .../air-node/AirNodeInitializer.h | 16 +- .../air-node/CMakeLists.txt | 0 .../pro-node => wedpr-main/air-node}/main.cpp | 33 +---- .../cem-node/CEMInitializer.cpp | 0 .../cem-node/CEMInitializer.h | 2 +- .../cem-node/CMakeLists.txt | 0 .../cem-node/main.cpp | 0 .../common/NodeStarter.h} | 33 +++-- .../gateway/CMakeLists.txt | 5 +- cpp/wedpr-main/gateway/GatewayInitializer.cpp | 83 +++++++++++ cpp/wedpr-main/gateway/GatewayInitializer.h | 49 +++++++ cpp/{ppc-main => wedpr-main}/gateway/main.cpp | 34 +---- .../mpc-node/CMakeLists.txt | 0 .../mpc-node/MPCInitializer.cpp | 0 .../mpc-node/MPCInitializer.h | 2 +- .../mpc-node/main.cpp | 0 .../pro-node/CMakeLists.txt | 3 +- .../pro-node/ProNodeInitializer.cpp | 92 ++++++++++++ cpp/wedpr-main/pro-node/ProNodeInitializer.h | 51 +++++++ cpp/wedpr-main/pro-node/main.cpp | 28 ++++ cpp/wedpr-protocol/grpc/Common.h | 37 +++++ .../grpc/client/FrontClient.cpp | 1 - cpp/wedpr-protocol/grpc/client/FrontClient.h | 6 +- .../grpc/client/GatewayClient.cpp | 1 + .../grpc/client/GatewayClient.h | 4 +- cpp/wedpr-protocol/grpc/client/GrpcClient.h | 9 +- .../grpc/client/RemoteFrontBuilder.cpp | 31 ++++ .../grpc/client/RemoteFrontBuilder.h} | 28 ++-- cpp/wedpr-protocol/grpc/server/GrpcServer.h | 8 +- .../ppc-front/ppc-front/Front.cpp | 9 +- .../ppc-front/ppc-front/Front.h | 11 +- .../ppc-front/ppc-front/FrontConfigImpl.h | 51 +++++++ .../ppc-front/ppc-front/FrontImpl.h | 2 +- .../ppc-front/ppc-front/LocalFrontBuilder.h | 40 +++++ cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp | 49 ------- cpp/wedpr-transport/ppc-rpc/src/Rpc.h | 1 - cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h | 2 - cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 18 ++- cpp/wedpr-transport/sdk/ProTransportImpl.h | 23 ++- cpp/wedpr-transport/sdk/TransportBuilder.cpp | 53 +++++++ cpp/wedpr-transport/sdk/TransportBuilder.h | 23 +-- 64 files changed, 825 insertions(+), 675 deletions(-) delete mode 100644 cpp/libinitializer/CMakeLists.txt delete mode 100644 cpp/libinitializer/FrontInitializer.cpp delete mode 100644 cpp/libinitializer/FrontInitializer.h delete mode 100644 cpp/libinitializer/ProFrontInitializer.h create mode 100644 cpp/ppc-framework/protocol/GrpcConfig.h delete mode 100644 cpp/ppc-main/CMakeLists.txt create mode 100644 cpp/wedpr-initializer/CMakeLists.txt rename cpp/{libinitializer => wedpr-initializer}/Common.h (100%) rename cpp/{libinitializer => wedpr-initializer}/Initializer.cpp (81%) rename cpp/{libinitializer => wedpr-initializer}/Initializer.h (84%) rename cpp/{libinitializer => wedpr-initializer}/ProtocolInitializer.h (100%) create mode 100644 cpp/wedpr-main/CMakeLists.txt rename cpp/{ppc-main => wedpr-main}/air-node/AirNodeInitializer.cpp (74%) rename cpp/{ppc-main => wedpr-main}/air-node/AirNodeInitializer.h (79%) rename cpp/{ppc-main => wedpr-main}/air-node/CMakeLists.txt (100%) rename cpp/{ppc-main/pro-node => wedpr-main/air-node}/main.cpp (54%) rename cpp/{ppc-main => wedpr-main}/cem-node/CEMInitializer.cpp (100%) rename cpp/{ppc-main => wedpr-main}/cem-node/CEMInitializer.h (97%) rename cpp/{ppc-main => wedpr-main}/cem-node/CMakeLists.txt (100%) rename cpp/{ppc-main => wedpr-main}/cem-node/main.cpp (100%) rename cpp/{ppc-main/air-node/main.cpp => wedpr-main/common/NodeStarter.h} (72%) rename cpp/{ppc-main => wedpr-main}/gateway/CMakeLists.txt (61%) create mode 100644 cpp/wedpr-main/gateway/GatewayInitializer.cpp create mode 100644 cpp/wedpr-main/gateway/GatewayInitializer.h rename cpp/{ppc-main => wedpr-main}/gateway/main.cpp (53%) rename cpp/{ppc-main => wedpr-main}/mpc-node/CMakeLists.txt (100%) rename cpp/{ppc-main => wedpr-main}/mpc-node/MPCInitializer.cpp (100%) rename cpp/{ppc-main => wedpr-main}/mpc-node/MPCInitializer.h (97%) rename cpp/{ppc-main => wedpr-main}/mpc-node/main.cpp (100%) rename cpp/{ppc-main => wedpr-main}/pro-node/CMakeLists.txt (74%) create mode 100644 cpp/wedpr-main/pro-node/ProNodeInitializer.cpp create mode 100644 cpp/wedpr-main/pro-node/ProNodeInitializer.h create mode 100644 cpp/wedpr-main/pro-node/main.cpp create mode 100644 cpp/wedpr-protocol/grpc/Common.h create mode 100644 cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp rename cpp/{ppc-framework/front/IFrontBuilder.h => wedpr-protocol/grpc/client/RemoteFrontBuilder.h} (58%) create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h create mode 100644 cpp/wedpr-transport/sdk/TransportBuilder.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c02bf0aa..dffbaea8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -88,7 +88,8 @@ set(ALL_SOURCE_LIST wedpr-storage/ppc-io wedpr-storage/ppc-storage wedpr-transport/ppc-gateway wedpr-transport/ppc-front wedpr-transport/ppc-http wedpr-transport/ppc-rpc wedpr-transport/sdk - wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE}) + wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE} + wedpr-initializer wedpr-main) if(BUILD_WEDPR_TOOLKIT) # fetch the python dependencies diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index b72cec18..1b30601a 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -63,8 +63,8 @@ set(RPC_TARGET "ppc-rpc") # libhelper set(HELPER_TARGET "ppc-helper") -# libinitializer -set(INIT_LIB init) +# wedpr-initializer +set(INIT_LIB "wedpr-inititializer") # ppc-cem set(CEM_TARGET "ppc-cem") diff --git a/cpp/libinitializer/CMakeLists.txt b/cpp/libinitializer/CMakeLists.txt deleted file mode 100644 index 7fec7d12..00000000 --- a/cpp/libinitializer/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -file(GLOB SRC_LIST "*.cpp") - -add_library(${INIT_LIB} ${SRC_LIST}) -target_link_libraries(${INIT_LIB} PUBLIC ${PROTOCOL_TARGET} ${CRYPTO_TARGET} ${FRONT_TARGET} ${LABELED_PSI_TARGET} ${RA2018_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${ECDH_MULTI_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${BS_ECDH_PSI_TARGET} ${PIR_TARGET} ${IO_TARGET} ${STORAGE_TARGET}) diff --git a/cpp/libinitializer/FrontInitializer.cpp b/cpp/libinitializer/FrontInitializer.cpp deleted file mode 100644 index 6cbd6702..00000000 --- a/cpp/libinitializer/FrontInitializer.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontInitializer.cpp - * @author: shawnhe - * @date 2022-10-20 - */ - -#include "FrontInitializer.h" - -using namespace ppctars; -using namespace ppc; -using namespace ppc::front; -using namespace ppc::protocol; -using namespace ppc::initializer; - -void FrontInitializer::start() -{ - if (!m_running.exchange(true)) - { - FRONT_LOG(INFO) << LOG_DESC("start the front"); - if (m_front) - { - m_front->start(); - } - if (m_gatewayReporter) - { - m_gatewayReporter->registerTimeoutHandler( - boost::bind(&FrontInitializer::reportOtherGateway, this)); - m_gatewayReporter->start(); - } - FRONT_LOG(INFO) << LOG_DESC("start the front success"); - } -} - -void FrontInitializer::stop() -{ - if (m_running.exchange(false)) - { - FRONT_LOG(INFO) << LOG_DESC("stop the front"); - if (m_front) - { - m_front->stop(); - } - if (m_gatewayReporter) - { - m_gatewayReporter->stop(); - } - FRONT_LOG(INFO) << LOG_DESC("stop the front success"); - } -} - -void FrontInitializer::init() -{ - FRONT_LOG(INFO) << LOG_BADGE("init front"); - - auto ioService = std::make_shared(); - // init front - m_frontFactory = std::make_shared(m_selfAgencyId, m_threadPool); - m_front = m_frontFactory->buildFront(ioService); - FRONT_LOG(INFO) << LOG_BADGE("init front success"); -} diff --git a/cpp/libinitializer/FrontInitializer.h b/cpp/libinitializer/FrontInitializer.h deleted file mode 100644 index f00491e2..00000000 --- a/cpp/libinitializer/FrontInitializer.h +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file FrontInitializer.h - * @author: shawnhe - * @date 2022-10-20 - */ - -#pragma once - -#include "ppc-framework/front/Channel.h" -#include "ppc-framework/front/FrontInterface.h" -#include "ppc-framework/protocol/PPCMessageFace.h" -#include "ppc-framework/rpc/RpcStatusInterface.h" -#include "ppc-front/ppc-front/Front.h" -#include "ppc-front/ppc-front/PPCChannelManager.h" -#include -#include - -namespace ppc -{ -namespace initializer -{ -class FrontInitializer -{ -public: - using Ptr = std::shared_ptr; - - explicit FrontInitializer(const std::string& _selfAgencyId, - std::shared_ptr _threadPool, - front::PPCMessageFaceFactory::Ptr _messageFactory) - : m_selfAgencyId(_selfAgencyId), - m_threadPool(std::move(_threadPool)), - m_messageFactory(std::move(_messageFactory)), - m_gatewayReporter(std::make_shared(3000, "gatewayReporter")) - { - init(); - } - - virtual ~FrontInitializer() { stop(); } - - virtual void start(); - virtual void stop(); - - const std::string& selfAgencyId() const { return m_selfAgencyId; } - std::shared_ptr threadPool() { return m_threadPool; } - - front::Front::Ptr front() { return m_front; } - - front::FrontFactory::Ptr frontFactory() { return m_frontFactory; } - - front::PPCMessageFaceFactory::Ptr messageFactory() { return m_messageFactory; } - - void setRpcStatus(rpc::RpcStatusInterface::Ptr _status) { m_rpcStatus = std::move(_status); } - -protected: - void init(); - // report the gateway endpoints of other parties to the gateway periodically - // Note: must set gatewayInterface into the front before call this function - // Note: must set RpcStorageInterface into the FrontInitializer before call this function - virtual void reportOtherGateway() - { - try - { - auto gatewayList = m_rpcStatus->listGateway(); - if (!gatewayList.empty()) - { - m_front->gatewayInterface()->registerGateway(gatewayList); - } - } - catch (std::exception const& e) - { - FRONT_LOG(WARNING) << LOG_DESC("reportOtherGateway exception") - << LOG_KV("exception", boost::diagnostic_information(e)); - } - m_gatewayReporter->restart(); - } - -protected: - std::string m_selfAgencyId; - std::shared_ptr m_threadPool; - front::PPCMessageFaceFactory::Ptr m_messageFactory; - - front::Front::Ptr m_front; - front::FrontFactory::Ptr m_frontFactory; - - std::atomic_bool m_running = {false}; - std::shared_ptr m_gatewayReporter; - rpc::RpcStatusInterface::Ptr m_rpcStatus; -}; -} // namespace initializer -} // namespace ppc \ No newline at end of file diff --git a/cpp/libinitializer/ProFrontInitializer.h b/cpp/libinitializer/ProFrontInitializer.h deleted file mode 100644 index 8355d968..00000000 --- a/cpp/libinitializer/ProFrontInitializer.h +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @file ProFrontInitializer.h - * @author: yujiechen - * @date 2022-11-25 - */ -#pragma once -#include "FrontInitializer.h" -#include "ppc-framework/gateway/GatewayInterface.h" -#include - -namespace ppc::initializer -{ -class ProFrontInitializer : public FrontInitializer -{ -public: - using Ptr = std::shared_ptr; - ProFrontInitializer(const std::string& _selfAgencyId, - std::shared_ptr _threadPool, - front::PPCMessageFaceFactory::Ptr _messageFactory) - : FrontInitializer(_selfAgencyId, _threadPool, _messageFactory), - m_statusReporter(std::make_shared(3000, "frontReporter")) - {} - ~ProFrontInitializer() override { stop(); } - - void start() override - { - FrontInitializer::start(); - if (m_statusReporter) - { - m_statusReporter->registerTimeoutHandler( - boost::bind(&ProFrontInitializer::reportStatusToGateway, this)); - m_statusReporter->start(); - } - } - - void stop() override - { - if (m_statusReporter) - { - m_statusReporter->stop(); - } - FrontInitializer::stop(); - } - -protected: - // report the endPoint to the gateway periodically - // Note: must set gatewayInterface into the front before call this function - virtual void reportStatusToGateway() - { - try - { - m_front->gatewayInterface()->registerFront(m_front->selfEndPoint(), nullptr); - } - catch (std::exception const& e) - { - FRONT_LOG(WARNING) << LOG_DESC("reportStatusToGateway exception") - << LOG_KV("exception", boost::diagnostic_information(e)); - } - m_statusReporter->restart(); - } - -private: - std::shared_ptr m_statusReporter; -}; -} // namespace ppc::initializer diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h index 6cee462a..affe5d9b 100644 --- a/cpp/ppc-framework/front/FrontConfig.h +++ b/cpp/ppc-framework/front/FrontConfig.h @@ -20,7 +20,10 @@ #pragma once #include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/GrpcConfig.h" +#include "ppc-framework/protocol/INodeInfo.h" #include +#include #include #include @@ -49,40 +52,58 @@ class FrontConfig virtual void appendGatewayInfo(ppc::protocol::EndPoint&& endpoint) { - // TODO:check the endpoint m_gatewayInfo.push_back(endpoint); } ppc::protocol::EndPoint const& selfEndPoint() const { return m_selfEndPoint; } void setSelfEndPoint(ppc::protocol::EndPoint const& endPoint) { m_selfEndPoint = endPoint; } - // TODO here - std::string gatewayEndPoints() { return ""; } + // refer to: https://github.com/grpc/grpc-node/issues/2066 + // grpc prefer to using ipv4:${host1}:${port1},${host2}:${port2} as target to support multiple + // servers + std::string gatewayGrpcTarget() + { + std::stringstream oss; + oss << "ipv4:"; + for (auto const& endPoint : m_gatewayInfo) + { + oss << endPoint.entryPoint() << ","; + } + return oss.str(); + } - std::string const& loadBalancePolicy() const { return m_loadBanlancePolicy; } - void setLoadBalancePolicy(std::string const& loadBanlancePolicy) + void setGrpcConfig(ppc::protocol::GrpcConfig::Ptr grpcConfig) { - m_loadBanlancePolicy = loadBanlancePolicy; + m_grpcConfig = std::move(grpcConfig); } + ppc::protocol::GrpcConfig::Ptr const& grpcConfig() const { return m_grpcConfig; } + + // generate the nodeInfo + virtual ppc::protocol::INodeInfo::Ptr generateNodeInfo() const = 0; + + virtual std::vector const& getComponents() const { return m_components; } + void setComponents(std::vector const& components) { m_components = components; } -private: - std::string m_loadBanlancePolicy = "round_robin"; +protected: + ppc::protocol::GrpcConfig::Ptr m_grpcConfig; ppc::protocol::EndPoint m_selfEndPoint; int m_threadPoolSize; std::string m_nodeID; std::vector m_gatewayInfo; + std::vector m_components; }; -class FrontConfigBuilder +inline std::string printFrontDesc(FrontConfig::Ptr const& config) { -public: - using Ptr = std::shared_ptr; - FrontConfigBuilder() = default; - virtual ~FrontConfigBuilder() = default; - - FrontConfig::Ptr build(int threadPoolSize, std::string nodeID) + if (!config) { - return std::make_shared(threadPoolSize, nodeID); + return "nullptr"; } -}; + std::ostringstream stringstream; + stringstream << LOG_KV("endPoint", config->selfEndPoint().entryPoint()) + << LOG_KV("nodeID", config->nodeID()) + << LOG_KV("poolSize", config->threadPoolSize()) + << LOG_KV("target", config->gatewayGrpcTarget()); + return stringstream.str(); +} } // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/front/FrontInterface.h b/cpp/ppc-framework/front/FrontInterface.h index e7648725..ff019e22 100644 --- a/cpp/ppc-framework/front/FrontInterface.h +++ b/cpp/ppc-framework/front/FrontInterface.h @@ -60,6 +60,8 @@ class FrontInterface virtual void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) = 0; + virtual void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, + std::function _handler) = 0; /** * @brief notice task info to gateway * @param _taskInfo the latest task information @@ -69,9 +71,6 @@ class FrontInterface // erase the task-info when task finished virtual bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) = 0; - // get the agencyList from the gateway - virtual void asyncGetAgencyList(GetAgencyListCallback _callback) = 0; - virtual std::string const& selfEndPoint() const { return m_selfEndPoint; } protected: diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index b5f94a2d..8de9ab09 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -40,7 +40,7 @@ class IFrontClient virtual void onReceiveMessage( ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) = 0; }; -class IFront : public virtual IFrontClient +class IFront : virtual public IFrontClient { public: using Ptr = std::shared_ptr; @@ -132,13 +132,6 @@ class IFrontBuilder IFrontBuilder() = default; virtual ~IFrontBuilder() = default; - /** - * @brief create the Front using specified config - * - * @param config the config used to build the Front - * @return IFront::Ptr he created Front - */ - virtual IFront::Ptr build(ppc::front::FrontConfig::Ptr config) const = 0; virtual IFrontClient::Ptr buildClient(std::string endPoint) const = 0; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h new file mode 100644 index 00000000..76b2ac17 --- /dev/null +++ b/cpp/ppc-framework/protocol/GrpcConfig.h @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcConfig.h + * @author: yujiechen + * @date 2024-09-02 + */ +#pragma once +#include "ppc-framework/protocol/EndPoint.h" +#include +#include + +namespace ppc::protocol +{ +struct GrpcServerConfig +{ + ppc::protocol::EndPoint endPoint; + + std::string listenEndPoint() const { return endPoint.listenEndPoint(); } +}; +class GrpcConfig +{ +public: + using Ptr = std::shared_ptr; + GrpcConfig() = default; + virtual ~GrpcConfig() = default; + + std::string const& loadBalancePolicy() const { return m_loadBanlancePolicy; } + void setLoadBalancePolicy(std::string const& loadBanlancePolicy) + { + m_loadBanlancePolicy = loadBanlancePolicy; + } + +private: + std::string m_loadBanlancePolicy = "round_robin"; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/rpc/RpcStatusInterface.h b/cpp/ppc-framework/rpc/RpcStatusInterface.h index 441a08a2..5a5b18d2 100644 --- a/cpp/ppc-framework/rpc/RpcStatusInterface.h +++ b/cpp/ppc-framework/rpc/RpcStatusInterface.h @@ -31,7 +31,6 @@ namespace ppc::rpc { - class RpcStatusInterface { public: @@ -45,8 +44,6 @@ class RpcStatusInterface virtual bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) = 0; virtual bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) = 0; virtual protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) = 0; - virtual bcos::Error::Ptr insertGateway( - const std::string& _agencyID, const std::string& _endpoint) = 0; virtual bcos::Error::Ptr deleteGateway(const std::string& _agencyID) = 0; virtual std::vector listGateway() = 0; }; diff --git a/cpp/ppc-framework/rpc/RpcTypeDef.h b/cpp/ppc-framework/rpc/RpcTypeDef.h index 3be4ee05..5a0fb493 100644 --- a/cpp/ppc-framework/rpc/RpcTypeDef.h +++ b/cpp/ppc-framework/rpc/RpcTypeDef.h @@ -36,7 +36,6 @@ enum class RpcError : int32_t std::string const RUN_TASK_METHOD = "runTask"; std::string const ASYNC_RUN_TASK_METHOD = "asyncRunTask"; std::string const GET_TASK_STATUS = "getTaskStatus"; -std::string const REGISTER_GATEWAY_URL = "registerGatewayUrl"; std::string const ASYNC_RUN_BS_MODE_TASK = "asyncRunBsModeTask"; std::string const FETCH_CIPHER = "fetchCipher"; diff --git a/cpp/ppc-main/CMakeLists.txt b/cpp/ppc-main/CMakeLists.txt deleted file mode 100644 index f73c486d..00000000 --- a/cpp/ppc-main/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -project(ppc-main VERSION ${VERSION}) - -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) -add_subdirectory(air-node) - -if (BUILD_CEM) - add_subdirectory(cem-node) -endif () -add_subdirectory(mpc-node) - -macro(compile_service SERVICE_SOURCE_PATH BINARY_NAME PKG_NAME) - add_subdirectory(${SERVICE_SOURCE_PATH}) - add_custom_command(OUTPUT ${PKG_NAME}.tgz - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/ppc-main/${SERVICE_SOURCE_PATH}/${BINARY_NAME} ${CMAKE_BINARY_DIR}/${PKG_NAME}/${PKG_NAME} - COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_BINARY_DIR} tar czfv ${PKG_NAME}.tgz ${PKG_NAME}/${PKG_NAME} - COMMAND ${CMAKE_COMMAND} -E copy ${PKG_NAME}.tgz tars/${PKG_NAME}.tgz - COMMENT "Compressing ${BINARY_NAME} into ${PKG_NAME}.tgz ..." - COMMAND ${CMAKE_COMMAND} -E rm ${PKG_NAME}.tgz - ) - add_custom_target(${PKG_NAME}-tar DEPENDS ${PKG_NAME}.tgz ${BINARY_NAME}) - list(APPEND SERVICE_TAR_LIST ${PKG_NAME}-tar) -endmacro() - -# gateway -# Note: the tar-pkg is only useful when using tars-admin -compile_service("gateway" ${GATEWAY_BINARY_NAME} ${GATEWAY_PKG_NAME}) -# ppc-node -compile_service("pro-node" ${PRO_BINARY_NAME} ${PPC_NODE_PKG_NAME}) - -# for make tar -add_custom_target(tar DEPENDS ${SERVICE_TAR_LIST}) - -unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h index b7cdc97d..0b54380e 100644 --- a/cpp/test-utils/FakeFront.h +++ b/cpp/test-utils/FakeFront.h @@ -202,7 +202,6 @@ class FakeFront : public FrontInterface } } - void asyncGetAgencyList(GetAgencyListCallback) override {} private: // the uuid to _callback diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h index 41f83430..7ee7dd80 100644 --- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h +++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h @@ -114,16 +114,6 @@ class PSIConfig int taskExpireTime() const { return m_taskExpireTime; } void setTaskExpireTime(int _taskExpireTime) { m_taskExpireTime = _taskExpireTime; } - void updateAgenyList(std::vector const& _agencyList) - { - bcos::UpgradableGuard l(x_agencyList); - if (m_agencyList != _agencyList) - { - bcos::UpgradeGuard ul(l); - m_agencyList = _agencyList; - } - } - std::vector agencyList() const { bcos::ReadGuard l(x_agencyList); @@ -165,6 +155,7 @@ class PSIConfig int m_taskExpireTime = 10000; // the agency list, for task-sync + // TODO: fetch from the gateway std::vector m_agencyList; mutable bcos::SharedMutex x_agencyList; }; diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h index 76aac8b0..f5b70e75 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h @@ -22,6 +22,8 @@ #include "Common.h" #include "NetworkConfig.h" #include "ParamChecker.h" +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/GrpcConfig.h" #include "ppc-framework/storage/CacheStorage.h" #include #include @@ -98,6 +100,7 @@ struct GatewayConfig bool disableCache; NetworkConfig networkConfig; + ppc::protocol::GrpcServerConfig grpcConfig; ppc::storage::CacheStorageConfig cacheStorageConfig; std::string nodeFileName; std::string nodePath; @@ -244,6 +247,10 @@ class PPCConfig int holdingMessageMinutes() const { return m_holdingMessageMinutes; } + ppc::front::FrontConfig::Ptr const& frontConfig() const { return m_frontConfig; } + + ppc::protocol::GrpcConfig::Ptr const& grpcConfig() const { return m_grpcConfig; } + private: virtual void loadRA2018Config(boost::property_tree::ptree const& _pt); virtual void loadEcdhPSIConfig(boost::property_tree::ptree const& _pt); @@ -283,6 +290,11 @@ class PPCConfig // the gateway holding message time, in minutes, default 30min int m_holdingMessageMinutes = 30; + // the front config + // TODO: parse the frontConfig + ppc::front::FrontConfig::Ptr m_frontConfig; + ppc::protocol::GrpcConfig::Ptr m_grpcConfig; + // the ra2018-psi config RA2018Config m_ra2018PSIConfig; // the storage config diff --git a/cpp/wedpr-initializer/CMakeLists.txt b/cpp/wedpr-initializer/CMakeLists.txt new file mode 100644 index 00000000..633aaf0b --- /dev/null +++ b/cpp/wedpr-initializer/CMakeLists.txt @@ -0,0 +1,7 @@ +file(GLOB SRC_LIST "*.cpp") + +add_library(${INIT_LIB} ${SRC_LIST}) +target_link_libraries(${INIT_LIB} PUBLIC + ${PROTOCOL_TARGET} ${CRYPTO_TARGET} ${WEDPR_TRANSPORT_SDK_TARGET} ${LABELED_PSI_TARGET} + ${RA2018_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} + ${ECDH_MULTI_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${BS_ECDH_PSI_TARGET} ${PIR_TARGET} ${IO_TARGET} ${STORAGE_TARGET}) diff --git a/cpp/libinitializer/Common.h b/cpp/wedpr-initializer/Common.h similarity index 100% rename from cpp/libinitializer/Common.h rename to cpp/wedpr-initializer/Common.h diff --git a/cpp/libinitializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp similarity index 81% rename from cpp/libinitializer/Initializer.cpp rename to cpp/wedpr-initializer/Initializer.cpp index cb5d68f7..0bd52833 100644 --- a/cpp/libinitializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -19,7 +19,6 @@ */ #include "Initializer.h" #include "Common.h" -#include "ProFrontInitializer.h" #include "ppc-crypto/src/ecc/ECDHCryptoFactoryImpl.h" #include "ppc-crypto/src/oprf/RA2018Oprf.h" #include "ppc-framework/protocol/Protocol.h" @@ -28,10 +27,12 @@ #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h" #include "ppc-psi/src/cm2020-psi/CM2020PSIFactory.h" #include "protocol/src/PPCMessage.h" +#include "wedpr-transport/sdk/TransportBuilder.h" #if 0 //TODO: optimize here #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h" #endif +#include "ppc-front/Front.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h" #include "ppc-psi/src/ecdh-psi/EcdhPSIFactory.h" #include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h" @@ -46,6 +47,7 @@ using namespace ppc::psi; using namespace ppc::pir; using namespace ppc::tools; using namespace ppc::crypto; +using namespace ppc::sdk; Initializer::Initializer(std::string const& _configPath) : m_configPath(_configPath) { @@ -54,7 +56,7 @@ Initializer::Initializer(std::string const& _configPath) : m_configPath(_configP m_config->loadConfig(_configPath); } -void Initializer::init(ppc::protocol::NodeArch _arch) +void Initializer::init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Ptr const& gateway) { INIT_LOG(INFO) << LOG_DESC("init the wedpr-component") << LOG_KV("arch", _arch); // load the protocol @@ -67,31 +69,20 @@ void Initializer::init(ppc::protocol::NodeArch _arch) auto frontThreadPool = std::make_shared("front", m_config->threadPoolSize()); // Note: must set the m_holdingMessageMinutes before init the node + TransportBuilder transportBuilder; if (_arch == ppc::protocol::NodeArch::AIR) { - m_frontInitializer = std::make_shared( - m_config->agencyID(), frontThreadPool, m_protocolInitializer->ppcMsgFactory()); - // load the gateway config - m_config->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, m_configPath); + m_transport = transportBuilder.build(SDKMode::AIR, m_config->frontConfig(), gateway); } else { - m_frontInitializer = std::make_shared( - m_config->agencyID(), frontThreadPool, m_protocolInitializer->ppcMsgFactory()); - m_agencyInfoFetcher = std::make_shared(3000, "agencyInfoFetcher"); - // Note: the timer start work only after calling start - auto self = weak_from_this(); - m_agencyInfoFetcher->registerTimeoutHandler([self]() { - auto init = self.lock(); - if (!init) - { - return; - } - init->fetchAgencyListPeriodically(); - }); + m_transport = transportBuilder.build(SDKMode::PRO, m_config->frontConfig(), nullptr); } + m_ppcFront = std::make_shared(m_transport->getFront()); + INIT_LOG(INFO) << LOG_DESC("init the frontService success") - << LOG_KV("agency", m_config->agencyID()) << LOG_KV("arch", _arch); + << LOG_KV("frontDetail", printFrontDesc(m_config->frontConfig())) + << LOG_KV("arch", _arch); auto cryptoBox = m_protocolInitializer->cryptoBox(); SQLStorage::Ptr sqlStorage = nullptr; @@ -123,9 +114,9 @@ void Initializer::init(ppc::protocol::NodeArch _arch) auto ra2018PSIFactory = std::make_shared(); auto oprf = std::make_shared( m_config->privateKey(), cryptoBox->eccCrypto(), cryptoBox->hashImpl()); - m_ra2018PSI = ra2018PSIFactory->createRA2018PSI(m_config->agencyID(), - m_frontInitializer->front(), m_config, oprf, m_protocolInitializer->binHashImpl(), - m_protocolInitializer->ppcMsgFactory(), sqlStorage, fileStorage, + m_ra2018PSI = ra2018PSIFactory->createRA2018PSI(m_config->agencyID(), m_ppcFront, m_config, + oprf, m_protocolInitializer->binHashImpl(), m_protocolInitializer->ppcMsgFactory(), + sqlStorage, fileStorage, std::make_shared("ra2018", m_config->threadPoolSize()), m_protocolInitializer->dataResourceLoader()); INIT_LOG(INFO) << LOG_DESC("init the ra2018-psi success"); @@ -133,8 +124,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch) // init the labeled-psi INIT_LOG(INFO) << LOG_DESC("init the labeled-psi"); auto labeledPSIFactory = std::make_shared(); - m_labeledPSI = labeledPSIFactory->buildLabeledPSI(m_config->agencyID(), - m_frontInitializer->front(), cryptoBox, + m_labeledPSI = labeledPSIFactory->buildLabeledPSI(m_config->agencyID(), m_ppcFront, cryptoBox, std::make_shared("t_labeled-psi", m_config->threadPoolSize()), m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes()); INIT_LOG(INFO) << LOG_DESC("init the labeled-psi success"); @@ -142,8 +132,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch) // init the cm2020-psi INIT_LOG(INFO) << LOG_DESC("init the cm2020-psi"); auto cm2020PSIFactory = std::make_shared(); - m_cm2020PSI = cm2020PSIFactory->buildCM2020PSI(m_config->agencyID(), - m_frontInitializer->front(), cryptoBox, + m_cm2020PSI = cm2020PSIFactory->buildCM2020PSI(m_config->agencyID(), m_ppcFront, cryptoBox, std::make_shared("t_cm2020-psi", m_config->threadPoolSize()), m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes(), m_config->cm2020PSIConfig().parallelism); @@ -153,8 +142,8 @@ void Initializer::init(ppc::protocol::NodeArch _arch) INIT_LOG(INFO) << LOG_DESC("create ecdh-psi"); auto ecdhPSIFactory = std::make_shared(); auto ecdhCryptoFactory = std::make_shared(m_config->privateKey()); - m_ecdhPSI = ecdhPSIFactory->createEcdhPSI(m_config, ecdhCryptoFactory, - m_frontInitializer->front(), m_protocolInitializer->ppcMsgFactory(), nullptr, + m_ecdhPSI = ecdhPSIFactory->createEcdhPSI(m_config, ecdhCryptoFactory, m_ppcFront, + m_protocolInitializer->ppcMsgFactory(), nullptr, m_protocolInitializer->dataResourceLoader()); INIT_LOG(INFO) << LOG_DESC("create ecdh-psi success"); @@ -164,7 +153,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch) INIT_LOG(INFO) << LOG_DESC("create ecdh-conn-psi"); auto ecdhConnPSIFactory = std::make_shared(); m_ecdhConnPSI = ecdhConnPSIFactory->createEcdhConnPSI(m_config, ecdhCryptoFactory, - m_frontInitializer->front(), m_protocolInitializer->ppcMsgFactory(), + m_ppcFront, m_protocolInitializer->ppcMsgFactory(), std::make_shared("t_ecdh-conn-psi", std::thread::hardware_concurrency()), m_protocolInitializer->dataResourceLoader()); INIT_LOG(INFO) << LOG_DESC("create ecdh-conn-psi success"); @@ -172,8 +161,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch) // init the ecdh-multi-psi INIT_LOG(INFO) << LOG_DESC("create ecdh-multi-psi"); auto ecdhMultiPSIFactory = std::make_shared(); - m_ecdhMultiPSI = ecdhMultiPSIFactory->createEcdhMultiPSI(m_config, m_frontInitializer->front(), - cryptoBox, + m_ecdhMultiPSI = ecdhMultiPSIFactory->createEcdhMultiPSI(m_config, m_ppcFront, cryptoBox, std::make_shared("t_ecdh-multi-psi", std::thread::hardware_concurrency()), m_protocolInitializer->dataResourceLoader()); INIT_LOG(INFO) << LOG_DESC("create ecdh-multi-psi success"); @@ -181,7 +169,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch) // init the ot-pir INIT_LOG(INFO) << LOG_DESC("create ot-pir"); auto otPIRFactory = std::make_shared(); - m_otPIR = otPIRFactory->buildOtPIR(m_config->agencyID(), m_frontInitializer->front(), cryptoBox, + m_otPIR = otPIRFactory->buildOtPIR(m_config->agencyID(), m_ppcFront, cryptoBox, std::make_shared("t_ot-pir", std::thread::hardware_concurrency()), m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes()); @@ -208,7 +196,7 @@ void Initializer::initMsgHandlers() INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ra2018PSI"); // register msg-handlers for ra2018-psi auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::RA_PSI_2PC, [weakRA2018PSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakRA2018PSI.lock(); @@ -222,7 +210,7 @@ void Initializer::initMsgHandlers() // register msg-handlers for labeled-psi INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for labeledPSI"); auto weakLabeledPSI = std::weak_ptr(m_labeledPSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC, [weakLabeledPSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakLabeledPSI.lock(); @@ -236,7 +224,7 @@ void Initializer::initMsgHandlers() // register msg-handlers for cm2020-psi INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for CM2020PSI"); auto weakCM2020PSI = std::weak_ptr(m_cm2020PSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::CM_PSI_2PC, [weakCM2020PSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakCM2020PSI.lock(); @@ -250,7 +238,7 @@ void Initializer::initMsgHandlers() INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-psi"); // register msg-handlers for ecdh-psi auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC, [weakEcdhPSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakEcdhPSI.lock(); @@ -264,7 +252,7 @@ void Initializer::initMsgHandlers() // register msg-handlers for ecdh-conn-psi /*INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-conn-psi"); auto weakEcdhConnPSI = std::weak_ptr(m_ecdhConnPSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN, [weakEcdhConnPSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakEcdhConnPSI.lock(); @@ -278,7 +266,7 @@ void Initializer::initMsgHandlers() // register msg-handlers for ecdh-multi-psi INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-multi-psi"); auto weakEcdhMultiPSI = std::weak_ptr(m_ecdhMultiPSI); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI, (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI, [weakEcdhMultiPSI](ppc::front::PPCMessageFace::Ptr _msg) { auto psi = weakEcdhMultiPSI.lock(); @@ -292,7 +280,7 @@ void Initializer::initMsgHandlers() INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ot-pir"); // register msg-handlers for ecdh-psi auto weakOtPIR = std::weak_ptr(m_otPIR); - m_frontInitializer->front()->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PIR, + m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PIR, (uint8_t)ppc::protocol::PSIAlgorithmType::OT_PIR_2PC, [weakOtPIR](ppc::front::PPCMessageFace::Ptr _msg) { auto pir = weakOtPIR.lock(); @@ -414,13 +402,9 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc) void Initializer::start() { - if (m_agencyInfoFetcher) - { - m_agencyInfoFetcher->start(); - } - if (m_frontInitializer) + if (m_transport) { - m_frontInitializer->start(); + m_transport->start(); } /*if (m_ecdhConnPSI) { @@ -460,14 +444,10 @@ void Initializer::start() void Initializer::stop() { - if (m_agencyInfoFetcher) - { - m_agencyInfoFetcher->stop(); - } // stop the network firstly - if (m_frontInitializer) + if (m_transport) { - m_frontInitializer->stop(); + m_transport->stop(); } /*if (m_ecdhConnPSI) { @@ -481,14 +461,14 @@ void Initializer::stop() { m_cm2020PSI->stop(); } - // if (m_ra2018PSI) - // { - // m_ra2018PSI->stop(); - // } - // if (m_labeledPSI) - // { - // m_labeledPSI->stop(); - // } + if (m_ra2018PSI) + { + m_ra2018PSI->stop(); + } + if (m_labeledPSI) + { + m_labeledPSI->stop(); + } // stop the ecdh-psi if (m_ecdhPSI) { @@ -503,40 +483,3 @@ void Initializer::stop() m_bsEcdhPSI->stop(); } } - -void Initializer::fetchAgencyListPeriodically() -{ - if (!m_agencyInfoFetcher) - { - return; - } - fetchAgencyList(); - m_agencyInfoFetcher->restart(); -} - -// fetch the agency-list for ecdh and ra2018 periodically in pro-mode -void Initializer::fetchAgencyList() -{ - auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI); - auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI); - m_frontInitializer->front()->asyncGetAgencyList( - [weakEcdhPSI, weakRA2018PSI]( - bcos::Error::Ptr _error, std::vector&& _agencyList) { - if (_error) - { - INIT_LOG(INFO) << LOG_DESC("asyncGetAgencyList failed") - << LOG_KV("code", _error->errorCode()); - return; - } - auto ecdhPsi = weakEcdhPSI.lock(); - if (ecdhPsi) - { - ecdhPsi->psiConfig()->updateAgenyList(_agencyList); - } - auto ra2018Psi = weakRA2018PSI.lock(); - if (ra2018Psi) - { - ra2018Psi->psiConfig()->updateAgenyList(_agencyList); - } - }); -} \ No newline at end of file diff --git a/cpp/libinitializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h similarity index 84% rename from cpp/libinitializer/Initializer.h rename to cpp/wedpr-initializer/Initializer.h index e5424733..78eba67d 100644 --- a/cpp/libinitializer/Initializer.h +++ b/cpp/wedpr-initializer/Initializer.h @@ -18,15 +18,18 @@ * @date 2022-11-14 */ #pragma once -#include "FrontInitializer.h" #include "ProtocolInitializer.h" +#include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/gateway/IGateway.h" #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/rpc/RpcTypeDef.h" #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h" #include "ppc-tools/src/config/PPCConfig.h" +#include "wedpr-transport/sdk/Transport.h" #include #include + namespace ppc::psi { class RA2018PSIImpl; @@ -51,31 +54,31 @@ class Initializer : public std::enable_shared_from_this virtual ~Initializer() { stop(); } // init the service - virtual void init(ppc::protocol::NodeArch _arch); + virtual void init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Ptr const& gateway); virtual void stop(); virtual void start(); ppc::tools::PPCConfig::Ptr config() { return m_config; } - FrontInitializer::Ptr const& frontInitializer() const { return m_frontInitializer; } + ppc::sdk::Transport::Ptr const& transport() const { return m_transport; } + ppc::front::FrontInterface::Ptr const& ppcFront() const { return m_ppcFront; } + ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } ProtocolInitializer::Ptr const& protocolInitializer() const { return m_protocolInitializer; } ppc::psi::BsEcdhPSIImpl::Ptr const& bsEcdhPsi() const { return m_bsEcdhPSI; } void registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc); - virtual void fetchAgencyList(); - protected: virtual void initMsgHandlers(); - void fetchAgencyListPeriodically(); + private: std::string m_configPath; ppc::tools::PPCConfig::Ptr m_config; ProtocolInitializer::Ptr m_protocolInitializer; - FrontInitializer::Ptr m_frontInitializer; - // timer to fetch all agency information - std::shared_ptr m_agencyInfoFetcher; + ppc::sdk::Transport::Ptr m_transport; + // created with transport + ppc::front::FrontInterface::Ptr m_ppcFront; // the ra2018-psi std::shared_ptr m_ra2018PSI; diff --git a/cpp/libinitializer/ProtocolInitializer.h b/cpp/wedpr-initializer/ProtocolInitializer.h similarity index 100% rename from cpp/libinitializer/ProtocolInitializer.h rename to cpp/wedpr-initializer/ProtocolInitializer.h diff --git a/cpp/wedpr-main/CMakeLists.txt b/cpp/wedpr-main/CMakeLists.txt new file mode 100644 index 00000000..44e9a218 --- /dev/null +++ b/cpp/wedpr-main/CMakeLists.txt @@ -0,0 +1,13 @@ +project(ppc-main VERSION ${VERSION}) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +add_subdirectory(air-node) +add_subdirectory(pro-node) +add_subdirectory(gateway) + +if (BUILD_CEM) + add_subdirectory(cem-node) +endif () +add_subdirectory(mpc-node) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/ppc-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp similarity index 74% rename from cpp/ppc-main/air-node/AirNodeInitializer.cpp rename to cpp/wedpr-main/air-node/AirNodeInitializer.cpp index 7bad21b7..82ee59ac 100644 --- a/cpp/ppc-main/air-node/AirNodeInitializer.cpp +++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp @@ -18,13 +18,13 @@ * @date 2022-11-14 */ #include "AirNodeInitializer.h" -#include "ppc-gateway/Gateway.h" -#include "ppc-gateway/GatewayConfigContext.h" +#include "ppc-front/LocalFrontBuilder.h" +#include "ppc-gateway/GatewayFactory.h" #include "ppc-rpc/src/RpcFactory.h" #include "ppc-rpc/src/RpcMemory.h" -#include "ppc-storage/src/redis/RedisStorage.h" using namespace ppc::protocol; +using namespace ppc::front; using namespace ppc::node; using namespace ppc::gateway; using namespace ppc::rpc; @@ -32,6 +32,10 @@ using namespace ppc::storage; using namespace ppc::initializer; using namespace bcos; +AirNodeInitializer::AirNodeInitializer() +{ + m_frontBuilder = std::make_shared(); +} void AirNodeInitializer::init(std::string const& _configPath) { // init the log @@ -44,31 +48,30 @@ void AirNodeInitializer::init(std::string const& _configPath) // init the node m_nodeInitializer = std::make_shared(_configPath); - m_nodeInitializer->init(ppc::protocol::NodeArch::AIR); - auto front = m_nodeInitializer->frontInitializer()->front(); - front->setSelfEndPoint("localhost"); // init the gateway initGateway(_configPath); - // set the gateway into front - front->setGatewayInterface(m_gateway); + // init the node + m_nodeInitializer->init(ppc::protocol::NodeArch::AIR, m_gateway); + // set the created front to the builder + m_frontBuilder->setFront(m_nodeInitializer->transport()->getFront()); + // register the NodeInfo + m_gateway->registerNodeInfo(m_nodeInitializer->config()->frontConfig()->generateNodeInfo()); INIT_LOG(INFO) << LOG_DESC("init the rpc"); // load the rpc config // not specify the certPath in air-mode m_nodeInitializer->config()->loadRpcConfig(nullptr, pt); // init RpcStatusInterface - RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(m_gateway); + RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(); + - m_nodeInitializer->frontInitializer()->setRpcStatus(rpcStatusInterface); auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID()); m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config()); m_rpc->setRpcStorage(rpcStatusInterface); m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); m_nodeInitializer->registerRpcHandler(m_rpc); - // Note: only can fetchAgencyList after the gatewayInterface has been setted into front - m_nodeInitializer->fetchAgencyList(); INIT_LOG(INFO) << LOG_DESC("init the rpc success"); } @@ -82,13 +85,8 @@ void AirNodeInitializer::initGateway(std::string const& _configPath) auto threadPool = std::make_shared( "gateway", config->gatewayConfig().networkConfig.threadPoolSize); - // Note: no need use redis as cache in-air-mode - GatewayFactory gatewayFactory; - auto gateway = gatewayFactory.buildGateway(ppc::protocol::NodeArch::AIR, config, nullptr, - m_nodeInitializer->protocolInitializer()->ppcMsgFactory(), threadPool); - auto frontInitializer = m_nodeInitializer->frontInitializer(); - gateway->registerFront(frontInitializer->front()->selfEndPoint(), frontInitializer->front()); - m_gateway = gateway; + GatewayFactory gatewayFactory(config); + m_gateway = gatewayFactory.build(m_frontBuilder); } void AirNodeInitializer::start() diff --git a/cpp/ppc-main/air-node/AirNodeInitializer.h b/cpp/wedpr-main/air-node/AirNodeInitializer.h similarity index 79% rename from cpp/ppc-main/air-node/AirNodeInitializer.h rename to cpp/wedpr-main/air-node/AirNodeInitializer.h index 260b39cb..0366a179 100644 --- a/cpp/ppc-main/air-node/AirNodeInitializer.h +++ b/cpp/wedpr-main/air-node/AirNodeInitializer.h @@ -18,22 +18,27 @@ * @date 2022-11-14 */ #pragma once -#include "libinitializer/Common.h" -#include "libinitializer/Initializer.h" -#include "ppc-framework/gateway/GatewayInterface.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/gateway/IGateway.h" +#include "wedpr-initializer/Common.h" +#include "wedpr-initializer/Initializer.h" #include #include namespace ppc::rpc { class Rpc; } +namespace ppc::front +{ +class LocalFrontBuilder; +} namespace ppc::node { class AirNodeInitializer { public: using Ptr = std::shared_ptr; - AirNodeInitializer() {} + AirNodeInitializer(); virtual ~AirNodeInitializer() { stop(); } virtual void init(std::string const& _configPath); @@ -46,7 +51,8 @@ class AirNodeInitializer private: bcos::BoostLogInitializer::Ptr m_logInitializer; ppc::initializer::Initializer::Ptr m_nodeInitializer; - ppc::gateway::GatewayInterface::Ptr m_gateway; + ppc::gateway::IGateway::Ptr m_gateway; std::shared_ptr m_rpc; + std::shared_ptr m_frontBuilder; }; } // namespace ppc::node \ No newline at end of file diff --git a/cpp/ppc-main/air-node/CMakeLists.txt b/cpp/wedpr-main/air-node/CMakeLists.txt similarity index 100% rename from cpp/ppc-main/air-node/CMakeLists.txt rename to cpp/wedpr-main/air-node/CMakeLists.txt diff --git a/cpp/ppc-main/pro-node/main.cpp b/cpp/wedpr-main/air-node/main.cpp similarity index 54% rename from cpp/ppc-main/pro-node/main.cpp rename to cpp/wedpr-main/air-node/main.cpp index 881c4304..b7cfd67f 100644 --- a/cpp/ppc-main/pro-node/main.cpp +++ b/cpp/wedpr-main/air-node/main.cpp @@ -1,4 +1,4 @@ -/** +/* * Copyright (C) 2022 WeDPR. * SPDX-License-Identifier: Apache-2.0 * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,30 +15,13 @@ * * @file main.cpp * @author: yujiechen - * @date 2022-11-25 + * @date 2022-11-14 */ -#include "ProNodeServiceApp.h" -#include "libhelper/CommandHelper.h" +#include "AirNodeInitializer.h" +#include "wedpr-main/common/NodeStarter.h" -using namespace ppctars; - -int main(int argc, char* argv[]) +int main(int argc, const char* argv[]) { - try - { - ppc::initAppCommandLine(argc, argv); - ProNodeServiceApp app; - app.main(argc, argv); - app.waitForShutdown(); - return 0; - } - catch (std::exception& e) - { - cerr << "ppc-pro-node std::exception:" << boost::diagnostic_information(e) << std::endl; - } - catch (...) - { - cerr << "ppc-pro-node unknown exception." << std::endl; - } - return -1; -} + auto initializer = std::make_shared(); + startProgram(argc, argv, "ppc-psi", initializer); +} \ No newline at end of file diff --git a/cpp/ppc-main/cem-node/CEMInitializer.cpp b/cpp/wedpr-main/cem-node/CEMInitializer.cpp similarity index 100% rename from cpp/ppc-main/cem-node/CEMInitializer.cpp rename to cpp/wedpr-main/cem-node/CEMInitializer.cpp diff --git a/cpp/ppc-main/cem-node/CEMInitializer.h b/cpp/wedpr-main/cem-node/CEMInitializer.h similarity index 97% rename from cpp/ppc-main/cem-node/CEMInitializer.h rename to cpp/wedpr-main/cem-node/CEMInitializer.h index 149e9872..1ba25899 100644 --- a/cpp/ppc-main/cem-node/CEMInitializer.h +++ b/cpp/wedpr-main/cem-node/CEMInitializer.h @@ -18,8 +18,8 @@ * @date 2022-11-19 */ #pragma once -#include "libinitializer/Common.h" #include "ppc-rpc/src/RpcFactory.h" +#include "wedpr-initializer/Common.h" #include #include namespace ppc::rpc diff --git a/cpp/ppc-main/cem-node/CMakeLists.txt b/cpp/wedpr-main/cem-node/CMakeLists.txt similarity index 100% rename from cpp/ppc-main/cem-node/CMakeLists.txt rename to cpp/wedpr-main/cem-node/CMakeLists.txt diff --git a/cpp/ppc-main/cem-node/main.cpp b/cpp/wedpr-main/cem-node/main.cpp similarity index 100% rename from cpp/ppc-main/cem-node/main.cpp rename to cpp/wedpr-main/cem-node/main.cpp diff --git a/cpp/ppc-main/air-node/main.cpp b/cpp/wedpr-main/common/NodeStarter.h similarity index 72% rename from cpp/ppc-main/air-node/main.cpp rename to cpp/wedpr-main/common/NodeStarter.h index 884f5298..95d8d671 100644 --- a/cpp/ppc-main/air-node/main.cpp +++ b/cpp/wedpr-main/common/NodeStarter.h @@ -17,15 +17,18 @@ * @author: yujiechen * @date 2022-11-14 */ -#include "AirNodeInitializer.h" + #include #include #include #include #include -using namespace ppc; -int main(int argc, const char* argv[]) +namespace ppc::node +{ +template +int startProgram( + int argc, const char* argv[], std::string const& binaryName, std::shared_ptr& starter) { /// set LC_ALL setDefaultOrCLocale(); @@ -43,35 +46,35 @@ int main(int argc, const char* argv[]) abort(); }); // get datetime and output welcome info - ExitHandler exitHandler; - signal(SIGTERM, &ExitHandler::exitHandler); - signal(SIGABRT, &ExitHandler::exitHandler); - signal(SIGINT, &ExitHandler::exitHandler); + ppc::ExitHandler exitHandler; + signal(SIGTERM, &ppc::ExitHandler::exitHandler); + signal(SIGABRT, &ppc::ExitHandler::exitHandler); + signal(SIGINT, &ppc::ExitHandler::exitHandler); // Note: the initializer must exist in the life time of the whole program - auto initializer = std::make_shared(); try { - auto param = initCommandLine(argc, argv); - initializer->init(param.configFilePath); - initializer->start(); + auto param = ppc::initCommandLine(argc, argv); + starter->init(param.configFilePath); + starter->start(); } catch (std::exception const& e) { printVersion(); std::cout << "[" << bcos::getCurrentDateTime() << "] "; - std::cout << "start ppc-psi failed, error:" << boost::diagnostic_information(e) + std::cout << "start " + binaryName + " failed, error:" << boost::diagnostic_information(e) << std::endl; return -1; } printVersion(); std::cout << "[" << bcos::getCurrentDateTime() << "] "; - std::cout << "The ppc-psi is running..." << std::endl; + std::cout << "The " + binaryName + "is running..." << std::endl; while (!exitHandler.shouldExit()) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } - initializer.reset(); + starter.reset(); std::cout << "[" << bcos::getCurrentDateTime() << "] "; - std::cout << "ppc-psi program exit normally." << std::endl; + std::cout << "The" + binaryName + " program exit normally." << std::endl; } +} // namespace ppc::node \ No newline at end of file diff --git a/cpp/ppc-main/gateway/CMakeLists.txt b/cpp/wedpr-main/gateway/CMakeLists.txt similarity index 61% rename from cpp/ppc-main/gateway/CMakeLists.txt rename to cpp/wedpr-main/gateway/CMakeLists.txt index 8406c395..062d9c66 100644 --- a/cpp/ppc-main/gateway/CMakeLists.txt +++ b/cpp/wedpr-main/gateway/CMakeLists.txt @@ -1,12 +1,11 @@ cmake_minimum_required(VERSION 3.14) -project(ppctars-Gateway) +project(wedpr-gateway) include_directories(${CMAKE_SOURCE_DIR}) aux_source_directory(./ SRC_LIST) -aux_source_directory(../../ppc-tars-service/GatewayService SRC_LIST) add_executable(${GATEWAY_BINARY_NAME} ${SRC_LIST}) -target_link_libraries(${GATEWAY_BINARY_NAME} ${GATEWAY_TARGET} ${STORAGE_TARGET} ${PROTOCOL_TARGET} ${HELPER_TARGET}) \ No newline at end of file +target_link_libraries(${GATEWAY_BINARY_NAME} ${GATEWAY_TARGET} ${STORAGE_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${HELPER_TARGET}) \ No newline at end of file diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.cpp b/cpp/wedpr-main/gateway/GatewayInitializer.cpp new file mode 100644 index 00000000..ca1c25c7 --- /dev/null +++ b/cpp/wedpr-main/gateway/GatewayInitializer.cpp @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayInitializer.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "GatewayInitializer.h" +#include "grpc/server/GatewayServer.h" +#include "grpc/server/GrpcServer.h" +#include "ppc-gateway/GatewayFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include "protobuf/NodeInfoImpl.h" +#include "wedpr-protocol/grpc/client/RemoteFrontBuilder.h" +#include "wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h" + +using namespace ppc::tools; +using namespace ppc::protocol; +using namespace bcos; +using namespace ppc::gateway; +using namespace ppc::front; + +void GatewayInitializer::init(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + INIT_LOG(INFO) << LOG_DESC("initLog success"); + + INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath; + auto config = std::make_shared(); + + config->loadGatewayConfig(ppc::protocol::NodeArch::PRO, nullptr, _configPath); + auto threadPool = std::make_shared( + "gateway", config->gatewayConfig().networkConfig.threadPoolSize); + + GatewayFactory gatewayFactory(config); + m_gateway = gatewayFactory.build(std::make_shared(config->grpcConfig())); + + m_server = std::make_shared(config->gatewayConfig().grpcConfig); + // register the gateway service + auto gatewayService = std::make_shared(m_gateway, + std::make_shared(), std::make_shared()); + m_server->registerService(gatewayService); +} + +void GatewayInitializer::start() +{ + if (m_gateway) + { + m_gateway->start(); + } + if (m_server) + { + m_server->start(); + } +} +void GatewayInitializer::stop() +{ + if (m_server) + { + m_server->stop(); + } + if (m_gateway) + { + m_gateway->start(); + } +} \ No newline at end of file diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.h b/cpp/wedpr-main/gateway/GatewayInitializer.h new file mode 100644 index 00000000..7cd15a01 --- /dev/null +++ b/cpp/wedpr-main/gateway/GatewayInitializer.h @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayInitializer.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "ppc-framework/gateway/IGateway.h" +#include +#include +#include + +#define INIT_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("GATEWAYInit") +namespace ppc::protocol +{ +class GrpcServer; +} +namespace ppc::gateway +{ +class GatewayInitializer +{ +public: + using Ptr = std::shared_ptr; + GatewayInitializer() = default; + virtual ~GatewayInitializer() { stop(); } + + virtual void init(std::string const& _configPath); + virtual void start(); + virtual void stop(); + +protected: + bcos::BoostLogInitializer::Ptr m_logInitializer; + ppc::gateway::IGateway::Ptr m_gateway; + std::shared_ptr m_server; +}; +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/ppc-main/gateway/main.cpp b/cpp/wedpr-main/gateway/main.cpp similarity index 53% rename from cpp/ppc-main/gateway/main.cpp rename to cpp/wedpr-main/gateway/main.cpp index 0212cf82..3f309bbd 100644 --- a/cpp/ppc-main/gateway/main.cpp +++ b/cpp/wedpr-main/gateway/main.cpp @@ -1,4 +1,4 @@ -/** +/* * Copyright (C) 2022 WeDPR. * SPDX-License-Identifier: Apache-2.0 * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,32 +15,14 @@ * * @file main.cpp * @author: yujiechen - * @date 2022-11-25 + * @date 2022-11-14 */ -#include "GatewayServiceApp.h" -#include "libhelper/CommandHelper.h" +#include "GatewayInitializer.h" +#include "wedpr-main/common/NodeStarter.h" -using namespace ppctars; - -int main(int argc, char* argv[]) +using namespace ppc::node; +int main(int argc, const char* argv[]) { - try - { - ppc::initAppCommandLine(argc, argv); - GatewayServiceApp app; - app.main(argc, argv); - app.waitForShutdown(); - - return 0; - } - catch (std::exception& e) - { - cerr << "ppc-gateway-service std::exception:" << boost::diagnostic_information(e) - << std::endl; - } - catch (...) - { - cerr << "ppc-gateway-service unknown exception." << std::endl; - } - return -1; + auto initializer = std::make_shared(); + startProgram(argc, argv, "ppc-gateway-service", initializer); } \ No newline at end of file diff --git a/cpp/ppc-main/mpc-node/CMakeLists.txt b/cpp/wedpr-main/mpc-node/CMakeLists.txt similarity index 100% rename from cpp/ppc-main/mpc-node/CMakeLists.txt rename to cpp/wedpr-main/mpc-node/CMakeLists.txt diff --git a/cpp/ppc-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp similarity index 100% rename from cpp/ppc-main/mpc-node/MPCInitializer.cpp rename to cpp/wedpr-main/mpc-node/MPCInitializer.cpp diff --git a/cpp/ppc-main/mpc-node/MPCInitializer.h b/cpp/wedpr-main/mpc-node/MPCInitializer.h similarity index 97% rename from cpp/ppc-main/mpc-node/MPCInitializer.h rename to cpp/wedpr-main/mpc-node/MPCInitializer.h index 1d92e675..4e0da526 100644 --- a/cpp/ppc-main/mpc-node/MPCInitializer.h +++ b/cpp/wedpr-main/mpc-node/MPCInitializer.h @@ -18,8 +18,8 @@ * @date 2023-03-24 */ #pragma once -#include "libinitializer/Common.h" #include "ppc-rpc/src/RpcFactory.h" +#include "wedpr-initializer/Common.h" #include #include namespace ppc::rpc diff --git a/cpp/ppc-main/mpc-node/main.cpp b/cpp/wedpr-main/mpc-node/main.cpp similarity index 100% rename from cpp/ppc-main/mpc-node/main.cpp rename to cpp/wedpr-main/mpc-node/main.cpp diff --git a/cpp/ppc-main/pro-node/CMakeLists.txt b/cpp/wedpr-main/pro-node/CMakeLists.txt similarity index 74% rename from cpp/ppc-main/pro-node/CMakeLists.txt rename to cpp/wedpr-main/pro-node/CMakeLists.txt index ad796b5a..40664ebc 100644 --- a/cpp/ppc-main/pro-node/CMakeLists.txt +++ b/cpp/wedpr-main/pro-node/CMakeLists.txt @@ -1,11 +1,10 @@ cmake_minimum_required(VERSION 3.14) -project(ppctars-node) +project(pro-psi-node) include_directories(${CMAKE_SOURCE_DIR}) aux_source_directory(./ SRC_LIST) -aux_source_directory(../../ppc-tars-service/FrontService SRC_LIST) add_executable(${PRO_BINARY_NAME} ${SRC_LIST}) target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc) diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp new file mode 100644 index 00000000..ab2a51be --- /dev/null +++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProNodeInitializer.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "ProNodeInitializer.h" +#include "ppc-rpc/src/RpcFactory.h" +#include "ppc-rpc/src/RpcMemory.h" +#include "wedpr-protocol/grpc/client/RemoteFrontBuilder.h" + +using namespace ppc::protocol; +using namespace ppc::node; +using namespace ppc::gateway; +using namespace ppc::rpc; +using namespace ppc::storage; +using namespace ppc::initializer; +using namespace bcos; + +ProNodeInitializer::ProNodeInitializer() {} + +void ProNodeInitializer::init(std::string const& _configPath) +{ + // init the log + boost::property_tree::ptree pt; + boost::property_tree::read_ini(_configPath, pt); + + m_logInitializer = std::make_shared(); + m_logInitializer->initLog(pt); + INIT_LOG(INFO) << LOG_DESC("initLog success"); + + // init the node + m_nodeInitializer = std::make_shared(_configPath); + + // init the node(no need to set the gateway) + m_nodeInitializer->init(ppc::protocol::NodeArch::PRO, nullptr); + + + INIT_LOG(INFO) << LOG_DESC("init the rpc"); + // load the rpc config + // not specify the certPath in air-mode + m_nodeInitializer->config()->loadRpcConfig(nullptr, pt); + // init RpcStatusInterface + RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(); + + + auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID()); + m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config()); + m_rpc->setRpcStorage(rpcStatusInterface); + m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); + m_nodeInitializer->registerRpcHandler(m_rpc); + + INIT_LOG(INFO) << LOG_DESC("init the rpc success"); +} + +void ProNodeInitializer::start() +{ + // start the node + if (m_nodeInitializer) + { + m_nodeInitializer->start(); + } + if (m_rpc) + { + m_rpc->start(); + } +} + +void ProNodeInitializer::stop() +{ + if (m_rpc) + { + m_rpc->stop(); + } + if (m_nodeInitializer) + { + m_nodeInitializer->stop(); + } +} \ No newline at end of file diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.h b/cpp/wedpr-main/pro-node/ProNodeInitializer.h new file mode 100644 index 00000000..054d1b84 --- /dev/null +++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.h @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file ProNodeInitializer.h + * @author: yujiechen + * @date 2022-11-14 + */ +#pragma once +#include "wedpr-initializer/Common.h" +#include "wedpr-initializer/Initializer.h" +#include +#include +namespace ppc::rpc +{ +class Rpc; +} +namespace ppc::front +{ +class RemoteFrontBuilder; +} +namespace ppc::node +{ +class ProNodeInitializer +{ +public: + using Ptr = std::shared_ptr; + ProNodeInitializer(); + virtual ~ProNodeInitializer() { stop(); } + + virtual void init(std::string const& _configPath); + virtual void start(); + virtual void stop(); + +private: + bcos::BoostLogInitializer::Ptr m_logInitializer; + ppc::initializer::Initializer::Ptr m_nodeInitializer; + std::shared_ptr m_rpc; +}; +} // namespace ppc::node \ No newline at end of file diff --git a/cpp/wedpr-main/pro-node/main.cpp b/cpp/wedpr-main/pro-node/main.cpp new file mode 100644 index 00000000..a122784b --- /dev/null +++ b/cpp/wedpr-main/pro-node/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file main.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "ProNodeInitializer.h" +#include "wedpr-main/common/NodeStarter.h" + +using namespace ppc::node; +int main(int argc, const char* argv[]) +{ + auto initializer = std::make_shared(); + startProgram(argc, argv, "ppc-pro-node", initializer); +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/Common.h b/cpp/wedpr-protocol/grpc/Common.h new file mode 100644 index 00000000..bf2a5d76 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/Common.h @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2021 FISCO BCOS. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2021-04-12 + */ +#pragma once +#include "ppc-framework/Common.h" +#include "ppc-framework/protocol/GrpcConfig.h" +#include + +namespace ppc::protocol +{ +inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr const& grpcConfig) +{ + grpc::ChannelArguments args; + if (grpcConfig == nullptr) + { + return args; + } + args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); + return args; +} +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp index 08e47ee2..7fe01a6f 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp @@ -21,7 +21,6 @@ #include "protobuf/RequestConverter.h" #include "wedpr-protocol/protobuf/Common.h" - using namespace ppc::protocol; using namespace ppc::proto; using namespace grpc; diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.h b/cpp/wedpr-protocol/grpc/client/FrontClient.h index 29e1a21a..acc4e96b 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.h +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.h @@ -23,12 +23,12 @@ namespace ppc::protocol { -class FrontClient : public ppc::front::IFrontClient, public GrpcClient +class FrontClient : public virtual ppc::front::IFrontClient, public GrpcClient { public: using Ptr = std::shared_ptr; - FrontClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) - : GrpcClient(channelConfig, endPoints) + FrontClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) + : GrpcClient(grpcConfig, endPoints), m_stub(ppc::proto::Front::NewStub(m_channel)) {} ~FrontClient() override = default; diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index 50a445ae..eef8c9be 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -19,6 +19,7 @@ */ #include "GatewayClient.h" #include "Common.h" +#include "Service.grpc.pb.h" #include "protobuf/RequestConverter.h" using namespace ppc; diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index 9a75a52b..ea964acd 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -27,8 +27,8 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient { public: using Ptr = std::shared_ptr; - GatewayClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) - : GrpcClient(channelConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel)) + GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) + : GrpcClient(grpcConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel)) {} ~GatewayClient() override = default; diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.h b/cpp/wedpr-protocol/grpc/client/GrpcClient.h index 86e44fb5..d1fda5af 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.h +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.h @@ -19,6 +19,8 @@ */ #pragma once #include "Service.grpc.pb.h" +#include "ppc-framework/protocol/GrpcConfig.h" +#include "wedpr-protocol/grpc/Common.h" #include namespace ppc::protocol @@ -28,15 +30,16 @@ class GrpcClient { public: using Ptr = std::shared_ptr; - GrpcClient(grpc::ChannelArguments const& channelConfig, std::string const& endPoints) - : m_channel( - grpc::CreateCustomChannel(endPoints, grpc::InsecureChannelCredentials(), channelConfig)) + GrpcClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) + : m_channel(grpc::CreateCustomChannel( + endPoints, grpc::InsecureChannelCredentials(), toChannelConfig(grpcConfig))) {} virtual ~GrpcClient() = default; std::shared_ptr const& channel() { return m_channel; } + protected: std::shared_ptr m_channel; }; diff --git a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp new file mode 100644 index 00000000..d00e1c98 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RemoteFrontBuilder.cpp + * @author: yujiechen + * @date 2024-09-4 + */ +#include "RemoteFrontBuilder.h" +// Note: it's better not to include generated grpc files in the header, since it will slow the +// compiler speed +#include "FrontClient.h" + +using namespace ppc::front; +using namespace ppc::protocol; + +IFrontClient::Ptr RemoteFrontBuilder::buildClient(std::string endPoint) const +{ + return std::make_shared(m_grpcConfig, endPoint); +} diff --git a/cpp/ppc-framework/front/IFrontBuilder.h b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h similarity index 58% rename from cpp/ppc-framework/front/IFrontBuilder.h rename to cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h index 50423645..44a3db31 100644 --- a/cpp/ppc-framework/front/IFrontBuilder.h +++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h @@ -13,29 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * @file IFront.h + * @file RemoteFrontBuilder.h * @author: yujiechen - * @date 2024-08-22 + * @date 2024-09-4 */ #pragma once -#include "IFront.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/GrpcConfig.h" namespace ppc::front { -class IFrontBuilder +class RemoteFrontBuilder : public IFrontBuilder { public: - using Ptr = std::shared_ptr; - IFrontBuilder() = default; - virtual ~IFrontBuilder() = default; + using Ptr = std::shared_ptr; + RemoteFrontBuilder(ppc::protocol::GrpcConfig::Ptr const& grpcConfig) : m_grpcConfig(grpcConfig) + {} + ~RemoteFrontBuilder() override = default; - /** - * @brief create the Front using specified config - * - * @param config the config used to build the Front - * @return IFront::Ptr he created Front - */ - virtual IFront::Ptr build(ppc::front::FrontConfig::Ptr config) const = 0; - virtual IFrontClient::Ptr buildClient(std::string endPoint) const = 0; + IFrontClient::Ptr buildClient(std::string endPoint) const override; + +private: + ppc::protocol::GrpcConfig::Ptr m_grpcConfig; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.h b/cpp/wedpr-protocol/grpc/server/GrpcServer.h index f5ac678e..71bb6156 100644 --- a/cpp/wedpr-protocol/grpc/server/GrpcServer.h +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.h @@ -18,7 +18,7 @@ * @date 2024-09-03 */ #pragma once -#include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/GrpcConfig.h" #include #include #include @@ -26,12 +26,6 @@ namespace ppc::protocol { // refer to: https://grpc.io/docs/languages/cpp/callback/ -struct GrpcServerConfig -{ - ppc::protocol::EndPoint endPoint; - - std::string listenEndPoint() const { return endPoint.listenEndPoint(); } -}; class GrpcServer { public: diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 23b379e0..8888b1fa 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -18,6 +18,7 @@ * @date 2022-10-20 */ #include "Front.h" +#include "FrontImpl.h" using namespace ppc; using namespace bcos; @@ -34,7 +35,8 @@ using namespace ppc::front; void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) { - auto routeInfo = m_front->routerInfoBuilder()->build(); + auto front = std::dynamic_pointer_cast(m_front); + auto routeInfo = front->routerInfoBuilder()->build(); routeInfo->setDstInst(_agencyID); routeInfo->setTopic(_message->taskID()); bcos::bytes data; @@ -86,7 +88,4 @@ bcos::Error::Ptr Front::notifyTaskInfo(std::string const& taskID) bcos::Error::Ptr Front::eraseTaskInfo(std::string const& _taskID) { m_front->unRegisterTopic(_taskID); -} - -// get the agencyList from the gateway -void Front::asyncGetAgencyList(GetAgencyListCallback _callback) {} +} \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h index 6394df73..b3a10370 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h @@ -19,8 +19,8 @@ */ #pragma once -#include "FrontImpl.h" #include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/PPCMessageFace.h" namespace ppc::front @@ -29,7 +29,7 @@ class Front : public FrontInterface, public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - Front(FrontImpl::Ptr front) : m_front(std::move(front)) {} + Front(IFront::Ptr front) : m_front(std::move(front)) {} ~Front() override {} /** @@ -56,12 +56,9 @@ class Front : public FrontInterface, public std::enable_shared_from_this // erase the task-info when task finished bcos::Error::Ptr eraseTaskInfo(std::string const& _taskID) override; - // get the agencyList from the gateway - void asyncGetAgencyList(GetAgencyListCallback _callback) override; - // register message handler for algorithm void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, - std::function _handler) + std::function _handler) override { uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; auto self = weak_from_this(); @@ -82,7 +79,7 @@ class Front : public FrontInterface, public std::enable_shared_from_this } private: - FrontImpl::Ptr m_front; + IFront::Ptr m_front; ppc::front::PPCMessageFaceFactory::Ptr m_messageFactory; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h new file mode 100644 index 00000000..2996b13a --- /dev/null +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file FrontConfigImpl.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#pragma once +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/INodeInfo.h" +#include + +namespace ppc::front +{ +class FrontConfigImpl : public FrontConfig +{ +public: + using Ptr = std::shared_ptr; + FrontConfigImpl(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, int threadPoolSize, + std::string nodeID) + : FrontConfig(threadPoolSize, nodeID), m_nodeInfoFactory(std::move(nodeInfoFactory)) + {} + + ~FrontConfigImpl() override = default; + + ppc::protocol::INodeInfo::Ptr generateNodeInfo() const override + { + auto nodeInfo = m_nodeInfoFactory->build( + bcos::bytesConstRef((bcos::byte*)m_nodeID.data(), m_nodeID.size()), + m_selfEndPoint.entryPoint()); + nodeInfo->setComponents(std::set(m_components.begin(), m_components.end())); + return nodeInfo; + } + +private: + ppc::protocol::INodeInfoFactory::Ptr m_nodeInfoFactory; +}; +} // namespace ppc::front diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index 3156b6c7..d89529d6 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -28,7 +28,7 @@ namespace ppc::front { -class FrontImpl : public IFront, public std::enable_shared_from_this +class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h b/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h new file mode 100644 index 00000000..022f6a83 --- /dev/null +++ b/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LocalFrontBuilder.h + * @author: yujiechen + * @date 2024-09-4 + */ +#pragma once +#include "ppc-framework/front/IFront.h" + +namespace ppc::front +{ +class LocalFrontBuilder : public IFrontBuilder +{ +public: + using Ptr = std::shared_ptr; + LocalFrontBuilder(IFront::Ptr front) { m_front = front; } + LocalFrontBuilder() = default; + ~LocalFrontBuilder() override = default; + + IFrontClient::Ptr buildClient(std::string endPoint) const override { return m_front; } + + void setFront(IFront::Ptr front) { m_front = std::move(front); } + +private: + IFront::Ptr m_front; +}; +} // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp index 66f0972c..5b21374e 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp +++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp @@ -59,10 +59,6 @@ Rpc::Rpc(std::shared_ptr _wsService, std::string const& // register the handler for GET_TASK_STATUS m_methodToHandler[GET_TASK_STATUS] = boost::bind(&Rpc::getTaskStatus, this, boost::placeholders::_1, boost::placeholders::_2); - // register the handler for REGISTER_GATEWAY_URL - m_methodToHandler[REGISTER_GATEWAY_URL] = boost::bind( - &Rpc::registerGatewayUrl, this, boost::placeholders::_1, boost::placeholders::_2); - // register ecdh bs mode methods m_methodToHandler[ASYNC_RUN_BS_MODE_TASK] = boost::bind( &Rpc::asyncRunBsModeTask, this, boost::placeholders::_1, boost::placeholders::_2); @@ -307,51 +303,6 @@ void Rpc::getTaskStatus(Json::Value const& _req, RespFunc _respFunc) _respFunc(result->error(), result->serializeToJson()); } -void Rpc::registerGatewayUrl(Json::Value const& _req, RespFunc _respFunc) -{ - if (!m_rpcStorage) - { - BOOST_THROW_EXCEPTION( - BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set")); - } - - if (!_req.isMember("id")) - { - BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the agencyID")); - } - auto agencyID = _req["id"].asString(); - - if (!_req.isMember("url")) - { - BOOST_THROW_EXCEPTION( - InvalidParameter() << errinfo_comment("Must specify the gateway url")); - } - auto agencyUrl = _req["url"].asString(); - - std::vector endpoints; - boost::split(endpoints, agencyUrl, boost::is_any_of(",")); - for (auto& endpoint : endpoints) - { - if (!checkEndpoint(endpoint)) - { - BOOST_THROW_EXCEPTION( - InvalidParameter() << bcos::errinfo_comment("Invalid endpoint: " + endpoint)); - } - } - Json::Value response; - auto error = m_rpcStorage->insertGateway(agencyID, agencyUrl); - if (error && error->errorCode()) - { - response["code"] = error->errorCode(); - response["message"] = error->errorMessage(); - _respFunc(error, std::move(response)); - return; - } - - response["code"] = 0; - response["message"] = "success"; - _respFunc(error, std::move(response)); -} void Rpc::asyncRunBsModeTask(Json::Value const& _req, RespFunc _respFunc) { diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h index 59eadce5..45871f29 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h +++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h @@ -122,7 +122,6 @@ class Rpc : public RpcInterface virtual void runTask(Json::Value const& _req, RespFunc _respFunc); virtual void asyncRunTask(Json::Value const& _req, RespFunc _respFunc); virtual void getTaskStatus(Json::Value const& _req, RespFunc _respFunc); - virtual void registerGatewayUrl(Json::Value const& _req, RespFunc _respFunc); virtual void asyncRunBsModeTask(Json::Value const& _req, RespFunc _respFunc); virtual void fetchCipher(Json::Value const& _req, RespFunc _respFunc); diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h index 30430027..78a447c5 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h @@ -40,8 +40,6 @@ class RpcMemory : public RpcStatusInterface bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) override; bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) override; protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) override; - bcos::Error::Ptr insertGateway( - const std::string& _agencyID, const std::string& _endpoint) override; bcos::Error::Ptr deleteGateway(const std::string& _agencyID) override; std::vector listGateway() override; diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp index 1573c0fa..37424c3b 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -19,7 +19,9 @@ */ #include "ProTransportImpl.h" #include "protocol/src/v1/MessageImpl.h" +#include "wedpr-protocol/grpc/client/GatewayClient.h" #include "wedpr-protocol/grpc/server/FrontServer.h" +#include "wedpr-protocol/grpc/server/GrpcServer.h" using namespace ppc::front; using namespace ppc::protocol; @@ -33,9 +35,8 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config) m_server = std::make_shared(grpcServerConfig); FrontFactory frontFactory; - grpc::ChannelArguments channelConfig; - channelConfig.SetLoadBalancingPolicyName(m_config->loadBalancePolicy()); - auto gateway = std::make_shared(channelConfig, m_config->gatewayEndPoints()); + auto gateway = + std::make_shared(m_config->grpcConfig(), m_config->gatewayGrpcTarget()); m_front = frontFactory.build(std::make_shared(), std::make_shared(), std::make_shared(), gateway, config); @@ -46,4 +47,15 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config) // register the frontService m_server->registerService(frontService); +} + +void ProTransportImpl::start() +{ + m_server->start(); + m_front->start(); +} +void ProTransportImpl::stop() +{ + m_server->stop(); + m_front->stop(); } \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index d32ba9e5..5756e8d4 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -18,10 +18,13 @@ * @date 2024-09-04 */ #pragma once - #include "TransportImpl.h" -#include "wedpr-protocol/grpc/client/GatewayClient.h" -#include "wedpr-protocol/grpc/server/GrpcServer.h" + +namespace ppc::protocol +{ +class GrpcServer; +} + namespace ppc::sdk { @@ -31,19 +34,11 @@ class ProTransportImpl : public Transport using Ptr = std::shared_ptr; ProTransportImpl(ppc::front::FrontConfig::Ptr config); - void start() override - { - m_server->start(); - m_front->start(); - } - void stop() override - { - m_server->stop(); - m_front->stop(); - } + void start() override; + void stop() override; protected: ppc::front::FrontConfig::Ptr m_config; - ppc::protocol::GrpcServer::Ptr m_server; + std::shared_ptr m_server; }; } // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/TransportBuilder.cpp new file mode 100644 index 00000000..8042b844 --- /dev/null +++ b/cpp/wedpr-transport/sdk/TransportBuilder.cpp @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file TransportBuilder.cpp + * @author: yujiechen + * @date 2024-09-04 + */ + +#include "TransportBuilder.h" +#include "ProTransportImpl.h" +#include "Transport.h" +#include "TransportImpl.h" +#include "ppc-front/FrontConfigImpl.h" +#include "protobuf/NodeInfoImpl.h" +#include + +using namespace ppc::sdk; + +Transport::Ptr TransportBuilder::build( + SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) +{ + switch (mode) + { + case SDKMode::AIR: + { + return std::make_shared(config, gateway); + } + case SDKMode::PRO: + { + return std::make_shared(config); + } + default: + throw std::runtime_error("Unsupported sdk mode, only support AIR/PRO mode!"); + } +} + +ppc::front::FrontConfig::Ptr TransportBuilder::buildConfig(int threadPoolSize, std::string nodeID) +{ + return std::make_shared( + std::make_shared(), threadPoolSize, nodeID); +} diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/TransportBuilder.h index f0492822..b2b57931 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.h +++ b/cpp/wedpr-transport/sdk/TransportBuilder.h @@ -18,9 +18,9 @@ * @date 2024-09-04 */ #pragma once -#include "ProTransportImpl.h" #include "Transport.h" -#include "TransportImpl.h" +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/gateway/IGateway.h" #include namespace ppc::sdk { @@ -37,21 +37,8 @@ class TransportBuilder virtual ~TransportBuilder() = default; Transport::Ptr build(SDKMode mode, ppc::front::FrontConfig::Ptr config, - ppc::gateway::IGateway::Ptr const& gateway) - { - switch (mode) - { - case SDKMode::AIR: - { - return std::make_shared(config, gateway); - } - case SDKMode::PRO: - { - return std::make_shared(config); - } - default: - throw std::runtime_error("Unsupported sdk mode, only support AIR/PRO mode!"); - } - } + ppc::gateway::IGateway::Ptr const& gateway); + + ppc::front::FrontConfig::Ptr buildConfig(int threadPoolSize, std::string nodeID); }; } // namespace ppc::sdk \ No newline at end of file From 89e8be1b6d1158fd94ce052953a05069fe7d0129 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 5 Sep 2024 14:29:54 +0800 Subject: [PATCH 025/120] complement config && optmize compile (#21) --- cpp/CMakeLists.txt | 8 - cpp/ppc-framework/front/FrontConfig.h | 50 ++++--- cpp/ppc-framework/protocol/EndPoint.h | 1 + cpp/ppc-framework/protocol/GrpcConfig.h | 8 +- .../psi/TaskFrameworkInterface.h | 47 ------ cpp/tools/build_ppc.sh | 59 ++++---- .../ppc-cem/src/CEMService.cpp | 4 +- cpp/wedpr-computing/ppc-cem/src/CEMService.h | 37 +++-- .../ppc-cem/src/wedpr_ffi_c_equality.h | 8 +- cpp/wedpr-computing/ppc-mpc/src/Common.h | 1 - .../ppc-mpc/src/MPCService.cpp | 1 + cpp/wedpr-computing/ppc-mpc/src/MPCService.h | 33 ++--- cpp/wedpr-computing/ppc-pir/src/Common.h | 27 ++-- cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h | 4 +- .../ppc-pir/src/OtPIRFactory.h | 11 +- cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp | 2 + cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h | 61 ++++---- .../ppc-psi/src/cm2020-psi/CM2020PSIFactory.h | 1 - .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp | 1 + .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.h | 12 +- .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp | 1 + .../src/ecdh-conn-psi/EcdhConnTaskState.h | 6 +- .../protocol/PSIConnMessageFactory.h | 1 - .../ecdh-multi-psi/EcdhMultiPSIFactory.cpp | 41 ++++++ .../src/ecdh-multi-psi/EcdhMultiPSIFactory.h | 39 +++-- .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp | 1 + .../ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp | 42 ++++++ .../ppc-psi/src/ecdh-psi/EcdhPSIFactory.h | 18 +-- .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp | 1 + .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.h | 4 +- .../ppc-psi/src/ecdh-psi/EcdhTaskState.h | 6 +- .../src/labeled-psi/LabeledPSIFactory.cpp | 35 +++++ .../src/labeled-psi/LabeledPSIFactory.h | 9 +- .../src/labeled-psi/LabeledPSIImpl.cpp | 1 + .../ppc-psi/src/labeled-psi/LabeledPSIImpl.h | 10 +- .../src/psi-framework/PSIFramework.cpp | 1 + .../ppc-psi/src/psi-framework/PSIFramework.h | 3 +- .../ppc-psi/src/psi-framework/TaskState.h | 15 +- .../src/ra2018-psi/RA2018PSIFactory.cpp | 52 +++++++ .../ppc-psi/src/ra2018-psi/RA2018PSIFactory.h | 28 +--- .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp | 1 + .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.h | 6 +- .../ppc-tools/src/config/CEMConfig.h | 33 +++++ .../ppc-tools/src/config/MPCConfig.h | 34 +++++ .../ppc-tools/src/config/PPCConfig.cpp | 138 ++++++++++++++---- .../ppc-tools/src/config/PPCConfig.h | 119 ++++++++------- .../ppc-tools/src/config/StorageConfig.h | 33 +++++ cpp/wedpr-initializer/Initializer.cpp | 5 +- cpp/wedpr-initializer/Initializer.h | 21 ++- .../air-node/AirNodeInitializer.cpp | 4 +- cpp/wedpr-main/cem-node/CEMInitializer.cpp | 2 +- cpp/wedpr-main/gateway/GatewayInitializer.cpp | 4 +- cpp/wedpr-main/mpc-node/MPCInitializer.cpp | 3 +- .../pro-node/ProNodeInitializer.cpp | 2 +- .../ppc-front/ppc-front/FrontConfigImpl.h | 26 ++++ .../ppc-gateway/GatewayConfigContext.cpp | 11 ++ .../ppc-gateway/GatewayConfigContext.h | 19 +-- .../ppc-gateway/GatewayConfigLoader.cpp | 1 + .../ppc-gateway/GatewayConfigLoader.h | 9 +- .../ppc-gateway/GatewayFactory.cpp | 23 ++- .../ppc-gateway/ppc-gateway/GatewayFactory.h | 12 +- .../ppc-gateway/test/demo/gateway_demo.cpp | 3 +- .../test/unittests/GatewayTest.cpp | 4 +- .../ppc-http/demo/http_demo.cpp | 2 +- .../ppc-http/src/HttpFactory.cpp | 10 +- .../ppc-http/src/HttpFactory.h | 10 +- cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp | 2 +- .../ppc-rpc/src/RpcFactory.cpp | 1 + cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h | 9 +- cpp/wedpr-transport/sdk/TransportBuilder.cpp | 10 +- cpp/wedpr-transport/sdk/TransportBuilder.h | 7 +- 71 files changed, 794 insertions(+), 460 deletions(-) delete mode 100644 cpp/ppc-framework/psi/TaskFrameworkInterface.h create mode 100644 cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp create mode 100644 cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp create mode 100644 cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp create mode 100644 cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp create mode 100644 cpp/wedpr-helper/ppc-tools/src/config/CEMConfig.h create mode 100644 cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h create mode 100644 cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dffbaea8..c7c0d094 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -72,14 +72,6 @@ set(CEM_SOURCE "") if(BUILD_CEM) set(CEM_SOURCE "wedpr-computing/ppc-cem") endif() -#set(ALL_SOURCE_LIST -# ${SDK_SOURCE_LIST} ppc-crypto -# libhelper libinitializer ppc-io ppc-protocol -# ppc-gateway ppc-front ppc-tars-protocol -# wedpr-helper/ppc-tools ppc-storage ppc-psi ppc-rpc -# ppc-http ppc-mpc ppc-pir -# ${CEM_SOURCE} ppc-main) - set(ALL_SOURCE_LIST ${SDK_SOURCE_LIST} wedpr-crypto/ppc-crypto diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h index affe5d9b..aa1938da 100644 --- a/cpp/ppc-framework/front/FrontConfig.h +++ b/cpp/ppc-framework/front/FrontConfig.h @@ -34,43 +34,32 @@ class FrontConfig { public: using Ptr = std::shared_ptr; + FrontConfig() = default; + FrontConfig(int threadPoolSize, std::string nodeID) : m_threadPoolSize(threadPoolSize), m_nodeID(std::move(nodeID)) {} virtual ~FrontConfig() = default; virtual int threadPoolSize() const { return m_threadPoolSize; } - virtual std::string const& nodeID() const { return m_nodeID; } - virtual std::vector const& gatewayInfo() const - { - return m_gatewayInfo; - } - virtual void setGatewayInfo(std::vector gatewayInfo) - { - m_gatewayInfo = std::move(gatewayInfo); - } + virtual void setThreadPoolSize(int threadPoolSize) { m_threadPoolSize = threadPoolSize; } - virtual void appendGatewayInfo(ppc::protocol::EndPoint&& endpoint) - { - m_gatewayInfo.push_back(endpoint); - } + virtual std::string const& nodeID() const { return m_nodeID; } + virtual void setNodeID(std::string const& nodeID) { m_nodeID = nodeID; } ppc::protocol::EndPoint const& selfEndPoint() const { return m_selfEndPoint; } + ppc::protocol::EndPoint& mutableSelfEndPoint() { return m_selfEndPoint; } + void setSelfEndPoint(ppc::protocol::EndPoint const& endPoint) { m_selfEndPoint = endPoint; } + void setGatewayGrpcTarget(std::string const& gatewayGrpcTarget) + { + m_gatewayGrpcTarget = gatewayGrpcTarget; + } // refer to: https://github.com/grpc/grpc-node/issues/2066 // grpc prefer to using ipv4:${host1}:${port1},${host2}:${port2} as target to support multiple // servers - std::string gatewayGrpcTarget() - { - std::stringstream oss; - oss << "ipv4:"; - for (auto const& endPoint : m_gatewayInfo) - { - oss << endPoint.entryPoint() << ","; - } - return oss.str(); - } + std::string const& gatewayGrpcTarget() const { return m_gatewayGrpcTarget; } void setGrpcConfig(ppc::protocol::GrpcConfig::Ptr grpcConfig) { @@ -84,15 +73,28 @@ class FrontConfig virtual std::vector const& getComponents() const { return m_components; } void setComponents(std::vector const& components) { m_components = components; } + std::vector& mutableComponents() { return m_components; } + protected: ppc::protocol::GrpcConfig::Ptr m_grpcConfig; ppc::protocol::EndPoint m_selfEndPoint; int m_threadPoolSize; std::string m_nodeID; - std::vector m_gatewayInfo; + std::string m_gatewayGrpcTarget; std::vector m_components; }; +class FrontConfigBuilder +{ +public: + using Ptr = std::shared_ptr; + FrontConfigBuilder() = default; + virtual ~FrontConfigBuilder() = default; + + virtual FrontConfig::Ptr build() const = 0; + virtual FrontConfig::Ptr build(int threadPoolSize, std::string nodeID) const = 0; +}; + inline std::string printFrontDesc(FrontConfig::Ptr const& config) { if (!config) diff --git a/cpp/ppc-framework/protocol/EndPoint.h b/cpp/ppc-framework/protocol/EndPoint.h index 099b7de5..293b6268 100644 --- a/cpp/ppc-framework/protocol/EndPoint.h +++ b/cpp/ppc-framework/protocol/EndPoint.h @@ -41,6 +41,7 @@ class EndPoint void setHost(std::string host) { m_host = std::move(host); } void setPort(uint16_t port) { m_port = port; } + void setListenIp(std::string const& listenIp) { m_listenIp = listenIp; } std::string entryPoint() const { return m_host + ":" + std::to_string(m_port); } diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h index 76b2ac17..3c915855 100644 --- a/cpp/ppc-framework/protocol/GrpcConfig.h +++ b/cpp/ppc-framework/protocol/GrpcConfig.h @@ -37,13 +37,13 @@ class GrpcConfig GrpcConfig() = default; virtual ~GrpcConfig() = default; - std::string const& loadBalancePolicy() const { return m_loadBanlancePolicy; } - void setLoadBalancePolicy(std::string const& loadBanlancePolicy) + std::string const& loadBalancePolicy() const { return m_loadBalancePolicy; } + void setLoadBalancePolicy(std::string const& loadBalancePolicy) { - m_loadBanlancePolicy = loadBanlancePolicy; + m_loadBalancePolicy = loadBalancePolicy; } private: - std::string m_loadBanlancePolicy = "round_robin"; + std::string m_loadBalancePolicy = "round_robin"; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/psi/TaskFrameworkInterface.h b/cpp/ppc-framework/psi/TaskFrameworkInterface.h deleted file mode 100644 index 12d461bb..00000000 --- a/cpp/ppc-framework/psi/TaskFrameworkInterface.h +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright (C) 2022 WeDPR. - * SPDX-License-Identifier: Apache-2.0 - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @brief the PSIFrameworkInterface - * @file PSIFrameworkInterface.h - * @author: yujiechen - * @date 2022-11-16 - */ -#pragma once -#include "../protocol/PPCMessageFace.h" -#include "../protocol/Task.h" -#include -#include -namespace ppc::psi -{ -using TaskResponseCallback = std::function; - -class PSIFrameworkInterface -{ -public: - using Ptr = std::shared_ptr; - PSIFrameworkInterface() = default; - virtual ~PSIFrameworkInterface() = default; - - // run task - virtual void asyncRunTask( - ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) = 0; - - // register to the front to get the message related to ra2018 - virtual void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg) = 0; - - virtual void start() = 0; - virtual void stop() = 0; -}; -} // namespace ppc::psi \ No newline at end of file diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh index 23c1fb99..730b518c 100644 --- a/cpp/tools/build_ppc.sh +++ b/cpp/tools/build_ppc.sh @@ -10,9 +10,10 @@ listen_ip="0.0.0.0" cdn_link_header="https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/FISCO-BCOS" OPENSSL_CMD="${HOME}/.fisco/tassl-1.1.1b" -port_start=(40300 10200) +port_start=(40300 10200 18000) gateway_listen_port=port_start[0] rpc_listen_port=port_start[1] +grpc_listen_port=port_start[2] use_ip_param= ip_param= @@ -357,16 +358,17 @@ generate_config_ini() { local rpc_listen_ip="${4}" local rpc_listen_port="${5}" - local agency_info="${6}" - local agency_id="${7}" - local index="${8}" + + local grpc_listen_ip="${6}" + local grpc_listen_port="${7}" + + local agency_id="${8}" + local index="${9}" cat <"${output}" [agency] ; the agency-id of self-party id = ${agency_id} - ; the agency info - ${agency_info} ; the private-key for the psi-server private_key_path = conf/node.pem ; disable the ra2018 or not, default enable ra2018 @@ -376,6 +378,18 @@ generate_config_ini() { ; task_timeout_minutes = 180 ; thread_count = 8 +[transport] + ; the endpoint information + listen_ip = ${grpc_listen_ip} + listen_port = ${grpc_listen_port} + host_ip = + ; the threadPoolSize + thread_count = 4 + ; the gatewayService endpoint information + service.gateway_target = + ; the components + service.components = + [crypto] sm_crypto = ${sm_mode} @@ -385,14 +399,19 @@ generate_config_ini() { ; thread_count = 4 ; ssl or sm ssl sm_ssl=${sm_mode} - ; the max allowed message size in MBytes, default is 100MBytes - max_allow_msg_size = 100 + ; the max allowed message size in MBytes, default is 100MBytes + max_allow_msg_size = 100 ;ssl connection switch, if disable the ssl connection, default: false ;disable_ssl = true ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes holding_msg_minutes = 30 - ;disable_cache = false ;reconnect_time = 10000 + ; the unreachable distance + ;unreachable_distance=10 + ;the dir that contains the connected endpoint information, e.g.nodes.json + ;nodes_path=./ + ; the file that configure the connected endpoint information + ; nodes_path=nodes.json [rpc] listen_ip=${rpc_listen_ip} @@ -411,22 +430,6 @@ generate_config_ini() { ; directory the certificates located in cert_path=./conf -[cache] - ; the cache type, only support redis now - type = 0 - proxy = 127.0.0.1:20002 - obServer = 127.0.0.1:10904 - cluster = REDIS_CLUSTER - user = 1194 - host = 127.0.0.1 - port = 6379 - password = - database = 1 - pool_size = 16 - ; the redis connection timeout, in ms, default is 500ms - connection_timeout = 500 - ; the redis socket timeout, in ms, default is 500ms - socket_timeout = 500 [storage] host = 127.0.0.1 @@ -813,7 +816,6 @@ deploy_nodes() fi local agency_index=0 - local agency_info="" # generate the ca-cert ca_dir="${output_dir}"/ca generate_ca_cert "${sm_mode}" "${ca_dir}" @@ -847,8 +849,6 @@ deploy_nodes() # generate the node-script generate_node_scripts "${node_dir}" local port=$((gateway_listen_port + node_count)) - # generate the agency_info - agency_info="${agency_info}agency.agency${agency_index}=${ip}:${port} " ((agency_index += 1)) set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) @@ -872,9 +872,10 @@ deploy_nodes() node_dir="${output_dir}/${ip}/node${node_count}" local gateway_port=$((gateway_listen_port + node_count)) local rpc_port=$((rpc_listen_port + node_count)) + local grpc_port=$((grpc_listen_port + node_count)) local agency_id="agency${count}" private_key=$(generate_private_key "${node_dir}/conf") - generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${count}" + generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) done diff --git a/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp b/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp index 696ed196..9e2734c5 100644 --- a/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp +++ b/cpp/wedpr-computing/ppc-cem/src/CEMService.cpp @@ -156,7 +156,7 @@ void CEMService::makeCiphertextEqualityMatch( } }); CEM_LOG(INFO) << LOG_DESC("ciphertext equality match request handle ok") - << LOG_KV("timecost(ms)", utcSteadyTime() - startT); + << LOG_KV("timecost(ms)", utcSteadyTime() - startT); } void CEMService::makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func) @@ -266,7 +266,7 @@ void CEMService::encryptDataset(Json::Value const& request, Json::Value& respons } }); CEM_LOG(INFO) << LOG_DESC("encrypt dataset request handle ok") - << LOG_KV("timecost(ms)", utcSteadyTime() - startT); + << LOG_KV("timecost(ms)", utcSteadyTime() - startT); } void CEMService::encryptDatasetRpc(Json::Value const& request, RespFunc func) diff --git a/cpp/wedpr-computing/ppc-cem/src/CEMService.h b/cpp/wedpr-computing/ppc-cem/src/CEMService.h index 394a4c4c..52be9c1d 100644 --- a/cpp/wedpr-computing/ppc-cem/src/CEMService.h +++ b/cpp/wedpr-computing/ppc-cem/src/CEMService.h @@ -19,19 +19,13 @@ */ #pragma once #include "Common.h" +#include "ppc-framework/rpc/RpcInterface.h" #include "ppc-io/src/FileLineReader.h" #include "ppc-io/src/FileLineWriter.h" -#include "ppc-rpc/src/RpcFactory.h" -#include "ppc-tools/src/config/PPCConfig.h" +#include "ppc-tools/src/config/CEMConfig.h" +#include "ppc-tools/src/config/StorageConfig.h" #include #include - -using namespace bcos; -using namespace ppc; -using namespace ppc::io; -using namespace ppc::rpc; -using namespace ppc::tools; - namespace ppc::cem { class CEMService @@ -41,23 +35,26 @@ class CEMService CEMService() = default; virtual ~CEMService() = default; - void makeCiphertextEqualityMatchRpc(Json::Value const& request, RespFunc func); - void makeCiphertextEqualityMatch(Json::Value const& request, Json::Value& response, ppc::protocol::DataResourceType _type); - void encryptDatasetRpc(Json::Value const& request, RespFunc func); + void makeCiphertextEqualityMatchRpc(Json::Value const& request, ppc::rpc::RespFunc func); + void makeCiphertextEqualityMatch( + Json::Value const& request, Json::Value& response, ppc::protocol::DataResourceType _type); + void encryptDatasetRpc(Json::Value const& request, ppc::rpc::RespFunc func); void encryptDataset(Json::Value const& request, Json::Value& response); - void setCEMConfig(CEMConfig const& cemConfig); - void setStorageConfig(StorageConfig const& storageConfig); + void setCEMConfig(ppc::tools::CEMConfig const& cemConfig); + void setStorageConfig(ppc::tools::StorageConfig const& storageConfig); void doCipherTextEqualityMatch(const Json::Value::Members& fieldNames, - const std::vector& fieldValues, LineReader::Ptr lineReader, + const std::vector& fieldValues, ppc::io::LineReader::Ptr lineReader, Json::Value& matchCount); - void doEncryptDataset(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter); - LineReader::Ptr initialize_lineReader(const std::string& _datasetId, ppc::protocol::DataResourceType _type); - LineWriter::Ptr initialize_lineWriter(const std::string& _datasetId, ppc::protocol::DataResourceType _type); + void doEncryptDataset(ppc::io::LineReader::Ptr lineReader, ppc::io::LineWriter::Ptr lineWriter); + ppc::io::LineReader::Ptr initialize_lineReader( + const std::string& _datasetId, ppc::protocol::DataResourceType _type); + ppc::io::LineWriter::Ptr initialize_lineWriter( + const std::string& _datasetId, ppc::protocol::DataResourceType _type); void renameSource(const std::string& _datasetId, ppc::protocol::DataResourceType _type); private: - CEMConfig m_cemConfig; - StorageConfig m_storageConfig; + ppc::tools::CEMConfig m_cemConfig; + ppc::tools::StorageConfig m_storageConfig; }; } // namespace ppc::cem \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h b/cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h index eae1e03c..8bbd8fdf 100644 --- a/cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h +++ b/cpp/wedpr-computing/ppc-cem/src/wedpr_ffi_c_equality.h @@ -12,14 +12,14 @@ extern "C" { /** * C interface for 'encrypt_message'. */ -int8_t wedpr_pairing_bls128_encrypt_message(const CInputBuffer *raw_plaintext, - COutputBuffer *output_ciphertext); +int8_t wedpr_pairing_bls128_encrypt_message( + const CInputBuffer* raw_plaintext, COutputBuffer* output_ciphertext); /** * C interface for 'equality_test'. */ -int8_t wedpr_pairing_bls128_equality_test(const CInputBuffer *raw_cipher1, - const CInputBuffer *raw_cipher2); +int8_t wedpr_pairing_bls128_equality_test( + const CInputBuffer* raw_cipher1, const CInputBuffer* raw_cipher2); } #endif diff --git a/cpp/wedpr-computing/ppc-mpc/src/Common.h b/cpp/wedpr-computing/ppc-mpc/src/Common.h index 38a7d492..574c2656 100644 --- a/cpp/wedpr-computing/ppc-mpc/src/Common.h +++ b/cpp/wedpr-computing/ppc-mpc/src/Common.h @@ -25,7 +25,6 @@ namespace ppc::mpc { - DERIVE_PPC_EXCEPTION(MpcCompilerNotExistException); DERIVE_PPC_EXCEPTION(RunMpcFailException); DERIVE_PPC_EXCEPTION(OpenPipeFailException); diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp index 6f071e5a..d1631677 100644 --- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp +++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp @@ -42,6 +42,7 @@ using namespace ppc::io; using namespace ppc::protocol; using namespace ppc::tools; using namespace ppc::storage; +using namespace ppc::rpc; void MPCService::doRun(Json::Value const& request, Json::Value& response) diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h index 9827d6bd..7539a67d 100644 --- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h +++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h @@ -19,19 +19,14 @@ */ #pragma once #include "Common.h" +#include "ppc-framework/rpc/RpcInterface.h" #include "ppc-io/src/FileLineReader.h" #include "ppc-io/src/FileLineWriter.h" -#include "ppc-rpc/src/RpcFactory.h" -#include "ppc-tools/src/config/PPCConfig.h" +#include "ppc-tools/src/config/MPCConfig.h" +#include "ppc-tools/src/config/StorageConfig.h" #include #include -using namespace bcos; -using namespace ppc; -using namespace ppc::io; -using namespace ppc::rpc; -using namespace ppc::tools; - namespace ppc::mpc { class MPCService @@ -41,10 +36,10 @@ class MPCService MPCService() = default; virtual ~MPCService() = default; - void runMpcRpc(Json::Value const& request, RespFunc func); - void killMpcRpc(Json::Value const& request, RespFunc func); - void setMPCConfig(MPCConfig const& mpcConfig); - void setStorageConfig(StorageConfig const& storageConfig); + void runMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func); + void killMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func); + void setMPCConfig(ppc::tools::MPCConfig const& mpcConfig); + void setStorageConfig(ppc::tools::StorageConfig const& storageConfig); JobInfo paramsToJobInfo(const Json::Value& params); void makeCommand(std::string& cmd, const JobInfo& jobInfo); @@ -55,14 +50,14 @@ class MPCService void execCommand(const std::string cmd, int& outExitStatus, std::string& outResult); void writeStringToFile(const std::string& content, const std::string& filePath); - void readAndSaveFile(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter); - LineReader::Ptr initialize_lineReader(const JobInfo& jobInfo, const std::string& readerFilePath, - ppc::protocol::DataResourceType type); - LineWriter::Ptr initialize_lineWriter(const JobInfo& jobInfo, const std::string& writerFilePath, - ppc::protocol::DataResourceType type); + void readAndSaveFile(ppc::io::LineReader::Ptr lineReader, ppc::io::LineWriter::Ptr lineWriter); + ppc::io::LineReader::Ptr initialize_lineReader(const JobInfo& jobInfo, + const std::string& readerFilePath, ppc::protocol::DataResourceType type); + ppc::io::LineWriter::Ptr initialize_lineWriter(const JobInfo& jobInfo, + const std::string& writerFilePath, ppc::protocol::DataResourceType type); private: - MPCConfig m_mpcConfig; - StorageConfig m_storageConfig; + ppc::tools::MPCConfig m_mpcConfig; + ppc::tools::StorageConfig m_storageConfig; }; } // namespace ppc::mpc diff --git a/cpp/wedpr-computing/ppc-pir/src/Common.h b/cpp/wedpr-computing/ppc-pir/src/Common.h index 77051444..561a89aa 100644 --- a/cpp/wedpr-computing/ppc-pir/src/Common.h +++ b/cpp/wedpr-computing/ppc-pir/src/Common.h @@ -19,10 +19,11 @@ */ #pragma once #include "ppc-framework/Common.h" +#include #include -#include #include #include +#include #include #include #include @@ -35,8 +36,6 @@ #include #include #include -#include -#include #define PIR_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PIR") @@ -46,7 +45,6 @@ namespace ppc::pir const std::string DEFAULT_PREFIX = "PPC_DEFAULT_MESG"; - enum class OTPIRMessageType : uint8_t { HELLO_RECEIVER = 0x01, @@ -63,7 +61,8 @@ enum class OTPIRRetCode : int inline bool isPrefixMatched(const bcos::bytes& _prefix, const char* begin, const char* end) { size_t prefixLen = _prefix.size(); - if (end - begin < prefixLen) return false; + if (end - begin < prefixLen) + return false; for (size_t i = 0; i < prefixLen; ++i) { if (_prefix[i] != *(begin + i)) @@ -73,7 +72,8 @@ inline bool isPrefixMatched(const bcos::bytes& _prefix, const char* begin, const } // 从Task param解析 -struct PirTaskMessage { +struct PirTaskMessage +{ std::string searchId; std::string requestAgencyId; std::string requestAgencyDataset; @@ -121,7 +121,7 @@ inline std::vector> readInputsByMmapWithPref ++end; } // 判断_prefix和buf中begin, end这一行的前缀是否相同 - if(isPrefixMatched(_prefix, begin, end)) + if (isPrefixMatched(_prefix, begin, end)) { std::string st(begin, end); std::stringstream ss(st); @@ -152,8 +152,8 @@ inline PirTaskMessage parseJson(std::string_view _param) Json::Value result; if (!reader.parse(_param.begin(), _param.end(), result)) { - BOOST_THROW_EXCEPTION(BCOS_ERROR( - (int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param: invalid json")); + BOOST_THROW_EXCEPTION( + BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param: invalid json")); } PIR_LOG(TRACE) << LOG_BADGE("result type") LOG_KV("result", result.type()); if (!result.isObject() || result.empty()) @@ -163,15 +163,18 @@ inline PirTaskMessage parseJson(std::string_view _param) } PirTaskMessage taskMessage; - if(!result.isMember("searchId")){ + if (!result.isMember("searchId")) + { BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param:: the param searchId not found")); } - if(!result.isMember("requestAgencyId")){ + if (!result.isMember("requestAgencyId")) + { BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param:: the param requestAgencyId not found")); } - if(!result.isMember("prefixLength")){ + if (!result.isMember("prefixLength")) + { BOOST_THROW_EXCEPTION(BCOS_ERROR((int)OTPIRRetCode::INVALID_TASK_PARAM, "invalid task param:: the param prefixLength not found")); } diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h index 9f6deb6f..3ecd2033 100644 --- a/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h +++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h @@ -31,11 +31,9 @@ #include "ppc-psi/src/PSIConfig.h" #include "protocol/src/PPCMessage.h" -using namespace ppc::psi; - namespace ppc::pir { -class OtPIRConfig : public PSIConfig +class OtPIRConfig : public ppc::psi::PSIConfig { public: using Ptr = std::shared_ptr; diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h index caf29cd9..eb86632f 100644 --- a/cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h +++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRFactory.h @@ -21,9 +21,6 @@ #include "OtPIRImpl.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/io/DataResourceLoader.h" -#include "ppc-tools/src/config/PPCConfig.h" - -using namespace ppc::psi; namespace ppc::pir { @@ -39,10 +36,10 @@ class OtPIRFactory bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes) { - auto config = std::make_shared(_selfParty, std::move(_front), - std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), - _holdingMessageMinutes); + auto config = + std::make_shared(_selfParty, std::move(_front), std::move(_cryptoBox), + std::move(_threadPool), std::move(_dataResourceLoader), _holdingMessageMinutes); return std::make_shared(config); } }; -} // namespace ppc::psi \ No newline at end of file +} // namespace ppc::pir \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp index cd39535b..d0d2d383 100644 --- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp +++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp @@ -33,6 +33,8 @@ using namespace bcos; using namespace ppc::io; using namespace ppc::protocol; using namespace ppc::pir; +using namespace ppc::task; +using namespace ppc::psi; OtPIRImpl::OtPIRImpl(const OtPIRConfig::Ptr& _config, unsigned _idleTimeMs) diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h index 5d3fea19..971714f7 100644 --- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h +++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h @@ -20,33 +20,25 @@ #pragma once #include "Common.h" // #include "OtPIR.h" +#include "BaseOT.h" +#include "OtPIRConfig.h" #include "ppc-framework/protocol/DataResource.h" -#include "ppc-rpc/src/RpcFactory.h" -#include "ppc-tools/src/config/PPCConfig.h" -#include -#include #include "ppc-framework/protocol/Task.h" #include "ppc-framework/task/TaskFrameworkInterface.h" -#include -#include #include "ppc-psi/src/psi-framework/TaskGuarder.h" -#include "OtPIRConfig.h" +#include "ppc-rpc/src/RpcFactory.h" +#include #include -#include "BaseOT.h" - - -using namespace bcos; -using namespace ppc; -using namespace ppc::rpc; -using namespace ppc::tools; -using namespace ppc::task; +#include +#include +#include namespace ppc::pir { class OtPIRImpl : public std::enable_shared_from_this, public bcos::Worker, - public psi::TaskGuarder, // taskGuarder并不一定属于psi-framework - public TaskFrameworkInterface + public psi::TaskGuarder, // taskGuarder并不一定属于psi-framework + public ppc::task::TaskFrameworkInterface // class OtPIRImpl : public std::enable_shared_from_this { public: @@ -58,13 +50,13 @@ class OtPIRImpl : public std::enable_shared_from_this, using OtPIRMsgQueuePtr = std::shared_ptr; // run task - void asyncRunTask( - ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + void asyncRunTask(ppc::protocol::Task::ConstPtr _task, + ppc::task::TaskResponseCallback&& _onTaskFinished) override; - void start() override; - void stop() override; + void start() override; + void stop() override; - // register to the front to get the message related to ot-pir + // register to the front to get the message related to ot-pir void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; void onReceivedErrorNotification(const std::string& _taskID) override; @@ -74,7 +66,7 @@ class OtPIRImpl : public std::enable_shared_from_this, // ot-pir main processing function // for ut to make this function public void executeWorker() override; - + void handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _message); void onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message); @@ -88,7 +80,8 @@ class OtPIRImpl : public std::enable_shared_from_this, // void encryptDataset(Json::Value const& request, Json::Value& response); // void setAYSConfig(AYSConfig const& aysConfig); - // std::vector prepareDataset(bcos::bytes sendObfuscatedHash, std::string datasetPath); + // std::vector prepareDataset(bcos::bytes sendObfuscatedHash, std::string + // datasetPath); protected: void asyncRunTask(); @@ -100,7 +93,8 @@ class OtPIRImpl : public std::enable_shared_from_this, // void runReceiverGenerateMessage(ppctars::SenderMessageParams senderMessageParams); // void runFinishSender(PirTaskMessage taskMessage); - void addTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) + void addTask( + ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished) { bcos::WriteGuard l(x_taskQueue); m_taskQueue.push({std::move(_task), std::move(_onTaskFinished)}); @@ -160,7 +154,7 @@ class OtPIRImpl : public std::enable_shared_from_this, bool m_enableOutputExists = false; // 为true时启动时会从配置中加载文件作为匹配源 bool m_enableMemoryFile = false; - protocol::DataResource m_resource; + ppc::protocol::DataResource m_resource; private: @@ -192,24 +186,21 @@ class OtPIRImpl : public std::enable_shared_from_this, // TODO: 改为prefix-message的map 预处理的数据集 - std::vector> messageKeypair; - std::queue > m_taskQueue; + std::vector> messageKeypair; + std::queue> + m_taskQueue; mutable bcos::SharedMutex x_taskQueue; - + // // setup的系统参数，分桶大小来决定披露前k个bit 目前版本由TASK先传入 // const int obfuscation_order = 6; std::string m_taskID; - TaskState::Ptr m_taskState; - protocol::TaskResult::Ptr m_taskResult; - - - + ppc::psi::TaskState::Ptr m_taskState; + ppc::protocol::TaskResult::Ptr m_taskResult; void waitSignal() { boost::unique_lock l(x_signal); m_signal.wait_for(l, boost::chrono::milliseconds(5)); } - }; } // namespace ppc::pir \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h index 4e1efc2e..9a047f7b 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h @@ -21,7 +21,6 @@ #include "CM2020PSIImpl.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/io/DataResourceLoader.h" -#include "ppc-tools/src/config/PPCConfig.h" namespace ppc::psi { diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp index 197670f9..d4534b5c 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp @@ -28,6 +28,7 @@ using namespace ppc::front; using namespace ppc::tools; using namespace ppc::crypto; using namespace ppc::io; +using namespace ppc::task; CM2020PSIImpl::CM2020PSIImpl(const CM2020PSIConfig::Ptr& _config, unsigned _idleTimeMs) : Worker("CM2020-PSI", _idleTimeMs), diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h index 7cd416e0..003fea3c 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h @@ -42,7 +42,7 @@ namespace ppc::psi { class CM2020PSIImpl : public bcos::Worker, public TaskGuarder, - public TaskFrameworkInterface, + public ppc::task::TaskFrameworkInterface, public std::enable_shared_from_this { public: @@ -54,8 +54,8 @@ class CM2020PSIImpl : public bcos::Worker, ~CM2020PSIImpl() override = default; - void asyncRunTask( - ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + void asyncRunTask(ppc::protocol::Task::ConstPtr _task, + ppc::task::TaskResponseCallback&& _onTaskFinished) override; // register to the front to get the message related to cm2020-psi void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; @@ -91,7 +91,8 @@ class CM2020PSIImpl : public bcos::Worker, void onReceiveResultCount(ppc::front::PPCMessageFace::Ptr _message); void onReceiveResults(ppc::front::PPCMessageFace::Ptr _message); - void addTask(ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) + void addTask( + ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished) { bcos::WriteGuard l(x_taskQueue); m_taskQueue.push({std::move(_task), std::move(_onTaskFinished)}); @@ -169,7 +170,8 @@ class CM2020PSIImpl : public bcos::Worker, std::shared_ptr m_thread; std::atomic m_parallelism; - std::queue > m_taskQueue; + std::queue > + m_taskQueue; mutable bcos::SharedMutex x_taskQueue; std::unordered_map m_receivers; diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp index 55e22be3..81b5339f 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp @@ -28,6 +28,7 @@ using namespace ppc::tools; using namespace ppc::crypto; using namespace ppc::io; using namespace bcos; +using namespace ppc::task; EcdhConnPSIImpl::EcdhConnPSIImpl(const EcdhConnPSIConfig::Ptr& _config, unsigned _idleTimeMs) : m_config(std::move(_config)), diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h index 831356d9..01ee4303 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnTaskState.h @@ -29,8 +29,8 @@ class EcdhConnTaskState : public TaskState { public: using Ptr = std::shared_ptr; - EcdhConnTaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, - bool _onlySelfRun = false) + EcdhConnTaskState(ppc::protocol::Task::ConstPtr const& _task, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false) : TaskState(_task, std::move(_callback), _onlySelfRun) {} ~EcdhConnTaskState() = default; @@ -77,7 +77,7 @@ class EcdhConnTaskStateFactory : public TaskStateFactory ~EcdhConnTaskStateFactory() override = default; EcdhConnTaskState::Ptr createConnTaskState(ppc::protocol::Task::ConstPtr const& _task, - TaskResponseCallback&& _callback, bool _onlySelfRun = false, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false, PSIConfig::Ptr _config = nullptr) { return std::make_shared(_task, std::move(_callback), false); diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h index 58280282..91718051 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.h @@ -22,7 +22,6 @@ #include "../Common.h" #include "PSIConnMessage.h" #include "ppc-framework/Common.h" -#include "ppc-tools/src/config/PPCConfig.h" namespace ppc::psi { diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp new file mode 100644 index 00000000..d78de97c --- /dev/null +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhMultiPSIFactory.cpp + */ + +#include "EcdhMultiPSIFactory.h" +#include "EcdhMultiPSIMessageFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" + +using namespace ppc::psi; +using namespace ppc::front; +using namespace ppc::tools; +using namespace ppc::io; +using namespace ppc::crypto; + +EcdhMultiPSIImpl::Ptr EcdhMultiPSIFactory::createEcdhMultiPSI(PPCConfig::Ptr const& _ppcConfig, + FrontInterface::Ptr _front, CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + DataResourceLoader::Ptr _dataResourceLoader) +{ + auto psiMsgFactory = std::make_shared(); + auto const& ecdhParam = _ppcConfig->ecdhMultiPSIConfig(); + auto _selfParty = _ppcConfig->agencyID(); + int _holdingMessageMinutes = _ppcConfig->holdingMessageMinutes(); + auto config = std::make_shared(_selfParty, std::move(_front), + std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), + ecdhParam.dataBatchSize, _holdingMessageMinutes, psiMsgFactory); + return std::make_shared(config); +} diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h index f6d0090b..a02b0ac8 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h @@ -1,10 +1,30 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhMultiPSIFactory.h + */ + #pragma once #include "EcdhMultiPSIImpl.h" -#include "EcdhMultiPSIMessageFactory.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/io/DataResourceLoader.h" -#include "ppc-tools/src/config/PPCConfig.h" +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::psi { class EcdhMultiPSIFactory @@ -14,18 +34,9 @@ class EcdhMultiPSIFactory EcdhMultiPSIFactory() = default; virtual ~EcdhMultiPSIFactory() = default; - virtual EcdhMultiPSIImpl::Ptr createEcdhMultiPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + virtual EcdhMultiPSIImpl::Ptr createEcdhMultiPSI( + std::shared_ptr const& _ppcConfig, ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, - bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader) - { - auto psiMsgFactory = std::make_shared(); - auto const& ecdhParam = _ppcConfig->ecdhMultiPSIConfig(); - auto _selfParty = _ppcConfig->agencyID(); - int _holdingMessageMinutes = _ppcConfig->holdingMessageMinutes(); - auto config = std::make_shared(_selfParty, std::move(_front), - std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader), - ecdhParam.dataBatchSize, _holdingMessageMinutes, psiMsgFactory); - return std::make_shared(config); - } + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader); }; } // namespace ppc::psi \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp index 24fa7194..e3e9b9bb 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp @@ -7,6 +7,7 @@ using namespace ppc::front; using namespace ppc::crypto; using namespace ppc::io; using namespace bcos; +using namespace ppc::task; EcdhMultiPSIImpl::EcdhMultiPSIImpl(const EcdhMultiPSIConfig::Ptr& _config, unsigned _idleTimeMs) diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp new file mode 100644 index 00000000..16d3aa94 --- /dev/null +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EcdhPSIFactory.cpp + * @author: yujiechen + * @date 2022-12-28 + */ +#include "EcdhPSIFactory.h" +#include "EcdhPSIMessageFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" +#include + +using namespace ppc::psi; +using namespace ppc::tools; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::io; + +EcdhPSIImpl::Ptr EcdhPSIFactory::createEcdhPSI(PPCConfig::Ptr const& _ppcConfig, + ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, FrontInterface::Ptr _front, + PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool, + DataResourceLoader::Ptr const& _dataResourceLoader) +{ + auto psiMsgFactory = std::make_shared(); + auto const& ecdhParam = _ppcConfig->ecdhPSIConfig(); + auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory, + _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize, + _ppcConfig->holdingMessageMinutes(), _threadPool); + return std::make_shared(config); +} diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h index 8513524a..2f0c8a4f 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h @@ -19,9 +19,11 @@ */ #pragma once #include "EcdhPSIImpl.h" -#include "EcdhPSIMessageFactory.h" -#include "ppc-tools/src/config/PPCConfig.h" #include +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::psi { class EcdhPSIFactory @@ -31,18 +33,10 @@ class EcdhPSIFactory EcdhPSIFactory() = default; virtual ~EcdhPSIFactory() = default; - virtual EcdhPSIImpl::Ptr createEcdhPSI(ppc::tools::PPCConfig::Ptr const& _ppcConfig, + virtual EcdhPSIImpl::Ptr createEcdhPSI(std::shared_ptr const& _ppcConfig, ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, ppc::front::FrontInterface::Ptr _front, ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool, - ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader) - { - auto psiMsgFactory = std::make_shared(); - auto const& ecdhParam = _ppcConfig->ecdhPSIConfig(); - auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory, - _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize, - _ppcConfig->holdingMessageMinutes(), _threadPool); - return std::make_shared(config); - } + ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader); }; } // namespace ppc::psi \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp index 051ee101..f792b190 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp @@ -25,6 +25,7 @@ using namespace ppc::protocol; using namespace ppc::crypto; using namespace ppc::io; using namespace bcos; +using namespace ppc::task; void EcdhPSIImpl::asyncRunTask( ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h index 4e8dad12..107280bf 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h @@ -40,8 +40,8 @@ class EcdhPSIImpl : public PSIFramework, public std::enable_shared_from_this; - EcdhTaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, - bool _onlySelfRun = false) + EcdhTaskState(ppc::protocol::Task::ConstPtr const& _task, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false) : TaskState(_task, std::move(_callback), _onlySelfRun) {} ~EcdhTaskState() override = default; @@ -59,7 +59,7 @@ class EcdhTaskStateFactory : public TaskStateFactory ~EcdhTaskStateFactory() override = default; TaskState::Ptr createTaskState(ppc::protocol::Task::ConstPtr const& _task, - TaskResponseCallback&& _callback, bool _onlySelfRun = false, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false, PSIConfig::Ptr _config = nullptr) override { return std::make_shared(_task, std::move(_callback), false); diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp new file mode 100644 index 00000000..3e6558b2 --- /dev/null +++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file LabeledPSIFactory.cpp + * @author: shawnhe + * @date 2022-11-14 + */ +#include "LabeledPSIFactory.h" + +using namespace ppc::psi; +using namespace ppc::crypto; +using namespace ppc::io; +using namespace ppc::front; + +LabeledPSIImpl::Ptr LabeledPSIFactory::buildLabeledPSI(std::string const& _selfParty, + FrontInterface::Ptr _front, CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, + DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes) +{ + auto config = + std::make_shared(_selfParty, std::move(_front), std::move(_cryptoBox), + std::move(_threadPool), std::move(_dataResourceLoader), _holdingMessageMinutes); + return std::make_shared(config); +} \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h index ba8d20ee..36bfeb08 100644 --- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h @@ -21,7 +21,6 @@ #include "LabeledPSIImpl.h" #include "ppc-framework/crypto/CryptoBox.h" #include "ppc-framework/io/DataResourceLoader.h" -#include "ppc-tools/src/config/PPCConfig.h" namespace ppc::psi { @@ -35,12 +34,6 @@ class LabeledPSIFactory virtual LabeledPSIImpl::Ptr buildLabeledPSI(std::string const& _selfParty, ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader, - int _holdingMessageMinutes) - { - auto config = - std::make_shared(_selfParty, std::move(_front), std::move(_cryptoBox), - std::move(_threadPool), std::move(_dataResourceLoader), _holdingMessageMinutes); - return std::make_shared(config); - } + int _holdingMessageMinutes); }; } // namespace ppc::psi \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp index c9726b47..290b1f83 100644 --- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp @@ -33,6 +33,7 @@ using namespace ppc::front; using namespace ppc::tools; using namespace ppc::crypto; using namespace ppc::io; +using namespace ppc::task; LabeledPSIImpl::LabeledPSIImpl(LabeledPSIConfig::Ptr _config, unsigned _idleTimeMs) : Worker("LabeledPSI", _idleTimeMs), diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h index 5765187b..9cd06c4e 100644 --- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h +++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h @@ -40,7 +40,7 @@ namespace ppc::psi { class LabeledPSIImpl : public bcos::Worker, public TaskGuarder, - public TaskFrameworkInterface, + public ppc::task::TaskFrameworkInterface, public std::enable_shared_from_this { public: @@ -53,8 +53,8 @@ class LabeledPSIImpl : public bcos::Worker, virtual ~LabeledPSIImpl() = default; // run task - void asyncRunTask( - ppc::protocol::Task::ConstPtr _task, TaskResponseCallback&& _onTaskFinished) override; + void asyncRunTask(ppc::protocol::Task::ConstPtr _task, + ppc::task::TaskResponseCallback&& _onTaskFinished) override; // register to the front to get the message related to labeled-psi void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override; @@ -71,8 +71,8 @@ class LabeledPSIImpl : public bcos::Worker, void executeWorker() override; protected: - void asyncRunSenderTask( - const ppc::protocol::Task::ConstPtr& _task, TaskResponseCallback&& _onTaskFinished); + void asyncRunSenderTask(const ppc::protocol::Task::ConstPtr& _task, + ppc::task::TaskResponseCallback&& _onTaskFinished); void checkFinishedTask(); diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp index 850b2804..fec26576 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp @@ -21,6 +21,7 @@ #include "../Common.h" #include "ppc-framework/protocol/GlobalConfig.h" +using namespace ppc::task; using namespace ppc::psi; using namespace ppc::io; using namespace ppc::protocol; diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h index 9f44414a..3bab2dc8 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h @@ -33,12 +33,11 @@ #define PSI_FRAMEWORK_LOG(LEVEL) \ BCOS_LOG(LEVEL) << LOG_BADGE("PSI") \ << LOG_BADGE((ppc::protocol::PSIAlgorithmType)m_psiConfig->algorithmType()) -using namespace ppc::task; namespace ppc::psi { using PSIMsgQueue = bcos::ConcurrentQueue; using PSIMsgQueuePtr = std::shared_ptr; -class PSIFramework : public bcos::Worker, public TaskFrameworkInterface +class PSIFramework : public bcos::Worker, public ppc::task::TaskFrameworkInterface { public: using Ptr = std::shared_ptr; diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h index 2552533c..24c2b3dc 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h @@ -30,15 +30,16 @@ #include #include #include -using namespace ppc::task; + namespace ppc::psi { class TaskState : public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - TaskState(ppc::protocol::Task::ConstPtr const& _task, TaskResponseCallback&& _callback, - bool _onlySelfRun = false, PSIConfig::Ptr _config = nullptr) + TaskState(ppc::protocol::Task::ConstPtr const& _task, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false, + PSIConfig::Ptr _config = nullptr) : m_task(_task), m_callback(std::move(_callback)), m_onlySelfRun(_onlySelfRun), @@ -49,8 +50,8 @@ class TaskState : public std::enable_shared_from_this virtual ~TaskState() {} - TaskResponseCallback const& callback() { return m_callback; } - TaskResponseCallback takeCallback() { return std::move(m_callback); } + ppc::task::TaskResponseCallback const& callback() { return m_callback; } + ppc::task::TaskResponseCallback takeCallback() { return std::move(m_callback); } bool onlySelfRun() { return m_onlySelfRun; } void setReader(io::LineReader::Ptr _reader, int64_t _readerParam) { @@ -400,7 +401,7 @@ class TaskState : public std::enable_shared_from_this protected: ppc::protocol::Task::ConstPtr m_task; - TaskResponseCallback m_callback; + ppc::task::TaskResponseCallback m_callback; bool m_onlySelfRun{false}; PSIConfig::Ptr m_config; uint64_t m_taskStartTime = 0; @@ -449,7 +450,7 @@ class TaskStateFactory virtual ~TaskStateFactory() = default; virtual TaskState::Ptr createTaskState(ppc::protocol::Task::ConstPtr const& _task, - TaskResponseCallback&& _callback, bool _onlySelfRun = false, + ppc::task::TaskResponseCallback&& _callback, bool _onlySelfRun = false, PSIConfig::Ptr _config = nullptr) { return std::make_shared( diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp new file mode 100644 index 00000000..5a589914 --- /dev/null +++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file RA2018PSIFactory.cpp + * @author: yujiechen + * @date 2022-11-14 + */ +#include "RA2018PSIFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" + +using namespace ppc::psi; +using namespace ppc::front; +using namespace ppc::crypto; +using namespace ppc::storage; +using namespace ppc::tools; +using namespace ppc::io; + +RA2018PSIImpl::Ptr RA2018PSIFactory::createRA2018PSI(std::string const& _selfParty, + FrontInterface::Ptr _front, PPCConfig::Ptr const& _config, + RA2018OprfInterface::Ptr const& _oprf, Hash::Ptr _binHashImpl, + PPCMessageFaceFactory::Ptr _ppcMsgFactory, SQLStorage::Ptr _storage, + FileStorage::Ptr _fileStorage, bcos::ThreadPool::Ptr _threadPool, + DataResourceLoader::Ptr _dataResourceLoader) +{ + auto const& ra2018Config = _config->ra2018PSIConfig(); + + auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl, + _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage, + _dataResourceLoader, _config->holdingMessageMinutes(), ra2018Config.dbName, + ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize, ra2018Config.dataBatchSize); + + RA2018PSIStorage::Ptr psiStorage = nullptr; + if (!_config->disableRA2018()) + { + psiStorage = std::make_shared(config); + // init the storage + psiStorage->init(); + } + return std::make_shared(config, psiStorage, 0, false, _config->disableRA2018()); +} diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h index a75c742a..ed107105 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h +++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h @@ -19,8 +19,11 @@ */ #pragma once #include "RA2018PSIImpl.h" -#include "ppc-tools/src/config/PPCConfig.h" #include "storage/RA2018PSIStorage.h" +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::psi { class RA2018PSIFactory @@ -31,28 +34,11 @@ class RA2018PSIFactory virtual ~RA2018PSIFactory() = default; virtual RA2018PSIImpl::Ptr createRA2018PSI(std::string const& _selfParty, - ppc::front::FrontInterface::Ptr _front, ppc::tools::PPCConfig::Ptr const& _config, + ppc::front::FrontInterface::Ptr _front, + std::shared_ptr const& _config, ppc::crypto::RA2018OprfInterface::Ptr const& _oprf, ppc::crypto::Hash::Ptr _binHashImpl, ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage, - bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader) - { - auto const& ra2018Config = _config->ra2018PSIConfig(); - - auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl, - _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage, - _dataResourceLoader, _config->holdingMessageMinutes(), ra2018Config.dbName, - ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize, ra2018Config.dataBatchSize); - - RA2018PSIStorage::Ptr psiStorage = nullptr; - if (!_config->disableRA2018()) - { - psiStorage = std::make_shared(config); - // init the storage - psiStorage->init(); - } - return std::make_shared( - config, psiStorage, 0, false, _config->disableRA2018()); - } + bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader); }; } // namespace ppc::psi \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp index 7decb401..7b98051b 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp @@ -28,6 +28,7 @@ using namespace ppc::protocol; using namespace bcos; using namespace ppc::front; using namespace ppc::io; +using namespace ppc::task; RA2018PSIImpl::RA2018PSIImpl(RA2018PSIConfig::Ptr const& _config, RA2018PSIStorage::Ptr const& _storage, unsigned _idleTimeMs, bool _waitResult, bool _disabled) diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h index d8d376b7..d47c7253 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h +++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h @@ -58,8 +58,8 @@ class RA2018PSIImpl : public PSIFramework, public std::enable_shared_from_this +#include + +namespace ppc::tools +{ +struct CEMConfig +{ + std::string datasetFilePath; + std::string datasetHDFSPath; + std::string ciphertextSuffix; + uint64_t readPerBatchLines; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h new file mode 100644 index 00000000..2e342582 --- /dev/null +++ b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MPCConfig.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include +#include + +namespace ppc::tools +{ +struct MPCConfig +{ + std::string datasetHDFSPath; + std::string jobPath; + std::string mpcRootPath; + std::string mpcRootPathNoGateway; + uint64_t readPerBatchLines; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index f644bd6d..127675fe 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -25,30 +25,20 @@ #include using namespace ppc::tools; +using namespace ppc::front; using namespace ppc::protocol; using namespace ppc::storage; using namespace bcos; -void PPCConfig::loadGatewayConfig( - NodeArch _arch, const char* _certPath, boost::property_tree::ptree const& _pt) +void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt) { // load the network config PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config"); // gateway default enable-ssl - loadNetworkConfig(m_gatewayConfig.networkConfig, _certPath, _pt, "gateway", - NetworkConfig::DefaultRpcListenPort, false); + loadNetworkConfig( + m_gatewayConfig.networkConfig, _pt, "gateway", NetworkConfig::DefaultRpcListenPort, false); PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config success"); - m_gatewayConfig.disableCache = _pt.get("gateway.disable_cache", false); - - // load the redis config - if (_arch == NodeArch::PRO && !m_gatewayConfig.disableCache) - { - PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the redis config"); - initRedisConfigForGateway(m_gatewayConfig.cacheStorageConfig, _pt); - PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the redis config success"); - } - m_gatewayConfig.nodePath = _pt.get("gateway.nodes_path", "./"); m_gatewayConfig.nodeFileName = _pt.get("gateway.nodes_file", "nodes.json"); @@ -75,12 +65,104 @@ void PPCConfig::loadGatewayConfig( } // load the holdingMessageMinutes, in minutes m_holdingMessageMinutes = loadHoldingMessageMinutes(_pt, "gateway.holding_msg_minutes"); + // load the grpcConfig + m_grpcConfig = loadGrpcConfig("transport", _pt); + // load the GrpcServerConfig + loadEndpointConfig(m_gatewayConfig.grpcServerConfig.endPoint, false, "transport", _pt); + // the agencyID + m_agencyID = _pt.get("agency.id", ""); + if (m_agencyID.empty()) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment("Must set agency.id")); + } PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("maxAllowedMsgSize", m_gatewayConfig.maxAllowedMsgSize) << LOG_KV("reconnectTime", m_gatewayConfig.reconnectTime) << LOG_KV("holdingMessageMinutes", m_holdingMessageMinutes); } +void PPCConfig::loadEndpointConfig(EndPoint& endPoint, bool requireHostIp, + std::string const& sectionName, boost::property_tree::ptree const& pt) +{ + // the host ip + auto hostIp = pt.get(sectionName + ".host_ip", "127.0.0.1"); + if (requireHostIp && hostIp.empty()) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Must specify the " + sectionName + ".host_ip!")); + } + endPoint.setHost(hostIp); + // the listen ip + auto listenIp = pt.get(sectionName + ".listen_ip", "0.0.0.0"); + if (listenIp.empty()) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Must specify the " + sectionName + ".listen_ip!")); + } + endPoint.setListenIp(listenIp); + // the listen port + auto listenPort = pt.get(sectionName + ".listen_port", 18000); + checkPort(sectionName + ".listen_port", listenPort); + endPoint.setPort(listenPort); +} + +void PPCConfig::loadFrontConfig( + FrontConfigBuilder::Ptr const& frontConfigBuilder, boost::property_tree::ptree const& pt) +{ + if (m_frontConfig == nullptr) + { + m_frontConfig = frontConfigBuilder->build(); + } + loadEndpointConfig(m_frontConfig->mutableSelfEndPoint(), true, "transport", pt); + // the thread_count + auto threadCount = pt.get("transport.thread_count", 4); + if (threadCount == 0) + { + threadCount = 2; + } + // TODO: load from the pem file + auto nodeID = pt.get("transport.nodeid", ""); + if (nodeID.empty()) + { + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Must specify the transport.nodeid!")); + } + m_frontConfig->setNodeID(nodeID); + + m_frontConfig->setThreadPoolSize(threadCount); + // the gateway targets + auto gatewayTargets = pt.get("transport.service.gateway_target", ""); + if (gatewayTargets.empty()) + { + BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( + "Must specify the transport.service.gateway_target!")); + } + // the components + auto components = pt.get("transport.service.components", ""); + boost::split(m_frontConfig->mutableComponents(), components, boost::is_any_of(",")); +} + +void PPCConfig::setPrivateKey(bcos::bytes const& _privateKey) +{ + m_privateKey = _privateKey; +} + +GrpcConfig::Ptr PPCConfig::loadGrpcConfig( + std::string const& sectionName, boost::property_tree::ptree const& pt) +{ + // the load balance config + auto loadBalancePolicy = + pt.get(sectionName + ".load_balance_policy", "round_robin"); + auto grpcConfig = std::make_shared(); + if (!loadBalancePolicy.empty()) + { + grpcConfig->setLoadBalancePolicy(loadBalancePolicy); + } + PPCConfig_LOG(INFO) << LOG_DESC("loadGrpcConfig") << LOG_KV("section", sectionName) + << LOG_KV("loadBalancePolicy", grpcConfig->loadBalancePolicy()); + return grpcConfig; +} + int PPCConfig::loadHoldingMessageMinutes( const boost::property_tree::ptree& _pt, std::string const& _section) { @@ -93,7 +175,7 @@ int PPCConfig::loadHoldingMessageMinutes( return holdingMessageMinutes; } -void PPCConfig::initRedisConfigForGateway( +void PPCConfig::loadCachedStorageConfig( CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt) { _redisConfig.type = ppc::protocol::CacheType(_pt.get("cache.type", 0)); @@ -130,9 +212,8 @@ void PPCConfig::initRedisConfigForGateway( } -void PPCConfig::loadNetworkConfig(NetworkConfig& _config, const char* _certPath, - boost::property_tree::ptree const& _pt, std::string const& _sectionName, int _defaultListenPort, - bool _defaultDisableSSl) +void PPCConfig::loadNetworkConfig(NetworkConfig& _config, boost::property_tree::ptree const& _pt, + std::string const& _sectionName, int _defaultListenPort, bool _defaultDisableSSl) { // the rpcListenIp _config.listenIp = _pt.get(_sectionName + ".listen_ip", "0.0.0.0"); @@ -166,15 +247,9 @@ void PPCConfig::loadNetworkConfig(NetworkConfig& _config, const char* _certPath, // enable sm-rpc or not _config.enableSM = _pt.get(_sectionName + ".sm_ssl", false); - // the rpc cert-path - if (_certPath == nullptr) - { - _config.certPath = _pt.get("cert.cert_path", "conf"); - } - else - { - _config.certPath = _certPath; - } + // the cert-path + _config.certPath = _pt.get("cert.cert_path", "conf"); + PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_KV("section", _sectionName) << LOG_KV("certPath", _config.certPath); @@ -409,7 +484,7 @@ int64_t PPCConfig::getDataBatchSize(std::string const& _section, int64_t _dataBa return dataBatchSize; } -void PPCConfig::loadCommonConfig(boost::property_tree::ptree const& _pt) +void PPCConfig::loadCommonNodeConfig(boost::property_tree::ptree const& _pt) { m_agencyID = _pt.get("agency.id", ""); if (m_agencyID.empty()) @@ -432,8 +507,9 @@ void PPCConfig::loadCommonConfig(boost::property_tree::ptree const& _pt) m_threadPoolSize = _pt.get( "agency.thread_count", static_cast(std::thread::hardware_concurrency() * 0.75)); - PPCConfig_LOG(INFO) << LOG_DESC("loadCommonConfig success") << LOG_KV("agencyID", m_agencyID) - << LOG_KV("dataLocation", m_dataLocation) << LOG_KV("smCrypto", m_smCrypto) + PPCConfig_LOG(INFO) << LOG_DESC("loadCommonNodeConfig success") + << LOG_KV("agencyID", m_agencyID) << LOG_KV("dataLocation", m_dataLocation) + << LOG_KV("smCrypto", m_smCrypto) << LOG_KV("taskTimeoutMinutes", m_taskTimeoutMinutes) << LOG_KV("threadPoolSize", m_threadPoolSize); } @@ -526,4 +602,4 @@ void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt) m_mpcConfig.mpcRootPathNoGateway = _pt.get("mpc.mpc_root_path_no_gateway", "./"); m_mpcConfig.readPerBatchLines = _pt.get("mpc.read_per_batch_lines", 100000); loadHDFSConfig(_pt); -} +} \ No newline at end of file diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h index f5b70e75..56b5249f 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h @@ -19,17 +19,21 @@ */ #pragma once #include "../cuckoo/Common.h" +#include "CEMConfig.h" #include "Common.h" +#include "MPCConfig.h" #include "NetworkConfig.h" #include "ParamChecker.h" +#include "StorageConfig.h" #include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/EndPoint.h" #include "ppc-framework/protocol/GrpcConfig.h" #include "ppc-framework/storage/CacheStorage.h" #include -#include #include #include #include +#include namespace ppc::tools { @@ -60,35 +64,6 @@ struct RA2018Config bool useHDFS = false; }; -struct StorageConfig -{ - ppc::protocol::SQLConnectionOption::Ptr sqlConnectionOpt; - ppc::protocol::FileStorageConnectionOption::Ptr fileStorageConnectionOpt; - ppc::protocol::RemoteStorageConnectionOption::Ptr remoteConnectionOpt; -}; - -struct CEMConfig -{ - std::string datasetFilePath; - std::string datasetHDFSPath; - std::string ciphertextSuffix; - uint64_t readPerBatchLines; -}; - -struct MPCConfig -{ - std::string datasetHDFSPath; - std::string jobPath; - std::string mpcRootPath; - std::string mpcRootPathNoGateway; - uint64_t readPerBatchLines; -}; - -// struct AYSConfig -// { -// std::string datasetPath; -// }; - struct GatewayConfig { // the max allowed message size, default is 100MBytes @@ -98,11 +73,11 @@ struct GatewayConfig constexpr static uint64_t MaxMsgSize = 1024 * 1024 * 1024; constexpr static int MinUnreachableDistance = 2; - bool disableCache; NetworkConfig networkConfig; - ppc::protocol::GrpcServerConfig grpcConfig; - ppc::storage::CacheStorageConfig cacheStorageConfig; + ppc::protocol::GrpcServerConfig grpcServerConfig; + // the file that configure the connected endpoint information std::string nodeFileName; + // the dir that contains the connected endpoint information, e.g.nodes.json std::string nodePath; uint64_t maxAllowedMsgSize = DefaultMaxAllowedMsgSize; int reconnectTime = 10000; @@ -149,13 +124,16 @@ class PPCConfig using ConstPtr = std::shared_ptr; PPCConfig() = default; virtual ~PPCConfig() = default; - void loadConfig(std::string const& _configPath) + // load the nodeConfig + void loadNodeConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + std::string const& _configPath) { - PPCConfig_LOG(INFO) << LOG_DESC("loadConfig") << LOG_KV("path", _configPath); + PPCConfig_LOG(INFO) << LOG_DESC("loadNodeConfig") << LOG_KV("path", _configPath); boost::property_tree::ptree iniConfig; boost::property_tree::read_ini(_configPath, iniConfig); // Note: must load common-config firstly since some ra-configs depends on the common-config - loadCommonConfig(iniConfig); + loadCommonNodeConfig(iniConfig); + loadFrontConfig(frontConfigBuilder, _configPath); loadRA2018Config(iniConfig); loadStorageConfig(iniConfig); loadEcdhPSIConfig(iniConfig); @@ -164,40 +142,42 @@ class PPCConfig loadEcdhConnPSIConfig(iniConfig); } - void loadRpcConfig(const char* _certPath, std::string const& _configPath) + void loadRpcConfig(std::string const& _configPath) { PPCConfig_LOG(INFO) << LOG_DESC("loadRpcConfig") << LOG_KV("path", _configPath); boost::property_tree::ptree iniConfig; boost::property_tree::read_ini(_configPath, iniConfig); - loadRpcConfig(_certPath, iniConfig); + loadRpcConfig(iniConfig); } - void loadGatewayConfig( - ppc::protocol::NodeArch _arch, const char* _certPath, std::string const& _configPath) + void loadGatewayConfig(std::string const& _configPath) { PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("path", _configPath); boost::property_tree::ptree iniConfig; boost::property_tree::read_ini(_configPath, iniConfig); - loadGatewayConfig(_arch, _certPath, iniConfig); + loadGatewayConfig(iniConfig); } - virtual void loadRpcConfig(const char* _certPath, boost::property_tree::ptree const& _pt) + void loadFrontConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + std::string const& _configPath) { - // rpc default disable-ssl - loadNetworkConfig( - m_rpcConfig, _certPath, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true); + PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig") << LOG_KV("path", _configPath); + boost::property_tree::ptree iniConfig; + boost::property_tree::read_ini(_configPath, iniConfig); + loadFrontConfig(frontConfigBuilder, iniConfig); + // load the grpcConfig + m_grpcConfig = loadGrpcConfig("transport", iniConfig); + m_frontConfig->setGrpcConfig(m_grpcConfig); } - virtual void loadGatewayConfig(ppc::protocol::NodeArch _arch, const char* _certPath, - boost::property_tree::ptree const& _pt); - - virtual void loadHDFSConfig(boost::property_tree::ptree const& _pt); - - virtual void loadSQLConfig(boost::property_tree::ptree const& _pt); + virtual void loadRpcConfig(boost::property_tree::ptree const& _pt) + { + // rpc default disable-ssl + loadNetworkConfig(m_rpcConfig, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true); + } - virtual void loadCEMConfig(boost::property_tree::ptree const& _pt); + virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt); - virtual void loadMPCConfig(boost::property_tree::ptree const& _pt); NetworkConfig const& rpcConfig() const { return m_rpcConfig; } // the gateway-config @@ -210,7 +190,6 @@ class PPCConfig CEMConfig const& cemConfig() const { return m_cemConfig; } MPCConfig const& mpcConfig() const { return m_mpcConfig; } std::string const& agencyID() const { return m_agencyID; } - void setAgencyID(std::string const& _agencyID) { m_agencyID = _agencyID; } bool smCrypto() const { return m_smCrypto; } std::string const& endpoint() const { return m_endpoint; } @@ -234,7 +213,7 @@ class PPCConfig OtPIRParam& mutableOtPIRParam() { return m_otPIRConfig; } bcos::bytes const& privateKey() const { return m_privateKey; } - void setPrivateKey(bcos::bytes const& _privateKey) { m_privateKey = _privateKey; } + void setPrivateKey(bcos::bytes const& _privateKey); std::string const& privateKeyPath() const { return m_privateKeyPath; } // for pro-mode @@ -251,18 +230,38 @@ class PPCConfig ppc::protocol::GrpcConfig::Ptr const& grpcConfig() const { return m_grpcConfig; } + // used by mpc initilizer + virtual void loadMPCConfig(boost::property_tree::ptree const& _pt); + // used by cem module + virtual void loadCEMConfig(boost::property_tree::ptree const& _pt); + private: + virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp, + std::string const& sectionName, boost::property_tree::ptree const& pt); + // load the front config + virtual void loadFrontConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + boost::property_tree::ptree const& pt); + // load the grpc config + ppc::protocol::GrpcConfig::Ptr loadGrpcConfig( + std::string const& sectionName, boost::property_tree::ptree const& pt); + + virtual void loadHDFSConfig(boost::property_tree::ptree const& _pt); + + virtual void loadSQLConfig(boost::property_tree::ptree const& _pt); + + virtual void loadRA2018Config(boost::property_tree::ptree const& _pt); virtual void loadEcdhPSIConfig(boost::property_tree::ptree const& _pt); virtual void loadCM2020PSIConfig(boost::property_tree::ptree const& _pt); virtual void loadEcdhMultiPSIConfig(boost::property_tree::ptree const& _pt); virtual void loadEcdhConnPSIConfig(boost::property_tree::ptree const& _pt); - virtual void loadCommonConfig(boost::property_tree::ptree const& _pt); + virtual void loadCommonNodeConfig(boost::property_tree::ptree const& _pt); virtual void loadStorageConfig(boost::property_tree::ptree const& _pt); + // Note: the gateway/rpc can share the loadNetworkConfig - void loadNetworkConfig(NetworkConfig& _config, const char* _certPath, - boost::property_tree::ptree const& _pt, std::string const& _sectionName, - int _defaultListenPort, bool _defaultDisableSSl); + void loadNetworkConfig(NetworkConfig& _config, boost::property_tree::ptree const& _pt, + std::string const& _sectionName, int _defaultListenPort, bool _defaultDisableSSl); + void checkPort(std::string const& _sectionName, int _port); void checkFileExists(std::string const& _filePath, bool _dir); @@ -274,7 +273,7 @@ class PPCConfig } } - void initRedisConfigForGateway( + void loadCachedStorageConfig( ppc::storage::CacheStorageConfig& _redisConfig, const boost::property_tree::ptree& _pt); int64_t getDataBatchSize(std::string const& _section, int64_t _dataBatchSize); diff --git a/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h new file mode 100644 index 00000000..8e3d7033 --- /dev/null +++ b/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file StorageConfig.h + * @author: yujiechen + * @date 2022-11-4 + */ +#pragma once +#include +#include +#include + +namespace ppc::tools +{ +struct StorageConfig +{ + ppc::protocol::SQLConnectionOption::Ptr sqlConnectionOpt; + ppc::protocol::FileStorageConnectionOption::Ptr fileStorageConnectionOpt; + ppc::protocol::RemoteStorageConnectionOption::Ptr remoteConnectionOpt; +}; +} // namespace ppc::tools \ No newline at end of file diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index 0bd52833..ad169481 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -33,10 +33,12 @@ #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h" #endif #include "ppc-front/Front.h" +#include "ppc-front/FrontConfigImpl.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h" #include "ppc-psi/src/ecdh-psi/EcdhPSIFactory.h" #include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h" #include "ppc-psi/src/ra2018-psi/RA2018PSIFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" #include #include @@ -51,9 +53,10 @@ using namespace ppc::sdk; Initializer::Initializer(std::string const& _configPath) : m_configPath(_configPath) { + m_transportBuilder = std::make_shared(); // load the config m_config = std::make_shared(); - m_config->loadConfig(_configPath); + m_config->loadNodeConfig(m_transportBuilder->frontConfigBuilder(), _configPath); } void Initializer::init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Ptr const& gateway) diff --git a/cpp/wedpr-initializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h index 78eba67d..d1ad5d01 100644 --- a/cpp/wedpr-initializer/Initializer.h +++ b/cpp/wedpr-initializer/Initializer.h @@ -24,7 +24,6 @@ #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/rpc/RpcTypeDef.h" #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h" -#include "ppc-tools/src/config/PPCConfig.h" #include "wedpr-transport/sdk/Transport.h" #include #include @@ -44,6 +43,16 @@ namespace ppc::pir { class OtPIRImpl; } // namespace ppc::pir + +namespace ppc::sdk +{ +class TransportBuilder; +} + +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::initializer { class Initializer : public std::enable_shared_from_this @@ -58,11 +67,11 @@ class Initializer : public std::enable_shared_from_this virtual void stop(); virtual void start(); - ppc::tools::PPCConfig::Ptr config() { return m_config; } + std::shared_ptr config() { return m_config; } ppc::sdk::Transport::Ptr const& transport() const { return m_transport; } ppc::front::FrontInterface::Ptr const& ppcFront() const { return m_ppcFront; } - ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } + std::shared_ptr const& config() const { return m_config; } ProtocolInitializer::Ptr const& protocolInitializer() const { return m_protocolInitializer; } ppc::psi::BsEcdhPSIImpl::Ptr const& bsEcdhPsi() const { return m_bsEcdhPSI; } @@ -74,10 +83,12 @@ class Initializer : public std::enable_shared_from_this private: std::string m_configPath; - ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_config; ProtocolInitializer::Ptr m_protocolInitializer; + + std::shared_ptr m_transportBuilder; ppc::sdk::Transport::Ptr m_transport; - // created with transport + // created using transport ppc::front::FrontInterface::Ptr m_ppcFront; // the ra2018-psi diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp index 82ee59ac..a05788e3 100644 --- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp +++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp @@ -61,7 +61,7 @@ void AirNodeInitializer::init(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("init the rpc"); // load the rpc config // not specify the certPath in air-mode - m_nodeInitializer->config()->loadRpcConfig(nullptr, pt); + m_nodeInitializer->config()->loadRpcConfig(pt); // init RpcStatusInterface RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(); @@ -80,7 +80,7 @@ void AirNodeInitializer::initGateway(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath; // not specify the certPath in air-mode auto config = m_nodeInitializer->config(); - config->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, _configPath); + config->loadGatewayConfig(_configPath); auto threadPool = std::make_shared( "gateway", config->gatewayConfig().networkConfig.threadPoolSize); diff --git a/cpp/wedpr-main/cem-node/CEMInitializer.cpp b/cpp/wedpr-main/cem-node/CEMInitializer.cpp index 7344386c..69c2efbe 100644 --- a/cpp/wedpr-main/cem-node/CEMInitializer.cpp +++ b/cpp/wedpr-main/cem-node/CEMInitializer.cpp @@ -40,7 +40,7 @@ void CEMInitializer::init(std::string const& _configPath) // load the rpc config auto ppcConfig = std::make_shared(); // not specify the certPath in air-mode - ppcConfig->loadRpcConfig(nullptr, pt); + ppcConfig->loadRpcConfig(pt); ppcConfig->loadCEMConfig(pt); // bool useMysql = pt.get("cem.use_mysql", false); auto storageConfig = ppcConfig->storageConfig(); diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.cpp b/cpp/wedpr-main/gateway/GatewayInitializer.cpp index ca1c25c7..b2657958 100644 --- a/cpp/wedpr-main/gateway/GatewayInitializer.cpp +++ b/cpp/wedpr-main/gateway/GatewayInitializer.cpp @@ -45,14 +45,14 @@ void GatewayInitializer::init(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath; auto config = std::make_shared(); - config->loadGatewayConfig(ppc::protocol::NodeArch::PRO, nullptr, _configPath); + config->loadGatewayConfig(_configPath); auto threadPool = std::make_shared( "gateway", config->gatewayConfig().networkConfig.threadPoolSize); GatewayFactory gatewayFactory(config); m_gateway = gatewayFactory.build(std::make_shared(config->grpcConfig())); - m_server = std::make_shared(config->gatewayConfig().grpcConfig); + m_server = std::make_shared(config->gatewayConfig().grpcServerConfig); // register the gateway service auto gatewayService = std::make_shared(m_gateway, std::make_shared(), std::make_shared()); diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp index 350e24fb..de8b8f1f 100644 --- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp +++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp @@ -19,6 +19,7 @@ */ #include "MPCInitializer.h" #include "ppc-mpc/src/MPCService.h" +#include "ppc-tools/src/config/PPCConfig.h" using namespace ppc::rpc; using namespace bcos; @@ -40,7 +41,7 @@ void MPCInitializer::init(std::string const& _configPath) // load the rpc config auto ppcConfig = std::make_shared(); // not specify the certPath in air-mode - ppcConfig->loadRpcConfig(nullptr, pt); + ppcConfig->loadRpcConfig(pt); ppcConfig->loadMPCConfig(pt); // bool useMysql = pt.get("mpc.use_mysql", false); auto storageConfig = ppcConfig->storageConfig(); diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp index ab2a51be..3e31a4d0 100644 --- a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp +++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp @@ -52,7 +52,7 @@ void ProNodeInitializer::init(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("init the rpc"); // load the rpc config // not specify the certPath in air-mode - m_nodeInitializer->config()->loadRpcConfig(nullptr, pt); + m_nodeInitializer->config()->loadRpcConfig(pt); // init RpcStatusInterface RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared(); diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h index 2996b13a..0e294403 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontConfigImpl.h @@ -29,6 +29,9 @@ class FrontConfigImpl : public FrontConfig { public: using Ptr = std::shared_ptr; + FrontConfigImpl(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory) + : FrontConfig(), m_nodeInfoFactory(std::move(nodeInfoFactory)) + {} FrontConfigImpl(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory, int threadPoolSize, std::string nodeID) : FrontConfig(threadPoolSize, nodeID), m_nodeInfoFactory(std::move(nodeInfoFactory)) @@ -45,6 +48,29 @@ class FrontConfigImpl : public FrontConfig return nodeInfo; } +private: + ppc::protocol::INodeInfoFactory::Ptr m_nodeInfoFactory; +}; + +class FrontConfigBuilderImpl : public FrontConfigBuilder +{ +public: + using Ptr = std::shared_ptr; + FrontConfigBuilderImpl(ppc::protocol::INodeInfoFactory::Ptr nodeInfoFactory) + : m_nodeInfoFactory(nodeInfoFactory) + {} + ~FrontConfigBuilderImpl() override = default; + + FrontConfig::Ptr build() const override + { + return std::make_shared(m_nodeInfoFactory); + } + FrontConfig::Ptr build(int threadPoolSize, std::string nodeID) const override + { + return std::make_shared( + m_nodeInfoFactory, threadPoolSize, std::move(nodeID)); + } + private: ppc::protocol::INodeInfoFactory::Ptr m_nodeInfoFactory; }; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp index 8243d133..374bf909 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.cpp @@ -19,11 +19,22 @@ */ #include "GatewayConfigContext.h" +#include using namespace bcos; using namespace ppc::gateway; using namespace bcos::boostssl::context; +GatewayConfigContext::GatewayConfigContext(std::shared_ptr _config) + : m_config(_config) +{ + if (!m_config->gatewayConfig().networkConfig.disableSsl) + { + GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig"); + initContextConfig(); + GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig success"); + } +} void GatewayConfigContext::initContextConfig() { m_contextConfig = std::make_shared(); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h index 259ceb71..33d936e8 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigContext.h @@ -23,32 +23,27 @@ #include "Common.h" #include "bcos-boostssl/context/NodeInfoTools.h" #include -#include #include #include #include +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::gateway { class GatewayConfigContext { public: using Ptr = std::shared_ptr; - GatewayConfigContext(ppc::tools::PPCConfig::Ptr _config) : m_config(_config) - { - if (!m_config->gatewayConfig().networkConfig.disableSsl) - { - GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig"); - initContextConfig(); - GATEWAY_LOG(INFO) << LOG_DESC("GatewayConfigContext: initContextConfig success"); - } - } + GatewayConfigContext(std::shared_ptr _config); virtual ~GatewayConfigContext() = default; [[nodiscard]] std::shared_ptr contextConfig() const { return m_contextConfig; } - ppc::tools::PPCConfig::Ptr const& config() const { return m_config; } + std::shared_ptr const& config() const { return m_config; } std::string const& nodeID() const { return m_nodeID; } @@ -56,7 +51,7 @@ class GatewayConfigContext void initContextConfig(); private: - ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_config; std::shared_ptr m_contextConfig; std::string m_nodeID; }; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp index 8d87f282..7f7d3bd3 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.cpp @@ -20,6 +20,7 @@ #include "GatewayConfigLoader.h" #include "Common.h" #include "bcos-utilities/FileUtility.h" +#include "ppc-tools/src/config/PPCConfig.h" #include using namespace ppc; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h index 1c6b418e..4d523797 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayConfigLoader.h @@ -19,8 +19,11 @@ */ #pragma once #include "bcos-boostssl/interfaces/NodeInfoDef.h" -#include "ppc-tools/src/config/PPCConfig.h" +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::gateway { class GatewayConfigLoader @@ -28,7 +31,7 @@ class GatewayConfigLoader public: using EndPointSet = std::set; using Ptr = std::shared_ptr; - GatewayConfigLoader(ppc::tools::PPCConfig::Ptr config) + GatewayConfigLoader(std::shared_ptr config) : m_config(std::move(config)), m_nodeIPEndpointSet(std::make_shared()) { loadP2pConnectedNodes(); @@ -46,7 +49,7 @@ class GatewayConfigLoader bool isValidPort(int port); private: - ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_config; std::shared_ptr m_nodeIPEndpointSet; }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp index 51f047f3..1349593c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp @@ -22,6 +22,7 @@ #include "bcos-boostssl/websocket/WsInitializer.h" #include "ppc-gateway/p2p/Service.h" #include "ppc-gateway/p2p/router/RouterTableImpl.h" +#include "ppc-tools/src/config/PPCConfig.h" #include "protocol/src/v1/MessageHeaderImpl.h" #include "protocol/src/v1/MessageImpl.h" @@ -33,27 +34,23 @@ using namespace ppc::gateway; using namespace bcos::boostssl::ws; using namespace bcos::boostssl; -WsConfig::Ptr GatewayFactory::createServiceConfig(GatewayConfig const& config) const +Service::Ptr GatewayFactory::buildService() const { + auto gwConfig = m_config->gatewayConfig(); auto wsConfig = std::make_shared(); wsConfig->setModel(WsModel::Mixed); - wsConfig->setListenIP(config.networkConfig.listenIp); - wsConfig->setListenPort(config.networkConfig.listenPort); - wsConfig->setSmSSL(config.networkConfig.enableSM); - wsConfig->setMaxMsgSize(config.maxAllowedMsgSize); - wsConfig->setReconnectPeriod(config.reconnectTime); + wsConfig->setListenIP(gwConfig.networkConfig.listenIp); + wsConfig->setListenPort(gwConfig.networkConfig.listenPort); + wsConfig->setSmSSL(gwConfig.networkConfig.enableSM); + wsConfig->setMaxMsgSize(gwConfig.maxAllowedMsgSize); + wsConfig->setReconnectPeriod(gwConfig.reconnectTime); // TODO: setHeartbeatPeriod, setSendMsgTimeout - wsConfig->setThreadPoolSize(config.networkConfig.threadPoolSize); + wsConfig->setThreadPoolSize(gwConfig.networkConfig.threadPoolSize); // connected peers wsConfig->setConnectPeers(m_gatewayConfig->nodeIPEndpointSetPtr()); - wsConfig->setDisableSsl(config.networkConfig.disableSsl); + wsConfig->setDisableSsl(gwConfig.networkConfig.disableSsl); wsConfig->setContextConfig(m_contextConfig->contextConfig()); - return wsConfig; -} -Service::Ptr GatewayFactory::buildService() const -{ - auto wsConfig = createServiceConfig(m_config->gatewayConfig()); auto wsInitializer = std::make_shared(); // set the messageFactory wsInitializer->setMessageFactory( diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h index 2393f240..bef50884 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.h @@ -23,15 +23,18 @@ #include "bcos-boostssl/websocket/WsConfig.h" #include "gateway/GatewayImpl.h" #include "ppc-gateway/p2p/Service.h" -#include "ppc-tools/src/config/PPCConfig.h" +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::gateway { class GatewayFactory { public: using Ptr = std::shared_ptr; - GatewayFactory(ppc::tools::PPCConfig::Ptr config) : m_config(std::move(config)) + GatewayFactory(std::shared_ptr config) : m_config(std::move(config)) { m_contextConfig = std::make_shared(m_config); m_gatewayConfig = std::make_shared(m_config); @@ -43,11 +46,8 @@ class GatewayFactory protected: Service::Ptr buildService() const; - bcos::boostssl::ws::WsConfig::Ptr createServiceConfig( - ppc::tools::GatewayConfig const& config) const; - private: - ppc::tools::PPCConfig::Ptr m_config; + std::shared_ptr m_config; GatewayConfigContext::Ptr m_contextConfig; GatewayConfigLoader::Ptr m_gatewayConfig; }; diff --git a/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp b/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp index aa28ad9b..bb8aca0e 100644 --- a/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp +++ b/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp @@ -56,8 +56,7 @@ int main(int argc, char* argv[]) // config auto config = std::make_shared(); // not specify the certPath in air-mode - config->loadGatewayConfig( - ppc::protocol::NodeArch::AIR, nullptr, "../../../../ppc-gateway/test/data/config0.ini"); + config->(nullptr, "../../../../ppc-gateway/test/data/config0.ini"); auto gatewayConfig = config->gatewayConfig(); // global thread pool diff --git a/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp b/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp index 49d9da28..a1ae49b3 100644 --- a/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp +++ b/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayTest.cpp @@ -21,8 +21,8 @@ #if 0 #include "ppc-gateway/ppc-gateway/Gateway.h" #include "MockCache.h" -#include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" #include "ppc-gateway/ppc-gateway/" +#include "ppc-gateway/ppc-gateway/GatewayConfigContext.h" #include "ppc-tools/src/config/PPCConfig.h" #include #include @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(test_readConfig) { auto ppcConfig = std::make_shared(); std::string configPath = "../../../ppc-gateway/test/data/config0.ini"; - ppcConfig->loadGatewayConfig(ppc::protocol::NodeArch::PRO, nullptr, configPath); + ppcConfig->loadGatewayConfig(configPath); auto config = std::make_shared(ppcConfig); auto const& gatewayConfig = ppcConfig->gatewayConfig(); BOOST_CHECK(gatewayConfig.networkConfig.disableSsl == true); diff --git a/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp b/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp index 463f078c..08ad495d 100644 --- a/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp +++ b/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp @@ -51,7 +51,7 @@ int main(int argc, char const* argv[]) auto param = initCommandLine(argc, argv); auto ppcConfig = std::make_shared(); // not specify the certPath in air-mode - ppcConfig->loadGatewayConfig(ppc::protocol::NodeArch::AIR, nullptr, param.configFilePath); + ppcConfig->loadGatewayConfig(param.configFilePath); auto httpFactory = std::make_shared("selfParty"); auto url = ppcConfig->gatewayConfig().networkConfig.url; auto http = httpFactory->buildHttp(ppcConfig); diff --git a/cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp b/cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp index cca22f21..0826944d 100644 --- a/cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp +++ b/cpp/wedpr-transport/ppc-http/src/HttpFactory.cpp @@ -19,6 +19,7 @@ */ #include "HttpFactory.h" +#include "ppc-tools/src/config/PPCConfig.h" #include #include @@ -37,7 +38,8 @@ Http::Ptr HttpFactory::buildHttp(ppc::tools::PPCConfig::ConstPtr _config) initializer->setConfig(wsConfig); initializer->initWsService(wsService); - return std::make_shared(wsService, m_selfPartyID, _config->gatewayConfig().networkConfig.token); + return std::make_shared( + wsService, m_selfPartyID, _config->gatewayConfig().networkConfig.token); } @@ -67,9 +69,9 @@ std::shared_ptr HttpFactory::initConfig( contextConfig->setCertConfig(certConfig); contextConfig->setSslType("ssl"); HTTP_LOG(INFO) << LOG_DESC("initConfig: http work in non-sm-ssl model") - << LOG_KV("caCert", certConfig.caCert) - << LOG_KV("nodeCert", certConfig.nodeCert) - << LOG_KV("nodeKey", certConfig.nodeKey); + << LOG_KV("caCert", certConfig.caCert) + << LOG_KV("nodeCert", certConfig.nodeCert) + << LOG_KV("nodeKey", certConfig.nodeKey); wsConfig->setContextConfig(contextConfig); return wsConfig; } diff --git a/cpp/wedpr-transport/ppc-http/src/HttpFactory.h b/cpp/wedpr-transport/ppc-http/src/HttpFactory.h index d3d063f5..f7aa0856 100644 --- a/cpp/wedpr-transport/ppc-http/src/HttpFactory.h +++ b/cpp/wedpr-transport/ppc-http/src/HttpFactory.h @@ -19,14 +19,16 @@ */ #pragma once #include "Http.h" -#include "ppc-tools/src/config/PPCConfig.h" #include namespace bcos::boostssl::ws { class WsConfig; } - +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::http { class HttpFactory @@ -36,11 +38,11 @@ class HttpFactory HttpFactory(std::string const& _selfPartyID) : m_selfPartyID(_selfPartyID) {} virtual ~HttpFactory() = default; - Http::Ptr buildHttp(ppc::tools::PPCConfig::ConstPtr _config); + Http::Ptr buildHttp(std::shared_ptr _config); private: std::shared_ptr initConfig( - ppc::tools::PPCConfig::ConstPtr _config); + std::shared_ptr _config); private: std::string m_selfPartyID; diff --git a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp index f51ecc3c..849502c2 100644 --- a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp +++ b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp @@ -48,7 +48,7 @@ int main(int argc, const char* argv[]) auto param = initCommandLine(argc, argv); auto ppcConfig = std::make_shared(); // not specify the certPath in air-mode - ppcConfig->loadRpcConfig(nullptr, param.configFilePath); + ppcConfig->loadRpcConfig(param.configFilePath); auto rpcFactory = std::make_shared("selfParty"); auto rpc = rpcFactory->buildRpc(ppcConfig); registerEchoHandler(rpc); diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp index a7e880ef..d665afe1 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp @@ -19,6 +19,7 @@ */ #include "RpcFactory.h" #include "ppc-storage/src/mysql/MySQLStorage.h" +#include "ppc-tools/src/config/PPCConfig.h" #include #include diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h index bfa04d9d..3e6b512e 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h @@ -19,12 +19,15 @@ */ #pragma once #include "Rpc.h" -#include "ppc-tools/src/config/PPCConfig.h" #include namespace bcos::boostssl::ws { class WsConfig; } +namespace ppc::tools +{ +class PPCConfig; +} namespace ppc::rpc { class RpcFactory @@ -34,12 +37,12 @@ class RpcFactory RpcFactory(std::string const& _selfPartyID) : m_selfPartyID(_selfPartyID) {} virtual ~RpcFactory() = default; - Rpc::Ptr buildRpc(ppc::tools::PPCConfig::ConstPtr _config); + Rpc::Ptr buildRpc(std::shared_ptr _config); private: std::shared_ptr initConfig( - ppc::tools::PPCConfig::ConstPtr _config); + std::shared_ptr _config); private: std::string m_selfPartyID; diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/TransportBuilder.cpp index 8042b844..6550efb6 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.cpp +++ b/cpp/wedpr-transport/sdk/TransportBuilder.cpp @@ -27,6 +27,13 @@ #include using namespace ppc::sdk; +using namespace ppc::front; + +TransportBuilder::TransportBuilder() +{ + m_frontConfigBuilder = std::make_shared( + std::make_shared()); +} Transport::Ptr TransportBuilder::build( SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) @@ -48,6 +55,5 @@ Transport::Ptr TransportBuilder::build( ppc::front::FrontConfig::Ptr TransportBuilder::buildConfig(int threadPoolSize, std::string nodeID) { - return std::make_shared( - std::make_shared(), threadPoolSize, nodeID); + return m_frontConfigBuilder->build(threadPoolSize, nodeID); } diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/TransportBuilder.h index b2b57931..cd6c3c56 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.h +++ b/cpp/wedpr-transport/sdk/TransportBuilder.h @@ -33,12 +33,17 @@ class TransportBuilder { public: using Ptr = std::shared_ptr; - TransportBuilder() = default; + TransportBuilder(); virtual ~TransportBuilder() = default; Transport::Ptr build(SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway); ppc::front::FrontConfig::Ptr buildConfig(int threadPoolSize, std::string nodeID); + + ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder() { return m_frontConfigBuilder; } + +private: + ppc::front::FrontConfigBuilder::Ptr m_frontConfigBuilder; }; } // namespace ppc::sdk \ No newline at end of file From 7b77c7bdd412051acfa733fb6076e2d8c606a2c6 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 5 Sep 2024 19:31:25 +0800 Subject: [PATCH 026/120] update build_ppc and ppc_builder (#22) --- cpp/tools/build_ppc.sh | 55 +++++- cpp/tools/ppc-builder/build_ppc.py | 7 +- .../ppc-builder/conf/config-example.toml | 78 +++----- cpp/tools/ppc-builder/src/common/utilities.py | 53 ++---- .../ppc-builder/src/config/cert_generator.py | 2 +- .../src/config/ppc_deploy_config.py | 119 ++++-------- .../config/ppc_gateway_config_generator.py | 172 +++++++----------- .../src/config/ppc_node_config_generator.py | 160 ++++++++-------- .../src/config/shell_script_generator.py | 75 ++++++++ .../src/config/tars_config_generator.py | 123 ------------- cpp/tools/ppc-builder/src/scripts/gen_cert.sh | 5 +- .../ppc-builder/src/tpl/config.ini.gateway | 38 ++-- cpp/tools/ppc-builder/src/tpl/config.ini.node | 22 ++- cpp/tools/ppc-builder/src/tpl/start.sh | 39 ++++ .../tpl/{tars_start_all.sh => start_all.sh} | 5 +- cpp/tools/ppc-builder/src/tpl/stop.sh | 35 ++++ .../src/tpl/{tars_stop_all.sh => stop_all.sh} | 5 +- cpp/tools/ppc-builder/src/tpl/tars.conf | 31 ---- cpp/tools/ppc-builder/src/tpl/tars_start.sh | 33 ---- cpp/tools/ppc-builder/src/tpl/tars_stop.sh | 31 ---- 20 files changed, 458 insertions(+), 630 deletions(-) create mode 100644 cpp/tools/ppc-builder/src/config/shell_script_generator.py delete mode 100644 cpp/tools/ppc-builder/src/config/tars_config_generator.py create mode 100755 cpp/tools/ppc-builder/src/tpl/start.sh rename cpp/tools/ppc-builder/src/tpl/{tars_start_all.sh => start_all.sh} (62%) create mode 100755 cpp/tools/ppc-builder/src/tpl/stop.sh rename cpp/tools/ppc-builder/src/tpl/{tars_stop_all.sh => stop_all.sh} (62%) delete mode 100644 cpp/tools/ppc-builder/src/tpl/tars.conf delete mode 100644 cpp/tools/ppc-builder/src/tpl/tars_start.sh delete mode 100644 cpp/tools/ppc-builder/src/tpl/tars_stop.sh diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh index 730b518c..509a0bd0 100644 --- a/cpp/tools/build_ppc.sh +++ b/cpp/tools/build_ppc.sh @@ -28,6 +28,8 @@ sm2_params="sm_sm2.param" sm_mode="false" days=36500 rsa_key_length=2048 +p2p_connected_conf_name="nodes.json" +file_dir="./" default_version="v1.1.0" compatibility_version=${default_version} @@ -338,7 +340,7 @@ Usage: -l [Required] "ip1:nodeNum1,ip2:nodeNum2" e.g:"192.168.0.1:2,192.168.0.2:3" -e [Optional] ppc-air-node binary exec -o [Optional] output directory, default ./nodes - -p [Optional] Default 40300,10200 means p2p_port start from 40300, rpc_port from 10200 + -p [Optional] Default 40300,10200,18000 means p2p_port start from 40300, rpc_port from 10200, grpc_port from 18000 -s [Optional] SM SSL connection or not, default is false -d [Optional] Disable ra2018 psi or not, default is false -h Help @@ -350,8 +352,39 @@ EOF exit 0 } + +generate_p2p_connected_conf() { + local output="${1}" + local ip_params="${2}" + local template="${3}" + + local p2p_host_list="" + if [[ "${template}" == "true" ]]; then + p2p_host_list="${ip_params}" + else + local ip_array=(${ip_params//,/ }) + local ip_length=${#ip_array[@]} + + local i=0 + for (( ; i < ip_length; i++)); do + local ip=${ip_array[i]} + local delim="" + if [[ $i == $((ip_length - 1)) ]]; then + delim="" + else + delim="," + fi + p2p_host_list="${p2p_host_list}\"${ip}\"${delim}" + done + fi + + cat <"${output}" +{"nodes":[${p2p_host_list}]} +EOF +} + # generate the config.ini -generate_config_ini() { +generate_node_config_ini() { local output="${1}" local gateway_listen_ip="${2}" local gateway_listen_port="${3}" @@ -364,6 +397,7 @@ generate_config_ini() { local agency_id="${8}" local index="${9}" + local nodeid="${10}" cat <"${output}" [agency] @@ -389,6 +423,7 @@ generate_config_ini() { service.gateway_target = ; the components service.components = + nodeid=${nodeid} [crypto] sm_crypto = ${sm_mode} @@ -396,6 +431,10 @@ generate_config_ini() { [gateway] listen_ip=${gateway_listen_ip} listen_port=${gateway_listen_port} + ;the dir that contains the connected endpoint information, e.g.nodes.json + ;nodes_path=${file_dir} + ; the file that configure the connected endpoint information + ; nodes_file=${p2p_connected_conf_name} ; thread_count = 4 ; ssl or sm ssl sm_ssl=${sm_mode} @@ -408,10 +447,6 @@ generate_config_ini() { ;reconnect_time = 10000 ; the unreachable distance ;unreachable_distance=10 - ;the dir that contains the connected endpoint information, e.g.nodes.json - ;nodes_path=./ - ; the file that configure the connected endpoint information - ; nodes_path=nodes.json [rpc] listen_ip=${rpc_listen_ip} @@ -786,6 +821,7 @@ generate_private_key() { fi ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" + ${OPENSSL_CMD} ec -text -noout -in "${output_path}/node.pem" 2>/dev/null | sed -n '7,11p' | tr -d ": \n" | awk '{print substr($0,3);}' | cat >"$output_path"/node.nodeid private_key=$(cat $output_path/node.privateKey) echo ${private_key} } @@ -819,6 +855,7 @@ deploy_nodes() # generate the ca-cert ca_dir="${output_dir}"/ca generate_ca_cert "${sm_mode}" "${ca_dir}" + connected_nodes="" for line in ${ip_array[*]}; do ip=${line%:*} num=${line#*:} @@ -849,7 +886,7 @@ deploy_nodes() # generate the node-script generate_node_scripts "${node_dir}" local port=$((gateway_listen_port + node_count)) - " + connected_nodes=${connected_nodes}"${ip}:${port}, " ((agency_index += 1)) set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) @@ -875,7 +912,9 @@ deploy_nodes() local grpc_port=$((grpc_listen_port + node_count)) local agency_id="agency${count}" private_key=$(generate_private_key "${node_dir}/conf") - generate_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" + node_id=$(cat "${node_dir}/conf/node.nodeid") + generate_node_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" "${node_id}" + generate_p2p_connected_conf "${node_dir}/${p2p_connected_conf_name}" "${connected_nodes}" "false" set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1)) ((++count)) done diff --git a/cpp/tools/ppc-builder/build_ppc.py b/cpp/tools/ppc-builder/build_ppc.py index e609a1c5..713d1801 100644 --- a/cpp/tools/ppc-builder/build_ppc.py +++ b/cpp/tools/ppc-builder/build_ppc.py @@ -1,11 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- # Note: here can't be refactored by autopep -import sys -sys.path.append("src/") - from controller import commandline_helper from common import utilities +import traceback +import sys +sys.path.append("src/") def main(): @@ -14,6 +14,7 @@ def main(): commandline_helper.execute_command(args) except Exception as error: utilities.log_error("%s" % error) + traceback.print_exc() if __name__ == "__main__": diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml index 167afe85..7225f7e7 100644 --- a/cpp/tools/ppc-builder/conf/config-example.toml +++ b/cpp/tools/ppc-builder/conf/config-example.toml @@ -1,6 +1,6 @@ -[tars] -tars_pkg_dir = "bin/" -app_name = "ppc1" +[env] +binary_path = "bin/" +deploy_dir = "ppc1" [crypto] # the gateway-service enable ssl or not, default enable ssl @@ -27,25 +27,11 @@ holding_msg_minutes = 30 listen_port=40300 # the thread count thread_count = 4 - # the cache config for the gateway - cache_type = 0 - cache_proxy = "" - cache_obServer = "" - cache_cluster = "" - cache_host = "127.0.0.1" - cache_port = 6379 - cache_password = "" - cache_database = 1 - cache_pool_size = 16 - # the redis connection timeout, in ms, default is 500ms - cache_connection_timeout = 500 - # the redis socket timeout, in ms, default is 500ms - cache_socket_timeout = 500 - # the tars config - # gateway tars server listen ip - tars_listen_ip="0.0.0.0" - # gateway tars server listen start port - tars_listen_port=40600 + # the grpc config + # gateway grpc server listen ip + grpc_listen_ip="0.0.0.0" + # gateway grpc server listen start port + grpc_listen_port=40600 # gateway connected peers, should be all of the gateway peers info [[agency.gateway.peers]] agency = "agency0" @@ -61,10 +47,12 @@ holding_msg_minutes = 30 deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" - # node tars server listen ip - tars_listen_ip="0.0.0.0" - # node tars server listen port - tars_listen_port=40402 + # node grpc server listen ip + grpc_listen_ip="0.0.0.0" + # node grpc server listen port + grpc_listen_port=40402 + # the components + components = "node" # the rpc config for the node [agency.node.rpc] listen_ip = "0.0.0.0" @@ -108,7 +96,7 @@ holding_msg_minutes = 30 token = "" # the gateway config [agency.node.gateway] - tars_endpoints = ["127.0.0.1:40600", "127.0.0.1:40601"] + gateway_grpc_target = ["127.0.0.1:40600", "127.0.0.1:40601"] [[agency]] @@ -124,25 +112,11 @@ holding_msg_minutes = 30 listen_port=40320 # the thread count thread_count = 4 - # the cache config for the gateway - cache_type = 0 - cache_proxy = "" - cache_obServer = "" - cache_cluster = "" - cache_host = "127.0.0.1" - cache_port = 6379 - cache_password = "" - cache_database = 2 - cache_pool_size = 16 - # the redis connection timeout, in ms, default is 500ms - cache_connection_timeout = 500 - # the redis socket timeout, in ms, default is 500ms - cache_socket_timeout = 500 - # the tars config - # gateway tars server listen ip - tars_listen_ip="0.0.0.0" - # gateway tars server listen start port - tars_listen_port=40620 + # the grpc config + # gateway grpc server listen ip + grpc_listen_ip="0.0.0.0" + # gateway grpc server listen start port + grpc_listen_port=40620 # gateway connected peers, should be all of the gateway peers info [[agency.gateway.peers]] agency = "agency0" @@ -158,10 +132,12 @@ holding_msg_minutes = 30 deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" - # node tars server listen ip - tars_listen_ip="0.0.0.0" - # node tars server listen port - tars_listen_port=40422 + # node grpc server listen ip + grpc_listen_ip="0.0.0.0" + # node grpc server listen port + grpc_listen_port=40422 + # the componet + components = "node" # the rpc config for the node [agency.node.rpc] listen_ip = "0.0.0.0" @@ -205,4 +181,4 @@ holding_msg_minutes = 30 token = "" # the gateway config [agency.node.gateway] - tars_endpoints = ["127.0.0.1:40620", "127.0.0.1:40621"] + gateway_grpc_target = ["127.0.0.1:40620", "127.0.0.1:40621"] diff --git a/cpp/tools/ppc-builder/src/common/utilities.py b/cpp/tools/ppc-builder/src/common/utilities.py index 34679544..5e4b3c32 100644 --- a/cpp/tools/ppc-builder/src/common/utilities.py +++ b/cpp/tools/ppc-builder/src/common/utilities.py @@ -15,14 +15,6 @@ class ServiceInfo: ssl_file_list = ["ca.crt", "ssl.key", "ssl.crt"] sm_ssl_file_list = ["sm_ca.crt", "sm_ssl.key", "sm_ssl.crt", "sm_enssl.key", "sm_enssl.crt"] - - node_service_postfix = "NodeService" - gateway_servant = "GatewayService" - gateway_servant_obj = ["GatewayServiceObj"] - - node_servant = ["FrontService"] - node_servant_object = ["FrontServiceObj"] - cert_generation_script_path = "src/scripts/gen_cert.sh" node_service_type = "node" gateway_service_type = "gateway" @@ -31,29 +23,26 @@ class ServiceInfo: class ConfigInfo: config_ini_file = "config.ini" - tars_config_file = "tars.conf" - tpl_abs_path = "src/tpl/" pwd_path = os.getcwd() node_config_tpl_path = os.path.join( pwd_path, tpl_abs_path, "config.ini.node") gateway_config_tpl_path = os.path.join( pwd_path, tpl_abs_path, "config.ini.gateway") - tars_config_tpl_path = os.path.join(pwd_path, tpl_abs_path, "tars.conf") - - tars_start_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_start.sh") - tars_stop_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_stop.sh") - - tars_start_all_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_start_all.sh") - tars_stop_all_tpl_path = os.path.join( - pwd_path, tpl_abs_path, "tars_stop_all.sh") ppc_gateway_binary_name = "ppc-gateway-service" ppc_node_binary_name = "ppc-pro-node" + start_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "start.sh") + stop_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "stop.sh") + + start_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "start_all.sh") + stop_all_tpl_path = os.path.join( + pwd_path, tpl_abs_path, "stop_all.sh") + class CommandInfo: generate_config = "genconfig" @@ -123,10 +112,6 @@ def mkfiledir(filepath): mkdir(parent_dir) -def generate_service_name(prefix, service_name): - return prefix + service_name - - def convert_bool_to_str(value): if value is True: return "true" @@ -162,7 +147,12 @@ def store_config(config_content, config_type, config_path, desc): mkdir(os.path.dirname(config_path)) with open(config_path, 'w') as configFile: - config_content.write(configFile) + if isinstance(config_content, str): + configFile.write(config_content) + elif isinstance(config_content, bytes): + configFile.write(config_content) + else: + config_content.write(configFile) log_info("* store %s config for %s success" % (config_type, desc)) return True @@ -176,14 +166,3 @@ def load_config(file_path): config_content.optionxform = str config_content.read(file_path) return config_content - - -def check_service_name(tag, service_name): - """ - Note: tars service name can only contain letters and numbers - """ - service_name_len = len(service_name) - ret = re.search(r'^[A-Za-z0-9]+', service_name).span() - if ret is None or (ret[0] != 0 or ret[1] != service_name_len): - raise Exception( - "the %s must be letters|numbers, invalid value: %s" % (tag, service_name)) diff --git a/cpp/tools/ppc-builder/src/config/cert_generator.py b/cpp/tools/ppc-builder/src/config/cert_generator.py index 77ae0fb9..8521ee8b 100644 --- a/cpp/tools/ppc-builder/src/config/cert_generator.py +++ b/cpp/tools/ppc-builder/src/config/cert_generator.py @@ -53,5 +53,5 @@ def generate_private_key(output_path): "* generate private key error, output_path: %s" % output_path) return (False, output) utilities.log_info( - "* generate private_key success, path: %s, private_key: %s" % (output_path, output)) + "* generate private_key success, path: %s, public_key: %s" % (output_path, output)) return (True, output) diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py index 285baf34..bdaffb85 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py +++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py @@ -4,40 +4,32 @@ from common import utilities -class TarsConfig: - """ - the tars config - """ - - def __init__(self, config): - self.config = config - self.app_name = utilities.get_value( - self.config, "tars", "app_name", None, True) - self.binary_path = utilities.get_value( - self.config, "tars", "tars_pkg_dir", None, True) - - class PeerInfo: def __init__(self, agency, endpoints): self.agency = agency self.endpoints = endpoints +class EnvConfig: + def __init__(self, config, section_name): + self.config = config + self.section_name = section_name + self.binary_path = utilities.get_value( + self.config, self.section_name, "binary_path", None, True) + self.deploy_dir = utilities.get_value( + self.config, self.section_name, "deploy_dir", None, True) + + class GatewayConfig: """ the gateway config """ - def __init__(self, app_name, agency_name, holding_msg_minutes, config, config_section, must_exist): + def __init__(self, agency_name, holding_msg_minutes, config, config_section, must_exist): self.config = config self.config_section = config_section - self.app_name = app_name self.holding_msg_minutes = holding_msg_minutes self.agency_name = agency_name - self.service_name = "%s%s" % ( - self.agency_name, utilities.ServiceInfo.gateway_servant) - self.servant_list = [utilities.ServiceInfo.gateway_servant] - self.servant_object_list = utilities.ServiceInfo.gateway_servant_obj # the deploy_ip self.deploy_ip = utilities.get_item_value( @@ -62,34 +54,12 @@ def __init__(self, app_name, agency_name, holding_msg_minutes, config, config_se peer, "endpoints", None, must_exist, "[[peers]]") self.peers.append(PeerInfo(agency, endpoints)) - self.cache_type = utilities.get_item_value( - self.config, "cache_type", 0, must_exist, config_section) - self.cache_proxy = utilities.get_item_value( - self.config, "cache_proxy", None, must_exist, config_section) - self.cache_obServer = utilities.get_item_value( - self.config, "cache_obServer", None, must_exist, config_section) - self.cache_cluster = utilities.get_item_value( - self.config, "cache_cluster", None, must_exist, config_section) - self.cache_host = utilities.get_item_value( - self.config, "cache_host", None, must_exist, config_section) - self.cache_port = utilities.get_item_value( - self.config, "cache_port", None, must_exist, config_section) - self.cache_password = utilities.get_item_value( - self.config, "cache_password", "", must_exist, config_section) - self.cache_database = utilities.get_item_value( - self.config, "cache_database", None, must_exist, config_section) - self.cache_pool_size = utilities.get_item_value( - self.config, "cache_pool_size", 16, False, config_section) - self.cache_connection_timeout = utilities.get_item_value( - self.config, "cache_connection_timeout", 500, False, config_section) - self.cache_socket_timeout = utilities.get_item_value( - self.config, "cache_socket_timeout", 500, False, config_section) - # the tars_listen_ip - self.tars_listen_ip = utilities.get_item_value( - self.config, "tars_listen_ip", "0.0.0.0", False, config_section) - # the tars_listen_port - self.tars_listen_port = utilities.get_item_value( - self.config, "tars_listen_port", None, must_exist, config_section) + # the grpc_listen_ip + self.grpc_listen_ip = utilities.get_item_value( + self.config, "grpc_listen_ip", "0.0.0.0", False, config_section) + # the grpc_listen_port + self.grpc_listen_port = utilities.get_item_value( + self.config, "grpc_listen_port", None, must_exist, config_section) class RpcConfig: @@ -183,16 +153,15 @@ class NodeGatewayConfig: the gateway config for the node """ - def __init__(self, app_name, agency_name, config, node_must_exists): + def __init__(self, agency_name, config, node_must_exists): self.config = config - self.app_name = app_name self.agency_name = agency_name self.desc = "[agency.node]" - self.endpoints = utilities.get_item_value( - self.config, "tars_endpoints", None, node_must_exists, self.desc) - # obtain the gateway name - self.service_name = "%s.%s%s" % ( - self.app_name, self.agency_name, utilities.ServiceInfo.gateway_servant) + self.gateway_grpc_target_array = utilities.get_item_value( + self.config, "gateway_grpc_target", None, node_must_exists, self.desc) + self.gateway_grpc_target = "ipv4:" + self.gateway_grpc_target += ','.join( + map(str, self.gateway_grpc_target_array)) class NodeConfig: @@ -200,28 +169,30 @@ class NodeConfig: the ppc-node config """ - def __init__(self, app_name, agency_name, holding_msg_minutes, config, must_exist): + def __init__(self, agency_name, holding_msg_minutes, config, must_exist): self.config = config self.section_name = "[[agency.node]]." self.holding_msg_minutes = holding_msg_minutes - self.app_name = app_name # set the agency_name self.agency_name = agency_name # disable ra2018 or not, default enable the ra2018 self.disable_ra2018 = utilities.get_item_value( self.config, "disable_ra2018", False, False, self.section_name) + # the components + self.components = utilities.get_item_value( + self.config, "components", None, False, self.section_name) # the deploy_ip self.deploy_ip = utilities.get_item_value( self.config, "deploy_ip", None, must_exist, self.section_name) # the node_name self.node_name = utilities.get_item_value( self.config, "node_name", None, must_exist, self.section_name) - # the tars_listen_ip - self.tars_listen_ip = utilities.get_item_value( - self.config, "tars_listen_ip", "0.0.0.0", False, self.section_name) - # the tars_listen_port - self.tars_listen_port = utilities.get_item_value( - self.config, "tars_listen_port", None, must_exist, self.section_name) + # the grpc_listen_ip + self.grpc_listen_ip = utilities.get_item_value( + self.config, "grpc_listen_ip", "0.0.0.0", False, self.section_name) + # the grpc_listen_port + self.grpc_listen_port = utilities.get_item_value( + self.config, "grpc_listen_port", None, must_exist, self.section_name) utilities.log_debug("load the node config success") # parse the rpc config @@ -276,24 +247,16 @@ def __init__(self, app_name, agency_name, holding_msg_minutes, config, must_exis self.gateway_config = None if gateway_config_object is not None: self.gateway_config = NodeGatewayConfig( - self.app_name, self.agency_name, gateway_config_object, must_exist) + self.agency_name, gateway_config_object, must_exist) utilities.log_debug("load the gateway success") - # set the server name - self.service_name = "%s%s%s" % ( - self.agency_name, self.node_name, utilities.ServiceInfo.node_service_postfix) - # set the servant name - self.servant_list = utilities.ServiceInfo.node_servant - self.servant_object_list = utilities.ServiceInfo.node_servant_object - class AgencyConfig: """ the agency config """ - def __init__(self, app_name, config, gateway_must_exists, node_must_exists): - self.app_name = app_name + def __init__(self, config, gateway_must_exists, node_must_exists): self.config = config self.section_name = "[[agency]]" # the agency-name @@ -309,9 +272,8 @@ def __init__(self, app_name, config, gateway_must_exists, node_must_exists): self.config, "gateway", None, gateway_must_exists, gateway_config_section_name) self.gateway_config = None if gateway_config_object is not None: - self.gateway_config = GatewayConfig( - self.app_name, self.agency_name, self.holding_msg_minutes, gateway_config_object, - gateway_config_section_name, gateway_must_exists) + self.gateway_config = GatewayConfig(self.agency_name, self.holding_msg_minutes, gateway_config_object, + gateway_config_section_name, gateway_must_exists) utilities.log_debug("load the gateway config success") # parse the node config @@ -323,7 +285,7 @@ def __init__(self, app_name, config, gateway_must_exists, node_must_exists): # TODO: check the node-name for node_object in node_config_list: node_config = NodeConfig( - self.app_name, self.agency_name, self.holding_msg_minutes, node_object, node_must_exists) + self.agency_name, self.holding_msg_minutes, node_object, node_must_exists) self.node_list[node_config.node_name] = node_config utilities.log_debug( "load node config for %s success" % node_config.node_name) @@ -353,8 +315,7 @@ def __init__(self, config, gateway_must_exists, node_must_exists): self.sm_crypto = utilities.get_value( self.config, crypto_section, "sm_crypto", False, False) utilities.log_debug("load the crypto config success") - # load the tars config - self.tars_config = TarsConfig(self.config) + self.env_config = EnvConfig(self.config, "env") # load the agency config # TODO: check duplicated case utilities.log_debug("load the agency config") @@ -363,7 +324,7 @@ def __init__(self, config, gateway_must_exists, node_must_exists): self.config, "agency", None, False, "[[agency]]") for agency_object in agency_list_object: agency_config = AgencyConfig( - self.tars_config.app_name, agency_object, gateway_must_exists, node_must_exists) + agency_object, gateway_must_exists, node_must_exists) self.agency_list[agency_config.agency_name] = agency_config utilities.log_debug( "load the agency config for %s success" % agency_config.agency_name) diff --git a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py index b75f7987..a92eea28 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py @@ -1,10 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import os +import json from common import utilities from config.binary_generator import BinaryGenerator from config.cert_generator import CertGenerator -from config.tars_config_generator import TarsConfigGenerator +from config.shell_script_generator import ShellScriptGenerator class PPCGatewayConfigGenerator: @@ -15,138 +16,123 @@ class PPCGatewayConfigGenerator: def __init__(self, config, output_dir): self.config = config self.output_dir = output_dir - self.with_tars = False self.binary_name = utilities.ConfigInfo.ppc_gateway_binary_name self.service_type = utilities.ServiceInfo.gateway_service_type def generate_gateway_config(self): - utilities.print_badge("* generate gateway config, app: %s" % - self.config.tars_config.app_name) + utilities.print_badge("* generate gateway config, deploy_dir: %s" % + self.config.env_config.deploy_dir) # generate the ca ret = CertGenerator.generate_ca_cert( self.config.gateway_sm_ssl, self.__generate_ca_cert_path__()) if ret is False: utilities.log_error( - "* generate ca-cert config for %s failed" % self.config.tars_config.app_name) + "* generate ca-cert config for %s failed" % self.config.env_config.deploy_dir) return False + connection_config = {} + connection_config["nodes"] = [] + connection_config_path_list = [] for agency_config in self.config.agency_list.values(): gateway_config = agency_config.gateway_config - ret = self.__generate_single_gateway_config__(gateway_config) + ret = self.__generate_single_gateway_config__( + gateway_config, connection_config, connection_config_path_list) if ret is False: return False + self.__generate_connection_info__( + connection_config_path_list, connection_config) utilities.print_badge( - "* generate gateway config success, app: %s" % self.config.tars_config.app_name) + "* generate gateway config success, deploy_dir: %s" % self.config.env_config.deploy_dir) return True - def __generate_single_gateway_config__(self, gateway_config): + def __generate_single_gateway_config__(self, gateway_config, connection_config, connection_config_path_list): # load the config from tpl_config_path - utilities.log_info("* generate config for ppc-gateway %s.%s" % - (gateway_config.app_name, gateway_config.service_name)) - # store the - # store the config.ini + utilities.log_info("* generate config for ppc-gateway") for ip_str in gateway_config.deploy_ip: ip_array = ip_str.split(":") ip = ip_array[0] + # generate the shell scripts for the given ip + ret = ShellScriptGenerator.generate_ip_shell_scripts( + self.__generate_ip_shell_scripts_output_path__(gateway_config.agency_name, ip), "start_all.sh", "stop_all.sh") + if ret is False: + return False node_count = 1 if len(ip_array) >= 2: node_count = int(ip_array[1]) for node_index in range(node_count): node_name = "node" + str(node_index) - utilities.print_badge("* generate config for ppc-gateway %s.%s.%s" % - (gateway_config.app_name, gateway_config.service_name, node_name)) + connection_config_path_list.append(os.path.join(self.__generate_node_path__( + gateway_config.agency_name, ip, node_name), "nodes.json")) + utilities.print_badge( + "* generate config for ppc-gateway %s.%s, deploy_ip: %s" % (gateway_config.agency_name, node_name, ip)) config_content = utilities.load_config( utilities.ConfigInfo.gateway_config_tpl_path) # load the common config self.__generate_common_config__(gateway_config, config_content) # load the gateway config listen_port = gateway_config.listen_port + node_index + connection_config["nodes"].append(ip + ":" + str(listen_port)) + grpc_listen_port = gateway_config.grpc_listen_port + node_index self.__generate_gateway_config_content__( - gateway_config, config_content, listen_port) - # load the cache config - self.__generate_cache_config__(gateway_config, config_content) - # generate the shell scripts for the given ip - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_output_path__(ip), "start_all.sh", "stop_all.sh") - if ret is False: - return False - # generate the start.sh/stop.sh for given service - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), "start.sh", - "stop.sh") - if ret is False: - return False + gateway_config, config_content, listen_port, grpc_listen_port) # generate the binary binary_path = os.path.join( - self.config.tars_config.binary_path, self.binary_name) + self.config.env_config.binary_path, self.binary_name) dst_binary_path = os.path.join( - self.__generate_ip_shell_scripts_for_given_service__(ip, gateway_config.service_name), + self.__generate_ip_shell_scripts_output_path__( + gateway_config.agency_name, ip), self.binary_name) ret = BinaryGenerator.generate_binary( binary_path, dst_binary_path) if ret is False: return False - # generate the shell scripts for the node - service_name = gateway_config.service_name - if self.with_tars is False: - service_name = utilities.ConfigInfo.ppc_gateway_binary_name - ret = TarsConfigGenerator.generate_node_shell_scripts( - self.__generate_node_path__(gateway_config, ip, node_name), service_name) - if ret is False: - return False # generate the ini config - config_output_path = self.__generate_conf_output_path__( - gateway_config, ip, node_name) ini_config_output_path = os.path.join( - config_output_path, utilities.ConfigInfo.config_ini_file) + self.__generate_node_path__(gateway_config.agency_name, ip, node_name), utilities.ConfigInfo.config_ini_file) ret = utilities.store_config( - config_content, "ini", ini_config_output_path, gateway_config.service_name) - if ret is False: - utilities.log_error("* generate config for ppc-gateway %s.%s, ip: %s failed" % - (gateway_config.app_name, gateway_config.service_name, ip)) - return False - # generate tars config - tars_config_path = os.path.join( - config_output_path, utilities.ConfigInfo.tars_config_file) - tars_listen_port = gateway_config.tars_listen_port + node_index - ret = TarsConfigGenerator.generate_and_store_tars_conf(utilities.ConfigInfo.tars_config_tpl_path, - tars_config_path, - gateway_config.app_name, - gateway_config.service_name, - gateway_config.servant_object_list, ip, - tars_listen_port) + config_content, "ini", ini_config_output_path, "config.ini") if ret is False: + utilities.log_error( + "* generate config for ppc-gateway, ip: %s failed" % (ip)) return False # generate the node config ret = CertGenerator.generate_node_cert(self.config.gateway_sm_ssl, self.__generate_ca_cert_path__( - ), self.__generate_conf_output_path__(gateway_config, ip, node_name)) + ), self.__generate_conf_output_path__(gateway_config.agency_name, ip, node_name)) if ret is False: utilities.log_error( - "* generate config for ppc-gateway %s.%s failed for generate the node config failed" % - (gateway_config.app_name, gateway_config.service_name)) + "* generate config for ppc-gateway failed for generate the node config failed") return False - utilities.print_badge("* generate config for ppc-gateway %s.%s.%s success" % - (gateway_config.app_name, gateway_config.service_name, node_name)) - utilities.log_info("* generate config for ppc-gateway %s.%s success" % - (gateway_config.app_name, gateway_config.service_name)) + ret = ShellScriptGenerator.generate_node_shell_scripts(self.__generate_node_path__( + gateway_config.agency_name, ip, node_name), self.binary_name) + if ret is False: + return False + utilities.print_badge("* generate config for ppc-gateway%s success" % + (node_name)) + utilities.log_info("* generate config for ppc-gateway success") + return True + + def __generate_connection_info__(self, connection_config_path_list, connection_config): + connection_str = json.dumps(connection_config) + for path in connection_config_path_list: + ret = utilities.store_config( + connection_str, "json", path, "nodes.json") + if ret is False: + utilities.log_error( + "* generate nodes.json failed, path: %s" % path) + return False return True def __generate_ca_cert_path__(self): - return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + return os.path.join(self.output_dir, self.service_type, "ca") - def __generate_node_path__(self, config, ip, node_name): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, - config.service_name, node_name) + def __generate_node_path__(self, agency_name, ip, node_name): + return os.path.join(self.output_dir, agency_name, ip, self.service_type, node_name) - def __generate_conf_output_path__(self, config, ip, node_name): - node_path = self.__generate_node_path__(config, ip, node_name) + def __generate_conf_output_path__(self, agency_name, ip, node_name): + node_path = self.__generate_node_path__(agency_name, ip, node_name) return os.path.join(node_path, "conf") - def __generate_ip_shell_scripts_output_path__(self, ip): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) - - def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): - ip_path = self.__generate_ip_shell_scripts_output_path__(ip) - return os.path.join(ip_path, service_name) + def __generate_ip_shell_scripts_output_path__(self, agency_name, ip): + return os.path.join(self.output_dir, agency_name, ip, self.service_type) def __generate_common_config__(self, config, config_content): """ @@ -157,12 +143,8 @@ def __generate_common_config__(self, config, config_content): config_content[section]["id"] = config.agency_name config_content["gateway"]["holding_msg_minutes"] = str( config.holding_msg_minutes) - # the peers info - for peer in config.peers: - key = "agency." + peer.agency - config_content[section][key] = str(','.join(peer.endpoints)) - def __generate_gateway_config_content__(self, config, config_content, listen_port): + def __generate_gateway_config_content__(self, config, config_content, listen_port, grpc_listen_port): """ generate the gateway config """ @@ -179,29 +161,7 @@ def __generate_gateway_config_content__(self, config, config_content, listen_por # disable ssl config_content[section]["disable_ssl"] = utilities.convert_bool_to_str( self.config.gateway_disable_ssl) - - def __generate_cache_config__(self, config, config_content): - """ - generate the cache config - """ - section = "cache" - config_content[section]["type"] = config.cache_type - config_content[section]["proxy"] = str(config.cache_proxy) - config_content[section]["obServer"] = str(config.cache_obServer) - config_content[section]["cluster"] = str(config.cache_cluster) - # the host - config_content[section]["host"] = config.cache_host - # the port - config_content[section]["port"] = str(config.cache_port) - # the password - config_content[section]["password"] = str(config.cache_password) - # the database - config_content[section]["database"] = str(config.cache_database) - # the pool_size - config_content[section]["pool_size"] = str(config.cache_pool_size) - # the connection_timeout - config_content[section]["connection_timeout"] = str( - config.cache_connection_timeout) - # the socket_timeout - config_content[section]["socket_timeout"] = str( - config.cache_socket_timeout) + section = "transport" + config_content[section]["listen_ip"] = config.grpc_listen_ip + # the listen port + config_content[section]["listen_port"] = str(config.grpc_listen_port) diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py index b3962b4c..3e9d8c2c 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py @@ -1,10 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import os +import sys from common import utilities from config.binary_generator import BinaryGenerator from config.cert_generator import CertGenerator -from config.tars_config_generator import TarsConfigGenerator +from config.shell_script_generator import ShellScriptGenerator class PPCNodeConfigGenerator: @@ -15,7 +16,6 @@ class PPCNodeConfigGenerator: def __init__(self, config, output_dir): self.config = config self.output_dir = output_dir - self.with_tars = False self.binary_name = utilities.ConfigInfo.ppc_node_binary_name self.service_type = utilities.ServiceInfo.node_service_type @@ -26,83 +26,74 @@ def generate_node_config(self): self.config.rpc_sm_ssl, self.__generate_ca_cert_path__()) if ret is False: utilities.log_error( - "* generate_node_config failed for generate ca error, app: %s" % (self.config.tars_config.app_name)) + "* generate_node_config failed for generate ca error") return False for agency_config in self.config.agency_list.values(): for node_config in agency_config.node_list.values(): for ip_str in node_config.deploy_ip: + if ret is False: + return False ip_array = ip_str.split(":") ip = ip_array[0] + # generate the shell scripts for the given ip + ret = ShellScriptGenerator.generate_ip_shell_scripts( + self.__generate_ip_output_path__(node_config.agency_name, ip), "start_all.sh", "stop_all.sh") node_count = 1 if len(ip_array) >= 2: node_count = int(ip_array[1]) for node_index in range(node_count): node_name = "node" + str(node_index) - if self.__generate_single_node_config__(node_config, ip, node_name, node_index) is False: + if self.__generate_single_node_config__(node_config, ip, node_name, node_config.agency_name, node_index) is False: return False utilities.print_badge("* generate_node_config success") return True - def __generate_single_node_config__(self, node_config, ip, node_name, node_index): - utilities.print_badge("* generate node config for %s.%s.%s, ip: %s" % - (node_config.app_name, node_config.service_name, node_name, ip)) - # generate the shell scripts for the given ip - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_output_path__(ip), "start_all.sh", "stop_all.sh") - if ret is False: - return False - # generate the start.sh/stop.sh for given service - ret = TarsConfigGenerator.generate_ip_shell_scripts( - self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), "start.sh", "stop.sh") - if ret is False: - return False + def __generate_single_node_config__(self, node_config, ip, node_name, agency_name, node_index): + utilities.print_badge("* generate node config for %s, ip: %s, agency: %s" % + (node_name, ip, agency_name)) # copy the binary binary_path = os.path.join( - self.config.tars_config.binary_path, self.binary_name) + self.config.env_config.binary_path, self.binary_name) dst_binary_path = os.path.join( - self.__generate_ip_shell_scripts_for_given_service__(ip, node_config.service_name), self.binary_name) + self.__generate_ip_output_path__(agency_name, ip), self.binary_name) ret = BinaryGenerator.generate_binary(binary_path, dst_binary_path) if ret is False: return False # generate the node config - node_path = self.__generate_node_conf_path__( - node_config, ip, node_name) + node_path = self.__generate_node_path__( + agency_name, ip, node_name) + private_key_path = self.__generate_node_conf_path__( - node_config, ip, node_name) - tars_output_path = private_key_path + agency_name, ip, node_name) if self.__generate_single_node_inner_config__(utilities.ConfigInfo.node_config_tpl_path, - utilities.ConfigInfo.tars_config_tpl_path, node_path, - private_key_path, tars_output_path, node_config, ip, + node_path, + private_key_path, node_config, ip, node_index) is False: - utilities.log_error("* generate node config for %s.%s, ip: %s failed" % - (node_config.app_name, node_config.service_name, ip)) + utilities.log_error("* generate node config, ip: %s failed" % + (ip)) return False - # generate the node shell_scipts - service_name = node_config.service_name - if self.with_tars is False: - service_name = utilities.ConfigInfo.ppc_node_binary_name - ret = TarsConfigGenerator.generate_node_shell_scripts( - self.__generate_node_path__(node_config, ip, node_name), service_name) - if ret is False: - return False # generate the node cert(for rpc) ret = CertGenerator.generate_node_cert(self.config.rpc_sm_ssl, self.__generate_ca_cert_path__( - ), self.__generate_node_conf_path__(node_config, ip, node_name)) + ), self.__generate_node_conf_path__(agency_name, ip, node_name)) + if ret is False: + utilities.log_error("* generate node config, ip: %s failed for generate rpc cert failed" % + (ip)) + return False + ret = ShellScriptGenerator.generate_node_shell_scripts( + self.__generate_node_path__(agency_name, ip, node_name), self.binary_name) if ret is False: - utilities.log_error("* generate node config for %s.%s, ip: %s failed for generate rpc cert failed" % - (node_config.app_name, node_config.service_name, ip)) return False - utilities.print_badge("* generate node config for %s.%s.%s, ip: %s success" % - (node_config.app_name, node_config.service_name, node_name, ip)) + utilities.print_badge("* generate node config %s, ip: %s.%s success" % + (node_name, agency_name, ip)) return True - def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl_path, node_path, private_key_path, - tars_output_path, node_config, ip, node_index): + def __generate_single_node_inner_config__(self, tpl_config_path, node_path, private_key_path, + node_config, ip, node_index): config_content = utilities.load_config(tpl_config_path) utilities.log_debug( - "__generate_single_node_config__, load config.ini from %s" % tpl_config_path) + "__generate_single_node_inner_config__, load config.ini from %s" % tpl_config_path) # generate the private key - (ret, private_key) = CertGenerator.generate_private_key(private_key_path) + (ret, node_id) = CertGenerator.generate_private_key(private_key_path) if ret is False: return False # load the common config @@ -111,7 +102,9 @@ def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl # load the rpc config self.__generate_rpc_config__( config_content, node_config.rpc_config, node_index) - # TODO: check the configuration + # load the transport config + self.__generate_transport_config__(config_content, + node_config, node_id, ip) # load the storage config self.__generate_storage_config__( config_content, node_config.storage_config) @@ -121,38 +114,28 @@ def __generate_single_node_inner_config__(self, tpl_config_path, tars_config_tpl # load the ra2018psi config self.__generate_ra2018psi_config__( config_content, node_config.ra2018psi_config) - # load the gateway tars config - self.__generate_tars_gateway_config__( - config_content, node_config.gateway_config) # store the config ini_config_output_path = os.path.join( node_path, utilities.ConfigInfo.config_ini_file) ret = utilities.store_config( - config_content, "ini", ini_config_output_path, node_config.service_name) + config_content, "ini", ini_config_output_path, "config.ini") if ret is False: return False - # load the tars config - self.__generate_tars_config__( - tars_config_tpl_path, tars_output_path, node_config, ip, node_index) return True def __generate_ca_cert_path__(self): - return os.path.join(self.output_dir, self.config.tars_config.app_name, "ca", self.service_type) + return os.path.join(self.output_dir, self.service_type, "ca") - def __generate_node_path__(self, node_config, ip, node_name): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type, - node_config.service_name, node_name) + def __generate_node_path__(self, agency_name, ip, node_name): + return os.path.join(self.output_dir, agency_name, ip, self.service_type, + node_name) - def __generate_node_conf_path__(self, node_config, ip, node_name): - node_path = self.__generate_node_path__(node_config, ip, node_name) + def __generate_node_conf_path__(self, agency_name, ip, node_name): + node_path = self.__generate_node_path__(agency_name, ip, node_name) return os.path.join(node_path, "conf") - def __generate_ip_output_path__(self, ip): - return os.path.join(self.output_dir, self.config.tars_config.app_name, ip, self.service_type) - - def __generate_ip_shell_scripts_for_given_service__(self, ip, service_name): - ip_path = self.__generate_ip_output_path__(ip) - return os.path.join(ip_path, service_name) + def __generate_ip_output_path__(self, agency_name, ip): + return os.path.join(self.output_dir, agency_name, ip, self.service_type) def __generate_common_config__(self, config_content, node_config): """ @@ -160,9 +143,6 @@ def __generate_common_config__(self, config_content, node_config): """ # the agency config config_content["agency"]["id"] = node_config.agency_name - # the holding_msg_minutes - config_content["tars_gateway"]["holding_msg_minutes"] = str( - node_config.holding_msg_minutes) # disable ra2018 or not config_content["agency"]["disable_ra2018"] = utilities.convert_bool_to_str( node_config.disable_ra2018) @@ -212,6 +192,31 @@ def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): hdfs_storage_config.name_node_port) config_content[section_name]["token"] = hdfs_storage_config.token + def __generate_transport_config__(self, config_content, node_config, node_id, deploy_ip): + """_summary_ + + Args: + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 18000 + host_ip = + ; the threadPoolSize + thread_count = 4 + ; the gatewayService endpoint information + service.gateway_target = + ; the components + service.components = + nodeid= + """ + section = "transport" + config_content[section]["listen_ip"] = node_config.grpc_listen_ip + config_content[section]["listen_port"] = str( + node_config.grpc_listen_port) + config_content[section]["host_ip"] = deploy_ip + config_content[section]["service.gateway_target"] = node_config.gateway_config.gateway_grpc_target + config_content[section]["service.components"] = node_config.components + config_content[section]["nodeid"] = node_id + def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): """ generate the ra2018psi config @@ -236,24 +241,3 @@ def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): ra2018psi_config.data_batch_size) config_content[section_name]["use_hdfs"] = utilities.convert_bool_to_str( ra2018psi_config.use_hdfs) - - def __generate_tars_gateway_config__(self, config_content, tars_gateway_config): - section_name = "tars_gateway" - config_content[section_name]["name"] = tars_gateway_config.service_name - i = 0 - for endpoint in tars_gateway_config.endpoints: - key = "proxy.%d" % i - config_content[section_name][key] = endpoint - i = i + 1 - - def __generate_tars_config__(self, tars_config_tpl_path, tars_config_dir, node_config, ip, node_index): - """ - generate the tars config - """ - tars_config_path = os.path.join( - tars_config_dir, utilities.ConfigInfo.tars_config_file) - # generate the tars config of the gateway-service - tars_listen_port = node_config.tars_listen_port + node_index - TarsConfigGenerator.generate_and_store_tars_conf(tars_config_tpl_path, tars_config_path, - node_config.app_name, node_config.service_name, - node_config.servant_object_list, ip, tars_listen_port) diff --git a/cpp/tools/ppc-builder/src/config/shell_script_generator.py b/cpp/tools/ppc-builder/src/config/shell_script_generator.py new file mode 100644 index 00000000..557e03ff --- /dev/null +++ b/cpp/tools/ppc-builder/src/config/shell_script_generator.py @@ -0,0 +1,75 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +from common import utilities +import os + + +class ShellScriptGenerator: + """ + generate the shell-scripts + """ + def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name): + start_all_path = os.path.join( + script_output_dir, start_shell_script_name) + utilities.mkdir(script_output_dir) + if os.path.exists(start_all_path) is False: + utilities.log_debug( + "* generate shell script, dst: %s" % start_all_path) + # start_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.start_all_tpl_path, start_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (start_all_path, output)) + return False + stop_all_path = os.path.join( + script_output_dir, stop_shell_script_name) + if os.path.exists(stop_all_path) is False: + # tars stop_all.sh + command = "cp %s %s" % ( + utilities.ConfigInfo.stop_all_tpl_path, stop_all_path) + (result, output) = utilities.execute_command_and_getoutput(command) + if result is False: + utilities.log_error( + "* generate %s failed, error: %s" % (stop_all_path, output)) + return False + utilities.log_debug( + "* generate_ip_shell_scripts success, output: %s" % script_output_dir) + return True + + def __update_binary__(file_path, output_path, binary_name): + command = "cp %s %s" % (file_path, output_path) + (ret, _) = utilities.execute_command_and_getoutput(command) + if ret is False: + return False + updated_config = "" + with open(output_path, 'r', encoding='UTF-8') as file: + updated_config = file.read() + updated_config = updated_config.replace( + "@BINARY_NAME@", binary_name) + with open(output_path, 'w', encoding='UTF-8') as file: + file.write(updated_config) + return True + + def generate_node_shell_scripts(script_output_dir, binary_name): + utilities.log_debug("* generate shell scripts for %s, dst: %s" % + (binary_name, script_output_dir)) + utilities.mkdir(script_output_dir) + # the start.sh + output_path = os.path.join(script_output_dir, "start.sh") + ret = ShellScriptGenerator.__update_binary__( + utilities.ConfigInfo.start_tpl_path, output_path, binary_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + # the stop.sh + output_path = os.path.join(script_output_dir, "stop.sh") + ret = ShellScriptGenerator.__update_binary__( + utilities.ConfigInfo.stop_tpl_path, output_path, binary_name) + if ret is False: + utilities.log_error( + "generate_node_shell_scripts %s error" % output_path) + return False + return True diff --git a/cpp/tools/ppc-builder/src/config/tars_config_generator.py b/cpp/tools/ppc-builder/src/config/tars_config_generator.py deleted file mode 100644 index 1540f486..00000000 --- a/cpp/tools/ppc-builder/src/config/tars_config_generator.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/python -# -*- coding: UTF-8 -*- -from common import utilities -import os - - -class TarsConfigGenerator: - """ - generate the tars-config - """ - def __generate_adapter_info__(app_name, server_name, servant_object, tars_listen_ip, tars_listen_port): - adapter_name = "%s.%s.%sAdapter" % ( - app_name, server_name, servant_object) - servant_name = "%s.%s.%s" % (app_name, server_name, servant_object) - result = "<%s>\n" % adapter_name - result = "%s\t\tallow\n" % result - result = "%s\t\tendpoint=tcp -h %s -p %d -t 60000\n" % ( - result, tars_listen_ip, tars_listen_port) - result = "%s\t\tmaxconns=100000\n\t\tprotocol=tars\n\t\tqueuecap=50000\n\t\tqueuetimeout=20000\n" % result - result = "%s\t\tservant=%s\n" % (result, servant_name) - result = "%s\t\tthreads=8\n\t\t\n" % (result, adapter_name) - return result - - def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name): - tars_start_all_path = os.path.join( - script_output_dir, start_shell_script_name) - utilities.mkdir(script_output_dir) - if os.path.exists(tars_start_all_path) is False: - utilities.log_debug( - "* generate shell script, dst: %s" % tars_start_all_path) - # tars_start_all.sh - command = "cp %s %s" % ( - utilities.ConfigInfo.tars_start_all_tpl_path, tars_start_all_path) - (result, output) = utilities.execute_command_and_getoutput(command) - if result is False: - utilities.log_error( - "* generate %s failed, error: %s" % (tars_start_all_path, output)) - return False - tars_stop_all_path = os.path.join( - script_output_dir, stop_shell_script_name) - if os.path.exists(tars_stop_all_path) is False: - # tars stop_all.sh - command = "cp %s %s" % ( - utilities.ConfigInfo.tars_stop_all_tpl_path, tars_stop_all_path) - (result, output) = utilities.execute_command_and_getoutput(command) - if result is False: - utilities.log_error( - "* generate %s failed, error: %s" % (tars_stop_all_path, output)) - return False - utilities.log_debug( - "* generate_ip_shell_scripts success, output: %s" % script_output_dir) - return True - - def __update_service_name__(file_path, output_path, service_name): - command = "cp %s %s" % (file_path, output_path) - (ret, output) = utilities.execute_command_and_getoutput(command) - if ret is False: - return False - updated_config = "" - with open(output_path, 'r', encoding='UTF-8') as file: - updated_config = file.read() - updated_config = updated_config.replace( - "@SERVICE_NAME@", "../" + service_name) - with open(output_path, 'w', encoding='UTF-8') as file: - file.write(updated_config) - return True - - def generate_node_shell_scripts(script_output_dir, service_name): - utilities.log_debug("* generate shell scripts for %s, dst: %s" % - (service_name, script_output_dir)) - utilities.mkdir(script_output_dir) - # the start.sh - output_path = os.path.join(script_output_dir, "start.sh") - ret = TarsConfigGenerator.__update_service_name__( - utilities.ConfigInfo.tars_start_tpl_path, output_path, service_name) - if ret is False: - utilities.log_error( - "generate_node_shell_scripts %s error" % output_path) - return False - # the stop.sh - output_path = os.path.join(script_output_dir, "stop.sh") - ret = TarsConfigGenerator.__update_service_name__( - utilities.ConfigInfo.tars_stop_tpl_path, output_path, service_name) - if ret is False: - utilities.log_error( - "generate_node_shell_scripts %s error" % output_path) - return False - return True - - def generate_and_store_tars_conf(conf_tpl_path, output_path, app_name, server_name, servant_object_list, tars_listen_ip, tars_listen_port): - utilities.log_debug("generate_and_store_tars_conf: %s" % conf_tpl_path) - utilities.mkfiledir(output_path) - # copy tpl to output_path - if os.path.exists(output_path): - utilities.log_error("the tars file %s already exists!") - return False - command = "cp %s %s" % (conf_tpl_path, output_path) - (ret, output) = utilities.execute_command_and_getoutput(command) - if ret is False: - utilities.log_error("copy tpl tars config %s to %s error: %s" % ( - conf_tpl_path, output_path, output)) - return False - # config - module_name = "%s.%s" % (app_name, server_name) - - tars_adapters = "" - for servant_obj in servant_object_list: - adapter_info = TarsConfigGenerator.__generate_adapter_info__( - app_name, server_name, servant_obj, tars_listen_ip, tars_listen_port) - tars_adapters = "%s%s" % (tars_adapters, adapter_info) - utilities.log_debug("tars_adapters: %s" % tars_adapters) - updated_config = "" - with open(output_path, 'r', encoding='UTF-8') as file: - updated_config = file.read() - updated_config = updated_config.replace("@TARS_APP@", app_name) - updated_config = updated_config.replace( - "@TARS_SERVER@", server_name) - updated_config = updated_config.replace( - "@MODULE_NAME@", module_name) - updated_config = updated_config.replace( - "@TARS_ADAPTERS@", tars_adapters) - with open(output_path, 'w', encoding='UTF-8') as file: - file.write(updated_config) diff --git a/cpp/tools/ppc-builder/src/scripts/gen_cert.sh b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh index 2ee333b6..187fa66e 100644 --- a/cpp/tools/ppc-builder/src/scripts/gen_cert.sh +++ b/cpp/tools/ppc-builder/src/scripts/gen_cert.sh @@ -435,8 +435,9 @@ generate_private_key_for_psi_server() { fi ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g' | cat > "$output_path/node.privateKey" - private_key=$(cat $output_path/node.privateKey) - echo ${private_key} + ${OPENSSL_CMD} ec -text -noout -in "${output_path}/node.pem" 2>/dev/null | sed -n '7,11p' | tr -d ": \n" | awk '{print substr($0,3);}' | cat >"$output_path"/node.nodeid + public_key=$(cat $output_path/node.nodeid) + echo ${public_key} } check_env() { diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway index a7319da2..8acc4990 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway @@ -1,7 +1,6 @@ [agency] ; the agency-id of self-party id = agency0 - ; the agency info [cert] ; directory the certificates located in @@ -10,30 +9,27 @@ [gateway] listen_ip=0.0.0.0 listen_port=40300 - ;thread_count = 4 + ;the dir that contains the connected endpoint information, e.g.nodes.json + ;nodes_path="./" + ; the file that configure the connected endpoint information + ; nodes_file=nodes.json + ; thread_count = 4 ; ssl or sm ssl sm_ssl=false + ; the max allowed message size in MBytes, default is 100MBytes + max_allow_msg_size = 100 ;ssl connection switch, if disable the ssl connection, default: false - disable_ssl = false + ;disable_ssl = true ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes holding_msg_minutes = 30 - ; disable_cache = false -[cache] - ; the cache type, only support redis now - type = 0 - proxy = 127.0.0.1:20002 - obServer = 127.0.0.1:10904 - cluster = REDIS_CLUSTER - user = 1194 - host = 127.0.0.1 - port = 6379 - password = - database = 1 - pool_size = 16 - ; the redis connection timeout, in ms, default is 500ms - connection_timeout = 500 - ; the redis socket timeout, in ms, default is 500ms - socket_timeout = 500 + ;reconnect_time = 10000 + ; the unreachable distance + ;unreachable_distance=10 + +[transport] + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 19000 [log] enable=true @@ -52,7 +48,5 @@ ; archive_path=./log/ compress_archive_file=true ; ; 0: no limit, in MB - ; max_archive_files=10 - ; ; 0: no limit, in MB ; max_archive_size=0 ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node index 4989f1d1..177536ac 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.node +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node @@ -30,12 +30,18 @@ ; directory the certificates located in cert_path=./conf -[tars_gateway] - ; the gateway service name - name = - ;the time of the gateway holding and waiting to dispatcher the unsynced task, in minutes - holding_msg_minutes = 30 - ; the gateway service endpoints +[transport] + ; the endpoint information + listen_ip = 0.0.0.0 + listen_port = 18000 + host_ip = + ; the threadPoolSize + thread_count = 4 + ; the gatewayService endpoint information + service.gateway_target = + ; the components + service.components = + nodeid= [storage] host = 127.0.0.1 @@ -103,8 +109,6 @@ ; if archive_path is empty, the archive function will be disabled ; archive_path=./log/ compress_archive_file=true - ; ; 0: no limit, in MB + ; 0: no limit, in MB ; max_archive_files=10 - ; ; 0: no limit, in MB - ; max_archive_size=0 ; min_free_space=0 \ No newline at end of file diff --git a/cpp/tools/ppc-builder/src/tpl/start.sh b/cpp/tools/ppc-builder/src/tpl/start.sh new file mode 100755 index 00000000..83921a4c --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/start.sh @@ -0,0 +1,39 @@ +#!/bin/bash +SHELL_FOLDER=$(cd $(dirname $0);pwd) + +LOG_ERROR() { + content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +binary=${SHELL_FOLDER}/../@BINARY_NAME@ +cd ${SHELL_FOLDER} +node=$(basename ${SHELL_FOLDER}) +node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') +if [ ! -z ${node_pid} ];then + echo " ${node} is running, pid is $node_pid." + exit 0 +else + nohup ${binary} -c config.ini >>nohup.out 2>&1 & + sleep 1.5 +fi +try_times=4 +i=0 +while [ $i -lt ${try_times} ] +do + node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') + success_flag=$(tail -n20 nohup.out | grep running) + if [[ ! -z ${node_pid} && ! -z "${success_flag}" ]];then + echo -e "\033[32m ${node} start successfully pid=${node_pid}\033[0m" + exit 0 + fi + sleep 0.5 + ((i=i+1)) +done +echo -e "\033[31m Exceed waiting time. Please try again to start ${node} \033[0m" +tail -n20 nohup.out diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh b/cpp/tools/ppc-builder/src/tpl/start_all.sh similarity index 62% rename from cpp/tools/ppc-builder/src/tpl/tars_start_all.sh rename to cpp/tools/ppc-builder/src/tpl/start_all.sh index 006f53b4..b0e20f1b 100644 --- a/cpp/tools/ppc-builder/src/tpl/tars_start_all.sh +++ b/cpp/tools/ppc-builder/src/tpl/start_all.sh @@ -5,10 +5,9 @@ cd "${dirpath}" dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) for dir in ${dirs[*]} do - if [[ -f "${dirpath}/${dir}/start.sh" ]];then - echo "try to start ${dirpath}/${dir}" + if [[ -f "${dirpath}/${dir}/config.ini" && -f "${dirpath}/${dir}/start.sh" ]];then + echo "try to start ${dir}" bash ${dirpath}/${dir}/start.sh & fi done wait - diff --git a/cpp/tools/ppc-builder/src/tpl/stop.sh b/cpp/tools/ppc-builder/src/tpl/stop.sh new file mode 100755 index 00000000..ae40e5ba --- /dev/null +++ b/cpp/tools/ppc-builder/src/tpl/stop.sh @@ -0,0 +1,35 @@ +#!/bin/bash +SHELL_FOLDER=$(cd $(dirname $0);pwd) + +LOG_ERROR() { + content=${1} + echo -e "\033[31m[ERROR] ${content}\033[0m" +} + +LOG_INFO() { + content=${1} + echo -e "\033[32m[INFO] ${content}\033[0m" +} + +binary=${SHELL_FOLDER}/../@BINARY_NAME@ +node=$(basename ${SHELL_FOLDER}) +node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') +try_times=10 +i=0 +if [ -z ${node_pid} ];then + echo " ${node} isn't running." + exit 0 +fi +[ ! -z ${node_pid} ] && kill ${node_pid} > /dev/null +while [ $i -lt ${try_times} ] +do + sleep 1 + node_pid=$(ps aux|grep ${binary}|grep -v grep|awk '{print $2}') + if [ -z ${node_pid} ];then + echo -e "\033[32m stop ${node} success.\033[0m" + exit 0 + fi + ((i=i+1)) +done +echo " Exceed maximum number of retries. Please try again to stop ${node}" +exit 1 diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh b/cpp/tools/ppc-builder/src/tpl/stop_all.sh similarity index 62% rename from cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh rename to cpp/tools/ppc-builder/src/tpl/stop_all.sh index 7a71beca..63cc7f48 100644 --- a/cpp/tools/ppc-builder/src/tpl/tars_stop_all.sh +++ b/cpp/tools/ppc-builder/src/tpl/stop_all.sh @@ -5,10 +5,9 @@ cd "${dirpath}" dirs=($(ls -l ${dirpath} | awk '/^d/ {print $NF}')) for dir in ${dirs[*]} do - if [[ -f "${dirpath}/${dir}/stop.sh" ]];then - echo "try to stop ${dirpath}/${dir}" + if [[ -f "${dirpath}/${dir}/config.ini" && -f "${dirpath}/${dir}/stop.sh" ]];then + echo "try to stop ${dir}" bash ${dirpath}/${dir}/stop.sh fi done wait - diff --git a/cpp/tools/ppc-builder/src/tpl/tars.conf b/cpp/tools/ppc-builder/src/tpl/tars.conf deleted file mode 100644 index f2157319..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars.conf +++ /dev/null @@ -1,31 +0,0 @@ - - - enableset=n - setdivision=NULL - - app=@TARS_APP@ - server=@TARS_SERVER@ - localip=127.0.0.1 - basepath=./conf/ - datapath=./.data/ - logpath=./log/ - logsize=100M - lognum=5 - logLevel=INFO - deactivating-timeout=3000 - activating-timeout=10000 - opencoroutine=0 - coroutinememsize=1G - coroutinestack=128K - closecout=0 - netthread=4 - @TARS_ADAPTERS@ - - - sync-invoke-timeout=3000000 - async-invoke-timeout=5000000 - asyncthread=8 - modulename=@MODULE_NAME@ - - - diff --git a/cpp/tools/ppc-builder/src/tpl/tars_start.sh b/cpp/tools/ppc-builder/src/tpl/tars_start.sh deleted file mode 100644 index a03b6fb4..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars_start.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd ${dirpath} - -service_name='@SERVICE_NAME@' -service=${dirpath}/${service_name} - -pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') -name=$(basename ${dirpath}) - -if [ ! -z ${pid} ];then - echo " ${service}/${name} is running, pid is ${pid}." - exit 0 -else - nohup ${service} --config=conf/tars.conf >>nohup.out 2>&1 & - sleep 1.5 -fi - -try_times=4 -i=0 -while [ $i -lt ${try_times} ] -do - pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - if [[ ! -z ${pid} ]];then - echo -e "\033[32m ${service}/${name} start successfully pid=${pid}\033[0m" - exit 0 - fi - sleep 0.5 - ((i=i+1)) -done -echo -e "\033[31m Exceed waiting time. Please try again to start ${service}/${name} \033[0m" -tail -n20 nohup.out - diff --git a/cpp/tools/ppc-builder/src/tpl/tars_stop.sh b/cpp/tools/ppc-builder/src/tpl/tars_stop.sh deleted file mode 100644 index 3e8931ce..00000000 --- a/cpp/tools/ppc-builder/src/tpl/tars_stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd ${dirpath} - -service_name='@SERVICE_NAME@' -service=${dirpath}/${service_name} -name=$(basename ${dirpath}) - -pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - -if [ -z ${pid} ];then - echo " ${service}/${name} isn't running." - exit 0 -fi - -kill ${pid} > /dev/null - -i=0 -try_times=10 -while [ $i -lt ${try_times} ] -do - sleep 1 - pid=$(ps aux|grep ${service}|grep -v grep|awk '{print $2}') - if [ -z ${pid} ];then - echo -e "\033[32m stop ${service}/${name} success.\033[0m" - exit 0 - fi - ((i=i+1)) -done -echo " Exceed maximum number of retries. Please try again to stop ${service}/${name}" -exit 1 From 4415b6e981fd446df968c0dcb6110c55b9c39c4b Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 6 Sep 2024 19:52:20 +0800 Subject: [PATCH 027/120] add ut for protocols && fix gatewayNodeInfo encode concurrency bug (#24) * fix initialize problems * add ut for Message * fix gatewayNodeInfo encode concurrency bug --- cpp/ppc-framework/gateway/IGateway.h | 8 +- cpp/ppc-framework/protocol/INodeInfo.h | 1 + cpp/ppc-framework/protocol/Message.h | 5 +- cpp/ppc-framework/protocol/PPCMessageFace.h | 15 +- cpp/test-utils/FakeFront.h | 10 +- cpp/test-utils/FakePPCMessage.h | 17 ++ .../tests/cm2020-psi/TestCM2020Impl.cpp | 2 +- .../ppc-psi/tests/labeled-psi/DataTools.h | 1 - .../tests/labeled-psi/TestLabeledPSIImpl.cpp | 2 +- .../tests/labeled-psi/TestSenderDB.cpp | 2 +- .../tests/ra2018-psi/TestEcdhPSIImpl.cpp | 6 +- .../tests/ra2018-psi/TestRA2018Impl.cpp | 2 +- .../ppc-psi/tests/ra2018-psi/mock/Common.h | 2 +- .../tests/ra2018-psi/mock/EcdhPSIFixture.h | 2 + .../ppc-tools/src/config/PPCConfig.cpp | 11 +- .../ppc-tools/src/config/PPCConfig.h | 16 +- cpp/wedpr-helper/ppc-utilities/Utilities.h | 13 +- cpp/wedpr-initializer/Initializer.cpp | 20 ++- cpp/wedpr-initializer/Initializer.h | 5 +- .../air-node/AirNodeInitializer.cpp | 4 +- cpp/wedpr-main/gateway/GatewayInitializer.cpp | 2 +- .../pro-node/ProNodeInitializer.cpp | 4 +- .../grpc/client/FrontClient.cpp | 4 +- .../grpc/client/GatewayClient.cpp | 8 +- .../grpc/client/GatewayClient.h | 7 +- .../grpc/server/FrontServer.cpp | 2 +- .../grpc/server/GatewayServer.cpp | 5 +- cpp/wedpr-protocol/proto/pb/Service.proto | 1 + cpp/wedpr-protocol/protobuf/CMakeLists.txt | 35 +--- .../protobuf/src/CMakeLists.txt | 27 ++++ .../protobuf/{ => src}/Common.h | 0 .../protobuf/{ => src}/NodeInfoImpl.cpp | 10 +- .../protobuf/{ => src}/NodeInfoImpl.h | 40 ++--- .../protobuf/{ => src}/RequestConverter.h | 4 +- .../protobuf/tests/CMakeLists.txt | 10 ++ .../protobuf/tests/NodeInfoImplTest.cpp | 71 +++++++++ cpp/wedpr-protocol/protobuf/tests/main.cpp | 2 + cpp/wedpr-protocol/protocol/CMakeLists.txt | 2 +- cpp/wedpr-protocol/protocol/src/Common.h | 29 ++++ .../protocol/src/PPCMessage.cpp | 58 +++++++ cpp/wedpr-protocol/protocol/src/PPCMessage.h | 30 ++-- .../protocol/src/v1/MessageHeaderImpl.cpp | 45 +++--- .../protocol/src/v1/MessageHeaderImpl.h | 6 +- .../protocol/src/v1/MessageImpl.cpp | 32 +++- .../protocol/src/v1/MessageImpl.h | 4 +- .../protocol/src/v1/MessagePayloadImpl.cpp | 4 +- .../protocol/tests/MessageTest.cpp | 150 ++++++++++++++++++ .../protocol/tests/PPCMessageTest.cpp | 127 ++++++++++++--- .../protocol/tests/TestTaskImpl.cpp | 2 +- cpp/wedpr-protocol/tars/CMakeLists.txt | 7 - .../ppc-front/ppc-front/Front.cpp | 5 +- .../ppc-front/ppc-front/Front.h | 2 +- .../ppc-front/ppc-front/FrontImpl.cpp | 13 +- .../ppc-front/ppc-front/FrontImpl.h | 4 +- .../test/unittests/PPCChannelTest.cpp | 9 +- .../ppc-gateway/ppc-gateway/Common.h | 2 + .../ppc-gateway/GatewayFactory.cpp | 4 +- .../ppc-gateway/gateway/GatewayImpl.cpp | 11 +- .../ppc-gateway/gateway/GatewayImpl.h | 7 +- .../gateway/router/GatewayNodeInfoImpl.cpp | 59 ++++--- .../gateway/router/GatewayNodeInfoImpl.h | 44 +++-- .../gateway/router/GatewayRouterManager.cpp | 13 +- .../gateway/router/LocalRouter.cpp | 13 ++ .../ppc-gateway/gateway/router/LocalRouter.h | 16 +- .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 97 +++++++---- .../ppc-gateway/ppc-gateway/p2p/Service.h | 2 +- .../ppc-gateway/p2p/router/RouterManager.cpp | 28 ++-- .../p2p/router/RouterTableImpl.cpp | 77 ++++----- .../unittests/GatewayNodeInfoImplTest.cpp | 135 ++++++++++++++++ .../ppc-gateway/test/unittests/MockCache.h | 103 ++++++------ .../ppc-rpc/src/RpcFactory.cpp | 2 +- cpp/wedpr-transport/sdk/TransportBuilder.cpp | 2 +- cpp/wedpr-transport/sdk/TransportImpl.h | 2 +- 73 files changed, 1104 insertions(+), 418 deletions(-) create mode 100644 cpp/wedpr-protocol/protobuf/src/CMakeLists.txt rename cpp/wedpr-protocol/protobuf/{ => src}/Common.h (100%) rename cpp/wedpr-protocol/protobuf/{ => src}/NodeInfoImpl.cpp (79%) rename cpp/wedpr-protocol/protobuf/{ => src}/NodeInfoImpl.h (67%) rename cpp/wedpr-protocol/protobuf/{ => src}/RequestConverter.h (98%) create mode 100644 cpp/wedpr-protocol/protobuf/tests/CMakeLists.txt create mode 100644 cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp create mode 100644 cpp/wedpr-protocol/protobuf/tests/main.cpp create mode 100644 cpp/wedpr-protocol/protocol/src/Common.h create mode 100644 cpp/wedpr-protocol/protocol/tests/MessageTest.cpp create mode 100644 cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayNodeInfoImplTest.cpp diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index 28b2fcd8..e4422b4a 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -58,11 +58,13 @@ class IGateway * @param callback callback */ virtual void asyncSendMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, - long timeout, ppc::protocol::ReceiveMsgFunc callback) = 0; + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, + bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) = 0; virtual void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) = 0; + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, + bcos::bytes&& payload) = 0; + virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; virtual bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; virtual bcos::Error::Ptr registerTopic( diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 680f0df9..46b29a8d 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -67,6 +67,7 @@ class INodeInfoFactory virtual INodeInfo::Ptr build() = 0; virtual INodeInfo::Ptr build(bcos::bytesConstRef nodeID, std::string const& endPoint) = 0; + virtual INodeInfo::Ptr build(bcos::bytesConstRef data) = 0; }; inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo) diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 95f7225e..fb0f39e6 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -199,8 +199,9 @@ class Message : virtual public bcos::boostssl::MessageFace protected: MessageHeader::Ptr m_header; - std::shared_ptr m_payload; - MessagePayload::Ptr m_frontMessage; + // Note: allocate here in case of wsService nullptr access caused coredump + std::shared_ptr m_payload = std::make_shared(); + MessagePayload::Ptr m_frontMessage = nullptr; }; class MessageHeaderBuilder diff --git a/cpp/ppc-framework/protocol/PPCMessageFace.h b/cpp/ppc-framework/protocol/PPCMessageFace.h index f6de1791..a2486ad2 100644 --- a/cpp/ppc-framework/protocol/PPCMessageFace.h +++ b/cpp/ppc-framework/protocol/PPCMessageFace.h @@ -33,10 +33,6 @@ namespace ppc { namespace front { -enum MessageExtFlag : uint16_t -{ - ResponseFlag = 0x0001, -}; class PPCMessageFace { public: @@ -86,7 +82,16 @@ class PPCMessageFaceFactory public: virtual ~PPCMessageFaceFactory() {} virtual PPCMessageFace::Ptr buildPPCMessage() = 0; - virtual PPCMessageFace::Ptr buildPPCMessage(ppc::protocol::Message::Ptr msg) = 0; + virtual PPCMessageFace::Ptr decodePPCMessage(ppc::protocol::Message::Ptr msg) = 0; + virtual ppc::protocol::Message::Ptr buildMessage( + ppc::protocol::MessageBuilder::Ptr const& msgBuilder, + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) = 0; + + virtual ppc::protocol::MessagePayload::Ptr buildMessage( + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) = 0; + virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesConstRef _data) = 0; virtual PPCMessageFace::Ptr buildPPCMessage(bcos::bytesPointer _buffer) = 0; virtual PPCMessageFace::Ptr buildPPCMessage(uint8_t _taskType, uint8_t _algorithmType, diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h index 0b54380e..18e7741b 100644 --- a/cpp/test-utils/FakeFront.h +++ b/cpp/test-utils/FakeFront.h @@ -37,8 +37,9 @@ class FakeFront : public FrontInterface FakeFront() = default; ~FakeFront() override = default; - void start() override {} - void stop() override {} + void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, + std::function _handler) override + {} void registerRA2018(std::string const& _agencyID, TaskFrameworkInterface::Ptr _psi) { @@ -73,11 +74,6 @@ class FakeFront : public FrontInterface m_agencyToOTPIR[_agencyID] = _pir; } - void onReceiveMessage(front::PPCMessageFace::Ptr, ErrorCallbackFunc) override - { - throw std::runtime_error("FakeFront: unimplemented interface onReceiveMessage!"); - } - void asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace::Ptr _message, uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _responseCallback) override { diff --git a/cpp/test-utils/FakePPCMessage.h b/cpp/test-utils/FakePPCMessage.h index a9070eae..8248bc9f 100644 --- a/cpp/test-utils/FakePPCMessage.h +++ b/cpp/test-utils/FakePPCMessage.h @@ -111,5 +111,22 @@ class FakePPCMessageFactory : public PPCMessageFaceFactory { throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); } + PPCMessageFace::Ptr decodePPCMessage(ppc::protocol::Message::Ptr msg) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } + ppc::protocol::Message::Ptr buildMessage(ppc::protocol::MessageBuilder::Ptr const& msgBuilder, + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } + + ppc::protocol::MessagePayload::Ptr buildMessage( + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) override + { + throw std::runtime_error("FakePPCMessageFactory: unimplemented interface!"); + } }; } // namespace ppc::test \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp index 866bcf54..95ff82e3 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp @@ -19,8 +19,8 @@ */ #include "FakeCM2020PSIFactory.h" -#include "ppc-protocol/src/JsonTaskImpl.h" #include "ppc-psi/src/cm2020-psi/CM2020PSIImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/FakeFront.h" #include "test-utils/FileTool.h" #include "test-utils/TaskMock.h" diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h index dbeabf52..03075d6f 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h +++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/DataTools.h @@ -25,7 +25,6 @@ namespace ppc::psi { - inline void genItemsLabels( ppc::io::DataBatch::Ptr _items, ppc::io::DataBatch::Ptr _labels, uint32_t _size) { diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp index c7ed5e01..77b1a7ac 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp @@ -19,9 +19,9 @@ */ #include "FakeLabeledPSIFactory.h" -#include "ppc-protocol/src/JsonTaskImpl.h" #include "ppc-psi/src/labeled-psi/LabeledPSIImpl.h" #include "ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/FileTool.h" #include "test-utils/TaskMock.h" #include diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp index d6c50960..2bf6c8ea 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp @@ -22,9 +22,9 @@ #include "ppc-crypto-core/src/hash/Sha256Hash.h" #include "ppc-crypto/src/ecc/Ed25519EccCrypto.h" #include "ppc-crypto/src/oprf/EcdhOprf.h" -#include "ppc-protocol/src/JsonTaskImpl.h" #include "ppc-psi/src/labeled-psi/core/LabeledPSIParams.h" #include "ppc-psi/src/labeled-psi/core/SenderDB.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/TaskMock.h" #include #include diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp index 862d57f9..766aa7df 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp @@ -20,7 +20,7 @@ #include "mock/Common.h" #include "mock/EcdhPSIFixture.h" // Note: it's better not to depends on the task-impl -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/TaskMock.h" #include #include @@ -61,8 +61,8 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS auto clientPSI = factory->createEcdhPSI(clientAgencyName, clientConfig); std::vector agencyList = {serverAgencyName, clientAgencyName}; - serverPSI->psiConfig()->updateAgenyList(agencyList); - clientPSI->psiConfig()->updateAgenyList(agencyList); + // serverPSI->psiConfig()->updateAgenyList(agencyList); + // clientPSI->psiConfig()->updateAgenyList(agencyList); // register the server-psi into the front factory->front()->registerEcdhPSI(serverAgencyName, serverPSI); diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp index 30ba8ff2..f25504ae 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp @@ -20,7 +20,7 @@ #include "mock/Common.h" #include "mock/RA2018PSIFixture.h" // Note: it's better not to depends on the task-impl -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/TaskMock.h" #include #include diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h index 9786e518..ed8e3b8f 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h @@ -18,7 +18,7 @@ * @date 2022-12-29 */ #pragma once -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/TaskMock.h" #include diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h index 6b155285..2bbbd065 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h @@ -27,6 +27,8 @@ #include "test-utils/FakeFront.h" #include "test-utils/FakePPCMessage.h" #include +#include + using namespace bcos; using namespace ppc::protocol; diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index 127675fe..c086ab3e 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -106,14 +106,13 @@ void PPCConfig::loadEndpointConfig(EndPoint& endPoint, bool requireHostIp, endPoint.setPort(listenPort); } -void PPCConfig::loadFrontConfig( +void PPCConfig::loadFrontConfig(bool requireTransport, FrontConfigBuilder::Ptr const& frontConfigBuilder, boost::property_tree::ptree const& pt) { if (m_frontConfig == nullptr) { m_frontConfig = frontConfigBuilder->build(); } - loadEndpointConfig(m_frontConfig->mutableSelfEndPoint(), true, "transport", pt); // the thread_count auto threadCount = pt.get("transport.thread_count", 4); if (threadCount == 0) @@ -128,8 +127,14 @@ void PPCConfig::loadFrontConfig( InvalidConfig() << errinfo_comment("Must specify the transport.nodeid!")); } m_frontConfig->setNodeID(nodeID); - m_frontConfig->setThreadPoolSize(threadCount); + + if (!requireTransport) + { + return; + } + + loadEndpointConfig(m_frontConfig->mutableSelfEndPoint(), true, "transport", pt); // the gateway targets auto gatewayTargets = pt.get("transport.service.gateway_target", ""); if (gatewayTargets.empty()) diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h index 56b5249f..df42f7c3 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h @@ -125,7 +125,8 @@ class PPCConfig PPCConfig() = default; virtual ~PPCConfig() = default; // load the nodeConfig - void loadNodeConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + void loadNodeConfig(bool requireTransport, + ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, std::string const& _configPath) { PPCConfig_LOG(INFO) << LOG_DESC("loadNodeConfig") << LOG_KV("path", _configPath); @@ -133,7 +134,7 @@ class PPCConfig boost::property_tree::read_ini(_configPath, iniConfig); // Note: must load common-config firstly since some ra-configs depends on the common-config loadCommonNodeConfig(iniConfig); - loadFrontConfig(frontConfigBuilder, _configPath); + loadFrontConfig(requireTransport, frontConfigBuilder, _configPath); loadRA2018Config(iniConfig); loadStorageConfig(iniConfig); loadEcdhPSIConfig(iniConfig); @@ -158,13 +159,14 @@ class PPCConfig loadGatewayConfig(iniConfig); } - void loadFrontConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + void loadFrontConfig(bool requireTransport, + ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, std::string const& _configPath) { PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig") << LOG_KV("path", _configPath); boost::property_tree::ptree iniConfig; boost::property_tree::read_ini(_configPath, iniConfig); - loadFrontConfig(frontConfigBuilder, iniConfig); + loadFrontConfig(requireTransport, frontConfigBuilder, iniConfig); // load the grpcConfig m_grpcConfig = loadGrpcConfig("transport", iniConfig); m_frontConfig->setGrpcConfig(m_grpcConfig); @@ -235,11 +237,15 @@ class PPCConfig // used by cem module virtual void loadCEMConfig(boost::property_tree::ptree const& _pt); + // for ut + void setAgencyID(std::string const& agencyID) { m_agencyID = agencyID; } + private: virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp, std::string const& sectionName, boost::property_tree::ptree const& pt); // load the front config - virtual void loadFrontConfig(ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, + virtual void loadFrontConfig(bool requireTransport, + ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder, boost::property_tree::ptree const& pt); // load the grpc config ppc::protocol::GrpcConfig::Ptr loadGrpcConfig( diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h index 13739719..7600e09d 100644 --- a/cpp/wedpr-helper/ppc-utilities/Utilities.h +++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h @@ -27,17 +27,22 @@ namespace ppc { +template inline uint64_t decodeNetworkBuffer( - bcos::bytes& _result, bcos::byte const* buffer, unsigned int bufferLen, uint64_t const offset) + T& _result, bcos::byte const* buffer, unsigned int bufferLen, uint64_t const offset) { uint64_t curOffset = offset; CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); + // Notice: operator* is higher priority than operator+, the () is essential auto dataLen = - boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)buffer + curOffset)); + boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)(buffer + curOffset))); curOffset += 2; + if (dataLen == 0) + { + return curOffset; + } CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); - _result.insert( - _result.end(), (bcos::byte*)buffer + curOffset, (bcos::byte*)buffer + curOffset + dataLen); + _result.assign((bcos::byte*)buffer + curOffset, (bcos::byte*)buffer + curOffset + dataLen); curOffset += dataLen; return curOffset; } diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index ad169481..b5848406 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -51,17 +51,25 @@ using namespace ppc::tools; using namespace ppc::crypto; using namespace ppc::sdk; -Initializer::Initializer(std::string const& _configPath) : m_configPath(_configPath) +Initializer::Initializer(ppc::protocol::NodeArch _arch, std::string const& _configPath) + : m_arch(_arch), m_configPath(_configPath) { m_transportBuilder = std::make_shared(); // load the config m_config = std::make_shared(); - m_config->loadNodeConfig(m_transportBuilder->frontConfigBuilder(), _configPath); + if (m_arch == ppc::protocol::NodeArch::PRO) + { + m_config->loadNodeConfig(true, m_transportBuilder->frontConfigBuilder(), _configPath); + } + else + { + m_config->loadNodeConfig(false, m_transportBuilder->frontConfigBuilder(), _configPath); + } } -void Initializer::init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Ptr const& gateway) +void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway) { - INIT_LOG(INFO) << LOG_DESC("init the wedpr-component") << LOG_KV("arch", _arch); + INIT_LOG(INFO) << LOG_DESC("init the wedpr-component") << LOG_KV("arch", m_arch); // load the protocol m_protocolInitializer = std::make_shared(); m_protocolInitializer->init(m_config); @@ -73,7 +81,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Pt // Note: must set the m_holdingMessageMinutes before init the node TransportBuilder transportBuilder; - if (_arch == ppc::protocol::NodeArch::AIR) + if (m_arch == ppc::protocol::NodeArch::AIR) { m_transport = transportBuilder.build(SDKMode::AIR, m_config->frontConfig(), gateway); } @@ -85,7 +93,7 @@ void Initializer::init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Pt INIT_LOG(INFO) << LOG_DESC("init the frontService success") << LOG_KV("frontDetail", printFrontDesc(m_config->frontConfig())) - << LOG_KV("arch", _arch); + << LOG_KV("arch", m_arch); auto cryptoBox = m_protocolInitializer->cryptoBox(); SQLStorage::Ptr sqlStorage = nullptr; diff --git a/cpp/wedpr-initializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h index d1ad5d01..90e8ae69 100644 --- a/cpp/wedpr-initializer/Initializer.h +++ b/cpp/wedpr-initializer/Initializer.h @@ -59,11 +59,11 @@ class Initializer : public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - Initializer(std::string const& _configPath); + Initializer(ppc::protocol::NodeArch _arch, std::string const& _configPath); virtual ~Initializer() { stop(); } // init the service - virtual void init(ppc::protocol::NodeArch _arch, ppc::gateway::IGateway::Ptr const& gateway); + virtual void init(ppc::gateway::IGateway::Ptr const& gateway); virtual void stop(); virtual void start(); @@ -82,6 +82,7 @@ class Initializer : public std::enable_shared_from_this private: + ppc::protocol::NodeArch m_arch; std::string m_configPath; std::shared_ptr m_config; ProtocolInitializer::Ptr m_protocolInitializer; diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp index a05788e3..ffcf605a 100644 --- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp +++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp @@ -47,12 +47,12 @@ void AirNodeInitializer::init(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("initLog success"); // init the node - m_nodeInitializer = std::make_shared(_configPath); + m_nodeInitializer = std::make_shared(ppc::protocol::NodeArch::AIR, _configPath); // init the gateway initGateway(_configPath); // init the node - m_nodeInitializer->init(ppc::protocol::NodeArch::AIR, m_gateway); + m_nodeInitializer->init(m_gateway); // set the created front to the builder m_frontBuilder->setFront(m_nodeInitializer->transport()->getFront()); // register the NodeInfo diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.cpp b/cpp/wedpr-main/gateway/GatewayInitializer.cpp index b2657958..0173bfb9 100644 --- a/cpp/wedpr-main/gateway/GatewayInitializer.cpp +++ b/cpp/wedpr-main/gateway/GatewayInitializer.cpp @@ -22,7 +22,7 @@ #include "grpc/server/GrpcServer.h" #include "ppc-gateway/GatewayFactory.h" #include "ppc-tools/src/config/PPCConfig.h" -#include "protobuf/NodeInfoImpl.h" +#include "protobuf/src/NodeInfoImpl.h" #include "wedpr-protocol/grpc/client/RemoteFrontBuilder.h" #include "wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h" diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp index 3e31a4d0..2ced6616 100644 --- a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp +++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp @@ -43,10 +43,10 @@ void ProNodeInitializer::init(std::string const& _configPath) INIT_LOG(INFO) << LOG_DESC("initLog success"); // init the node - m_nodeInitializer = std::make_shared(_configPath); + m_nodeInitializer = std::make_shared(ppc::protocol::NodeArch::PRO, _configPath); // init the node(no need to set the gateway) - m_nodeInitializer->init(ppc::protocol::NodeArch::PRO, nullptr); + m_nodeInitializer->init(nullptr); INIT_LOG(INFO) << LOG_DESC("init the rpc"); diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp index 7fe01a6f..a2037613 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp @@ -18,8 +18,8 @@ * @date 2024-09-02 */ #include "FrontClient.h" -#include "protobuf/RequestConverter.h" -#include "wedpr-protocol/protobuf/Common.h" +#include "protobuf/src/RequestConverter.h" +#include "wedpr-protocol/protobuf/src/Common.h" using namespace ppc::protocol; using namespace ppc::proto; diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index eef8c9be..ad7c7276 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -20,7 +20,7 @@ #include "GatewayClient.h" #include "Common.h" #include "Service.grpc.pb.h" -#include "protobuf/RequestConverter.h" +#include "protobuf/src/RequestConverter.h" using namespace ppc; using namespace ppc::proto; @@ -29,10 +29,10 @@ using namespace ppc::gateway; using namespace ppc::protocol; void GatewayClient::asyncSendMessage(RouteType routeType, - MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout, - ReceiveMsgFunc callback) + MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload, + long timeout, ReceiveMsgFunc callback) { - auto request = generateRequest(routeType, routeInfo, std::move(payload), timeout); + auto request = generateRequest(traceID, routeType, routeInfo, std::move(payload), timeout); ClientContext context; auto response = std::make_shared(); m_stub->async()->asyncSendMessage(&context, request.get(), response.get(), diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index ea964acd..69513248 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -52,11 +52,12 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient * @param callback callback */ void asyncSendMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, - long timeout, ppc::protocol::ReceiveMsgFunc callback) override; + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, + bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) override; void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, + bcos::bytes&& payload) override {} bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override; bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) override; diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp index ee040e6f..8b1182e5 100644 --- a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp @@ -19,7 +19,7 @@ */ #include "FrontServer.h" #include "Common.h" -#include "protobuf/RequestConverter.h" +#include "protobuf/src/RequestConverter.h" #include using namespace ppc::proto; diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp index 91f690c4..3b5caf69 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp @@ -19,7 +19,7 @@ */ #include "GatewayServer.h" #include "Common.h" -#include "protobuf/RequestConverter.h" +#include "protobuf/src/RequestConverter.h" using namespace ppc::protocol; using namespace grpc; @@ -33,7 +33,8 @@ ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* conte bcos::bytes payloadData(sendedMsg->payload().begin(), sendedMsg->payload().end()); auto routeInfo = generateRouteInfo(m_routeInfoBuilder, sendedMsg->routeinfo()); m_gateway->asyncSendMessage((ppc::protocol::RouteType)sendedMsg->routetype(), routeInfo, - std::move(payloadData), sendedMsg->timeout(), [reactor, reply](bcos::Error::Ptr error) { + sendedMsg->traceid(), std::move(payloadData), sendedMsg->timeout(), + [reactor, reply](bcos::Error::Ptr error) { toSerializedError(reply, error); reactor->Finish(Status::OK); }); diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto index eb241af8..e46d91b0 100644 --- a/cpp/wedpr-protocol/proto/pb/Service.proto +++ b/cpp/wedpr-protocol/proto/pb/Service.proto @@ -28,6 +28,7 @@ message SendedMessageRequest{ RouteInfo routeInfo = 2; bytes payload = 3; int64 timeout = 4; + string traceID = 5; }; service Front { rpc onReceiveMessage (ReceivedMessage) returns (Error) {} diff --git a/cpp/wedpr-protocol/protobuf/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/CMakeLists.txt index 6dd7bc81..615270ec 100644 --- a/cpp/wedpr-protocol/protobuf/CMakeLists.txt +++ b/cpp/wedpr-protocol/protobuf/CMakeLists.txt @@ -1,27 +1,8 @@ -# proto generation -set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/pb) - -file(GLOB_RECURSE MESSAGES_PROTOS "${PROTO_INPUT_PATH}/*.proto") - -find_program(PROTOC_BINARY protoc REQUIRED) - -# create PROTO_OUTPUT_PATH -file(MAKE_DIRECTORY ${PROTO_OUTPUT_PATH}) -foreach(proto_file ${MESSAGES_PROTOS}) - get_filename_component(basename ${proto_file} NAME_WE) - set(generated_file ${PROTO_OUTPUT_PATH}/${basename}.pb.cc) - - list(APPEND MESSAGES_SRCS ${generated_file}) - - message("Command: protoc --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file}") - add_custom_command( - OUTPUT ${generated_file} - COMMAND ${PROTOC_BINARY} --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file} - COMMENT "Generating ${generated_file} from ${proto_file}" - VERBATIM - ) -endforeach() - -file(GLOB_RECURSE SRCS *.cpp) -add_library(${PB_PROTOCOL_TARGET} ${SRCS} ${MESSAGES_SRCS}) -target_link_libraries(${PB_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET}) \ No newline at end of file +project(ppc-protocol VERSION ${VERSION}) + +add_subdirectory(src) +if (TESTS) + enable_testing() + set(CTEST_OUTPUT_ON_FAILURE TRUE) + add_subdirectory(tests) +endif () diff --git a/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt new file mode 100644 index 00000000..7aaa0e0d --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt @@ -0,0 +1,27 @@ +# proto generation +set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/pb) + +file(GLOB_RECURSE MESSAGES_PROTOS "${PROTO_INPUT_PATH}/*.proto") + +find_program(PROTOC_BINARY protoc REQUIRED) + +# create PROTO_OUTPUT_PATH +file(MAKE_DIRECTORY ${PROTO_OUTPUT_PATH}) +foreach(proto_file ${MESSAGES_PROTOS}) + get_filename_component(basename ${proto_file} NAME_WE) + set(generated_file ${PROTO_OUTPUT_PATH}/${basename}.pb.cc) + + list(APPEND MESSAGES_SRCS ${generated_file}) + + message("Command: protoc --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file}") + add_custom_command( + OUTPUT ${generated_file} + COMMAND ${PROTOC_BINARY} --cpp_out ${PROTO_OUTPUT_PATH} -I ${PROTO_INPUT_PATH} ${proto_file} + COMMENT "Generating ${generated_file} from ${proto_file}" + VERBATIM + ) +endforeach() + +file(GLOB_RECURSE SRCS *.cpp) +add_library(${PB_PROTOCOL_TARGET} ${SRCS} ${MESSAGES_SRCS}) +target_link_libraries(${PB_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} protobuf::libprotobuf ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/Common.h b/cpp/wedpr-protocol/protobuf/src/Common.h similarity index 100% rename from cpp/wedpr-protocol/protobuf/Common.h rename to cpp/wedpr-protocol/protobuf/src/Common.h diff --git a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp similarity index 79% rename from cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp rename to cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp index 742d53a8..93f0f66a 100644 --- a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.cpp +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp @@ -28,13 +28,13 @@ void NodeInfoImpl::encode(bcos::bytes& data) const // set the components for (auto const& component : m_components) { - m_inner()->add_components(component); + m_rawNodeInfo->add_components(component); } - encodePBObject(data, m_inner()); + encodePBObject(data, m_rawNodeInfo); } void NodeInfoImpl::decode(bcos::bytesConstRef data) { - decodePBObject(m_inner(), data); - m_components = - std::set(m_inner()->components().begin(), m_inner()->components().end()); + decodePBObject(m_rawNodeInfo, data); + m_components = std::set( + m_rawNodeInfo->components().begin(), m_rawNodeInfo->components().end()); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h similarity index 67% rename from cpp/wedpr-protocol/protobuf/NodeInfoImpl.h rename to cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h index a81ad5c0..ed8f6a68 100644 --- a/cpp/wedpr-protocol/protobuf/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h @@ -29,27 +29,28 @@ class NodeInfoImpl : public INodeInfo { public: using Ptr = std::shared_ptr; - explicit NodeInfoImpl(std::function inner) : m_inner(std::move(inner)) + NodeInfoImpl() { m_rawNodeInfo = std::make_shared(); } + explicit NodeInfoImpl(std::shared_ptr rawNodeInfo) + : m_rawNodeInfo(rawNodeInfo) {} - NodeInfoImpl() : m_inner([inner = ppc::proto::NodeInfo()]() mutable { return &inner; }) {} + NodeInfoImpl(bcos::bytesConstRef const& data) : NodeInfoImpl() { decode(data); } - NodeInfoImpl(bcos::bytesConstRef const& nodeID) : NodeInfoImpl() + NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) : NodeInfoImpl() { - m_inner()->set_nodeid(nodeID.data(), nodeID.size()); + m_rawNodeInfo->set_nodeid(nodeID.data(), nodeID.size()); + m_rawNodeInfo->set_endpoint(endPoint); } - NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) - : NodeInfoImpl(nodeID) - { - m_inner()->set_endpoint(endPoint); - } - ~NodeInfoImpl() override = default; + ~NodeInfoImpl() override {} void setNodeID(bcos::bytesConstRef nodeID) override { - m_inner()->set_nodeid(nodeID.data(), nodeID.size()); + m_rawNodeInfo->set_nodeid(nodeID.data(), nodeID.size()); + } + void setEndPoint(std::string const& endPoint) override + { + m_rawNodeInfo->set_endpoint(endPoint); } - void setEndPoint(std::string const& endPoint) override { m_inner()->set_endpoint(endPoint); } void setComponents(std::set const& components) override { @@ -57,17 +58,17 @@ class NodeInfoImpl : public INodeInfo } std::set const& components() const override { return m_components; } - std::string const& endPoint() const override { return m_inner()->endpoint(); } + std::string const& endPoint() const override { return m_rawNodeInfo->endpoint(); } bcos::bytesConstRef nodeID() const override { - return {reinterpret_cast(m_inner()->nodeid().data()), - m_inner()->nodeid().size()}; + return {reinterpret_cast(m_rawNodeInfo->nodeid().data()), + m_rawNodeInfo->nodeid().size()}; } void encode(bcos::bytes& data) const override; void decode(bcos::bytesConstRef data) override; - std::function innerFunc() { return m_inner; } + std::shared_ptr rawNodeInfo() { return m_rawNodeInfo; } void setFront(std::shared_ptr&& front) override { @@ -78,7 +79,7 @@ class NodeInfoImpl : public INodeInfo private: std::shared_ptr m_front; std::set m_components; - std::function m_inner; + std::shared_ptr m_rawNodeInfo; }; class NodeInfoFactory : public INodeInfoFactory @@ -90,7 +91,10 @@ class NodeInfoFactory : public INodeInfoFactory INodeInfo::Ptr build() override { return std::make_shared(); } - + INodeInfo::Ptr build(bcos::bytesConstRef data) override + { + return std::make_shared(data); + } INodeInfo::Ptr build(bcos::bytesConstRef nodeID, std::string const& endPoint) override { return std::make_shared(nodeID, endPoint); diff --git a/cpp/wedpr-protocol/protobuf/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h similarity index 98% rename from cpp/wedpr-protocol/protobuf/RequestConverter.h rename to cpp/wedpr-protocol/protobuf/src/RequestConverter.h index faefc75d..4f1544bb 100644 --- a/cpp/wedpr-protocol/protobuf/RequestConverter.h +++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h @@ -42,10 +42,12 @@ inline MessageOptionalHeader::Ptr generateRouteInfo( return routeInfo; } -inline std::shared_ptr generateRequest(RouteType routeType, +inline std::shared_ptr generateRequest(std::string const& traceID, + RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout) { auto request = std::make_shared(); + request->set_traceid(traceID); request->set_routetype(uint16_t(routeType)); // set the route information request->mutable_routeinfo()->set_topic(routeInfo->topic()); diff --git a/cpp/wedpr-protocol/protobuf/tests/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/tests/CMakeLists.txt new file mode 100644 index 00000000..c7310f2f --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/tests/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE SOURCES "*.cpp" "*.h") + +# cmake settings +set(TEST_BINARY_NAME test-protobuf) + +add_executable(${TEST_BINARY_NAME} ${SOURCES}) +target_include_directories(${TEST_BINARY_NAME} PRIVATE .) + +target_link_libraries(${TEST_BINARY_NAME} ${PB_PROTOCOL_TARGET} TBB::tbb ${BOOST_UNIT_TEST}) +add_test(NAME test-protobuf WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp b/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp new file mode 100644 index 00000000..a234a8e1 --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file NodeInfoImplTest.cpp + * @author: yujiechen + * @date 2024-09-06 + */ + +#include "protobuf/src/NodeInfoImpl.h" +#include +#include + +using namespace ppc; +using namespace ppc::protocol; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(NodeInfoTest, TestPromptFixture) + +INodeInfo::Ptr fakeNodeInfo(INodeInfoFactory::Ptr factory, std::string const& nodeID, + std::string const& endPoint, std::set const& components) +{ + auto nodeInfo = factory->build(); + nodeInfo->setNodeID(bcos::bytesConstRef((bcos::byte*)nodeID.data(), nodeID.size())); + nodeInfo->setEndPoint(endPoint); + nodeInfo->setComponents(components); + return nodeInfo; +} + + +void testNodeInfoEncodeDecode(INodeInfoFactory::Ptr factory, INodeInfo::Ptr nodeInfo) +{ + bcos::bytes encodedData; + nodeInfo->encode(encodedData); + + auto decodedNodeInfo = factory->build(bcos::ref(encodedData)); + BOOST_CHECK(nodeInfo->nodeID().toBytes() == decodedNodeInfo->nodeID().toBytes()); + BOOST_CHECK(nodeInfo->endPoint() == decodedNodeInfo->endPoint()); + auto const& components = nodeInfo->components(); + for (auto const& decodedComp : decodedNodeInfo->components()) + { + BOOST_CHECK(components.count(decodedComp)); + } +} + +BOOST_AUTO_TEST_CASE(testNodeInfo) +{ + auto nodeInfoFactory = std::make_shared(); + std::string nodeID = "testn+NodeID"; + std::string endPoint = "testEndpoint"; + std::set components; + for (int i = 0; i < 100; i++) + { + components.insert("component_" + std::to_string(i)); + } + auto nodeInfo = fakeNodeInfo(nodeInfoFactory, nodeID, endPoint, components); + testNodeInfoEncodeDecode(nodeInfoFactory, nodeInfo); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/tests/main.cpp b/cpp/wedpr-protocol/protobuf/tests/main.cpp new file mode 100644 index 00000000..2c06f9b5 --- /dev/null +++ b/cpp/wedpr-protocol/protobuf/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MAIN +#include \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/CMakeLists.txt b/cpp/wedpr-protocol/protocol/CMakeLists.txt index 615270ec..cda79d6d 100644 --- a/cpp/wedpr-protocol/protocol/CMakeLists.txt +++ b/cpp/wedpr-protocol/protocol/CMakeLists.txt @@ -1,4 +1,4 @@ -project(ppc-protocol VERSION ${VERSION}) +project(ppc-protobuf VERSION ${VERSION}) add_subdirectory(src) if (TESTS) diff --git a/cpp/wedpr-protocol/protocol/src/Common.h b/cpp/wedpr-protocol/protocol/src/Common.h new file mode 100644 index 00000000..9875b303 --- /dev/null +++ b/cpp/wedpr-protocol/protocol/src/Common.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2024-9-6 + */ + +#pragma once + +#include "ppc-framework/Common.h" +#include + +namespace ppc::gateway +{ +#define PROTOCOL_LOG(LEVEL) BCOS_LOG(LEVEL) << "[PROTOCOL]" +} // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp index 6c2a0fa0..a316a773 100644 --- a/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp @@ -24,6 +24,7 @@ using namespace bcos; using namespace ppc::front; +using namespace ppc::protocol; void PPCMessage::encode(bytes& _buffer) { @@ -137,4 +138,61 @@ std::map PPCMessage::decodeMap(const std::string& _enc } return maps; +} + +// Note: this interface is used after the MessagePayload(frontMessage) has been decoded; this +// interface passed some meta information to the ppcMessage +PPCMessageFace::Ptr PPCMessageFactory::decodePPCMessage(Message::Ptr msg) +{ + auto ppcMsg = buildPPCMessage(); + auto frontMsg = msg->frontMessage(); + // Note: this field is been setted when onReceiveMessage + if (frontMsg) + { + ppcMsg->setSeq(frontMsg->seq()); + ppcMsg->setUuid(frontMsg->traceID()); + if (frontMsg->isRespPacket()) + { + ppcMsg->setResponse(); + } + ppcMsg->decode(bcos::ref(frontMsg->data())); + } + if (msg->header() && msg->header()->optionalField()) + { + auto const& routeInfo = msg->header()->optionalField(); + ppcMsg->setTaskID(routeInfo->topic()); + ppcMsg->setSender(routeInfo->srcInst()); + } + return ppcMsg; +} + +Message::Ptr PPCMessageFactory::buildMessage(MessageBuilder::Ptr const& msgBuilder, + MessagePayloadBuilder::Ptr const& msgPayloadBuilder, PPCMessageFace::Ptr const& ppcMessage) +{ + auto msg = msgBuilder->build(); + msg->header()->optionalField()->setTopic(ppcMessage->taskID()); + msg->header()->optionalField()->setSrcInst(ppcMessage->sender()); + + auto payload = buildMessage(msgPayloadBuilder, ppcMessage); + auto payloadData = std::make_shared(); + payload->encode(*payloadData); + msg->setPayload(std::move(payloadData)); + return msg; +} + +MessagePayload::Ptr PPCMessageFactory::buildMessage( + MessagePayloadBuilder::Ptr const& msgPayloadBuilder, PPCMessageFace::Ptr const& ppcMessage) +{ + auto payload = msgPayloadBuilder->build(); + payload->setSeq(ppcMessage->seq()); + if (ppcMessage->response()) + { + payload->setRespPacket(); + } + payload->setTraceID(ppcMessage->uuid()); + + bcos::bytes ppcMsgData; + ppcMessage->encode(ppcMsgData); + payload->setData(std::move(ppcMsgData)); + return payload; } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.h b/cpp/wedpr-protocol/protocol/src/PPCMessage.h index fc6983f6..148e4fd3 100644 --- a/cpp/wedpr-protocol/protocol/src/PPCMessage.h +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.h @@ -148,27 +148,15 @@ class PPCMessageFactory : public PPCMessageFaceFactory return msg; } - PPCMessageFace::Ptr buildPPCMessage(ppc::protocol::Message::Ptr msg) override - { - auto ppcMsg = buildPPCMessage(); - auto frontMsg = msg->frontMessage(); - if (frontMsg) - { - ppcMsg->setSeq(frontMsg->seq()); - ppcMsg->setUuid(frontMsg->traceID()); - if (frontMsg->isRespPacket()) - { - ppcMsg->setResponse(); - } - } - if (msg->header() && msg->header()->optionalField()) - { - auto const& routeInfo = msg->header()->optionalField(); - ppcMsg->setTaskID(routeInfo->topic()); - ppcMsg->setSender(routeInfo->srcInst()); - } - return ppcMsg; - } + PPCMessageFace::Ptr decodePPCMessage(ppc::protocol::Message::Ptr msg) override; + + ppc::protocol::Message::Ptr buildMessage(ppc::protocol::MessageBuilder::Ptr const& msgBuilder, + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) override; + + ppc::protocol::MessagePayload::Ptr buildMessage( + ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder, + PPCMessageFace::Ptr const& ppcMessage) override; }; } // namespace front diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp index 117f304c..cc03557d 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp @@ -50,6 +50,10 @@ void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const uint16_t dstInstLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstInst.size()); buffer.insert(buffer.end(), (byte*)&dstInstLen, (byte*)&dstInstLen + 2); buffer.insert(buffer.end(), m_dstInst.begin(), m_dstInst.end()); + // the topic + uint16_t topicLen = boost::asio::detail::socket_ops::host_to_network_short(m_topic.size()); + buffer.insert(buffer.end(), (byte*)&topicLen, (byte*)&topicLen + 2); + buffer.insert(buffer.end(), m_topic.begin(), m_topic.end()); } @@ -66,15 +70,13 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con // srcNode offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), offset); // source inst - bcos::bytes sourceInst; - offset = decodeNetworkBuffer(sourceInst, data.data(), data.size(), offset); - m_srcInst = std::string(sourceInst.begin(), sourceInst.end()); + offset = decodeNetworkBuffer(m_srcInst, data.data(), data.size(), offset); // dstNode offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset); - // dstInst, TODO: optimize here - bcos::bytes dstInstData; - offset = decodeNetworkBuffer(dstInstData, data.data(), data.size(), offset); - m_dstInst = std::string(dstInstData.begin(), dstInstData.end()); + // dstInst + offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset); + // topic + offset = decodeNetworkBuffer(m_topic, data.data(), data.size(), offset); return offset; } @@ -96,19 +98,29 @@ void MessageHeaderImpl::encode(bcos::bytes& buffer) const // the traceID, 2+Bytes uint16_t traceIDLen = boost::asio::detail::socket_ops::host_to_network_short(m_traceID.size()); buffer.insert(buffer.end(), (byte*)&traceIDLen, (byte*)&traceIDLen + 2); - buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end()); + if (m_traceID.size() > 0) + { + buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end()); + } // srcGwNode, 2+Bytes uint16_t srcGwNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_srcGwNode.size()); buffer.insert(buffer.end(), (byte*)&srcGwNodeLen, (byte*)&srcGwNodeLen + 2); - buffer.insert(buffer.end(), m_srcGwNode.begin(), m_srcGwNode.end()); + if (m_srcGwNode.size() > 0) + { + buffer.insert(buffer.end(), m_srcGwNode.begin(), m_srcGwNode.end()); + } // dstGwNode, 2+Bytes uint16_t dstGwNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstGwNode.size()); buffer.insert(buffer.end(), (byte*)&dstGwNodeLen, (byte*)&dstGwNodeLen + 2); - buffer.insert(buffer.end(), m_dstGwNode.begin(), m_dstGwNode.end()); + if (m_dstGwNode.size() > 0) + { + buffer.insert(buffer.end(), m_dstGwNode.begin(), m_dstGwNode.end()); + } if (!hasOptionalField()) { + m_length = buffer.size(); return; } // encode the optionalField @@ -137,18 +149,11 @@ int64_t MessageHeaderImpl::decode(bcos::bytesConstRef data) m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); pointer += 2; // the traceID - bcos::bytes traceIDData; - auto offset = - decodeNetworkBuffer(traceIDData, data.data(), data.size(), (pointer - data.data())); - m_traceID = std::string(traceIDData.begin(), traceIDData.end()); + auto offset = decodeNetworkBuffer(m_traceID, data.data(), data.size(), (pointer - data.data())); // srcGwNode - bcos::bytes srcGWNodeData; - offset = decodeNetworkBuffer(srcGWNodeData, data.data(), data.size(), offset); - m_srcGwNode = std::string(srcGWNodeData.begin(), srcGWNodeData.end()); + offset = decodeNetworkBuffer(m_srcGwNode, data.data(), data.size(), offset); // dstGwNode - bcos::bytes dstGWNodeData; - offset = decodeNetworkBuffer(dstGWNodeData, data.data(), data.size(), offset); - m_dstGwNode = std::string(dstGWNodeData.begin(), dstGWNodeData.end()); + offset = decodeNetworkBuffer(m_dstGwNode, data.data(), data.size(), offset); // optionalField if (hasOptionalField()) { diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h index b48caec7..0212c20b 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h @@ -56,7 +56,11 @@ class MessageHeaderImpl : public MessageHeader public: using Ptr = std::shared_ptr; MessageHeaderImpl() { m_optionalField = std::make_shared(); } - MessageHeaderImpl(bcos::bytesConstRef data) { decode(data); } + MessageHeaderImpl(bcos::bytesConstRef data) + { + m_optionalField = std::make_shared(); + decode(data); + } ~MessageHeaderImpl() override {} void encode(bcos::bytes& buffer) const override; diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp index 09f4183a..361029e6 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp @@ -19,6 +19,7 @@ */ #include "MessageImpl.h" +#include "../Common.h" using namespace bcos; using namespace ppc::protocol; @@ -26,21 +27,30 @@ using namespace ppc::protocol; bool MessageImpl::encode(bcos::bytes& _buffer) { // encode the header - bcos::bytes headerData; - m_header->encode(headerData); + m_header->encode(_buffer); // encode the payload - if (m_payload) + if (m_payload && m_payload->size() > 0) { - headerData.insert(headerData.end(), m_payload->begin(), m_payload->end()); + _buffer.insert(_buffer.end(), m_payload->begin(), m_payload->end()); } } bool MessageImpl::encode(bcos::boostssl::EncodedMsg& encodedMsg) { - // header - m_header->encode(encodedMsg.header); - // assign the payload back - encodedMsg.payload = m_payload; + try + { + // header + m_header->encode(encodedMsg.header); + // assign the payload back + encodedMsg.payload = m_payload; + return true; + } + catch (std::exception const& e) + { + PROTOCOL_LOG(WARNING) << LOG_DESC("encode message failed") + << LOG_KV("error", boost::diagnostic_information(e)); + return false; + } } int64_t MessageImpl::decode(bytesConstRef buffer) @@ -53,6 +63,11 @@ int64_t MessageImpl::decode(bytesConstRef buffer) } // decode the header m_header = m_headerBuilder->build(buffer); + // no payload case + if (buffer.size() <= m_header->length()) + { + return buffer.size(); + } // decode the payload if (!m_payload) { @@ -60,4 +75,5 @@ int64_t MessageImpl::decode(bytesConstRef buffer) } m_payload->clear(); m_payload->insert(m_payload->end(), buffer.data() + m_header->length(), buffer.end()); + return buffer.size(); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h index e261a7a2..8fb053fe 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h @@ -33,7 +33,9 @@ class MessageImpl : public Message using Ptr = std::shared_ptr; MessageImpl(MessageHeaderBuilder::Ptr headerBuilder, size_t maxMessageLen) : m_headerBuilder(std::move(headerBuilder)), m_maxMessageLen(maxMessageLen) - {} + { + m_header = m_headerBuilder->build(); + } MessageImpl( MessageHeaderBuilder::Ptr headerBuilder, size_t maxMessageLen, bcos::bytesConstRef buffer) : MessageImpl(headerBuilder, maxMessageLen) diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp index 7f1f676a..7e2f37b8 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp @@ -71,10 +71,8 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); pointer += 2; // the traceID - bcos::bytes traceID; auto offset = - decodeNetworkBuffer(traceID, buffer.data(), buffer.size(), (pointer - buffer.data())); - m_traceID = std::string(traceID.begin(), traceID.end()); + decodeNetworkBuffer(m_traceID, buffer.data(), buffer.size(), (pointer - buffer.data())); // data return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp new file mode 100644 index 00000000..53977bbe --- /dev/null +++ b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file MessageTest.cpp + * @author: shawnhe + * @date 2022-10-28 + */ + +#include "protocol/src/v1/MessageHeaderImpl.h" +#include "protocol/src/v1/MessageImpl.h" +#include +#include + +using namespace ppc; +using namespace ppc::protocol; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(MessageTest, TestPromptFixture) + +void checkMsg(Message::Ptr msg, Message::Ptr decodedMsg) +{ + if (msg->header()) + { + BOOST_CHECK(msg->header()->version() == decodedMsg->header()->version()); + BOOST_CHECK(msg->header()->traceID() == decodedMsg->header()->traceID()); + BOOST_CHECK(msg->header()->srcGwNode() == decodedMsg->header()->srcGwNode()); + BOOST_CHECK(msg->header()->dstGwNode() == decodedMsg->header()->dstGwNode()); + BOOST_CHECK(msg->header()->packetType() == decodedMsg->header()->packetType()); + BOOST_CHECK(msg->header()->ttl() == decodedMsg->header()->ttl()); + BOOST_CHECK(msg->header()->ext() == decodedMsg->header()->ext()); + } + auto routeInfo = msg->header()->optionalField(); + auto decodedRouteInfo = decodedMsg->header()->optionalField(); + if (routeInfo) + { + BOOST_CHECK(routeInfo->topic() == decodedRouteInfo->topic()); + BOOST_CHECK(routeInfo->componentType() == decodedRouteInfo->componentType()); + BOOST_CHECK(routeInfo->srcNode() == decodedRouteInfo->srcNode()); + BOOST_CHECK(routeInfo->srcInst() == decodedRouteInfo->srcInst()); + BOOST_CHECK(routeInfo->dstNode() == decodedRouteInfo->dstNode()); + BOOST_CHECK(routeInfo->dstInst() == decodedRouteInfo->dstInst()); + } + if (msg->payload()) + { + BOOST_CHECK(*(msg->payload()) == *(decodedMsg->payload())); + } +} + +Message::Ptr fakeMsg(MessageBuilder::Ptr msgBuilder, int version, std::string const& traceID, + std::string const& srcGwNode, std::string const& dstGwNode, int packetType, int ttl, int ext, + std::string topic, std::string const& componentType, bcos::bytes const& srcNode, + std::string const& srcInst, bcos::bytes const& dstNode, std::string const& dstInst, + std::shared_ptr payload) +{ + auto msg = msgBuilder->build(); + msg->header()->setVersion(version); + msg->header()->setTraceID(traceID); + msg->header()->setSrcGwNode(srcGwNode); + msg->header()->setDstGwNode(dstGwNode); + msg->header()->setPacketType(packetType); + msg->header()->setTTL(ttl); + msg->header()->setExt(ext); + msg->header()->optionalField()->setTopic(topic); + msg->header()->optionalField()->setComponentType(componentType); + msg->header()->optionalField()->setSrcNode(srcNode); + msg->header()->optionalField()->setSrcInst(srcInst); + msg->header()->optionalField()->setDstNode(dstNode); + msg->header()->optionalField()->setDstInst(dstInst); + msg->setPayload(payload); + return msg; +} + +void checkEncodeDecode(MessageBuilder::Ptr msgBuilder, Message::Ptr const& msg) +{ + // encode + bcos::bytes encodedData; + msg->encode(encodedData); + + // decode + auto decodedMsg = msgBuilder->build(bcos::ref(encodedData)); + checkMsg(msg, decodedMsg); +} + +BOOST_AUTO_TEST_CASE(testMessage) +{ + auto msgBuilder = + std::make_shared(std::make_shared()); + int version = 1000; + int packetType = 2344; + int ttl = 123; + int ext = 1000; + std::string traceID = ""; + std::string srcGwNode = ""; + std::string dstGwNode = ""; + std::string topic = ""; + std::string componentType = ""; + bcos::bytes srcNode; + bcos::bytes dstNode; + std::string srcInst; + std::string dstInst; + std::shared_ptr payload; + + auto msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, + topic, componentType, srcNode, srcInst, dstNode, dstInst, payload); + checkEncodeDecode(msgBuilder, msg); + // with payload + payload = std::make_shared(); + std::string payloadData = "payloadf@#$@#$sdfs234"; + *payload = bcos::bytes(payloadData.begin(), payloadData.end()); + msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, + componentType, srcNode, srcInst, dstNode, dstInst, payload); + checkEncodeDecode(msgBuilder, msg); + + // with header router + traceID = "1233"; + srcGwNode = "srcGwNode"; + dstGwNode = "dstGwNode"; + msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, + componentType, srcNode, srcInst, dstNode, dstInst, payload); + checkEncodeDecode(msgBuilder, msg); + + // with optional field + std::string srcNodeData = "sdwerwer"; + srcNode = bcos::bytes(srcNodeData.begin(), srcNodeData.end()); + std::string dstNodeData = "dstswerwer"; + dstNode = bcos::bytes(dstNodeData.begin(), dstNodeData.end()); + dstInst = "dstInst"; + srcInst = "srcInst"; + componentType = "compp,ad"; + topic = "topcisdf"; + + msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, + componentType, srcNode, srcInst, dstNode, dstInst, payload); + msg->setPacketType((uint16_t)ppc::gateway::GatewayPacketType::P2PMessage); + checkEncodeDecode(msgBuilder, msg); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp index 316abdcf..1a8fea15 100644 --- a/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp +++ b/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp @@ -18,8 +18,11 @@ * @date 2022-10-28 */ -#include "ppc-protocol/src/PPCMessage.h" +#include "protocol/src/PPCMessage.h" #include "ppc-framework/protocol/Protocol.h" +#include "protocol/src/v1/MessageHeaderImpl.h" +#include "protocol/src/v1/MessageImpl.h" +#include "protocol/src/v1/MessagePayloadImpl.h" #include #include #include @@ -28,17 +31,12 @@ using namespace ppc; using namespace ppc::front; using namespace bcos::test; +using namespace ppc::protocol; BOOST_FIXTURE_TEST_SUITE(PPCMessageTest, TestPromptFixture) -void testEncodeAndDecode(PPCMessageFace::Ptr _message) +void checkPPCMessage(PPCMessageFace::Ptr _message, PPCMessageFace::Ptr newMsg) { - auto payload = std::make_shared(); - _message->encode(*payload); - - auto messageFactory = std::make_shared(); - auto newMsg = messageFactory->buildPPCMessage(payload); - BOOST_CHECK(newMsg->version() == _message->version()); BOOST_CHECK(newMsg->taskType() == _message->taskType()); BOOST_CHECK(newMsg->algorithmType() == _message->algorithmType()); @@ -46,7 +44,6 @@ void testEncodeAndDecode(PPCMessageFace::Ptr _message) BOOST_CHECK(newMsg->seq() == _message->seq()); BOOST_CHECK(newMsg->taskID() == _message->taskID()); BOOST_CHECK(newMsg->sender() == _message->sender()); - BOOST_CHECK(newMsg->ext() == _message->ext()); BOOST_CHECK(newMsg->uuid() == _message->uuid()); BOOST_CHECK(newMsg->data()->size() == _message->data()->size()); auto newMsgHeader = newMsg->header(); @@ -58,24 +55,108 @@ void testEncodeAndDecode(PPCMessageFace::Ptr _message) BOOST_CHECK(messageHeader["x-http-request"] == "2222222"); } -BOOST_AUTO_TEST_CASE(test_ppcMesage) +void checkPayloadMsg(MessagePayload::Ptr payloadMsg, MessagePayload::Ptr decodedPayload) +{ + BOOST_CHECK(payloadMsg->version() == decodedPayload->version()); + BOOST_CHECK(payloadMsg->seq() == decodedPayload->seq()); + BOOST_CHECK(payloadMsg->traceID() == decodedPayload->traceID()); + BOOST_CHECK(payloadMsg->data() == decodedPayload->data()); + BOOST_CHECK(payloadMsg->ext() == decodedPayload->ext()); +} + +void checkMsg(Message::Ptr msg, Message::Ptr decodedMsg) +{ + if (msg->header()) + { + BOOST_CHECK(msg->header()->version() == decodedMsg->header()->version()); + BOOST_CHECK(msg->header()->traceID() == decodedMsg->header()->traceID()); + BOOST_CHECK(msg->header()->srcGwNode() == decodedMsg->header()->srcGwNode()); + BOOST_CHECK(msg->header()->dstGwNode() == decodedMsg->header()->dstGwNode()); + BOOST_CHECK(msg->header()->packetType() == decodedMsg->header()->packetType()); + BOOST_CHECK(msg->header()->ttl() == decodedMsg->header()->ttl()); + BOOST_CHECK(msg->header()->ext() == decodedMsg->header()->ext()); + } + auto routeInfo = msg->header()->optionalField(); + auto decodedRouteInfo = decodedMsg->header()->optionalField(); + if (routeInfo) + { + BOOST_CHECK(routeInfo->topic() == decodedRouteInfo->topic()); + BOOST_CHECK(routeInfo->componentType() == decodedRouteInfo->componentType()); + BOOST_CHECK(routeInfo->srcNode() == decodedRouteInfo->srcNode()); + BOOST_CHECK(routeInfo->srcInst() == decodedRouteInfo->srcInst()); + BOOST_CHECK(routeInfo->dstNode() == decodedRouteInfo->dstNode()); + BOOST_CHECK(routeInfo->dstInst() == decodedRouteInfo->dstInst()); + } +} + +PPCMessageFace::Ptr fakePPCMessage(PPCMessageFactory::Ptr messageFactory, int version, int taskType, + int algorithmType, int messageType, std::string const& taskID, int seq, std::string const& uuid, + std::string const& srcInst, std::shared_ptr data, + std::map const& header) { - auto messageFactory = std::make_shared(); auto message = messageFactory->buildPPCMessage(); - message->setVersion(1); - message->setTaskType(uint8_t(protocol::TaskType::PSI)); - message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC)); - message->setMessageType(4); - message->setSeq(5); - message->setTaskID("12345678"); - message->setSender("1001"); - message->setExt(10); - message->setUuid("uuid1245"); - message->setData(std::make_shared(10, 'a')); + message->setVersion(version); + message->setTaskType(taskType); + message->setAlgorithmType(algorithmType); + message->setMessageType(messageType); + message->setTaskID(taskID); + message->setSeq(seq); + message->setUuid(uuid); + message->setSender(srcInst); + message->setData(data); + message->setHeader(header); + return message; +} + + +BOOST_AUTO_TEST_CASE(test_ppcMesage) +{ + int version = 1; + int taskType = uint8_t(protocol::TaskType::PSI); + int algorithmType = uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC); + int messageType = 4; + std::string taskID = "12345678"; + int seq = 5; + std::string uuid = "uuid1245"; + std::string srcInst = "from"; + std::string dstInst = "dst"; + auto data = std::make_shared(10, 'a'); std::map head = { {"x-http-session", "111111"}, {"x-http-request", "2222222"}}; - message->setHeader(head); - testEncodeAndDecode(message); + + auto messageFactory = std::make_shared(); + auto ppcMessage = fakePPCMessage(messageFactory, version, taskType, algorithmType, messageType, + taskID, seq, uuid, srcInst, data, head); + + auto payloadBuilder = std::make_shared(); + auto msgBuilder = + std::make_shared(std::make_shared()); + + auto msg = messageFactory->buildMessage(msgBuilder, payloadBuilder, ppcMessage); + auto payloadMsg = messageFactory->buildMessage(payloadBuilder, ppcMessage); + bcos::bytes payloadData; + payloadMsg->encode(payloadData); + + auto decodedPayload = payloadBuilder->build(bcos::ref(payloadData)); + checkPayloadMsg(payloadMsg, decodedPayload); + + auto decodedPayload2 = payloadBuilder->build(bcos::ref(*(msg->payload()))); + checkPayloadMsg(payloadMsg, decodedPayload); + + bcos::bytes msgData; + msg->encode(msgData); + + auto decodedMsg = msgBuilder->build(bcos::ref(msgData)); + checkMsg(msg, decodedMsg); + + decodedMsg->setFrontMessage(decodedPayload2); + auto decodedPPCMsg = messageFactory->decodePPCMessage(decodedMsg); + checkPPCMessage(ppcMessage, decodedPPCMsg); + + // invalid case + std::string invalidStr = "sdfsinvalidsfwre"; + bcos::bytes invalidData(invalidStr.begin(), invalidStr.end()); + BOOST_CHECK_THROW(msgBuilder->build(bcos::ref(invalidData)), std::exception); } BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp b/cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp index 33fc5a4c..dbe5c8ce 100644 --- a/cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp +++ b/cpp/wedpr-protocol/protocol/tests/TestTaskImpl.cpp @@ -18,7 +18,7 @@ * @date 2022-10-19 */ -#include "ppc-protocol/src/JsonTaskImpl.h" +#include "protocol/src/JsonTaskImpl.h" #include "test-utils/TaskMock.h" #include #include diff --git a/cpp/wedpr-protocol/tars/CMakeLists.txt b/cpp/wedpr-protocol/tars/CMakeLists.txt index 7d0e2f8e..7ae5a697 100644 --- a/cpp/wedpr-protocol/tars/CMakeLists.txt +++ b/cpp/wedpr-protocol/tars/CMakeLists.txt @@ -39,10 +39,3 @@ target_include_directories(${TARS_PROTOCOL_TARGET} PUBLIC $ $) target_link_libraries(${TARS_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} tarscpp::tarsutil) - -# ut -if (TESTS) - enable_testing() - set(CTEST_OUTPUT_ON_FAILURE TRUE) - add_subdirectory(test) -endif () diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 8888b1fa..01a1f179 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -42,7 +42,8 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace bcos::bytes data; _message->encode(data); auto self = weak_from_this(); - m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_AGENCY, routeInfo, std::move(data), + // ROUTE_THROUGH_TOPIC will hold the topic + m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data), _message->seq(), _timeout, _callback, [self, _agencyID, _respCallback]( Error::Ptr error, Message::Ptr msg, SendResponseFunction resFunc) { @@ -66,7 +67,7 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace } // get the agencyID _respCallback(error, msg->header()->optionalField()->srcInst(), - front->m_messageFactory->buildPPCMessage(msg), responseCallback); + front->m_messageFactory->decodePPCMessage(msg), responseCallback); }); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h index b3a10370..30cc315a 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h @@ -74,7 +74,7 @@ class Front : public FrontInterface, public std::enable_shared_from_this _handler(nullptr); return; } - _handler(front->m_messageFactory->buildPPCMessage(msg)); + _handler(front->m_messageFactory->decodePPCMessage(msg)); }); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index 467e4bf6..470462a2 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -134,8 +134,8 @@ void FrontImpl::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr m_callbackManager->addCallback(traceID, timeout, callback); auto self = weak_from_this(); // send the message to the gateway - asyncSendMessageToGateway(false, std::move(frontMessage), routeType, routeInfo, timeout, - [self, traceID, routeInfo, errorCallback](bcos::Error::Ptr error) { + asyncSendMessageToGateway(false, std::move(frontMessage), routeType, traceID, routeInfo, + timeout, [self, traceID, routeInfo, errorCallback](bcos::Error::Ptr error) { auto front = self.lock(); if (!front) { @@ -181,7 +181,7 @@ void FrontImpl::handleCallback( // set the srcNodeID routerInfo->setSrcNode(message->header()->optionalField()->dstNode()); front->asyncSendMessageToGateway(true, std::move(frontMessage), - RouteType::ROUTE_THROUGH_NODEID, routerInfo, 0, + RouteType::ROUTE_THROUGH_NODEID, message->header()->traceID(), routerInfo, 0, [routerInfo](bcos::Error::Ptr error) { if (!error || error->errorCode() == 0) { @@ -196,8 +196,8 @@ void FrontImpl::handleCallback( } void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::Ptr&& frontMessage, - RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, long timeout, - ReceiveMsgFunc callback) + RouteType routeType, std::string const& traceID, MessageOptionalHeader::Ptr const& routeInfo, + long timeout, ReceiveMsgFunc callback) { if (responsePacket) { @@ -206,7 +206,8 @@ void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::P routeInfo->setSrcNode(m_nodeID); auto payload = std::make_shared(); frontMessage->encode(*payload); - m_gatewayClient->asyncSendMessage(routeType, routeInfo, std::move(*payload), timeout, callback); + m_gatewayClient->asyncSendMessage( + routeType, routeInfo, traceID, std::move(*payload), timeout, callback); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index d89529d6..32700b7b 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -158,8 +158,8 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ private: void asyncSendMessageToGateway(bool responsePacket, ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, long timeout, - ppc::protocol::ReceiveMsgFunc callback); + std::string const& traceID, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, + long timeout, ppc::protocol::ReceiveMsgFunc callback); void handleCallback(bcos::Error::Ptr const& error, std::string const& traceID, ppc::protocol::Message::Ptr message); diff --git a/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp b/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp index d0d2d6cf..e5f1ad9e 100644 --- a/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp +++ b/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp @@ -17,6 +17,7 @@ * @author: shawnhe * @date 2022-10-29 */ +#if 0 #include "ppc-front/ppc-front/PPCChannel.h" #include "ppc-front/ppc-front/PPCChannelManager.h" @@ -42,7 +43,7 @@ BOOST_AUTO_TEST_CASE(test_ppcChannel) auto channelManager = std::make_shared(ioService, front); // Note: must start here, otherwise the ioservice will not work - front->start(); + //front->start(); // register message handler channelManager->registerMsgHandlerForChannel( @@ -112,7 +113,7 @@ BOOST_AUTO_TEST_CASE(test_ppcChannel) std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - front->stop(); + //front->stop(); } - -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END() +#endif \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h index c6dc211c..5a81cc8c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h @@ -31,6 +31,8 @@ namespace ppc::gateway { #define GATEWAY_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY]" +#define SERVICE_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][SERVICE]" +#define SERVICE_ROUTER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][SERVICE][ROUTER]" // HTTP HEADER DEFINE #define HEAD_TASK_ID "x-ptp-session-id" diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp index 1349593c..a31a184d 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp @@ -19,6 +19,7 @@ */ #include "GatewayFactory.h" #include "Common.h" +#include "bcos-boostssl/utility/NewTimer.h" #include "bcos-boostssl/websocket/WsInitializer.h" #include "ppc-gateway/p2p/Service.h" #include "ppc-gateway/p2p/router/RouterTableImpl.h" @@ -59,7 +60,8 @@ Service::Ptr GatewayFactory::buildService() const wsInitializer->setConfig(wsConfig); auto p2pService = std::make_shared(m_contextConfig->nodeID(), std::make_shared(), m_config->gatewayConfig().unreachableDistance, - "Service"); + "Gateway-Service"); + p2pService->setTimerFactory(std::make_shared()); p2pService->setNodeEndpoints(m_gatewayConfig->nodeIPEndpointSet()); wsInitializer->initWsService(p2pService); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 4abad7a4..49c28a70 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -84,13 +84,14 @@ void GatewayImpl::stop() } void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, - MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) + MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload) { // dispatcher to all the local front routeInfo->setDstNode(bcos::bytes()); routeInfo->setSrcInst(m_agency); auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload)); + p2pMessage->setSeq(traceID); p2pMessage->setPacketType((uint16_t)GatewayPacketType::BroadcastMessage); m_localRouter->dispatcherMessage(p2pMessage, nullptr); @@ -100,14 +101,16 @@ void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType, - ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout, - ReceiveMsgFunc callback) + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, + bcos::bytes&& payload, long timeout, ReceiveMsgFunc callback) { routeInfo->setSrcInst(m_agency); // check the localRouter auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload)); - + p2pMessage->setSeq(traceID); p2pMessage->setPacketType((uint16_t)GatewayPacketType::P2PMessage); + GATEWAY_LOG(INFO) << LOG_DESC("##### asyncSendMessage") + << LOG_KV("msg", printMessage(p2pMessage)); auto nodeList = m_localRouter->chooseReceiver(p2pMessage); // case send to the same agency if (!nodeList.empty()) diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 66d692bf..12a43d49 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -54,11 +54,12 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_thisp2pnodeid(); + return m_rawGatewayInfo->p2pnodeid(); } // the agency std::string const& GatewayNodeInfoImpl::agency() const { - return m_inner()->agency(); + return m_rawGatewayInfo->agency(); } uint32_t GatewayNodeInfoImpl::statusSeq() const { - return m_inner()->statusseq(); + return m_rawGatewayInfo->statusseq(); } void GatewayNodeInfoImpl::setStatusSeq(uint32_t statusSeq) { - m_inner()->set_statusseq(statusSeq); + m_rawGatewayInfo->set_statusseq(statusSeq); } // get the node information by nodeID @@ -58,18 +58,35 @@ INodeInfo::Ptr GatewayNodeInfoImpl::nodeInfo(bcos::bytes const& nodeID) const return nullptr; } +void GatewayNodeInfoImpl::updateNodeList() +{ + // Note: can't use clear_nodelist here, for clear_nodelist will destroy the allocated nodelist, + // and cause double release coredump + releaseWithoutDestory(); + // re-encode nodeList + for (auto const& it : m_nodeList) + { + auto nodeInfo = std::dynamic_pointer_cast(it.second); + m_rawGatewayInfo->mutable_nodelist()->UnsafeArenaAddAllocated( + nodeInfo->rawNodeInfo().get()); + } +} + bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info) { auto nodeID = info->nodeID().toBytes(); auto existedNodeInfo = nodeInfo(nodeID); - // update the info - if (existedNodeInfo == nullptr || !existedNodeInfo->equal(info)) + // the node info has not been updated + if (existedNodeInfo != nullptr && existedNodeInfo->equal(info)) + { + return false; + } { bcos::WriteGuard l(x_nodeList); m_nodeList[nodeID] = info; - return true; + updateNodeList(); } - return false; + return true; } void GatewayNodeInfoImpl::removeNodeInfo(bcos::bytes const& nodeID) @@ -84,6 +101,7 @@ void GatewayNodeInfoImpl::removeNodeInfo(bcos::bytes const& nodeID) } bcos::UpgradeGuard ul(l); m_nodeList.erase(it); + updateNodeList(); } // remove the topic info { @@ -184,30 +202,21 @@ void GatewayNodeInfoImpl::unRegisterTopic(bcos::bytes const& nodeID, std::string void GatewayNodeInfoImpl::encode(bcos::bytes& data) const { - m_inner()->clear_nodelist(); - { - bcos::ReadGuard l(x_nodeList); - // encode nodeList - for (auto const& it : m_nodeList) - { - auto nodeInfo = std::dynamic_pointer_cast(it.second); - m_inner()->mutable_nodelist()->UnsafeArenaAddAllocated(nodeInfo->innerFunc()()); - } - } - encodePBObject(data, m_inner()); + encodePBObject(data, m_rawGatewayInfo); } void GatewayNodeInfoImpl::decode(bcos::bytesConstRef data) { - decodePBObject(m_inner(), data); + decodePBObject(m_rawGatewayInfo, data); { bcos::WriteGuard l(x_nodeList); // decode into m_nodeList m_nodeList.clear(); - for (int i = 0; i < m_inner()->nodelist_size(); i++) + for (int i = 0; i < m_rawGatewayInfo->nodelist_size(); i++) { - auto nodeInfoPtr = std::make_shared( - [m_entry = m_inner()->nodelist(i)]() mutable { return &m_entry; }); + std::shared_ptr rawNodeInfo( + m_rawGatewayInfo->mutable_nodelist(i)); + auto nodeInfoPtr = std::make_shared(rawNodeInfo); m_nodeList.insert(std::make_pair(nodeInfoPtr->nodeID().toBytes(), nodeInfoPtr)); } } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index 4e3ba366..92a74f17 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -29,21 +29,18 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo { public: using Ptr = std::shared_ptr; + GatewayNodeInfoImpl() : m_rawGatewayInfo(std::make_shared()) {} GatewayNodeInfoImpl(std::string const& p2pNodeID, std::string const& agency) - : m_inner([inner = ppc::proto::GatewayNodeInfo()]() mutable { return &inner; }) + : GatewayNodeInfoImpl() { - m_inner()->set_p2pnodeid(p2pNodeID); - m_inner()->set_agency(agency); - } - ~GatewayNodeInfoImpl() override - { - auto allocatedNodeListSize = m_inner()->nodelist_size(); - for (int i = 0; i < allocatedNodeListSize; i++) - { - m_inner()->mutable_nodelist()->UnsafeArenaReleaseLast(); - } + m_rawGatewayInfo->set_p2pnodeid(p2pNodeID); + m_rawGatewayInfo->set_agency(agency); } + GatewayNodeInfoImpl(bcos::bytesConstRef data) : GatewayNodeInfoImpl() { decode(data); } + + ~GatewayNodeInfoImpl() override { releaseWithoutDestory(); } + // the gateway nodeID std::string const& p2pNodeID() const override; // the agency @@ -71,21 +68,38 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo std::map nodeList() const override { - bcos::WriteGuard l(x_nodeList); + bcos::ReadGuard l(x_nodeList); return m_nodeList; } uint32_t statusSeq() const override; void setStatusSeq(uint32_t statusSeq) override; - virtual uint16_t nodeSize() const override { return m_nodeList.size(); } + virtual uint16_t nodeSize() const override + { + bcos::ReadGuard l(x_nodeList); + return m_nodeList.size(); + } + +private: + void updateNodeList(); + + void releaseWithoutDestory() + { + // return back the ownership to nodeList to shared_ptr + auto allocatedNodeListSize = m_rawGatewayInfo->nodelist_size(); + for (int i = 0; i < allocatedNodeListSize; i++) + { + m_rawGatewayInfo->mutable_nodelist()->UnsafeArenaReleaseLast(); + } + } private: - std::function m_inner; + std::shared_ptr m_rawGatewayInfo; // NodeID => nodeInfo std::map m_nodeList; mutable bcos::SharedMutex x_nodeList; - // NodeID=>topics + // NodeID=>topics(Note serialized) using Topics = std::set; std::map m_topicInfo; mutable bcos::SharedMutex x_topicInfo; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp index a26844e4..6106e6bc 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp @@ -93,7 +93,8 @@ void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSessi return; } // status changed, request for the nodeStatus - GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveNodeSeqMessage") << LOG_KV("from", from) + GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveNodeSeqMessage") + << LOG_KV("from", printP2PIDElegantly(from)) << LOG_KV("statusSeq", statusSeq); m_service->asyncSendMessageByP2PNodeID( (uint16_t)GatewayPacketType::RequestNodeStatus, from, std::make_shared()); @@ -138,11 +139,11 @@ void GatewayRouterManager::onReceiveRequestNodeStatusMsg( if (!nodeStatusData) { GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveRequestNodeStatusMsg: generate nodeInfo error") - << LOG_KV("from", from); + << LOG_KV("from", printP2PIDElegantly(from)); return; } GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveRequestNodeStatusMsg: response the latest nodeStatus") - << LOG_KV("from", from); + << LOG_KV("from", printP2PIDElegantly(from)); m_service->asyncSendMessageByP2PNodeID( (uint16_t)GatewayPacketType::ResponseNodeStatus, from, nodeStatusData); } @@ -157,7 +158,8 @@ void GatewayRouterManager::onRecvResponseNodeStatusMsg(MessageFace::Ptr msg, WsS p2pMessage->header()->srcGwNode() : session->nodeId(); - GATEWAY_LOG(INFO) << LOG_DESC("onRecvResponseNodeStatusMsg") << LOG_KV("from", from) + GATEWAY_LOG(INFO) << LOG_DESC("onRecvResponseNodeStatusMsg") + << LOG_KV("from", printP2PIDElegantly(from)) << LOG_KV("statusSeq", nodeStatus->statusSeq()) << LOG_KV("agency", nodeStatus->agency()); updatePeerNodeStatus(from, nodeStatus); @@ -176,7 +178,8 @@ void GatewayRouterManager::updatePeerNodeStatus( UpgradeGuard ul(l); m_p2pID2Seq[p2pID] = statusSeq; } - GATEWAY_LOG(INFO) << LOG_DESC("updatePeerNodeStatus") << LOG_KV("from", p2pID) + GATEWAY_LOG(INFO) << LOG_DESC("updatePeerNodeStatus") + << LOG_KV("from", printP2PIDElegantly(p2pID)) << LOG_KV("statusSeq", status->statusSeq()) << LOG_KV("agency", status->agency()); m_peerRouter->updateGatewayInfo(status); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 2db66e76..55b3ea67 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -20,14 +20,27 @@ #include "LocalRouter.h" #include "ppc-framework/Common.h" #include "ppc-framework/gateway/GatewayProtocol.h" +#include "ppc-gateway/Common.h" using namespace bcos; using namespace ppc::protocol; using namespace ppc::gateway; +bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo) +{ + GATEWAY_LOG(INFO) << LOG_DESC("registerNodeInfo") << printNodeInfo(nodeInfo); + nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); + auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); + if (ret) + { + increaseSeq(); + } + return ret; +} // Note: the change of the topic will not trigger router-update void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& topic) { + GATEWAY_LOG(INFO) << LOG_DESC("registerTopic") << LOG_KV("topic", topic); m_routerInfo->registerTopic(_nodeID.toBytes(), topic); // try to dispatch the cacheInfo if (!m_cache) diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index 2bd18e7f..5cf5a7ef 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -30,26 +30,16 @@ class LocalRouter { public: using Ptr = std::shared_ptr; - LocalRouter(GatewayNodeInfoFactory::Ptr nodeInfoFactory, + LocalRouter(GatewayNodeInfoFactory::Ptr gatewayNodeInfoFactory, ppc::front::IFrontBuilder::Ptr frontBuilder, MessageCache::Ptr msgCache) - : m_routerInfo(std::move(nodeInfoFactory->build())), + : m_routerInfo(gatewayNodeInfoFactory->build()), m_frontBuilder(std::move(frontBuilder)), m_cache(std::move(msgCache)) {} virtual ~LocalRouter() = default; - virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) - { - nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); - auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); - if (ret) - { - increaseSeq(); - } - return ret; - } - + virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo); virtual void unRegisterNode(bcos::bytes const& nodeID) { m_routerInfo->removeNodeInfo(nodeID); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp index fd36e94e..c60fa00d 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -40,7 +40,7 @@ Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _rou m_routerTable->setNodeID(m_nodeID); m_routerTable->setUnreachableDistance(unreachableDistance); - GATEWAY_LOG(INFO) << LOG_DESC("create P2PService") << LOG_KV("module", _moduleName); + SERVICE_LOG(INFO) << LOG_DESC("create P2PService") << LOG_KV("module", _moduleName); WsService::registerConnectHandler( boost::bind(&Service::onP2PConnect, this, boost::placeholders::_1)); WsService::registerDisconnectHandler( @@ -50,7 +50,8 @@ Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _rou void Service::onP2PConnect(WsSession::Ptr _session) { - GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect") << LOG_KV("p2pid", _session->nodeId()) + SERVICE_LOG(INFO) << LOG_DESC("Receive new p2p connection") + << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); @@ -59,8 +60,8 @@ void Service::onP2PConnect(WsSession::Ptr _session) // the session already connected if (it != m_nodeID2Session.end() && it->second->isConnected()) { - GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect, drop the duplicated connection") - << LOG_KV("nodeID", _session->nodeId()) + SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect, drop the duplicated connection") + << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); _session->drop(WsError::UserDisconnect); updateNodeIDInfo(_session); @@ -70,28 +71,54 @@ void Service::onP2PConnect(WsSession::Ptr _session) if (_session->nodeId() == m_nodeID) { updateNodeIDInfo(_session); - GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect, drop the node-self connection") - << LOG_KV("nodeID", _session->nodeId()) + SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect, drop the node-self connection") + << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); _session->drop(WsError::UserDisconnect); return; } - // the new session - updateNodeIDInfo(_session); + + + ///// Note: here allow all new session, even the ip not configured(support dynamic access) + bool updated = updateNodeIDInfo(_session); + // hit the m_nodeID2Session if (it != m_nodeID2Session.end()) { + // the old session has already been connected, and the new session endPoint is not + // configured + if (it->second->isConnected() && !updated) + { + SERVICE_LOG(INFO) << LOG_DESC( + "onP2PConnect, drop the new not-configurated session, remain " + "the old session") + << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) + << LOG_KV("endPoint", _session->endPoint()) + << LOG_KV("oldEndPoint", it->second->endPoint()); + _session->drop(WsError::UserDisconnect); + return; + } + SERVICE_LOG(INFO) << LOG_DESC( + "onP2PConnect, drop the old not-configurated session, replace " + "with the new session") + << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) + << LOG_KV("endPoint", _session->endPoint()) + << LOG_KV("oldEndPoint", it->second->endPoint()); + if (it->second->isConnected()) + { + it->second->drop(WsError::UserDisconnect); + } it->second = _session; + return; } - else - { - m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); - } - GATEWAY_LOG(INFO) << LOG_DESC("onP2PConnect established") << LOG_KV("p2pid", _session->nodeId()) + // the new session + m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); + SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect established new session") + << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); } -void Service::updateNodeIDInfo(WsSession::Ptr const& _session) +bool Service::updateNodeIDInfo(WsSession::Ptr const& _session) { bcos::WriteGuard l(x_configuredNode2ID); std::string p2pNodeID = _session->nodeId(); @@ -99,16 +126,16 @@ void Service::updateNodeIDInfo(WsSession::Ptr const& _session) if (it != m_configuredNode2ID.end()) { it->second = p2pNodeID; - GATEWAY_LOG(INFO) << LOG_DESC("updateNodeIDInfo: update the nodeID") - << LOG_KV("nodeid", p2pNodeID) - << LOG_KV("endpoint", _session->endPoint()); - } - else - { - GATEWAY_LOG(INFO) << LOG_DESC("updateNodeIDInfo can't find endpoint") + SERVICE_LOG(INFO) << LOG_DESC("updateNodeIDInfo: update the nodeID") << LOG_KV("nodeid", p2pNodeID) << LOG_KV("endpoint", _session->endPoint()); + return true; } + + SERVICE_LOG(INFO) << LOG_DESC("updateNodeIDInfo can't find endpoint") + << LOG_KV("nodeid", printP2PIDElegantly(p2pNodeID)) + << LOG_KV("endpoint", _session->endPoint()); + return false; } void Service::removeSessionInfo(WsSession::Ptr const& _session) @@ -117,8 +144,8 @@ void Service::removeSessionInfo(WsSession::Ptr const& _session) auto it = m_nodeID2Session.find(_session->nodeId()); if (it != m_nodeID2Session.end()) { - GATEWAY_LOG(INFO) << "onP2PDisconnectand remove from m_nodeID2Session" - << LOG_KV("p2pid", _session->nodeId()) + SERVICE_LOG(INFO) << "onP2PDisconnectand remove from m_nodeID2Session" + << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); m_nodeID2Session.erase(it); @@ -132,7 +159,8 @@ void Service::onP2PDisconnect(WsSession::Ptr _session) UpgradableGuard l(x_configuredNode2ID); for (auto& it : m_configuredNode2ID) { - if (it.second == _session->nodeId()) + // reset the nodeID of the dropped session(except the node-self) to empty + if (m_nodeID != _session->nodeId() && it.second == _session->nodeId()) { UpgradeGuard ul(l); it.second.clear(); @@ -158,6 +186,9 @@ void Service::reconnect() continue; } unconnectedPeers->insert(it.first); + SERVICE_LOG(DEBUG) << LOG_DESC("ready to reconnect") + << LOG_KV("endpoint", + it.first.address() + ":" + std::to_string(it.first.port())); } } setReconnectedPeers(unconnectedPeers); @@ -201,7 +232,7 @@ void Service::asyncSendMessageWithForward( return asyncSendMessage(dstNodeID, msg, options, respFunc); } // with nextHop, send the message to nextHop - GATEWAY_LOG(TRACE) << LOG_DESC("asyncSendMessageByNodeID") << printMessage(p2pMsg); + SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageByNodeID") << printMessage(p2pMsg); return asyncSendMessage(nextHop, msg, options, respFunc); } @@ -216,6 +247,10 @@ void Service::asyncSendMessage( { return; } + if (msg->seq().empty()) + { + msg->setSeq(m_messageFactory->newSeq()); + } auto session = getSessionByNodeID(dstNodeID); if (session) { @@ -231,13 +266,13 @@ void Service::asyncSendMessage( " failed for no network established, msg: " + printWsMessage(msg)); respFunc(std::move(error), nullptr, nullptr); } - GATEWAY_LOG(WARNING) + SERVICE_LOG(WARNING) << LOG_DESC("asyncSendMessageByNodeID failed for no network established, msg detail:") << printWsMessage(msg); } catch (std::exception const& e) { - GATEWAY_LOG(ERROR) << "asyncSendMessageByNodeID" << LOG_KV("dstNode", dstNodeID) + SERVICE_LOG(ERROR) << "asyncSendMessageByNodeID" << LOG_KV("dstNode", dstNodeID) << LOG_KV("what", boost::diagnostic_information(e)); if (respFunc) { @@ -254,7 +289,7 @@ void Service::onRecvMessage(MessageFace::Ptr _msg, std::shared_ptr _s // find the dstNode if (p2pMsg->header()->dstGwNode().empty() || p2pMsg->header()->dstGwNode() == m_nodeID) { - GATEWAY_LOG(TRACE) << LOG_DESC("onRecvMessage, dispatch for find the dst node") + SERVICE_LOG(TRACE) << LOG_DESC("onRecvMessage, dispatch for find the dst node") << printMessage(p2pMsg); WsService::onRecvMessage(_msg, _session); return; @@ -262,7 +297,7 @@ void Service::onRecvMessage(MessageFace::Ptr _msg, std::shared_ptr _s // forward the message if (p2pMsg->header()->ttl() >= m_routerTable->unreachableDistance()) { - GATEWAY_LOG(WARNING) << LOG_DESC("onRecvMessage: ttl expired") << printMessage(p2pMsg); + SERVICE_LOG(WARNING) << LOG_DESC("onRecvMessage: ttl expired") << printMessage(p2pMsg); return; } p2pMsg->header()->setTTL(p2pMsg->header()->ttl() + 1); @@ -283,7 +318,7 @@ void Service::asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, Option } catch (std::exception& e) { - GATEWAY_LOG(WARNING) << LOG_BADGE("asyncBroadcastMessage exception") + SERVICE_LOG(WARNING) << LOG_BADGE("asyncBroadcastMessage exception") << LOG_KV("msg", printWsMessage(msg)) << LOG_KV("error", boost::diagnostic_information(e)); } @@ -318,6 +353,6 @@ void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& WsSessions sessions; sessions.emplace_back(session); WsService::asyncSendMessage(sessions, respMessage); - GATEWAY_LOG(TRACE) << "sendRespMessageBySession" << LOG_KV("resp", printMessage(respMessage)) + SERVICE_LOG(TRACE) << "sendRespMessageBySession" << LOG_KV("resp", printMessage(respMessage)) << LOG_KV("payload size", payload->size()); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h index 3be13906..c41e0c1f 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h @@ -79,7 +79,7 @@ class Service : public bcos::boostssl::ws::WsService void reconnect() override; - void updateNodeIDInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); + bool updateNodeIDInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); void removeSessionInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); bcos::boostssl::ws::WsSession::Ptr getSessionByNodeID(std::string const& _nodeID); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp index 00f3e7c7..77a765fb 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp @@ -70,9 +70,10 @@ void RouterManager::onReceiveRouterSeq(MessageFace::Ptr msg, WsSession::Ptr sess { return; } - GATEWAY_LOG(INFO) << LOG_BADGE("onReceiveRouterSeq") - << LOG_DESC("receive router seq and request router table") - << LOG_KV("peer", session->nodeId()) << LOG_KV("seq", statusSeq); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("onReceiveRouterSeq") + << LOG_DESC("receive router seq and request router table") + << LOG_KV("peer", printP2PIDElegantly(session->nodeId())) + << LOG_KV("seq", statusSeq); // request router table to peer auto p2pMsg = std::dynamic_pointer_cast(msg); auto dstP2PNodeID = (!p2pMsg->header()->srcGwNode().empty()) ? p2pMsg->header()->srcGwNode() : @@ -99,17 +100,19 @@ void RouterManager::onReceivePeersRouterTable(MessageFace::Ptr msg, WsSession::P { auto routerTable = m_service->routerTableFactory()->createRouterTable(ref(*(msg->payload()))); - GATEWAY_LOG(INFO) << LOG_BADGE("onReceivePeersRouterTable") << LOG_KV("peer", session->nodeId()) - << LOG_KV("entrySize", routerTable->routerEntries().size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("onReceivePeersRouterTable") + << LOG_KV("peer", printP2PIDElegantly(session->nodeId())) + << LOG_KV("entrySize", routerTable->routerEntries().size()); joinRouterTable(session->nodeId(), routerTable); } // receive routerTable request from peer void RouterManager::onReceiveRouterTableRequest(MessageFace::Ptr msg, WsSession::Ptr session) { - GATEWAY_LOG(INFO) << LOG_BADGE("onReceiveRouterTableRequest") - << LOG_KV("peer", session->nodeId()) - << LOG_KV("entrySize", m_service->routerTable()->routerEntries().size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("onReceiveRouterTableRequest") + << LOG_KV("peer", printP2PIDElegantly(session->nodeId())) + << LOG_KV( + "entrySize", m_service->routerTable()->routerEntries().size()); auto routerTableData = std::make_shared(); m_service->routerTable()->encode(*routerTableData); @@ -135,8 +138,8 @@ void RouterManager::joinRouterTable( } } - GATEWAY_LOG(INFO) << LOG_BADGE("joinRouterTable") << LOG_DESC("create router entry") - << LOG_KV("dst", _generatedFrom); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("joinRouterTable") << LOG_DESC("create router entry") + << LOG_KV("dst", printP2PIDElegantly(_generatedFrom)); auto entry = m_service->routerTableFactory()->createRouterEntry(); entry->setDstNode(_generatedFrom); @@ -147,8 +150,9 @@ void RouterManager::joinRouterTable( } if (!updated) { - GATEWAY_LOG(DEBUG) << LOG_BADGE("joinRouterTable") << LOG_DESC("router table not updated") - << LOG_KV("dst", _generatedFrom); + SERVICE_ROUTER_LOG(DEBUG) << LOG_BADGE("joinRouterTable") + << LOG_DESC("router table not updated") + << LOG_KV("dst", printP2PIDElegantly(_generatedFrom)); return; } onP2PNodesUnreachable(unreachableNodes); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp index f8a2a22a..f3cc3071 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp @@ -69,9 +69,10 @@ bool RouterTable::erase(std::set& _unreachableNodes, std::string co it->second->clearNextHop(); _unreachableNodes.insert(it->second->dstNode()); - GATEWAY_LOG(INFO) << LOG_BADGE("erase") << LOG_DESC("make the router unreachable") - << LOG_KV("dst", _p2pNodeID) << LOG_KV("distance", it->second->distance()) - << LOG_KV("size", m_routerEntries.size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("erase") << LOG_DESC("make the router unreachable") + << LOG_KV("dst", printP2PIDElegantly(_p2pNodeID)) + << LOG_KV("distance", it->second->distance()) + << LOG_KV("size", m_routerEntries.size()); updated = true; } // update the router-entry with nextHop equal to _p2pNodeID to be unreachable @@ -94,13 +95,13 @@ void RouterTable::updateDistanceForAllRouterEntries( entry->clearNextHop(); _unreachableNodes.insert(entry->dstNode()); } - GATEWAY_LOG(INFO) << LOG_BADGE("updateDistanceForAllRouterEntries") - << LOG_DESC( - "update entry since the nextHop distance has been updated") - << LOG_KV("dst", entry->dstNode()) << LOG_KV("nextHop", _nextHop) - << LOG_KV("distance", entry->distance()) - << LOG_KV("oldDistance", oldDistance) - << LOG_KV("size", m_routerEntries.size()); + SERVICE_ROUTER_LOG(INFO) + << LOG_BADGE("updateDistanceForAllRouterEntries") + << LOG_DESC("update entry since the nextHop distance has been updated") + << LOG_KV("dst", printP2PIDElegantly(entry->dstNode())) + << LOG_KV("nextHop", printP2PIDElegantly(_nextHop)) + << LOG_KV("distance", entry->distance()) << LOG_KV("oldDistance", oldDistance) + << LOG_KV("size", m_routerEntries.size()); } } } @@ -111,10 +112,10 @@ bool RouterTable::update(std::set& _unreachableNodes, if (c_fileLogLevel <= TRACE) [[unlikely]] { - GATEWAY_LOG(TRACE) << LOG_BADGE("update") << LOG_DESC("receive entry") - << LOG_KV("dst", printP2PIDElegantly(_entry->dstNode())) - << LOG_KV("distance", _entry->distance()) - << LOG_KV("from", _generatedFrom); + SERVICE_ROUTER_LOG(TRACE) << LOG_BADGE("update") << LOG_DESC("receive entry") + << LOG_KV("dst", printP2PIDElegantly(_entry->dstNode())) + << LOG_KV("distance", _entry->distance()) + << LOG_KV("from", printP2PIDElegantly(_generatedFrom)); } auto ret = updateDstNodeEntry(_generatedFrom, _entry); // the dst entry has not been updated @@ -168,12 +169,12 @@ bool RouterTable::updateDstNodeEntry( _entry->setNextHop(_generatedFrom); } m_routerEntries.insert(std::make_pair(_entry->dstNode(), _entry)); - GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") - << LOG_DESC("insert new entry into the routerTable") - << LOG_KV("distance", _entry->distance()) - << LOG_KV("dst", _entry->dstNode()) - << LOG_KV("nextHop", _entry->nextHop()) - << LOG_KV("size", m_routerEntries.size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC("insert new entry into the routerTable") + << LOG_KV("distance", _entry->distance()) + << LOG_KV("dst", printP2PIDElegantly(_entry->dstNode())) + << LOG_KV("nextHop", printP2PIDElegantly(_entry->nextHop())) + << LOG_KV("size", m_routerEntries.size()); return true; } @@ -189,13 +190,13 @@ bool RouterTable::updateDstNodeEntry( currentEntry->setNextHop(_generatedFrom); } currentEntry->setDistance(distance); - GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") - << LOG_DESC("discover smaller distance, update entry") - << LOG_KV("distance", currentEntry->distance()) - << LOG_KV("oldDistance", currentDistance) - << LOG_KV("dst", _entry->dstNode()) - << LOG_KV("nextHop", _entry->nextHop()) - << LOG_KV("size", m_routerEntries.size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC("discover smaller distance, update entry") + << LOG_KV("distance", currentEntry->distance()) + << LOG_KV("oldDistance", currentDistance) + << LOG_KV("dst", printP2PIDElegantly(_entry->dstNode())) + << LOG_KV("nextHop", printP2PIDElegantly(_entry->nextHop())) + << LOG_KV("size", m_routerEntries.size()); return true; } // the distance information for the nextHop changed @@ -216,14 +217,14 @@ bool RouterTable::updateDstNodeEntry( { currentEntry->clearNextHop(); } - GATEWAY_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") - << LOG_DESC( - "distance of the nextHop entry " - "updated, update the current entry") - << LOG_KV("dst", currentEntry->dstNode()) - << LOG_KV("nextHop", currentEntry->nextHop()) - << LOG_KV("distance", currentEntry->distance()) - << LOG_KV("size", m_routerEntries.size()); + SERVICE_ROUTER_LOG(INFO) << LOG_BADGE("updateDstNodeEntry") + << LOG_DESC( + "distance of the nextHop entry " + "updated, update the current entry") + << LOG_KV("dst", printP2PIDElegantly(currentEntry->dstNode())) + << LOG_KV("nextHop", printP2PIDElegantly(currentEntry->nextHop())) + << LOG_KV("distance", currentEntry->distance()) + << LOG_KV("size", m_routerEntries.size()); return true; } return false; @@ -264,9 +265,9 @@ std::set RouterTable::getAllReachableNode() std::stringstream nodes; std::for_each(reachableNodes.begin(), reachableNodes.end(), [&](const auto& item) { nodes << printP2PIDElegantly(item) << ","; }); - GATEWAY_LOG(TRACE) << LOG_BADGE("getAllReachableNode") - << LOG_KV("nodes size", reachableNodes.size()) - << LOG_KV("nodes", nodes.str()); + SERVICE_ROUTER_LOG(TRACE) + << LOG_BADGE("getAllReachableNode") << LOG_KV("nodes size", reachableNodes.size()) + << LOG_KV("nodes", nodes.str()); } return reachableNodes; diff --git a/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayNodeInfoImplTest.cpp b/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayNodeInfoImplTest.cpp new file mode 100644 index 00000000..0a2f80e4 --- /dev/null +++ b/cpp/wedpr-transport/ppc-gateway/test/unittests/GatewayNodeInfoImplTest.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GatewayNodeInfoImplTest.cpp + * @author: yujiechen + * @date 2024-09-06 + */ +#include "ppc-gateway/gateway/router/GatewayNodeInfoImpl.h" +#include "ppc-gateway/gateway/router/GatewayNodeInfo.h" +#include "protobuf/src/NodeInfoImpl.h" +#include +#include +#include + +using namespace ppc; +using namespace ppc::protocol; +using namespace ppc::gateway; +using namespace bcos::test; + +BOOST_FIXTURE_TEST_SUITE(NodeInfoTest, TestPromptFixture) + +INodeInfo::Ptr fakeNodeInfo(INodeInfoFactory::Ptr factory, std::string const& nodeID, + std::string const& endPoint, std::set const& components) +{ + auto nodeInfo = + factory->build(bcos::bytesConstRef((bcos::byte*)nodeID.data(), nodeID.size()), endPoint); + // nodeInfo->setNodeID(bcos::bytesConstRef((bcos::byte*)nodeID.data(), nodeID.size())); + // nodeInfo->setEndPoint(endPoint); + nodeInfo->setComponents(components); + return nodeInfo; +} + +void checkNodeInfo(INodeInfo::Ptr nodeInfo, INodeInfo::Ptr decodedNodeInfo) +{ + BOOST_CHECK(nodeInfo->nodeID().toBytes() == decodedNodeInfo->nodeID().toBytes()); + BOOST_CHECK(nodeInfo->endPoint() == decodedNodeInfo->endPoint()); + auto const& components = nodeInfo->components(); + for (auto const& decodedComp : decodedNodeInfo->components()) + { + BOOST_CHECK(components.count(decodedComp)); + } +} + +void testNodeInfoEncodeDecode(INodeInfoFactory::Ptr factory, INodeInfo::Ptr nodeInfo) +{ + bcos::bytes encodedData; + nodeInfo->encode(encodedData); + + auto decodedNodeInfo = factory->build(bcos::ref(encodedData)); + checkNodeInfo(nodeInfo, decodedNodeInfo); +} + +void registerNode(GatewayNodeInfoImpl::Ptr gatewayNodeInfo, int nodeSize) +{ + auto nodeInfoFactory = std::make_shared(); + std::string nodeID = "testn+NodeID"; + std::string endPoint = "testEndpoint"; + for (int i = 0; i < nodeSize; i++) + { + std::set components; + for (int j = 0; j < 100; j++) + { + components.insert("component_" + std::to_string(i) + "_" + std::to_string(j)); + } + auto populatedNodeID = nodeID + std::to_string(i); + auto nodeInfo = fakeNodeInfo(nodeInfoFactory, populatedNodeID, endPoint, components); + testNodeInfoEncodeDecode(nodeInfoFactory, nodeInfo); + gatewayNodeInfo->tryAddNodeInfo(nodeInfo); + } +} + +GatewayNodeInfo::Ptr fakeGatewayNodeInfo(GatewayNodeInfoFactory::Ptr factory, uint32_t statusSeq) +{ + auto gatewayNodeInfo = factory->build(); + gatewayNodeInfo->setStatusSeq(statusSeq); + return gatewayNodeInfo; +} + +void checkGatewayNodeInfo(GatewayNodeInfoImpl::Ptr gatewayNodeInfo, int expectedNodeSize, + int expectedStatusSeq, std::string const& expectedAgency, std::string const& expectedP2pNode) +{ + BOOST_CHECK(gatewayNodeInfo->nodeSize() == expectedNodeSize); + BOOST_CHECK(gatewayNodeInfo->statusSeq() == expectedStatusSeq); + BOOST_CHECK(gatewayNodeInfo->agency() == expectedAgency); + BOOST_CHECK(gatewayNodeInfo->p2pNodeID() == expectedP2pNode); + + // check concurrency + tbb::parallel_for(tbb::blocked_range(0U, 5), [&](auto const& range) { + bcos::bytes dataTemp; + gatewayNodeInfo->encode(dataTemp); + }); + bcos::bytes data; + gatewayNodeInfo->encode(data); + + auto decodedNodeInfo = std::make_shared(bcos::ref(data)); + BOOST_CHECK(gatewayNodeInfo->nodeSize() == decodedNodeInfo->nodeSize()); + BOOST_CHECK(gatewayNodeInfo->statusSeq() == decodedNodeInfo->statusSeq()); + BOOST_CHECK(gatewayNodeInfo->agency() == decodedNodeInfo->agency()); + BOOST_CHECK(gatewayNodeInfo->p2pNodeID() == decodedNodeInfo->p2pNodeID()); + + auto nodeList = decodedNodeInfo->nodeList(); + auto originNodeList = gatewayNodeInfo->nodeList(); + for (auto const& it : nodeList) + { + auto decodedNodeInfo = it.second; + auto originNodeInfo = originNodeList.at(it.first); + checkNodeInfo(originNodeInfo, decodedNodeInfo); + } +} + +BOOST_AUTO_TEST_CASE(testGatewayNodeInfo) +{ + std::string agency = "agency"; + std::string p2pNode = "p2p23"; + int statusSeq = 123343234234; + auto factory = std::make_shared(p2pNode, agency); + auto gatewayNodeInfo = + std::dynamic_pointer_cast(fakeGatewayNodeInfo(factory, statusSeq)); + registerNode(gatewayNodeInfo, 10); + checkGatewayNodeInfo(gatewayNodeInfo, 10, statusSeq, agency, p2pNode); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h b/cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h index a974faed..73c7e0f0 100644 --- a/cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h +++ b/cpp/wedpr-transport/ppc-gateway/test/unittests/MockCache.h @@ -21,66 +21,65 @@ #include "ppc-framework/storage/CacheStorage.h" #include -namespace ppc::mock { -class MockCache : public storage::CacheStorage { +namespace ppc::mock +{ +class MockCache : public storage::CacheStorage +{ public: - using Ptr = std::shared_ptr; - MockCache() = default; - ~MockCache() override {} + using Ptr = std::shared_ptr; + MockCache() = default; + ~MockCache() override {} - /// Note: all these interfaces throws exception when error happened - /** - * @brief: check whether the key exists - * @param _key: key - * @return whether the key exists - */ - bool exists(const std::string &_key) override { - return m_kv.find(_key) != m_kv.end(); - } + /// Note: all these interfaces throws exception when error happened + /** + * @brief: check whether the key exists + * @param _key: key + * @return whether the key exists + */ + bool exists(const std::string& _key) override { return m_kv.find(_key) != m_kv.end(); } - /** - * @brief: set key value - * @param _expirationTime: timeout of key, seconds - */ - void setValue(const std::string &_key, const std::string &_value, - int32_t _expirationSeconds = -1) override { - m_kv.emplace(_key, _value); - } - - /** - * @brief: get value by key - * @param _key: key - * @return value - */ - std::optional getValue(const std::string &_key) override { - auto it = m_kv.find(_key); - if (it == m_kv.end()) { - return std::nullopt; + /** + * @brief: set key value + * @param _expirationTime: timeout of key, seconds + */ + void setValue(const std::string& _key, const std::string& _value, + int32_t _expirationSeconds = -1) override + { + m_kv.emplace(_key, _value); } - return it->second; - } + /** + * @brief: get value by key + * @param _key: key + * @return value + */ + std::optional getValue(const std::string& _key) override + { + auto it = m_kv.find(_key); + if (it == m_kv.end()) + { + return std::nullopt; + } + + return it->second; + } - /** - * @brief: set a timeout on key - * @param _expirationTime: timeout of key, ms - * @return whether setting is successful - */ - bool expireKey(const std::string &_key, uint32_t _expirationTime) override { - return true; - } + /** + * @brief: set a timeout on key + * @param _expirationTime: timeout of key, ms + * @return whether setting is successful + */ + bool expireKey(const std::string& _key, uint32_t _expirationTime) override { return true; } - /** - * @brief: delete key - * @param _key: key - * @return the number of key deleted - */ - uint64_t deleteKey(const std::string &_key) override { - return m_kv.erase(_key); - } + /** + * @brief: delete key + * @param _key: key + * @return the number of key deleted + */ + uint64_t deleteKey(const std::string& _key) override { return m_kv.erase(_key); } private: - std::unordered_map> m_kv; + std::unordered_map> m_kv; }; -} // namespace ppc::mock \ No newline at end of file +} // namespace ppc::mock \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp index d665afe1..a1db0739 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp @@ -31,7 +31,7 @@ Rpc::Ptr RpcFactory::buildRpc(ppc::tools::PPCConfig::ConstPtr _config) { auto wsConfig = initConfig(_config); // create the wsConfig - auto wsService = std::make_shared(); + auto wsService = std::make_shared("WeDPR-RPC"); wsService->setTimerFactory(std::make_shared()); auto initializer = std::make_shared(); diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/TransportBuilder.cpp index 6550efb6..cec1a6aa 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.cpp +++ b/cpp/wedpr-transport/sdk/TransportBuilder.cpp @@ -23,7 +23,7 @@ #include "Transport.h" #include "TransportImpl.h" #include "ppc-front/FrontConfigImpl.h" -#include "protobuf/NodeInfoImpl.h" +#include "protobuf/src/NodeInfoImpl.h" #include using namespace ppc::sdk; diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/TransportImpl.h index c32e20d3..a400b416 100644 --- a/cpp/wedpr-transport/sdk/TransportImpl.h +++ b/cpp/wedpr-transport/sdk/TransportImpl.h @@ -21,7 +21,7 @@ #include "Transport.h" #include "ppc-framework/gateway/IGateway.h" #include "ppc-front/FrontFactory.h" -#include "protobuf/NodeInfoImpl.h" +#include "protobuf/src/NodeInfoImpl.h" #include "protocol/src/v1/MessageHeaderImpl.h" #include "protocol/src/v1/MessagePayloadImpl.h" From 7db7ad00ef3eca74eeeef40a2f041bd2cdd8cb1b Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 6 Sep 2024 23:03:29 +0800 Subject: [PATCH 028/120] optimize log && fix ut (#25) * optimize log * fix ut --- cpp/cmake/IncludeDirectories.cmake | 1 + cpp/ppc-framework/Common.h | 5 + cpp/ppc-framework/front/IFront.h | 1 + cpp/ppc-framework/protocol/INodeInfo.h | 4 +- .../ppc-cem/tests/TestCEMService.cpp | 2 +- .../ppc-cem/tests/data/config.ini | 2 +- .../ppc-mpc/tests/TestMPCService.cpp | 2 +- .../ppc-pir/tests/FakeOtPIRFactory.h | 70 ++-- .../ppc-pir/tests/TestBaseOT.cpp | 384 +++++++++--------- cpp/wedpr-computing/ppc-pir/tests/test.json | 2 +- cpp/wedpr-computing/ppc-psi/src/PSIConfig.h | 7 + .../tests/bs-ecdh-psi/TestBsEcdhCache.cpp | 7 +- .../tests/cm2020-psi/TestCM2020Impl.cpp | 66 +-- .../tests/labeled-psi/TestLabeledPSIImpl.cpp | 15 +- .../tests/ra2018-psi/TestEcdhPSIImpl.cpp | 21 +- .../tests/ra2018-psi/TestRA2018Impl.cpp | 15 +- .../ppc-io/tests/TestLineReader.cpp | 6 +- cpp/wedpr-transport/ppc-front/CMakeLists.txt | 10 +- .../ppc-front/test/CMakeLists.txt | 2 +- .../ppc-gateway/ppc-gateway/Common.h | 3 + .../ppc-gateway/gateway/GatewayImpl.cpp | 32 +- .../gateway/router/GatewayRouterManager.cpp | 42 +- .../gateway/router/LocalRouter.cpp | 8 +- .../gateway/router/PeerRouterTable.cpp | 8 +- .../ppc-gateway/test/CMakeLists.txt | 2 +- 25 files changed, 400 insertions(+), 317 deletions(-) diff --git a/cpp/cmake/IncludeDirectories.cmake b/cpp/cmake/IncludeDirectories.cmake index 5d1643be..4c0e78d4 100644 --- a/cpp/cmake/IncludeDirectories.cmake +++ b/cpp/cmake/IncludeDirectories.cmake @@ -7,6 +7,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-computing) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-crypto) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-storage) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-helper) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-crypto/sdk) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/ppc-front) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/ppc-gateway) diff --git a/cpp/ppc-framework/Common.h b/cpp/ppc-framework/Common.h index 2ae44dfd..077a1969 100644 --- a/cpp/ppc-framework/Common.h +++ b/cpp/ppc-framework/Common.h @@ -77,6 +77,11 @@ inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC); } +template +inline std::string printNodeID(T const& nodeID) +{ + return std::string(nodeID.begin(), nodeID.begin() + 8); +} #if ENABLE_CPU_FEATURES #if X86 diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 8de9ab09..27e9bfe0 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -123,6 +123,7 @@ class IFront : virtual public IFrontClient * @param topic the topic to unregister */ virtual void unRegisterTopic(std::string const& topic) = 0; + }; class IFrontBuilder diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 46b29a8d..c8789773 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -18,6 +18,7 @@ * @date 2024-08-26 */ #pragma once +#include "ppc-framework/Common.h" #include #include #include @@ -77,7 +78,8 @@ inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo) return "nullptr"; } std::ostringstream stringstream; - stringstream << LOG_KV("endPoint", nodeInfo->endPoint()); + stringstream << LOG_KV("endPoint", nodeInfo->endPoint()) + << LOG_KV("nodeID", printNodeID(nodeInfo->nodeID())); std::string components = ""; for (auto const& it : nodeInfo->components()) { diff --git a/cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp b/cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp index cbcfc668..5214af38 100644 --- a/cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp +++ b/cpp/wedpr-computing/ppc-cem/tests/TestCEMService.cpp @@ -33,7 +33,7 @@ BOOST_FIXTURE_TEST_SUITE(cMServiceTest, TestPromptFixture) BOOST_AUTO_TEST_CASE(testCEMService) { auto cemService = std::make_shared(); - std::string configPath{"../../../ppc-cem/tests/data/config.ini"}; + std::string configPath{"../../../../wedpr-computing/ppc-cem/tests/data/config.ini"}; boost::property_tree::ptree pt; boost::property_tree::read_ini(configPath, pt); // load the cm config diff --git a/cpp/wedpr-computing/ppc-cem/tests/data/config.ini b/cpp/wedpr-computing/ppc-cem/tests/data/config.ini index c5a93e32..5cf1c6f8 100644 --- a/cpp/wedpr-computing/ppc-cem/tests/data/config.ini +++ b/cpp/wedpr-computing/ppc-cem/tests/data/config.ini @@ -6,7 +6,7 @@ [cem] use_mysql = false dataset_hdfs_path = /user/ppc/admin - dataset_file_path = ../../../ppc-cem/tests/data + dataset_file_path = ../../../../wedpr-computing/ppc-cem/tests/data ciphertext_suffix = -encrypted read_per_batch_lines = 100000 diff --git a/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp b/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp index 51277660..75e427cc 100644 --- a/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp +++ b/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp @@ -34,7 +34,7 @@ BOOST_FIXTURE_TEST_SUITE(MPCServiceTest, TestPromptFixture) BOOST_AUTO_TEST_CASE(testMPCService) { auto mpcService = std::make_shared(); - std::string configPath{"../../../ppc-mpc/tests/data/config.ini"}; + std::string configPath{"../../../../wedpr-computing/ppc-mpc/tests/data/config.ini"}; boost::property_tree::ptree pt; boost::property_tree::read_ini(configPath, pt); // load the mpc config diff --git a/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h b/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h index c38a1b71..23ee2179 100644 --- a/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h +++ b/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h @@ -38,51 +38,55 @@ using namespace ppc::io; using namespace ppc::front; using namespace ppc::tools; -namespace ppc::test { - -class FakeOtPIRImpl : public OtPIRImpl { +namespace ppc::test +{ +class FakeOtPIRImpl : public OtPIRImpl +{ public: - using Ptr = std::shared_ptr; - FakeOtPIRImpl(OtPIRConfig::Ptr const &_config, unsigned _idleTimeMs = 0) - : OtPIRImpl(_config, _idleTimeMs) { - m_enableOutputExists = true; - } - ~FakeOtPIRImpl() override = default; + using Ptr = std::shared_ptr; + FakeOtPIRImpl(OtPIRConfig::Ptr const& _config, unsigned _idleTimeMs = 0) + : OtPIRImpl(_config, _idleTimeMs) + { + m_enableOutputExists = true; + } + ~FakeOtPIRImpl() override = default; }; -class FakeOtPIRFactory : public OtPIRFactory { +class FakeOtPIRFactory : public OtPIRFactory +{ public: - using Ptr = std::shared_ptr; + using Ptr = std::shared_ptr; - FakeOtPIRFactory() + FakeOtPIRFactory() : m_front(std::make_shared()), m_dataResourceLoader(std::make_shared( nullptr, nullptr, nullptr, nullptr, nullptr, nullptr)), - m_threadPool(std::make_shared("ot-pir", 4)) { - auto hashImpl = std::make_shared(); - auto eccCrypto = std::make_shared( - hashImpl, ppc::protocol::ECCCurve::P256); - m_cryptoBox = std::make_shared(hashImpl, eccCrypto); - } + m_threadPool(std::make_shared("ot-pir", 4)) + { + auto hashImpl = std::make_shared(); + auto eccCrypto = + std::make_shared(hashImpl, ppc::protocol::ECCCurve::P256); + m_cryptoBox = std::make_shared(hashImpl, eccCrypto); + } - ~FakeOtPIRFactory() override = default; + ~FakeOtPIRFactory() override = default; - OtPIRImpl::Ptr createOtPIR(std::string const &_selfParty) { - auto config = - std::make_shared(_selfParty, m_front, m_cryptoBox, - m_threadPool, m_dataResourceLoader, 1); + OtPIRImpl::Ptr createOtPIR(std::string const& _selfParty) + { + auto config = std::make_shared( + _selfParty, m_front, m_cryptoBox, m_threadPool, m_dataResourceLoader, 1); - return std::make_shared(config); - } + return std::make_shared(config); + } - DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } - FakeFront::Ptr front() { return m_front; } - CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } + DataResourceLoaderImpl::Ptr resourceLoader() { return m_dataResourceLoader; } + FakeFront::Ptr front() { return m_front; } + CryptoBox::Ptr cryptoBox() { return m_cryptoBox; } private: - FakeFront::Ptr m_front; - DataResourceLoaderImpl::Ptr m_dataResourceLoader; - ThreadPool::Ptr m_threadPool; - CryptoBox::Ptr m_cryptoBox; + FakeFront::Ptr m_front; + DataResourceLoaderImpl::Ptr m_dataResourceLoader; + ThreadPool::Ptr m_threadPool; + CryptoBox::Ptr m_cryptoBox; }; -} // namespace ppc::test \ No newline at end of file +} // namespace ppc::test \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp index 9f5477fb..b76d867b 100644 --- a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp +++ b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp @@ -37,216 +37,216 @@ using namespace ppc::pir; using namespace bcos; using namespace bcos::test; -namespace ppc::test { +namespace ppc::test +{ BOOST_FIXTURE_TEST_SUITE(OtPIRest, TestPromptFixture) -BOOST_AUTO_TEST_CASE(testBaseOT) { - // 统计函数执行时间 - std::cout << "testBaseOT" << std::endl; - - auto eccFactory = std::make_shared(); - auto hashFactory = std::make_shared(); - auto hash = - hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512); - EccCrypto::Ptr ecc = - eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hash); - // auto aysService = std::make_shared(); - // auto aes = std::make_shared( - // OpenSSLAES::AESType::AES128, - // SymCrypto::OperationMode::CBC, _seed, bcos::bytes()); - - std::string datasetPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; - // std::cout<< "aysService->prepareDataset" << std::endl; - std::string prefix = "testmsg1"; - bcos::bytes sendObfuscatedHash(prefix.begin(), prefix.end()); - - uint32_t obfuscatedOrder = 10; - auto baseOT = std::make_shared(ecc, hash); - auto messageKeypair = baseOT->prepareDataset(sendObfuscatedHash, datasetPath); - // for(auto iter = messageKeypair.begin(); iter != messageKeypair.end(); - // ++iter) - // { - // std::string pairKey(iter->first.begin(), iter->first.end()); - // std::string pairValue(iter->second.begin(), iter->second.end()); - // // for(uint32_t i = 0; i < messageKeypair.size(); ++i) { - // // std::cout<< "pairKey:"<< pairKey << std::endl; - // // std::cout<< "pairValue:"<< pairValue << std::endl; - // } - auto start = std::chrono::high_resolution_clock::now(); - - std::string choice = "testmsg1100"; - // std::cout<< "baseOT->senderGenerateCipher" << std::endl; - auto senderMessage = baseOT->senderGenerateCipher( - bcos::bytes(choice.begin(), choice.end()), obfuscatedOrder); - // std::cout<< "baseOT->receiverGenerateMessage" << std::endl; - auto receiverMessage = baseOT->receiverGenerateMessage( - senderMessage.pointX, senderMessage.pointY, messageKeypair, - senderMessage.pointZ); - - // std::cout<< "baseOT->finishSender" << std::endl; - auto result = baseOT->finishSender( - senderMessage.scalarBlidingB, receiverMessage.pointWList, - receiverMessage.encryptMessagePair, receiverMessage.encryptCipher); - - auto end = std::chrono::high_resolution_clock::now(); - - auto duration = - std::chrono::duration_cast(end - start); - std::cout << "执行时间 time: " << duration.count() << " microseconds" - << std::endl; - - if (result.size() == 0) { - std::cout << "final result: message not found" << std::endl; - } else { - std::cout << "final result: " << std::string(result.begin(), result.end()) - << std::endl; - } - // for(uint32_t i = 0; i < result.size(); ++i) - // { - // std::cout<< std::string(result[i].begin(), result[i].end()) << - // std::endl; - // } - - BOOST_CHECK(true); +BOOST_AUTO_TEST_CASE(testBaseOT) +{ + // 统计函数执行时间 + std::cout << "testBaseOT" << std::endl; + + auto eccFactory = std::make_shared(); + auto hashFactory = std::make_shared(); + auto hash = hashFactory->createHashImpl((int8_t)ppc::protocol::HashImplName::SHA512); + EccCrypto::Ptr ecc = eccFactory->createEccCrypto((int8_t)ppc::protocol::ECCCurve::P256, hash); + // auto aysService = std::make_shared(); + // auto aes = std::make_shared( + // OpenSSLAES::AESType::AES128, + // SymCrypto::OperationMode::CBC, _seed, bcos::bytes()); + + std::string datasetPath = "../../../../wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv"; + // std::cout<< "aysService->prepareDataset" << std::endl; + std::string prefix = "testmsg1"; + bcos::bytes sendObfuscatedHash(prefix.begin(), prefix.end()); + + uint32_t obfuscatedOrder = 10; + auto baseOT = std::make_shared(ecc, hash); + auto messageKeypair = baseOT->prepareDataset(sendObfuscatedHash, datasetPath); + // for(auto iter = messageKeypair.begin(); iter != messageKeypair.end(); + // ++iter) + // { + // std::string pairKey(iter->first.begin(), iter->first.end()); + // std::string pairValue(iter->second.begin(), iter->second.end()); + // // for(uint32_t i = 0; i < messageKeypair.size(); ++i) { + // // std::cout<< "pairKey:"<< pairKey << std::endl; + // // std::cout<< "pairValue:"<< pairValue << std::endl; + // } + auto start = std::chrono::high_resolution_clock::now(); + + std::string choice = "testmsg1100"; + // std::cout<< "baseOT->senderGenerateCipher" << std::endl; + auto senderMessage = + baseOT->senderGenerateCipher(bcos::bytes(choice.begin(), choice.end()), obfuscatedOrder); + // std::cout<< "baseOT->receiverGenerateMessage" << std::endl; + auto receiverMessage = baseOT->receiverGenerateMessage( + senderMessage.pointX, senderMessage.pointY, messageKeypair, senderMessage.pointZ); + + // std::cout<< "baseOT->finishSender" << std::endl; + auto result = baseOT->finishSender(senderMessage.scalarBlidingB, receiverMessage.pointWList, + receiverMessage.encryptMessagePair, receiverMessage.encryptCipher); + + auto end = std::chrono::high_resolution_clock::now(); + + auto duration = std::chrono::duration_cast(end - start); + std::cout << "执行时间 time: " << duration.count() << " microseconds" << std::endl; + + if (result.size() == 0) + { + std::cout << "final result: message not found" << std::endl; + } + else + { + std::cout << "final result: " << std::string(result.begin(), result.end()) << std::endl; + } + // for(uint32_t i = 0; i < result.size(); ++i) + // { + // std::cout<< std::string(result[i].begin(), result[i].end()) << + // std::endl; + // } + + BOOST_CHECK(true); } -void testOTPIR(FakeOtPIRFactory::Ptr _factory, OtPIRImpl::Ptr _sender, - OtPIRImpl::Ptr _receiver, - ppc::protocol::Task::ConstPtr _senderPirTask, - ppc::protocol::Task::ConstPtr _receiverPirTask, - std::vector const &_expectedPIRResult, - bool _expectedSuccess, int _expectedErrorCode = 0) { - std::atomic flag = 0; - - _sender->asyncRunTask( - _senderPirTask, [_senderPirTask, _expectedSuccess, _expectedErrorCode, - &flag](ppc::protocol::TaskResult::Ptr &&_response) { - if (_expectedSuccess) { - BOOST_CHECK(_response->error() == nullptr || - _response->error()->errorCode() == 0); - BOOST_CHECK(_response->taskID() == _senderPirTask->id()); - auto result = _response->error(); - BOOST_CHECK(result == nullptr || result->errorCode() == 0); - } else { - BOOST_CHECK(_response->error() != nullptr); - auto result = _response->error(); - BOOST_CHECK(result != nullptr); - BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); +void testOTPIR(FakeOtPIRFactory::Ptr _factory, OtPIRImpl::Ptr _sender, OtPIRImpl::Ptr _receiver, + ppc::protocol::Task::ConstPtr _senderPirTask, ppc::protocol::Task::ConstPtr _receiverPirTask, + std::vector const& _expectedPIRResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + std::atomic flag = 0; + + _sender->asyncRunTask(_senderPirTask, [_senderPirTask, _expectedSuccess, _expectedErrorCode, + &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _senderPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); } - flag++; - }); - _sender->start(); - - _receiver->asyncRunTask( - _receiverPirTask, [_receiverPirTask, _expectedSuccess, - &flag](ppc::protocol::TaskResult::Ptr &&_response) { - if (_expectedSuccess) { - BOOST_CHECK(_response->error() == nullptr || - _response->error()->errorCode() == 0); - BOOST_CHECK(_response->taskID() == _receiverPirTask->id()); - auto result = _response->error(); - BOOST_CHECK(result == nullptr || result->errorCode() == 0); - } else { - BOOST_CHECK(_response->error() != nullptr); - auto result = _response->error(); - BOOST_CHECK(result != nullptr); + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + BOOST_CHECK(_response->error()->errorCode() == _expectedErrorCode); } flag++; - }); - _receiver->start(); - - // wait for the task finish and check - while (flag < 2) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - _sender->stop(); - _receiver->stop(); - - // if (_expectedSuccess && !_expectedPIRResult.empty()) - // { - // checkTaskPIRResult(_factory->resourceLoader(), _receiverPirTask, - // _expectedPIRResult.size(), - // _expectedPIRResult); - // } + }); + _sender->start(); + + _receiver->asyncRunTask(_receiverPirTask, + [_receiverPirTask, _expectedSuccess, &flag](ppc::protocol::TaskResult::Ptr&& _response) { + if (_expectedSuccess) + { + BOOST_CHECK(_response->error() == nullptr || _response->error()->errorCode() == 0); + BOOST_CHECK(_response->taskID() == _receiverPirTask->id()); + auto result = _response->error(); + BOOST_CHECK(result == nullptr || result->errorCode() == 0); + } + else + { + BOOST_CHECK(_response->error() != nullptr); + auto result = _response->error(); + BOOST_CHECK(result != nullptr); + } + flag++; + }); + _receiver->start(); + + // wait for the task finish and check + while (flag < 2) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + _sender->stop(); + _receiver->stop(); + + // if (_expectedSuccess && !_expectedPIRResult.empty()) + // { + // checkTaskPIRResult(_factory->resourceLoader(), _receiverPirTask, + // _expectedPIRResult.size(), + // _expectedPIRResult); + // } } -void testOTPIRImplFunc(const std::string &_taskID, const std::string &_params, - bool _syncResults, PartyResource::Ptr _senderParty, - PartyResource::Ptr _receiverParty, - std::vector const &_expectedPIRResult, - bool _expectedSuccess, int _expectedErrorCode = 0) { - auto factory = std::make_shared(); - - // fake the sender - std::string senderAgencyName = "sender"; - auto senderPIR = factory->createOtPIR("sender"); - - // fake the receiver - std::string receiverAgencyName = _receiverParty->id(); - auto receiverPIR = factory->createOtPIR(receiverAgencyName); - - // register the server-pir into the front - factory->front()->registerOTPIR(senderAgencyName, senderPIR); - factory->front()->registerOTPIR(receiverAgencyName, receiverPIR); - - // trigger the pir task - auto senderPIRTask = std::make_shared(senderAgencyName); - senderPIRTask->setId(_taskID); - senderPIRTask->setParam(_params); - senderPIRTask->setSelf(_senderParty); - senderPIRTask->addParty(_receiverParty); - senderPIRTask->setSyncResultToPeer(_syncResults); - senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); - senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); - - auto receiverPIRTask = std::make_shared(receiverAgencyName); - receiverPIRTask->setId(_taskID); - receiverPIRTask->setParam(_params); - receiverPIRTask->setSelf(_receiverParty); - receiverPIRTask->addParty(_senderParty); - receiverPIRTask->setSyncResultToPeer(_syncResults); - receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); - receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); - - testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, - _expectedPIRResult, _expectedSuccess, _expectedErrorCode); +void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, bool _syncResults, + PartyResource::Ptr _senderParty, PartyResource::Ptr _receiverParty, + std::vector const& _expectedPIRResult, bool _expectedSuccess, + int _expectedErrorCode = 0) +{ + auto factory = std::make_shared(); + + // fake the sender + std::string senderAgencyName = "sender"; + auto senderPIR = factory->createOtPIR("sender"); + + // fake the receiver + std::string receiverAgencyName = _receiverParty->id(); + auto receiverPIR = factory->createOtPIR(receiverAgencyName); + + // register the server-pir into the front + factory->front()->registerOTPIR(senderAgencyName, senderPIR); + factory->front()->registerOTPIR(receiverAgencyName, receiverPIR); + + // trigger the pir task + auto senderPIRTask = std::make_shared(senderAgencyName); + senderPIRTask->setId(_taskID); + senderPIRTask->setParam(_params); + senderPIRTask->setSelf(_senderParty); + senderPIRTask->addParty(_receiverParty); + senderPIRTask->setSyncResultToPeer(_syncResults); + senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + auto receiverPIRTask = std::make_shared(receiverAgencyName); + receiverPIRTask->setId(_taskID); + receiverPIRTask->setParam(_params); + receiverPIRTask->setSelf(_receiverParty); + receiverPIRTask->addParty(_senderParty); + receiverPIRTask->setSyncResultToPeer(_syncResults); + receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC); + receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR); + + testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, _expectedPIRResult, + _expectedSuccess, _expectedErrorCode); } -BOOST_AUTO_TEST_CASE(testNormalOtPIRCase) { - std::string otSearchPath = "../../../ppc-pir/tests/data/AysPreDataset.csv"; - std::string outputPath = "../../../ppc-pir/tests/data/output.csv"; +BOOST_AUTO_TEST_CASE(testNormalOtPIRCase) +{ + std::string otSearchPath = "../../../../wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv"; + std::string outputPath = "../../../../wedpr-computing/ppc-pir/tests/data/output.csv"; - uint32_t count = 513; - // prepareInputs(senderPath, count, receiverPath, count, count); + uint32_t count = 513; + // prepareInputs(senderPath, count, receiverPath, count, count); - auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, - "sender", "senderPartyResource", "sender_inputs", - protocol::DataResourceType::FILE, ""); - auto senderOutputDesc = std::make_shared(); - senderOutputDesc->setPath(outputPath); - senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); + auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Client, "sender", + "senderPartyResource", "sender_inputs", protocol::DataResourceType::FILE, ""); + auto senderOutputDesc = std::make_shared(); + senderOutputDesc->setPath(outputPath); + senderParty->mutableDataResource()->setOutputDesc(senderOutputDesc); - auto receiverParty = - mockParty((uint16_t)ppc::protocol::PartyType::Server, "receiver", - "receiverPartyResource", "receiver_inputs", - DataResourceType::FILE, otSearchPath); + auto receiverParty = mockParty((uint16_t)ppc::protocol::PartyType::Server, "receiver", + "receiverPartyResource", "receiver_inputs", DataResourceType::FILE, otSearchPath); - // auto receiverOutputDesc = std::make_shared(); - // receiverOutputDesc->setPath(outputPath); - // receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); + // auto receiverOutputDesc = std::make_shared(); + // receiverOutputDesc->setPath(outputPath); + // receiverParty->mutableDataResource()->setOutputDesc(receiverOutputDesc); - std::vector expectedResult; - for (uint32_t i = 0; i < count; i++) { - expectedResult.emplace_back(std::to_string(100000 + i)); - } + std::vector expectedResult; + for (uint32_t i = 0; i < count; i++) + { + expectedResult.emplace_back(std::to_string(100000 + i)); + } - std::string jobParams = "{\"searchId\":\"testmsg1100\",\"requestAgencyId\":" - "\"receiver\",\"prefixLength\":6}"; + std::string jobParams = + "{\"searchId\":\"testmsg1100\",\"requestAgencyId\":" + "\"receiver\",\"prefixLength\":6}"; - testOTPIRImplFunc("0x12345678", jobParams, true, senderParty, receiverParty, - expectedResult, true, 0); + testOTPIRImplFunc( + "0x12345678", jobParams, true, senderParty, receiverParty, expectedResult, true, 0); } BOOST_AUTO_TEST_SUITE_END() -} // namespace ppc::test +} // namespace ppc::test diff --git a/cpp/wedpr-computing/ppc-pir/tests/test.json b/cpp/wedpr-computing/ppc-pir/tests/test.json index 08df6b4d..37bf327b 100644 --- a/cpp/wedpr-computing/ppc-pir/tests/test.json +++ b/cpp/wedpr-computing/ppc-pir/tests/test.json @@ -1,6 +1,6 @@ { "searchId": "testmsg1100", "requestAgencyId": "1002", - "requestAgencyDataset": "../../../ppc-pir/tests/data/AysPreDataset.csv", + "requestAgencyDataset": "../../../../wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv", "prefixLength": 6 } \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h index 7ee7dd80..72ee8bbe 100644 --- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h +++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h @@ -120,6 +120,13 @@ class PSIConfig return m_agencyList; } + // for ut + void setAgencyList(std::vector const& agencyList) + { + bcos::WriteGuard l(x_agencyList); + m_agencyList = agencyList; + } + protected: ppc::front::PPCMessageFace::Ptr generatePPCMsg( std::string const& _taskID, PSIMessageInterface::Ptr const& _msg, uint32_t _seq) diff --git a/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp b/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp index 82409479..05eb185d 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp @@ -83,9 +83,10 @@ BOOST_AUTO_TEST_CASE(testEcdhEnc) BOOST_AUTO_TEST_CASE(testBsEcdh) { - auto inputPath = "../../../ppc-psi/tests/bs-ecdh-psi/data/inputs.csv"; - auto outputPath = "../../../ppc-psi/tests/bs-ecdh-psi/data/outputs.csv"; - boost::filesystem::create_directory("../../../ppc-psi/tests/bs-ecdh-psi/data"); + auto inputPath = "../../../../wedpr-computing/ppc-psi/tests/bs-ecdh-psi/data/inputs.csv"; + auto outputPath = "../../../../../wedpr-computing/ppc-psi/tests/bs-ecdh-psi/data/outputs.csv"; + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/bs-ecdh-psi/data"); auto count = 123456; prepareInputs(inputPath, count); diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp index 95ff82e3..bb9f5bf6 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp @@ -169,13 +169,18 @@ void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _param BOOST_AUTO_TEST_CASE(testNormalCM2020PSICase) { - std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; - std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; - std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; - std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + std::string senderPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; uint32_t count = 513; - boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data"); prepareInputs(senderPath, count, receiverPath, count, count); auto senderParty = mockParty((uint16_t)ppc::protocol::PartyType::Server, "sender", @@ -208,14 +213,19 @@ BOOST_AUTO_TEST_CASE(testNormalCM2020PSICase) BOOST_AUTO_TEST_CASE(testBigDataCM2020PSICase) { - std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; - std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; - std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; - std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + std::string senderPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; uint32_t count = 1000000, common = 100; - boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data"); prepareInputs(senderPath, count, receiverPath, count, common); auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", @@ -242,18 +252,23 @@ BOOST_AUTO_TEST_CASE(testBigDataCM2020PSICase) BOOST_AUTO_TEST_CASE(testErrorCM2020PSICase) { - std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; - std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; - std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; - std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + std::string senderPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; uint32_t count = 1234; - boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data"); prepareInputs(senderPath, count, receiverPath, count, count); - auto senderParty = - mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", "sender_inputs", - DataResourceType::FILE, "../../../ppc-psi/tests/cm2020-psi/data/not_existed.csv"); + auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", + "sender_inputs", DataResourceType::FILE, + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/not_existed.csv"); auto senderOutputDesc = std::make_shared(); senderOutputDesc->setPath(senderOutputPath); @@ -271,13 +286,18 @@ BOOST_AUTO_TEST_CASE(testErrorCM2020PSICase) BOOST_AUTO_TEST_CASE(testRawDataCM2020PSICase) { - std::string senderPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; - std::string receiverPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; - std::string senderOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/sender_out.csv"; - std::string receiverOutputPath = "../../../ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; + std::string senderPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_inputs.csv"; + std::string receiverPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_inputs.csv"; + std::string senderOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/sender_out.csv"; + std::string receiverOutputPath = + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data/receiver_out.csv"; uint32_t count = 1234; - boost::filesystem::create_directory("../../../ppc-psi/tests/cm2020-psi/data"); + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/cm2020-psi/data"); prepareInputs(senderPath, count, receiverPath, count, count); auto reader = std::make_shared(senderPath); diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp index 77b1a7ac..3a030eae 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp @@ -42,10 +42,12 @@ BOOST_FIXTURE_TEST_SUITE(LabeledPSIImplTest, TestPromptFixture) void runSetup(LabeledPSIImpl::Ptr _sender) { - std::string senderPath = "../../../ppc-psi/tests/labeled-psi/data/sender.csv"; + std::string senderPath = + "../../../../wedpr-computing/ppc-psi/tests/labeled-psi/data/sender.csv"; uint32_t count = 10000; - boost::filesystem::create_directory("../../../ppc-psi/tests/labeled-psi/data"); + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/labeled-psi/data"); prepareItemsAndLabels(senderPath, count); auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", @@ -77,8 +79,10 @@ void runSetup(LabeledPSIImpl::Ptr _sender) void saveCache(LabeledPSIImpl::Ptr _sender) { - std::string cachePath = "../../../ppc-psi/tests/labeled-psi/data/sender_cache.txt"; - boost::filesystem::create_directory("../../../ppc-psi/tests/labeled-psi/data"); + std::string cachePath = + "../../../../wedpr-computing/ppc-psi/tests/labeled-psi/data/sender_cache.txt"; + boost::filesystem::create_directory( + "../../../../wedpr-computing/ppc-psi/tests/labeled-psi/data"); auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", "sender_inputs", DataResourceType::FILE, ""); @@ -111,7 +115,8 @@ void saveCache(LabeledPSIImpl::Ptr _sender) void loadCache(LabeledPSIImpl::Ptr _sender) { - std::string cachePath = "../../../ppc-psi/tests/labeled-psi/data/sender_cache.txt"; + std::string cachePath = + "../../../../wedpr-computing/ppc-psi/tests/labeled-psi/data/sender_cache.txt"; auto senderParty = mockParty(uint16_t(PartyType::Server), "sender", "senderPartyResource", "sender_inputs", DataResourceType::FILE, cachePath); diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp index 766aa7df..a7df31a2 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp @@ -61,8 +61,8 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS auto clientPSI = factory->createEcdhPSI(clientAgencyName, clientConfig); std::vector agencyList = {serverAgencyName, clientAgencyName}; - // serverPSI->psiConfig()->updateAgenyList(agencyList); - // clientPSI->psiConfig()->updateAgenyList(agencyList); + serverPSI->psiConfig()->setAgencyList(agencyList); + clientPSI->psiConfig()->setAgencyList(agencyList); // register the server-psi into the front factory->front()->registerEcdhPSI(serverAgencyName, serverPSI); @@ -109,8 +109,10 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS void testNormalCase(std::string const& _outputPrefix, int64_t _dataBatchSize) { - std::string serverDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; - std::string clientDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string serverDataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string clientDataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; std::string outputPath = _outputPrefix + "psiResult.txt"; std::vector expectedResult; for (int i = 1; i < 49; i++) @@ -135,8 +137,9 @@ BOOST_AUTO_TEST_CASE(testNormalEcdhPSICase) BOOST_AUTO_TEST_CASE(testECDHMissingResourceDataCase) { std::string serverPSIDataPath = - "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate-missing.txt"; - std::string clientPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate-missing.txt"; + std::string clientPSIDataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; std::string outputPath = "psiResult_MissingResourceData.txt"; std::vector expectedResult; testEcdhImplFunc(1000, serverPSIDataPath, clientPSIDataPath, outputPath, true, false, @@ -144,8 +147,10 @@ BOOST_AUTO_TEST_CASE(testECDHMissingResourceDataCase) } BOOST_AUTO_TEST_CASE(testECDHMisMatchTaskID) { - std::string serverPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; - std::string clientPSIDataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string serverPSIDataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string clientPSIDataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; std::string outputPath = "psiResult_MismatchTaskID.txt"; std::vector expectedResult; testEcdhImplFunc(1000, serverPSIDataPath, clientPSIDataPath, outputPath, true, false, diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp index f25504ae..074766ea 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp @@ -187,9 +187,11 @@ void testRA2018PSIImplFunc(int _dataBatchSize, CuckoofilterOption::Ptr option, void testNormalCase(std::string const& _outputPrefix, CuckoofilterOption::Ptr option, uint64_t _dataBatchSize = 10000) { - std::string dataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; - std::string psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; - std::string appendPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt"; + std::string dataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string psiPath = "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string appendPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/append-full-evaluate.txt"; std::string outputPath = _outputPrefix + "psiResult.txt"; std::vector expectedResult; for (int i = 1; i < 49; i++) @@ -200,7 +202,7 @@ void testNormalCase(std::string const& _outputPrefix, CuckoofilterOption::Ptr op expectedResult, 0, false, appendPath, std::vector()); // with psi2.txt as input - psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi2.txt"; + psiPath = "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi2.txt"; expectedResult.clear(); for (int i = 2000; i < 4000; i += 3) { @@ -224,8 +226,9 @@ BOOST_AUTO_TEST_CASE(testMisMatchResourceIDCase) option->tagBits = 32; option->maxKickOutCount = 10; option->capacity = 200000; - std::string dataPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; - std::string psiPath = "../../../ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; + std::string dataPath = + "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/fullevaluate.txt"; + std::string psiPath = "../../../../wedpr-computing/ppc-psi/tests/ra2018-psi/mock-data/psi.txt"; std::string outputPath = "psiResult_MismatchResource.txt"; std::vector expectedResult; testRA2018PSIImplFunc(10000, option, dataPath, psiPath, outputPath, true, false, expectedResult, diff --git a/cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp b/cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp index 47c0a074..2050a9c9 100644 --- a/cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp +++ b/cpp/wedpr-storage/ppc-io/tests/TestLineReader.cpp @@ -146,7 +146,7 @@ BOOST_AUTO_TEST_CASE(testDataBatch) } BOOST_AUTO_TEST_CASE(testFileLineReader) { - std::string filePath = "../../../ppc-io/tests/data/testData"; + std::string filePath = "../../../../wedpr-storage/ppc-io/tests/data/testData"; uint64_t fileTotalLine = 10000; std::vector fileData; for (uint64_t i = 1; i <= fileTotalLine; i++) @@ -173,11 +173,11 @@ BOOST_AUTO_TEST_CASE(testFileLineReader) << ", pageSize:" << pageSize << std::endl; // case4: the windows file - auto windowsFilePath = "../../../ppc-io/tests/data/windows_file.txt"; + auto windowsFilePath = "../../../../wedpr-storage/ppc-io/tests/data/windows_file.txt"; auto lineReader = factory->createLineReader(windowsFilePath, pageSize, '\n'); auto windowsDataBatch = lineReader->next(-1, DataSchema::Bytes); - auto linuxFilePath = "../../../ppc-io/tests/data/linux_file.txt"; + auto linuxFilePath = "../../../../wedpr-storage/ppc-io/tests/data/linux_file.txt"; lineReader = factory->createLineReader(linuxFilePath, pageSize, '\n'); auto linuxDataBatch = lineReader->next(-1, DataSchema::Bytes); BOOST_CHECK(windowsDataBatch->size() == linuxDataBatch->size()); diff --git a/cpp/wedpr-transport/ppc-front/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/CMakeLists.txt index ae55348b..efd94094 100644 --- a/cpp/wedpr-transport/ppc-front/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-front/CMakeLists.txt @@ -6,8 +6,8 @@ aux_source_directory(ppc-front SRCS) add_library(${FRONT_TARGET} ${SRCS}) target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) -if (TESTS) - enable_testing() - set(CTEST_OUTPUT_ON_FAILURE TRUE) - add_subdirectory(test) -endif () \ No newline at end of file +#if (TESTS) +# enable_testing() +# set(CTEST_OUTPUT_ON_FAILURE TRUE) +# add_subdirectory(test) +#endif () \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/test/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/test/CMakeLists.txt index d1b76fd6..af0aa6db 100644 --- a/cpp/wedpr-transport/ppc-front/test/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-front/test/CMakeLists.txt @@ -10,4 +10,4 @@ target_compile_options(${FRONT_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unused- target_link_libraries(${FRONT_TEST_BINARY_NAME} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) -add_test(NAME test-front WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ppc-front/test/unittests COMMAND ${FRONT_TEST_BINARY_NAME}) \ No newline at end of file +add_test(NAME test-front WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${FRONT_TEST_BINARY_NAME}) \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h index 5a81cc8c..ecf94259 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/Common.h @@ -33,6 +33,9 @@ namespace ppc::gateway #define GATEWAY_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY]" #define SERVICE_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][SERVICE]" #define SERVICE_ROUTER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][SERVICE][ROUTER]" +#define LOCAL_ROUTER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][LOCAL_ROUTER]" +#define PEER_ROUTER_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][PEER_ROUTER]" +#define ROUTER_MGR_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][ROUTER_MGR]" // HTTP HEADER DEFINE #define HEAD_TASK_ID "x-ptp-session-id" diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 49c28a70..b9b5cccd 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -63,9 +63,18 @@ void GatewayImpl::start() return; } m_running = true; - m_service->start(); - m_p2pRouterManager->start(); - m_gatewayRouterManager->start(); + if (m_service) + { + m_service->start(); + } + if (m_p2pRouterManager) + { + m_p2pRouterManager->start(); + } + if (m_gatewayRouterManager) + { + m_gatewayRouterManager->start(); + } GATEWAY_LOG(INFO) << LOG_DESC("Start gateway success"); } @@ -77,10 +86,19 @@ void GatewayImpl::stop() return; } m_running = false; - m_service->stop(); - m_p2pRouterManager->stop(); - m_gatewayRouterManager->stop(); - GATEWAY_LOG(INFO) << LOG_DESC("Stop gateway success"); + if (m_service) + { + m_service->stop(); + } + if (m_p2pRouterManager) + { + m_p2pRouterManager->stop(); + } + if (m_gatewayRouterManager) + { + m_gatewayRouterManager->stop(); + } + PEER_ROUTER_LOG(INFO) << LOG_DESC("Stop gateway success"); } void GatewayImpl::asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp index 6106e6bc..3380eaef 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp @@ -58,24 +58,24 @@ void GatewayRouterManager::start() { if (m_running) { - GATEWAY_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been started"); + ROUTER_MGR_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been started"); return; } m_running = true; m_timer->start(); - GATEWAY_LOG(INFO) << LOG_DESC("start GatewayRouterManager success"); + ROUTER_MGR_LOG(INFO) << LOG_DESC("start GatewayRouterManager success"); } void GatewayRouterManager::stop() { if (!m_running) { - GATEWAY_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been stopped"); + ROUTER_MGR_LOG(INFO) << LOG_DESC("GatewayRouterManager has already been stopped"); return; } m_running = false; m_timer->stop(); - GATEWAY_LOG(INFO) << LOG_DESC("stop GatewayRouterManager success"); + ROUTER_MGR_LOG(INFO) << LOG_DESC("stop GatewayRouterManager success"); } void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSession::Ptr session) @@ -93,9 +93,9 @@ void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSessi return; } // status changed, request for the nodeStatus - GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveNodeSeqMessage") - << LOG_KV("from", printP2PIDElegantly(from)) - << LOG_KV("statusSeq", statusSeq); + ROUTER_MGR_LOG(TRACE) + << LOG_DESC("onReceiveNodeSeqMessage: statusChanged, request the lastest nodeStatus") + << LOG_KV("node", printP2PIDElegantly(from)) << LOG_KV("statusSeq", statusSeq); m_service->asyncSendMessageByP2PNodeID( (uint16_t)GatewayPacketType::RequestNodeStatus, from, std::make_shared()); } @@ -122,7 +122,7 @@ void GatewayRouterManager::broadcastStatusSeq() auto statusSeq = boost::asio::detail::socket_ops::host_to_network_long(seq); auto payload = std::make_shared((byte*)&statusSeq, (byte*)&statusSeq + 4); message->setPayload(payload); - GATEWAY_LOG(TRACE) << LOG_DESC("broadcastStatusSeq") << LOG_KV("seq", seq); + ROUTER_MGR_LOG(TRACE) << LOG_DESC("broadcastStatusSeq") << LOG_KV("seq", seq); m_service->asyncBroadcastMessage(message); } @@ -138,12 +138,14 @@ void GatewayRouterManager::onReceiveRequestNodeStatusMsg( auto nodeStatusData = m_localRouter->generateNodeStatus(); if (!nodeStatusData) { - GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveRequestNodeStatusMsg: generate nodeInfo error") - << LOG_KV("from", printP2PIDElegantly(from)); + ROUTER_MGR_LOG(WARNING) << LOG_DESC( + "onReceiveRequestNodeStatusMsg: generate nodeInfo error") + << LOG_KV("from", printP2PIDElegantly(from)); return; } - GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveRequestNodeStatusMsg: response the latest nodeStatus") - << LOG_KV("from", printP2PIDElegantly(from)); + ROUTER_MGR_LOG(TRACE) << LOG_DESC( + "onReceiveRequestNodeStatusMsg: response the latest nodeStatus") + << LOG_KV("node", printP2PIDElegantly(from)); m_service->asyncSendMessageByP2PNodeID( (uint16_t)GatewayPacketType::ResponseNodeStatus, from, nodeStatusData); } @@ -158,10 +160,10 @@ void GatewayRouterManager::onRecvResponseNodeStatusMsg(MessageFace::Ptr msg, WsS p2pMessage->header()->srcGwNode() : session->nodeId(); - GATEWAY_LOG(INFO) << LOG_DESC("onRecvResponseNodeStatusMsg") - << LOG_KV("from", printP2PIDElegantly(from)) - << LOG_KV("statusSeq", nodeStatus->statusSeq()) - << LOG_KV("agency", nodeStatus->agency()); + ROUTER_MGR_LOG(INFO) << LOG_DESC("onRecvResponseNodeStatusMsg") + << LOG_KV("node", printP2PIDElegantly(from)) + << LOG_KV("statusSeq", nodeStatus->statusSeq()) + << LOG_KV("agency", nodeStatus->agency()); updatePeerNodeStatus(from, nodeStatus); } @@ -178,9 +180,9 @@ void GatewayRouterManager::updatePeerNodeStatus( UpgradeGuard ul(l); m_p2pID2Seq[p2pID] = statusSeq; } - GATEWAY_LOG(INFO) << LOG_DESC("updatePeerNodeStatus") - << LOG_KV("from", printP2PIDElegantly(p2pID)) - << LOG_KV("statusSeq", status->statusSeq()) - << LOG_KV("agency", status->agency()); + ROUTER_MGR_LOG(INFO) << LOG_DESC("updatePeerNodeStatus") + << LOG_KV("node", printP2PIDElegantly(p2pID)) + << LOG_KV("statusSeq", status->statusSeq()) + << LOG_KV("agency", status->agency()); m_peerRouter->updateGatewayInfo(status); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 55b3ea67..e286d2fd 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -28,11 +28,12 @@ using namespace ppc::gateway; bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo) { - GATEWAY_LOG(INFO) << LOG_DESC("registerNodeInfo") << printNodeInfo(nodeInfo); + LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo") << printNodeInfo(nodeInfo); nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); if (ret) { + LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo success") << printNodeInfo(nodeInfo); increaseSeq(); } return ret; @@ -40,7 +41,8 @@ bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo) // Note: the change of the topic will not trigger router-update void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& topic) { - GATEWAY_LOG(INFO) << LOG_DESC("registerTopic") << LOG_KV("topic", topic); + LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerTopic") << LOG_KV("topic", topic) + << LOG_KV("nodeID", printNodeID(_nodeID)); m_routerInfo->registerTopic(_nodeID.toBytes(), topic); // try to dispatch the cacheInfo if (!m_cache) @@ -65,6 +67,8 @@ void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const& // Note: the change of the topic will not trigger router-update void LocalRouter::unRegisterTopic(bcos::bytesConstRef _nodeID, std::string const& topic) { + LOCAL_ROUTER_LOG(INFO) << LOG_DESC("unRegisterTopic") << LOG_KV("topic", topic) + << LOG_KV("nodeID", printNodeID(_nodeID)); m_routerInfo->unRegisterTopic(_nodeID.toBytes(), topic); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index e5367eda..b9ccb092 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -28,6 +28,8 @@ using namespace ppc::protocol; void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) { + PEER_ROUTER_LOG(INFO) << LOG_DESC("updateGatewayInfo") + << LOG_KV("detail", printNodeStatus(gatewayInfo)); auto nodeList = gatewayInfo->nodeList(); bcos::WriteGuard l(x_mutex); for (auto const& it : nodeList) @@ -138,9 +140,9 @@ void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& m { continue; } - GATEWAY_LOG(TRACE) << LOG_DESC("asyncBroadcastMessage") - << LOG_KV("nodeID", printP2PIDElegantly(selectedNode->p2pNodeID())) - << LOG_KV("msg", printMessage(msg)); + PEER_ROUTER_LOG(TRACE) << LOG_DESC("asyncBroadcastMessage") + << LOG_KV("nodeID", printP2PIDElegantly(selectedNode->p2pNodeID())) + << LOG_KV("msg", printMessage(msg)); m_service->asyncSendMessageByNodeID(selectedNode->p2pNodeID(), msg); } } diff --git a/cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt index 8a952e2f..8b5234a0 100644 --- a/cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-gateway/test/CMakeLists.txt @@ -14,4 +14,4 @@ target_compile_options(${GATEWAY_TEST_BINARY_NAME} PRIVATE -Wno-error -Wno-unuse target_link_libraries(${GATEWAY_TEST_BINARY_NAME} ${GATEWAY_TARGET} ${FRONT_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST}) -add_test(NAME test-gateway WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ppc-gateway/test/unittests COMMAND ${GATEWAY_TEST_BINARY_NAME}) \ No newline at end of file +add_test(NAME test-gateway WORKING_DIRECTORY WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${GATEWAY_TEST_BINARY_NAME}) \ No newline at end of file From 6d72991a6bd4e152234e17a553c19a6ee1549313 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Mon, 9 Sep 2024 10:23:36 +0800 Subject: [PATCH 029/120] Feature milestone2 (#26) * fix gateway status sync logic * add health-check logic * support broadcast * add keepAlive logic for front registerNode --- cpp/ppc-framework/front/IFront.h | 5 +- cpp/ppc-framework/protocol/GrpcConfig.h | 27 +++- cpp/ppc-framework/protocol/Handler.h | 41 ++++++ .../ppc-tools/src/config/PPCConfig.cpp | 3 +- .../ppc-tools/src/config/PPCConfig.h | 3 +- cpp/wedpr-main/gateway/GatewayInitializer.cpp | 16 ++- cpp/wedpr-main/gateway/GatewayInitializer.h | 5 + cpp/wedpr-protocol/grpc/Common.h | 6 + cpp/wedpr-protocol/grpc/client/Common.h | 3 +- .../grpc/client/GatewayClient.cpp | 56 +++++--- cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 101 +++++++++++++ cpp/wedpr-protocol/grpc/client/GrpcClient.h | 24 +++- .../grpc/client/HealthCheckTimer.cpp | 135 ++++++++++++++++++ .../grpc/client/HealthCheckTimer.h | 52 +++++++ .../grpc/client/RemoteFrontBuilder.cpp | 17 ++- .../grpc/client/RemoteFrontBuilder.h | 14 +- cpp/wedpr-protocol/grpc/server/FrontServer.h | 8 ++ cpp/wedpr-protocol/grpc/server/GrpcServer.cpp | 10 +- cpp/wedpr-protocol/grpc/server/GrpcServer.h | 6 +- .../proto/pb/ServiceHealth.proto | 63 ++++++++ .../ppc-front/ppc-front/LocalFrontBuilder.h | 6 +- .../ppc-gateway/gateway/GatewayImpl.cpp | 15 +- .../gateway/router/GatewayNodeInfo.h | 4 +- .../gateway/router/LocalRouter.cpp | 6 +- .../ppc-gateway/gateway/router/LocalRouter.h | 5 +- .../gateway/router/PeerRouterTable.cpp | 18 +++ cpp/wedpr-transport/sdk/Common.h | 29 ++++ cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 41 +++++- cpp/wedpr-transport/sdk/ProTransportImpl.h | 11 +- 29 files changed, 670 insertions(+), 60 deletions(-) create mode 100644 cpp/ppc-framework/protocol/Handler.h create mode 100644 cpp/wedpr-protocol/grpc/client/GrpcClient.cpp create mode 100644 cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp create mode 100644 cpp/wedpr-protocol/grpc/client/HealthCheckTimer.h create mode 100644 cpp/wedpr-protocol/proto/pb/ServiceHealth.proto create mode 100644 cpp/wedpr-transport/sdk/Common.h diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 27e9bfe0..561723dd 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -19,6 +19,7 @@ */ #pragma once #include "FrontConfig.h" +#include "ppc-framework/protocol/Handler.h" #include "ppc-framework/protocol/INodeInfo.h" #include "ppc-framework/protocol/Message.h" #include "ppc-framework/protocol/RouteType.h" @@ -123,7 +124,6 @@ class IFront : virtual public IFrontClient * @param topic the topic to unregister */ virtual void unRegisterTopic(std::string const& topic) = 0; - }; class IFrontBuilder @@ -133,6 +133,7 @@ class IFrontBuilder IFrontBuilder() = default; virtual ~IFrontBuilder() = default; - virtual IFrontClient::Ptr buildClient(std::string endPoint) const = 0; + virtual IFrontClient::Ptr buildClient(std::string endPoint, + std::function onUnHealthHandler, bool removeHandlerOnUnhealth) const = 0; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h index 3c915855..b4dae30d 100644 --- a/cpp/ppc-framework/protocol/GrpcConfig.h +++ b/cpp/ppc-framework/protocol/GrpcConfig.h @@ -24,11 +24,26 @@ namespace ppc::protocol { -struct GrpcServerConfig +class GrpcServerConfig { - ppc::protocol::EndPoint endPoint; +public: + using Ptr = std::shared_ptr; + GrpcServerConfig() = default; + GrpcServerConfig(EndPoint endPoint, bool enableHealthCheck) + : m_endPoint(std::move(endPoint)), m_enableHealthCheck(enableHealthCheck) + {} + std::string listenEndPoint() const { return m_endPoint.listenEndPoint(); } + + void setEndPoint(EndPoint endPoint) { m_endPoint = endPoint; } + void setEnableHealthCheck(bool enableHealthCheck) { m_enableHealthCheck = enableHealthCheck; } - std::string listenEndPoint() const { return endPoint.listenEndPoint(); } + EndPoint const& endPoint() const { return m_endPoint; } + EndPoint& mutableEndPoint() { return m_endPoint; } + bool enableHealthCheck() const { return m_enableHealthCheck; } + +protected: + ppc::protocol::EndPoint m_endPoint; + bool m_enableHealthCheck = false; }; class GrpcConfig { @@ -43,7 +58,11 @@ class GrpcConfig m_loadBalancePolicy = loadBalancePolicy; } -private: + bool enableHealthCheck() const { return m_enableHealthCheck; } + void setEnableHealthCheck(bool enableHealthCheck) { m_enableHealthCheck = enableHealthCheck; } + +protected: + bool m_enableHealthCheck = false; std::string m_loadBalancePolicy = "round_robin"; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/ppc-framework/protocol/Handler.h b/cpp/ppc-framework/protocol/Handler.h new file mode 100644 index 00000000..aecf774b --- /dev/null +++ b/cpp/ppc-framework/protocol/Handler.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file EndPoint.h + * @author: yujiechen + * @date 2024-08-22 + */ + +#pragma once +#include +#include +#include + +namespace ppc::protocol +{ +struct HealthCheckHandler +{ + using Ptr = std::shared_ptr; + HealthCheckHandler(std::string const& _serviceName) : serviceName(_serviceName) {} + + std::string serviceName; + // handler used to check the health + std::function checkHealthHandler; + // handler called when the service un-health + std::function onUnHealthHandler; + // remove the handler when the service un-health + bool removeHandlerOnUnhealth = true; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index c086ab3e..530be4c0 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -68,7 +68,8 @@ void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt) // load the grpcConfig m_grpcConfig = loadGrpcConfig("transport", _pt); // load the GrpcServerConfig - loadEndpointConfig(m_gatewayConfig.grpcServerConfig.endPoint, false, "transport", _pt); + loadEndpointConfig( + m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt); // the agencyID m_agencyID = _pt.get("agency.id", ""); if (m_agencyID.empty()) diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h index df42f7c3..d66db91e 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h @@ -74,7 +74,8 @@ struct GatewayConfig constexpr static int MinUnreachableDistance = 2; NetworkConfig networkConfig; - ppc::protocol::GrpcServerConfig grpcServerConfig; + ppc::protocol::GrpcServerConfig::Ptr grpcServerConfig = + std::make_shared(); // the file that configure the connected endpoint information std::string nodeFileName; // the dir that contains the connected endpoint information, e.g.nodes.json diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.cpp b/cpp/wedpr-main/gateway/GatewayInitializer.cpp index 0173bfb9..c5610fda 100644 --- a/cpp/wedpr-main/gateway/GatewayInitializer.cpp +++ b/cpp/wedpr-main/gateway/GatewayInitializer.cpp @@ -23,6 +23,7 @@ #include "ppc-gateway/GatewayFactory.h" #include "ppc-tools/src/config/PPCConfig.h" #include "protobuf/src/NodeInfoImpl.h" +#include "wedpr-protocol/grpc/client/HealthCheckTimer.h" #include "wedpr-protocol/grpc/client/RemoteFrontBuilder.h" #include "wedpr-protocol/protocol/src/v1/MessageHeaderImpl.h" @@ -50,7 +51,10 @@ void GatewayInitializer::init(std::string const& _configPath) "gateway", config->gatewayConfig().networkConfig.threadPoolSize); GatewayFactory gatewayFactory(config); - m_gateway = gatewayFactory.build(std::make_shared(config->grpcConfig())); + // default 1min + m_healthChecker = std::make_shared(60 * 1000); + m_gateway = gatewayFactory.build( + std::make_shared(config->grpcConfig(), m_healthChecker)); m_server = std::make_shared(config->gatewayConfig().grpcServerConfig); // register the gateway service @@ -69,15 +73,23 @@ void GatewayInitializer::start() { m_server->start(); } + if (m_healthChecker) + { + m_healthChecker->start(); + } } void GatewayInitializer::stop() { + if (m_healthChecker) + { + m_healthChecker->stop(); + } if (m_server) { m_server->stop(); } if (m_gateway) { - m_gateway->start(); + m_gateway->stop(); } } \ No newline at end of file diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.h b/cpp/wedpr-main/gateway/GatewayInitializer.h index 7cd15a01..2fdacded 100644 --- a/cpp/wedpr-main/gateway/GatewayInitializer.h +++ b/cpp/wedpr-main/gateway/GatewayInitializer.h @@ -28,6 +28,10 @@ namespace ppc::protocol { class GrpcServer; } +namespace ppc::protocol +{ +class HealthCheckTimer; +}; namespace ppc::gateway { class GatewayInitializer @@ -44,6 +48,7 @@ class GatewayInitializer protected: bcos::BoostLogInitializer::Ptr m_logInitializer; ppc::gateway::IGateway::Ptr m_gateway; + std::shared_ptr m_healthChecker; std::shared_ptr m_server; }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/Common.h b/cpp/wedpr-protocol/grpc/Common.h index bf2a5d76..b04f85d5 100644 --- a/cpp/wedpr-protocol/grpc/Common.h +++ b/cpp/wedpr-protocol/grpc/Common.h @@ -32,6 +32,12 @@ inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr con return args; } args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); + if (grpcConfig->enableHealthCheck()) + { + args.SetServiceConfigJSON( + "{\"healthCheckConfig\": " + "{\"serviceName\": \"\"}}"); + } return args; } } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/Common.h b/cpp/wedpr-protocol/grpc/client/Common.h index f1c32ab3..c7b3ab2f 100644 --- a/cpp/wedpr-protocol/grpc/client/Common.h +++ b/cpp/wedpr-protocol/grpc/client/Common.h @@ -21,4 +21,5 @@ #include "ppc-framework/Common.h" #define GRPC_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GRPC][CLIENT]" -#define GATEWAY_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][CLIENT]" \ No newline at end of file +#define GATEWAY_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][CLIENT]" +#define HEALTH_LOG(LEVEL) BCOS_LOG(LEVEL) << "[HEALTH]" \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index ad7c7276..92580244 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -42,36 +42,52 @@ void GatewayClient::asyncSendMessage(RouteType routeType, bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) { - auto request = toNodeInfoRequest(nodeInfo); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = m_stub->registerNodeInfo(&context, *request, response.get()); - return toError(status, std::move(*response)); + broadCast([nodeInfo](ChannelInfo const& channel) { + std::unique_ptr stub( + ppc::proto::Gateway::NewStub(channel.channel)); + auto request = toNodeInfoRequest(nodeInfo); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = stub->registerNodeInfo(&context, *request, response.get()); + return toError(status, std::move(*response)); + }); } bcos::Error::Ptr GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) { - auto request = toNodeInfoRequest(nodeID, ""); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = m_stub->unRegisterNodeInfo(&context, *request, response.get()); - return toError(status, std::move(*response)); + broadCast([nodeID](ChannelInfo const& channel) { + std::unique_ptr stub( + ppc::proto::Gateway::NewStub(channel.channel)); + auto request = toNodeInfoRequest(nodeID, ""); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = stub->unRegisterNodeInfo(&context, *request, response.get()); + return toError(status, std::move(*response)); + }); } bcos::Error::Ptr GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) { - auto request = toNodeInfoRequest(nodeID, topic); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = m_stub->registerTopic(&context, *request, response.get()); - return toError(status, std::move(*response)); + broadCast([nodeID, topic](ChannelInfo const& channel) { + std::unique_ptr stub( + ppc::proto::Gateway::NewStub(channel.channel)); + auto request = toNodeInfoRequest(nodeID, topic); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = stub->registerTopic(&context, *request, response.get()); + return toError(status, std::move(*response)); + }); } bcos::Error::Ptr GatewayClient::unRegisterTopic( bcos::bytesConstRef nodeID, std::string const& topic) { - auto request = toNodeInfoRequest(nodeID, topic); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = m_stub->unRegisterTopic(&context, *request, response.get()); - return toError(status, std::move(*response)); + broadCast([nodeID, topic](ChannelInfo const& channel) { + std::unique_ptr stub( + ppc::proto::Gateway::NewStub(channel.channel)); + auto request = toNodeInfoRequest(nodeID, topic); + ClientContext context; + std::shared_ptr response = std::make_shared(); + auto status = stub->unRegisterTopic(&context, *request, response.get()); + return toError(status, std::move(*response)); + }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp new file mode 100644 index 00000000..2ba6e503 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp @@ -0,0 +1,101 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file GrpcClient.cpp + * @author: yujiechen + * @date 2024-09-02 + */ +#include "GrpcClient.h" +#include "Common.h" + +using namespace ppc::protocol; +using namespace ppc::proto; +using namespace grpc; +using namespace grpc::health::v1; + +GrpcClient::GrpcClient( + ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) + : m_grpcConfig(grpcConfig), + m_channel(grpc::CreateCustomChannel( + endPoints, grpc::InsecureChannelCredentials(), toChannelConfig(grpcConfig))), + m_healthCheckStub(grpc::health::v1::Health::NewStub(m_channel)) +{ + std::vector endPointList; + boost::split(endPointList, endPoints, boost::is_any_of(",")); + // create the broadcast channels + for (auto const& endPoint : endPointList) + { + GRPC_CLIENT_LOG(INFO) << LOG_DESC("create broacast-channel, endpoint: ") << endPoint; + m_broadcastChannels.push_back( + {endPoint, grpc::CreateCustomChannel(endPoint, grpc::InsecureChannelCredentials(), + toChannelConfig(grpcConfig))}); + } +} + +bool GrpcClient::checkHealth() +{ + try + { + ClientContext context; + HealthCheckResponse response; + auto status = + m_healthCheckStub->Check(&context, HealthCheckRequest::default_instance(), &response); + if (!status.ok()) + { + GRPC_CLIENT_LOG(WARNING) + << LOG_DESC("GrpcClient check health failed") << LOG_KV("code", status.error_code()) + << LOG_KV("msg", status.error_message()); + return false; + } + return true; + } + catch (std::exception const& e) + { + GRPC_CLIENT_LOG(WARNING) << LOG_DESC("GrpcClient check health exception") + << LOG_KV("error", boost::diagnostic_information(e)); + return false; + } +} + +bcos::Error::Ptr GrpcClient::broadCast( + std::function callback) +{ + auto result = std::make_shared(0, ""); + for (auto const& channel : m_broadcastChannels) + { + try + { + if (channel.channel->GetState(false) == GRPC_CHANNEL_SHUTDOWN) + { + GRPC_CLIENT_LOG(INFO) << LOG_DESC("Ignore unconnected channel") + << LOG_KV("endpoint", channel.endPoint); + continue; + } + auto error = callback(channel); + if (error && error->errorCode() != 0) + { + result->setErrorCode(error->errorCode()); + result->setErrorMessage(result->errorMessage() + error->errorMessage() + "; "); + } + } + catch (std::exception const& e) + { + GRPC_CLIENT_LOG(WARNING) + << LOG_DESC("registerNodeInfo exception") << LOG_KV("remote", channel.endPoint) + << LOG_KV("error", boost::diagnostic_information(e)); + } + } + return result; +} \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.h b/cpp/wedpr-protocol/grpc/client/GrpcClient.h index d1fda5af..39d1adaa 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.h +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.h @@ -19,28 +19,44 @@ */ #pragma once #include "Service.grpc.pb.h" +#include "ServiceHealth.grpc.pb.h" #include "ppc-framework/protocol/GrpcConfig.h" #include "wedpr-protocol/grpc/Common.h" +#include #include namespace ppc::protocol { +struct ChannelInfo +{ + std::string endPoint; + std::shared_ptr channel; +}; // refer to: https://grpc.io/docs/languages/cpp/callback/ class GrpcClient { public: using Ptr = std::shared_ptr; - GrpcClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) - : m_channel(grpc::CreateCustomChannel( - endPoints, grpc::InsecureChannelCredentials(), toChannelConfig(grpcConfig))) - {} + GrpcClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints); virtual ~GrpcClient() = default; std::shared_ptr const& channel() { return m_channel; } + bool checkHealth(); protected: + virtual bcos::Error::Ptr broadCast( + std::function callback); + +protected: + ppc::protocol::GrpcConfig::Ptr m_grpcConfig; + // the channel std::shared_ptr m_channel; + + // the broadcast channel + std::vector m_broadcastChannels; + // the healthcheck stub + std::unique_ptr m_healthCheckStub; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp new file mode 100644 index 00000000..4b18b0df --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp @@ -0,0 +1,135 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HealthCheckTimer.cpp + * @author: yujiechen + * @date 2024-09-6 + */ +#include "HealthCheckTimer.h" +#include "Common.h" + +using namespace ppc::protocol; +using namespace ppc::front; + +HealthCheckTimer::HealthCheckTimer(int periodMs) : m_periodMs(periodMs) +{ + m_timer = std::make_shared(m_periodMs, "healthChecker"); +} + +void HealthCheckTimer::start() +{ + if (m_running == true) + { + HEALTH_LOG(INFO) << LOG_DESC("HealthCheckTimer has already been started!"); + return; + } + m_running = true; + + auto self = weak_from_this(); + m_timer->registerTimeoutHandler([self]() { + auto healthChecker = self.lock(); + if (!healthChecker) + { + return; + } + healthChecker->checkHealth(); + healthChecker->m_timer->restart(); + }); + if (m_timer) + { + m_timer->start(); + } + HEALTH_LOG(INFO) << LOG_DESC("start the HealthCheckTimer success"); +} + +void HealthCheckTimer::stop() +{ + if (m_running == false) + { + HEALTH_LOG(INFO) << LOG_DESC("HealthCheckTimer has already been stopped!"); + return; + } + m_running = false; + if (m_timer) + { + m_timer->stop(); + } + HEALTH_LOG(INFO) << LOG_DESC("stop the HealthCheckTimer success"); +} + + +void HealthCheckTimer::registerHealthCheckHandler(HealthCheckHandler::Ptr healthCheckHandler) +{ + if (!healthCheckHandler) + { + return; + } + bcos::WriteGuard l(x_healthCheckHandlers); + m_healthCheckHandlers[healthCheckHandler->serviceName] = healthCheckHandler; +} + + +void HealthCheckTimer::checkHealth() +{ + try + { + std::map handlers; + { + bcos::ReadGuard l(x_healthCheckHandlers); + handlers = m_healthCheckHandlers; + } + std::set serviceToRemove; + for (auto const& it : handlers) + { + auto handler = it.second; + bool health = true; + if (handler->checkHealthHandler) + { + health = handler->checkHealthHandler(); + } + if (health) + { + continue; + } + HEALTH_LOG(WARNING) << LOG_DESC("Detect unHealth service: ") << it.first; + if (handler->onUnHealthHandler) + { + handler->onUnHealthHandler(); + } + if (handler->removeHandlerOnUnhealth) + { + serviceToRemove.insert(it.first); + } + } + m_timer->restart(); + if (serviceToRemove.empty()) + { + return; + } + { + bcos::WriteGuard l(x_healthCheckHandlers); + for (auto const& service : serviceToRemove) + { + m_healthCheckHandlers.erase(service); + } + } + } + catch (std::exception const& e) + { + HEALTH_LOG(WARNING) << LOG_DESC("checkHealth exception, error: ") + << LOG_KV("error", boost::diagnostic_information(e)); + m_timer->restart(); + } +} diff --git a/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.h b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.h new file mode 100644 index 00000000..39dc3eb1 --- /dev/null +++ b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file HealthCheckTimer.h + * @author: yujiechen + * @date 2024-09-6 + */ +#pragma once +#include "ppc-framework/front/IFront.h" +#include +#include +#include + +namespace ppc::protocol +{ +class HealthCheckTimer : public std::enable_shared_from_this +{ +public: + using Ptr = std::shared_ptr; + HealthCheckTimer(int periodMs); + virtual ~HealthCheckTimer() = default; + + virtual void start(); + virtual void stop(); + + + void registerHealthCheckHandler(HealthCheckHandler::Ptr healthCheckHandler); + +protected: + void checkHealth(); + +private: + std::map m_healthCheckHandlers; + mutable bcos::SharedMutex x_healthCheckHandlers; + + int m_periodMs = 3000; + std::shared_ptr m_timer; + bool m_running = false; +}; +} // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp index d00e1c98..a2d06fa9 100644 --- a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp +++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp @@ -22,10 +22,23 @@ // compiler speed #include "FrontClient.h" +using namespace ppc::protocol; using namespace ppc::front; using namespace ppc::protocol; -IFrontClient::Ptr RemoteFrontBuilder::buildClient(std::string endPoint) const +IFrontClient::Ptr RemoteFrontBuilder::buildClient(std::string endPoint, + std::function onUnHealthHandler, bool removeHandlerOnUnhealth) const { - return std::make_shared(m_grpcConfig, endPoint); + auto frontClient = std::make_shared(m_grpcConfig, endPoint); + if (m_healthChecker) + { + auto healthCheckHandler = std::make_shared("front" + endPoint); + healthCheckHandler->checkHealthHandler = [frontClient]() { + return frontClient->checkHealth(); + }; + healthCheckHandler->onUnHealthHandler = onUnHealthHandler; + healthCheckHandler->removeHandlerOnUnhealth = removeHandlerOnUnhealth; + m_healthChecker->registerHealthCheckHandler(healthCheckHandler); + } + return frontClient; } diff --git a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h index 44a3db31..8a6a6030 100644 --- a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h +++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h @@ -18,6 +18,7 @@ * @date 2024-09-4 */ #pragma once +#include "HealthCheckTimer.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/GrpcConfig.h" @@ -27,13 +28,20 @@ class RemoteFrontBuilder : public IFrontBuilder { public: using Ptr = std::shared_ptr; - RemoteFrontBuilder(ppc::protocol::GrpcConfig::Ptr const& grpcConfig) : m_grpcConfig(grpcConfig) - {} + RemoteFrontBuilder(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, + ppc::protocol::HealthCheckTimer::Ptr healthChecker) + : m_grpcConfig(grpcConfig), m_healthChecker(healthChecker) + { + // Note: the front enable health-check + m_grpcConfig->setEnableHealthCheck(true); + } ~RemoteFrontBuilder() override = default; - IFrontClient::Ptr buildClient(std::string endPoint) const override; + IFrontClient::Ptr buildClient(std::string endPoint, std::function onUnHealthHandler, + bool removeHandlerOnUnhealth) const override; private: ppc::protocol::GrpcConfig::Ptr m_grpcConfig; + ppc::protocol::HealthCheckTimer::Ptr m_healthChecker; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.h b/cpp/wedpr-protocol/grpc/server/FrontServer.h index 988c093c..f8e0ae96 100644 --- a/cpp/wedpr-protocol/grpc/server/FrontServer.h +++ b/cpp/wedpr-protocol/grpc/server/FrontServer.h @@ -19,6 +19,7 @@ */ #pragma once #include "Service.grpc.pb.h" +#include #include #include #include @@ -37,7 +38,14 @@ class FrontServer : public ppc::proto::Front::CallbackService grpc::ServerUnaryReactor* onReceiveMessage(grpc::CallbackServerContext* context, const ppc::proto::ReceivedMessage* receivedMsg, ppc::proto::Error* reply) override; + void setHealthCheckService(grpc::HealthCheckServiceInterface* healthCheckService) + { + m_healthCheckService = healthCheckService; + m_healthCheckService->SetServingStatus(true); + } + private: + grpc::HealthCheckServiceInterface* m_healthCheckService = nullptr; ppc::front::IFront::Ptr m_front; ppc::protocol::MessageBuilder::Ptr m_msgBuilder; }; diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp index 8ad659b9..de95ab48 100644 --- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp @@ -29,14 +29,18 @@ void GrpcServer::start() if (m_running) { GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer has already been started!") - << LOG_KV("listenEndPoint", m_config.listenEndPoint()); + << LOG_KV("listenEndPoint", m_config->listenEndPoint()); return; } m_running = true; + if (m_config->enableHealthCheck()) + { + grpc::EnableDefaultHealthCheckService(true); + } grpc::reflection::InitProtoReflectionServerBuilderPlugin(); grpc::ServerBuilder builder; // without authentication - builder.AddListeningPort(m_config.listenEndPoint(), grpc::InsecureServerCredentials()); + builder.AddListeningPort(m_config->listenEndPoint(), grpc::InsecureServerCredentials()); // register the service for (auto const& service : m_bindingServices) { @@ -44,7 +48,7 @@ void GrpcServer::start() } m_server = std::unique_ptr(builder.BuildAndStart()); GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer start success!") - << LOG_KV("listenEndPoint", m_config.listenEndPoint()); + << LOG_KV("listenEndPoint", m_config->listenEndPoint()); } void GrpcServer::stop() diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.h b/cpp/wedpr-protocol/grpc/server/GrpcServer.h index 71bb6156..9d70f5a6 100644 --- a/cpp/wedpr-protocol/grpc/server/GrpcServer.h +++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.h @@ -30,7 +30,7 @@ class GrpcServer { public: using Ptr = std::shared_ptr; - GrpcServer(GrpcServerConfig const& config) : m_config(config) {} + GrpcServer(GrpcServerConfig::Ptr const& config) : m_config(config) {} virtual ~GrpcServer() = default; virtual void start(); @@ -41,9 +41,11 @@ class GrpcServer m_bindingServices.emplace_back(std::move(service)); } + std::unique_ptr const& server() const { return m_server; } + private: bool m_running = false; - GrpcServerConfig m_config; + GrpcServerConfig::Ptr m_config; std::unique_ptr m_server; std::vector> m_bindingServices; diff --git a/cpp/wedpr-protocol/proto/pb/ServiceHealth.proto b/cpp/wedpr-protocol/proto/pb/ServiceHealth.proto new file mode 100644 index 00000000..38843ff1 --- /dev/null +++ b/cpp/wedpr-protocol/proto/pb/ServiceHealth.proto @@ -0,0 +1,63 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + +syntax = "proto3"; + +package grpc.health.v1; + +option csharp_namespace = "Grpc.Health.V1"; +option go_package = "google.golang.org/grpc/health/grpc_health_v1"; +option java_multiple_files = true; +option java_outer_classname = "HealthProto"; +option java_package = "io.grpc.health.v1"; + +message HealthCheckRequest { + string service = 1; +} + +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; // Used only by the Watch method. + } + ServingStatus status = 1; +} + +service Health { + // If the requested service is unknown, the call will fail with status + // NOT_FOUND. + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); + + // Performs a watch for the serving status of the requested service. + // The server will immediately send back a message indicating the current + // serving status. It will then subsequently send a new message whenever + // the service's serving status changes. + // + // If the requested service is unknown when the call is received, the + // server will send a message setting the serving status to + // SERVICE_UNKNOWN but will *not* terminate the call. If at some + // future point, the serving status of the service becomes known, the + // server will send a new message with the service's serving status. + // + // If the call terminates with status UNIMPLEMENTED, then clients + // should assume this method is not supported and should not retry the + // call. If the call terminates with any other status (including OK), + // clients should retry the call with appropriate exponential backoff. + rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse); +} diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h b/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h index 022f6a83..9dc04758 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/LocalFrontBuilder.h @@ -30,7 +30,11 @@ class LocalFrontBuilder : public IFrontBuilder LocalFrontBuilder() = default; ~LocalFrontBuilder() override = default; - IFrontClient::Ptr buildClient(std::string endPoint) const override { return m_front; } + IFrontClient::Ptr buildClient(std::string, std::function onUnHealthHandler, + bool removeHandlerOnUnhealth) const override + { + return m_front; + } void setFront(IFront::Ptr front) { m_front = std::move(front); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index b9b5cccd..d80079d0 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -206,7 +206,20 @@ void GatewayImpl::onReceiveBroadcastMessage(MessageFace::Ptr msg, WsSession::Ptr bcos::Error::Ptr GatewayImpl::registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) { - m_localRouter->registerNodeInfo(nodeInfo); + auto self = weak_from_this(); + m_localRouter->registerNodeInfo( + nodeInfo, + [nodeInfo, self]() { + // remove the unhealth node + GATEWAY_LOG(INFO) << LOG_DESC("Remove the unhealth node") << printNodeInfo(nodeInfo); + auto gateway = self.lock(); + if (!gateway) + { + return; + } + gateway->m_localRouter->unRegisterNode(nodeInfo->nodeID().toBytes()); + }, + true); return nullptr; } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index 9338f07c..be8d8eaf 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -82,8 +82,8 @@ using GatewayNodeInfos = std::set; inline std::string printNodeStatus(GatewayNodeInfo::Ptr const& status) { std::ostringstream stringstream; - stringstream << LOG_KV("p2pNodeID", status->p2pNodeID()) << LOG_KV("agency", status->agency()) - << LOG_KV("statusSeq", status->statusSeq()) + stringstream << LOG_KV("p2pNodeID", printP2PIDElegantly(status->p2pNodeID())) + << LOG_KV("agency", status->agency()) << LOG_KV("statusSeq", status->statusSeq()) << LOG_KV("nodeSize", status->nodeSize()); return stringstream.str(); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index e286d2fd..baebc6eb 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -26,10 +26,12 @@ using namespace bcos; using namespace ppc::protocol; using namespace ppc::gateway; -bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo) +bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo, + std::function onUnHealthHandler, bool removeHandlerOnUnhealth) { LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo") << printNodeInfo(nodeInfo); - nodeInfo->setFront(m_frontBuilder->buildClient(nodeInfo->endPoint())); + nodeInfo->setFront(m_frontBuilder->buildClient( + nodeInfo->endPoint(), onUnHealthHandler, removeHandlerOnUnhealth)); auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); if (ret) { diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index 5cf5a7ef..30b5f244 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -39,7 +39,8 @@ class LocalRouter virtual ~LocalRouter() = default; - virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo); + virtual bool registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo, + std::function onUnHealthHandler, bool removeHandlerOnUnhealth); virtual void unRegisterNode(bcos::bytes const& nodeID) { m_routerInfo->removeNodeInfo(nodeID); @@ -59,6 +60,7 @@ class LocalRouter std::shared_ptr generateNodeStatus() { auto data = std::make_shared(); + m_routerInfo->setStatusSeq(m_statusSeq); m_routerInfo->encode(*data); return data; } @@ -73,6 +75,7 @@ class LocalRouter private: ppc::front::IFrontBuilder::Ptr m_frontBuilder; + GatewayNodeInfo::Ptr m_routerInfo; std::atomic m_statusSeq{1}; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index b9ccb092..3da0fbe9 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -32,6 +32,24 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) << LOG_KV("detail", printNodeStatus(gatewayInfo)); auto nodeList = gatewayInfo->nodeList(); bcos::WriteGuard l(x_mutex); + // remove the origin information of the gateway + auto it = m_nodeID2GatewayInfos.begin(); + for (; it != m_nodeID2GatewayInfos.end();) + { + auto& gatewayInfos = it->second; + auto ptr = gatewayInfos.find(gatewayInfo); + if (ptr != gatewayInfos.end()) + { + gatewayInfos.erase(ptr); + } + if (gatewayInfos.empty()) + { + it = m_nodeID2GatewayInfos.erase(it); + continue; + } + it++; + } + // insert new information for the gateway for (auto const& it : nodeList) { // update nodeID => gatewayInfos diff --git a/cpp/wedpr-transport/sdk/Common.h b/cpp/wedpr-transport/sdk/Common.h new file mode 100644 index 00000000..e7a1831f --- /dev/null +++ b/cpp/wedpr-transport/sdk/Common.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: shawnhe + * @date 2022-10-23 + */ + +#pragma once + +#include "ppc-framework/Common.h" +#include + +namespace ppc::sdk +{ +#define TRANSPORT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[TRASPORT]" +} // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp index 37424c3b..2ff873ed 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -18,6 +18,7 @@ * @date 2024-09-04 */ #include "ProTransportImpl.h" +#include "Common.h" #include "protocol/src/v1/MessageImpl.h" #include "wedpr-protocol/grpc/client/GatewayClient.h" #include "wedpr-protocol/grpc/server/FrontServer.h" @@ -28,34 +29,62 @@ using namespace ppc::protocol; using namespace ppc::sdk; -ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config) - : m_config(std::move(config)) +ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keepAlivePeriodMs) + : m_config(std::move(config)), m_keepAlivePeriodMs(keepAlivePeriodMs) { - GrpcServerConfig grpcServerConfig{config->selfEndPoint()}; + // default enable health-check + auto grpcServerConfig = std::make_shared(config->selfEndPoint(), true); m_server = std::make_shared(grpcServerConfig); FrontFactory frontFactory; - auto gateway = + m_gateway = std::make_shared(m_config->grpcConfig(), m_config->gatewayGrpcTarget()); m_front = frontFactory.build(std::make_shared(), std::make_shared(), - std::make_shared(), gateway, config); + std::make_shared(), m_gateway, config); auto msgBuilder = std::make_shared(std::make_shared()); auto frontService = std::make_shared(msgBuilder, m_front); - + frontService->setHealthCheckService(m_server->server()->GetHealthCheckService()); // register the frontService m_server->registerService(frontService); } void ProTransportImpl::start() { + m_timer = std::make_shared(m_keepAlivePeriodMs, "frontKeepAlive"); + auto self = weak_from_this(); + m_timer->registerTimeoutHandler([self]() { + auto transport = self.lock(); + if (!transport) + { + return; + } + transport->keepAlive(); + }); + m_timer->start(); m_server->start(); m_front->start(); } + void ProTransportImpl::stop() { + m_timer->stop(); m_server->stop(); m_front->stop(); +} + +void ProTransportImpl::keepAlive() +{ + try + { + m_gateway->registerNodeInfo(m_config->generateNodeInfo()); + } + catch (std::exception const& e) + { + TRANSPORT_LOG(WARNING) << LOG_DESC("keepAlive exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + m_timer->restart(); } \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index 5756e8d4..c89b33ef 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -19,6 +19,7 @@ */ #pragma once #include "TransportImpl.h" +#include "bcos-utilities/Timer.h" namespace ppc::protocol { @@ -28,17 +29,23 @@ class GrpcServer; namespace ppc::sdk { -class ProTransportImpl : public Transport +class ProTransportImpl : public Transport, public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - ProTransportImpl(ppc::front::FrontConfig::Ptr config); + ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keepAlivePeriodMs = 3000); void start() override; void stop() override; +protected: + void keepAlive(); + protected: ppc::front::FrontConfig::Ptr m_config; + ppc::gateway::IGateway::Ptr m_gateway; std::shared_ptr m_server; + int m_keepAlivePeriodMs; + std::shared_ptr m_timer; }; } // namespace ppc::sdk \ No newline at end of file From f89c9bc169cf57d80fc9c634fc8c2131ad9d0cd1 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 10 Sep 2024 02:16:44 +0800 Subject: [PATCH 030/120] add asyncGetPeers and asyncGetAgencies (#27) * add asyncGetPeers and asyncGetAgencies * add getPeers rpc implementation * fix gateway bugs * add registerMessageHandler to support dispatcher message by component --- cpp/cmake/BuildInfo.cmake | 2 +- cpp/ppc-framework/Common.h | 22 ---- cpp/ppc-framework/Helper.h | 48 ++++++++ cpp/ppc-framework/front/FrontInterface.h | 4 + cpp/ppc-framework/front/IFront.h | 4 + cpp/ppc-framework/gateway/IGateway.h | 4 + cpp/ppc-framework/protocol/INodeInfo.h | 5 +- cpp/ppc-framework/protocol/Message.h | 10 +- cpp/ppc-framework/protocol/Protocol.h | 1 - cpp/ppc-framework/rpc/RpcTypeDef.h | 1 + cpp/test-utils/FakeFront.h | 19 ++++ cpp/wedpr-computing/ppc-psi/src/PSIConfig.h | 18 +-- .../tests/ra2018-psi/TestEcdhPSIImpl.cpp | 7 +- cpp/wedpr-helper/ppc-utilities/Utilities.h | 13 ++- cpp/wedpr-initializer/Initializer.cpp | 12 +- .../air-node/AirNodeInitializer.cpp | 2 +- cpp/wedpr-main/cem-node/CEMInitializer.cpp | 2 +- cpp/wedpr-main/common/NodeStarter.h | 4 +- cpp/wedpr-main/mpc-node/MPCInitializer.cpp | 2 +- .../pro-node/ProNodeInitializer.cpp | 3 +- .../grpc/client/GatewayClient.cpp | 27 +++++ .../grpc/client/GatewayClient.h | 4 + .../grpc/server/GatewayServer.cpp | 53 +++++++++ .../grpc/server/GatewayServer.h | 6 + cpp/wedpr-protocol/proto/pb/Service.proto | 16 +++ .../protobuf/src/CMakeLists.txt | 2 +- .../protobuf/src/NodeInfoImpl.cpp | 13 +++ .../protobuf/src/NodeInfoImpl.h | 2 + .../ppc-front/ppc-front/CallbackManager.cpp | 52 +++++++-- .../ppc-front/ppc-front/CallbackManager.h | 11 ++ .../ppc-front/ppc-front/Front.cpp | 78 ++++++++++++- .../ppc-front/ppc-front/Front.h | 24 +++- .../ppc-front/ppc-front/FrontFactory.cpp | 2 + .../ppc-front/ppc-front/FrontImpl.h | 12 ++ .../ppc-gateway/gateway/GatewayImpl.cpp | 62 +++++++++- .../ppc-gateway/gateway/GatewayImpl.h | 4 + .../gateway/SendMessageWithRetry.cpp | 4 +- .../gateway/cache/MessageCache.cpp | 2 +- .../gateway/router/GatewayNodeInfo.h | 3 + .../gateway/router/GatewayNodeInfoImpl.cpp | 28 +++++ .../gateway/router/GatewayNodeInfoImpl.h | 2 + .../gateway/router/GatewayRouterManager.cpp | 2 +- .../gateway/router/LocalRouter.cpp | 4 +- .../ppc-gateway/gateway/router/LocalRouter.h | 2 + .../gateway/router/PeerRouterTable.cpp | 12 ++ .../gateway/router/PeerRouterTable.h | 8 ++ .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 107 +++++++++--------- .../ppc-gateway/ppc-gateway/p2p/Service.h | 4 +- .../ppc-gateway/p2p/router/RouterManager.cpp | 1 + .../p2p/router/RouterTableImpl.cpp | 1 + cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp | 2 +- cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp | 34 +++++- cpp/wedpr-transport/ppc-rpc/src/Rpc.h | 7 +- .../ppc-rpc/src/RpcFactory.cpp | 5 +- cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h | 5 +- cpp/wedpr-transport/sdk/ProTransportImpl.h | 1 - cpp/wedpr-transport/sdk/Transport.h | 6 +- cpp/wedpr-transport/sdk/TransportImpl.h | 1 + 58 files changed, 637 insertions(+), 155 deletions(-) create mode 100644 cpp/ppc-framework/Helper.h diff --git a/cpp/cmake/BuildInfo.cmake b/cpp/cmake/BuildInfo.cmake index 628b7c23..7ef9778c 100644 --- a/cpp/cmake/BuildInfo.cmake +++ b/cpp/cmake/BuildInfo.cmake @@ -27,7 +27,7 @@ function(create_build_info) # Generate header file containing useful build information add_custom_target(BuildInfo.h ALL WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} - COMMAND ${CMAKE_COMMAND} -DPPC_SOURCE_DIR="${PROJECT_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -DPPC_SOURCE_DIR="${PROJECT_SOURCE_DIR}/.." -DPPC_BUILDINFO_IN="${CMAKE_CURRENT_SOURCE_DIR}/cmake/templates/BuildInfo.h.in" -DPPC_DST_DIR="${PROJECT_BINARY_DIR}/include" -DPPC_CMAKE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/cmake" diff --git a/cpp/ppc-framework/Common.h b/cpp/ppc-framework/Common.h index 077a1969..cfd3f7f3 100644 --- a/cpp/ppc-framework/Common.h +++ b/cpp/ppc-framework/Common.h @@ -61,28 +61,6 @@ DERIVE_PPC_EXCEPTION(DataSchemaNotSetted); DERIVE_PPC_EXCEPTION(UnsupportedDataSchema); DERIVE_PPC_EXCEPTION(WeDPRException); -constexpr static int MAX_PORT = 65535; -constexpr static int DEFAULT_SECURITY_PARAM = 128; - -constexpr static size_t RSA_PUBLIC_KEY_PREFIX = 18; -constexpr static size_t RSA_PUBLIC_KEY_TRUNC = 8; -constexpr static size_t RSA_PUBLIC_KEY_TRUNC_LENGTH = 26; - -inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept -{ - if (p2pId.length() < RSA_PUBLIC_KEY_TRUNC_LENGTH) - { - return p2pId; - } - return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC); -} - -template -inline std::string printNodeID(T const& nodeID) -{ - return std::string(nodeID.begin(), nodeID.begin() + 8); -} - #if ENABLE_CPU_FEATURES #if X86 static const cpu_features::X86Features CPU_FEATURES = cpu_features::GetX86Info().features; diff --git a/cpp/ppc-framework/Helper.h b/cpp/ppc-framework/Helper.h new file mode 100644 index 00000000..13f72c46 --- /dev/null +++ b/cpp/ppc-framework/Helper.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Common.h + * @author: yujiechen + * @date 2022-10-20 + */ +#pragma once +#include +#include +namespace ppc +{ +constexpr static int MAX_PORT = 65535; +constexpr static int DEFAULT_SECURITY_PARAM = 128; + +constexpr static size_t RSA_PUBLIC_KEY_PREFIX = 18; +constexpr static size_t RSA_PUBLIC_KEY_TRUNC = 8; +constexpr static size_t RSA_PUBLIC_KEY_TRUNC_LENGTH = 26; + +inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept +{ + if (p2pId.length() < RSA_PUBLIC_KEY_TRUNC_LENGTH) + { + return p2pId; + } + return p2pId.substr(RSA_PUBLIC_KEY_PREFIX, RSA_PUBLIC_KEY_TRUNC); +} + + +template +inline std::string_view printNodeID(T const& nodeID) +{ + size_t offset = nodeID.size() >= 8 ? 8 : nodeID.size(); + return std::string_view((const char*)nodeID.data(), offset); +} +} // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/front/FrontInterface.h b/cpp/ppc-framework/front/FrontInterface.h index ff019e22..16691928 100644 --- a/cpp/ppc-framework/front/FrontInterface.h +++ b/cpp/ppc-framework/front/FrontInterface.h @@ -73,6 +73,10 @@ class FrontInterface virtual std::string const& selfEndPoint() const { return m_selfEndPoint; } + virtual std::vector agencies() const = 0; + virtual void start() = 0; + virtual void stop() = 0; + protected: // the selfEndPoint for the air-mode-node can be localhost std::string m_selfEndPoint = "localhost"; diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 561723dd..324d8124 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -71,6 +71,8 @@ class IFront : virtual public IFrontClient virtual void registerTopicHandler( std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) = 0; + virtual void registerMessageHandler( + std::string const& componentType, ppc::protocol::MessageDispatcherCallback callback) = 0; /** * @brief async send message * @@ -99,6 +101,8 @@ class IFront : virtual public IFrontClient virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; + virtual void asyncGetAgencies( + std::function)> callback) = 0; /** * @brief register the nodeInfo to the gateway diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index e4422b4a..d2489217 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -65,6 +65,10 @@ class IGateway ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload) = 0; + virtual void asyncGetPeers(std::function callback) = 0; + virtual void asyncGetAgencies( + std::function)> callback) = 0; + virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; virtual bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; virtual bcos::Error::Ptr registerTopic( diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index c8789773..2eb5e1d1 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -18,8 +18,9 @@ * @date 2024-08-26 */ #pragma once -#include "ppc-framework/Common.h" +#include "ppc-framework/Helper.h" #include +#include #include #include #include @@ -58,6 +59,8 @@ class INodeInfo { return (nodeID() == info->nodeID()) && (components() == info->components()); } + + virtual void toJson(Json::Value& jsonObject) const = 0; }; class INodeInfoFactory { diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index fb0f39e6..8fb013d3 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -18,9 +18,9 @@ * @date 2024-08-22 */ #pragma once -#include "../Common.h" #include "MessagePayload.h" #include "RouteType.h" +#include "ppc-framework/Helper.h" #include #include #include @@ -249,10 +249,10 @@ inline std::string printOptionalField(MessageOptionalHeader::Ptr optionalHeader) std::ostringstream stringstream; stringstream << LOG_KV("topic", optionalHeader->topic()) << LOG_KV("componentType", optionalHeader->componentType()) - << LOG_KV("srcNode", *(bcos::toHexString(optionalHeader->srcNode()))) - << LOG_KV("dstNode", *(bcos::toHexString(optionalHeader->dstNode()))) - << LOG_KV("dstInst", optionalHeader->dstInst()); - + << LOG_KV("srcNode", printNodeID(optionalHeader->srcNode())) + << LOG_KV("dstNode", printNodeID(optionalHeader->dstNode())) + << LOG_KV("srcInst", printNodeID(optionalHeader->srcInst())) + << LOG_KV("dstInst", printNodeID(optionalHeader->dstInst())); return stringstream.str(); } diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h index 889b7104..c9c6c850 100644 --- a/cpp/ppc-framework/protocol/Protocol.h +++ b/cpp/ppc-framework/protocol/Protocol.h @@ -396,7 +396,6 @@ inline std::ostream& operator<<(std::ostream& _out, HashImplName const& _type) enum class MessageType : uint16_t { - GatewayMessage = 0x0000, RpcRequest = 0x1000, // the rpc request type }; diff --git a/cpp/ppc-framework/rpc/RpcTypeDef.h b/cpp/ppc-framework/rpc/RpcTypeDef.h index 5a0fb493..208fee40 100644 --- a/cpp/ppc-framework/rpc/RpcTypeDef.h +++ b/cpp/ppc-framework/rpc/RpcTypeDef.h @@ -36,6 +36,7 @@ enum class RpcError : int32_t std::string const RUN_TASK_METHOD = "runTask"; std::string const ASYNC_RUN_TASK_METHOD = "asyncRunTask"; std::string const GET_TASK_STATUS = "getTaskStatus"; +std::string const GET_PEERS = "getPeers"; std::string const ASYNC_RUN_BS_MODE_TASK = "asyncRunBsModeTask"; std::string const FETCH_CIPHER = "fetchCipher"; diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h index 18e7741b..44b249ef 100644 --- a/cpp/test-utils/FakeFront.h +++ b/cpp/test-utils/FakeFront.h @@ -198,6 +198,21 @@ class FakeFront : public FrontInterface } } + // for ut + void setAgencyList(std::vector const& agencyList) + { + bcos::WriteGuard l(x_agencyList); + m_agencyList = agencyList; + } + + std::vector agencies() const override + { + bcos::ReadGuard l(x_agencyList); + return m_agencyList; + } + + void start() override {} + void stop() override {} private: // the uuid to _callback @@ -231,5 +246,9 @@ class FakeFront : public FrontInterface std::map m_uuidToCallback; bcos::Mutex m_mutex; std::atomic m_uuid = 0; + + // the agency list, for task-sync + std::vector m_agencyList; + mutable bcos::SharedMutex x_agencyList; }; } // namespace ppc::test \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h index 72ee8bbe..898c172d 100644 --- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h +++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h @@ -114,18 +114,7 @@ class PSIConfig int taskExpireTime() const { return m_taskExpireTime; } void setTaskExpireTime(int _taskExpireTime) { m_taskExpireTime = _taskExpireTime; } - std::vector agencyList() const - { - bcos::ReadGuard l(x_agencyList); - return m_agencyList; - } - - // for ut - void setAgencyList(std::vector const& agencyList) - { - bcos::WriteGuard l(x_agencyList); - m_agencyList = agencyList; - } + std::vector agencyList() const { return m_front->agencies(); } protected: ppc::front::PPCMessageFace::Ptr generatePPCMsg( @@ -160,10 +149,5 @@ class PSIConfig // the task-expire time int m_taskExpireTime = 10000; - - // the agency list, for task-sync - // TODO: fetch from the gateway - std::vector m_agencyList; - mutable bcos::SharedMutex x_agencyList; }; } // namespace ppc::psi \ No newline at end of file diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp index a7df31a2..5c973bed 100644 --- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp @@ -61,8 +61,11 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS auto clientPSI = factory->createEcdhPSI(clientAgencyName, clientConfig); std::vector agencyList = {serverAgencyName, clientAgencyName}; - serverPSI->psiConfig()->setAgencyList(agencyList); - clientPSI->psiConfig()->setAgencyList(agencyList); + auto serverFront = std::dynamic_pointer_cast(serverPSI->psiConfig()->front()); + serverFront->setAgencyList(agencyList); + + auto clientFront = std::dynamic_pointer_cast(clientPSI->psiConfig()->front()); + clientFront->setAgencyList(agencyList); // register the server-psi into the front factory->front()->registerEcdhPSI(serverAgencyName, serverPSI); diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h index 7600e09d..7b989243 100644 --- a/cpp/wedpr-helper/ppc-utilities/Utilities.h +++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h @@ -18,12 +18,13 @@ * @date 2024-08-23 */ #pragma once - #include "ppc-framework/Common.h" #include #include #include #include +#include +#include namespace ppc { @@ -52,4 +53,14 @@ inline std::string generateUUID() static thread_local auto uuid_gen = boost::uuids::basic_random_generator(); return boost::uuids::to_string(uuid_gen()); } +template +inline std::string printVector(std::vector const& list) +{ + std::stringstream oss; + for (auto const& it : list) + { + oss << it << ","; + } + return oss.str(); +} } // namespace ppc \ No newline at end of file diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index b5848406..4e7c4eec 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -74,7 +74,6 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway) m_protocolInitializer = std::make_shared(); m_protocolInitializer->init(m_config); - auto ppcMessageFactory = std::make_shared(); // init the frontService INIT_LOG(INFO) << LOG_DESC("init the frontService") << LOG_KV("agency", m_config->agencyID()); auto frontThreadPool = std::make_shared("front", m_config->threadPoolSize()); @@ -89,7 +88,8 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway) { m_transport = transportBuilder.build(SDKMode::PRO, m_config->frontConfig(), nullptr); } - m_ppcFront = std::make_shared(m_transport->getFront()); + m_ppcFront = + std::make_shared(std::make_shared(), m_transport->getFront()); INIT_LOG(INFO) << LOG_DESC("init the frontService success") << LOG_KV("frontDetail", printFrontDesc(m_config->frontConfig())) @@ -413,9 +413,9 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc) void Initializer::start() { - if (m_transport) + if (m_ppcFront) { - m_transport->start(); + m_ppcFront->start(); } /*if (m_ecdhConnPSI) { @@ -456,9 +456,9 @@ void Initializer::start() void Initializer::stop() { // stop the network firstly - if (m_transport) + if (m_ppcFront) { - m_transport->stop(); + m_ppcFront->stop(); } /*if (m_ecdhConnPSI) { diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp index ffcf605a..af88181d 100644 --- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp +++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp @@ -67,7 +67,7 @@ void AirNodeInitializer::init(std::string const& _configPath) auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID()); - m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config()); + m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config(), m_gateway); m_rpc->setRpcStorage(rpcStatusInterface); m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); m_nodeInitializer->registerRpcHandler(m_rpc); diff --git a/cpp/wedpr-main/cem-node/CEMInitializer.cpp b/cpp/wedpr-main/cem-node/CEMInitializer.cpp index 69c2efbe..e228fa5a 100644 --- a/cpp/wedpr-main/cem-node/CEMInitializer.cpp +++ b/cpp/wedpr-main/cem-node/CEMInitializer.cpp @@ -46,7 +46,7 @@ void CEMInitializer::init(std::string const& _configPath) auto storageConfig = ppcConfig->storageConfig(); auto cemConfig = ppcConfig->cemConfig(); auto rpcFactory = std::make_shared(ppcConfig->agencyID()); - m_rpc = rpcFactory->buildRpc(ppcConfig); + m_rpc = rpcFactory->buildRpc(ppcConfig, nullptr); auto cemService = std::make_shared(); cemService->setCEMConfig(cemConfig); cemService->setStorageConfig(storageConfig); diff --git a/cpp/wedpr-main/common/NodeStarter.h b/cpp/wedpr-main/common/NodeStarter.h index 95d8d671..e33ad7ef 100644 --- a/cpp/wedpr-main/common/NodeStarter.h +++ b/cpp/wedpr-main/common/NodeStarter.h @@ -68,13 +68,13 @@ int startProgram( } printVersion(); std::cout << "[" << bcos::getCurrentDateTime() << "] "; - std::cout << "The " + binaryName + "is running..." << std::endl; + std::cout << "The " + binaryName + " is running..." << std::endl; while (!exitHandler.shouldExit()) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); } starter.reset(); std::cout << "[" << bcos::getCurrentDateTime() << "] "; - std::cout << "The" + binaryName + " program exit normally." << std::endl; + std::cout << "The " + binaryName + " program exit normally." << std::endl; } } // namespace ppc::node \ No newline at end of file diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp index de8b8f1f..6998876a 100644 --- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp +++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp @@ -47,7 +47,7 @@ void MPCInitializer::init(std::string const& _configPath) auto storageConfig = ppcConfig->storageConfig(); auto mpcConfig = ppcConfig->mpcConfig(); auto rpcFactory = std::make_shared(ppcConfig->agencyID()); - m_rpc = rpcFactory->buildRpc(ppcConfig); + m_rpc = rpcFactory->buildRpc(ppcConfig, nullptr); auto mpcService = std::make_shared(); mpcService->setMPCConfig(mpcConfig); mpcService->setStorageConfig(storageConfig); diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp index 2ced6616..ec0ce5cc 100644 --- a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp +++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp @@ -58,7 +58,8 @@ void ProNodeInitializer::init(std::string const& _configPath) auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID()); - m_rpc = rpcFactory->buildRpc(m_nodeInitializer->config()); + m_rpc = rpcFactory->buildRpc( + m_nodeInitializer->config(), m_nodeInitializer->transport()->gateway()); m_rpc->setRpcStorage(rpcStatusInterface); m_rpc->setBsEcdhPSI(m_nodeInitializer->bsEcdhPsi()); m_nodeInitializer->registerRpcHandler(m_rpc); diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index 92580244..7b55c1a3 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -39,6 +39,33 @@ void GatewayClient::asyncSendMessage(RouteType routeType, [callback, response](Status status) { callback(toError(status, std::move(*response))); }); } +void GatewayClient::asyncGetPeers(std::function callback) +{ + auto response = std::make_shared(); + ClientContext context; + auto request = std::make_shared(); + m_stub->async()->asyncGetPeers( + &context, request.get(), response.get(), [callback, response](Status status) { + callback(toError(status, std::move(*response->mutable_error())), response->peersinfo()); + }); +} + +void GatewayClient::asyncGetAgencies( + std::function)> callback) +{ + auto response = std::make_shared(); + ClientContext context; + auto request = std::make_shared(); + m_stub->async()->asyncGetAgencies( + &context, request.get(), response.get(), [callback, response](Status status) { + std::vector agencies; + for (int i = 0; i < response->agencies_size(); i++) + { + agencies.emplace_back(response->agencies(i)); + } + callback(toError(status, std::move(*response->mutable_error())), agencies); + }); +} bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) { diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index 69513248..58c79cec 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -55,6 +55,10 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) override; + void asyncGetPeers(std::function callback) override; + void asyncGetAgencies( + std::function)> callback) override; + void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload) override diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp index 3b5caf69..8b770a06 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp @@ -51,6 +51,59 @@ ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* conte return reactor.get(); } +grpc::ServerUnaryReactor* GatewayServer::asyncGetPeers( + grpc::CallbackServerContext* context, const ppc::proto::Empty*, ppc::proto::PeersInfo* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + m_gateway->asyncGetPeers([reactor, reply](bcos::Error::Ptr error, std::string peersInfo) { + toSerializedError(reply->mutable_error(), error); + reply->set_peersinfo(std::move(peersInfo)); + reactor->Finish(Status::OK); + }); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("asyncGetPeers exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply->mutable_error(), + std::make_shared( + -1, "asyncGetPeers failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + +grpc::ServerUnaryReactor* GatewayServer::asyncGetAgencies( + grpc::CallbackServerContext* context, const ppc::proto::Empty*, ppc::proto::AgenciesInfo* reply) +{ + std::shared_ptr reactor(context->DefaultReactor()); + try + { + m_gateway->asyncGetAgencies( + [reactor, reply](bcos::Error::Ptr error, std::vector agencies) { + toSerializedError(reply->mutable_error(), error); + for (auto const& it : agencies) + { + reply->add_agencies(it); + } + reactor->Finish(Status::OK); + }); + } + catch (std::exception const& e) + { + GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("asyncGetAgencies exception") + << LOG_KV("error", boost::diagnostic_information(e)); + toSerializedError(reply->mutable_error(), + std::make_shared(-1, + "asyncGetAgencies failed for : " + std::string(boost::diagnostic_information(e)))); + reactor->Finish(Status::OK); + } + return reactor.get(); +} + + ServerUnaryReactor* GatewayServer::registerNodeInfo(CallbackServerContext* context, const ppc::proto::NodeInfo* serializedNodeInfo, ppc::proto::Error* reply) { diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.h b/cpp/wedpr-protocol/grpc/server/GatewayServer.h index d287a6c8..7c62048d 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.h +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.h @@ -39,6 +39,12 @@ class GatewayServer : public ppc::proto::Gateway::CallbackService grpc::ServerUnaryReactor* asyncSendMessage(grpc::CallbackServerContext* context, const ppc::proto::SendedMessageRequest* sendedMsg, ppc::proto::Error* reply) override; + grpc::ServerUnaryReactor* asyncGetPeers(grpc::CallbackServerContext* context, + const ppc::proto::Empty* request, ppc::proto::PeersInfo* reply) override; + grpc::ServerUnaryReactor* asyncGetAgencies(grpc::CallbackServerContext* context, + const ppc::proto::Empty* request, ppc::proto::AgenciesInfo* reply) override; + + grpc::ServerUnaryReactor* registerNodeInfo(grpc::CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override; diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto index e46d91b0..f531f08f 100644 --- a/cpp/wedpr-protocol/proto/pb/Service.proto +++ b/cpp/wedpr-protocol/proto/pb/Service.proto @@ -30,11 +30,27 @@ message SendedMessageRequest{ int64 timeout = 4; string traceID = 5; }; + +message AgenciesInfo{ + Error error = 1; + repeated string agencies = 2; +}; + +message PeersInfo{ + Error error = 1; + string peersInfo = 2; +}; +message Empty{ + +}; + service Front { rpc onReceiveMessage (ReceivedMessage) returns (Error) {} } service Gateway{ rpc asyncSendMessage(SendedMessageRequest) returns(Error){} + rpc asyncGetPeers(Empty)returns(PeersInfo){} + rpc asyncGetAgencies(Empty)returns(AgenciesInfo){} rpc registerNodeInfo(NodeInfo) returns(Error){} rpc unRegisterNodeInfo(NodeInfo)returns(Error){} rpc registerTopic(NodeInfo) returns(Error){} diff --git a/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt index 7aaa0e0d..8b23ec29 100644 --- a/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt +++ b/cpp/wedpr-protocol/protobuf/src/CMakeLists.txt @@ -24,4 +24,4 @@ endforeach() file(GLOB_RECURSE SRCS *.cpp) add_library(${PB_PROTOCOL_TARGET} ${SRCS} ${MESSAGES_SRCS}) -target_link_libraries(${PB_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} protobuf::libprotobuf ${CPU_FEATURES_LIB}) \ No newline at end of file +target_link_libraries(${PB_PROTOCOL_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} jsoncpp_static protobuf::libprotobuf ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp index 93f0f66a..59cf149e 100644 --- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp @@ -37,4 +37,17 @@ void NodeInfoImpl::decode(bcos::bytesConstRef data) decodePBObject(m_rawNodeInfo, data); m_components = std::set( m_rawNodeInfo->components().begin(), m_rawNodeInfo->components().end()); +} + +void NodeInfoImpl::toJson(Json::Value& jsonObject) const +{ + jsonObject["nodeID"] = std::string(nodeID().begin(), nodeID().end()); + jsonObject["endPoint"] = endPoint(); + Json::Value componentsInfo(Json::arrayValue); + auto componentsList = components(); + for (auto const& it : componentsList) + { + componentsInfo.append(it); + } + jsonObject["components"] = componentsInfo; } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h index ed8f6a68..02f8b31d 100644 --- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h @@ -76,6 +76,8 @@ class NodeInfoImpl : public INodeInfo } std::shared_ptr const& getFront() const override { return m_front; } + void toJson(Json::Value& jsonObject) const override; + private: std::shared_ptr m_front; std::set m_components; diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp index 51031f5b..3f500292 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp @@ -128,21 +128,49 @@ void CallbackManager::registerTopicHandler( m_topicHandlers.insert(std::make_pair(topic, callback)); } +void CallbackManager::registerMessageHandler( + std::string const& componentType, ppc::protocol::MessageDispatcherCallback callback) +{ + bcos::WriteGuard l(x_msgHandlers); + m_msgHandlers.insert(std::make_pair(componentType, callback)); +} + +MessageDispatcherCallback CallbackManager::getHandlerByTopic(std::string const& topic) +{ + bcos::ReadGuard l(x_topicHandlers); + auto it = m_topicHandlers.find(topic); + if (it != m_topicHandlers.end()) + { + return it->second; + } + return nullptr; +} + +MessageDispatcherCallback CallbackManager::getHandlerByComponentType( + std::string const& componentType) +{ + bcos::ReadGuard l(x_msgHandlers); + auto it = m_msgHandlers.find(componentType); + if (it != m_msgHandlers.end()) + { + return it->second; + } + return nullptr; +} + void CallbackManager::onReceiveMessage(std::string const& topic, Message::Ptr msg) { - MessageDispatcherCallback callback = nullptr; + auto callback = getHandlerByTopic(topic); + if (!callback) { - bcos::ReadGuard l(x_topicHandlers); - auto it = m_topicHandlers.find(topic); - if (it == m_topicHandlers.end()) - { - FRONT_LOG(DEBUG) << LOG_DESC( - "onReceiveMessage: not find the handler, put into the buffer") - << LOG_KV("topic", topic); - addMsgCache(topic, msg); - return; - } - callback = it->second; + callback = getHandlerByComponentType(msg->header()->optionalField()->componentType()); + } + if (!callback) + { + FRONT_LOG(DEBUG) << LOG_DESC("onReceiveMessage: not find the handler, put into the buffer") + << LOG_KV("topic", topic); + addMsgCache(topic, msg); + return; } m_threadPool->enqueue([callback, msg]() { try diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h index 95894d52..819faa05 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.h @@ -64,6 +64,9 @@ class CallbackManager : public std::enable_shared_from_this virtual void registerTopicHandler( std::string const& topic, ppc::protocol::MessageDispatcherCallback callback); + virtual void registerMessageHandler( + std::string const& componentType, ppc::protocol::MessageDispatcherCallback callback); + virtual ppc::protocol::Message::Ptr pop(std::string const& topic, int timeoutMs) { auto it = m_msgCache.find(topic); @@ -94,6 +97,10 @@ class CallbackManager : public std::enable_shared_from_this msgQueue->push(std::move(msg)); } + ppc::protocol::MessageDispatcherCallback getHandlerByTopic(std::string const& topic); + ppc::protocol::MessageDispatcherCallback getHandlerByComponentType( + std::string const& componentType); + private: bcos::ThreadPool::Ptr m_threadPool; std::shared_ptr m_ioService; @@ -105,6 +112,10 @@ class CallbackManager : public std::enable_shared_from_this std::map m_topicHandlers; mutable bcos::SharedMutex x_topicHandlers; + // componentType => messageDispatcherCallback + std::map m_msgHandlers; + mutable bcos::SharedMutex x_msgHandlers; + // the messageCache for the message with no topic handler defined uint64_t m_maxMsgCacheSize = 10000; // TODO: check the queueSize diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 01a1f179..a3259cd6 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -19,11 +19,71 @@ */ #include "Front.h" #include "FrontImpl.h" +#include "ppc-utilities/Utilities.h" using namespace ppc; using namespace bcos; using namespace ppc::protocol; using namespace ppc::front; + +Front::Front(ppc::front::PPCMessageFaceFactory::Ptr ppcMsgFactory, IFront::Ptr front) + : m_messageFactory(std::move(ppcMsgFactory)), m_front(std::move(front)) +{ + m_fetcher = std::make_shared(60 * 1000, "metaFetcher"); + m_fetcher->registerTimeoutHandler([this]() { + try + { + fetchGatewayMetaInfo(); + } + catch (std::exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("fetch the gateway information failed") + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + +void Front::start() +{ + m_front->start(); + m_fetcher->start(); +} + +void Front::stop() +{ + m_fetcher->stop(); + m_front->stop(); +} + +void Front::fetchGatewayMetaInfo() +{ + auto self = weak_from_this(); + m_front->asyncGetAgencies([self](bcos::Error::Ptr error, std::vector agencies) { + auto front = self.lock(); + if (!front) + { + return; + } + if (error && error->errorCode() != 0) + { + FRONT_LOG(WARNING) << LOG_DESC("asyncGetAgencies failed") + << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + return; + } + bcos::UpgradableGuard l(front->x_agencyList); + if (front->m_agencyList == agencies) + { + return; + } + bcos::UpgradeGuard ul(l); + front->m_agencyList = agencies; + FRONT_LOG(INFO) << LOG_DESC("Update agencies information") + << LOG_KV("agencies", printVector(agencies)); + }); + m_fetcher->restart(); +} + /** * @brief: send message to other party by gateway * @param _agencyID: agency ID of receiver @@ -39,14 +99,16 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace auto routeInfo = front->routerInfoBuilder()->build(); routeInfo->setDstInst(_agencyID); routeInfo->setTopic(_message->taskID()); + auto type = ((uint16_t)_message->taskType() << 8) | _message->algorithmType(); + routeInfo->setComponentType(std::to_string(type)); bcos::bytes data; _message->encode(data); auto self = weak_from_this(); - // ROUTE_THROUGH_TOPIC will hold the topic - m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data), - _message->seq(), _timeout, _callback, - [self, _agencyID, _respCallback]( - Error::Ptr error, Message::Ptr msg, SendResponseFunction resFunc) { + ppc::protocol::MessageCallback msgCallback = nullptr; + if (_respCallback) + { + msgCallback = [self, _agencyID, _respCallback]( + Error::Ptr error, Message::Ptr msg, SendResponseFunction resFunc) { auto front = self.lock(); if (!front) { @@ -68,7 +130,11 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace // get the agencyID _respCallback(error, msg->header()->optionalField()->srcInst(), front->m_messageFactory->decodePPCMessage(msg), responseCallback); - }); + }; + } + // ROUTE_THROUGH_TOPIC will hold the topic + m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data), + _message->seq(), _timeout, _callback, msgCallback); } // send response when receiving message from given agencyID diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h index 30cc315a..4efa9b5d 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h @@ -19,6 +19,7 @@ */ #pragma once +#include "bcos-utilities/Timer.h" #include "ppc-framework/front/FrontInterface.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/PPCMessageFace.h" @@ -29,9 +30,12 @@ class Front : public FrontInterface, public std::enable_shared_from_this { public: using Ptr = std::shared_ptr; - Front(IFront::Ptr front) : m_front(std::move(front)) {} + Front(ppc::front::PPCMessageFaceFactory::Ptr ppcMsgFactory, IFront::Ptr front); ~Front() override {} + void start() override; + void stop() override; + /** * @brief: send message to other party by gateway * @param _agencyID: agency ID of receiver @@ -62,7 +66,7 @@ class Front : public FrontInterface, public std::enable_shared_from_this { uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; auto self = weak_from_this(); - m_front->registerTopicHandler( + m_front->registerMessageHandler( std::to_string(type), [self, _handler](ppc::protocol::Message::Ptr msg) { auto front = self.lock(); if (!front) @@ -78,8 +82,24 @@ class Front : public FrontInterface, public std::enable_shared_from_this }); } + std::vector agencies() const override + { + bcos::ReadGuard l(x_agencyList); + return m_agencyList; + } + +protected: + void fetchGatewayMetaInfo(); + private: IFront::Ptr m_front; + + // the agency list + std::vector m_agencyList; + mutable bcos::SharedMutex x_agencyList; + + std::shared_ptr m_fetcher; + ppc::front::PPCMessageFaceFactory::Ptr m_messageFactory; }; } // namespace ppc::front \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp index 40b56ef1..eaac6f7a 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp @@ -31,6 +31,8 @@ IFront::Ptr FrontFactory::build(INodeInfoFactory::Ptr nodeInfoFactory, auto nodeInfo = nodeInfoFactory->build( bcos::bytesConstRef((bcos::byte*)config->nodeID().data(), config->nodeID().size()), config->selfEndPoint().entryPoint()); + FRONT_LOG(INFO) << LOG_DESC("build front") << LOG_KV("nodeID", config->nodeID()) + << LOG_KV("endPoint", config->selfEndPoint().entryPoint()); return std::make_shared(threadPool, nodeInfo, messageFactory, routerInfoBuilder, gateway, std::make_shared()); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index 32700b7b..b1c36b66 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -104,6 +104,12 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ m_callbackManager->registerTopicHandler(topic, callback); } + void registerMessageHandler(std::string const& componentType, + ppc::protocol::MessageDispatcherCallback callback) override + { + m_callbackManager->registerMessageHandler(componentType, callback); + } + /** * @brief register the nodeInfo to the gateway * @param nodeInfo the nodeInfo @@ -135,6 +141,12 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ m_gatewayClient->registerTopic(bcos::ref(m_nodeID), topic); } + void asyncGetAgencies( + std::function)> callback) override + { + m_gatewayClient->asyncGetAgencies(callback); + } + /** * @brief unRegister the topic * diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index d80079d0..18163caa 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -127,8 +127,7 @@ void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType, auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload)); p2pMessage->setSeq(traceID); p2pMessage->setPacketType((uint16_t)GatewayPacketType::P2PMessage); - GATEWAY_LOG(INFO) << LOG_DESC("##### asyncSendMessage") - << LOG_KV("msg", printMessage(p2pMessage)); + GATEWAY_LOG(TRACE) << LOG_DESC("asyncSendMessage") << LOG_KV("msg", printMessage(p2pMessage)); auto nodeList = m_localRouter->chooseReceiver(p2pMessage); // case send to the same agency if (!nodeList.empty()) @@ -239,4 +238,63 @@ bcos::Error::Ptr GatewayImpl::unRegisterTopic(bcos::bytesConstRef nodeID, std::s { m_localRouter->unRegisterTopic(nodeID, topic); return nullptr; +} + +void GatewayImpl::asyncGetPeers(std::function callback) +{ + if (!callback) + { + return; + } + try + { + auto infos = m_peerRouter->gatewayInfos(); + Json::Value peers; + peers["agency"] = m_agency; + peers["nodeID"] = m_service->nodeID(); + // add the local gatewayInfo + Json::Value localGatewayInfo; + m_localRouter->routerInfo()->toJson(localGatewayInfo); + peers["gateway"] = localGatewayInfo; + peers["peers"] = Json::Value(Json::arrayValue); + for (auto const& it : infos) + { + auto gatewayInfoList = it.second; + Json::Value agencyGatewayInfo; + agencyGatewayInfo["agency"] = it.first; + Json::Value peersInfo(Json::arrayValue); + for (auto const& gatewayInfo : gatewayInfoList) + { + Json::Value gatewayJson; + gatewayInfo->toJson(gatewayJson); + peersInfo.append(gatewayJson); + } + agencyGatewayInfo["gateway"] = peersInfo; + peers["peers"].append(agencyGatewayInfo); + } + Json::FastWriter fastWriter; + std::string statusStr = fastWriter.write(peers); + callback(nullptr, statusStr); + } + catch (std::exception const& e) + { + GATEWAY_LOG(WARNING) << LOG_DESC("asyncGetPeers exception") + << LOG_KV("error", boost::diagnostic_information(e)); + callback( + std::make_shared( + -1, "asyncGetPeers exception for " + std::string(boost::diagnostic_information(e))), + ""); + } +} + +void GatewayImpl::asyncGetAgencies( + std::function)> callback) +{ + if (!callback) + { + return; + } + auto agencies = m_peerRouter->agencies(); + agencies.emplace_back(m_agency); + callback(nullptr, agencies); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 12a43d49..4cc4af78 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -67,6 +67,10 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this callback) override; + void asyncGetAgencies( + std::function)> callback) override; + protected: virtual void onReceiveP2PMessage( bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp index 19093b32..b83b07ff 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp @@ -34,9 +34,9 @@ GatewayNodeInfo::Ptr SendMessageWithRetry::chooseP2pNode() RecursiveGuard lock(x_mutex); if (!m_dstNodeList.empty()) { - auto selectedNode = m_dstNodeList.begin(); + auto selectedNode = *(m_dstNodeList.begin()); m_dstNodeList.erase(m_dstNodeList.begin()); - return *selectedNode; + return selectedNode; } return nullptr; } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp index 9e8b2cc6..de96b110 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp @@ -33,7 +33,7 @@ void MessageCache::insertCache( GATEWAY_LOG(DEBUG) << LOG_BADGE("MessageCache: insertCache") << LOG_KV("topic", topic); bcos::ReadGuard l(x_msgCache); auto it = m_msgCache.find(topic); - if (it == m_msgCache.end()) + if (it != m_msgCache.end()) { it->second->messages.emplace_back(MessageInfo{msg, callback}); return; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index be8d8eaf..4ed8d833 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -18,7 +18,9 @@ * @date 2024-08-26 */ #pragma once +#include "ppc-framework/Helper.h" #include "ppc-framework/protocol/INodeInfo.h" +#include "ppc-utilities/Utilities.h" #include #include #include @@ -58,6 +60,7 @@ class GatewayNodeInfo virtual std::map nodeList() const = 0; virtual uint16_t nodeSize() const = 0; + virtual void toJson(Json::Value& jsonObject) const = 0; }; class GatewayNodeInfoFactory diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp index c6c24422..fb14954b 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -155,6 +155,17 @@ std::vector> GatewayNodeInfoImpl::choo bool selectAll, std::string const& topic) const { std::vector> result; + // empty topic means broadcast message to all front + if (topic.empty()) + { + bcos::ReadGuard l(x_nodeList); + for (auto const& it : m_nodeList) + { + result.emplace_back(it.second->getFront()); + } + return result; + } + // the topic specified bcos::ReadGuard l(x_topicInfo); for (auto const& it : m_topicInfo) { @@ -220,4 +231,21 @@ void GatewayNodeInfoImpl::decode(bcos::bytesConstRef data) m_nodeList.insert(std::make_pair(nodeInfoPtr->nodeID().toBytes(), nodeInfoPtr)); } } +} + +void GatewayNodeInfoImpl::toJson(Json::Value& jsonObject) const +{ + bcos::ReadGuard l(x_nodeList); + jsonObject["gatewayNodeID"] = p2pNodeID(); + jsonObject["agency"] = agency(); + + auto agencyNodeList = nodeList(); + Json::Value frontList(Json::arrayValue); + for (auto const& it : agencyNodeList) + { + Json::Value nodeInfo; + it.second->toJson(nodeInfo); + frontList.append(nodeInfo); + } + jsonObject["frontList"] = frontList; } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index 92a74f17..13f0bf25 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -80,6 +80,8 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo return m_nodeList.size(); } + void toJson(Json::Value& jsonObject) const override; + private: void updateNodeList(); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp index 3380eaef..7534e3b0 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp @@ -18,6 +18,7 @@ * @date 2024-08-26 */ #include "GatewayRouterManager.h" +#include "ppc-framework/Helper.h" #include "ppc-framework/gateway/GatewayProtocol.h" #include @@ -82,7 +83,6 @@ void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSessi { auto statusSeq = boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)msg->payload()->data())); - auto p2pMessage = std::dynamic_pointer_cast(msg); auto const& from = (p2pMessage->header()->srcGwNode().size() > 0) ? p2pMessage->header()->srcGwNode() : diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index baebc6eb..2b99dec6 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -18,7 +18,7 @@ * @date 2024-08-26 */ #include "LocalRouter.h" -#include "ppc-framework/Common.h" +#include "ppc-framework/Helper.h" #include "ppc-framework/gateway/GatewayProtocol.h" #include "ppc-gateway/Common.h" @@ -77,7 +77,7 @@ void LocalRouter::unRegisterTopic(bcos::bytesConstRef _nodeID, std::string const bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc callback, bool holding) { auto frontList = chooseReceiver(msg); - // send success + // find the front if (!frontList.empty()) { for (auto const& front : frontList) diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index 30b5f244..a3b33f2f 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -66,6 +66,8 @@ class LocalRouter } uint32_t statusSeq() { return m_statusSeq; } + GatewayNodeInfo::Ptr const& routerInfo() const { return m_routerInfo; } + private: uint32_t increaseSeq() { diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index 3da0fbe9..abb55789 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -19,6 +19,7 @@ */ #include "PeerRouterTable.h" #include "ppc-framework/Common.h" +#include "ppc-framework/Helper.h" #include using namespace bcos; @@ -67,6 +68,17 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) m_agency2GatewayInfos[gatewayInfo->agency()].insert(gatewayInfo); } +std::vector PeerRouterTable::agencies() const +{ + std::vector agencies; + bcos::ReadGuard l(x_mutex); + for (auto const& it : m_agency2GatewayInfos) + { + agencies.emplace_back(it.first); + } + return agencies; +} + GatewayNodeInfos PeerRouterTable::selectRouter( RouteType const& routeType, Message::Ptr const& msg) const { diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h index 7a433303..7a286443 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h @@ -40,6 +40,14 @@ class PeerRouterTable virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const; + std::vector agencies() const; + + std::map gatewayInfos() const + { + bcos::ReadGuard l(x_mutex); + return m_agency2GatewayInfos; + } + private: virtual GatewayNodeInfos selectRouterByNodeID(ppc::protocol::Message::Ptr const& msg) const; virtual GatewayNodeInfos selectRouterByComponent(ppc::protocol::Message::Ptr const& msg) const; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp index c60fa00d..3e96a31f 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -17,10 +17,10 @@ * @author: yujiechen * @date 2024-08-26 */ - #include "Service.h" #include "bcos-boostssl/websocket/WsError.h" #include "ppc-framework/Common.h" +#include "ppc-framework/Helper.h" using namespace bcos; using namespace ppc; @@ -57,14 +57,13 @@ void Service::onP2PConnect(WsSession::Ptr _session) RecursiveGuard l(x_nodeID2Session); auto it = m_nodeID2Session.find(_session->nodeId()); - // the session already connected if (it != m_nodeID2Session.end() && it->second->isConnected()) { SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect, drop the duplicated connection") << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); - _session->drop(WsError::UserDisconnect); updateNodeIDInfo(_session); + _session->drop(WsError::UserDisconnect); return; } // the node-self @@ -77,41 +76,18 @@ void Service::onP2PConnect(WsSession::Ptr _session) _session->drop(WsError::UserDisconnect); return; } - - ///// Note: here allow all new session, even the ip not configured(support dynamic access) bool updated = updateNodeIDInfo(_session); // hit the m_nodeID2Session if (it != m_nodeID2Session.end()) { - // the old session has already been connected, and the new session endPoint is not - // configured - if (it->second->isConnected() && !updated) - { - SERVICE_LOG(INFO) << LOG_DESC( - "onP2PConnect, drop the new not-configurated session, remain " - "the old session") - << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) - << LOG_KV("endPoint", _session->endPoint()) - << LOG_KV("oldEndPoint", it->second->endPoint()); - _session->drop(WsError::UserDisconnect); - return; - } - SERVICE_LOG(INFO) << LOG_DESC( - "onP2PConnect, drop the old not-configurated session, replace " - "with the new session") - << LOG_KV("nodeID", printP2PIDElegantly(_session->nodeId())) - << LOG_KV("endPoint", _session->endPoint()) - << LOG_KV("oldEndPoint", it->second->endPoint()); - if (it->second->isConnected()) - { - it->second->drop(WsError::UserDisconnect); - } it->second = _session; - return; } - // the new session - m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); + else + { + // the new session + m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); + } SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect established new session") << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); @@ -127,7 +103,7 @@ bool Service::updateNodeIDInfo(WsSession::Ptr const& _session) { it->second = p2pNodeID; SERVICE_LOG(INFO) << LOG_DESC("updateNodeIDInfo: update the nodeID") - << LOG_KV("nodeid", p2pNodeID) + << LOG_KV("nodeid", printP2PIDElegantly(p2pNodeID)) << LOG_KV("endpoint", _session->endPoint()); return true; } @@ -138,37 +114,58 @@ bool Service::updateNodeIDInfo(WsSession::Ptr const& _session) return false; } -void Service::removeSessionInfo(WsSession::Ptr const& _session) +bool Service::removeSessionInfo(WsSession::Ptr const& _session) { RecursiveGuard l(x_nodeID2Session); auto it = m_nodeID2Session.find(_session->nodeId()); - if (it != m_nodeID2Session.end()) + if (it != m_nodeID2Session.end() && it->second->endPointInfo() == _session->endPointInfo()) { - SERVICE_LOG(INFO) << "onP2PDisconnectand remove from m_nodeID2Session" + SERVICE_LOG(INFO) << "onP2PDisconnect: remove from m_nodeID2Session" << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); m_nodeID2Session.erase(it); + return true; } + return false; } + void Service::onP2PDisconnect(WsSession::Ptr _session) { // remove the session information - removeSessionInfo(_session); + if (!removeSessionInfo(_session)) + { + return; + } // update the session nodeID to empty UpgradableGuard l(x_configuredNode2ID); for (auto& it : m_configuredNode2ID) { - // reset the nodeID of the dropped session(except the node-self) to empty - if (m_nodeID != _session->nodeId() && it.second == _session->nodeId()) + // the node-self, no need to reset the nodeID + if (m_nodeID == _session->nodeId()) { - UpgradeGuard ul(l); - it.second.clear(); - break; + continue; } + // not with the same nodeID, can't reset the nodeID + if (it.second != _session->nodeId()) + { + continue; + } + UpgradeGuard ul(l); + it.second.clear(); + SERVICE_LOG(INFO) << "onP2PDisconnect: clear the nodeID information" + << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) + << LOG_KV("endpoint", _session->endPoint()); + break; } } +bool Service::nodeConnected(std::string const& nodeID) +{ + bcos::RecursiveGuard l(x_nodeID2Session); + return m_nodeID2Session.count(nodeID); +} + void Service::reconnect() { // obtain the un-connected peers information @@ -181,14 +178,12 @@ void Service::reconnect() { continue; } - if (!it.second.empty() && isConnected(it.first)) + if (!it.second.empty() && nodeConnected(it.second)) { continue; } unconnectedPeers->insert(it.first); - SERVICE_LOG(DEBUG) << LOG_DESC("ready to reconnect") - << LOG_KV("endpoint", - it.first.address() + ":" + std::to_string(it.first.port())); + SERVICE_LOG(DEBUG) << LOG_DESC("ready to reconnect") << LOG_KV("endpoint", it.first); } } setReconnectedPeers(unconnectedPeers); @@ -232,7 +227,7 @@ void Service::asyncSendMessageWithForward( return asyncSendMessage(dstNodeID, msg, options, respFunc); } // with nextHop, send the message to nextHop - SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageByNodeID") << printMessage(p2pMsg); + SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageWithForward") << printMessage(p2pMsg); return asyncSendMessage(nextHop, msg, options, respFunc); } @@ -258,11 +253,10 @@ void Service::asyncSendMessage( sessions.emplace_back(session); return WsService::asyncSendMessage(sessions, msg, options, respFunc); } - if (respFunc) { Error::Ptr error = std::make_shared( - -1, "send message to " + dstNodeID + + -1, "send message to " + std::string(printP2PIDElegantly(dstNodeID)) + " failed for no network established, msg: " + printWsMessage(msg)); respFunc(std::move(error), nullptr, nullptr); } @@ -272,12 +266,14 @@ void Service::asyncSendMessage( } catch (std::exception const& e) { - SERVICE_LOG(ERROR) << "asyncSendMessageByNodeID" << LOG_KV("dstNode", dstNodeID) + SERVICE_LOG(ERROR) << "asyncSendMessageByNodeID" + << LOG_KV("dstNode", printP2PIDElegantly(dstNodeID)) << LOG_KV("what", boost::diagnostic_information(e)); if (respFunc) { - respFunc(std::make_shared(-1, "send message to " + dstNodeID + " failed for " + - boost::diagnostic_information(e)), + respFunc(std::make_shared( + -1, "send message to " + std::string(printP2PIDElegantly(dstNodeID)) + + " failed for " + boost::diagnostic_information(e)), nullptr, nullptr); } } @@ -311,8 +307,14 @@ void Service::asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, Option auto reachableNodes = m_routerTable->getAllReachableNode(); try { + if (msg->seq().empty()) + { + msg->setSeq(m_messageFactory->newSeq()); + } for (auto const& node : reachableNodes) { + auto p2pMsg = std::dynamic_pointer_cast(msg); + p2pMsg->header()->setDstGwNode(node); asyncSendMessageByNodeID(node, msg, options); } } @@ -354,5 +356,6 @@ void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& sessions.emplace_back(session); WsService::asyncSendMessage(sessions, respMessage); SERVICE_LOG(TRACE) << "sendRespMessageBySession" << LOG_KV("resp", printMessage(respMessage)) - << LOG_KV("payload size", payload->size()); + << LOG_KV("payloadSize", + respMessage->payload() ? respMessage->payload()->size() : 0); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h index c41e0c1f..377d1161 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h @@ -77,10 +77,12 @@ class Service : public bcos::boostssl::ws::WsService virtual void onP2PConnect(bcos::boostssl::ws::WsSession::Ptr _session); virtual void onP2PDisconnect(bcos::boostssl::ws::WsSession::Ptr _session); + virtual bool nodeConnected(std::string const& nodeID); + void reconnect() override; bool updateNodeIDInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); - void removeSessionInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); + bool removeSessionInfo(bcos::boostssl::ws::WsSession::Ptr const& _session); bcos::boostssl::ws::WsSession::Ptr getSessionByNodeID(std::string const& _nodeID); virtual void asyncSendMessageWithForward(std::string const& dstNodeID, diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp index 77a765fb..ce25385c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp @@ -18,6 +18,7 @@ * @date 2024-08-26 */ #include "RouterManager.h" +#include "ppc-framework/Helper.h" #include "ppc-framework/gateway/GatewayProtocol.h" #include "ppc-framework/protocol/Message.h" #include diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp index f3cc3071..729af315 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterTableImpl.cpp @@ -18,6 +18,7 @@ * @date 2022-5-24 */ #include "RouterTableImpl.h" +#include "ppc-framework/Helper.h" #include "ppc-gateway/Common.h" using namespace bcos; diff --git a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp index 849502c2..252e3ba4 100644 --- a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp +++ b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp @@ -50,7 +50,7 @@ int main(int argc, const char* argv[]) // not specify the certPath in air-mode ppcConfig->loadRpcConfig(param.configFilePath); auto rpcFactory = std::make_shared("selfParty"); - auto rpc = rpcFactory->buildRpc(ppcConfig); + auto rpc = rpcFactory->buildRpc(ppcConfig, nullptr); registerEchoHandler(rpc); // start the rpc rpc->start(); diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp index 5b21374e..f2a78a09 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp +++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp @@ -31,10 +31,11 @@ using namespace ppc::rpc; using namespace ppc::tools; using namespace ppc::protocol; -Rpc::Rpc(std::shared_ptr _wsService, std::string const& _selfPartyID, - std::string const& _token, std::string const& _prePath) +Rpc::Rpc(std::shared_ptr _wsService, ppc::gateway::IGateway::Ptr gateway, + std::string const& _selfPartyID, std::string const& _token, std::string const& _prePath) : m_prePath(_prePath), m_wsService(std::move(_wsService)), + m_gateway(std::move(gateway)), m_taskFactory(std::make_shared(_selfPartyID, _prePath)), m_token(_token) { @@ -74,7 +75,8 @@ Rpc::Rpc(std::shared_ptr _wsService, std::string const& boost::bind(&Rpc::killBsModeTask, this, boost::placeholders::_1, boost::placeholders::_2); m_methodToHandler[UPDATE_BS_MODE_TASK_STATUS] = boost::bind( &Rpc::updateBsModeTaskStatus, this, boost::placeholders::_1, boost::placeholders::_2); - + m_methodToHandler[GET_PEERS] = + boost::bind(&Rpc::getPeers, this, boost::placeholders::_1, boost::placeholders::_2); RPC_LOG(INFO) << LOG_DESC("init rpc success") << LOG_KV("selfParty", _selfPartyID); } @@ -345,6 +347,32 @@ void Rpc::sendEcdhCipher(Json::Value const& _req, RespFunc _respFunc) _respFunc(result->error(), result->serializeToJson()); } +void Rpc::getPeers(Json::Value const& _req, RespFunc _respFunc) +{ + if (m_gateway == nullptr) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR(-1, "the gateway not initialized!")); + } + m_gateway->asyncGetPeers([_respFunc](bcos::Error::Ptr error, std::string peersInfo) { + try + { + Json::Value root; + Json::Reader jsonReader; + + if (!jsonReader.parse(peersInfo, root)) + { + BOOST_THROW_EXCEPTION(BCOS_ERROR(-1, "Invalid json string: " + peersInfo)); + } + _respFunc(error, std::move(root)); + } + catch (std::exception const& e) + { + RPC_LOG(WARNING) << LOG_DESC("getPeers exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); +} + void Rpc::sendPartnerCipher(Json::Value const& _req, RespFunc _respFunc) { if (!m_bsEcdhPSI) diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h index 45871f29..fe19b420 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h +++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h @@ -19,6 +19,7 @@ */ #pragma once #include "ppc-framework/front/FrontInterface.h" +#include "ppc-framework/gateway/IGateway.h" #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/rpc/RpcStatusInterface.h" #include "protocol/src/JsonTaskImpl.h" @@ -35,7 +36,8 @@ class Rpc : public RpcInterface { public: using Ptr = std::shared_ptr; - Rpc(std::shared_ptr _wsService, std::string const& _selfPartyID, + Rpc(std::shared_ptr _wsService, + ppc::gateway::IGateway::Ptr gateway, std::string const& _selfPartyID, std::string const& _token, std::string const& _prePath = "data"); ~Rpc() override { stop(); } void start() override @@ -131,11 +133,14 @@ class Rpc : public RpcInterface virtual void killBsModeTask(Json::Value const& _req, RespFunc _respFunc); virtual void updateBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc); + virtual void getPeers(Json::Value const& _req, RespFunc _respFunc); + void checkHostResource(); private: std::string m_prePath; std::shared_ptr m_wsService; + ppc::gateway::IGateway::Ptr m_gateway; RpcStatusInterface::Ptr m_rpcStorage; // Note: here use jsonTaskFactory to decrease the overhead to convert json::value to string when diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp index a1db0739..92c1ed22 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp @@ -27,7 +27,8 @@ using namespace bcos; using namespace ppc::rpc; using namespace ppc::tools; -Rpc::Ptr RpcFactory::buildRpc(ppc::tools::PPCConfig::ConstPtr _config) +Rpc::Ptr RpcFactory::buildRpc( + ppc::tools::PPCConfig::ConstPtr _config, ppc::gateway::IGateway::Ptr gateway) { auto wsConfig = initConfig(_config); // create the wsConfig @@ -39,7 +40,7 @@ Rpc::Ptr RpcFactory::buildRpc(ppc::tools::PPCConfig::ConstPtr _config) initializer->initWsService(wsService); auto rpc = std::make_shared( - wsService, m_selfPartyID, _config->rpcConfig().token, _config->dataLocation()); + wsService, gateway, m_selfPartyID, _config->rpcConfig().token, _config->dataLocation()); rpc->setMinNeededMemory(_config->rpcConfig().minNeededMemoryGB); return rpc; } diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h index 3e6b512e..91f94ff1 100644 --- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h +++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.h @@ -19,6 +19,7 @@ */ #pragma once #include "Rpc.h" +#include "ppc-framework/gateway/IGateway.h" #include namespace bcos::boostssl::ws { @@ -37,8 +38,8 @@ class RpcFactory RpcFactory(std::string const& _selfPartyID) : m_selfPartyID(_selfPartyID) {} virtual ~RpcFactory() = default; - Rpc::Ptr buildRpc(std::shared_ptr _config); - + Rpc::Ptr buildRpc( + std::shared_ptr _config, ppc::gateway::IGateway::Ptr gateway); private: std::shared_ptr initConfig( diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index c89b33ef..ed29cbdd 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -43,7 +43,6 @@ class ProTransportImpl : public Transport, public std::enable_shared_from_this

m_server; int m_keepAlivePeriodMs; std::shared_ptr m_timer; diff --git a/cpp/wedpr-transport/sdk/Transport.h b/cpp/wedpr-transport/sdk/Transport.h index cf9230fb..c9c36148 100644 --- a/cpp/wedpr-transport/sdk/Transport.h +++ b/cpp/wedpr-transport/sdk/Transport.h @@ -19,6 +19,7 @@ */ #pragma once #include "ppc-framework/front/IFront.h" +#include "ppc-framework/gateway/IGateway.h" namespace ppc::sdk { class Transport @@ -31,9 +32,12 @@ class Transport virtual void start() { m_front->start(); } virtual void stop() { m_front->stop(); } - virtual ppc::front::IFront::Ptr const& getFront() { return m_front; } + virtual ppc::front::IFront::Ptr const& getFront() const { return m_front; } + + virtual ppc::gateway::IGateway::Ptr const& gateway() const { return m_gateway; } protected: ppc::front::IFront::Ptr m_front; + ppc::gateway::IGateway::Ptr m_gateway; }; } // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/TransportImpl.h index a400b416..fbd9e3e6 100644 --- a/cpp/wedpr-transport/sdk/TransportImpl.h +++ b/cpp/wedpr-transport/sdk/TransportImpl.h @@ -33,6 +33,7 @@ class TransportImpl : public Transport TransportImpl(ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) : m_config(std::move(config)) { + m_gateway = gateway; ppc::front::FrontFactory frontFactory; m_front = frontFactory.build(std::make_shared(), std::make_shared(), From 98895592734760d0fa703744df5e7bba245dffe9 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 10 Sep 2024 17:09:59 +0800 Subject: [PATCH 031/120] add asyncSendResponse && fix MessagePayload decode error when the data size over 65556 (#29) * fix sendResponse bug * fix registerTopic/unregisterTopic bug * add asyncSendResponse implementation * fix MessagePayload decode error when the data size over 65556 --- cpp/ppc-framework/front/FrontInterface.h | 4 +- cpp/ppc-framework/front/IFront.h | 11 +++-- cpp/ppc-framework/protocol/INodeInfo.h | 3 +- cpp/ppc-framework/protocol/Message.h | 1 + cpp/ppc-framework/protocol/PPCMessageFace.h | 3 ++ cpp/ppc-framework/protocol/RouteType.h | 3 ++ cpp/test-utils/FakeFront.h | 4 +- cpp/test-utils/FakePPCMessage.h | 6 +++ cpp/wedpr-computing/ppc-psi/src/PSIConfig.h | 10 ++-- .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp | 2 +- .../src/psi-framework/PSIFramework.cpp | 15 +++--- .../ppc-psi/src/psi-framework/PSIFramework.h | 2 +- .../interfaces/PSIMessageInterface.h | 7 ++- cpp/wedpr-helper/ppc-utilities/Utilities.h | 25 +++++++--- .../protocol/src/JsonTaskImpl.cpp | 15 ++++-- .../protocol/src/PPCMessage.cpp | 7 +-- cpp/wedpr-protocol/protocol/src/PPCMessage.h | 6 +++ .../protocol/src/v1/MessageHeaderImpl.cpp | 20 ++++---- .../protocol/src/v1/MessagePayloadImpl.cpp | 6 +-- .../protocol/tests/MessageTest.cpp | 10 ++++ .../ppc-io/src/DataResourceLoaderImpl.cpp | 8 ++++ .../ppc-front/ppc-front/Front.cpp | 46 +++++++++++++++++-- .../ppc-front/ppc-front/Front.h | 25 ++-------- .../ppc-front/ppc-front/FrontImpl.cpp | 19 +++++++- .../ppc-front/ppc-front/FrontImpl.h | 19 ++++---- .../ppc-gateway/gateway/GatewayImpl.cpp | 2 +- .../gateway/SendMessageWithRetry.cpp | 4 +- .../gateway/router/GatewayNodeInfo.h | 2 +- .../gateway/router/GatewayNodeInfoImpl.cpp | 6 ++- .../gateway/router/GatewayNodeInfoImpl.h | 2 +- .../gateway/router/LocalRouter.cpp | 10 ++-- .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 14 +++++- 32 files changed, 221 insertions(+), 96 deletions(-) diff --git a/cpp/ppc-framework/front/FrontInterface.h b/cpp/ppc-framework/front/FrontInterface.h index 16691928..767afc02 100644 --- a/cpp/ppc-framework/front/FrontInterface.h +++ b/cpp/ppc-framework/front/FrontInterface.h @@ -57,8 +57,8 @@ class FrontInterface uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) = 0; // send response when receiving message from given agencyID - virtual void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, - front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) = 0; + virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + front::PPCMessageFace::Ptr message, ErrorCallbackFunc _callback) = 0; virtual void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, std::function _handler) = 0; diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 324d8124..25073ac3 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -93,6 +93,9 @@ class IFront : virtual public IFrontClient long timeout, ppc::protocol::ReceiveMsgFunc errorCallback, ppc::protocol::MessageCallback callback) = 0; + virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0; + // the sync interface for async_send_message virtual bcos::Error::Ptr push(ppc::protocol::RouteType routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, @@ -108,26 +111,26 @@ class IFront : virtual public IFrontClient * @brief register the nodeInfo to the gateway * @param nodeInfo the nodeInfo */ - virtual void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; + virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; /** * @brief unRegister the nodeInfo to the gateway */ - virtual void unRegisterNodeInfo() = 0; + virtual bcos::Error::Ptr unRegisterNodeInfo() = 0; /** * @brief register the topic * * @param topic the topic to register */ - virtual void registerTopic(std::string const& topic) = 0; + virtual bcos::Error::Ptr registerTopic(std::string const& topic) = 0; /** * @brief unRegister the topic * * @param topic the topic to unregister */ - virtual void unRegisterTopic(std::string const& topic) = 0; + virtual bcos::Error::Ptr unRegisterTopic(std::string const& topic) = 0; }; class IFrontBuilder diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index 2eb5e1d1..c949e8ed 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -57,7 +57,8 @@ class INodeInfo virtual bool equal(INodeInfo::Ptr const& info) { - return (nodeID() == info->nodeID()) && (components() == info->components()); + return (nodeID().toBytes() == info->nodeID().toBytes()) && + (components() == info->components()); } virtual void toJson(Json::Value& jsonObject) const = 0; diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 8fb013d3..3d78b432 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -265,6 +265,7 @@ inline std::string printMessage(Message::Ptr const& _msg) std::ostringstream stringstream; stringstream << LOG_KV("from", _msg->header()->srcP2PNodeIDView()) << LOG_KV("to", _msg->header()->dstP2PNodeIDView()) + << LOG_KV("routeType", (ppc::protocol::RouteType)_msg->header()->routeType()) << LOG_KV("ttl", _msg->header()->ttl()) << LOG_KV("rsp", _msg->header()->isRespPacket()) << LOG_KV("traceID", _msg->header()->traceID()) diff --git a/cpp/ppc-framework/protocol/PPCMessageFace.h b/cpp/ppc-framework/protocol/PPCMessageFace.h index a2486ad2..04fc9a76 100644 --- a/cpp/ppc-framework/protocol/PPCMessageFace.h +++ b/cpp/ppc-framework/protocol/PPCMessageFace.h @@ -54,7 +54,10 @@ class PPCMessageFace virtual std::string const& taskID() const = 0; virtual void setTaskID(std::string const&) = 0; virtual std::string const& sender() const = 0; + virtual bcos::bytes const& senderNode() const = 0; virtual void setSender(std::string const&) = 0; + virtual void setSenderNode(bcos::bytes const&) = 0; + virtual std::shared_ptr data() const = 0; virtual void setData(std::shared_ptr) = 0; virtual std::map header() = 0; diff --git a/cpp/ppc-framework/protocol/RouteType.h b/cpp/ppc-framework/protocol/RouteType.h index cb7feb96..0d481345 100644 --- a/cpp/ppc-framework/protocol/RouteType.h +++ b/cpp/ppc-framework/protocol/RouteType.h @@ -45,6 +45,9 @@ inline std::ostream& operator<<(std::ostream& _out, RouteType const& _type) case RouteType::ROUTE_THROUGH_AGENCY: _out << "RouteThroughAgency"; break; + case RouteType::ROUTE_THROUGH_TOPIC: + _out << "RouteThroughTopic"; + break; default: _out << "UnknownRouteType"; break; diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h index 44b249ef..ab1c9709 100644 --- a/cpp/test-utils/FakeFront.h +++ b/cpp/test-utils/FakeFront.h @@ -184,7 +184,7 @@ class FakeFront : public FrontInterface bcos::Error::Ptr eraseTaskInfo(std::string const&) override { return nullptr; } // send response when receiving message from given agencyID - void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, + void asyncSendResponse(bcos::bytes const& peer, std::string const& _uuid, front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override { if (m_uuidToCallback.count(_uuid)) @@ -193,7 +193,7 @@ class FakeFront : public FrontInterface removeCallback(_uuid); if (callback) { - callback(nullptr, _agencyID, _message, nullptr); + callback(nullptr, std::string(peer.begin(), peer.end()), _message, nullptr); } } } diff --git a/cpp/test-utils/FakePPCMessage.h b/cpp/test-utils/FakePPCMessage.h index 8248bc9f..44fa9d56 100644 --- a/cpp/test-utils/FakePPCMessage.h +++ b/cpp/test-utils/FakePPCMessage.h @@ -74,7 +74,13 @@ class FakePPCMessage : public PPCMessageFace // set the message to be response void setResponse() override { m_response = true; } + + bcos::bytes const& senderNode() const override { return m_senderNode; } + + void setSenderNode(bcos::bytes const& senderNode) override { m_senderNode = senderNode; } + private: + bcos::bytes m_senderNode; uint8_t m_version; uint8_t m_taskType; uint8_t m_algorithmType; diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h index 898c172d..4b67e599 100644 --- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h +++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h @@ -20,6 +20,7 @@ #pragma once #include "Common.h" #include "bcos-utilities/Common.h" +#include "ppc-framework/Helper.h" #include "ppc-framework/front/FrontInterface.h" #include "ppc-framework/io/DataResourceLoader.h" #include "ppc-framework/protocol/Protocol.h" @@ -96,14 +97,15 @@ class PSIConfig _responseCallback); } - void asyncSendResponse(std::string const& _peerID, std::string const& _taskID, + void asyncSendResponse(bcos::bytes const& fromNode, std::string const& _taskID, std::string const& _uuid, PSIMessageInterface::Ptr const& _msg, ppc::front::ErrorCallbackFunc _callback, uint32_t _seq = 0) { auto ppcMsg = generatePPCMsg(_taskID, _msg, _seq); - PSI_LOG(TRACE) << LOG_DESC("sendResponse") << LOG_KV("peer", _peerID) << printPPCMsg(ppcMsg) - << LOG_KV("msgType", (int)_msg->packetType()) << LOG_KV("uuid", _uuid); - m_front->asyncSendResponse(_peerID, _uuid, ppcMsg, _callback); + PSI_LOG(TRACE) << LOG_DESC("sendResponse") << LOG_KV("peer", printNodeID(fromNode)) + << printPPCMsg(ppcMsg) << LOG_KV("msgType", (int)_msg->packetType()) + << LOG_KV("uuid", _uuid); + m_front->asyncSendResponse(fromNode, _uuid, ppcMsg, _callback); } ppc::io::DataResourceLoader::Ptr const& dataResourceLoader() const diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp index f792b190..3162328d 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp @@ -566,7 +566,7 @@ void EcdhPSIImpl::onHandshakeResponse(PSIMessageInterface::Ptr const& _msg) psiMsg->setErrorCode(0); psiMsg->setErrorMessage("success"); auto startT = bcos::utcSteadyTime(); - m_config->asyncSendResponse(taskState->peerID(), taskState->task()->id(), _msg->uuid(), psiMsg, + m_config->asyncSendResponse(_msg->fromNode(), taskState->task()->id(), _msg->uuid(), psiMsg, [this, startT, _msg](bcos::Error::Ptr _error) { if (!_error || _error->errorCode() == 0) { diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp index fec26576..6010656e 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp @@ -150,6 +150,7 @@ void PSIFramework::onReceiveMessage(PPCMessageFace::Ptr _msg) psiMsg->setTaskID(_msg->taskID()); psiMsg->setSeq(_msg->seq()); psiMsg->setUUID(_msg->uuid()); + psiMsg->setFromNode(_msg->senderNode()); m_msgQueue->push(psiMsg); PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("onReceiveMessage") << printPSIMessage(psiMsg) << LOG_KV("uuid", _msg->uuid()); @@ -699,7 +700,7 @@ void PSIFramework::sendHandshakeRequest(TaskState::Ptr const& _taskState) void PSIFramework::responsePSIResultSyncStatus(int32_t _code, std::string const& _msg, - std::string const& _peer, std::string const& _taskID, std::string const& _uuid, uint32_t _seq) + bcos::bytes const& _peer, std::string const& _taskID, std::string const& _uuid, uint32_t _seq) { // response to the client auto psiMsg = @@ -732,15 +733,17 @@ void PSIFramework::handlePSIResultSyncMsg(PSIMessageInterface::Ptr _resultSyncMs << printPSIMessage(_resultSyncMsg); std::string msg = "sync psi result for task " + _resultSyncMsg->taskID() + " failed for task not found!"; - responsePSIResultSyncStatus((int32_t)PSIRetCode::TaskNotFound, msg, _resultSyncMsg->from(), - _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), _resultSyncMsg->seq()); + responsePSIResultSyncStatus((int32_t)PSIRetCode::TaskNotFound, msg, + _resultSyncMsg->fromNode(), _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), + _resultSyncMsg->seq()); return; } try { taskState->storePSIResult(m_dataResourceLoader, _resultSyncMsg->takeData()); - responsePSIResultSyncStatus((int32_t)PSIRetCode::Success, "success", _resultSyncMsg->from(), - _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), _resultSyncMsg->seq()); + responsePSIResultSyncStatus((int32_t)PSIRetCode::Success, "success", + _resultSyncMsg->fromNode(), _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), + _resultSyncMsg->seq()); } catch (std::exception const& e) { @@ -749,7 +752,7 @@ void PSIFramework::handlePSIResultSyncMsg(PSIMessageInterface::Ptr _resultSyncMs auto errorMessage = "sync psi result for " + _resultSyncMsg->taskID() + " failed, error: " + std::string(boost::diagnostic_information(e)); responsePSIResultSyncStatus((int32_t)PSIRetCode::SyncPSIResultFailed, errorMessage, - _resultSyncMsg->from(), _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), + _resultSyncMsg->fromNode(), _resultSyncMsg->taskID(), _resultSyncMsg->uuid(), _resultSyncMsg->seq()); // cancel the task auto error = BCOS_ERROR_PTR((int32_t)PSIRetCode::SyncPSIResultFailed, errorMessage); diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h index 3bab2dc8..66238319 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h @@ -189,7 +189,7 @@ class PSIFramework : public bcos::Worker, public ppc::task::TaskFrameworkInterfa m_signalled.wait_for(l, boost::chrono::milliseconds(5)); } void responsePSIResultSyncStatus(int32_t _code, std::string const& _msg, - std::string const& _peer, std::string const& _taskID, std::string const& _uuid, + bcos::bytes const& _peer, std::string const& _taskID, std::string const& _uuid, uint32_t _seq); void broadcastSyncTaskInfo( diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h index c4263477..3632c74f 100644 --- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h +++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h @@ -76,7 +76,8 @@ class PSIMessageInterface virtual void setTaskID(std::string const& _taskID) { m_taskID = _taskID; } virtual void setSeq(uint32_t _seq) { m_seq = _seq; } virtual void setFrom(std::string const& _from) { m_from = _from; } - + virtual void setFromNode(bcos::bytes const& fromNode) { m_fromNode = fromNode; } + virtual bcos::bytes fromNode() const { return m_fromNode; } virtual std::string const& taskID() const { return m_taskID; } virtual uint32_t seq() const { return m_seq; } @@ -88,7 +89,11 @@ class PSIMessageInterface private: std::string m_taskID; int32_t m_seq; + // the agency std::string m_from; + // the fromNode + bcos::bytes m_fromNode; + std::string m_uuid; }; diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h index 7b989243..a3d90fb2 100644 --- a/cpp/wedpr-helper/ppc-utilities/Utilities.h +++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h @@ -29,20 +29,31 @@ namespace ppc { template -inline uint64_t decodeNetworkBuffer( - T& _result, bcos::byte const* buffer, unsigned int bufferLen, uint64_t const offset) +inline uint64_t decodeNetworkBuffer(T& _result, bcos::byte const* buffer, unsigned int bufferLen, + uint64_t const offset, bool largeBuffer = false) { uint64_t curOffset = offset; - CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); // Notice: operator* is higher priority than operator+, the () is essential - auto dataLen = - boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)(buffer + curOffset))); - curOffset += 2; + uint32_t dataLen = 0; + if (largeBuffer) + { + CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset + 4, bufferLen); + dataLen = boost::asio::detail::socket_ops::network_to_host_long( + *((uint32_t*)(buffer + curOffset))); + curOffset += 4; + } + else + { + CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset + 2, bufferLen); + dataLen = boost::asio::detail::socket_ops::network_to_host_short( + *((uint16_t*)(buffer + curOffset))); + curOffset += 2; + } if (dataLen == 0) { return curOffset; } - CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset, bufferLen); + CHECK_OFFSET_WITH_THROW_EXCEPTION(curOffset + dataLen, bufferLen); _result.assign((bcos::byte*)buffer + curOffset, (bcos::byte*)buffer + curOffset + dataLen); curOffset += dataLen; return curOffset; diff --git a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp index 5378e6a8..f71e47ce 100644 --- a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp @@ -211,10 +211,17 @@ void JsonTaskImpl::decodeDataResourceDesc(DataResourceDesc::Ptr _desc, Json::Val BOOST_THROW_EXCEPTION( InvalidParameter() << errinfo_comment("The \"..\" cannot be in the path")); } - boost::filesystem::path prePath(m_prePath); - boost::filesystem::path inputPath(path); - boost::filesystem::path filePath(prePath / inputPath); - _desc->setPath(filePath.string()); + if (path.starts_with("/")) + { + _desc->setPath(path); + } + else + { + boost::filesystem::path prePath(m_prePath); + boost::filesystem::path inputPath(path); + boost::filesystem::path filePath(prePath / inputPath); + _desc->setPath(filePath.string()); + } } else { diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp index a316a773..efb7861f 100644 --- a/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.cpp @@ -17,8 +17,8 @@ * @author: shawnhe * @date 2022-10-19 */ - #include "PPCMessage.h" +#include "Common.h" #include #include @@ -98,7 +98,7 @@ int64_t PPCMessage::decode(uint32_t _length, bcos::byte* _data) p += dataLength; } - if (p) + if (p < _data + _length) { m_header.insert(m_header.begin(), p, _data + _length); } @@ -149,19 +149,20 @@ PPCMessageFace::Ptr PPCMessageFactory::decodePPCMessage(Message::Ptr msg) // Note: this field is been setted when onReceiveMessage if (frontMsg) { + ppcMsg->decode(bcos::ref(frontMsg->data())); ppcMsg->setSeq(frontMsg->seq()); ppcMsg->setUuid(frontMsg->traceID()); if (frontMsg->isRespPacket()) { ppcMsg->setResponse(); } - ppcMsg->decode(bcos::ref(frontMsg->data())); } if (msg->header() && msg->header()->optionalField()) { auto const& routeInfo = msg->header()->optionalField(); ppcMsg->setTaskID(routeInfo->topic()); ppcMsg->setSender(routeInfo->srcInst()); + ppcMsg->setSenderNode(routeInfo->srcNode()); } return ppcMsg; } diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.h b/cpp/wedpr-protocol/protocol/src/PPCMessage.h index 148e4fd3..4ce25faf 100644 --- a/cpp/wedpr-protocol/protocol/src/PPCMessage.h +++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.h @@ -83,6 +83,10 @@ class PPCMessage : public PPCMessageFace // set the message to be response void setResponse() override { m_isResponse = true; } + void setSenderNode(bcos::bytes const& senderNode) override { m_senderNode = senderNode; } + + bcos::bytes const& senderNode() const override { return m_senderNode; } + protected: std::string encodeMap(const std::map& _map); std::map decodeMap(const std::string& _encval); @@ -95,6 +99,8 @@ class PPCMessage : public PPCMessageFace uint32_t m_seq = 0; std::string m_taskID; std::string m_sender; + bcos::bytes m_senderNode; + bool m_isResponse; // the uuid used to find the response-callback std::string m_uuid; diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp index cc03557d..6690dd07 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp @@ -65,18 +65,19 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con auto pointer = data.data() + offset; m_componentType = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); bcos::bytes componentType; - offset = decodeNetworkBuffer(componentType, data.data(), data.size(), (pointer - data.data())); + offset = decodeNetworkBuffer( + componentType, data.data(), data.size(), (pointer - data.data()), false); m_componentType = std::string(componentType.begin(), componentType.end()); // srcNode - offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_srcNode, data.data(), data.size(), offset, false); // source inst - offset = decodeNetworkBuffer(m_srcInst, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_srcInst, data.data(), data.size(), offset, false); // dstNode - offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset, false); // dstInst - offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset, false); // topic - offset = decodeNetworkBuffer(m_topic, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_topic, data.data(), data.size(), offset, false); return offset; } @@ -149,11 +150,12 @@ int64_t MessageHeaderImpl::decode(bcos::bytesConstRef data) m_ext = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); pointer += 2; // the traceID - auto offset = decodeNetworkBuffer(m_traceID, data.data(), data.size(), (pointer - data.data())); + auto offset = + decodeNetworkBuffer(m_traceID, data.data(), data.size(), (pointer - data.data()), false); // srcGwNode - offset = decodeNetworkBuffer(m_srcGwNode, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_srcGwNode, data.data(), data.size(), offset, false); // dstGwNode - offset = decodeNetworkBuffer(m_dstGwNode, data.data(), data.size(), offset); + offset = decodeNetworkBuffer(m_dstGwNode, data.data(), data.size(), offset, false); // optionalField if (hasOptionalField()) { diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp index 7e2f37b8..07c4327f 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp @@ -42,8 +42,8 @@ int64_t MessagePayloadImpl::encode(bcos::bytes& buffer) const buffer.insert(buffer.end(), (byte*)&traceIDLen, (byte*)&traceIDLen + 2); buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end()); // data - uint16_t dataLen = boost::asio::detail::socket_ops::host_to_network_short(m_data.size()); - buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 2); + uint32_t dataLen = boost::asio::detail::socket_ops::host_to_network_long(m_data.size()); + buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 4); buffer.insert(buffer.end(), m_data.begin(), m_data.end()); // update the length m_length = buffer.size(); @@ -74,5 +74,5 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) auto offset = decodeNetworkBuffer(m_traceID, buffer.data(), buffer.size(), (pointer - buffer.data())); // data - return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset); + return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset, true); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp index 53977bbe..cf7bfce5 100644 --- a/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp +++ b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp @@ -123,6 +123,16 @@ BOOST_AUTO_TEST_CASE(testMessage) componentType, srcNode, srcInst, dstNode, dstInst, payload); checkEncodeDecode(msgBuilder, msg); + // with payload over 65535 + for (uint32_t i = 0; i < 10000000; i++) + { + payload->emplace_back(i); + } + msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, + componentType, srcNode, srcInst, dstNode, dstInst, payload); + checkEncodeDecode(msgBuilder, msg); + + // with header router traceID = "1233"; srcGwNode = "srcGwNode"; diff --git a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp index c312bed5..279984c3 100644 --- a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp +++ b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp @@ -232,6 +232,14 @@ void DataResourceLoaderImpl::checkResourceExists( switch (_desc->type()) { case (int)(DataResourceType::FILE): + { + if (boost::filesystem::exists(boost::filesystem::path(_desc->path()))) + { + BOOST_THROW_EXCEPTION(LoadDataResourceException() << bcos::errinfo_comment( + "The file: " + _desc->path() + " already exists!")); + } + break; + } case (int)(DataResourceType::HDFS): { auto storage = _storage; diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index a3259cd6..153e45ea 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -138,9 +138,13 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace } // send response when receiving message from given agencyID -void Front::asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, - front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) -{} +void Front::asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + PPCMessageFace::Ptr message, ErrorCallbackFunc _callback) +{ + bcos::bytes data; + message->encode(data); + m_front->asyncSendResponse(dstNode, traceID, std::move(data), 0, _callback); +} /** * @brief notice task info to gateway @@ -148,11 +152,43 @@ void Front::asyncSendResponse(const std::string& _agencyID, std::string const& _ */ bcos::Error::Ptr Front::notifyTaskInfo(std::string const& taskID) { - m_front->registerTopic(taskID); + return m_front->registerTopic(taskID); } // erase the task-info when task finished bcos::Error::Ptr Front::eraseTaskInfo(std::string const& _taskID) { - m_front->unRegisterTopic(_taskID); + FRONT_LOG(INFO) << LOG_DESC("eraseTaskInfo") << LOG_KV("front", m_front); + return m_front->unRegisterTopic(_taskID); +} + +// register message handler for algorithm +void Front::registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, + std::function _handler) +{ + uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; + auto self = weak_from_this(); + m_front->registerMessageHandler( + std::to_string(type), [self, type, _handler](ppc::protocol::Message::Ptr msg) { + auto front = self.lock(); + if (!front) + { + return; + } + try + { + if (msg == nullptr) + { + _handler(nullptr); + return; + } + _handler(front->m_messageFactory->decodePPCMessage(msg)); + } + catch (std::exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("Call handler for component failed") + << LOG_KV("componentType", type) + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h index 4efa9b5d..5ddd224b 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.h @@ -48,9 +48,8 @@ class Front : public FrontInterface, public std::enable_shared_from_this uint32_t _timeout, ErrorCallbackFunc _callback, CallbackFunc _respCallback) override; // send response when receiving message from given agencyID - void asyncSendResponse(const std::string& _agencyID, std::string const& _uuid, - front::PPCMessageFace::Ptr _message, ErrorCallbackFunc _callback) override; - + void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + front::PPCMessageFace::Ptr message, ErrorCallbackFunc _callback) override; /** * @brief notice task info to gateway * @param _taskInfo the latest task information @@ -62,25 +61,7 @@ class Front : public FrontInterface, public std::enable_shared_from_this // register message handler for algorithm void registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, - std::function _handler) override - { - uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType; - auto self = weak_from_this(); - m_front->registerMessageHandler( - std::to_string(type), [self, _handler](ppc::protocol::Message::Ptr msg) { - auto front = self.lock(); - if (!front) - { - return; - } - if (msg == nullptr) - { - _handler(nullptr); - return; - } - _handler(front->m_messageFactory->decodePPCMessage(msg)); - }); - } + std::function _handler) override; std::vector agencies() const override { diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index 470462a2..920ceeb2 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -37,7 +37,7 @@ FrontImpl::FrontImpl(std::shared_ptr threadPool, m_gatewayClient(gateway) { m_nodeID = m_nodeInfo->nodeID().toBytes(); - m_callbackManager = std::make_shared(m_threadPool, ioService); + m_callbackManager = std::make_shared(m_threadPool, m_ioService); } /** @@ -106,6 +106,23 @@ void FrontImpl::stop() } } +void FrontImpl::asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) +{ + // generate the frontMessage + auto frontMessage = m_messageFactory->build(); + frontMessage->setTraceID(traceID); + frontMessage->setSeq(seq); + frontMessage->setData(std::move(payload)); + + auto routeInfo = m_routerInfoBuilder->build(); + routeInfo->setSrcNode(m_nodeID); + routeInfo->setDstNode(dstNode); + + asyncSendMessageToGateway(true, std::move(frontMessage), RouteType::ROUTE_THROUGH_NODEID, + traceID, routeInfo, -1, errorCallback); +} + /** * @brief async send message * diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index b1c36b66..b60f1e87 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -114,20 +114,20 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ * @brief register the nodeInfo to the gateway * @param nodeInfo the nodeInfo */ - void registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override + bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) override { FRONT_LOG(INFO) << LOG_DESC("registerNodeInfo") << LOG_KV("nodeInfo", printNodeInfo(m_nodeInfo)); - m_gatewayClient->registerNodeInfo(m_nodeInfo); + return m_gatewayClient->registerNodeInfo(m_nodeInfo); } /** * @brief unRegister the nodeInfo to the gateway */ - void unRegisterNodeInfo() override + bcos::Error::Ptr unRegisterNodeInfo() override { FRONT_LOG(INFO) << LOG_DESC("unRegisterNodeInfo"); - m_gatewayClient->unRegisterNodeInfo(bcos::ref(m_nodeID)); + return m_gatewayClient->unRegisterNodeInfo(bcos::ref(m_nodeID)); } /** @@ -135,10 +135,10 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ * * @param topic the topic to register */ - void registerTopic(std::string const& topic) override + bcos::Error::Ptr registerTopic(std::string const& topic) override { FRONT_LOG(INFO) << LOG_DESC("register topic: ") << topic; - m_gatewayClient->registerTopic(bcos::ref(m_nodeID), topic); + return m_gatewayClient->registerTopic(bcos::ref(m_nodeID), topic); } void asyncGetAgencies( @@ -152,10 +152,10 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ * * @param topic the topic to unregister */ - void unRegisterTopic(std::string const& topic) override + bcos::Error::Ptr unRegisterTopic(std::string const& topic) override { FRONT_LOG(INFO) << LOG_DESC("unregister topic: ") << topic; - m_gatewayClient->unRegisterTopic(bcos::ref(m_nodeID), topic); + return m_gatewayClient->unRegisterTopic(bcos::ref(m_nodeID), topic); } ppc::protocol::MessageOptionalHeaderBuilder::Ptr const routerInfoBuilder() const @@ -167,6 +167,9 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ return m_messageFactory; } + void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) override; + private: void asyncSendMessageToGateway(bool responsePacket, ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType, diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 18163caa..7a997afc 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -190,7 +190,7 @@ void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr sessi "onReceiveP2PMessage failed to find the node that can dispatch this message") << LOG_KV("msg", printMessage(p2pMessage)); callback(std::make_shared(CommonError::NotFoundFrontServiceDispatchMsg, - "unable to find the ndoe to dispatcher this message, message detail: " + + "unable to find the node to dispatcher this message, message detail: " + printMessage(p2pMessage))); } } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp index b83b07ff..eeb724f8 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.cpp @@ -46,8 +46,8 @@ void SendMessageWithRetry::trySendMessage() { if (m_dstNodeList.empty()) { - GATEWAY_LOG(DEBUG) << LOG_DESC("Gateway::SendMessageWithRetry") - << LOG_DESC("unable to send the message") << printMessage(m_p2pMessage); + GATEWAY_LOG(DEBUG) << LOG_DESC("Gateway::SendMessageWithRetry: unable to send the message") + << printMessage(m_p2pMessage); if (m_respFunc) { m_respFunc(std::make_shared( diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index 4ed8d833..470d4b3b 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -50,7 +50,7 @@ class GatewayNodeInfo virtual std::vector> chooseRouterByAgency( bool selectAll) const = 0; virtual std::vector> chooseRouterByTopic( - bool selectAll, std::string const& topic) const = 0; + bool selectAll, bcos::bytes const& fromNode, std::string const& topic) const = 0; virtual void encode(bcos::bytes& data) const = 0; virtual void decode(bcos::bytesConstRef data) = 0; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp index fb14954b..0dda811e 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -18,6 +18,7 @@ * @date 2024-08-26 */ #include "GatewayNodeInfoImpl.h" +#include "ppc-gateway/Common.h" #include "wedpr-protocol/protobuf/src/Common.h" #include "wedpr-protocol/protobuf/src/NodeInfoImpl.h" #include "wedpr-protocol/tars/Common.h" @@ -152,7 +153,7 @@ std::vector> GatewayNodeInfoImpl::choo } std::vector> GatewayNodeInfoImpl::chooseRouterByTopic( - bool selectAll, std::string const& topic) const + bool selectAll, bcos::bytes const& fromNode, std::string const& topic) const { std::vector> result; // empty topic means broadcast message to all front @@ -174,7 +175,8 @@ std::vector> GatewayNodeInfoImpl::choo { selectedNode = nodeInfo(it.first); } - if (selectedNode != nullptr) + // ignore the fromNode + if (selectedNode != nullptr && selectedNode->nodeID().toBytes() != fromNode) { result.emplace_back(selectedNode->getFront()); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index 13f0bf25..fbe3a205 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -61,7 +61,7 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo std::vector> chooseRouterByAgency( bool selectAll) const override; std::vector> chooseRouterByTopic( - bool selectAll, std::string const& topic) const override; + bool selectAll, bcos::bytes const& fromNode, std::string const& topic) const override; void registerTopic(bcos::bytes const& nodeID, std::string const& topic) override; void unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) override; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 2b99dec6..3a7c035f 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -103,7 +103,8 @@ std::vector LocalRouter::chooseReceiver( ppc::protocol::Message::Ptr const& msg) { std::vector receivers; - if (msg->header()->optionalField()->dstInst() != m_routerInfo->agency()) + auto const& dstInst = msg->header()->optionalField()->dstInst(); + if (!dstInst.empty() && dstInst != m_routerInfo->agency()) { return receivers; } @@ -123,17 +124,20 @@ std::vector LocalRouter::chooseReceiver( } case (uint16_t)RouteType::ROUTE_THROUGH_COMPONENT: { + // Note: should check the dstInst when route-by-component return m_routerInfo->chooseRouteByComponent( selectAll, msg->header()->optionalField()->componentType()); } case (uint16_t)RouteType::ROUTE_THROUGH_AGENCY: { + // Note: should check the dstInst when route-by-agency return m_routerInfo->chooseRouterByAgency(selectAll); } case (uint16_t)RouteType::ROUTE_THROUGH_TOPIC: { - return m_routerInfo->chooseRouterByTopic( - selectAll, msg->header()->optionalField()->topic()); + // Note: should ignore the srcNode when route-by-topic + return m_routerInfo->chooseRouterByTopic(selectAll, + msg->header()->optionalField()->srcNode(), msg->header()->optionalField()->topic()); } default: BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment( diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp index 3e96a31f..dfca0e5c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -40,7 +40,8 @@ Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _rou m_routerTable->setNodeID(m_nodeID); m_routerTable->setUnreachableDistance(unreachableDistance); - SERVICE_LOG(INFO) << LOG_DESC("create P2PService") << LOG_KV("module", _moduleName); + SERVICE_LOG(INFO) << LOG_DESC("create P2PService") << LOG_KV("module", _moduleName) + << LOG_KV("nodeID", printP2PIDElegantly(m_nodeID)); WsService::registerConnectHandler( boost::bind(&Service::onP2PConnect, this, boost::placeholders::_1)); WsService::registerDisconnectHandler( @@ -224,10 +225,12 @@ void Service::asyncSendMessageWithForward( auto nextHop = m_routerTable->getNextHop(dstNodeID); if (nextHop.empty()) { + SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessage directly") << printMessage(p2pMsg); return asyncSendMessage(dstNodeID, msg, options, respFunc); } // with nextHop, send the message to nextHop - SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageWithForward") << printMessage(p2pMsg); + SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageWithForward to nextHop") + << printMessage(p2pMsg); return asyncSendMessage(nextHop, msg, options, respFunc); } @@ -346,7 +349,14 @@ void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& requestMsg->header()->optionalField()->srcNode()); respMessage->header()->optionalField()->setSrcNode( requestMsg->header()->optionalField()->dstNode()); + + respMessage->header()->optionalField()->setDstInst( + requestMsg->header()->optionalField()->srcInst()); + respMessage->header()->optionalField()->setSrcInst( + requestMsg->header()->optionalField()->dstInst()); } + respMessage->header()->setSrcGwNode(requestMsg->header()->dstGwNode()); + respMessage->header()->setDstGwNode(requestMsg->header()->srcGwNode()); respMessage->header()->setTraceID(requestMsg->header()->traceID()); respMessage->header()->setRespPacket(); respMessage->header()->setRouteType(ppc::protocol::RouteType::ROUTE_THROUGH_NODEID); From 7c679e754afa15e2b04e91218b09ca04e8f7a10e Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:16:32 +0800 Subject: [PATCH 032/120] update secure lr (#28) --- python/ppc_common/ppc_crypto/ihc_cipher.py | 3 + python/ppc_model/common/model_setting.py | 2 +- python/ppc_model/common/protocol.py | 2 + python/ppc_model/datasets/dataset.py | 12 +- .../feature_binning/feature_binning.py | 2 +- .../ppc_model/datasets/test/test_dataset.py | 10 +- python/ppc_model/metrics/loss.py | 18 +- python/ppc_model/ppc_model_app.py | 3 + .../secure_lgbm/secure_lgbm_context.py | 22 +- .../test/test_secure_lgbm_context.py | 38 +-- .../secure_lgbm/vertical/active_party.py | 2 +- .../ppc_model/secure_lgbm/vertical/booster.py | 8 +- .../secure_lgbm/vertical/passive_party.py | 2 +- python/ppc_model/secure_lr/__init__.py | 0 .../ppc_model/secure_lr/secure_lr_context.py | 220 +++++++++++++ .../secure_lr/secure_lr_training_engine.py | 40 +++ python/ppc_model/secure_lr/test/__init__.py | 0 .../secure_lr/test/test_enc_matmul.py | 64 ++++ python/ppc_model/secure_lr/test/test_lr.py | 207 ++++++++++++ .../test_secure_lr_performance_training.py | 179 ++++++++++ .../secure_lr/test/test_secure_lr_training.py | 188 +++++++++++ .../ppc_model/secure_lr/vertical/__init__.py | 4 + .../secure_lr/vertical/active_party.py | 165 ++++++++++ .../ppc_model/secure_lr/vertical/booster.py | 308 ++++++++++++++++++ .../secure_lr/vertical/passive_party.py | 125 +++++++ 25 files changed, 1574 insertions(+), 50 deletions(-) create mode 100644 python/ppc_model/secure_lr/__init__.py create mode 100644 python/ppc_model/secure_lr/secure_lr_context.py create mode 100644 python/ppc_model/secure_lr/secure_lr_training_engine.py create mode 100644 python/ppc_model/secure_lr/test/__init__.py create mode 100644 python/ppc_model/secure_lr/test/test_enc_matmul.py create mode 100644 python/ppc_model/secure_lr/test/test_lr.py create mode 100644 python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py create mode 100644 python/ppc_model/secure_lr/test/test_secure_lr_training.py create mode 100644 python/ppc_model/secure_lr/vertical/__init__.py create mode 100644 python/ppc_model/secure_lr/vertical/active_party.py create mode 100644 python/ppc_model/secure_lr/vertical/booster.py create mode 100644 python/ppc_model/secure_lr/vertical/passive_party.py diff --git a/python/ppc_common/ppc_crypto/ihc_cipher.py b/python/ppc_common/ppc_crypto/ihc_cipher.py index 54061208..a0f7c132 100644 --- a/python/ppc_common/ppc_crypto/ihc_cipher.py +++ b/python/ppc_common/ppc_crypto/ihc_cipher.py @@ -19,6 +19,9 @@ def __add__(self, other): cipher_left = self.c_left + other.c_left cipher_right = self.c_right + other.c_right return IhcCiphertext(cipher_left, cipher_right) + + def __mul__(self, num: int): + return IhcCiphertext(num * self.c_left, num * self.c_right) def __eq__(self, other): return self.c_left == other.c_left and self.c_right == other.c_right diff --git a/python/ppc_model/common/model_setting.py b/python/ppc_model/common/model_setting.py index ab139ec0..4d1aa98e 100644 --- a/python/ppc_model/common/model_setting.py +++ b/python/ppc_model/common/model_setting.py @@ -36,7 +36,7 @@ def __init__(self, model_dict): "iv_thresh", 0.1, model_dict, False)) self.use_goss = common_func.get_config_value( "use_goss", False, model_dict, False) - self.test_dataset_percentage = float(common_func.get_config_value( + self.test_size = float(common_func.get_config_value( "test_dataset_percentage", 0.3, model_dict, False)) self.learning_rate = float(common_func.get_config_value( "learning_rate", 0.1, model_dict, False)) diff --git a/python/ppc_model/common/protocol.py b/python/ppc_model/common/protocol.py index 2ec8f65d..31bc4ded 100644 --- a/python/ppc_model/common/protocol.py +++ b/python/ppc_model/common/protocol.py @@ -15,6 +15,8 @@ class ModelTask(Enum): FEATURE_ENGINEERING = "FEATURE_ENGINEERING" XGB_TRAINING = "XGB_TRAINING" XGB_PREDICTING = "XGB_PREDICTING" + LR_TRAINING = "LR_TRAINING" + LR_PREDICTING = "LR_PREDICTING" class TaskStatus(Enum): diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py index a8db9c58..1e3a33ff 100644 --- a/python/ppc_model/datasets/dataset.py +++ b/python/ppc_model/datasets/dataset.py @@ -18,11 +18,11 @@ def __init__(self, ctx: SecureLGBMContext, model_data=None, delimiter: str = ' ' self.selected_col_file = ctx.selected_col_file self.is_label_holder = ctx.is_label_holder self.algorithm_type = ctx.algorithm_type - self.test_size = ctx.lgbm_params.test_size - self.random_state = ctx.lgbm_params.random_state - self.eval_set_column = ctx.lgbm_params.eval_set_column - self.train_set_value = ctx.lgbm_params.train_set_value - self.eval_set_value = ctx.lgbm_params.eval_set_value + self.test_size = ctx.model_params.test_size + self.random_state = ctx.model_params.random_state + self.eval_set_column = ctx.model_params.eval_set_column + self.train_set_value = ctx.model_params.train_set_value + self.eval_set_value = ctx.model_params.eval_set_value self.ctx = ctx self.train_X = None @@ -197,7 +197,7 @@ def _construct_dataset(self): and not os.path.exists(self.selected_col_file): try: self.ctx.remote_selected_col_file = os.path.join( - self.ctx.lgbm_params.training_job_id, self.ctx.SELECTED_COL_FILE) + self.ctx.model_params.training_job_id, self.ctx.SELECTED_COL_FILE) ResultFileHandling._download_file(self.ctx.components.storage_client, self.selected_col_file, self.ctx.remote_selected_col_file) self._dataset_fe_selected(self.selected_col_file, 'id') diff --git a/python/ppc_model/datasets/feature_binning/feature_binning.py b/python/ppc_model/datasets/feature_binning/feature_binning.py index 91c5ab3d..9baa230c 100644 --- a/python/ppc_model/datasets/feature_binning/feature_binning.py +++ b/python/ppc_model/datasets/feature_binning/feature_binning.py @@ -8,7 +8,7 @@ class FeatureBinning: def __init__(self, ctx: Context): self.ctx = ctx - self.params = ctx.lgbm_params + self.params = ctx.model_params self.data = None self.data_bin = None self.data_split = None diff --git a/python/ppc_model/datasets/test/test_dataset.py b/python/ppc_model/datasets/test/test_dataset.py index e7519454..8fec5b8e 100644 --- a/python/ppc_model/datasets/test/test_dataset.py +++ b/python/ppc_model/datasets/test/test_dataset.py @@ -73,7 +73,7 @@ def test_random_split_dataset(self): } } task_info = SecureLGBMContext(args, self.components) - print(task_info.lgbm_params.get_all_params()) + print(task_info.model_params.get_all_params()) # 模拟构造主动方数据集 dataset_with_y = SecureDataset(task_info, self.df_with_y) @@ -99,7 +99,7 @@ def test_random_split_dataset(self): } } task_info = SecureLGBMContext(args, self.components) - print(task_info.lgbm_params.get_all_params()) + print(task_info.model_params.get_all_params()) # 模拟构造被动方数据集 dataset_without_y = SecureDataset(task_info, self.df_without_y) @@ -128,7 +128,7 @@ def test_customized_split_dataset(self): } } task_info = SecureLGBMContext(args, self.components) - print(task_info.lgbm_params.get_all_params()) + print(task_info.model_params.get_all_params()) # 模拟构造主动方数据集 task_info.eval_column_file = self.eval_column_file @@ -158,7 +158,7 @@ def test_predict_dataset(self): 'model_dict': {} } task_info = SecureLGBMContext(args, self.components) - print(task_info.lgbm_params.get_all_params()) + print(task_info.model_params.get_all_params()) # 模拟构造主动方数据集 task_info.model_prepare_file = self.df_with_y_file @@ -184,7 +184,7 @@ def test_iv_selected_dataset(self): 'model_dict': {} } task_info = SecureLGBMContext(args, self.components) - print(task_info.lgbm_params.get_all_params()) + print(task_info.model_params.get_all_params()) # 模拟构造主动方数据集 task_info.model_prepare_file = self.df_with_y_file diff --git a/python/ppc_model/metrics/loss.py b/python/ppc_model/metrics/loss.py index b2b4e6b3..817d4f7c 100644 --- a/python/ppc_model/metrics/loss.py +++ b/python/ppc_model/metrics/loss.py @@ -7,7 +7,7 @@ class Loss: class BinaryLoss(Loss): - def __init__(self, objective: str) -> None: + def __init__(self, objective: str = None) -> None: super().__init__() self.objective = objective @@ -30,3 +30,19 @@ def compute_loss(y_true: np.ndarray, y_pred: np.ndarray): epsilon = 1e-15 y_pred = np.clip(y_pred, epsilon, 1 - epsilon) return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred)) + + @staticmethod + def dot_product(x, theta): + if x.ndim == 1: + x.reshape(1, len(x)) + if theta.ndim == 1: + theta.reshape(len(theta), 1) + g = np.matmul(x, theta) + return g + + @staticmethod + def inference(g): + # h = np.divide(np.exp(g), np.exp(g) + 1) + # 近似 + h = 0.125 * g + return h diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py index 79060c82..dc5865ea 100644 --- a/python/ppc_model/ppc_model_app.py +++ b/python/ppc_model/ppc_model_app.py @@ -1,6 +1,7 @@ # Note: here can't be refactored by autopep from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine +from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine from ppc_model.network.http.restx import api from ppc_model.network.http.model_controller import ns2 as log_namespace @@ -49,6 +50,8 @@ def register_task_handler(): ModelTask.XGB_TRAINING, SecureLGBMTrainingEngine.run) task_manager.register_task_handler( ModelTask.XGB_PREDICTING, SecureLGBMPredictionEngine.run) + task_manager.register_task_handler( + ModelTask.LR_TRAINING, SecureLRTrainingEngine.run) def model_serve(): diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py index 5475185b..ea536a14 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -221,28 +221,28 @@ def __init__(self, else: self.dataset_file_path = None - self.lgbm_params = SecureLGBMParams() + self.model_params = SecureLGBMParams() model_setting = ModelSetting(args['model_dict']) - self.set_lgbm_params(model_setting) + self.set_model_params(model_setting) if model_setting.train_features is not None and len(model_setting.train_features) > 0: - self.lgbm_params.train_feature = model_setting.train_features.split( + self.model_params.train_feature = model_setting.train_features.split( ',') - self.lgbm_params.n_estimators = model_setting.num_trees - self.lgbm_params.feature_rate = model_setting.colsample_bytree - self.lgbm_params.min_split_gain = model_setting.gamma - self.lgbm_params.random_state = model_setting.seed + self.model_params.n_estimators = model_setting.num_trees + self.model_params.feature_rate = model_setting.colsample_bytree + self.model_params.min_split_gain = model_setting.gamma + self.model_params.random_state = model_setting.seed self.sync_file_list = {} if self.algorithm_type == AlgorithmType.Train.name: self.set_sync_file() - def set_lgbm_params(self, model_setting: ModelSetting): + def set_model_params(self, model_setting: ModelSetting): """设置lgbm参数""" - self.lgbm_params.set_model_setting(model_setting) + self.model_params.set_model_setting(model_setting) - def get_lgbm_params(self): + def get_model_params(self): """获取lgbm参数""" - return self.lgbm_params + return self.model_params def set_sync_file(self): self.sync_file_list['metrics_iteration'] = [self.metrics_iteration_file, self.remote_metrics_iteration_file] diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py index 2797194c..b04b9218 100644 --- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py @@ -11,7 +11,7 @@ class TestSecureLGBMContext(unittest.TestCase): components.config_data = {'JOB_TEMP_DIR': '/tmp'} components.mock_logger = MockLogger() - def test_get_lgbm_params(self): + def test_get_model_params(self): args = { 'job_id': 'j-123', @@ -26,15 +26,15 @@ def test_get_lgbm_params(self): } task_info = SecureLGBMContext(args, self.components) - lgbm_params = task_info.get_lgbm_params() + model_params = task_info.get_model_params() # 打印LGBMModel默认参数 - print(lgbm_params._get_params()) + print(model_params._get_params()) # 默认自定义参数为空字典 - assert lgbm_params.get_params() == {} - # assert lgbm_params.get_all_params() != lgbm_params._get_params() + assert model_params.get_params() == {} + # assert model_params.get_all_params() != model_params._get_params() - def test_set_lgbm_params(self): + def test_set_model_params(self): args = { 'job_id': 'j-123', @@ -49,28 +49,28 @@ def test_set_lgbm_params(self): 'objective': 'regression', 'n_estimators': 6, 'max_depth': 3, - 'test_size': 0.2, + 'test_dataset_percentage': 0.2, 'use_goss': 1 } } task_info = SecureLGBMContext(args, self.components) - lgbm_params = task_info.get_lgbm_params() + model_params = task_info.get_model_params() # 打印SecureLGBMParams自定义参数 - print(lgbm_params.get_params()) + print(model_params.get_params()) # 打印SecureLGBMParams所有参数 - print(lgbm_params.get_all_params()) + print(model_params.get_all_params()) - assert lgbm_params.get_params() == args['model_dict'] - self.assertEqual(lgbm_params.get_all_params()[ - 'learning_rate'], lgbm_params._get_params()['learning_rate']) - self.assertEqual(lgbm_params.learning_rate, - lgbm_params._get_params()['learning_rate']) - self.assertEqual(lgbm_params.n_estimators, + # assert model_params.get_params() == args['model_dict'] + self.assertEqual(model_params.get_all_params()[ + 'learning_rate'], model_params._get_params()['learning_rate']) + self.assertEqual(model_params.learning_rate, + model_params._get_params()['learning_rate']) + self.assertEqual(model_params.n_estimators, args['model_dict']['n_estimators']) - self.assertEqual(lgbm_params.test_size, - args['model_dict']['test_size']) - self.assertEqual(lgbm_params.use_goss, args['model_dict']['use_goss']) + self.assertEqual(model_params.test_size, + args['model_dict']['test_dataset_percentage']) + self.assertEqual(model_params.use_goss, args['model_dict']['use_goss']) if __name__ == "__main__": diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py index 4078ec35..ddec192a 100644 --- a/python/ppc_model/secure_lgbm/vertical/active_party.py +++ b/python/ppc_model/secure_lgbm/vertical/active_party.py @@ -27,7 +27,7 @@ class VerticalLGBMActiveParty(VerticalBooster): def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: super().__init__(ctx, dataset) - self.params = ctx.lgbm_params + self.params = ctx.model_params self._loss_func = BinaryLoss(self.params.objective) self._all_feature_name = [dataset.feature_name] self._all_feature_num = len(dataset.feature_name) diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index c191b6f2..9bfab7b6 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -33,8 +33,8 @@ def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: self._test_weights = None self._test_praba = None - random.seed(ctx.lgbm_params.random_state) - np.random.seed(ctx.lgbm_params.random_state) + random.seed(ctx.model_params.random_state) + np.random.seed(ctx.model_params.random_state) def _build_tree(self, *args, **kwargs): @@ -250,9 +250,9 @@ def load_model(self, file_path=None): file_path, self.ctx.MODEL_DATA_FILE) if self.ctx.algorithm_type == AlgorithmType.Predict.name: self.ctx.remote_feature_bin_file = os.path.join( - self.ctx.lgbm_params.training_job_id, self.ctx.FEATURE_BIN_FILE) + self.ctx.model_params.training_job_id, self.ctx.FEATURE_BIN_FILE) self.ctx.remote_model_data_file = os.path.join( - self.ctx.lgbm_params.training_job_id, self.ctx.MODEL_DATA_FILE) + self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) ResultFileHandling._download_file(self.ctx.components.storage_client, self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) diff --git a/python/ppc_model/secure_lgbm/vertical/passive_party.py b/python/ppc_model/secure_lgbm/vertical/passive_party.py index 321f3651..31e47826 100644 --- a/python/ppc_model/secure_lgbm/vertical/passive_party.py +++ b/python/ppc_model/secure_lgbm/vertical/passive_party.py @@ -15,7 +15,7 @@ class VerticalLGBMPassiveParty(VerticalBooster): def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: super().__init__(ctx, dataset) - self.params = ctx.lgbm_params + self.params = ctx.model_params self.log = ctx.components.logger() self.log.info( f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') diff --git a/python/ppc_model/secure_lr/__init__.py b/python/ppc_model/secure_lr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lr/secure_lr_context.py b/python/ppc_model/secure_lr/secure_lr_context.py new file mode 100644 index 00000000..4ecebb9f --- /dev/null +++ b/python/ppc_model/secure_lr/secure_lr_context.py @@ -0,0 +1,220 @@ +import os +from enum import Enum +from typing import Any, Dict +from sklearn.base import BaseEstimator + +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_common.ppc_crypto.phe_factory import PheCipherFactory +from ppc_model.common.context import Context +from ppc_model.common.initializer import Initializer +from ppc_model.common.protocol import TaskRole +from ppc_common.ppc_utils import common_func +from ppc_model.common.model_setting import ModelSetting + + +class LRModel(BaseEstimator): + + def __init__( + self, + epochs: int = 10, + batch_size: int = 8, + learning_rate: float = 0.1, + random_state: int = None, + n_jobs: int = None, + **kwargs + ): + + self.epochs = epochs + self.batch_size = batch_size + self.learning_rate = learning_rate + self.random_state = random_state + self.n_jobs = n_jobs + self._other_params: Dict[str, Any] = {} + self.set_params(**kwargs) + + def get_params(self, deep: bool = True) -> Dict[str, Any]: + """Get parameters for this estimator. + + Parameters + ---------- + deep : bool, optional (default=True) + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : dict + Parameter names mapped to their values. + """ + params = super().get_params(deep=deep) + params.update(self._other_params) + return params + + def set_model_setting(self, model_setting: ModelSetting) -> "LRModel": + # 获取对象的所有属性名 + attrs = dir(model_setting) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + setattr(self, attr, getattr(model_setting, attr)) + except Exception as e: + pass + return self + + def set_params(self, **params: Any) -> "LRModel": + """Set the parameters of this estimator. + + Parameters + ---------- + **params + Parameter names with their new values. + + Returns + ------- + self : object + Returns self. + """ + for key, value in params.items(): + setattr(self, key, value) + if hasattr(self, f"_{key}"): + setattr(self, f"_{key}", value) + self._other_params[key] = value + return self + + +class ModelTaskParams(LRModel): + def __init__( + self, + test_size: float = 0.3, + feature_rate: float = 1.0, + eval_set_column: str = None, + train_set_value: str = None, + eval_set_value: str = None, + train_feats: str = None, + verbose_eval: int = 1, + categorical_feature: list = [], + silent: bool = False + ): + + super().__init__() + + self.test_size = test_size + self.feature_rate = feature_rate + self.eval_set_column = eval_set_column + self.train_set_value = train_set_value + self.eval_set_value = eval_set_value + self.train_feature = train_feats + self.verbose_eval = verbose_eval + self.silent = silent + self.lr = self.learning_rate + self.categorical_feature = categorical_feature + self.categorical_idx = [] + self.my_categorical_idx = [] + + +class SecureLRParams(ModelTaskParams): + + def __init__(self): + super().__init__() + + def _get_params(self): + """返回LRClassifier所有参数""" + return LRModel().get_params() + + def get_all_params(self): + """返回SecureLRParams所有参数""" + # 获取对象的所有属性名 + attrs = dir(self) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + # 使用getattr来获取属性的值 + value = getattr(self, attr) + # 检查value是否可调用（例如，方法或函数），如果是，则不打印其值 + if not callable(value): + params[attr] = value + except Exception as e: + pass + return params + + +class SecureLRContext(Context): + + def __init__(self, + args, + components: Initializer + ): + + if args['is_label_holder']: + role = TaskRole.ACTIVE_PARTY + else: + role = TaskRole.PASSIVE_PARTY + + super().__init__(args['job_id'], + args['task_id'], + components, + role) + + self.phe = PheCipherFactory.build_phe( + components.homo_algorithm, components.public_key_length) + self.codec = PheCipherFactory.build_codec(components.homo_algorithm) + self.is_label_holder = args['is_label_holder'] + self.result_receiver_id_list = args['result_receiver_id_list'] + self.participant_id_list = args['participant_id_list'] + self.model_predict_algorithm = common_func.get_config_value( + "model_predict_algorithm", None, args, False) + self.algorithm_type = args['algorithm_type'] + if 'dataset_id' in args and args['dataset_id'] is not None: + self.dataset_file_path = os.path.join( + self.workspace, args['dataset_id']) + else: + self.dataset_file_path = None + + self.model_params = SecureLRParams() + model_setting = ModelSetting(args['model_dict']) + self.set_model_params(model_setting) + if model_setting.train_features is not None and len(model_setting.train_features) > 0: + self.model_params.train_feature = model_setting.train_features.split( + ',') + self.model_params.random_state = model_setting.seed + self.sync_file_list = {} + if self.algorithm_type == AlgorithmType.Train.name: + self.set_sync_file() + + def set_model_params(self, model_setting: ModelSetting): + """设置lr参数""" + self.model_params.set_model_setting(model_setting) + + def get_model_params(self): + """获取lr参数""" + return self.model_params + + def set_sync_file(self): + self.sync_file_list['summary_evaluation'] = [self.summary_evaluation_file, self.remote_summary_evaluation_file] + self.sync_file_list['train_ks_table'] = [self.train_metric_ks_table, self.remote_train_metric_ks_table] + self.sync_file_list['train_metric_roc'] = [self.train_metric_roc_file, self.remote_train_metric_roc_file] + self.sync_file_list['train_metric_ks'] = [self.train_metric_ks_file, self.remote_train_metric_ks_file] + self.sync_file_list['train_metric_pr'] = [self.train_metric_pr_file, self.remote_train_metric_pr_file] + self.sync_file_list['train_metric_acc'] = [self.train_metric_acc_file, self.remote_train_metric_acc_file] + self.sync_file_list['test_ks_table'] = [self.test_metric_ks_table, self.remote_test_metric_ks_table] + self.sync_file_list['test_metric_roc'] = [self.test_metric_roc_file, self.remote_test_metric_roc_file] + self.sync_file_list['test_metric_ks'] = [self.test_metric_ks_file, self.remote_test_metric_ks_file] + self.sync_file_list['test_metric_pr'] = [self.test_metric_pr_file, self.remote_test_metric_pr_file] + self.sync_file_list['test_metric_acc'] = [self.test_metric_acc_file, self.remote_test_metric_acc_file] + + +class LRMessage(Enum): + FEATURE_NAME = "FEATURE_NAME" + ENC_D_LIST = "ENC_D_LIST" + ENC_D_HIST = "ENC_D_HIST" + D_MATMUL = "D_MATMUL" + PREDICT_LEAF_MASK = "PREDICT_LEAF_MASK" + TEST_LEAF_MASK = "PREDICT_TEST_LEAF_MASK" + VALID_LEAF_MASK = "PREDICT_VALID_LEAF_MASK" + PREDICT_PRABA = "PREDICT_PRABA" diff --git a/python/ppc_model/secure_lr/secure_lr_training_engine.py b/python/ppc_model/secure_lr/secure_lr_training_engine.py new file mode 100644 index 00000000..c848fd76 --- /dev/null +++ b/python/ppc_model/secure_lr/secure_lr_training_engine.py @@ -0,0 +1,40 @@ +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.protocol import TaskRole, ModelTask +from ppc_model.common.global_context import components +from ppc_model.interface.task_engine import TaskEngine +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lr.secure_lr_context import SecureLRContext +from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty + + +class SecureLRTrainingEngine(TaskEngine): + task_type = ModelTask.LR_TRAINING + + @staticmethod + def run(args): + + task_info = SecureLRContext(args, components) + secure_dataset = SecureDataset(task_info) + + if task_info.role == TaskRole.ACTIVE_PARTY: + booster = VerticalLRActiveParty(task_info, secure_dataset) + elif task_info.role == TaskRole.PASSIVE_PARTY: + booster = VerticalLRPassiveParty(task_info, secure_dataset) + else: + raise PpcException(PpcErrorCode.ROLE_TYPE_ERROR.get_code(), + PpcErrorCode.ROLE_TYPE_ERROR.get_message()) + + booster.fit() + booster.save_model() + + # 获取训练集和验证集的预测概率值 + train_praba = booster.get_train_praba() + test_praba = booster.get_test_praba() + + # 获取训练集和验证集的预测值评估指标 + Evaluation(task_info, secure_dataset, train_praba, test_praba) + ModelPlot(booster) + ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lr/test/__init__.py b/python/ppc_model/secure_lr/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_lr/test/test_enc_matmul.py b/python/ppc_model/secure_lr/test/test_enc_matmul.py new file mode 100644 index 00000000..6dc921b7 --- /dev/null +++ b/python/ppc_model/secure_lr/test/test_enc_matmul.py @@ -0,0 +1,64 @@ +import unittest +import numpy as np + +from ppc_model.common.initializer import Initializer +from ppc_model.secure_lr.secure_lr_context import SecureLRContext +from ppc_model.secure_lr.vertical.booster import VerticalBooster + + +ACTIVE_PARTY = 'ACTIVE_PARTY' + +job_id = 'j-1234' +task_id = 't-1234' + +model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'epochs': 1, + 'batch_size': 8, + 'feature_rate': 0.8, + 'random_state': 2024 +} + +args = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [], + 'participant_id_list': [], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict +} + + +class TestEncMatmul(unittest.TestCase): + + def test_enc_matmul(self): + active_components = Initializer(log_config_path='', config_path='') + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + task_info = SecureLRContext(args, active_components) + + # 15个特征，batch_size: 8 + arr = np.array([2, 4, -5, 0, 9, -7, 12, 3]) + np.random.seed(0) + # x = np.random.randint(-10, 10, size=(15, 8)) + x = np.random.randint(0, 10, size=(15, 8)) + enc_arr = task_info.phe.encrypt_batch_parallel((arr).astype('object')) + enc_x_d = VerticalBooster.enc_matmul(x, enc_arr) + x_d_rec = np.array(task_info.phe.decrypt_batch(enc_x_d), dtype='object') + x_d_rec[x_d_rec > 2**(task_info.phe.key_length-1)] -= 2**(task_info.phe.key_length) + + assert (np.matmul(x, arr) == x_d_rec).all() + + arr_ = VerticalBooster.rounding_d(arr) + x_ = VerticalBooster.rounding_d(x) + enc_arr = task_info.phe.encrypt_batch_parallel((arr_).astype('object')) + enc_x_d = VerticalBooster.enc_matmul(x_, enc_arr) + x_d_rec = np.array(task_info.phe.decrypt_batch(enc_x_d), dtype='object') + x_d_rec = VerticalBooster.recover_d(task_info, x_d_rec, is_square=True) + + assert (np.matmul(x, arr) == x_d_rec).all() diff --git a/python/ppc_model/secure_lr/test/test_lr.py b/python/ppc_model/secure_lr/test/test_lr.py new file mode 100644 index 00000000..7ed377ba --- /dev/null +++ b/python/ppc_model/secure_lr/test/test_lr.py @@ -0,0 +1,207 @@ +import unittest +import threading +import traceback +import numpy as np +from sklearn.datasets import load_breast_cancer + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.network.stub import ModelStub +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.metrics.loss import BinaryLoss +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.secure_lr.secure_lr_context import SecureLRContext +from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty +from ppc_model.secure_lr.vertical.booster import VerticalBooster + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + + +def mock_args(): + job_id = 'j-1234' + task_id = 't-1234' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'epochs': 1, + 'batch_size': 8, + 'feature_rate': 0.8, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroXGB', + 'model_dict': model_dict + } + + return args_a, args_b + + +cancer = load_breast_cancer() +X = cancer.data +y = cancer.target + +df = SecureDataset.assembling_dataset(X, y) +df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df) + +args_a, args_b = mock_args() + +active_components = Initializer(log_config_path='', config_path='') +active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} +active_components.mock_logger = MockLogger() +task_info_a = SecureLRContext(args_a, active_components) + +# df --------------------------------------------- +secure_dataset = SecureDataset(task_info_a, df) +max_iter = VerticalBooster._init_iter( + secure_dataset.train_X.shape[0], 3, 8) + +train_praba = VerticalBooster._init_praba(secure_dataset.train_X.shape[0]) +train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1]) +bias = 0 + +# for _ in range(max_iter): +for i in range(1): + idx = VerticalBooster._get_sample_idx(i, secure_dataset.train_X.shape[0], size = 8) + x_, y_ = secure_dataset.train_X[idx], secure_dataset.train_y[idx] + + g = BinaryLoss.dot_product(x_, train_weights) + bias + # h = 0.5 + BinaryLoss.inference(g) + h = BinaryLoss.sigmoid(g) + d = h - y_ + deriv = np.matmul(x_.T, d) / x_.shape[0] + deriv_bias = np.sum(d) / x_.shape[0] + print(deriv) + + train_weights -= 0.1 * deriv.astype('float') + bias -= 0.1 * deriv_bias + print(train_weights) + print(bias) + +g = BinaryLoss.dot_product(secure_dataset.train_X, train_weights) + bias +train_praba = BinaryLoss.sigmoid(g) +auc = Evaluation.fevaluation(secure_dataset.train_y, train_praba)['auc'] +print(auc) + + +# df --------------------------------------------- +# not bias +train_praba = VerticalBooster._init_praba(secure_dataset.train_X.shape[0]) +train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1]) + +# for _ in range(max_iter): +for i in range(2): + idx = VerticalBooster._get_sample_idx(i, secure_dataset.train_X.shape[0], size = 8) + x_, y_ = secure_dataset.train_X[idx], secure_dataset.train_y[idx] + + g = BinaryLoss.dot_product(x_, train_weights) + h = 0.5 + BinaryLoss.inference(g) + # h = BinaryLoss.sigmoid(g) + d = h - y_ + deriv = np.matmul(x_.T, d) / x_.shape[0] + print(deriv) + + train_weights -= 0.1 * deriv.astype('float') + print(train_weights) + +g = BinaryLoss.dot_product(secure_dataset.train_X, train_weights) +# train_praba = 0.5 + BinaryLoss.inference(g) +train_praba = BinaryLoss.sigmoid(g) +auc = Evaluation.fevaluation(secure_dataset.train_y, train_praba)['auc'] +print(auc) + + +# df --------------------------------------------- +# MinMaxScaler +from sklearn.preprocessing import MinMaxScaler + +# 创建MinMaxScaler对象 +scaler = MinMaxScaler() + +# 拟合并转换数据 +train_X = scaler.fit_transform(secure_dataset.train_X) + +train_praba = VerticalBooster._init_praba(secure_dataset.train_X.shape[0]) +train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1]) + +for i in range(2): + idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size = 8) + x_, y_ = train_X[idx], secure_dataset.train_y[idx] + + g = BinaryLoss.dot_product(x_, train_weights) + h = 0.5 + BinaryLoss.inference(g) + # h = BinaryLoss.sigmoid(g) + d = h - y_ + deriv = np.matmul(x_.T, d) / x_.shape[0] + print(deriv) + + train_weights -= 0.1 * deriv.astype('float') + print(train_weights) + +g = BinaryLoss.dot_product(train_X, train_weights) +# train_praba = 0.5 + BinaryLoss.inference(g) +train_praba = BinaryLoss.sigmoid(g) +auc = Evaluation.fevaluation(secure_dataset.train_y, train_praba)['auc'] +print(auc) + + +# StandardScaler +from sklearn.preprocessing import StandardScaler + +# 创建MinMaxScaler对象 +scaler = StandardScaler() + +# 拟合并转换数据 +train_X = scaler.fit_transform(secure_dataset.train_X) + +train_praba = VerticalBooster._init_praba(secure_dataset.train_X.shape[0]) +train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1]) + +for i in range(2): + idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size = 8) + x_, y_ = train_X[idx], secure_dataset.train_y[idx] + + g = BinaryLoss.dot_product(x_, train_weights) + h = 0.5 + BinaryLoss.inference(g) + # h = BinaryLoss.sigmoid(g) + d = h - y_ + deriv = np.matmul(x_.T, d) / x_.shape[0] + print(deriv) + + train_weights -= 0.1 * deriv.astype('float') + print(train_weights) + +g = BinaryLoss.dot_product(train_X, train_weights) +# train_praba = 0.5 + BinaryLoss.inference(g) +train_praba = BinaryLoss.sigmoid(g) +auc = Evaluation.fevaluation(secure_dataset.train_y, train_praba)['auc'] +print(auc) + diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py new file mode 100644 index 00000000..3c99d12b --- /dev/null +++ b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py @@ -0,0 +1,179 @@ +import unittest +import threading +import traceback + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.network.stub import ModelStub +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext +from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + +data_size = 1000 +feature_dim = 20 + + +def mock_args(): + job_id = 'j-1111' + task_id = 't-1111' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'epochs': 1, + 'batch_size': 8, + 'feature_rate': 0.8, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroLR', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroLR', + 'model_dict': model_dict + } + + return args_a, args_b + + +class TestXgboostTraining(unittest.TestCase): + + def setUp(self): + self._active_rpc_client = RpcClientMock() + self._passive_rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._active_stub = ModelStub( + agency_id=ACTIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._active_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._passive_stub = ModelStub( + agency_id=PASSIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._passive_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._active_rpc_client.set_message_handler( + self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler( + self._active_stub.on_message_received) + + def test_fit(self): + args_a, args_b = mock_args() + + active_components = Initializer(log_config_path='', config_path='') + active_components.stub = self._active_stub + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLGBMContext(args_a, active_components) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=True) + secure_dataset_a = SecureDataset(task_info_a, model_data) + booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a) + print(secure_dataset_a.feature_name) + print(secure_dataset_a.train_idx.shape) + print(secure_dataset_a.train_X.shape) + print(secure_dataset_a.train_y.shape) + print(secure_dataset_a.test_idx.shape) + print(secure_dataset_a.test_X.shape) + print(secure_dataset_a.test_y.shape) + + passive_components = Initializer(log_config_path='', config_path='') + passive_components.stub = self._passive_stub + passive_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} + passive_components.mock_logger = MockLogger() + task_info_b = SecureLGBMContext(args_b, passive_components) + model_data = SecureDataset.simulate_dataset( + data_size, feature_dim, has_label=False) + secure_dataset_b = SecureDataset(task_info_b, model_data) + booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b) + print(secure_dataset_b.feature_name) + print(secure_dataset_b.train_idx.shape) + print(secure_dataset_b.train_X.shape) + print(secure_dataset_b.test_idx.shape) + print(secure_dataset_b.test_X.shape) + + def active_worker(): + try: + booster_a.fit() + booster_a.save_model() + train_praba = booster_a.get_train_praba() + test_praba = booster_a.get_test_praba() + Evaluation(task_info_a, secure_dataset_a, + train_praba, test_praba) + ResultFileHandling(task_info_a) + booster_a.load_model() + booster_a.predict() + test_praba = booster_a.get_test_praba() + task_info_a.algorithm_type = 'Predict' + task_info_a.sync_file_list = {} + Evaluation(task_info_a, secure_dataset_a, + test_praba=test_praba) + ResultFileHandling(task_info_a) + except Exception as e: + task_info_a.components.logger().info(traceback.format_exc()) + + def passive_worker(): + try: + booster_b.fit() + booster_b.save_model() + train_praba = booster_b.get_train_praba() + test_praba = booster_b.get_test_praba() + Evaluation(task_info_b, secure_dataset_b, + train_praba, test_praba) + ResultFileHandling(task_info_b) + booster_b.load_model() + booster_b.predict() + test_praba = booster_b.get_test_praba() + task_info_b.algorithm_type = 'Predict' + task_info_b.sync_file_list = {} + Evaluation(task_info_b, secure_dataset_b, + test_praba=test_praba) + ResultFileHandling(task_info_b) + except Exception as e: + task_info_b.components.logger().info(traceback.format_exc()) + + thread_lgbm_active = threading.Thread(target=active_worker, args=()) + thread_lgbm_active.start() + + thread_lgbm_passive = threading.Thread(target=passive_worker, args=()) + thread_lgbm_passive.start() + + thread_lgbm_active.join() + thread_lgbm_passive.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_training.py new file mode 100644 index 00000000..14a9bce5 --- /dev/null +++ b/python/ppc_model/secure_lr/test/test_secure_lr_training.py @@ -0,0 +1,188 @@ +import unittest +import threading +import traceback +from sklearn.datasets import load_breast_cancer + +from ppc_model.common.initializer import Initializer +from ppc_common.ppc_mock.mock_objects import MockLogger +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_model.network.stub import ModelStub +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.model_plot import ModelPlot +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.common.mock.rpc_client_mock import RpcClientMock +from ppc_model.secure_lr.secure_lr_context import SecureLRContext +from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty + + +ACTIVE_PARTY = 'ACTIVE_PARTY' +PASSIVE_PARTY = 'PASSIVE_PARTY' + + +def mock_args(): + job_id = 'j-1234' + task_id = 't-1234' + + model_dict = { + 'objective': 'regression', + 'categorical_feature': [], + 'train_features': "", + 'epochs': 1, + 'batch_size': 8, + 'feature_rate': 0.8, + 'random_state': 2024 + } + + args_a = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': True, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroLR', + 'model_dict': model_dict + } + + args_b = { + 'job_id': job_id, + 'task_id': task_id, + 'is_label_holder': False, + 'result_receiver_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'participant_id_list': [ACTIVE_PARTY, PASSIVE_PARTY], + 'model_predict_algorithm': None, + 'algorithm_type': 'Train', + 'algorithm_subtype': 'HeteroLR', + 'model_dict': model_dict + } + + return args_a, args_b + + +class TestXgboostTraining(unittest.TestCase): + + cancer = load_breast_cancer() + X = cancer.data + y = cancer.target + + # MinMaxScaler + from sklearn.preprocessing import MinMaxScaler + # 创建MinMaxScaler对象 + scaler = MinMaxScaler() + # 拟合并转换数据 + X = scaler.fit_transform(X) + + df = SecureDataset.assembling_dataset(X, y) + df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df) + + def setUp(self): + self._active_rpc_client = RpcClientMock() + self._passive_rpc_client = RpcClientMock() + self._thread_event_manager = ThreadEventManager() + self._active_stub = ModelStub( + agency_id=ACTIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._active_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._passive_stub = ModelStub( + agency_id=PASSIVE_PARTY, + thread_event_manager=self._thread_event_manager, + rpc_client=self._passive_rpc_client, + send_retry_times=3, + retry_interval_s=0.1 + ) + self._active_rpc_client.set_message_handler( + self._passive_stub.on_message_received) + self._passive_rpc_client.set_message_handler( + self._active_stub.on_message_received) + + def test_fit(self): + args_a, args_b = mock_args() + plot_lock = threading.Lock() + + active_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock) + active_components.stub = self._active_stub + active_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY} + active_components.mock_logger = MockLogger() + task_info_a = SecureLRContext(args_a, active_components) + secure_dataset_a = SecureDataset(task_info_a, self.df_with_y) + booster_a = VerticalLRActiveParty(task_info_a, secure_dataset_a) + print(secure_dataset_a.feature_name) + print(secure_dataset_a.train_idx.shape) + print(secure_dataset_a.train_X.shape) + print(secure_dataset_a.train_y.shape) + print(secure_dataset_a.test_idx.shape) + print(secure_dataset_a.test_X.shape) + print(secure_dataset_a.test_y.shape) + + passive_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock) + passive_components.stub = self._passive_stub + passive_components.config_data = { + 'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY} + passive_components.mock_logger = MockLogger() + task_info_b = SecureLRContext(args_b, passive_components) + secure_dataset_b = SecureDataset(task_info_b, self.df_without_y) + booster_b = VerticalLRPassiveParty(task_info_b, secure_dataset_b) + print(secure_dataset_b.feature_name) + print(secure_dataset_b.train_idx.shape) + print(secure_dataset_b.train_X.shape) + print(secure_dataset_b.test_idx.shape) + print(secure_dataset_b.test_X.shape) + + def active_worker(): + try: + booster_a.fit() + # booster_a.save_model() + # train_praba = booster_a.get_train_praba() + # test_praba = booster_a.get_test_praba() + # Evaluation(task_info_a, secure_dataset_a, + # train_praba, test_praba) + # ResultFileHandling(task_info_a) + # booster_a.load_model() + # booster_a.predict() + # test_praba = booster_a.get_test_praba() + # task_info_a.algorithm_type = 'Predict' + # task_info_a.sync_file_list = {} + # Evaluation(task_info_a, secure_dataset_a, + # test_praba=test_praba) + # ResultFileHandling(task_info_a) + except Exception as e: + task_info_a.components.logger().info(traceback.format_exc()) + + def passive_worker(): + try: + booster_b.fit() + # booster_b.save_model() + # train_praba = booster_b.get_train_praba() + # test_praba = booster_b.get_test_praba() + # Evaluation(task_info_b, secure_dataset_b, + # train_praba, test_praba) + # ResultFileHandling(task_info_b) + # booster_b.load_model() + # booster_b.predict() + # test_praba = booster_b.get_test_praba() + # task_info_b.algorithm_type = 'Predict' + # task_info_b.sync_file_list = {} + # Evaluation(task_info_b, secure_dataset_b, + # test_praba=test_praba) + # ResultFileHandling(task_info_b) + except Exception as e: + task_info_b.components.logger().info(traceback.format_exc()) + + thread_lr_active = threading.Thread(target=active_worker, args=()) + thread_lr_active.start() + + thread_lr_passive = threading.Thread(target=passive_worker, args=()) + thread_lr_passive.start() + + thread_lr_active.join() + thread_lr_passive.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/ppc_model/secure_lr/vertical/__init__.py b/python/ppc_model/secure_lr/vertical/__init__.py new file mode 100644 index 00000000..bf7ebebe --- /dev/null +++ b/python/ppc_model/secure_lr/vertical/__init__.py @@ -0,0 +1,4 @@ +from ppc_model.secure_lr.vertical.active_party import VerticalLRActiveParty +from ppc_model.secure_lr.vertical.passive_party import VerticalLRPassiveParty + +__all__ = ["VerticalLRActiveParty", "VerticalLRPassiveParty"] diff --git a/python/ppc_model/secure_lr/vertical/active_party.py b/python/ppc_model/secure_lr/vertical/active_party.py new file mode 100644 index 00000000..1d9e3a2f --- /dev/null +++ b/python/ppc_model/secure_lr/vertical/active_party.py @@ -0,0 +1,165 @@ +import itertools +import time + +import numpy as np +from pandas import DataFrame + +from ppc_common.deps_services.serialize_type import SerializeType +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceStore +from ppc_common.ppc_ml.feature.feature_importance import FeatureImportanceType +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo, IterationRequest +from ppc_common.ppc_utils import utils +from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection +from ppc_model.datasets.data_reduction.sampling import Sampling +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.metrics.loss import BinaryLoss +from ppc_model.secure_lr.secure_lr_context import SecureLRContext, LRMessage +from ppc_model.secure_lr.vertical.booster import VerticalBooster + + +class VerticalLRActiveParty(VerticalBooster): + + def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None: + super().__init__(ctx, dataset) + self.params = ctx.model_params + self._loss_func = BinaryLoss() + self._all_feature_name = [dataset.feature_name] + self._all_feature_num = len(dataset.feature_name) + self.log = ctx.components.logger() + self.storage_client = ctx.components.storage_client + self.log.info( + f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + + def fit( + self, + *args, + **kwargs, + ) -> None: + self.log.info( + f'task {self.ctx.task_id}: Starting the lr on the active party.') + self._init_active_data() + + max_iter = self._init_iter(self.dataset.train_X.shape[0], + self.params.epochs, self.params.batch_size) + for _ in range(max_iter): + self._iter_id += 1 + start_time = time.time() + self.log.info( + f'task {self.ctx.task_id}: Starting iter-{self._iter_id} in active party.') + + # 初始化 + idx, feature_select = self._init_each_iter() + self.log.info( + f'task {self.ctx.task_id}: feature select: {len(feature_select)}, {feature_select}.') + + # 构建 + self._build_iter(feature_select, idx) + + # 预测 + self._train_praba = self._predict_tree(self.dataset.train_X, LRMessage.PREDICT_LEAF_MASK.value) + # print('train_praba', set(self._train_praba)) + + # 评估 + if not self.params.silent and self.dataset.train_y is not None: + auc = Evaluation.fevaluation( + self.dataset.train_y, self._train_praba)['auc'] + self.log.info( + f'task {self.ctx.task_id}: iter-{self._iter_id}, auc: {auc}.') + self.log.info(f'task {self.ctx.task_id}: Ending iter-{self._iter_id}, ' + f'time_costs: {time.time() - start_time}s.') + + # 预测验证集 + self._test_praba = self._predict_tree(self.dataset.test_X, LRMessage.TEST_LEAF_MASK.value) + if not self.params.silent and self.dataset.test_y is not None: + auc = Evaluation.fevaluation( + self.dataset.test_y, self._test_praba)['auc'] + self.log.info( + f'task {self.ctx.task_id}: iter-{self._iter_id}, test auc: {auc}.') + + self._end_active_data() + + def transform(self, transform_data: DataFrame) -> DataFrame: + ... + + def predict(self, dataset: SecureDataset = None) -> np.ndarray: + start_time = time.time() + if dataset is None: + dataset = self.dataset + + test_praba = self._predict_tree(dataset.test_X, LRMessage.VALID_LEAF_MASK.value) + self._test_praba = test_praba + + if dataset.test_y is not None: + auc = Evaluation.fevaluation(dataset.test_y, test_praba)['auc'] + self.log.info(f'task {self.ctx.task_id}: predict test auc: {auc}.') + self.log.info( + f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + + self._end_active_data(is_train=False) + + def _init_active_data(self): + + # 初始化预测值和权重 + self._train_praba = self._init_praba(self.dataset.train_X.shape[0]) + self._train_weights = self._init_weight(self.dataset.train_X.shape[1]) + self._test_weights = self._init_weight(self.dataset.test_X.shape[1]) + self._iter_id = 0 + + # 初始化所有参与方的特征 + for i in range(1, len(self.ctx.participant_id_list)): + feature_name_bytes = self._receive_byte_data( + self.ctx, LRMessage.FEATURE_NAME.value, i) + self._all_feature_name.append( + [s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + self._all_feature_num += len([s.decode('utf-8') + for s in feature_name_bytes.split(b' ') if s]) + + self.log.info(f'task {self.ctx.task_id}: total feature number:{self._all_feature_num}, ' + f'total feature name: {self._all_feature_name}.') + self.params.categorical_idx = self._get_categorical_idx( + list(itertools.chain(*self._all_feature_name)), self.params.categorical_feature) + self.params.my_categorical_idx = self._get_categorical_idx( + self.dataset.feature_name, self.params.categorical_feature) + + def _build_iter(self, feature_select, idx): + + x_, y_ = self.dataset.train_X[idx], self.dataset.train_y[idx] + + g = self._loss_func.dot_product(x_, self._train_weights) + h = 0.5 + self._loss_func.inference(g) + d = h - y_ + + self._send_d_instance_list(d) + public_key_list, d_other_list, partner_index_list = self._receive_d_instance_list() + deriv = self._calculate_deriv(x_, d, partner_index_list, d_other_list) + + self._train_weights -= self.params.learning_rate * deriv.astype('float') + self._train_weights[~np.isin(np.arange(len(self._train_weights)), feature_select)] = 0 + + def _predict_tree(self, X, key_type): + train_g = self._loss_func.dot_product(X, self._train_weights) + for i in range(1, len(self.ctx.participant_id_list)): + train_g_other = np.frombuffer( + self._receive_byte_data(self.ctx, key_type, i), dtype='float') + train_g += train_g_other + return self._loss_func.sigmoid(train_g) + + def _end_active_data(self, is_train=True): + if is_train: + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LRMessage.PREDICT_PRABA.value}_train', + self._train_praba.astype('float').tobytes(), partner_index) + + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LRMessage.PREDICT_PRABA.value}_test', + self._test_praba.astype('float').tobytes(), partner_index) + + else: + for partner_index in range(1, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] in self.ctx.result_receiver_id_list: + self._send_byte_data(self.ctx, f'{LRMessage.PREDICT_PRABA.value}_predict', + self._test_praba.astype('float').tobytes(), partner_index) diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py new file mode 100644 index 00000000..68a68d5f --- /dev/null +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -0,0 +1,308 @@ +import os +import time +import random +import json +import itertools +import numpy as np + +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.interface.model_base import VerticalModel +from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.common.protocol import PheMessage +from ppc_model.network.stub import PushRequest, PullRequest +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.secure_lr.secure_lr_context import SecureLRContext, LRMessage + + +# 抽离sgb的公共部分 +class VerticalBooster(VerticalModel): + def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None: + super().__init__(ctx) + self.dataset = dataset + self._stub = ctx.components.stub + + self._iter_id = None + + self._train_weights = None + self._train_praba = None + self._test_weights = None + self._test_praba = None + + random.seed(ctx.model_params.random_state) + np.random.seed(ctx.model_params.random_state) + + def _build_tree(self, *args, **kwargs): + + raise NotImplementedError + + def _predict_tree(self, *args, **kwargs): + + raise NotImplementedError + + @staticmethod + def _init_praba(n): + return np.full(n, 0.5) + + @staticmethod + def _init_weight(n): + return np.zeros(n, dtype=float) + + @staticmethod + def _init_iter(n, epochs, batch_size): + return round(n*epochs/batch_size) + + @staticmethod + def _get_sample_idx(i, n, size): + start_idx = (i * size) % n + end_idx = start_idx + size + if end_idx <= n: + idx = list(range(start_idx, end_idx)) + else: + head_idx = end_idx - n + idx = list(range(start_idx, n)) + list(range(head_idx)) + return idx + + @staticmethod + def _get_categorical_idx(feature_name, categorical_feature=[]): + categorical_idx = [] + if len(categorical_feature) > 0: + for i in categorical_feature: + if i in feature_name: + categorical_idx.append(feature_name.index(i)) + return categorical_idx + + def _init_each_iter(self): + + idx = self._get_sample_idx(self._iter_id-1, self.dataset.train_X.shape[0], + size = self.params.batch_size) + feature_select = FeatureSelection.feature_selecting( + list(self.dataset.feature_name), + self.params.train_feature, self.params.feature_rate) + + return idx, feature_select + + def _send_d_instance_list(self, d): + + d_list = self.rounding_d(d) + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + + start_time = time.time() + self.log.info(f'task {self.ctx.task_id}: Starting iter-{self._iter_id} ' + f'encrypt d in {my_agency_id} party.') + enc_dlist = self.ctx.phe.encrypt_batch_parallel((d_list).astype('object')) + self.log.info(f'task {self.ctx.task_id}: Finished iter-{self._iter_id} ' + f'encrypt d time_costs: {time.time() - start_time}.') + + for partner_index in range(len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + self._send_enc_data(self.ctx, f'{LRMessage.ENC_D_LIST.value}_{self._iter_id}', + enc_dlist, partner_index) + + def _receive_d_instance_list(self): + + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + + public_key_list = [] + d_other_list = [] + partner_index_list = [] + for partner_index in range(len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + public_key, enc_d = self._receive_enc_data( + self.ctx, f'{LRMessage.ENC_D_LIST.value}_{self._iter_id}', partner_index) + public_key_list.append(public_key) + d_other_list.append(np.array(enc_d)) + partner_index_list.append(partner_index) + + return public_key_list, d_other_list, partner_index_list + + def _calculate_deriv(self, x_, d, partner_index_list, d_other_list): + + x = self.rounding_d(x_) + deriv = np.matmul(x_.T, d) / x_.shape[0] + for i, partner_index in enumerate(partner_index_list): + # 计算明文*密文 matmul + # deriv_other_i = np.matmul(x.T, d_other_list[i]) + deriv_other_i = self.enc_matmul(x.T, d_other_list[i]) + + # 发送密文，接受密文并解密 + self._send_enc_data(self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', + deriv_other_i, partner_index) + _, enc_deriv_i = self._receive_enc_data( + self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', partner_index) + deriv_i_rec = np.array(self.ctx.phe.decrypt_batch(enc_deriv_i), dtype='object') + deriv_i = self.recover_d(self.ctx, deriv_i_rec, is_square=True) / x_.shape[0] + + # 发送明文，接受明文并计算 + self._send_byte_data(self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', + deriv_i.astype('float').tobytes(), partner_index) + deriv_x_i = np.frombuffer(self._receive_byte_data( + self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', partner_index), dtype=np.float) + self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') + self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv_x_i: {deriv_x_i}.') + deriv += deriv_x_i + self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') + return deriv + + def _calculate_deriv1(self, x_, d, partner_index_list, d_other_list): + + x = self.rounding_d(x_) + deriv = np.matmul(x_.T, d) / x_.shape[0] + for i, partner_index in enumerate(partner_index_list): + # TODO：重载方法，目前支持np.array(enc_dlist).sum()的方式，不支持明文*密文 + # deriv_other_i = np.matmul(x.T, d_other_list[i]) + deriv_other_i = self.enc_matmul(x.T, d_other_list[i]) + self._send_enc_data(self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', + deriv_other_i, partner_index) + _, enc_deriv_i = self._receive_enc_data( + self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', partner_index) + deriv_i = np.array(self.ctx.phe.decrypt_batch(enc_deriv_i), dtype='object') + deriv += (self.recover_d(self.ctx, deriv_i, is_square=True) / x_.shape[0]) + return deriv + + def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=False): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + if matrix_data: + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=PheMessage.packing_2dim_data( + ctx.codec, ctx.phe.public_key, enc_data) + )) + else: + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=PheMessage.packing_data( + ctx.codec, ctx.phe.public_key, enc_data) + )) + + log.info( + f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " + f"data_length: {len(enc_data)}, time_costs: {time.time() - start_time}s") + + def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + byte_data = self._stub.pull(PullRequest( + sender=partner_id, + task_id=ctx.task_id, + key=key_type + )) + + if matrix_data: + public_key, enc_data = PheMessage.unpacking_2dim_data( + ctx.codec, byte_data) + else: + public_key, enc_data = PheMessage.unpacking_data( + ctx.codec, byte_data) + + log.info( + f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + return public_key, enc_data + + def _send_byte_data(self, ctx, key_type, byte_data, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + self._stub.push(PushRequest( + receiver=partner_id, + task_id=ctx.task_id, + key=key_type, + data=byte_data + )) + + log.info( + f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + + def _receive_byte_data(self, ctx, key_type, partner_index): + log = ctx.components.logger() + start_time = time.time() + partner_id = ctx.participant_id_list[partner_index] + + byte_data = self._stub.pull(PullRequest( + sender=partner_id, + task_id=ctx.task_id, + key=key_type + )) + + log.info( + f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " + f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") + return byte_data + + def save_model(self, file_path=None): + log = self.ctx.components.logger() + if file_path is not None: + self.ctx.model_data_file = os.path.join( + file_path, self.ctx.MODEL_DATA_FILE) + + if not os.path.exists(self.ctx.model_data_file): + serial_weight = list(self._train_weights) + with open(self.ctx.model_data_file, 'w') as f: + json.dump(serial_weight, f) + ResultFileHandling._upload_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + log.info( + f"task {self.ctx.task_id}: Saved serial_weight to {self.ctx.model_data_file} finished.") + + def load_model(self, file_path=None): + log = self.ctx.components.logger() + if file_path is not None: + self.ctx.model_data_file = os.path.join( + file_path, self.ctx.MODEL_DATA_FILE) + if self.ctx.algorithm_type == AlgorithmType.Predict.name: + self.ctx.remote_model_data_file = os.path.join( + self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) + + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + + with open(self.ctx.model_data_file, 'r') as f: + serial_weight = json.load(f) + self._train_weights = np.array(serial_weight) + log.info( + f"task {self.ctx.task_id}: Load serial_weight from {self.ctx.model_data_file} finished.") + + def get_weights(self): + return self._train_weights + + def get_train_praba(self): + return self._train_praba + + def get_test_praba(self): + return self._test_praba + + @staticmethod + def enc_matmul(arr, enc): + result = [] + for i in range(len(arr)): + # arr[i] * enc # 需要将密文放在前面 + result.append((enc * arr[i]).sum()) + return np.array(result) + + @staticmethod + def rounding_d(d_list: np.ndarray, expand=1000): + return (d_list * expand).astype('int') + + @staticmethod + def recover_d(ctx, d_sum_list: np.ndarray, is_square=False, expand=1000): + + d_sum_list[d_sum_list > 2**(ctx.phe.key_length-1)] -= 2**(ctx.phe.key_length) + + if is_square: + return (d_sum_list / expand / expand).astype('float') + else: + return (d_sum_list / expand).astype('float') diff --git a/python/ppc_model/secure_lr/vertical/passive_party.py b/python/ppc_model/secure_lr/vertical/passive_party.py new file mode 100644 index 00000000..46101559 --- /dev/null +++ b/python/ppc_model/secure_lr/vertical/passive_party.py @@ -0,0 +1,125 @@ +import itertools +import multiprocessing +import time +import numpy as np +from pandas import DataFrame + +from ppc_common.ppc_utils import utils +from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo, IterationRequest +from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.metrics.loss import BinaryLoss +from ppc_model.secure_lr.secure_lr_context import SecureLRContext, LRMessage +from ppc_model.secure_lr.vertical.booster import VerticalBooster + + +class VerticalLRPassiveParty(VerticalBooster): + + def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None: + super().__init__(ctx, dataset) + self.params = ctx.model_params + self._loss_func = BinaryLoss() + self.log = ctx.components.logger() + self.log.info( + f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') + + def fit( + self, + *args, + **kwargs, + ) -> None: + self.log.info( + f'task {self.ctx.task_id}: Starting the lr on the passive party.') + self._init_passive_data() + + max_iter = self._init_iter(self.dataset.train_X.shape[0], + self.params.epochs, self.params.batch_size) + for _ in range(max_iter): + self._iter_id += 1 + start_time = time.time() + self.log.info( + f'task {self.ctx.task_id}: Starting iter-{self._iter_id} in passive party.') + + # 初始化 + idx, feature_select = self._init_each_iter() + self.log.info( + f'task {self.ctx.task_id}: feature select: {len(feature_select)}, {feature_select}.') + + # 构建 + self._build_iter(feature_select, idx) + + # 预测 + self._predict_tree(self.dataset.train_X, LRMessage.PREDICT_LEAF_MASK.value) + self.log.info(f'task {self.ctx.task_id}: Ending iter-{self._iter_id}, ' + f'time_costs: {time.time() - start_time}s.') + + # 预测验证集 + self._predict_tree(self.dataset.test_X, LRMessage.TEST_LEAF_MASK.value) + + self._end_passive_data() + + def transform(self, transform_data: DataFrame) -> DataFrame: + ... + + def predict(self, dataset: SecureDataset = None) -> np.ndarray: + start_time = time.time() + if dataset is None: + dataset = self.dataset + + self._predict_tree(dataset.test_X, LRMessage.VALID_LEAF_MASK.value) + self.log.info( + f'task {self.ctx.task_id}: Ending predict, time_costs: {time.time() - start_time}s.') + + self._end_passive_data(is_train=False) + + def _init_passive_data(self): + + # 初始化预测值和权重 + self._train_praba = self._init_praba(self.dataset.train_X.shape[0]) + self._train_weights = self._init_weight(self.dataset.train_X.shape[1]) + self._test_weights = self._init_weight(self.dataset.test_X.shape[1]) + self._iter_id = 0 + + # 初始化参与方特征 + self._send_byte_data(self.ctx, LRMessage.FEATURE_NAME.value, + b''.join(s.encode('utf-8') + b' ' for s in self.dataset.feature_name), 0) + self.params.my_categorical_idx = self._get_categorical_idx( + self.dataset.feature_name, self.params.categorical_feature) + + def _build_iter(self, feature_select, idx): + + x_ = self.dataset.train_X[idx] + + g = self._loss_func.dot_product(x_, self._train_weights) + h = self._loss_func.inference(g) + d = h + + self._send_d_instance_list(d) + public_key_list, d_other_list, partner_index_list = self._receive_d_instance_list() + deriv = self._calculate_deriv(x_, d, partner_index_list, d_other_list) + + self._train_weights -= self.params.learning_rate * deriv.astype('float') + self._train_weights[~np.isin(np.arange(len(self._train_weights)), feature_select)] = 0 + + def _predict_tree(self, X, key_type): + train_g = self._loss_func.dot_product(X, self._train_weights) + self._send_byte_data(self.ctx, f'{key_type}', + train_g.astype('float').tobytes(), 0) + + def _end_passive_data(self, is_train=True): + + if self.ctx.components.config_data['AGENCY_ID'] in self.ctx.result_receiver_id_list: + if is_train: + self._train_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LRMessage.PREDICT_PRABA.value}_train', 0), dtype=np.float) + + self._test_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LRMessage.PREDICT_PRABA.value}_test', 0), dtype=np.float) + + else: + self._test_praba = np.frombuffer( + self._receive_byte_data( + self.ctx, f'{LRMessage.PREDICT_PRABA.value}_predict', 0), dtype=np.float) From a2b1126149a8dd92ba04c08981a7d5f3d785a81c Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 11 Sep 2024 00:52:12 +0800 Subject: [PATCH 033/120] fix grpc client and server caused crash && fix pro-node initialize (#30) * fix build_ppc script * fix pro-node initialize * fix grpc client and server caused crash * update gateway-router-table when the node become unreachable --- cpp/ppc-framework/front/IFront.h | 2 +- cpp/ppc-framework/gateway/IGateway.h | 2 +- cpp/ppc-framework/protocol/GrpcConfig.h | 4 + cpp/tools/build_ppc.sh | 11 +-- cpp/tools/build_wedpr_cem.sh | 5 +- cpp/tools/ppc-builder/build_ppc.py | 4 +- .../ppc-builder/conf/config-example.toml | 4 +- .../src/config/ppc_node_config_generator.py | 14 ++-- .../ppc-builder/src/tpl/config.ini.gateway | 5 +- cpp/tools/ppc-builder/src/tpl/config.ini.node | 11 +-- .../ppc-cem/tests/data/config.ini | 2 +- .../ppc-tools/src/config/PPCConfig.cpp | 14 ++-- cpp/wedpr-helper/ppc-utilities/Utilities.h | 2 +- cpp/wedpr-initializer/Initializer.cpp | 8 ++ cpp/wedpr-protocol/grpc/Common.h | 12 ++- cpp/wedpr-protocol/grpc/client/Common.h | 1 + .../grpc/client/FrontClient.cpp | 7 +- .../grpc/client/GatewayClient.cpp | 41 +++++----- .../grpc/client/GatewayClient.h | 2 +- cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 1 + .../grpc/client/HealthCheckTimer.cpp | 3 +- .../grpc/server/FrontServer.cpp | 5 +- .../grpc/server/GatewayServer.cpp | 30 +++---- .../protobuf/src/RequestConverter.h | 9 ++- .../ppc-front/ppc-front/Front.cpp | 9 ++- .../ppc-front/ppc-front/FrontImpl.cpp | 2 + .../ppc-front/ppc-front/FrontImpl.h | 2 +- .../ppc-gateway/gateway/GatewayImpl.cpp | 35 ++++++++- .../ppc-gateway/gateway/GatewayImpl.h | 2 +- .../gateway/cache/MessageCache.cpp | 2 +- .../gateway/router/GatewayNodeInfo.h | 1 + .../gateway/router/GatewayNodeInfoImpl.h | 12 ++- .../gateway/router/GatewayRouterManager.cpp | 12 +++ .../gateway/router/GatewayRouterManager.h | 2 + .../gateway/router/LocalRouter.cpp | 14 +++- .../gateway/router/PeerRouterTable.cpp | 78 +++++++++++++++---- .../gateway/router/PeerRouterTable.h | 12 ++- .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 3 +- .../ppc-gateway/ppc-gateway/p2p/Service.h | 47 +++++++++++ .../ppc-gateway/p2p/router/RouterManager.cpp | 73 ++++++++++++++--- .../ppc-gateway/p2p/router/RouterManager.h | 3 + cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 14 ++-- cpp/wedpr-transport/sdk/ProTransportImpl.h | 4 +- 43 files changed, 394 insertions(+), 132 deletions(-) diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 25073ac3..76cdbcad 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -105,7 +105,7 @@ class IFront : virtual public IFrontClient virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; virtual void asyncGetAgencies( - std::function)> callback) = 0; + std::function)> callback) = 0; /** * @brief register the nodeInfo to the gateway diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index d2489217..7ccc53c5 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -67,7 +67,7 @@ class IGateway virtual void asyncGetPeers(std::function callback) = 0; virtual void asyncGetAgencies( - std::function)> callback) = 0; + std::function)> callback) = 0; virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; virtual bcos::Error::Ptr unRegisterNodeInfo(bcos::bytesConstRef nodeID) = 0; diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h index b4dae30d..108b22c1 100644 --- a/cpp/ppc-framework/protocol/GrpcConfig.h +++ b/cpp/ppc-framework/protocol/GrpcConfig.h @@ -60,9 +60,13 @@ class GrpcConfig bool enableHealthCheck() const { return m_enableHealthCheck; } void setEnableHealthCheck(bool enableHealthCheck) { m_enableHealthCheck = enableHealthCheck; } + void setEnableDnslookup(bool enableDnslookup) { m_enableDnslookup = enableDnslookup; } + + bool enableDnslookup() const { return m_enableDnslookup; } protected: bool m_enableHealthCheck = false; std::string m_loadBalancePolicy = "round_robin"; + bool m_enableDnslookup = false; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh index 509a0bd0..c7ec3f7a 100644 --- a/cpp/tools/build_ppc.sh +++ b/cpp/tools/build_ppc.sh @@ -35,7 +35,7 @@ default_version="v1.1.0" compatibility_version=${default_version} command="deploy" -disable_ra2018="false" +disable_ra2018="true" LOG_WARN() { local content=${1} @@ -420,9 +420,9 @@ generate_node_config_ini() { ; the threadPoolSize thread_count = 4 ; the gatewayService endpoint information - service.gateway_target = + gateway_target = ; the components - service.components = + components = nodeid=${nodeid} [crypto] @@ -524,8 +524,9 @@ generate_node_config_ini() { ; MB max_log_file_size=200 ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message - format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% - enable_rotate_by_hour=false + ;format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + format=%Severity%|%TimeStamp%|%Message% + enable_rotate_by_hour=true log_name_pattern=ppcs-psi4ef.log ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log rotate_name_pattern=log_%Y%m%d.%H%M.log diff --git a/cpp/tools/build_wedpr_cem.sh b/cpp/tools/build_wedpr_cem.sh index 5cb54e9d..0a2a6ed2 100644 --- a/cpp/tools/build_wedpr_cem.sh +++ b/cpp/tools/build_wedpr_cem.sh @@ -377,8 +377,9 @@ generate_config_ini() { ; MB max_log_file_size=200 ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message - format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% - enable_rotate_by_hour=false + ;format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + format=%Severity%|%TimeStamp%|%Message% + enable_rotate_by_hour=true log_name_pattern=ppcs-psi4ef.log ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log rotate_name_pattern=log_%Y%m%d.%H%M.log diff --git a/cpp/tools/ppc-builder/build_ppc.py b/cpp/tools/ppc-builder/build_ppc.py index 713d1801..21d98853 100644 --- a/cpp/tools/ppc-builder/build_ppc.py +++ b/cpp/tools/ppc-builder/build_ppc.py @@ -1,11 +1,11 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- # Note: here can't be refactored by autopep +import sys +sys.path.append("src/") from controller import commandline_helper from common import utilities import traceback -import sys -sys.path.append("src/") def main(): diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml index 7225f7e7..a27685dd 100644 --- a/cpp/tools/ppc-builder/conf/config-example.toml +++ b/cpp/tools/ppc-builder/conf/config-example.toml @@ -43,7 +43,7 @@ holding_msg_minutes = 30 # configuration for the ppc-node [[agency.node]] # disable the ra2018 psi or not, default enable ra2018 - disable_ra2018 = false + disable_ra2018 = true deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" @@ -128,7 +128,7 @@ holding_msg_minutes = 30 # configuration for the ppc-node [[agency.node]] # disable the ra2018 psi or not, default enable ra2018 - disable_ra2018 = false + disable_ra2018 = true deploy_ip=["127.0.0.1:2"] # node name, Notice: node_name in the same agency and group must be unique node_name = "node0" diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py index 3e9d8c2c..615d1d70 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py @@ -104,7 +104,7 @@ def __generate_single_node_inner_config__(self, tpl_config_path, node_path, priv config_content, node_config.rpc_config, node_index) # load the transport config self.__generate_transport_config__(config_content, - node_config, node_id, ip) + node_config, node_id, ip, node_index) # load the storage config self.__generate_storage_config__( config_content, node_config.storage_config) @@ -192,7 +192,7 @@ def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config): hdfs_storage_config.name_node_port) config_content[section_name]["token"] = hdfs_storage_config.token - def __generate_transport_config__(self, config_content, node_config, node_id, deploy_ip): + def __generate_transport_config__(self, config_content, node_config, node_id, deploy_ip, node_index): """_summary_ Args: @@ -203,18 +203,18 @@ def __generate_transport_config__(self, config_content, node_config, node_id, de ; the threadPoolSize thread_count = 4 ; the gatewayService endpoint information - service.gateway_target = + gateway_target = ; the components - service.components = + components = nodeid= """ section = "transport" config_content[section]["listen_ip"] = node_config.grpc_listen_ip config_content[section]["listen_port"] = str( - node_config.grpc_listen_port) + node_config.grpc_listen_port + node_index) config_content[section]["host_ip"] = deploy_ip - config_content[section]["service.gateway_target"] = node_config.gateway_config.gateway_grpc_target - config_content[section]["service.components"] = node_config.components + config_content[section]["gateway_target"] = node_config.gateway_config.gateway_grpc_target + config_content[section]["components"] = node_config.components config_content[section]["nodeid"] = node_id def __generate_ra2018psi_config__(self, config_content, ra2018psi_config): diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway index 8acc4990..10afa936 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.gateway +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.gateway @@ -39,8 +39,9 @@ ; MB max_log_file_size=200 ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message - format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% - enable_rotate_by_hour=false + ;format=%Severity%|ppcs-gateway|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + format=%Severity%|%TimeStamp%|%Message% + enable_rotate_by_hour=true log_name_pattern=ppcs-gateway.log ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log rotate_name_pattern=log_%Y%m%d.%H%M.log diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node index 177536ac..d47d7883 100644 --- a/cpp/tools/ppc-builder/src/tpl/config.ini.node +++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node @@ -4,7 +4,7 @@ ; the private key path for the psi-server private_key_path = conf/node.pem ; disable the ra2018 or not, default enable ra2018 - disable_ra2018 = false + disable_ra2018 = true ; the path that allows programs to access ; data_location = data ; task_timeout_minutes = 180 @@ -38,9 +38,9 @@ ; the threadPoolSize thread_count = 4 ; the gatewayService endpoint information - service.gateway_target = + gateway_target = ; the components - service.components = + components = nodeid= [storage] @@ -101,8 +101,9 @@ ; MB max_log_file_size=200 ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message - format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% - enable_rotate_by_hour=false + ;format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% + format=%Severity%|%TimeStamp%|%Message% + enable_rotate_by_hour=true log_name_pattern=ppcs-psi4ef.log ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log rotate_name_pattern=log_%Y%m%d.%H%M.log diff --git a/cpp/wedpr-computing/ppc-cem/tests/data/config.ini b/cpp/wedpr-computing/ppc-cem/tests/data/config.ini index 5cf1c6f8..c9562ec0 100644 --- a/cpp/wedpr-computing/ppc-cem/tests/data/config.ini +++ b/cpp/wedpr-computing/ppc-cem/tests/data/config.ini @@ -56,7 +56,7 @@ max_log_file_size=200 ; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message #format=%Severity%|ppcs-psi4ef|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message% - #enable_rotate_by_hour=false + #enable_rotate_by_hour=true #log_name_pattern=ppcs-psi4ef.log ; Y,m,d,H,M,S are supported, N is the sequence number log_%Y%m%d.%H%M%S.%N.log #rotate_name_pattern=log_%Y%m%d.%H%M.log diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index 530be4c0..d8dadc32 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -129,7 +129,8 @@ void PPCConfig::loadFrontConfig(bool requireTransport, } m_frontConfig->setNodeID(nodeID); m_frontConfig->setThreadPoolSize(threadCount); - + PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig and not require the transport") + << printFrontDesc(m_frontConfig); if (!requireTransport) { return; @@ -137,15 +138,18 @@ void PPCConfig::loadFrontConfig(bool requireTransport, loadEndpointConfig(m_frontConfig->mutableSelfEndPoint(), true, "transport", pt); // the gateway targets - auto gatewayTargets = pt.get("transport.service.gateway_target", ""); + auto gatewayTargets = pt.get("transport.gateway_target", ""); if (gatewayTargets.empty()) { - BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment( - "Must specify the transport.service.gateway_target!")); + BOOST_THROW_EXCEPTION( + InvalidConfig() << errinfo_comment("Must specify the transport.gateway_target!")); } + m_frontConfig->setGatewayGrpcTarget(gatewayTargets); // the components - auto components = pt.get("transport.service.components", ""); + auto components = pt.get("transport.components", ""); boost::split(m_frontConfig->mutableComponents(), components, boost::is_any_of(",")); + + PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig") << printFrontDesc(m_frontConfig); } void PPCConfig::setPrivateKey(bcos::bytes const& _privateKey) diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h index a3d90fb2..c5e52009 100644 --- a/cpp/wedpr-helper/ppc-utilities/Utilities.h +++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h @@ -65,7 +65,7 @@ inline std::string generateUUID() return boost::uuids::to_string(uuid_gen()); } template -inline std::string printVector(std::vector const& list) +inline std::string printVector(T const& list) { std::stringstream oss; for (auto const& it : list) diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index 4e7c4eec..14ceef33 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -413,6 +413,10 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc) void Initializer::start() { + if (m_transport) + { + m_transport->start(); + } if (m_ppcFront) { m_ppcFront->start(); @@ -455,6 +459,10 @@ void Initializer::start() void Initializer::stop() { + if (m_transport) + { + m_transport->stop(); + } // stop the network firstly if (m_ppcFront) { diff --git a/cpp/wedpr-protocol/grpc/Common.h b/cpp/wedpr-protocol/grpc/Common.h index b04f85d5..c0b2d0fb 100644 --- a/cpp/wedpr-protocol/grpc/Common.h +++ b/cpp/wedpr-protocol/grpc/Common.h @@ -31,13 +31,23 @@ inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr con { return args; } - args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); + // TODO: when enable round_robin load-balance policy, the program will be exited on dns resolver + // args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); if (grpcConfig->enableHealthCheck()) { args.SetServiceConfigJSON( "{\"healthCheckConfig\": " "{\"serviceName\": \"\"}}"); } + // disable dns lookup + if (!grpcConfig->enableDnslookup()) + { + args.SetInt("grpc.enable_dns_srv_lookup", 0); + } + else + { + args.SetInt("grpc.enable_dns_srv_lookup", 1); + } return args; } } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/Common.h b/cpp/wedpr-protocol/grpc/client/Common.h index c7b3ab2f..99a59a19 100644 --- a/cpp/wedpr-protocol/grpc/client/Common.h +++ b/cpp/wedpr-protocol/grpc/client/Common.h @@ -22,4 +22,5 @@ #define GRPC_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GRPC][CLIENT]" #define GATEWAY_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GATEWAY][CLIENT]" +#define FRONT_CLIENT_LOG(LEVEL) BCOS_LOG(LEVEL) << "[FRONT][CLIENT]" #define HEALTH_LOG(LEVEL) BCOS_LOG(LEVEL) << "[HEALTH]" \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp index a2037613..c934a442 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp @@ -18,6 +18,7 @@ * @date 2024-09-02 */ #include "FrontClient.h" +#include "Common.h" #include "protobuf/src/RequestConverter.h" #include "wedpr-protocol/protobuf/src/Common.h" @@ -33,8 +34,8 @@ void FrontClient::onReceiveMessage(ppc::protocol::Message::Ptr const& msg, Recei msg->encode(encodedData); receivedMsg.set_data(encodedData.data(), encodedData.size()); - ClientContext context; + auto context = std::make_shared(); auto response = std::make_shared(); - m_stub->async()->onReceiveMessage(&context, &receivedMsg, response.get(), - [response, callback](Status status) { callback(toError(status, std::move(*response))); }); + m_stub->async()->onReceiveMessage(context.get(), &receivedMsg, response.get(), + [response, callback](Status status) { callback(toError(status, *response)); }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index 7b55c1a3..8089f947 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -33,88 +33,89 @@ void GatewayClient::asyncSendMessage(RouteType routeType, long timeout, ReceiveMsgFunc callback) { auto request = generateRequest(traceID, routeType, routeInfo, std::move(payload), timeout); - ClientContext context; + auto context = std::make_shared(); auto response = std::make_shared(); - m_stub->async()->asyncSendMessage(&context, request.get(), response.get(), - [callback, response](Status status) { callback(toError(status, std::move(*response))); }); + m_stub->async()->asyncSendMessage(context.get(), request.get(), response.get(), + [callback, response](Status status) { callback(toError(status, *response)); }); } void GatewayClient::asyncGetPeers(std::function callback) { auto response = std::make_shared(); - ClientContext context; + auto context = std::make_shared(); auto request = std::make_shared(); m_stub->async()->asyncGetPeers( - &context, request.get(), response.get(), [callback, response](Status status) { - callback(toError(status, std::move(*response->mutable_error())), response->peersinfo()); + context.get(), request.get(), response.get(), [callback, response](Status status) { + callback(toError(status, response->error()), response->peersinfo()); }); } void GatewayClient::asyncGetAgencies( - std::function)> callback) + std::function)> callback) { auto response = std::make_shared(); - ClientContext context; + auto context = std::make_shared(); auto request = std::make_shared(); m_stub->async()->asyncGetAgencies( - &context, request.get(), response.get(), [callback, response](Status status) { - std::vector agencies; + context.get(), request.get(), response.get(), [callback, response](Status status) { + std::set agencies; for (int i = 0; i < response->agencies_size(); i++) { - agencies.emplace_back(response->agencies(i)); + agencies.insert(response->agencies(i)); } - callback(toError(status, std::move(*response->mutable_error())), agencies); + callback(toError(status, response->error()), agencies); }); } bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) { - broadCast([nodeInfo](ChannelInfo const& channel) { + return broadCast([nodeInfo](ChannelInfo const& channel) { std::unique_ptr stub( ppc::proto::Gateway::NewStub(channel.channel)); auto request = toNodeInfoRequest(nodeInfo); ClientContext context; std::shared_ptr response = std::make_shared(); auto status = stub->registerNodeInfo(&context, *request, response.get()); - return toError(status, std::move(*response)); + auto result = toError(status, *response); + return result; }); } bcos::Error::Ptr GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) { - broadCast([nodeID](ChannelInfo const& channel) { + return broadCast([nodeID](ChannelInfo const& channel) { std::unique_ptr stub( ppc::proto::Gateway::NewStub(channel.channel)); auto request = toNodeInfoRequest(nodeID, ""); ClientContext context; std::shared_ptr response = std::make_shared(); auto status = stub->unRegisterNodeInfo(&context, *request, response.get()); - return toError(status, std::move(*response)); + return toError(status, *response); }); } bcos::Error::Ptr GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) { - broadCast([nodeID, topic](ChannelInfo const& channel) { + return broadCast([nodeID, topic](ChannelInfo const& channel) { std::unique_ptr stub( ppc::proto::Gateway::NewStub(channel.channel)); auto request = toNodeInfoRequest(nodeID, topic); ClientContext context; std::shared_ptr response = std::make_shared(); auto status = stub->registerTopic(&context, *request, response.get()); - return toError(status, std::move(*response)); + return toError(status, *response); }); } bcos::Error::Ptr GatewayClient::unRegisterTopic( bcos::bytesConstRef nodeID, std::string const& topic) { - broadCast([nodeID, topic](ChannelInfo const& channel) { + return broadCast([nodeID, topic](ChannelInfo const& channel) { std::unique_ptr stub( ppc::proto::Gateway::NewStub(channel.channel)); auto request = toNodeInfoRequest(nodeID, topic); ClientContext context; std::shared_ptr response = std::make_shared(); auto status = stub->unRegisterTopic(&context, *request, response.get()); - return toError(status, std::move(*response)); + return toError(status, *response); }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index 58c79cec..0d2415ae 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -57,7 +57,7 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient void asyncGetPeers(std::function callback) override; void asyncGetAgencies( - std::function)> callback) override; + std::function)> callback) override; void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp index 2ba6e503..6516e1ed 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp @@ -48,6 +48,7 @@ bool GrpcClient::checkHealth() { try { + HEALTH_LOG(TRACE) << LOG_DESC("checkHealth"); ClientContext context; HealthCheckResponse response; auto status = diff --git a/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp index 4b18b0df..aa9f9223 100644 --- a/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp +++ b/cpp/wedpr-protocol/grpc/client/HealthCheckTimer.cpp @@ -45,7 +45,6 @@ void HealthCheckTimer::start() return; } healthChecker->checkHealth(); - healthChecker->m_timer->restart(); }); if (m_timer) { @@ -78,6 +77,8 @@ void HealthCheckTimer::registerHealthCheckHandler(HealthCheckHandler::Ptr health } bcos::WriteGuard l(x_healthCheckHandlers); m_healthCheckHandlers[healthCheckHandler->serviceName] = healthCheckHandler; + HEALTH_LOG(INFO) << LOG_DESC("registerHealthCheckHandler for ") + << healthCheckHandler->serviceName; } diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp index 8b1182e5..90e7cebe 100644 --- a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp @@ -29,13 +29,14 @@ using namespace grpc; ServerUnaryReactor* FrontServer::onReceiveMessage( CallbackServerContext* context, const ReceivedMessage* receivedMsg, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { // decode the request auto msg = m_msgBuilder->build(bcos::bytesConstRef( (bcos::byte*)receivedMsg->data().data(), receivedMsg->data().size())); m_front->onReceiveMessage(msg, [reactor, reply](bcos::Error::Ptr error) { + FRONT_SERVER_LOG(TRACE) << LOG_DESC("onReceiveMessage"); toSerializedError(reply, error); reactor->Finish(Status::OK); }); @@ -48,5 +49,5 @@ ServerUnaryReactor* FrontServer::onReceiveMessage( std::make_shared(-1, std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp index 8b770a06..e25fcb14 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp @@ -26,7 +26,7 @@ using namespace grpc; ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* context, const ppc::proto::SendedMessageRequest* sendedMsg, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { // TODO: optimize here @@ -48,13 +48,13 @@ ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* conte "handle message failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } grpc::ServerUnaryReactor* GatewayServer::asyncGetPeers( grpc::CallbackServerContext* context, const ppc::proto::Empty*, ppc::proto::PeersInfo* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { m_gateway->asyncGetPeers([reactor, reply](bcos::Error::Ptr error, std::string peersInfo) { @@ -72,17 +72,17 @@ grpc::ServerUnaryReactor* GatewayServer::asyncGetPeers( -1, "asyncGetPeers failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } grpc::ServerUnaryReactor* GatewayServer::asyncGetAgencies( grpc::CallbackServerContext* context, const ppc::proto::Empty*, ppc::proto::AgenciesInfo* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { m_gateway->asyncGetAgencies( - [reactor, reply](bcos::Error::Ptr error, std::vector agencies) { + [reactor, reply](bcos::Error::Ptr error, std::set agencies) { toSerializedError(reply->mutable_error(), error); for (auto const& it : agencies) { @@ -100,14 +100,14 @@ grpc::ServerUnaryReactor* GatewayServer::asyncGetAgencies( "asyncGetAgencies failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } ServerUnaryReactor* GatewayServer::registerNodeInfo(CallbackServerContext* context, const ppc::proto::NodeInfo* serializedNodeInfo, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { auto nodeInfo = toNodeInfo(m_nodeInfoFactory, *serializedNodeInfo); @@ -124,13 +124,13 @@ ServerUnaryReactor* GatewayServer::registerNodeInfo(CallbackServerContext* conte "registerNodeInfo failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } ServerUnaryReactor* GatewayServer::unRegisterNodeInfo( CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { auto result = m_gateway->unRegisterNodeInfo( @@ -147,13 +147,13 @@ ServerUnaryReactor* GatewayServer::unRegisterNodeInfo( std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } ServerUnaryReactor* GatewayServer::registerTopic( CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { auto result = m_gateway->registerTopic( @@ -171,13 +171,13 @@ ServerUnaryReactor* GatewayServer::registerTopic( -1, "registerTopic failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } ServerUnaryReactor* GatewayServer::unRegisterTopic( CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) { - std::shared_ptr reactor(context->DefaultReactor()); + ServerUnaryReactor* reactor(context->DefaultReactor()); try { auto result = m_gateway->unRegisterTopic( @@ -195,5 +195,5 @@ ServerUnaryReactor* GatewayServer::unRegisterTopic( "unRegisterTopic failed for : " + std::string(boost::diagnostic_information(e)))); reactor->Finish(Status::OK); } - return reactor.get(); + return reactor; } \ No newline at end of file diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h index 4f1544bb..7c84d79d 100644 --- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h +++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h @@ -39,12 +39,13 @@ inline MessageOptionalHeader::Ptr generateRouteInfo( routeInfo->setDstNode( bcos::bytes(serializedRouteInfo.dstnode().begin(), serializedRouteInfo.dstnode().end())); routeInfo->setDstInst(serializedRouteInfo.dstinst()); + routeInfo->setTopic(serializedRouteInfo.topic()); return routeInfo; } -inline std::shared_ptr generateRequest(std::string const& traceID, - RouteType routeType, - MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout) +inline std::shared_ptr generateRequest(std::string const& traceID, + RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, + long timeout) { auto request = std::make_shared(); request->set_traceid(traceID); @@ -106,7 +107,7 @@ inline INodeInfo::Ptr toNodeInfo( return nodeInfo; } -inline bcos::Error::Ptr toError(grpc::Status const& status, ppc::proto::Error&& error) +inline bcos::Error::Ptr toError(grpc::Status const& status, ppc::proto::Error const& error) { if (!status.ok()) { diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 153e45ea..23dc4dec 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -58,7 +58,7 @@ void Front::stop() void Front::fetchGatewayMetaInfo() { auto self = weak_from_this(); - m_front->asyncGetAgencies([self](bcos::Error::Ptr error, std::vector agencies) { + m_front->asyncGetAgencies([self](bcos::Error::Ptr error, std::set agencies) { auto front = self.lock(); if (!front) { @@ -71,15 +71,16 @@ void Front::fetchGatewayMetaInfo() << LOG_KV("msg", error->errorMessage()); return; } + std::vector agencyList(agencies.begin(), agencies.end()); bcos::UpgradableGuard l(front->x_agencyList); - if (front->m_agencyList == agencies) + if (front->m_agencyList == agencyList) { return; } bcos::UpgradeGuard ul(l); - front->m_agencyList = agencies; + front->m_agencyList = agencyList; FRONT_LOG(INFO) << LOG_DESC("Update agencies information") - << LOG_KV("agencies", printVector(agencies)); + << LOG_KV("agencies", printVector(agencyList)); }); m_fetcher->restart(); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index 920ceeb2..f88f6bc4 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -223,6 +223,8 @@ void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::P routeInfo->setSrcNode(m_nodeID); auto payload = std::make_shared(); frontMessage->encode(*payload); + FRONT_LOG(TRACE) << LOG_DESC("asyncSendMessageToGateway") << LOG_KV("traceID", traceID) + << LOG_KV("route", printOptionalField(routeInfo)); m_gatewayClient->asyncSendMessage( routeType, routeInfo, traceID, std::move(*payload), timeout, callback); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index b60f1e87..ca751374 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -142,7 +142,7 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ } void asyncGetAgencies( - std::function)> callback) override + std::function)> callback) override { m_gatewayClient->asyncGetAgencies(callback); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 7a997afc..29ebc29b 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -42,7 +42,7 @@ GatewayImpl::GatewayImpl(Service::Ptr const& service, m_gatewayInfoFactory(std::make_shared(service->nodeID(), agency)), m_localRouter(std::make_shared( m_gatewayInfoFactory, m_frontBuilder, std::make_shared(ioService))), - m_peerRouter(std::make_shared(m_service)) + m_peerRouter(std::make_shared(m_service, m_gatewayInfoFactory)) { m_service->registerMsgHandler((uint16_t)GatewayPacketType::P2PMessage, boost::bind(&GatewayImpl::onReceiveP2PMessage, this, boost::placeholders::_1, @@ -53,6 +53,34 @@ GatewayImpl::GatewayImpl(Service::Ptr const& service, boost::placeholders::_2)); m_gatewayRouterManager = std::make_shared( m_service, m_gatewayInfoFactory, m_localRouter, m_peerRouter); + + m_service->registerOnNewSession([this](WsSession::Ptr _session) { + if (!_session) + { + return; + } + m_p2pRouterManager->onNewSession(_session->nodeId()); + }); + + m_service->registerOnDeleteSession([this](WsSession::Ptr _session) { + if (!_session) + { + return; + } + m_p2pRouterManager->onEraseSession(_session->nodeId()); + }); + + m_p2pRouterManager->registerUnreachableHandler([this](std::string const& unreachableNode) { + m_gatewayRouterManager->removeUnreachableP2pNode(unreachableNode); + }); + + m_service->registerDisconnectHandler([this](WsSession::Ptr _session) { + if (!_session) + { + return; + } + m_gatewayRouterManager->removeUnreachableP2pNode(_session->nodeId()); + }); } void GatewayImpl::start() @@ -287,14 +315,13 @@ void GatewayImpl::asyncGetPeers(std::function cal } } -void GatewayImpl::asyncGetAgencies( - std::function)> callback) +void GatewayImpl::asyncGetAgencies(std::function)> callback) { if (!callback) { return; } auto agencies = m_peerRouter->agencies(); - agencies.emplace_back(m_agency); + agencies.insert(m_agency); callback(nullptr, agencies); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 4cc4af78..87e10fa5 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -69,7 +69,7 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this callback) override; void asyncGetAgencies( - std::function)> callback) override; + std::function)> callback) override; protected: virtual void onReceiveP2PMessage( diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp index de96b110..e16d0fa4 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/cache/MessageCache.cpp @@ -30,7 +30,7 @@ void MessageCache::insertCache( std::string const& topic, ppc::protocol::Message::Ptr const& msg, ReceiveMsgFunc callback) { // hold the message - GATEWAY_LOG(DEBUG) << LOG_BADGE("MessageCache: insertCache") << LOG_KV("topic", topic); + GATEWAY_LOG(DEBUG) << LOG_BADGE("MessageCache: insertCache") << printMessage(msg); bcos::ReadGuard l(x_msgCache); auto it = m_msgCache.find(topic); if (it != m_msgCache.end()) diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index 470d4b3b..cfa4362e 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -71,6 +71,7 @@ class GatewayNodeInfoFactory virtual ~GatewayNodeInfoFactory() = default; virtual GatewayNodeInfo::Ptr build() const = 0; + virtual GatewayNodeInfo::Ptr build(std::string const& p2pNode) const = 0; }; struct GatewayNodeInfoCmp { diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index fbe3a205..553af157 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -30,10 +30,13 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo public: using Ptr = std::shared_ptr; GatewayNodeInfoImpl() : m_rawGatewayInfo(std::make_shared()) {} - GatewayNodeInfoImpl(std::string const& p2pNodeID, std::string const& agency) - : GatewayNodeInfoImpl() + GatewayNodeInfoImpl(std::string const& p2pNodeID) : GatewayNodeInfoImpl() { m_rawGatewayInfo->set_p2pnodeid(p2pNodeID); + } + GatewayNodeInfoImpl(std::string const& p2pNodeID, std::string const& agency) + : GatewayNodeInfoImpl(p2pNodeID) + { m_rawGatewayInfo->set_agency(agency); } @@ -121,6 +124,11 @@ class GatewayNodeInfoFactoryImpl : public GatewayNodeInfoFactory return std::make_shared(m_p2pNodeID, m_agency); } + GatewayNodeInfo::Ptr build(std::string const& p2pNode) const override + { + return std::make_shared(p2pNode); + } + private: std::string m_p2pNodeID; std::string m_agency; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp index 7534e3b0..b22b73a4 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp @@ -79,6 +79,18 @@ void GatewayRouterManager::stop() ROUTER_MGR_LOG(INFO) << LOG_DESC("stop GatewayRouterManager success"); } +void GatewayRouterManager::removeUnreachableP2pNode(std::string const& p2pNode) +{ + ROUTER_MGR_LOG(INFO) << LOG_DESC("removeUnreachableP2pNode") + << LOG_KV("p2pid", printP2PIDElegantly(p2pNode)); + { + // remove statusSeq info + WriteGuard l(x_p2pID2Seq); + m_p2pID2Seq.erase(p2pNode); + } + m_peerRouter->removeP2PID(p2pNode); +} + void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSession::Ptr session) { auto statusSeq = diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h index 4cf9b0ba..679eb08e 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h @@ -36,6 +36,8 @@ class GatewayRouterManager virtual void start(); virtual void stop(); + void removeUnreachableP2pNode(std::string const& p2pNode); + protected: virtual void onReceiveNodeSeqMessage( bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 3a7c035f..e9a7e25a 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -30,12 +30,14 @@ bool LocalRouter::registerNodeInfo(ppc::protocol::INodeInfo::Ptr nodeInfo, std::function onUnHealthHandler, bool removeHandlerOnUnhealth) { LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo") << printNodeInfo(nodeInfo); - nodeInfo->setFront(m_frontBuilder->buildClient( - nodeInfo->endPoint(), onUnHealthHandler, removeHandlerOnUnhealth)); auto ret = m_routerInfo->tryAddNodeInfo(nodeInfo); if (ret) { - LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo success") << printNodeInfo(nodeInfo); + // only create the frontClient when update + nodeInfo->setFront(m_frontBuilder->buildClient( + nodeInfo->endPoint(), onUnHealthHandler, removeHandlerOnUnhealth)); + LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerNodeInfo: update the node") + << printNodeInfo(nodeInfo); increaseSeq(); } return ret; @@ -86,6 +88,12 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call } return true; } + // the case broadcast failed + if (msg->header() && msg->header()->optionalField() && + msg->header()->optionalField()->topic().empty()) + { + return false; + } if (!holding) { return false; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index abb55789..6063e355 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -32,6 +32,35 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) PEER_ROUTER_LOG(INFO) << LOG_DESC("updateGatewayInfo") << LOG_KV("detail", printNodeStatus(gatewayInfo)); auto nodeList = gatewayInfo->nodeList(); + + removeP2PNodeIDFromNodeIDInfos(gatewayInfo); + insertGatewayInfo(gatewayInfo); +} + +void PeerRouterTable::insertGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) +{ + auto nodeList = gatewayInfo->nodeList(); + bcos::WriteGuard l(x_mutex); + // insert new information for the gateway + for (auto const& it : nodeList) + { + // update nodeID => gatewayInfos + if (!m_nodeID2GatewayInfos.count(it.first)) + { + m_nodeID2GatewayInfos.insert(std::make_pair(it.first, GatewayNodeInfos())); + } + m_nodeID2GatewayInfos[it.first].insert(gatewayInfo); + } + if (!m_agency2GatewayInfos.count(gatewayInfo->agency())) + { + m_agency2GatewayInfos.insert(std::make_pair(gatewayInfo->agency(), GatewayNodeInfos())); + } + // update agency => gatewayInfos + m_agency2GatewayInfos[gatewayInfo->agency()].insert(gatewayInfo); +} + +void PeerRouterTable::removeP2PNodeIDFromNodeIDInfos(GatewayNodeInfo::Ptr const& gatewayInfo) +{ bcos::WriteGuard l(x_mutex); // remove the origin information of the gateway auto it = m_nodeID2GatewayInfos.begin(); @@ -50,31 +79,50 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo) } it++; } - // insert new information for the gateway - for (auto const& it : nodeList) +} + +void PeerRouterTable::removeP2PNodeIDFromAgencyInfos(std::string const& p2pNode) +{ + bcos::WriteGuard l(x_mutex); + for (auto it = m_agency2GatewayInfos.begin(); it != m_agency2GatewayInfos.end();) { - // update nodeID => gatewayInfos - if (!m_nodeID2GatewayInfos.count(it.first)) + auto& gatewayInfos = it->second; + for (auto pGateway = gatewayInfos.begin(); pGateway != gatewayInfos.end();) { - m_nodeID2GatewayInfos.insert(std::make_pair(it.first, GatewayNodeInfos())); + if ((*pGateway)->p2pNodeID() == p2pNode) + { + pGateway = gatewayInfos.erase(pGateway); + continue; + } + pGateway++; } - m_nodeID2GatewayInfos[it.first].insert(gatewayInfo); - } - if (!m_agency2GatewayInfos.count(gatewayInfo->agency())) - { - m_agency2GatewayInfos.insert(std::make_pair(gatewayInfo->agency(), GatewayNodeInfos())); + if (gatewayInfos.empty()) + { + it = m_agency2GatewayInfos.erase(it); + continue; + } + it++; } - // update agency => gatewayInfos - m_agency2GatewayInfos[gatewayInfo->agency()].insert(gatewayInfo); } -std::vector PeerRouterTable::agencies() const +void PeerRouterTable::removeP2PID(std::string const& p2pNode) +{ + PEER_ROUTER_LOG(INFO) << LOG_DESC("PeerRouterTable: removeP2PID") + << LOG_KV("p2pID", printP2PIDElegantly(p2pNode)); + // remove P2PNode from m_nodeID2GatewayInfos + auto gatewayInfo = m_gatewayInfoFactory->build(p2pNode); + removeP2PNodeIDFromNodeIDInfos(gatewayInfo); + // remove P2PNode from m_agency2GatewayInfos + removeP2PNodeIDFromAgencyInfos(p2pNode); +} + +std::set PeerRouterTable::agencies() const { - std::vector agencies; + std::set agencies; bcos::ReadGuard l(x_mutex); for (auto const& it : m_agency2GatewayInfos) { - agencies.emplace_back(it.first); + agencies.insert(it.first); } return agencies; } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h index 7a286443..d9819420 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h @@ -31,16 +31,19 @@ class PeerRouterTable { public: using Ptr = std::shared_ptr; - PeerRouterTable(Service::Ptr service) : m_service(std::move(service)) {} + PeerRouterTable(Service::Ptr service, GatewayNodeInfoFactory::Ptr gatewayInfoFactory) + : m_service(std::move(service)), m_gatewayInfoFactory(std::move(gatewayInfoFactory)) + {} virtual ~PeerRouterTable() = default; virtual void updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo); + virtual void removeP2PID(std::string const& p2pNode); virtual GatewayNodeInfos selectRouter( ppc::protocol::RouteType const& routeType, ppc::protocol::Message::Ptr const& msg) const; virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const; - std::vector agencies() const; + std::set agencies() const; std::map gatewayInfos() const { @@ -52,10 +55,13 @@ class PeerRouterTable virtual GatewayNodeInfos selectRouterByNodeID(ppc::protocol::Message::Ptr const& msg) const; virtual GatewayNodeInfos selectRouterByComponent(ppc::protocol::Message::Ptr const& msg) const; virtual GatewayNodeInfos selectRouterByAgency(ppc::protocol::Message::Ptr const& msg) const; - + void removeP2PNodeIDFromNodeIDInfos(GatewayNodeInfo::Ptr const& gatewayInfo); + void insertGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo); + void removeP2PNodeIDFromAgencyInfos(std::string const& p2pNode); private: Service::Ptr m_service; + GatewayNodeInfoFactory::Ptr m_gatewayInfoFactory; // nodeID => p2pNodes std::map m_nodeID2GatewayInfos; // agency => p2pNodes diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp index dfca0e5c..88a4c4e6 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -48,7 +48,6 @@ Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _rou boost::bind(&Service::onP2PDisconnect, this, boost::placeholders::_1)); } - void Service::onP2PConnect(WsSession::Ptr _session) { SERVICE_LOG(INFO) << LOG_DESC("Receive new p2p connection") @@ -88,6 +87,7 @@ void Service::onP2PConnect(WsSession::Ptr _session) { // the new session m_nodeID2Session.insert(std::make_pair(_session->nodeId(), _session)); + callNewSessionHandlers(_session); } SERVICE_LOG(INFO) << LOG_DESC("onP2PConnect established new session") << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) @@ -126,6 +126,7 @@ bool Service::removeSessionInfo(WsSession::Ptr const& _session) << LOG_KV("endpoint", _session->endPoint()); m_nodeID2Session.erase(it); + callDeleteSessionHandlers(_session); return true; } return false; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h index 377d1161..20b525b1 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h @@ -70,6 +70,17 @@ class Service : public bcos::boostssl::ws::WsService } } + // handlers called when new-session + void registerOnNewSession(std::function _handler) + { + m_newSessionHandlers.emplace_back(_handler); + } + // handlers called when delete-session + void registerOnDeleteSession(std::function _handler) + { + m_deleteSessionHandlers.emplace_back(_handler); + } + protected: void onRecvMessage(bcos::boostssl::MessageFace::Ptr _msg, bcos::boostssl::ws::WsSession::Ptr _session) override; @@ -93,6 +104,37 @@ class Service : public bcos::boostssl::ws::WsService bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::Options options, bcos::boostssl::ws::RespCallBack respFunc); + virtual void callNewSessionHandlers(bcos::boostssl::ws::WsSession::Ptr _session) + { + try + { + for (auto const& handler : m_newSessionHandlers) + { + handler(_session); + } + } + catch (std::exception const& e) + { + SERVICE_LOG(WARNING) << LOG_DESC("callNewSessionHandlers exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + } + virtual void callDeleteSessionHandlers(bcos::boostssl::ws::WsSession::Ptr _session) + { + try + { + for (auto const& handler : m_deleteSessionHandlers) + { + handler(_session); + } + } + catch (std::exception const& e) + { + SERVICE_LOG(WARNING) << LOG_DESC("callDeleteSessionHandlers exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + } + protected: std::string m_nodeID; // nodeID=>session @@ -105,5 +147,10 @@ class Service : public bcos::boostssl::ws::WsService // configuredNode=>nodeID std::map m_configuredNode2ID; mutable bcos::SharedMutex x_configuredNode2ID; + + // handlers called when new-session + std::vector> m_newSessionHandlers; + // handlers called when delete-session + std::vector> m_deleteSessionHandlers; }; } // namespace ppc::gateway \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp index ce25385c..b4970643 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp @@ -165,19 +165,27 @@ void RouterManager::joinRouterTable( // called when the nodes become unreachable void RouterManager::onP2PNodesUnreachable(std::set const& _p2pNodeIDs) { - std::vector> handlers; + try { - ReadGuard readGuard(x_unreachableHandlers); - handlers = m_unreachableHandlers; - } - // TODO: async here - for (auto const& node : _p2pNodeIDs) - { - for (auto const& it : m_unreachableHandlers) + std::vector> handlers; + { + ReadGuard readGuard(x_unreachableHandlers); + handlers = m_unreachableHandlers; + } + // TODO: async here + for (auto const& node : _p2pNodeIDs) { - it(node); + for (auto const& it : m_unreachableHandlers) + { + it(node); + } } } + catch (std::exception const& e) + { + SERVICE_ROUTER_LOG(WARNING) << LOG_DESC("onP2PNodesUnreachable exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } } void RouterManager::broadcastRouterSeq() @@ -191,4 +199,51 @@ void RouterManager::broadcastRouterSeq() message->setPayload(std::make_shared((byte*)&statusSeq, (byte*)&statusSeq + 4)); // the router table should only exchange between neighbor m_service->broadcastMessage(message); +} + +std::set RouterManager::onEraseSession(std::string const& sessionNodeID) +{ + eraseSeq(sessionNodeID); + std::set unreachableNodes; + if (m_service->routerTable()->erase(unreachableNodes, sessionNodeID)) + { + m_statusSeq++; + broadcastRouterSeq(); + } + onP2PNodesUnreachable(unreachableNodes); + SERVICE_ROUTER_LOG(INFO) << LOG_DESC("onEraseSession") + << LOG_KV("dst", printP2PIDElegantly(sessionNodeID)); + return unreachableNodes; +} + +bool RouterManager::eraseSeq(std::string const& _p2pNodeID) +{ + UpgradableGuard l(x_node2Seq); + if (!m_node2Seq.count(_p2pNodeID)) + { + return false; + } + UpgradeGuard ul(l); + m_node2Seq.erase(_p2pNodeID); + return true; +} + +std::set RouterManager::onNewSession(std::string const& sessionNodeID) +{ + std::set unreachableNodes; + auto entry = m_service->routerTableFactory()->createRouterEntry(); + entry->setDstNode(sessionNodeID); + entry->setDistance(0); + if (!m_service->routerTable()->update(unreachableNodes, m_service->nodeID(), entry)) + { + SERVICE_ROUTER_LOG(INFO) << LOG_DESC("onNewSession: RouterTable not changed") + << LOG_KV("dst", printP2PIDElegantly(sessionNodeID)); + return unreachableNodes; + } + m_statusSeq++; + broadcastRouterSeq(); + SERVICE_ROUTER_LOG(INFO) << LOG_DESC("onNewSession: update routerTable") + << LOG_KV("dst", printP2PIDElegantly(sessionNodeID)); + onP2PNodesUnreachable(unreachableNodes); + return unreachableNodes; } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h index 64ebf4c8..ab50a887 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h @@ -41,6 +41,8 @@ class RouterManager virtual void start(); virtual void stop(); + std::set onEraseSession(std::string const& sessionNodeID); + std::set onNewSession(std::string const& sessionNodeID); private: void onReceiveRouterSeq( @@ -54,6 +56,7 @@ class RouterManager bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session); void joinRouterTable(std::string const& _generatedFrom, RouterTableInterface::Ptr _routerTable); + bool eraseSeq(std::string const& _p2pNodeID); void onP2PNodesUnreachable(std::set const& _p2pNodeIDs); diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp index 2ff873ed..3763e339 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -32,8 +32,9 @@ using namespace ppc::sdk; ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keepAlivePeriodMs) : m_config(std::move(config)), m_keepAlivePeriodMs(keepAlivePeriodMs) { - // default enable health-check - auto grpcServerConfig = std::make_shared(config->selfEndPoint(), true); + // Note: since the config has been moved away, should not use the `config`, use `m_config` + // instead default enable health-check + auto grpcServerConfig = std::make_shared(m_config->selfEndPoint(), true); m_server = std::make_shared(grpcServerConfig); FrontFactory frontFactory; @@ -41,14 +42,13 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keep std::make_shared(m_config->grpcConfig(), m_config->gatewayGrpcTarget()); m_front = frontFactory.build(std::make_shared(), std::make_shared(), - std::make_shared(), m_gateway, config); + std::make_shared(), m_gateway, m_config); auto msgBuilder = std::make_shared(std::make_shared()); - auto frontService = std::make_shared(msgBuilder, m_front); - frontService->setHealthCheckService(m_server->server()->GetHealthCheckService()); + m_frontService = std::make_shared(msgBuilder, m_front); // register the frontService - m_server->registerService(frontService); + m_server->registerService(m_frontService); } void ProTransportImpl::start() @@ -65,6 +65,8 @@ void ProTransportImpl::start() }); m_timer->start(); m_server->start(); + // Note: the server is inited after start + m_frontService->setHealthCheckService(m_server->server()->GetHealthCheckService()); m_front->start(); } diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index ed29cbdd..76e2ce3a 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -24,7 +24,8 @@ namespace ppc::protocol { class GrpcServer; -} +class FrontServer; +} // namespace ppc::protocol namespace ppc::sdk @@ -44,6 +45,7 @@ class ProTransportImpl : public Transport, public std::enable_shared_from_this

m_server; + std::shared_ptr m_frontService; int m_keepAlivePeriodMs; std::shared_ptr m_timer; }; From f5919afd629bcce521e4c3a716b6def80722468a Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 11 Sep 2024 19:22:08 +0800 Subject: [PATCH 034/120] fix grpc crash caused by clientContext not alive && reuse broadcast stubs && fix sendResponse bug (#31) * reuse broadcast stubs * fix sendResponse bug --- cpp/ppc-framework/protocol/GrpcConfig.h | 4 +- .../config/ppc_gateway_config_generator.py | 2 +- .../ppc-tools/src/config/PPCConfig.cpp | 4 +- cpp/wedpr-initializer/Initializer.cpp | 8 +- cpp/wedpr-initializer/Initializer.h | 2 +- cpp/wedpr-protocol/grpc/Common.h | 3 +- .../grpc/client/FrontClient.cpp | 11 ++- .../grpc/client/GatewayClient.cpp | 99 +++++++++++++------ .../grpc/client/GatewayClient.h | 5 +- cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 4 +- .../grpc/client/RemoteFrontBuilder.h | 5 +- .../protobuf/src/RequestConverter.h | 12 +-- .../ppc-gateway/gateway/GatewayImpl.cpp | 9 +- .../gateway/router/LocalRouter.cpp | 20 +++- .../ppc-gateway/ppc-gateway/p2p/Service.cpp | 15 +-- 15 files changed, 129 insertions(+), 74 deletions(-) diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h index 108b22c1..c0f2704e 100644 --- a/cpp/ppc-framework/protocol/GrpcConfig.h +++ b/cpp/ppc-framework/protocol/GrpcConfig.h @@ -43,7 +43,7 @@ class GrpcServerConfig protected: ppc::protocol::EndPoint m_endPoint; - bool m_enableHealthCheck = false; + bool m_enableHealthCheck = true; }; class GrpcConfig { @@ -65,7 +65,7 @@ class GrpcConfig bool enableDnslookup() const { return m_enableDnslookup; } protected: - bool m_enableHealthCheck = false; + bool m_enableHealthCheck = true; std::string m_loadBalancePolicy = "round_robin"; bool m_enableDnslookup = false; }; diff --git a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py index a92eea28..2c06abbb 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py +++ b/cpp/tools/ppc-builder/src/config/ppc_gateway_config_generator.py @@ -164,4 +164,4 @@ def __generate_gateway_config_content__(self, config, config_content, listen_por section = "transport" config_content[section]["listen_ip"] = config.grpc_listen_ip # the listen port - config_content[section]["listen_port"] = str(config.grpc_listen_port) + config_content[section]["listen_port"] = str(grpc_listen_port) diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp index d8dadc32..1a18fad4 100644 --- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp +++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp @@ -129,10 +129,10 @@ void PPCConfig::loadFrontConfig(bool requireTransport, } m_frontConfig->setNodeID(nodeID); m_frontConfig->setThreadPoolSize(threadCount); - PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig and not require the transport") - << printFrontDesc(m_frontConfig); if (!requireTransport) { + PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig and not require the transport") + << printFrontDesc(m_frontConfig); return; } diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index 14ceef33..26b120d9 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -52,17 +52,19 @@ using namespace ppc::crypto; using namespace ppc::sdk; Initializer::Initializer(ppc::protocol::NodeArch _arch, std::string const& _configPath) - : m_arch(_arch), m_configPath(_configPath) + : m_arch((uint16_t)_arch), m_configPath(_configPath) { m_transportBuilder = std::make_shared(); // load the config m_config = std::make_shared(); - if (m_arch == ppc::protocol::NodeArch::PRO) + if (m_arch == (uint16_t)ppc::protocol::NodeArch::PRO) { + INIT_LOG(INFO) << LOG_DESC("loadNodeConfig for pro node"); m_config->loadNodeConfig(true, m_transportBuilder->frontConfigBuilder(), _configPath); } else { + INIT_LOG(INFO) << LOG_DESC("loadNodeConfig for air node"); m_config->loadNodeConfig(false, m_transportBuilder->frontConfigBuilder(), _configPath); } } @@ -80,7 +82,7 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway) // Note: must set the m_holdingMessageMinutes before init the node TransportBuilder transportBuilder; - if (m_arch == ppc::protocol::NodeArch::AIR) + if (m_arch == (uint16_t)ppc::protocol::NodeArch::AIR) { m_transport = transportBuilder.build(SDKMode::AIR, m_config->frontConfig(), gateway); } diff --git a/cpp/wedpr-initializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h index 90e8ae69..4a49702b 100644 --- a/cpp/wedpr-initializer/Initializer.h +++ b/cpp/wedpr-initializer/Initializer.h @@ -82,7 +82,7 @@ class Initializer : public std::enable_shared_from_this private: - ppc::protocol::NodeArch m_arch; + uint16_t m_arch; std::string m_configPath; std::shared_ptr m_config; ProtocolInitializer::Ptr m_protocolInitializer; diff --git a/cpp/wedpr-protocol/grpc/Common.h b/cpp/wedpr-protocol/grpc/Common.h index c0b2d0fb..b38c7e52 100644 --- a/cpp/wedpr-protocol/grpc/Common.h +++ b/cpp/wedpr-protocol/grpc/Common.h @@ -31,8 +31,7 @@ inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr con { return args; } - // TODO: when enable round_robin load-balance policy, the program will be exited on dns resolver - // args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); + args.SetLoadBalancingPolicyName(grpcConfig->loadBalancePolicy()); if (grpcConfig->enableHealthCheck()) { args.SetServiceConfigJSON( diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp index c934a442..cd3b9b39 100644 --- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp @@ -29,13 +29,16 @@ using namespace grpc; void FrontClient::onReceiveMessage(ppc::protocol::Message::Ptr const& msg, ReceiveMsgFunc callback) { // TODO: optimize here - ReceivedMessage receivedMsg; + std::unique_ptr request(new ReceivedMessage()); bcos::bytes encodedData; msg->encode(encodedData); - receivedMsg.set_data(encodedData.data(), encodedData.size()); + request->set_data(encodedData.data(), encodedData.size()); + // The ClientContext instance used for creating an rpc must remain alive and valid for the + // lifetime of the rpc auto context = std::make_shared(); auto response = std::make_shared(); - m_stub->async()->onReceiveMessage(context.get(), &receivedMsg, response.get(), - [response, callback](Status status) { callback(toError(status, *response)); }); + // lambda keeps the lifecycle for clientContext + m_stub->async()->onReceiveMessage(context.get(), request.get(), response.get(), + [context, response, callback](Status status) { callback(toError(status, *response)); }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index 8089f947..9a71005b 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -28,15 +28,29 @@ using namespace grpc; using namespace ppc::gateway; using namespace ppc::protocol; +GatewayClient::GatewayClient( + ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) + : GrpcClient(grpcConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel)) +{ + for (auto const& channel : m_broadcastChannels) + { + m_broadcastStubs.insert( + std::make_pair(channel.endPoint, ppc::proto::Gateway::NewStub(channel.channel))); + } +} + void GatewayClient::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, std::string const& traceID, bcos::bytes&& payload, long timeout, ReceiveMsgFunc callback) { - auto request = generateRequest(traceID, routeType, routeInfo, std::move(payload), timeout); + std::unique_ptr request( + generateRequest(traceID, routeType, routeInfo, std::move(payload), timeout)); auto context = std::make_shared(); auto response = std::make_shared(); + // lambda keeps the lifecycle for clientContext m_stub->async()->asyncSendMessage(context.get(), request.get(), response.get(), - [callback, response](Status status) { callback(toError(status, *response)); }); + [context, traceID, callback, response]( + Status status) { callback(toError(status, *response)); }); } void GatewayClient::asyncGetPeers(std::function callback) @@ -44,8 +58,9 @@ void GatewayClient::asyncGetPeers(std::function(); auto context = std::make_shared(); auto request = std::make_shared(); + // lambda keeps the lifecycle for clientContext m_stub->async()->asyncGetPeers( - context.get(), request.get(), response.get(), [callback, response](Status status) { + context.get(), request.get(), response.get(), [context, callback, response](Status status) { callback(toError(status, response->error()), response->peersinfo()); }); } @@ -56,8 +71,9 @@ void GatewayClient::asyncGetAgencies( auto response = std::make_shared(); auto context = std::make_shared(); auto request = std::make_shared(); + // lambda keeps the lifecycle for clientContext m_stub->async()->asyncGetAgencies( - context.get(), request.get(), response.get(), [callback, response](Status status) { + context.get(), request.get(), response.get(), [context, callback, response](Status status) { std::set agencies; for (int i = 0; i < response->agencies_size(); i++) { @@ -69,13 +85,18 @@ void GatewayClient::asyncGetAgencies( bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) { - return broadCast([nodeInfo](ChannelInfo const& channel) { - std::unique_ptr stub( - ppc::proto::Gateway::NewStub(channel.channel)); - auto request = toNodeInfoRequest(nodeInfo); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = stub->registerNodeInfo(&context, *request, response.get()); + std::unique_ptr request(toNodeInfoRequest(nodeInfo)); + return broadCast([&](ChannelInfo const& channel) { + if (!m_broadcastStubs.count(channel.endPoint)) + { + return make_shared( + -1, "registerNodeInfo failed for not find stub for endPoint: " + channel.endPoint); + } + auto const& stub = m_broadcastStubs.at(channel.endPoint); + + auto context = std::make_shared(); + auto response = std::make_shared(); + auto status = stub->registerNodeInfo(context.get(), *request, response.get()); auto result = toError(status, *response); return result; }); @@ -83,25 +104,35 @@ bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo) bcos::Error::Ptr GatewayClient::unRegisterNodeInfo(bcos::bytesConstRef nodeID) { - return broadCast([nodeID](ChannelInfo const& channel) { - std::unique_ptr stub( - ppc::proto::Gateway::NewStub(channel.channel)); - auto request = toNodeInfoRequest(nodeID, ""); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = stub->unRegisterNodeInfo(&context, *request, response.get()); + std::unique_ptr request(toNodeInfoRequest(nodeID, "")); + return broadCast([&](ChannelInfo const& channel) { + if (!m_broadcastStubs.count(channel.endPoint)) + { + return make_shared(-1, + "unRegisterNodeInfo failed for not find stub for endPoint: " + channel.endPoint); + } + auto const& stub = m_broadcastStubs.at(channel.endPoint); + + auto context = std::make_shared(); + auto response = std::make_shared(); + auto status = stub->unRegisterNodeInfo(context.get(), *request, response.get()); return toError(status, *response); }); } bcos::Error::Ptr GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) { - return broadCast([nodeID, topic](ChannelInfo const& channel) { - std::unique_ptr stub( - ppc::proto::Gateway::NewStub(channel.channel)); - auto request = toNodeInfoRequest(nodeID, topic); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = stub->registerTopic(&context, *request, response.get()); + std::unique_ptr request(toNodeInfoRequest(nodeID, topic)); + return broadCast([&](ChannelInfo const& channel) { + if (!m_broadcastStubs.count(channel.endPoint)) + { + return make_shared( + -1, "registerTopic failed for not find stub for endPoint: " + channel.endPoint); + } + auto const& stub = m_broadcastStubs.at(channel.endPoint); + + auto context = std::make_shared(); + auto response = std::make_shared(); + auto status = stub->registerTopic(context.get(), *request, response.get()); return toError(status, *response); }); } @@ -109,13 +140,17 @@ bcos::Error::Ptr GatewayClient::registerTopic(bcos::bytesConstRef nodeID, std::s bcos::Error::Ptr GatewayClient::unRegisterTopic( bcos::bytesConstRef nodeID, std::string const& topic) { - return broadCast([nodeID, topic](ChannelInfo const& channel) { - std::unique_ptr stub( - ppc::proto::Gateway::NewStub(channel.channel)); - auto request = toNodeInfoRequest(nodeID, topic); - ClientContext context; - std::shared_ptr response = std::make_shared(); - auto status = stub->unRegisterTopic(&context, *request, response.get()); + std::unique_ptr request(toNodeInfoRequest(nodeID, topic)); + return broadCast([&](ChannelInfo const& channel) { + if (!m_broadcastStubs.count(channel.endPoint)) + { + return make_shared( + -1, "unRegisterTopic failed for not find stub for endPoint: " + channel.endPoint); + } + auto const& stub = m_broadcastStubs.at(channel.endPoint); + auto context = std::make_shared(); + auto response = std::make_shared(); + auto status = stub->unRegisterTopic(context.get(), *request, response.get()); return toError(status, *response); }); } \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index 0d2415ae..2206e000 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -27,9 +27,7 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient { public: using Ptr = std::shared_ptr; - GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints) - : GrpcClient(grpcConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel)) - {} + GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints); ~GatewayClient() override = default; @@ -70,5 +68,6 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient private: std::unique_ptr m_stub; + std::map> m_broadcastStubs; }; } // namespace ppc::protocol \ No newline at end of file diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp index 6516e1ed..c0b3d0a5 100644 --- a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp @@ -37,7 +37,7 @@ GrpcClient::GrpcClient( // create the broadcast channels for (auto const& endPoint : endPointList) { - GRPC_CLIENT_LOG(INFO) << LOG_DESC("create broacast-channel, endpoint: ") << endPoint; + GRPC_CLIENT_LOG(INFO) << LOG_DESC("create broadcast-channel, endpoint: ") << endPoint; m_broadcastChannels.push_back( {endPoint, grpc::CreateCustomChannel(endPoint, grpc::InsecureChannelCredentials(), toChannelConfig(grpcConfig))}); @@ -94,7 +94,7 @@ bcos::Error::Ptr GrpcClient::broadCast( catch (std::exception const& e) { GRPC_CLIENT_LOG(WARNING) - << LOG_DESC("registerNodeInfo exception") << LOG_KV("remote", channel.endPoint) + << LOG_DESC("GrpcClient broadCast exception") << LOG_KV("remote", channel.endPoint) << LOG_KV("error", boost::diagnostic_information(e)); } } diff --git a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h index 8a6a6030..3770f6a0 100644 --- a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h +++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.h @@ -31,10 +31,7 @@ class RemoteFrontBuilder : public IFrontBuilder RemoteFrontBuilder(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, ppc::protocol::HealthCheckTimer::Ptr healthChecker) : m_grpcConfig(grpcConfig), m_healthChecker(healthChecker) - { - // Note: the front enable health-check - m_grpcConfig->setEnableHealthCheck(true); - } + {} ~RemoteFrontBuilder() override = default; IFrontClient::Ptr buildClient(std::string endPoint, std::function onUnHealthHandler, diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h index 7c84d79d..7641103f 100644 --- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h +++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h @@ -43,11 +43,11 @@ inline MessageOptionalHeader::Ptr generateRouteInfo( return routeInfo; } -inline std::shared_ptr generateRequest(std::string const& traceID, +inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& traceID, RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, long timeout) { - auto request = std::make_shared(); + auto request = new ppc::proto::SendedMessageRequest(); request->set_traceid(traceID); request->set_routetype(uint16_t(routeType)); // set the route information @@ -65,18 +65,18 @@ inline std::shared_ptr generateRequest(std::st return request; } -inline std::shared_ptr toNodeInfoRequest( +inline ppc::proto::NodeInfo* toNodeInfoRequest( bcos::bytesConstRef const& nodeID, std::string const& topic) { - auto request = std::make_shared(); + auto request = new ppc::proto::NodeInfo(); request->set_nodeid(nodeID.data(), nodeID.size()); request->set_topic(topic); return request; } -inline std::shared_ptr toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo) +inline ppc::proto::NodeInfo* toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo) { - auto request = std::make_shared(); + auto request = new ppc::proto::NodeInfo(); if (!nodeInfo) { return request; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index 29ebc29b..fe2becda 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -190,18 +190,25 @@ void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr sessi // try to dispatcher to the front auto p2pMessage = std::dynamic_pointer_cast(msg); auto self = std::weak_ptr(shared_from_this()); + // Note: the callback can only been called once since it binds the callback seq auto callback = [p2pMessage, session, self](Error::Ptr error) { auto gateway = self.lock(); if (!gateway) { return; } + // Note: no need to sendResponse for the response packet + if (p2pMessage->isRespPacket()) + { + return; + } std::string errorCode = std::to_string(CommonError::SUCCESS); if (error && error->errorCode() != 0) { GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveP2PMessage: dispatcherMessage failed") << LOG_KV("code", error->errorCode()) - << LOG_KV("msg", error->errorMessage()); + << LOG_KV("msg", error->errorMessage()) + << printMessage(p2pMessage); errorCode = std::to_string(error->errorCode()); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index e9a7e25a..513f6b5f 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -82,9 +82,27 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call // find the front if (!frontList.empty()) { + // Note: the callback can only been called once since it binds the callback seq + int i = 0; for (auto const& front : frontList) { - front->onReceiveMessage(msg, callback); + if (i == 0) + { + front->onReceiveMessage(msg, callback); + } + else + { + front->onReceiveMessage(msg, [](bcos::Error::Ptr error) { + if (!error || error->errorCode() == 0) + { + return; + } + LOCAL_ROUTER_LOG(WARNING) << LOG_DESC("dispatcherMessage to front failed") + << LOG_KV("code", error->errorMessage()) + << LOG_KV("msg", error->errorMessage()); + }); + } + i++; } return true; } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp index 88a4c4e6..12dcb06b 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp @@ -54,7 +54,6 @@ void Service::onP2PConnect(WsSession::Ptr _session) << LOG_KV("p2pid", printP2PIDElegantly(_session->nodeId())) << LOG_KV("endpoint", _session->endPoint()); - RecursiveGuard l(x_nodeID2Session); auto it = m_nodeID2Session.find(_session->nodeId()); if (it != m_nodeID2Session.end() && it->second->isConnected()) @@ -203,18 +202,13 @@ WsSession::Ptr Service::getSessionByNodeID(std::string const& _nodeID) return it->second; } +// Note: this only called by the sender; will not been called when forward void Service::asyncSendMessageByNodeID( std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc) { auto p2pMsg = std::dynamic_pointer_cast(msg); - if (p2pMsg->header()->dstGwNode().empty()) - { - p2pMsg->header()->setDstGwNode(dstNodeID); - } - if (p2pMsg->header()->srcGwNode().empty()) - { - p2pMsg->header()->setSrcGwNode(m_nodeID); - } + p2pMsg->header()->setDstGwNode(dstNodeID); + p2pMsg->header()->setSrcGwNode(m_nodeID); return asyncSendMessageWithForward(dstNodeID, msg, options, respFunc); } @@ -366,7 +360,8 @@ void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& WsSessions sessions; sessions.emplace_back(session); WsService::asyncSendMessage(sessions, respMessage); - SERVICE_LOG(TRACE) << "sendRespMessageBySession" << LOG_KV("resp", printMessage(respMessage)) + SERVICE_LOG(TRACE) << "sendRespMessageBySession: " << LOG_KV("resp", printMessage(respMessage)) + << LOG_KV("sessionNode", printP2PIDElegantly(session->nodeId())) << LOG_KV("payloadSize", respMessage->payload() ? respMessage->payload()->size() : 0); } \ No newline at end of file From 03747f780cd55003fad869c6ba93858b92f3de8a Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 13 Sep 2024 10:57:49 +0800 Subject: [PATCH 035/120] update java-sdk wrapper (#32) * update java-sdk wrapper * optimize compile --- cpp/CMakeLists.txt | 9 +- cpp/cmake/Dependencies.cmake | 10 +- cpp/cmake/Options.cmake | 8 + cpp/cmake/TargetSettings.cmake | 5 +- cpp/ppc-framework/Helper.h | 4 +- cpp/ppc-framework/front/FrontConfig.h | 2 + cpp/ppc-framework/front/IFront.h | 140 ++++++++++++++++- cpp/ppc-framework/protocol/Message.h | 38 +++++ cpp/ppc-framework/protocol/MessagePayload.h | 6 + cpp/vcpkg.json | 16 ++ cpp/wedpr-transport/ppc-front/CMakeLists.txt | 2 +- .../ppc-front/ppc-front/Front.cpp | 2 +- .../ppc-front/ppc-front/FrontImpl.cpp | 9 +- .../ppc-front/ppc-front/FrontImpl.h | 4 +- .../ppc-front/ppc-front/PPCChannel.h | 2 - .../ppc-front/ppc-front/PPCChannelManager.h | 2 - .../sdk-wrapper/java/CMakeLists.txt | 11 +- .../sdk-wrapper/java/bindings/build.gradle | 61 ++++---- .../wedpr/sdk/jni/common/JniLibLoader.java | 10 +- .../java/swig/wedpr_java_transport.i | 148 ++++++++++++++++-- cpp/wedpr-transport/sdk/ProTransportImpl.cpp | 12 +- cpp/wedpr-transport/sdk/ProTransportImpl.h | 2 +- cpp/wedpr-transport/sdk/Transport.cpp | 33 ++++ cpp/wedpr-transport/sdk/Transport.h | 30 +++- cpp/wedpr-transport/sdk/TransportBuilder.cpp | 2 +- cpp/wedpr-transport/sdk/TransportBuilder.h | 10 +- cpp/wedpr-transport/sdk/TransportImpl.h | 13 +- 27 files changed, 501 insertions(+), 90 deletions(-) create mode 100644 cpp/wedpr-transport/sdk/Transport.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c7c0d094..45d13348 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -72,6 +72,12 @@ set(CEM_SOURCE "") if(BUILD_CEM) set(CEM_SOURCE "wedpr-computing/ppc-cem") endif() + +set(TRANSPORT_SDK_SOURCE_LIST + wedpr-protocol + wedpr-transport/ppc-front + wedpr-transport/sdk wedpr-transport/sdk-wrapper) + set(ALL_SOURCE_LIST ${SDK_SOURCE_LIST} wedpr-crypto/ppc-crypto @@ -92,7 +98,6 @@ if(BUILD_WEDPR_TOOLKIT) include(swig) message(STATUS "Getting SWIG for Windows: ...DONE") endif() - add_subdirectory(wedpr-transport/sdk-wrapper) endif() if(BUILD_ALL) @@ -101,6 +106,8 @@ elseif(BUILD_UDF) add_sources("${UDF_SOURCE_LIST}") elseif(BUILD_SDK) add_sources("${SDK_SOURCE_LIST}") +elseif(BUILD_WEDPR_TOOLKIT) + add_sources("${TRANSPORT_SDK_SOURCE_LIST}") endif() ########### set the sources end ########### diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake index 4a2183ca..d8e1bf62 100644 --- a/cpp/cmake/Dependencies.cmake +++ b/cpp/cmake/Dependencies.cmake @@ -21,17 +21,19 @@ find_package(unofficial-sodium CONFIG REQUIRED) ##### the full-dependencies ##### -if(BUILD_ALL) - find_package(TBB REQUIRED) +if(BUILD_ALL OR BUILD_WEDPR_TOOLKIT) find_package(jsoncpp REQUIRED) - + find_package(TBB REQUIRED) + find_package(gRPC REQUIRED) find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED) +endif() + +if(BUILD_ALL) # tcmalloc include(ProjectTCMalloc) find_package(SEAL REQUIRED) find_package(Kuku REQUIRED) - find_package(gRPC REQUIRED) # APSI: Note: APSI depends on seal 4.0 and Kuku 2.1 include(ProjectAPSI) diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 9b092e36..ab319fb3 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -130,10 +130,18 @@ macro(configure_project) set(VISIBILITY_FLAG "") set(BUILD_ALL OFF) endif() + if (BUILD_WEDPR_TOOLKIT) + set(VISIBILITY_FLAG "") + set(BUILD_ALL OFF) + endif() if (BUILD_ALL) # install all dependencies list(APPEND VCPKG_MANIFEST_FEATURES "all") endif() + if (BUILD_WEDPR_TOOLKIT) + # install wedpr dependencies + list(APPEND VCPKG_MANIFEST_FEATURES "toolkit") + endif() if(ENABLE_SSE) # enable sse for libhdfs3 list(APPEND VCPKG_MANIFEST_FEATURES "sse") diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 1b30601a..47573b12 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -140,5 +140,6 @@ set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk) set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport") set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT}) -set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport") -set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/${WEDPR_JAVA_TRANSPORT}) \ No newline at end of file +set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport_jni") +set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/generated/${WEDPR_JAVA_TRANSPORT}) +set(WEDPR_JAVA_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native) diff --git a/cpp/ppc-framework/Helper.h b/cpp/ppc-framework/Helper.h index 13f72c46..d084cff7 100644 --- a/cpp/ppc-framework/Helper.h +++ b/cpp/ppc-framework/Helper.h @@ -18,8 +18,8 @@ * @date 2022-10-20 */ #pragma once -#include #include +#include namespace ppc { constexpr static int MAX_PORT = 65535; @@ -42,7 +42,7 @@ inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept template inline std::string_view printNodeID(T const& nodeID) { - size_t offset = nodeID.size() >= 8 ? 8 : nodeID.size(); + size_t offset = nodeID.size() >= 15 ? 15 : nodeID.size(); return std::string_view((const char*)nodeID.data(), offset); } } // namespace ppc \ No newline at end of file diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h index aa1938da..8af6ca05 100644 --- a/cpp/ppc-framework/front/FrontConfig.h +++ b/cpp/ppc-framework/front/FrontConfig.h @@ -73,6 +73,8 @@ class FrontConfig virtual std::vector const& getComponents() const { return m_components; } void setComponents(std::vector const& components) { m_components = components; } + void addComponent(std::string const& component) { m_components.emplace_back(component); } + std::vector& mutableComponents() { return m_components; } protected: diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 76cdbcad..1c0c49ea 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -41,6 +41,59 @@ class IFrontClient virtual void onReceiveMessage( ppc::protocol::Message::Ptr const& _msg, ppc::protocol::ReceiveMsgFunc _callback) = 0; }; + +///////// the callback definition for sdk wrapper ///////// +class ErrorCallback +{ +public: + using Ptr = std::shared_ptr; + ErrorCallback() = default; + virtual ~ErrorCallback() {} + + virtual void onError(bcos::Error::Ptr error) = 0; +}; + +class MessageDispatcherHandler +{ +public: + using Ptr = std::shared_ptr; + MessageDispatcherHandler() = default; + virtual ~MessageDispatcherHandler() {} + + virtual void onMessage(ppc::protocol::Message::Ptr msg) = 0; +}; + +class SendResponseHandler +{ +public: + using Ptr = std::shared_ptr; + SendResponseHandler(ppc::protocol::SendResponseFunction responseFunc) + : m_responseFunc(responseFunc) + {} + virtual ~SendResponseHandler() {} + + virtual void sendResponse(std::shared_ptr&& payload) + { + m_responseFunc(std::move(payload)); + } + +private: + ppc::protocol::SendResponseFunction m_responseFunc; +}; + +class IMessageHandler +{ +public: + using Ptr = std::shared_ptr; + IMessageHandler() = default; + virtual ~IMessageHandler() {} + + virtual void onMessage(bcos::Error::Ptr e, ppc::protocol::Message::Ptr msg, + SendResponseHandler sendResponseHandler) = 0; +}; + +///////// the callback definition for sdk wrapper ///////// + class IFront : virtual public IFrontClient { public: @@ -71,8 +124,23 @@ class IFront : virtual public IFrontClient virtual void registerTopicHandler( std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) = 0; + /////// to simplify SDK wrapper //// + virtual void registerTopicHandler( + std::string const& topic, MessageDispatcherHandler::Ptr callback) + { + registerTopicHandler(topic, populateMessageDispatcherCallback(callback)); + } + virtual void registerMessageHandler( std::string const& componentType, ppc::protocol::MessageDispatcherCallback callback) = 0; + + /////// to simplify SDK wrapper //// + virtual void registerMessageHandler( + std::string const& componentType, MessageDispatcherHandler::Ptr callback) + { + registerMessageHandler(componentType, populateMessageDispatcherCallback(callback)); + } + /** * @brief async send message * @@ -88,19 +156,53 @@ class IFront : virtual public IFrontClient * @param timeout timeout * @param callback callback */ - virtual void asyncSendMessage(ppc::protocol::RouteType routeType, + virtual void asyncSendMessage(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout, ppc::protocol::ReceiveMsgFunc errorCallback, ppc::protocol::MessageCallback callback) = 0; + /////// to simplify SDK wrapper //// + + // !!! Note: the 'payload' type(char*) should not been changed, since it used to pass-in java + // byte[] data + virtual void asyncSendMessage(uint16_t routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, + uint64_t payloadSize, int seq, long timeout, ErrorCallback::Ptr errorCallback, + IMessageHandler::Ptr msgHandler) + { + // TODO: optimize here + bcos::bytes copyedPayload(payload, payload + payloadSize); + asyncSendMessage(routeType, routeInfo, std::move(copyedPayload), seq, timeout, + populateErrorCallback(errorCallback), populateMsgCallback(msgHandler)); + } + virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0; + /////// to simplify SDK wrapper //// + + // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java + // byte[] data + virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + bcos::bytes&& payload, int seq, ErrorCallback::Ptr errorCallback) + { + asyncSendResponse( + dstNode, traceID, std::move(payload), seq, populateErrorCallback(errorCallback)); + } + // the sync interface for async_send_message - virtual bcos::Error::Ptr push(ppc::protocol::RouteType routeType, + virtual bcos::Error::Ptr push(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout) = 0; + // TODO: optmize here + virtual bcos::Error::Ptr push(uint16_t routeType, + ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, + uint64_t payloadSize, int seq, long timeout) + { + bcos::bytes copyedPayload(payload, payload + payloadSize); + return push(routeType, routeInfo, std::move(copyedPayload), seq, timeout); + } virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; @@ -131,6 +233,40 @@ class IFront : virtual public IFrontClient * @param topic the topic to unregister */ virtual bcos::Error::Ptr unRegisterTopic(std::string const& topic) = 0; + + +private: + ppc::protocol::ReceiveMsgFunc populateErrorCallback(ErrorCallback::Ptr errorCallback) + { + if (errorCallback == nullptr) + { + return nullptr; + } + return [errorCallback](bcos::Error::Ptr error) { errorCallback->onError(error); }; + } + + ppc::protocol::MessageDispatcherCallback populateMessageDispatcherCallback( + MessageDispatcherHandler::Ptr handler) + { + if (handler == nullptr) + { + return nullptr; + } + return [handler](ppc::protocol::Message::Ptr msg) { handler->onMessage(msg); }; + } + + ppc::protocol::MessageCallback populateMsgCallback(IMessageHandler::Ptr msgHandler) + { + if (msgHandler == nullptr) + { + return nullptr; + } + return [msgHandler](bcos::Error::Ptr e, ppc::protocol::Message::Ptr msg, + ppc::protocol::SendResponseFunction resFunc) { + SendResponseHandler sendResponseHandler(resFunc); + msgHandler->onMessage(e, msg, sendResponseHandler); + }; + } }; class IFrontBuilder diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 3d78b432..2988fc4b 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -21,6 +21,7 @@ #include "MessagePayload.h" #include "RouteType.h" #include "ppc-framework/Helper.h" +#include "ppc-framework/libwrapper/Buffer.h" #include #include #include @@ -46,12 +47,34 @@ class MessageOptionalHeader // the source nodeID that send the message virtual bcos::bytes const& srcNode() const { return m_srcNode; } + /// for swig-wrapper(pass the binary data) + OutputBuffer srcNodeBuffer() const + { + // Note: this will be copied to java through jni + return OutputBuffer{(unsigned char*)m_srcNode.data(), m_srcNode.size()}; + } + virtual void setSrcNode(bcos::bytes const& srcNode) { m_srcNode = srcNode; } + // !!! Note: the first paramater type should not been changed, for it's used for pass-in java + // byte[] into c bytes + virtual void setSrcNode(char* data, uint64_t length) { m_srcNode.assign(data, data + length); } + // the target nodeID that should receive the message virtual bcos::bytes const& dstNode() const { return m_dstNode; } + + // for swig-wrapper(pass the binary to java) + OutputBuffer dstNodeBuffer() const + { + // Note: this will be copied to java through jni + return OutputBuffer{(unsigned char*)m_dstNode.data(), m_dstNode.size()}; + } virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; } + // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in + // java byte[] into c bytes + virtual void setDstNode(char* data, uint64_t length) { m_dstNode.assign(data, data + length); } + // the target agency that need receive the message virtual std::string const& dstInst() const { return m_dstInst; } virtual void setDstInst(std::string const& dstInst) { m_dstInst = dstInst; } @@ -184,6 +207,16 @@ class Message : virtual public bcos::boostssl::MessageFace } std::shared_ptr payload() const override { return m_payload; } + // for swig wrapper + OutputBuffer payloadBuffer() const + { + if (!m_payload) + { + return OutputBuffer{nullptr, 0}; + } + return OutputBuffer{(unsigned char*)m_payload->data(), m_payload->size()}; + } + void setPayload(std::shared_ptr _payload) override { m_payload = std::move(_payload); @@ -196,6 +229,11 @@ class Message : virtual public bcos::boostssl::MessageFace MessagePayload::Ptr const& frontMessage() const { return m_frontMessage; } + // Note: swig wrapper require define all methods + virtual bool encode(bcos::bytes& _buffer) = 0; + // encode and return the {header, payload} + virtual bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) = 0; + virtual int64_t decode(bcos::bytesConstRef _buffer) = 0; protected: MessageHeader::Ptr m_header; diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h index f5fbf9e3..a8e6a57c 100644 --- a/cpp/ppc-framework/protocol/MessagePayload.h +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -18,6 +18,7 @@ * @date 2024-08-22 */ #pragma once +#include "ppc-framework/libwrapper/Buffer.h" #include #include @@ -42,6 +43,11 @@ class MessagePayload virtual void setVersion(uint8_t version) { m_version = version; } // data virtual bcos::bytes const& data() const { return m_data; } + // for swig wrapper here + virtual OutputBuffer dataBuffer() const + { + return OutputBuffer{(unsigned char*)m_data.data(), m_data.size()}; + } virtual void setData(bcos::bytes&& data) { m_data = std::move(data); } virtual void setData(bcos::bytes const& data) { m_data = data; } // the seq diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index a56cfe04..83449a54 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -103,6 +103,22 @@ "tbb", "libxml2" ] + }, + "toolkit": { + "description": "tooklit dependencies", + "dependencies": [ + { + "name": "bcos-boostssl", + "version>=": "3.2.3" + }, + { + "name": "grpc", + "version>=": "1.51.1" + }, + "tarscpp", + "tbb", + "libxml2" + ] } } } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/CMakeLists.txt b/cpp/wedpr-transport/ppc-front/CMakeLists.txt index efd94094..8966cbae 100644 --- a/cpp/wedpr-transport/ppc-front/CMakeLists.txt +++ b/cpp/wedpr-transport/ppc-front/CMakeLists.txt @@ -4,7 +4,7 @@ aux_source_directory(ppc-front SRCS) add_library(${FRONT_TARGET} ${SRCS}) -target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc) +target_link_libraries(${FRONT_TARGET} PUBLIC ${TARS_PROTOCOL_TARGET} TBB::tbb) #if (TESTS) # enable_testing() diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 23dc4dec..7a205b7b 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -134,7 +134,7 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace }; } // ROUTE_THROUGH_TOPIC will hold the topic - m_front->asyncSendMessage(RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data), + m_front->asyncSendMessage((uint16_t)RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data), _message->seq(), _timeout, _callback, msgCallback); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index f88f6bc4..58aae243 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -138,7 +138,7 @@ void FrontImpl::asyncSendResponse(bcos::bytes const& dstNode, std::string const& * @param timeout timeout * @param callback callback */ -void FrontImpl::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, +void FrontImpl::asyncSendMessage(uint16_t routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout, ReceiveMsgFunc errorCallback, MessageCallback callback) { @@ -151,8 +151,9 @@ void FrontImpl::asyncSendMessage(RouteType routeType, MessageOptionalHeader::Ptr m_callbackManager->addCallback(traceID, timeout, callback); auto self = weak_from_this(); // send the message to the gateway - asyncSendMessageToGateway(false, std::move(frontMessage), routeType, traceID, routeInfo, - timeout, [self, traceID, routeInfo, errorCallback](bcos::Error::Ptr error) { + asyncSendMessageToGateway(false, std::move(frontMessage), (ppc::protocol::RouteType)routeType, + traceID, routeInfo, timeout, + [self, traceID, routeInfo, errorCallback](bcos::Error::Ptr error) { auto front = self.lock(); if (!front) { @@ -265,7 +266,7 @@ void FrontImpl::onReceiveMessage(Message::Ptr const& msg, ReceiveMsgFunc callbac } // the sync interface for asyncSendMessage -bcos::Error::Ptr FrontImpl::push(RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, +bcos::Error::Ptr FrontImpl::push(uint16_t routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout) { auto promise = std::make_shared>(); diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index ca751374..84000656 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -52,7 +52,7 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ */ void stop() override; - bcos::Error::Ptr push(ppc::protocol::RouteType routeType, + bcos::Error::Ptr push(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout) override; /** @@ -70,7 +70,7 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ * @param timeout timeout * @param callback callback */ - void asyncSendMessage(ppc::protocol::RouteType routeType, + void asyncSendMessage(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq, long timeout, ppc::protocol::ReceiveMsgFunc errorCallback, ppc::protocol::MessageCallback callback) override; diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h index f881a915..3f5e4edf 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannel.h @@ -23,7 +23,6 @@ #include "Common.h" #include "Front.h" #include "ppc-framework/front/Channel.h" -#include #include namespace ppc::front @@ -43,7 +42,6 @@ class PPCChannel : public Channel, public std::enable_shared_from_this().swap(m_handlers); std::unordered_map().swap(m_messages); - MallocExtension::instance()->ReleaseFreeMemory(); FRONT_LOG(INFO) << LOG_DESC("the PPCChannel destroyed"); } diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h index 3e4b6bd4..506b2416 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/PPCChannelManager.h @@ -21,7 +21,6 @@ #pragma once #include "PPCChannel.h" -#include #include namespace ppc::front @@ -37,7 +36,6 @@ class PPCChannelManager : public std::enable_shared_from_this { std::unordered_map().swap(m_holdingMessages); std::unordered_map().swap(m_channels); - MallocExtension::instance()->ReleaseFreeMemory(); FRONT_LOG(INFO) << LOG_DESC("the PPCChannelManager destroyed"); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt index d4d36ffd..9276c9e6 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt @@ -1,19 +1,22 @@ -file(GLOB_RECURSE SRCS *.i) +file(GLOB_RECURSE SRCS wedpr_java_transport.i) set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) -set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.transport") +set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.generated") set_property(SOURCE swig/wedpr_java_transport.i PROPERTY COMPILE_OPTIONS -package ${WEDPR_TRANSPORT_PACKAGE}) +file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_DIR}) +file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_LIB_DIR}) + swig_add_library(${WEDPR_JAVA_TRANSPORT} TYPE MODULE LANGUAGE java OUTPUT_DIR ${WEDPR_JAVA_TRANSPORT_DIR} SOURCES ${SRCS}) -SET(LIBRARY_OUTPUT_PATH ${WEDPR_JAVA_TRANSPORT_DIR}) -message("LIBRARY_OUTPUT_PATH: ${WEDPR_JAVA_TRANSPORT_DIR}") +SET(LIBRARY_OUTPUT_PATH ${WEDPR_JAVA_TRANSPORT_LIB_DIR}) +message("LIBRARY_OUTPUT_PATH: ${WEDPR_JAVA_TRANSPORT_LIB_DIR}") set_target_properties(${WEDPR_JAVA_TRANSPORT} PROPERTIES SWIG_USE_TARGET_INCLUDE_DIRECTORIES ON diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index c455b8ad..bbceb950 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -3,9 +3,24 @@ plugins { id 'idea' id 'java-library' id 'maven-publish' - //id 'org.ajoberstar.grgit' version '4.1.1' - //id "de.undercouch.download" version "4.1.2" -// id 'com.github.sherter.google-java-format' version '0.9' + id 'org.ajoberstar.grgit' version '4.1.1' + id "de.undercouch.download" version "4.1.2" + id 'com.github.sherter.google-java-format' version '0.9' +} + +// Additional attribute definition +ext { + if (!project.hasProperty("ossrhUsername")) { + ossrhUsername="xxx" + } + + if (!project.hasProperty("ossrhPassword")) { + ossrhPassword="xxx" + } + lombokVersion = "1.18.32" + junitVersion = "4.13.2" + log4jVersion = "2.23.0" + jmhVersion = "1.36" } println("Notice: current gradle version is " + gradle.gradleVersion) @@ -23,15 +38,15 @@ repositories { mavenLocal() } -// googleJavaFormat { -// toolVersion = '1.7' -// options style: 'AOSP' -// source = sourceSets*.allJava -// include '**/*.java' -// exclude '**/*Test.java' -// exclude '**/Test*.java' -// exclude '**/Mock*.java' -// } + googleJavaFormat { + toolVersion = '1.7' + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' + exclude '**/*Test.java' + exclude '**/Test*.java' + exclude '**/Mock*.java' + } sourceSets { main { @@ -61,28 +76,20 @@ sourceSets { dependencies { api ('org.slf4j:slf4j-api:1.7.36') - jmhImplementation 'org.openjdk.jmh:jmh-core:1.36' - jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.36' + implementation("org.projectlombok:lombok:${lombokVersion}") + jmhImplementation ("org.openjdk.jmh:jmh-core:${jmhVersion}") + jmhAnnotationProcessor ("org.openjdk.jmh:jmh-generator-annprocess:${jmhVersion}") + + testImplementation ("org.slf4j:slf4j-log4j12:${log4jVersion}") + testImplementation ("junit:junit:${junitVersion}") - testImplementation ('org.slf4j:slf4j-log4j12:1.7.36') - testImplementation ('junit:junit:4.13.2') + annotationProcessor("org.projectlombok:lombok:${lombokVersion}") } archivesBaseName = 'wedpr-java-transport-jni' group = 'com.webank.wedpr' version = '1.0.0' -// Additional attribute definition -ext { - if (!project.hasProperty("ossrhUsername")) { - ossrhUsername="xxx" - } - - if (!project.hasProperty("ossrhPassword")) { - ossrhPassword="xxx" - } -} - jar { exclude '**/*.xml' exclude '**/*.properties' diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java index 9af0301e..f30e9165 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java @@ -37,7 +37,7 @@ public final class JniLibLoader { public static final String MAC = "mac"; public static final String WIN = "win"; - public static final String NATIVE_RESOURCE_LIB_NAME = "wedpr-java-transport-jni"; + public static final String NATIVE_RESOURCE_LIB_NAME = "wedpr_java_transport_jni"; public static final String NATIVE_RESOURCE_HOME = "/META-INF/native"; public static final String NATIVE_WIN_DEPS_DIR = NATIVE_RESOURCE_HOME + "/win/"; public static final String WIN_DEPS_FILE_LIST = "file.list"; @@ -51,13 +51,13 @@ public final class JniLibLoader { // 1. initialize workdir // -Dcom.webank.wedpr.workdir - String workdir = System.getProperty("com.webank.wedpr.workdir"); + String workdir = System.getProperty("com.webank.wedpr.sdk.jni.workdir"); if (workdir != null) { try { File f = new File(workdir); f.mkdirs(); WORKDIR = f; - logger.info("initialize workdir, -Dcom.webank.wedpr.workdir: {}", WORKDIR); + logger.info("initialize workdir, -Dcom.webank.wedpr.sdk.jni.workdir: {}", WORKDIR); } catch (Exception e) { logger.debug("initialize workdir, cannot mkdir workdir: {}, e: ", workdir, e); workdir = null; @@ -101,9 +101,9 @@ public static String getLibName(String baseName) { } else if (osName.contains(MAC)) { String arch = getArch(); if ("arm".equals(arch)) { - return "lib" + baseName + "-aarch64" + ".dylib"; + return "lib" + baseName + "-aarch64" + ".jnilib"; } - return "lib" + baseName + ".dylib"; + return "lib" + baseName + ".jnilib"; } else { throw new RuntimeException("unrecognized OS: " + osName); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index 05a504c8..35dfdd9b 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -1,4 +1,4 @@ -%module wedpr_java_transport +%module(directors="1") wedpr_java_transport %include "stdint.i" %include "typemaps.i" @@ -25,50 +25,178 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long); #endif // defined(SWIGWORDSIZE64) #endif // defined(SWIGJAVA) + %include %include %include %include %include +%include - +// shared_ptr definition %shared_ptr(ppc::front::FrontConfig); +%shared_ptr(ppc::front::IFront); +%shared_ptr(ppc::front::IFrontClient); +// the callbacks +%shared_ptr(ppc::front::ErrorCallback); +%shared_ptr(ppc::front::MessageDispatcherHandler); +%shared_ptr(ppc::front::IMessageHandler); + +%shared_ptr(ppc::gateway::IGateway); %shared_ptr(bcos::Error); +%shared_ptr(bcos::bytes); %shared_ptr(ppc::protocol::Message); -%shared_ptr(ppc::protocol::MessageOptionalHeader) +%shared_ptr(ppc::protocol::MessageOptionalHeader); +%shared_ptr(ppc::protocol::MessageHeader); +%shared_ptr(ppc::protocol::MessagePayload); +%shared_ptr(ppc::protocol::MessageBuilder); +%shared_ptr(ppc::protocol::MessageHeaderBuilder); +%shared_ptr(ppc::protocol::MessagePayloadBuilder); +%shared_ptr(ppc::protocol::MessageOptionalHeaderBuilder); +%shared_ptr(ppc::protocol::GrpcConfig); %shared_ptr(ppc::sdk::Transport); + %{ #define SWIG_FILE_WITH_INIT +#include +#include #include #include "wedpr-transport/sdk/TransportBuilder.h" #include "wedpr-transport/sdk/Transport.h" +#include "ppc-framework/libwrapper/Buffer.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/RouteType.h" #include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/GrpcConfig.h" #include +#include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/MessagePayload.h" %} +namespace ppc::sdk{ + class Transport; + class TransportBuilder; +} + +namespace ppc::gateway{ + class IGateway; +} + +namespace ppc::protocol{ + class Message; + class MessageHeader; + class MessagePayload; + class MessageOptionalHeader; + class MessageBuilder; + class MessageHeaderBuilder; + class MessagePayloadBuilder; + class MessageOptionalHeaderBuilder; + class EndPoint; + class GrpcConfig; + class RouteType; +} + namespace ppc::front{ class FrontConfig; class IFront; + class IFrontClient; class FrontImpl; class FrontBuilderImpl; - class RouteType; class GatewayEndPoint; + class ErrorCallback; + class MessageDispatcherHandler; + class IMessageHandler; + class SendResponseHandler; } -namespace ppc::sdk{ - class Transport; - class TransportBuilder; +namespace std{ + class vector; } -%template(SharedFrontConfig) std::shared_ptr; +namespace bcos{ + using byte = uint8_t; + using bytes = std::vector; + class Error; +} + +// define shared_ptr objects %template(SharedBcosError) std::shared_ptr; + +%template(SharedFrontConfig) std::shared_ptr; +%template(SharedGrpcConfig) std::shared_ptr; + +%template(SharedFront) std::shared_ptr; +%template(SharedFrontClient) std::shared_ptr; + +%template(SharedErrorCallback) std::shared_ptr; +%template(SharedMessageDispatcherHandler) std::shared_ptr; +%template(SharedIMessageHandler) std::shared_ptr; + +%template(SharedGateway) std::shared_ptr; + %template(SharedMessage) std::shared_ptr; +%template(SharedMessageHeader) std::shared_ptr; +%template(SharedMessagePayload) std::shared_ptr; %template(SharedRouteInfo) std::shared_ptr; -%template(SharedTransport) std::shared_ptr; + +%template(SharedMessageBuilder) std::shared_ptr; +%template(SharedMessageHeaderBuilder) std::shared_ptr; +%template(SharedMessagePayloadBuilder) std::shared_ptr; +%template(SharedRouteInfoBuilder) std::shared_ptr; + +%template(ubytes) std::vector; +%template(ibytes) std::vector; + +/// callbacks +%feature("director") ppc::front::ErrorCallback; +%feature("director") ppc::front::MessageDispatcherHandler; +%feature("director") ppc::front::IMessageHandler; + +// Note: the field data should equal to the fieldMap of class or the function +%include various.i +// this means convert all (char*) to byte[] +%apply char *BYTE {char * }; + + +%typemap(jni) OutputBuffer "jbyteArray" +%typemap(jtype) OutputBuffer "byte[]" +%typemap(jstype) OutputBuffer "byte[]" +%typemap(in) OutputBuffer { + $1.data = (unsigned char *) JCALL2(GetByteArrayElements, jenv, $input, 0); + $1.len = JCALL1(GetArrayLength, jenv, $input); +} +%typemap(argout) OutputBuffer { + JCALL3(ReleaseByteArrayElements, jenv, $input, (jbyte *) $1.data, 0); +} +// Note: will cause copy herer +%typemap(out) OutputBuffer { + $result = JCALL1(NewByteArray, jenv, $1.len); + JCALL4(SetByteArrayRegion, jenv, $result, 0, $1.len, (jbyte *) $1.data); + delete[] $1.data; +} +%typemap(javain) OutputBuffer "$javainput" +%typemap(javaout) OutputBuffer { return $jnicall; } + +/* +///// tests /// +%inline { +} +///// tests /// +*/ + +// define the interface should been exposed +%include "bcos-utilities/Error.h" +%include "ppc-framework/libwrapper/Buffer.h" +%include "ppc-framework/front/FrontConfig.h" +%include "ppc-framework/protocol/EndPoint.h" +%include "ppc-framework/protocol/GrpcConfig.h" +%include "ppc-framework/protocol/Message.h" +%include "ppc-framework/protocol/MessagePayload.h" + +%include "ppc-framework/front/IFront.h" %include "wedpr-transport/sdk/TransportBuilder.h" %include "wedpr-transport/sdk/Transport.h" -%include "ppc-framework/front/IFront.h" \ No newline at end of file + diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp index 3763e339..b2a50563 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.cpp @@ -19,7 +19,6 @@ */ #include "ProTransportImpl.h" #include "Common.h" -#include "protocol/src/v1/MessageImpl.h" #include "wedpr-protocol/grpc/client/GatewayClient.h" #include "wedpr-protocol/grpc/server/FrontServer.h" #include "wedpr-protocol/grpc/server/GrpcServer.h" @@ -30,7 +29,7 @@ using namespace ppc::sdk; ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keepAlivePeriodMs) - : m_config(std::move(config)), m_keepAlivePeriodMs(keepAlivePeriodMs) + : Transport(config), m_keepAlivePeriodMs(keepAlivePeriodMs) { // Note: since the config has been moved away, should not use the `config`, use `m_config` // instead default enable health-check @@ -40,13 +39,10 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keep FrontFactory frontFactory; m_gateway = std::make_shared(m_config->grpcConfig(), m_config->gatewayGrpcTarget()); - m_front = frontFactory.build(std::make_shared(), - std::make_shared(), - std::make_shared(), m_gateway, m_config); + m_front = frontFactory.build(std::make_shared(), m_msgPayloadBuilder, + m_routeInfoBuilder, m_gateway, m_config); - auto msgBuilder = - std::make_shared(std::make_shared()); - m_frontService = std::make_shared(msgBuilder, m_front); + m_frontService = std::make_shared(m_msgBuilder, m_front); // register the frontService m_server->registerService(m_frontService); } diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/ProTransportImpl.h index 76e2ce3a..cb5d4873 100644 --- a/cpp/wedpr-transport/sdk/ProTransportImpl.h +++ b/cpp/wedpr-transport/sdk/ProTransportImpl.h @@ -20,6 +20,7 @@ #pragma once #include "TransportImpl.h" #include "bcos-utilities/Timer.h" +#include "protocol/src/v1/MessageImpl.h" namespace ppc::protocol { @@ -43,7 +44,6 @@ class ProTransportImpl : public Transport, public std::enable_shared_from_this

m_server; std::shared_ptr m_frontService; int m_keepAlivePeriodMs; diff --git a/cpp/wedpr-transport/sdk/Transport.cpp b/cpp/wedpr-transport/sdk/Transport.cpp new file mode 100644 index 00000000..ea7b112f --- /dev/null +++ b/cpp/wedpr-transport/sdk/Transport.cpp @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2023 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file Transport.cpp + * @author: yujiechen + * @date 2024-09-04 + */ +#include "Transport.h" +#include "protocol/src/v1/MessageHeaderImpl.h" +#include "protocol/src/v1/MessageImpl.h" +#include "protocol/src/v1/MessagePayloadImpl.h" + +using namespace ppc::sdk; + +Transport::Transport(ppc::front::FrontConfig::Ptr config) + : m_config(std::move(config)), + m_msgPayloadBuilder(std::make_shared()), + m_msgHeaderBuilder(std::make_shared()), + m_msgBuilder(std::make_shared(m_msgHeaderBuilder)), + m_routeInfoBuilder(std::make_shared()) +{} diff --git a/cpp/wedpr-transport/sdk/Transport.h b/cpp/wedpr-transport/sdk/Transport.h index c9c36148..c6175c91 100644 --- a/cpp/wedpr-transport/sdk/Transport.h +++ b/cpp/wedpr-transport/sdk/Transport.h @@ -18,15 +18,19 @@ * @date 2024-09-04 */ #pragma once +#include "ppc-framework/front/FrontConfig.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/gateway/IGateway.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/MessagePayload.h" + namespace ppc::sdk { class Transport { public: using Ptr = std::shared_ptr; - Transport() = default; + Transport(ppc::front::FrontConfig::Ptr config); virtual ~Transport() = default; virtual void start() { m_front->start(); } @@ -36,8 +40,32 @@ class Transport virtual ppc::gateway::IGateway::Ptr const& gateway() const { return m_gateway; } + virtual ppc::protocol::MessagePayloadBuilder::Ptr const& msgPayloadBuilder() const + { + return m_msgPayloadBuilder; + } + virtual ppc::protocol::MessageBuilder::Ptr const& msgBuilder() const { return m_msgBuilder; } + virtual ppc::protocol::MessageHeaderBuilder::Ptr const& msgHeaderBuilder() const + { + return m_msgHeaderBuilder; + } + + virtual ppc::protocol::MessageOptionalHeaderBuilder::Ptr const& routeInfoBuilder() const + { + return m_routeInfoBuilder; + } + + virtual ppc::front::FrontConfig::Ptr const& getConfig() const { return m_config; } + protected: ppc::front::IFront::Ptr m_front; ppc::gateway::IGateway::Ptr m_gateway; + + ppc::front::FrontConfig::Ptr m_config; + + ppc::protocol::MessageHeaderBuilder::Ptr m_msgHeaderBuilder; + ppc::protocol::MessagePayloadBuilder::Ptr m_msgPayloadBuilder; + ppc::protocol::MessageOptionalHeaderBuilder::Ptr m_routeInfoBuilder; + ppc::protocol::MessageBuilder::Ptr m_msgBuilder; }; } // namespace ppc::sdk \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/TransportBuilder.cpp index cec1a6aa..f0cc6a5b 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.cpp +++ b/cpp/wedpr-transport/sdk/TransportBuilder.cpp @@ -36,7 +36,7 @@ TransportBuilder::TransportBuilder() } Transport::Ptr TransportBuilder::build( - SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) + SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr gateway) { switch (mode) { diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/TransportBuilder.h index cd6c3c56..d8ffa95b 100644 --- a/cpp/wedpr-transport/sdk/TransportBuilder.h +++ b/cpp/wedpr-transport/sdk/TransportBuilder.h @@ -36,8 +36,14 @@ class TransportBuilder TransportBuilder(); virtual ~TransportBuilder() = default; - Transport::Ptr build(SDKMode mode, ppc::front::FrontConfig::Ptr config, - ppc::gateway::IGateway::Ptr const& gateway); + Transport::Ptr build( + SDKMode mode, ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr gateway); + + // Note: the swig-wrapper can't receive the null shared_ptr + Transport::Ptr buildProTransport(ppc::front::FrontConfig::Ptr config) + { + return build(SDKMode::PRO, config, nullptr); + } ppc::front::FrontConfig::Ptr buildConfig(int threadPoolSize, std::string nodeID); diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/TransportImpl.h index fbd9e3e6..5ebda37b 100644 --- a/cpp/wedpr-transport/sdk/TransportImpl.h +++ b/cpp/wedpr-transport/sdk/TransportImpl.h @@ -22,8 +22,7 @@ #include "ppc-framework/gateway/IGateway.h" #include "ppc-front/FrontFactory.h" #include "protobuf/src/NodeInfoImpl.h" -#include "protocol/src/v1/MessageHeaderImpl.h" -#include "protocol/src/v1/MessagePayloadImpl.h" + namespace ppc::sdk { @@ -31,17 +30,15 @@ class TransportImpl : public Transport { public: TransportImpl(ppc::front::FrontConfig::Ptr config, ppc::gateway::IGateway::Ptr const& gateway) - : m_config(std::move(config)) + : Transport(config) + { m_gateway = gateway; + ppc::front::FrontFactory frontFactory; m_front = frontFactory.build(std::make_shared(), - std::make_shared(), - std::make_shared(), gateway, m_config); + m_msgPayloadBuilder, m_routeInfoBuilder, gateway, m_config); } ~TransportImpl() override = default; - -protected: - ppc::front::FrontConfig::Ptr m_config; }; } // namespace ppc::sdk From 31da0c1b5a7a7ba54f86c2beaf500c216d7cac6c Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 13 Sep 2024 11:02:42 +0800 Subject: [PATCH 036/120] add swig generated code (#33) --- .../wedpr/sdk/jni/generated/EndPoint.java | 91 ++ .../webank/wedpr/sdk/jni/generated/Error.java | 100 +++ .../sdk/jni/generated/ErrorCallback.java | 66 ++ .../wedpr/sdk/jni/generated/FrontConfig.java | 115 +++ .../sdk/jni/generated/FrontConfigBuilder.java | 62 ++ .../sdk/jni/generated/FrontMsgExtFlag.java | 54 ++ .../wedpr/sdk/jni/generated/GrpcConfig.java | 70 ++ .../sdk/jni/generated/GrpcServerConfig.java | 89 ++ .../wedpr/sdk/jni/generated/IFront.java | 300 +++++++ .../sdk/jni/generated/IFrontBuilder.java | 64 ++ .../wedpr/sdk/jni/generated/IFrontClient.java | 60 ++ .../sdk/jni/generated/IMessageHandler.java | 74 ++ .../wedpr/sdk/jni/generated/InputBuffer.java | 72 ++ .../wedpr/sdk/jni/generated/Message.java | 136 +++ .../sdk/jni/generated/MessageBuilder.java | 70 ++ .../generated/MessageDispatcherHandler.java | 68 ++ .../sdk/jni/generated/MessageHeader.java | 153 ++++ .../jni/generated/MessageHeaderBuilder.java | 64 ++ .../jni/generated/MessageOptionalHeader.java | 129 +++ .../MessageOptionalHeaderBuilder.java | 58 ++ .../sdk/jni/generated/MessagePayload.java | 109 +++ .../jni/generated/MessagePayloadBuilder.java | 54 ++ .../wedpr/sdk/jni/generated/OutputBuffer.java | 72 ++ .../wedpr/sdk/jni/generated/SDKMode.java | 53 ++ ...SWIGTYPE_p_bcos__boostssl__EncodedMsg.java | 30 + ...PE_p_bcos__boostssl__MessageFace__Ptr.java | 30 + .../SWIGTYPE_p_bcos__bytesConstRef.java | 30 + .../SWIGTYPE_p_ppc__gateway__IGateway.java | 30 + ...WIGTYPE_p_ppc__gateway__IGateway__Ptr.java | 30 + ...GTYPE_p_ppc__protocol__INodeInfo__Ptr.java | 30 + .../SWIGTYPE_p_ppc__protocol__RouteType.java | 30 + .../generated/SWIGTYPE_p_std__exception.java | 29 + .../SWIGTYPE_p_std__functionT_void_fF_t.java | 30 + ...id_fstd__shared_ptrT_bcos__Error_tF_t.java | 32 + ...s__Error_t_std__setT_std__string_tF_t.java | 37 + ...d__shared_ptrT_bcos__bytes_t_RRF_tF_t.java | 37 + ...fstd__shared_ptrT_bcos__bytes_t_RRF_t.java | 32 + ...ared_ptrT_ppc__protocol__Message_tF_t.java | 32 + ...TYPE_p_std__shared_ptrT_bcos__bytes_t.java | 30 + ...ptrT_ppc__front__FrontConfigBuilder_t.java | 32 + .../jni/generated/SWIGTYPE_p_std__string.java | 29 + .../SWIGTYPE_p_std__string_view.java | 30 + ...SWIGTYPE_p_std__vectorT_std__string_t.java | 30 + .../generated/SWIGTYPE_p_unsigned_char.java | 29 + .../jni/generated/SendResponseHandler.java | 64 ++ .../sdk/jni/generated/SharedBcosError.java | 54 ++ .../jni/generated/SharedErrorCallback.java | 54 ++ .../wedpr/sdk/jni/generated/SharedFront.java | 54 ++ .../sdk/jni/generated/SharedFrontClient.java | 54 ++ .../sdk/jni/generated/SharedFrontConfig.java | 54 ++ .../sdk/jni/generated/SharedGateway.java | 54 ++ .../sdk/jni/generated/SharedGrpcConfig.java | 54 ++ .../jni/generated/SharedIMessageHandler.java | 54 ++ .../sdk/jni/generated/SharedMessage.java | 54 ++ .../jni/generated/SharedMessageBuilder.java | 54 ++ .../SharedMessageDispatcherHandler.java | 54 ++ .../jni/generated/SharedMessageHeader.java | 54 ++ .../generated/SharedMessageHeaderBuilder.java | 54 ++ .../jni/generated/SharedMessagePayload.java | 54 ++ .../SharedMessagePayloadBuilder.java | 54 ++ .../sdk/jni/generated/SharedRouteInfo.java | 54 ++ .../jni/generated/SharedRouteInfoBuilder.java | 54 ++ .../wedpr/sdk/jni/generated/Transport.java | 89 ++ .../sdk/jni/generated/TransportBuilder.java | 86 ++ .../wedpr/sdk/jni/generated/ibytes.java | 163 ++++ .../wedpr/sdk/jni/generated/ubytes.java | 163 ++++ .../jni/generated/wedpr_java_transport.java | 29 + .../generated/wedpr_java_transportJNI.java | 831 ++++++++++++++++++ 68 files changed, 5145 insertions(+) create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/EndPoint.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Error.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ErrorCallback.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfigBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontMsgExtFlag.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IMessageHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/InputBuffer.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/OutputBuffer.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SDKMode.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__EncodedMsg.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__bytesConstRef.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway__Ptr.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__RouteType.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__exception.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string_view.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_unsigned_char.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SendResponseHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedBcosError.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedErrorCallback.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFront.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontClient.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGateway.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGrpcConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedIMessageHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessage.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageDispatcherHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeader.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeaderBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayload.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayloadBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfo.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfoBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ibytes.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ubytes.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/EndPoint.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/EndPoint.java new file mode 100644 index 00000000..74228f26 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/EndPoint.java @@ -0,0 +1,91 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +/** the endpoint */ +public class EndPoint { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected EndPoint(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(EndPoint obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(EndPoint obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_EndPoint(swigCPtr); + } + swigCPtr = 0; + } + } + + public EndPoint() { + this(wedpr_java_transportJNI.new_EndPoint__SWIG_0(), true); + } + + public EndPoint(String host, int port) { + this(wedpr_java_transportJNI.new_EndPoint__SWIG_1(host, port), true); + } + + public String host() { + return wedpr_java_transportJNI.EndPoint_host(swigCPtr, this); + } + + public int port() { + return wedpr_java_transportJNI.EndPoint_port(swigCPtr, this); + } + + public void setHost(String host) { + wedpr_java_transportJNI.EndPoint_setHost(swigCPtr, this, host); + } + + public void setPort(int port) { + wedpr_java_transportJNI.EndPoint_setPort(swigCPtr, this, port); + } + + public void setListenIp(String listenIp) { + wedpr_java_transportJNI.EndPoint_setListenIp(swigCPtr, this, listenIp); + } + + public String entryPoint() { + return wedpr_java_transportJNI.EndPoint_entryPoint(swigCPtr, this); + } + + public String listenEndPoint() { + return wedpr_java_transportJNI.EndPoint_listenEndPoint(swigCPtr, this); + } + + public String listenIp() { + return wedpr_java_transportJNI.EndPoint_listenIp(swigCPtr, this); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Error.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Error.java new file mode 100644 index 00000000..f37f589e --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Error.java @@ -0,0 +1,100 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class Error { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected Error(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(Error obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_Error(swigCPtr); + } + swigCPtr = 0; + } + } + + public static Error buildError( + byte[] func, byte[] file, int line, int errorCode, String errorMessage) { + return new Error( + wedpr_java_transportJNI.Error_buildError__SWIG_0( + func, file, line, errorCode, errorMessage), + true); + } + + public static Error buildError( + byte[] func, byte[] file, int line, int errorCode, String errorMessage, Error prev) { + return new Error( + wedpr_java_transportJNI.Error_buildError__SWIG_1( + func, file, line, errorCode, errorMessage, Error.getCPtr(prev), prev), + true); + } + + public static Error buildError( + byte[] func, + byte[] file, + int line, + int errorCode, + String errorMessage, + SWIGTYPE_p_std__exception prev) { + return new Error( + wedpr_java_transportJNI.Error_buildError__SWIG_2( + func, + file, + line, + errorCode, + errorMessage, + SWIGTYPE_p_std__exception.getCPtr(prev)), + true); + } + + public Error() { + this(wedpr_java_transportJNI.new_Error__SWIG_0(), true); + } + + public Error(long _errorCode, String _errorMessage) { + this(wedpr_java_transportJNI.new_Error__SWIG_1(_errorCode, _errorMessage), true); + } + + public long errorCode() { + return wedpr_java_transportJNI.Error_errorCode(swigCPtr, this); + } + + public String errorMessage() { + return wedpr_java_transportJNI.Error_errorMessage(swigCPtr, this); + } + + public void setErrorCode(long _errorCode) { + wedpr_java_transportJNI.Error_setErrorCode(swigCPtr, this, _errorCode); + } + + public void setErrorMessage(String _errorMessage) { + wedpr_java_transportJNI.Error_setErrorMessage(swigCPtr, this, _errorMessage); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ErrorCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ErrorCallback.java new file mode 100644 index 00000000..3b58fc66 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ErrorCallback.java @@ -0,0 +1,66 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class ErrorCallback { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected ErrorCallback(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(ErrorCallback obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_ErrorCallback(swigCPtr); + } + swigCPtr = 0; + } + } + + protected void swigDirectorDisconnect() { + swigSetCMemOwn(false); + delete(); + } + + public void swigReleaseOwnership() { + swigSetCMemOwn(false); + wedpr_java_transportJNI.ErrorCallback_change_ownership(this, swigCPtr, false); + } + + public void swigTakeOwnership() { + swigSetCMemOwn(true); + wedpr_java_transportJNI.ErrorCallback_change_ownership(this, swigCPtr, true); + } + + public ErrorCallback() { + this(wedpr_java_transportJNI.new_ErrorCallback(), true); + wedpr_java_transportJNI.ErrorCallback_director_connect(this, swigCPtr, true, true); + } + + public void onError(Error error) { + wedpr_java_transportJNI.ErrorCallback_onError(swigCPtr, this, Error.getCPtr(error), error); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java new file mode 100644 index 00000000..c7de68ff --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java @@ -0,0 +1,115 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class FrontConfig { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected FrontConfig(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(FrontConfig obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_FrontConfig(swigCPtr); + } + swigCPtr = 0; + } + } + + public int threadPoolSize() { + return wedpr_java_transportJNI.FrontConfig_threadPoolSize(swigCPtr, this); + } + + public void setThreadPoolSize(int threadPoolSize) { + wedpr_java_transportJNI.FrontConfig_setThreadPoolSize(swigCPtr, this, threadPoolSize); + } + + public String nodeID() { + return wedpr_java_transportJNI.FrontConfig_nodeID(swigCPtr, this); + } + + public void setNodeID(String nodeID) { + wedpr_java_transportJNI.FrontConfig_setNodeID(swigCPtr, this, nodeID); + } + + public EndPoint selfEndPoint() { + return new EndPoint( + wedpr_java_transportJNI.FrontConfig_selfEndPoint(swigCPtr, this), false); + } + + public EndPoint mutableSelfEndPoint() { + return new EndPoint( + wedpr_java_transportJNI.FrontConfig_mutableSelfEndPoint(swigCPtr, this), false); + } + + public void setSelfEndPoint(EndPoint endPoint) { + wedpr_java_transportJNI.FrontConfig_setSelfEndPoint( + swigCPtr, this, EndPoint.getCPtr(endPoint), endPoint); + } + + public void setGatewayGrpcTarget(String gatewayGrpcTarget) { + wedpr_java_transportJNI.FrontConfig_setGatewayGrpcTarget(swigCPtr, this, gatewayGrpcTarget); + } + + public String gatewayGrpcTarget() { + return wedpr_java_transportJNI.FrontConfig_gatewayGrpcTarget(swigCPtr, this); + } + + public void setGrpcConfig(GrpcConfig grpcConfig) { + wedpr_java_transportJNI.FrontConfig_setGrpcConfig( + swigCPtr, this, GrpcConfig.getCPtr(grpcConfig), grpcConfig); + } + + public GrpcConfig grpcConfig() { + long cPtr = wedpr_java_transportJNI.FrontConfig_grpcConfig(swigCPtr, this); + return (cPtr == 0) ? null : new GrpcConfig(cPtr, true); + } + + public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr generateNodeInfo() { + return new SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr( + wedpr_java_transportJNI.FrontConfig_generateNodeInfo(swigCPtr, this), true); + } + + public SWIGTYPE_p_std__vectorT_std__string_t getComponents() { + return new SWIGTYPE_p_std__vectorT_std__string_t( + wedpr_java_transportJNI.FrontConfig_getComponents(swigCPtr, this), false); + } + + public void setComponents(SWIGTYPE_p_std__vectorT_std__string_t components) { + wedpr_java_transportJNI.FrontConfig_setComponents( + swigCPtr, this, SWIGTYPE_p_std__vectorT_std__string_t.getCPtr(components)); + } + + public void addComponent(String component) { + wedpr_java_transportJNI.FrontConfig_addComponent(swigCPtr, this, component); + } + + public SWIGTYPE_p_std__vectorT_std__string_t mutableComponents() { + return new SWIGTYPE_p_std__vectorT_std__string_t( + wedpr_java_transportJNI.FrontConfig_mutableComponents(swigCPtr, this), false); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfigBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfigBuilder.java new file mode 100644 index 00000000..bbdcb1a0 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfigBuilder.java @@ -0,0 +1,62 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class FrontConfigBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected FrontConfigBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(FrontConfigBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(FrontConfigBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_FrontConfigBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public FrontConfig build() { + long cPtr = wedpr_java_transportJNI.FrontConfigBuilder_build__SWIG_0(swigCPtr, this); + return (cPtr == 0) ? null : new FrontConfig(cPtr, true); + } + + public FrontConfig build(int threadPoolSize, String nodeID) { + long cPtr = + wedpr_java_transportJNI.FrontConfigBuilder_build__SWIG_1( + swigCPtr, this, threadPoolSize, nodeID); + return (cPtr == 0) ? null : new FrontConfig(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontMsgExtFlag.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontMsgExtFlag.java new file mode 100644 index 00000000..5cdc7443 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontMsgExtFlag.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public final class FrontMsgExtFlag { + public static final FrontMsgExtFlag Response = + new FrontMsgExtFlag("Response", wedpr_java_transportJNI.FrontMsgExtFlag_Response_get()); + + public final int swigValue() { + return swigValue; + } + + public String toString() { + return swigName; + } + + public static FrontMsgExtFlag swigToEnum(int swigValue) { + if (swigValue < swigValues.length + && swigValue >= 0 + && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; + for (int i = 0; i < swigValues.length; i++) + if (swigValues[i].swigValue == swigValue) return swigValues[i]; + throw new IllegalArgumentException( + "No enum " + FrontMsgExtFlag.class + " with value " + swigValue); + } + + private FrontMsgExtFlag(String swigName) { + this.swigName = swigName; + this.swigValue = swigNext++; + } + + private FrontMsgExtFlag(String swigName, int swigValue) { + this.swigName = swigName; + this.swigValue = swigValue; + swigNext = swigValue + 1; + } + + private FrontMsgExtFlag(String swigName, FrontMsgExtFlag swigEnum) { + this.swigName = swigName; + this.swigValue = swigEnum.swigValue; + swigNext = this.swigValue + 1; + } + + private static FrontMsgExtFlag[] swigValues = {Response}; + private static int swigNext = 0; + private final int swigValue; + private final String swigName; +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java new file mode 100644 index 00000000..35d8e79d --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java @@ -0,0 +1,70 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class GrpcConfig { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected GrpcConfig(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(GrpcConfig obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_GrpcConfig(swigCPtr); + } + swigCPtr = 0; + } + } + + public GrpcConfig() { + this(wedpr_java_transportJNI.new_GrpcConfig(), true); + } + + public String loadBalancePolicy() { + return wedpr_java_transportJNI.GrpcConfig_loadBalancePolicy(swigCPtr, this); + } + + public void setLoadBalancePolicy(String loadBalancePolicy) { + wedpr_java_transportJNI.GrpcConfig_setLoadBalancePolicy(swigCPtr, this, loadBalancePolicy); + } + + public boolean enableHealthCheck() { + return wedpr_java_transportJNI.GrpcConfig_enableHealthCheck(swigCPtr, this); + } + + public void setEnableHealthCheck(boolean enableHealthCheck) { + wedpr_java_transportJNI.GrpcConfig_setEnableHealthCheck(swigCPtr, this, enableHealthCheck); + } + + public void setEnableDnslookup(boolean enableDnslookup) { + wedpr_java_transportJNI.GrpcConfig_setEnableDnslookup(swigCPtr, this, enableDnslookup); + } + + public boolean enableDnslookup() { + return wedpr_java_transportJNI.GrpcConfig_enableDnslookup(swigCPtr, this); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java new file mode 100644 index 00000000..179abdc3 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java @@ -0,0 +1,89 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class GrpcServerConfig { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected GrpcServerConfig(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(GrpcServerConfig obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(GrpcServerConfig obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_GrpcServerConfig(swigCPtr); + } + swigCPtr = 0; + } + } + + public GrpcServerConfig() { + this(wedpr_java_transportJNI.new_GrpcServerConfig__SWIG_0(), true); + } + + public GrpcServerConfig(EndPoint endPoint, boolean enableHealthCheck) { + this( + wedpr_java_transportJNI.new_GrpcServerConfig__SWIG_1( + EndPoint.getCPtr(endPoint), endPoint, enableHealthCheck), + true); + } + + public String listenEndPoint() { + return wedpr_java_transportJNI.GrpcServerConfig_listenEndPoint(swigCPtr, this); + } + + public void setEndPoint(EndPoint endPoint) { + wedpr_java_transportJNI.GrpcServerConfig_setEndPoint( + swigCPtr, this, EndPoint.getCPtr(endPoint), endPoint); + } + + public void setEnableHealthCheck(boolean enableHealthCheck) { + wedpr_java_transportJNI.GrpcServerConfig_setEnableHealthCheck( + swigCPtr, this, enableHealthCheck); + } + + public EndPoint endPoint() { + return new EndPoint( + wedpr_java_transportJNI.GrpcServerConfig_endPoint(swigCPtr, this), false); + } + + public EndPoint mutableEndPoint() { + return new EndPoint( + wedpr_java_transportJNI.GrpcServerConfig_mutableEndPoint(swigCPtr, this), false); + } + + public boolean enableHealthCheck() { + return wedpr_java_transportJNI.GrpcServerConfig_enableHealthCheck(swigCPtr, this); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java new file mode 100644 index 00000000..75164877 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java @@ -0,0 +1,300 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class IFront extends IFrontClient { + private transient long swigCPtr; + private transient boolean swigCMemOwnDerived; + + protected IFront(long cPtr, boolean cMemoryOwn) { + super(wedpr_java_transportJNI.IFront_SWIGSmartPtrUpcast(cPtr), true); + swigCMemOwnDerived = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(IFront obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwnDerived = own; + super.swigSetCMemOwn(own); + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwnDerived) { + swigCMemOwnDerived = false; + wedpr_java_transportJNI.delete_IFront(swigCPtr); + } + swigCPtr = 0; + } + super.delete(); + } + + /** + * start the IFront
+ *
+ */ + public void start() { + wedpr_java_transportJNI.IFront_start(swigCPtr, this); + } + + /** + * stop the IFront
+ *
+ */ + public void stop() { + wedpr_java_transportJNI.IFront_stop(swigCPtr, this); + } + + /** + *
+ * + * @param topic the topic
+ * @param callback the callback called when receive specified topic + */ + public void registerTopicHandler( + String topic, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t callback) { + wedpr_java_transportJNI.IFront_registerTopicHandler__SWIG_0( + swigCPtr, + this, + topic, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t + .getCPtr(callback)); + } + + public void registerTopicHandler(String topic, MessageDispatcherHandler callback) { + wedpr_java_transportJNI.IFront_registerTopicHandler__SWIG_1( + swigCPtr, this, topic, MessageDispatcherHandler.getCPtr(callback), callback); + } + + public void registerMessageHandler( + String componentType, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t callback) { + wedpr_java_transportJNI.IFront_registerMessageHandler__SWIG_0( + swigCPtr, + this, + componentType, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t + .getCPtr(callback)); + } + + public void registerMessageHandler(String componentType, MessageDispatcherHandler callback) { + wedpr_java_transportJNI.IFront_registerMessageHandler__SWIG_1( + swigCPtr, + this, + componentType, + MessageDispatcherHandler.getCPtr(callback), + callback); + } + + /** + * async send message
+ *
+ * + * @param routeType the route type
+ * @param routeInfo the route info, include
+ * - topic the topic
+ * - dstInst the dst agency(must set when 'route by agency' and 'route by
+ * component')
+ * - dstNodeID the dst nodeID(must set when 'route by nodeID')
+ * - componentType the componentType(must set when 'route by component')
+ * @param payload the payload to send
+ * @param seq the message seq
+ * @param timeout timeout
+ * @param callback callback + */ + public void asyncSendMessage( + int routeType, + MessageOptionalHeader routeInfo, + ubytes payload, + int seq, + int timeout, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t errorCallback, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t + callback) { + wedpr_java_transportJNI.IFront_asyncSendMessage__SWIG_0( + swigCPtr, + this, + routeType, + MessageOptionalHeader.getCPtr(routeInfo), + routeInfo, + ubytes.swigRelease(payload), + payload, + seq, + timeout, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.getCPtr( + errorCallback), + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t + .getCPtr(callback)); + } + + public void asyncSendMessage( + int routeType, + MessageOptionalHeader routeInfo, + byte[] payload, + java.math.BigInteger payloadSize, + int seq, + int timeout, + ErrorCallback errorCallback, + IMessageHandler msgHandler) { + wedpr_java_transportJNI.IFront_asyncSendMessage__SWIG_1( + swigCPtr, + this, + routeType, + MessageOptionalHeader.getCPtr(routeInfo), + routeInfo, + payload, + payloadSize, + seq, + timeout, + ErrorCallback.getCPtr(errorCallback), + errorCallback, + IMessageHandler.getCPtr(msgHandler), + msgHandler); + } + + public void asyncSendResponse( + ubytes dstNode, + String traceID, + ubytes payload, + int seq, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t errorCallback) { + wedpr_java_transportJNI.IFront_asyncSendResponse__SWIG_0( + swigCPtr, + this, + ubytes.getCPtr(dstNode), + dstNode, + traceID, + ubytes.swigRelease(payload), + payload, + seq, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.getCPtr( + errorCallback)); + } + + public void asyncSendResponse( + ubytes dstNode, String traceID, ubytes payload, int seq, ErrorCallback errorCallback) { + wedpr_java_transportJNI.IFront_asyncSendResponse__SWIG_1( + swigCPtr, + this, + ubytes.getCPtr(dstNode), + dstNode, + traceID, + ubytes.swigRelease(payload), + payload, + seq, + ErrorCallback.getCPtr(errorCallback), + errorCallback); + } + + public Error push( + int routeType, MessageOptionalHeader routeInfo, ubytes payload, int seq, int timeout) { + long cPtr = + wedpr_java_transportJNI.IFront_push__SWIG_0( + swigCPtr, + this, + routeType, + MessageOptionalHeader.getCPtr(routeInfo), + routeInfo, + ubytes.swigRelease(payload), + payload, + seq, + timeout); + return (cPtr == 0) ? null : new Error(cPtr, true); + } + + public Error push( + int routeType, + MessageOptionalHeader routeInfo, + byte[] payload, + java.math.BigInteger payloadSize, + int seq, + int timeout) { + long cPtr = + wedpr_java_transportJNI.IFront_push__SWIG_1( + swigCPtr, + this, + routeType, + MessageOptionalHeader.getCPtr(routeInfo), + routeInfo, + payload, + payloadSize, + seq, + timeout); + return (cPtr == 0) ? null : new Error(cPtr, true); + } + + public Message pop(String topic, int timeoutMs) { + long cPtr = wedpr_java_transportJNI.IFront_pop(swigCPtr, this, topic, timeoutMs); + return (cPtr == 0) ? null : new Message(cPtr, true); + } + + public Message peek(String topic) { + long cPtr = wedpr_java_transportJNI.IFront_peek(swigCPtr, this, topic); + return (cPtr == 0) ? null : new Message(cPtr, true); + } + + public void asyncGetAgencies( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t + callback) { + wedpr_java_transportJNI.IFront_asyncGetAgencies( + swigCPtr, + this, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t + .getCPtr(callback)); + } + + /** + * register the nodeInfo to the gateway
+ * + * @param nodeInfo the nodeInfo + */ + public Error registerNodeInfo(SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr nodeInfo) { + long cPtr = + wedpr_java_transportJNI.IFront_registerNodeInfo( + swigCPtr, this, SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.getCPtr(nodeInfo)); + return (cPtr == 0) ? null : new Error(cPtr, true); + } + + /** unRegister the nodeInfo to the gateway */ + public Error unRegisterNodeInfo() { + long cPtr = wedpr_java_transportJNI.IFront_unRegisterNodeInfo(swigCPtr, this); + return (cPtr == 0) ? null : new Error(cPtr, true); + } + + /** + * register the topic
+ *
+ * + * @param topic the topic to register + */ + public Error registerTopic(String topic) { + long cPtr = wedpr_java_transportJNI.IFront_registerTopic(swigCPtr, this, topic); + return (cPtr == 0) ? null : new Error(cPtr, true); + } + + /** + * unRegister the topic
+ *
+ * + * @param topic the topic to unregister + */ + public Error unRegisterTopic(String topic) { + long cPtr = wedpr_java_transportJNI.IFront_unRegisterTopic(swigCPtr, this, topic); + return (cPtr == 0) ? null : new Error(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontBuilder.java new file mode 100644 index 00000000..23e0fdab --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontBuilder.java @@ -0,0 +1,64 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class IFrontBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected IFrontBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(IFrontBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(IFrontBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_IFrontBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public IFrontClient buildClient( + String endPoint, + SWIGTYPE_p_std__functionT_void_fF_t onUnHealthHandler, + boolean removeHandlerOnUnhealth) { + long cPtr = + wedpr_java_transportJNI.IFrontBuilder_buildClient( + swigCPtr, + this, + endPoint, + SWIGTYPE_p_std__functionT_void_fF_t.getCPtr(onUnHealthHandler), + removeHandlerOnUnhealth); + return (cPtr == 0) ? null : new IFrontClient(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java new file mode 100644 index 00000000..ac777ec8 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java @@ -0,0 +1,60 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class IFrontClient { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected IFrontClient(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(IFrontClient obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_IFrontClient(swigCPtr); + } + swigCPtr = 0; + } + } + + /** + * : receive message from gateway, call by gateway
+ *
+ * + * @return void + */ + public void onReceiveMessage( + Message _msg, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t _callback) { + wedpr_java_transportJNI.IFrontClient_onReceiveMessage( + swigCPtr, + this, + Message.getCPtr(_msg), + _msg, + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.getCPtr( + _callback)); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IMessageHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IMessageHandler.java new file mode 100644 index 00000000..260ce3a7 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IMessageHandler.java @@ -0,0 +1,74 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class IMessageHandler { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected IMessageHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(IMessageHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_IMessageHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + protected void swigDirectorDisconnect() { + swigSetCMemOwn(false); + delete(); + } + + public void swigReleaseOwnership() { + swigSetCMemOwn(false); + wedpr_java_transportJNI.IMessageHandler_change_ownership(this, swigCPtr, false); + } + + public void swigTakeOwnership() { + swigSetCMemOwn(true); + wedpr_java_transportJNI.IMessageHandler_change_ownership(this, swigCPtr, true); + } + + public IMessageHandler() { + this(wedpr_java_transportJNI.new_IMessageHandler(), true); + wedpr_java_transportJNI.IMessageHandler_director_connect(this, swigCPtr, true, true); + } + + public void onMessage(Error e, Message msg, SendResponseHandler sendResponseHandler) { + wedpr_java_transportJNI.IMessageHandler_onMessage( + swigCPtr, + this, + Error.getCPtr(e), + e, + Message.getCPtr(msg), + msg, + SendResponseHandler.getCPtr(sendResponseHandler), + sendResponseHandler); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/InputBuffer.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/InputBuffer.java new file mode 100644 index 00000000..d07438aa --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/InputBuffer.java @@ -0,0 +1,72 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class InputBuffer { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected InputBuffer(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(InputBuffer obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(InputBuffer obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_InputBuffer(swigCPtr); + } + swigCPtr = 0; + } + } + + public void setData(SWIGTYPE_p_unsigned_char value) { + wedpr_java_transportJNI.InputBuffer_data_set( + swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); + } + + public SWIGTYPE_p_unsigned_char getData() { + long cPtr = wedpr_java_transportJNI.InputBuffer_data_get(swigCPtr, this); + return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); + } + + public void setLen(java.math.BigInteger value) { + wedpr_java_transportJNI.InputBuffer_len_set(swigCPtr, this, value); + } + + public java.math.BigInteger getLen() { + return wedpr_java_transportJNI.InputBuffer_len_get(swigCPtr, this); + } + + public InputBuffer() { + this(wedpr_java_transportJNI.new_InputBuffer(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java new file mode 100644 index 00000000..68091c7c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java @@ -0,0 +1,136 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class Message { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected Message(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(Message obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_Message(swigCPtr); + } + swigCPtr = 0; + } + } + + public MessageHeader header() { + long cPtr = wedpr_java_transportJNI.Message_header(swigCPtr, this); + return (cPtr == 0) ? null : new MessageHeader(cPtr, true); + } + + public void setHeader(MessageHeader header) { + wedpr_java_transportJNI.Message_setHeader( + swigCPtr, this, MessageHeader.getCPtr(header), header); + } + + /** the overloaed implementation === */ + public int version() { + return wedpr_java_transportJNI.Message_version(swigCPtr, this); + } + + public void setVersion(int version) { + wedpr_java_transportJNI.Message_setVersion(swigCPtr, this, version); + } + + public int packetType() { + return wedpr_java_transportJNI.Message_packetType(swigCPtr, this); + } + + public void setPacketType(int packetType) { + wedpr_java_transportJNI.Message_setPacketType(swigCPtr, this, packetType); + } + + public String seq() { + return wedpr_java_transportJNI.Message_seq(swigCPtr, this); + } + + public void setSeq(String traceID) { + wedpr_java_transportJNI.Message_setSeq(swigCPtr, this, traceID); + } + + public int ext() { + return wedpr_java_transportJNI.Message_ext(swigCPtr, this); + } + + public void setExt(int ext) { + wedpr_java_transportJNI.Message_setExt(swigCPtr, this, ext); + } + + public boolean isRespPacket() { + return wedpr_java_transportJNI.Message_isRespPacket(swigCPtr, this); + } + + public void setRespPacket() { + wedpr_java_transportJNI.Message_setRespPacket(swigCPtr, this); + } + + public long length() { + return wedpr_java_transportJNI.Message_length(swigCPtr, this); + } + + public ubytes payload() { + long cPtr = wedpr_java_transportJNI.Message_payload(swigCPtr, this); + return (cPtr == 0) ? null : new ubytes(cPtr, true); + } + + public byte[] payloadBuffer() { + return wedpr_java_transportJNI.Message_payloadBuffer(swigCPtr, this); + } + + public void setPayload(ubytes _payload) { + wedpr_java_transportJNI.Message_setPayload( + swigCPtr, this, ubytes.getCPtr(_payload), _payload); + } + + public void setFrontMessage(MessagePayload frontMessage) { + wedpr_java_transportJNI.Message_setFrontMessage( + swigCPtr, this, MessagePayload.getCPtr(frontMessage), frontMessage); + } + + public MessagePayload frontMessage() { + long cPtr = wedpr_java_transportJNI.Message_frontMessage(swigCPtr, this); + return (cPtr == 0) ? null : new MessagePayload(cPtr, true); + } + + public boolean encode(ubytes _buffer) { + return wedpr_java_transportJNI.Message_encode__SWIG_0( + swigCPtr, this, ubytes.getCPtr(_buffer), _buffer); + } + + public boolean encode(SWIGTYPE_p_bcos__boostssl__EncodedMsg _encodedMsg) { + return wedpr_java_transportJNI.Message_encode__SWIG_1( + swigCPtr, this, SWIGTYPE_p_bcos__boostssl__EncodedMsg.getCPtr(_encodedMsg)); + } + + public long decode(SWIGTYPE_p_bcos__bytesConstRef _buffer) { + return wedpr_java_transportJNI.Message_decode( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(_buffer)); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java new file mode 100644 index 00000000..769d0dae --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java @@ -0,0 +1,70 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageBuilder { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public Message build() { + long cPtr = wedpr_java_transportJNI.MessageBuilder_build__SWIG_0(swigCPtr, this); + return (cPtr == 0) ? null : new Message(cPtr, true); + } + + public Message build(SWIGTYPE_p_bcos__bytesConstRef buffer) { + long cPtr = + wedpr_java_transportJNI.MessageBuilder_build__SWIG_1( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(buffer)); + return (cPtr == 0) ? null : new Message(cPtr, true); + } + + public Message build( + SWIGTYPE_p_ppc__protocol__RouteType routeType, + MessageOptionalHeader routeInfo, + ubytes payload) { + long cPtr = + wedpr_java_transportJNI.MessageBuilder_build__SWIG_2( + swigCPtr, + this, + SWIGTYPE_p_ppc__protocol__RouteType.getCPtr(routeType), + MessageOptionalHeader.getCPtr(routeInfo), + routeInfo, + ubytes.swigRelease(payload), + payload); + return (cPtr == 0) ? null : new Message(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler.java new file mode 100644 index 00000000..7a0ee807 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler.java @@ -0,0 +1,68 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageDispatcherHandler { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageDispatcherHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageDispatcherHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageDispatcherHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + protected void swigDirectorDisconnect() { + swigSetCMemOwn(false); + delete(); + } + + public void swigReleaseOwnership() { + swigSetCMemOwn(false); + wedpr_java_transportJNI.MessageDispatcherHandler_change_ownership(this, swigCPtr, false); + } + + public void swigTakeOwnership() { + swigSetCMemOwn(true); + wedpr_java_transportJNI.MessageDispatcherHandler_change_ownership(this, swigCPtr, true); + } + + public MessageDispatcherHandler() { + this(wedpr_java_transportJNI.new_MessageDispatcherHandler(), true); + wedpr_java_transportJNI.MessageDispatcherHandler_director_connect( + this, swigCPtr, true, true); + } + + public void onMessage(Message msg) { + wedpr_java_transportJNI.MessageDispatcherHandler_onMessage( + swigCPtr, this, Message.getCPtr(msg), msg); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java new file mode 100644 index 00000000..f8c215a4 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java @@ -0,0 +1,153 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageHeader { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageHeader(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageHeader obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageHeader(swigCPtr); + } + swigCPtr = 0; + } + } + + public void encode(ubytes buffer) { + wedpr_java_transportJNI.MessageHeader_encode( + swigCPtr, this, ubytes.getCPtr(buffer), buffer); + } + + public long decode(SWIGTYPE_p_bcos__bytesConstRef data) { + return wedpr_java_transportJNI.MessageHeader_decode( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(data)); + } + + public short version() { + return wedpr_java_transportJNI.MessageHeader_version(swigCPtr, this); + } + + public void setVersion(int version) { + wedpr_java_transportJNI.MessageHeader_setVersion(swigCPtr, this, version); + } + + public String traceID() { + return wedpr_java_transportJNI.MessageHeader_traceID(swigCPtr, this); + } + + public void setTraceID(String traceID) { + wedpr_java_transportJNI.MessageHeader_setTraceID(swigCPtr, this, traceID); + } + + public String srcGwNode() { + return wedpr_java_transportJNI.MessageHeader_srcGwNode(swigCPtr, this); + } + + public void setSrcGwNode(String srcGwNode) { + wedpr_java_transportJNI.MessageHeader_setSrcGwNode(swigCPtr, this, srcGwNode); + } + + public String dstGwNode() { + return wedpr_java_transportJNI.MessageHeader_dstGwNode(swigCPtr, this); + } + + public void setDstGwNode(String dstGwNode) { + wedpr_java_transportJNI.MessageHeader_setDstGwNode(swigCPtr, this, dstGwNode); + } + + public int packetType() { + return wedpr_java_transportJNI.MessageHeader_packetType(swigCPtr, this); + } + + public void setPacketType(int packetType) { + wedpr_java_transportJNI.MessageHeader_setPacketType(swigCPtr, this, packetType); + } + + public short ttl() { + return wedpr_java_transportJNI.MessageHeader_ttl(swigCPtr, this); + } + + public void setTTL(int ttl) { + wedpr_java_transportJNI.MessageHeader_setTTL(swigCPtr, this, ttl); + } + + public int ext() { + return wedpr_java_transportJNI.MessageHeader_ext(swigCPtr, this); + } + + public void setExt(int ext) { + wedpr_java_transportJNI.MessageHeader_setExt(swigCPtr, this, ext); + } + + public MessageOptionalHeader optionalField() { + long cPtr = wedpr_java_transportJNI.MessageHeader_optionalField(swigCPtr, this); + return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); + } + + public void setOptionalField(MessageOptionalHeader optionalField) { + wedpr_java_transportJNI.MessageHeader_setOptionalField( + swigCPtr, this, MessageOptionalHeader.getCPtr(optionalField), optionalField); + } + + public int length() { + return wedpr_java_transportJNI.MessageHeader_length(swigCPtr, this); + } + + public boolean isRespPacket() { + return wedpr_java_transportJNI.MessageHeader_isRespPacket(swigCPtr, this); + } + + public void setRespPacket() { + wedpr_java_transportJNI.MessageHeader_setRespPacket(swigCPtr, this); + } + + public SWIGTYPE_p_std__string_view srcP2PNodeIDView() { + return new SWIGTYPE_p_std__string_view( + wedpr_java_transportJNI.MessageHeader_srcP2PNodeIDView(swigCPtr, this), true); + } + + public SWIGTYPE_p_std__string_view dstP2PNodeIDView() { + return new SWIGTYPE_p_std__string_view( + wedpr_java_transportJNI.MessageHeader_dstP2PNodeIDView(swigCPtr, this), true); + } + + public int routeType() { + return wedpr_java_transportJNI.MessageHeader_routeType(swigCPtr, this); + } + + public void setRouteType(SWIGTYPE_p_ppc__protocol__RouteType type) { + wedpr_java_transportJNI.MessageHeader_setRouteType( + swigCPtr, this, SWIGTYPE_p_ppc__protocol__RouteType.getCPtr(type)); + } + + public boolean hasOptionalField() { + return wedpr_java_transportJNI.MessageHeader_hasOptionalField(swigCPtr, this); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java new file mode 100644 index 00000000..4b3db731 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java @@ -0,0 +1,64 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageHeaderBuilder { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageHeaderBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageHeaderBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageHeaderBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public MessageHeader build(SWIGTYPE_p_bcos__bytesConstRef _data) { + long cPtr = + wedpr_java_transportJNI.MessageHeaderBuilder_build__SWIG_0( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(_data)); + return (cPtr == 0) ? null : new MessageHeader(cPtr, true); + } + + public MessageHeader build() { + long cPtr = wedpr_java_transportJNI.MessageHeaderBuilder_build__SWIG_1(swigCPtr, this); + return (cPtr == 0) ? null : new MessageHeader(cPtr, true); + } + + public MessageOptionalHeader build(MessageOptionalHeader optionalHeader) { + long cPtr = + wedpr_java_transportJNI.MessageHeaderBuilder_build__SWIG_2( + swigCPtr, + this, + MessageOptionalHeader.getCPtr(optionalHeader), + optionalHeader); + return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java new file mode 100644 index 00000000..8ad6a360 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java @@ -0,0 +1,129 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageOptionalHeader { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageOptionalHeader(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageOptionalHeader obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageOptionalHeader(swigCPtr); + } + swigCPtr = 0; + } + } + + public void encode(ubytes buffer) { + wedpr_java_transportJNI.MessageOptionalHeader_encode( + swigCPtr, this, ubytes.getCPtr(buffer), buffer); + } + + public long decode(SWIGTYPE_p_bcos__bytesConstRef data, java.math.BigInteger _offset) { + return wedpr_java_transportJNI.MessageOptionalHeader_decode( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(data), _offset); + } + + public String componentType() { + return wedpr_java_transportJNI.MessageOptionalHeader_componentType(swigCPtr, this); + } + + public void setComponentType(String componentType) { + wedpr_java_transportJNI.MessageOptionalHeader_setComponentType( + swigCPtr, this, componentType); + } + + public ubytes srcNode() { + return new ubytes( + wedpr_java_transportJNI.MessageOptionalHeader_srcNode(swigCPtr, this), true); + } + + /** for swig-wrapper(pass the binary data) */ + public byte[] srcNodeBuffer() { + return wedpr_java_transportJNI.MessageOptionalHeader_srcNodeBuffer(swigCPtr, this); + } + + public void setSrcNode(ubytes srcNode) { + wedpr_java_transportJNI.MessageOptionalHeader_setSrcNode__SWIG_0( + swigCPtr, this, ubytes.getCPtr(srcNode), srcNode); + } + + public void setSrcNode(byte[] data, java.math.BigInteger length) { + wedpr_java_transportJNI.MessageOptionalHeader_setSrcNode__SWIG_1( + swigCPtr, this, data, length); + } + + public ubytes dstNode() { + return new ubytes( + wedpr_java_transportJNI.MessageOptionalHeader_dstNode(swigCPtr, this), true); + } + + public byte[] dstNodeBuffer() { + return wedpr_java_transportJNI.MessageOptionalHeader_dstNodeBuffer(swigCPtr, this); + } + + public void setDstNode(ubytes dstNode) { + wedpr_java_transportJNI.MessageOptionalHeader_setDstNode__SWIG_0( + swigCPtr, this, ubytes.getCPtr(dstNode), dstNode); + } + + public void setDstNode(byte[] data, java.math.BigInteger length) { + wedpr_java_transportJNI.MessageOptionalHeader_setDstNode__SWIG_1( + swigCPtr, this, data, length); + } + + public String dstInst() { + return wedpr_java_transportJNI.MessageOptionalHeader_dstInst(swigCPtr, this); + } + + public void setDstInst(String dstInst) { + wedpr_java_transportJNI.MessageOptionalHeader_setDstInst(swigCPtr, this, dstInst); + } + + public String topic() { + return wedpr_java_transportJNI.MessageOptionalHeader_topic(swigCPtr, this); + } + + public void setTopic(SWIGTYPE_p_std__string topic) { + wedpr_java_transportJNI.MessageOptionalHeader_setTopic__SWIG_0( + swigCPtr, this, SWIGTYPE_p_std__string.swigRelease(topic)); + } + + public void setTopic(String topic) { + wedpr_java_transportJNI.MessageOptionalHeader_setTopic__SWIG_1(swigCPtr, this, topic); + } + + public String srcInst() { + return wedpr_java_transportJNI.MessageOptionalHeader_srcInst(swigCPtr, this); + } + + public void setSrcInst(String srcInst) { + wedpr_java_transportJNI.MessageOptionalHeader_setSrcInst(swigCPtr, this, srcInst); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java new file mode 100644 index 00000000..16187d11 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java @@ -0,0 +1,58 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessageOptionalHeaderBuilder { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessageOptionalHeaderBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessageOptionalHeaderBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessageOptionalHeaderBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public MessageOptionalHeader build(MessageOptionalHeader optionalHeader) { + long cPtr = + wedpr_java_transportJNI.MessageOptionalHeaderBuilder_build__SWIG_0( + swigCPtr, + this, + MessageOptionalHeader.getCPtr(optionalHeader), + optionalHeader); + return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); + } + + public MessageOptionalHeader build() { + long cPtr = + wedpr_java_transportJNI.MessageOptionalHeaderBuilder_build__SWIG_1(swigCPtr, this); + return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java new file mode 100644 index 00000000..8c62f8aa --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java @@ -0,0 +1,109 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessagePayload { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessagePayload(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessagePayload obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessagePayload(swigCPtr); + } + swigCPtr = 0; + } + } + + public long encode(ubytes buffer) { + return wedpr_java_transportJNI.MessagePayload_encode( + swigCPtr, this, ubytes.getCPtr(buffer), buffer); + } + + public long decode(SWIGTYPE_p_bcos__bytesConstRef data) { + return wedpr_java_transportJNI.MessagePayload_decode( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(data)); + } + + public short version() { + return wedpr_java_transportJNI.MessagePayload_version(swigCPtr, this); + } + + public void setVersion(short version) { + wedpr_java_transportJNI.MessagePayload_setVersion(swigCPtr, this, version); + } + + public ubytes data() { + return new ubytes(wedpr_java_transportJNI.MessagePayload_data(swigCPtr, this), true); + } + + public byte[] dataBuffer() { + return wedpr_java_transportJNI.MessagePayload_dataBuffer(swigCPtr, this); + } + + public void setData(ubytes data) { + wedpr_java_transportJNI.MessagePayload_setData__SWIG_0( + swigCPtr, this, ubytes.swigRelease(data), data); + } + + public int seq() { + return wedpr_java_transportJNI.MessagePayload_seq(swigCPtr, this); + } + + public void setSeq(int seq) { + wedpr_java_transportJNI.MessagePayload_setSeq(swigCPtr, this, seq); + } + + public long length() { + return wedpr_java_transportJNI.MessagePayload_length(swigCPtr, this); + } + + public String traceID() { + return wedpr_java_transportJNI.MessagePayload_traceID(swigCPtr, this); + } + + public void setTraceID(String traceID) { + wedpr_java_transportJNI.MessagePayload_setTraceID(swigCPtr, this, traceID); + } + + public int ext() { + return wedpr_java_transportJNI.MessagePayload_ext(swigCPtr, this); + } + + public void setExt(int ext) { + wedpr_java_transportJNI.MessagePayload_setExt(swigCPtr, this, ext); + } + + public void setRespPacket() { + wedpr_java_transportJNI.MessagePayload_setRespPacket(swigCPtr, this); + } + + public boolean isRespPacket() { + return wedpr_java_transportJNI.MessagePayload_isRespPacket(swigCPtr, this); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java new file mode 100644 index 00000000..c4d1f674 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class MessagePayloadBuilder { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected MessagePayloadBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(MessagePayloadBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_MessagePayloadBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public MessagePayload build() { + long cPtr = wedpr_java_transportJNI.MessagePayloadBuilder_build__SWIG_0(swigCPtr, this); + return (cPtr == 0) ? null : new MessagePayload(cPtr, true); + } + + public MessagePayload build(SWIGTYPE_p_bcos__bytesConstRef buffer) { + long cPtr = + wedpr_java_transportJNI.MessagePayloadBuilder_build__SWIG_1( + swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(buffer)); + return (cPtr == 0) ? null : new MessagePayload(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/OutputBuffer.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/OutputBuffer.java new file mode 100644 index 00000000..86eb3187 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/OutputBuffer.java @@ -0,0 +1,72 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class OutputBuffer { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected OutputBuffer(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(OutputBuffer obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(OutputBuffer obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_OutputBuffer(swigCPtr); + } + swigCPtr = 0; + } + } + + public void setData(SWIGTYPE_p_unsigned_char value) { + wedpr_java_transportJNI.OutputBuffer_data_set( + swigCPtr, this, SWIGTYPE_p_unsigned_char.getCPtr(value)); + } + + public SWIGTYPE_p_unsigned_char getData() { + long cPtr = wedpr_java_transportJNI.OutputBuffer_data_get(swigCPtr, this); + return (cPtr == 0) ? null : new SWIGTYPE_p_unsigned_char(cPtr, false); + } + + public void setLen(java.math.BigInteger value) { + wedpr_java_transportJNI.OutputBuffer_len_set(swigCPtr, this, value); + } + + public java.math.BigInteger getLen() { + return wedpr_java_transportJNI.OutputBuffer_len_get(swigCPtr, this); + } + + public OutputBuffer() { + this(wedpr_java_transportJNI.new_OutputBuffer(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SDKMode.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SDKMode.java new file mode 100644 index 00000000..c5c9f44f --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SDKMode.java @@ -0,0 +1,53 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public final class SDKMode { + public static final SDKMode AIR = new SDKMode("AIR", wedpr_java_transportJNI.SDKMode_AIR_get()); + public static final SDKMode PRO = new SDKMode("PRO", wedpr_java_transportJNI.SDKMode_PRO_get()); + + public final int swigValue() { + return swigValue; + } + + public String toString() { + return swigName; + } + + public static SDKMode swigToEnum(int swigValue) { + if (swigValue < swigValues.length + && swigValue >= 0 + && swigValues[swigValue].swigValue == swigValue) return swigValues[swigValue]; + for (int i = 0; i < swigValues.length; i++) + if (swigValues[i].swigValue == swigValue) return swigValues[i]; + throw new IllegalArgumentException("No enum " + SDKMode.class + " with value " + swigValue); + } + + private SDKMode(String swigName) { + this.swigName = swigName; + this.swigValue = swigNext++; + } + + private SDKMode(String swigName, int swigValue) { + this.swigName = swigName; + this.swigValue = swigValue; + swigNext = swigValue + 1; + } + + private SDKMode(String swigName, SDKMode swigEnum) { + this.swigName = swigName; + this.swigValue = swigEnum.swigValue; + swigNext = this.swigValue + 1; + } + + private static SDKMode[] swigValues = {AIR, PRO}; + private static int swigNext = 0; + private final int swigValue; + private final String swigName; +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__EncodedMsg.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__EncodedMsg.java new file mode 100644 index 00000000..bac116db --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__EncodedMsg.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_bcos__boostssl__EncodedMsg { + private transient long swigCPtr; + + protected SWIGTYPE_p_bcos__boostssl__EncodedMsg( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_bcos__boostssl__EncodedMsg() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_bcos__boostssl__EncodedMsg obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_bcos__boostssl__EncodedMsg obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.java new file mode 100644 index 00000000..426dd9c4 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr { + private transient long swigCPtr; + + protected SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__bytesConstRef.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__bytesConstRef.java new file mode 100644 index 00000000..3016ab17 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_bcos__bytesConstRef.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_bcos__bytesConstRef { + private transient long swigCPtr; + + protected SWIGTYPE_p_bcos__bytesConstRef( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_bcos__bytesConstRef() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_bcos__bytesConstRef obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_bcos__bytesConstRef obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway.java new file mode 100644 index 00000000..556ac652 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_ppc__gateway__IGateway { + private transient long swigCPtr; + + protected SWIGTYPE_p_ppc__gateway__IGateway( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_ppc__gateway__IGateway() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_ppc__gateway__IGateway obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_ppc__gateway__IGateway obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway__Ptr.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway__Ptr.java new file mode 100644 index 00000000..73e52ea9 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__gateway__IGateway__Ptr.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_ppc__gateway__IGateway__Ptr { + private transient long swigCPtr; + + protected SWIGTYPE_p_ppc__gateway__IGateway__Ptr( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_ppc__gateway__IGateway__Ptr() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_ppc__gateway__IGateway__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_ppc__gateway__IGateway__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.java new file mode 100644 index 00000000..69a74312 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr { + private transient long swigCPtr; + + protected SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__RouteType.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__RouteType.java new file mode 100644 index 00000000..850a8086 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_ppc__protocol__RouteType.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_ppc__protocol__RouteType { + private transient long swigCPtr; + + protected SWIGTYPE_p_ppc__protocol__RouteType( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_ppc__protocol__RouteType() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_ppc__protocol__RouteType obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_ppc__protocol__RouteType obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__exception.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__exception.java new file mode 100644 index 00000000..e2a77bb3 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__exception.java @@ -0,0 +1,29 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__exception { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__exception(long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__exception() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__exception obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__exception obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fF_t.java new file mode 100644 index 00000000..e3043d71 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fF_t.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__functionT_void_fF_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__functionT_void_fF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__functionT_void_fF_t() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__functionT_void_fF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__functionT_void_fF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.java new file mode 100644 index 00000000..9d7de5ce --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.java @@ -0,0 +1,32 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java new file mode 100644 index 00000000..a0f6b854 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java @@ -0,0 +1,37 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public +class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t { + private transient long swigCPtr; + + protected + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t + obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t + obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java new file mode 100644 index 00000000..8b62167d --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java @@ -0,0 +1,37 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public +class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t { + private transient long swigCPtr; + + protected + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t + obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t + obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t.java new file mode 100644 index 00000000..859373ad --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t.java @@ -0,0 +1,32 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java new file mode 100644 index 00000000..073a71ab --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java @@ -0,0 +1,32 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t.java new file mode 100644 index 00000000..8573008b --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t.java new file mode 100644 index 00000000..58d25258 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t.java @@ -0,0 +1,32 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t() { + swigCPtr = 0; + } + + protected static long getCPtr( + SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease( + SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string.java new file mode 100644 index 00000000..89720225 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string.java @@ -0,0 +1,29 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__string { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__string(long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__string() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__string obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__string obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string_view.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string_view.java new file mode 100644 index 00000000..d087fcd7 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__string_view.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__string_view { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__string_view( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__string_view() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__string_view obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__string_view obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java new file mode 100644 index 00000000..2ab7be1f --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java @@ -0,0 +1,30 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_std__vectorT_std__string_t { + private transient long swigCPtr; + + protected SWIGTYPE_p_std__vectorT_std__string_t( + long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_std__vectorT_std__string_t() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_std__vectorT_std__string_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_std__vectorT_std__string_t obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_unsigned_char.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_unsigned_char.java new file mode 100644 index 00000000..b7f2ac03 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_unsigned_char.java @@ -0,0 +1,29 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SWIGTYPE_p_unsigned_char { + private transient long swigCPtr; + + protected SWIGTYPE_p_unsigned_char(long cPtr, @SuppressWarnings("unused") boolean futureUse) { + swigCPtr = cPtr; + } + + protected SWIGTYPE_p_unsigned_char() { + swigCPtr = 0; + } + + protected static long getCPtr(SWIGTYPE_p_unsigned_char obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SWIGTYPE_p_unsigned_char obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SendResponseHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SendResponseHandler.java new file mode 100644 index 00000000..6ab774ab --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SendResponseHandler.java @@ -0,0 +1,64 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SendResponseHandler { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SendResponseHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SendResponseHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SendResponseHandler obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SendResponseHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + public SendResponseHandler( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t responseFunc) { + this( + wedpr_java_transportJNI.new_SendResponseHandler( + SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t + .getCPtr(responseFunc)), + true); + } + + public void sendResponse(SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t payload) { + wedpr_java_transportJNI.SendResponseHandler_sendResponse( + swigCPtr, this, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t.swigRelease(payload)); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedBcosError.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedBcosError.java new file mode 100644 index 00000000..87fff366 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedBcosError.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedBcosError { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedBcosError(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedBcosError obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedBcosError obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedBcosError(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedBcosError() { + this(wedpr_java_transportJNI.new_SharedBcosError(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedErrorCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedErrorCallback.java new file mode 100644 index 00000000..4b53ca7f --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedErrorCallback.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedErrorCallback { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedErrorCallback(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedErrorCallback obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedErrorCallback obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedErrorCallback(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedErrorCallback() { + this(wedpr_java_transportJNI.new_SharedErrorCallback(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFront.java new file mode 100644 index 00000000..26f4d825 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFront.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedFront { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedFront(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedFront obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedFront obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedFront(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedFront() { + this(wedpr_java_transportJNI.new_SharedFront(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontClient.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontClient.java new file mode 100644 index 00000000..0ae0e986 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontClient.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedFrontClient { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedFrontClient(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedFrontClient obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedFrontClient obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedFrontClient(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedFrontClient() { + this(wedpr_java_transportJNI.new_SharedFrontClient(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontConfig.java new file mode 100644 index 00000000..a0d898ec --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedFrontConfig.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedFrontConfig { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedFrontConfig(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedFrontConfig obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedFrontConfig obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedFrontConfig(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedFrontConfig() { + this(wedpr_java_transportJNI.new_SharedFrontConfig(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGateway.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGateway.java new file mode 100644 index 00000000..64ef6146 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGateway.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedGateway { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedGateway(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedGateway obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedGateway obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedGateway(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedGateway() { + this(wedpr_java_transportJNI.new_SharedGateway(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGrpcConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGrpcConfig.java new file mode 100644 index 00000000..4bd5497c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGrpcConfig.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedGrpcConfig { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedGrpcConfig(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedGrpcConfig obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedGrpcConfig obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedGrpcConfig(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedGrpcConfig() { + this(wedpr_java_transportJNI.new_SharedGrpcConfig(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedIMessageHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedIMessageHandler.java new file mode 100644 index 00000000..45703dcd --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedIMessageHandler.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedIMessageHandler { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedIMessageHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedIMessageHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedIMessageHandler obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedIMessageHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedIMessageHandler() { + this(wedpr_java_transportJNI.new_SharedIMessageHandler(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessage.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessage.java new file mode 100644 index 00000000..4e30cedb --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessage.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessage { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessage(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessage obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessage obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessage(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessage() { + this(wedpr_java_transportJNI.new_SharedMessage(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageBuilder.java new file mode 100644 index 00000000..b45e5eff --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageBuilder.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessageBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessageBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessageBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessageBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessageBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessageBuilder() { + this(wedpr_java_transportJNI.new_SharedMessageBuilder(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageDispatcherHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageDispatcherHandler.java new file mode 100644 index 00000000..6b3fd639 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageDispatcherHandler.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessageDispatcherHandler { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessageDispatcherHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessageDispatcherHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessageDispatcherHandler obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessageDispatcherHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessageDispatcherHandler() { + this(wedpr_java_transportJNI.new_SharedMessageDispatcherHandler(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeader.java new file mode 100644 index 00000000..7040dd6c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeader.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessageHeader { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessageHeader(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessageHeader obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessageHeader obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessageHeader(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessageHeader() { + this(wedpr_java_transportJNI.new_SharedMessageHeader(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeaderBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeaderBuilder.java new file mode 100644 index 00000000..fc0464ed --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessageHeaderBuilder.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessageHeaderBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessageHeaderBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessageHeaderBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessageHeaderBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessageHeaderBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessageHeaderBuilder() { + this(wedpr_java_transportJNI.new_SharedMessageHeaderBuilder(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayload.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayload.java new file mode 100644 index 00000000..6a52b280 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayload.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessagePayload { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessagePayload(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessagePayload obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessagePayload obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessagePayload(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessagePayload() { + this(wedpr_java_transportJNI.new_SharedMessagePayload(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayloadBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayloadBuilder.java new file mode 100644 index 00000000..19b3a20a --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedMessagePayloadBuilder.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedMessagePayloadBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedMessagePayloadBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedMessagePayloadBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedMessagePayloadBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedMessagePayloadBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedMessagePayloadBuilder() { + this(wedpr_java_transportJNI.new_SharedMessagePayloadBuilder(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfo.java new file mode 100644 index 00000000..d3ae668b --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfo.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedRouteInfo { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedRouteInfo(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedRouteInfo obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedRouteInfo obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedRouteInfo(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedRouteInfo() { + this(wedpr_java_transportJNI.new_SharedRouteInfo(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfoBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfoBuilder.java new file mode 100644 index 00000000..7cec4144 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedRouteInfoBuilder.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedRouteInfoBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedRouteInfoBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedRouteInfoBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedRouteInfoBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedRouteInfoBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedRouteInfoBuilder() { + this(wedpr_java_transportJNI.new_SharedRouteInfoBuilder(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java new file mode 100644 index 00000000..285e1c55 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java @@ -0,0 +1,89 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class Transport { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected Transport(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(Transport obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_Transport(swigCPtr); + } + swigCPtr = 0; + } + } + + public Transport(FrontConfig config) { + this(wedpr_java_transportJNI.new_Transport(FrontConfig.getCPtr(config), config), true); + } + + public void start() { + wedpr_java_transportJNI.Transport_start(swigCPtr, this); + } + + public void stop() { + wedpr_java_transportJNI.Transport_stop(swigCPtr, this); + } + + public IFront getFront() { + long cPtr = wedpr_java_transportJNI.Transport_getFront(swigCPtr, this); + return (cPtr == 0) ? null : new IFront(cPtr, true); + } + + public SWIGTYPE_p_ppc__gateway__IGateway__Ptr gateway() { + return new SWIGTYPE_p_ppc__gateway__IGateway__Ptr( + wedpr_java_transportJNI.Transport_gateway(swigCPtr, this), false); + } + + public MessagePayloadBuilder msgPayloadBuilder() { + long cPtr = wedpr_java_transportJNI.Transport_msgPayloadBuilder(swigCPtr, this); + return (cPtr == 0) ? null : new MessagePayloadBuilder(cPtr, true); + } + + public MessageBuilder msgBuilder() { + long cPtr = wedpr_java_transportJNI.Transport_msgBuilder(swigCPtr, this); + return (cPtr == 0) ? null : new MessageBuilder(cPtr, true); + } + + public MessageHeaderBuilder msgHeaderBuilder() { + long cPtr = wedpr_java_transportJNI.Transport_msgHeaderBuilder(swigCPtr, this); + return (cPtr == 0) ? null : new MessageHeaderBuilder(cPtr, true); + } + + public MessageOptionalHeaderBuilder routeInfoBuilder() { + long cPtr = wedpr_java_transportJNI.Transport_routeInfoBuilder(swigCPtr, this); + return (cPtr == 0) ? null : new MessageOptionalHeaderBuilder(cPtr, true); + } + + public FrontConfig getConfig() { + long cPtr = wedpr_java_transportJNI.Transport_getConfig(swigCPtr, this); + return (cPtr == 0) ? null : new FrontConfig(cPtr, true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java new file mode 100644 index 00000000..ed489a57 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java @@ -0,0 +1,86 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class TransportBuilder { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected TransportBuilder(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(TransportBuilder obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(TransportBuilder obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_TransportBuilder(swigCPtr); + } + swigCPtr = 0; + } + } + + public TransportBuilder() { + this(wedpr_java_transportJNI.new_TransportBuilder(), true); + } + + public Transport build( + SDKMode mode, FrontConfig config, SWIGTYPE_p_ppc__gateway__IGateway__Ptr gateway) { + long cPtr = + wedpr_java_transportJNI.TransportBuilder_build( + swigCPtr, + this, + mode.swigValue(), + FrontConfig.getCPtr(config), + config, + SWIGTYPE_p_ppc__gateway__IGateway__Ptr.getCPtr(gateway)); + return (cPtr == 0) ? null : new Transport(cPtr, true); + } + + public Transport buildProTransport(FrontConfig config) { + long cPtr = + wedpr_java_transportJNI.TransportBuilder_buildProTransport( + swigCPtr, this, FrontConfig.getCPtr(config), config); + return (cPtr == 0) ? null : new Transport(cPtr, true); + } + + public FrontConfig buildConfig(int threadPoolSize, String nodeID) { + long cPtr = + wedpr_java_transportJNI.TransportBuilder_buildConfig( + swigCPtr, this, threadPoolSize, nodeID); + return (cPtr == 0) ? null : new FrontConfig(cPtr, true); + } + + public SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t frontConfigBuilder() { + return new SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t( + wedpr_java_transportJNI.TransportBuilder_frontConfigBuilder(swigCPtr, this), false); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ibytes.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ibytes.java new file mode 100644 index 00000000..8b48eaf1 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ibytes.java @@ -0,0 +1,163 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class ibytes extends java.util.AbstractList implements java.util.RandomAccess { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected ibytes(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(ibytes obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(ibytes obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_ibytes(swigCPtr); + } + swigCPtr = 0; + } + } + + public ibytes(byte[] initialElements) { + this(); + reserve(initialElements.length); + + for (byte element : initialElements) { + add(element); + } + } + + public ibytes(Iterable initialElements) { + this(); + for (byte element : initialElements) { + add(element); + } + } + + public Byte get(int index) { + return doGet(index); + } + + public Byte set(int index, Byte e) { + return doSet(index, e); + } + + public boolean add(Byte e) { + modCount++; + doAdd(e); + return true; + } + + public void add(int index, Byte e) { + modCount++; + doAdd(index, e); + } + + public Byte remove(int index) { + modCount++; + return doRemove(index); + } + + protected void removeRange(int fromIndex, int toIndex) { + modCount++; + doRemoveRange(fromIndex, toIndex); + } + + public int size() { + return doSize(); + } + + public int capacity() { + return doCapacity(); + } + + public void reserve(int n) { + doReserve(n); + } + + public ibytes() { + this(wedpr_java_transportJNI.new_ibytes__SWIG_0(), true); + } + + public ibytes(ibytes other) { + this(wedpr_java_transportJNI.new_ibytes__SWIG_1(ibytes.getCPtr(other), other), true); + } + + public boolean isEmpty() { + return wedpr_java_transportJNI.ibytes_isEmpty(swigCPtr, this); + } + + public void clear() { + wedpr_java_transportJNI.ibytes_clear(swigCPtr, this); + } + + public ibytes(int count, byte value) { + this(wedpr_java_transportJNI.new_ibytes__SWIG_2(count, value), true); + } + + private int doCapacity() { + return wedpr_java_transportJNI.ibytes_doCapacity(swigCPtr, this); + } + + private void doReserve(int n) { + wedpr_java_transportJNI.ibytes_doReserve(swigCPtr, this, n); + } + + private int doSize() { + return wedpr_java_transportJNI.ibytes_doSize(swigCPtr, this); + } + + private void doAdd(byte x) { + wedpr_java_transportJNI.ibytes_doAdd__SWIG_0(swigCPtr, this, x); + } + + private void doAdd(int index, byte x) { + wedpr_java_transportJNI.ibytes_doAdd__SWIG_1(swigCPtr, this, index, x); + } + + private byte doRemove(int index) { + return wedpr_java_transportJNI.ibytes_doRemove(swigCPtr, this, index); + } + + private byte doGet(int index) { + return wedpr_java_transportJNI.ibytes_doGet(swigCPtr, this, index); + } + + private byte doSet(int index, byte val) { + return wedpr_java_transportJNI.ibytes_doSet(swigCPtr, this, index, val); + } + + private void doRemoveRange(int fromIndex, int toIndex) { + wedpr_java_transportJNI.ibytes_doRemoveRange(swigCPtr, this, fromIndex, toIndex); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ubytes.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ubytes.java new file mode 100644 index 00000000..38c274e0 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/ubytes.java @@ -0,0 +1,163 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class ubytes extends java.util.AbstractList implements java.util.RandomAccess { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected ubytes(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(ubytes obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(ubytes obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_ubytes(swigCPtr); + } + swigCPtr = 0; + } + } + + public ubytes(short[] initialElements) { + this(); + reserve(initialElements.length); + + for (short element : initialElements) { + add(element); + } + } + + public ubytes(Iterable initialElements) { + this(); + for (short element : initialElements) { + add(element); + } + } + + public Short get(int index) { + return doGet(index); + } + + public Short set(int index, Short e) { + return doSet(index, e); + } + + public boolean add(Short e) { + modCount++; + doAdd(e); + return true; + } + + public void add(int index, Short e) { + modCount++; + doAdd(index, e); + } + + public Short remove(int index) { + modCount++; + return doRemove(index); + } + + protected void removeRange(int fromIndex, int toIndex) { + modCount++; + doRemoveRange(fromIndex, toIndex); + } + + public int size() { + return doSize(); + } + + public int capacity() { + return doCapacity(); + } + + public void reserve(int n) { + doReserve(n); + } + + public ubytes() { + this(wedpr_java_transportJNI.new_ubytes__SWIG_0(), true); + } + + public ubytes(ubytes other) { + this(wedpr_java_transportJNI.new_ubytes__SWIG_1(ubytes.getCPtr(other), other), true); + } + + public boolean isEmpty() { + return wedpr_java_transportJNI.ubytes_isEmpty(swigCPtr, this); + } + + public void clear() { + wedpr_java_transportJNI.ubytes_clear(swigCPtr, this); + } + + public ubytes(int count, short value) { + this(wedpr_java_transportJNI.new_ubytes__SWIG_2(count, value), true); + } + + private int doCapacity() { + return wedpr_java_transportJNI.ubytes_doCapacity(swigCPtr, this); + } + + private void doReserve(int n) { + wedpr_java_transportJNI.ubytes_doReserve(swigCPtr, this, n); + } + + private int doSize() { + return wedpr_java_transportJNI.ubytes_doSize(swigCPtr, this); + } + + private void doAdd(short x) { + wedpr_java_transportJNI.ubytes_doAdd__SWIG_0(swigCPtr, this, x); + } + + private void doAdd(int index, short x) { + wedpr_java_transportJNI.ubytes_doAdd__SWIG_1(swigCPtr, this, index, x); + } + + private short doRemove(int index) { + return wedpr_java_transportJNI.ubytes_doRemove(swigCPtr, this, index); + } + + private short doGet(int index) { + return wedpr_java_transportJNI.ubytes_doGet(swigCPtr, this, index); + } + + private short doSet(int index, short val) { + return wedpr_java_transportJNI.ubytes_doSet(swigCPtr, this, index, val); + } + + private void doRemoveRange(int fromIndex, int toIndex) { + wedpr_java_transportJNI.ubytes_doRemoveRange(swigCPtr, this, fromIndex, toIndex); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java new file mode 100644 index 00000000..c8f1cdc8 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java @@ -0,0 +1,29 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class wedpr_java_transport { + public static String printFrontDesc(FrontConfig config) { + return wedpr_java_transportJNI.printFrontDesc(FrontConfig.getCPtr(config), config); + } + + public static String printOptionalField(MessageOptionalHeader optionalHeader) { + return wedpr_java_transportJNI.printOptionalField( + MessageOptionalHeader.getCPtr(optionalHeader), optionalHeader); + } + + public static String printMessage(Message _msg) { + return wedpr_java_transportJNI.printMessage(Message.getCPtr(_msg), _msg); + } + + public static String printWsMessage(SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr _msg) { + return wedpr_java_transportJNI.printWsMessage( + SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.getCPtr(_msg)); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java new file mode 100644 index 00000000..debebac1 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java @@ -0,0 +1,831 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class wedpr_java_transportJNI { + public static final native long new_SharedBcosError(); + + public static final native void delete_SharedBcosError(long jarg1); + + public static final native long new_SharedFrontConfig(); + + public static final native void delete_SharedFrontConfig(long jarg1); + + public static final native long new_SharedGrpcConfig(); + + public static final native void delete_SharedGrpcConfig(long jarg1); + + public static final native long new_SharedFront(); + + public static final native void delete_SharedFront(long jarg1); + + public static final native long new_SharedFrontClient(); + + public static final native void delete_SharedFrontClient(long jarg1); + + public static final native long new_SharedErrorCallback(); + + public static final native void delete_SharedErrorCallback(long jarg1); + + public static final native long new_SharedMessageDispatcherHandler(); + + public static final native void delete_SharedMessageDispatcherHandler(long jarg1); + + public static final native long new_SharedIMessageHandler(); + + public static final native void delete_SharedIMessageHandler(long jarg1); + + public static final native long new_SharedGateway(); + + public static final native void delete_SharedGateway(long jarg1); + + public static final native long new_SharedMessage(); + + public static final native void delete_SharedMessage(long jarg1); + + public static final native long new_SharedMessageHeader(); + + public static final native void delete_SharedMessageHeader(long jarg1); + + public static final native long new_SharedMessagePayload(); + + public static final native void delete_SharedMessagePayload(long jarg1); + + public static final native long new_SharedRouteInfo(); + + public static final native void delete_SharedRouteInfo(long jarg1); + + public static final native long new_SharedMessageBuilder(); + + public static final native void delete_SharedMessageBuilder(long jarg1); + + public static final native long new_SharedMessageHeaderBuilder(); + + public static final native void delete_SharedMessageHeaderBuilder(long jarg1); + + public static final native long new_SharedMessagePayloadBuilder(); + + public static final native void delete_SharedMessagePayloadBuilder(long jarg1); + + public static final native long new_SharedRouteInfoBuilder(); + + public static final native void delete_SharedRouteInfoBuilder(long jarg1); + + public static final native long new_ubytes__SWIG_0(); + + public static final native long new_ubytes__SWIG_1(long jarg1, ubytes jarg1_); + + public static final native boolean ubytes_isEmpty(long jarg1, ubytes jarg1_); + + public static final native void ubytes_clear(long jarg1, ubytes jarg1_); + + public static final native long new_ubytes__SWIG_2(int jarg1, short jarg2); + + public static final native int ubytes_doCapacity(long jarg1, ubytes jarg1_); + + public static final native void ubytes_doReserve(long jarg1, ubytes jarg1_, int jarg2); + + public static final native int ubytes_doSize(long jarg1, ubytes jarg1_); + + public static final native void ubytes_doAdd__SWIG_0(long jarg1, ubytes jarg1_, short jarg2); + + public static final native void ubytes_doAdd__SWIG_1( + long jarg1, ubytes jarg1_, int jarg2, short jarg3); + + public static final native short ubytes_doRemove(long jarg1, ubytes jarg1_, int jarg2); + + public static final native short ubytes_doGet(long jarg1, ubytes jarg1_, int jarg2); + + public static final native short ubytes_doSet( + long jarg1, ubytes jarg1_, int jarg2, short jarg3); + + public static final native void ubytes_doRemoveRange( + long jarg1, ubytes jarg1_, int jarg2, int jarg3); + + public static final native void delete_ubytes(long jarg1); + + public static final native long new_ibytes__SWIG_0(); + + public static final native long new_ibytes__SWIG_1(long jarg1, ibytes jarg1_); + + public static final native boolean ibytes_isEmpty(long jarg1, ibytes jarg1_); + + public static final native void ibytes_clear(long jarg1, ibytes jarg1_); + + public static final native long new_ibytes__SWIG_2(int jarg1, byte jarg2); + + public static final native int ibytes_doCapacity(long jarg1, ibytes jarg1_); + + public static final native void ibytes_doReserve(long jarg1, ibytes jarg1_, int jarg2); + + public static final native int ibytes_doSize(long jarg1, ibytes jarg1_); + + public static final native void ibytes_doAdd__SWIG_0(long jarg1, ibytes jarg1_, byte jarg2); + + public static final native void ibytes_doAdd__SWIG_1( + long jarg1, ibytes jarg1_, int jarg2, byte jarg3); + + public static final native byte ibytes_doRemove(long jarg1, ibytes jarg1_, int jarg2); + + public static final native byte ibytes_doGet(long jarg1, ibytes jarg1_, int jarg2); + + public static final native byte ibytes_doSet(long jarg1, ibytes jarg1_, int jarg2, byte jarg3); + + public static final native void ibytes_doRemoveRange( + long jarg1, ibytes jarg1_, int jarg2, int jarg3); + + public static final native void delete_ibytes(long jarg1); + + public static final native long Error_buildError__SWIG_0( + byte[] jarg1, byte[] jarg2, int jarg3, int jarg4, String jarg5); + + public static final native long Error_buildError__SWIG_1( + byte[] jarg1, + byte[] jarg2, + int jarg3, + int jarg4, + String jarg5, + long jarg6, + Error jarg6_); + + public static final native long Error_buildError__SWIG_2( + byte[] jarg1, byte[] jarg2, int jarg3, int jarg4, String jarg5, long jarg6); + + public static final native long new_Error__SWIG_0(); + + public static final native long new_Error__SWIG_1(long jarg1, String jarg2); + + public static final native void delete_Error(long jarg1); + + public static final native long Error_errorCode(long jarg1, Error jarg1_); + + public static final native String Error_errorMessage(long jarg1, Error jarg1_); + + public static final native void Error_setErrorCode(long jarg1, Error jarg1_, long jarg2); + + public static final native void Error_setErrorMessage(long jarg1, Error jarg1_, String jarg2); + + public static final native void InputBuffer_data_set( + long jarg1, InputBuffer jarg1_, long jarg2); + + public static final native long InputBuffer_data_get(long jarg1, InputBuffer jarg1_); + + public static final native void InputBuffer_len_set( + long jarg1, InputBuffer jarg1_, java.math.BigInteger jarg2); + + public static final native java.math.BigInteger InputBuffer_len_get( + long jarg1, InputBuffer jarg1_); + + public static final native long new_InputBuffer(); + + public static final native void delete_InputBuffer(long jarg1); + + public static final native void OutputBuffer_data_set( + long jarg1, OutputBuffer jarg1_, long jarg2); + + public static final native long OutputBuffer_data_get(long jarg1, OutputBuffer jarg1_); + + public static final native void OutputBuffer_len_set( + long jarg1, OutputBuffer jarg1_, java.math.BigInteger jarg2); + + public static final native java.math.BigInteger OutputBuffer_len_get( + long jarg1, OutputBuffer jarg1_); + + public static final native long new_OutputBuffer(); + + public static final native void delete_OutputBuffer(long jarg1); + + public static final native void delete_FrontConfig(long jarg1); + + public static final native int FrontConfig_threadPoolSize(long jarg1, FrontConfig jarg1_); + + public static final native void FrontConfig_setThreadPoolSize( + long jarg1, FrontConfig jarg1_, int jarg2); + + public static final native String FrontConfig_nodeID(long jarg1, FrontConfig jarg1_); + + public static final native void FrontConfig_setNodeID( + long jarg1, FrontConfig jarg1_, String jarg2); + + public static final native long FrontConfig_selfEndPoint(long jarg1, FrontConfig jarg1_); + + public static final native long FrontConfig_mutableSelfEndPoint(long jarg1, FrontConfig jarg1_); + + public static final native void FrontConfig_setSelfEndPoint( + long jarg1, FrontConfig jarg1_, long jarg2, EndPoint jarg2_); + + public static final native void FrontConfig_setGatewayGrpcTarget( + long jarg1, FrontConfig jarg1_, String jarg2); + + public static final native String FrontConfig_gatewayGrpcTarget(long jarg1, FrontConfig jarg1_); + + public static final native void FrontConfig_setGrpcConfig( + long jarg1, FrontConfig jarg1_, long jarg2, GrpcConfig jarg2_); + + public static final native long FrontConfig_grpcConfig(long jarg1, FrontConfig jarg1_); + + public static final native long FrontConfig_generateNodeInfo(long jarg1, FrontConfig jarg1_); + + public static final native long FrontConfig_getComponents(long jarg1, FrontConfig jarg1_); + + public static final native void FrontConfig_setComponents( + long jarg1, FrontConfig jarg1_, long jarg2); + + public static final native void FrontConfig_addComponent( + long jarg1, FrontConfig jarg1_, String jarg2); + + public static final native long FrontConfig_mutableComponents(long jarg1, FrontConfig jarg1_); + + public static final native void delete_FrontConfigBuilder(long jarg1); + + public static final native long FrontConfigBuilder_build__SWIG_0( + long jarg1, FrontConfigBuilder jarg1_); + + public static final native long FrontConfigBuilder_build__SWIG_1( + long jarg1, FrontConfigBuilder jarg1_, int jarg2, String jarg3); + + public static final native String printFrontDesc(long jarg1, FrontConfig jarg1_); + + public static final native long new_EndPoint__SWIG_0(); + + public static final native long new_EndPoint__SWIG_1(String jarg1, int jarg2); + + public static final native void delete_EndPoint(long jarg1); + + public static final native String EndPoint_host(long jarg1, EndPoint jarg1_); + + public static final native int EndPoint_port(long jarg1, EndPoint jarg1_); + + public static final native void EndPoint_setHost(long jarg1, EndPoint jarg1_, String jarg2); + + public static final native void EndPoint_setPort(long jarg1, EndPoint jarg1_, int jarg2); + + public static final native void EndPoint_setListenIp(long jarg1, EndPoint jarg1_, String jarg2); + + public static final native String EndPoint_entryPoint(long jarg1, EndPoint jarg1_); + + public static final native String EndPoint_listenEndPoint(long jarg1, EndPoint jarg1_); + + public static final native String EndPoint_listenIp(long jarg1, EndPoint jarg1_); + + public static final native long new_GrpcServerConfig__SWIG_0(); + + public static final native long new_GrpcServerConfig__SWIG_1( + long jarg1, EndPoint jarg1_, boolean jarg2); + + public static final native String GrpcServerConfig_listenEndPoint( + long jarg1, GrpcServerConfig jarg1_); + + public static final native void GrpcServerConfig_setEndPoint( + long jarg1, GrpcServerConfig jarg1_, long jarg2, EndPoint jarg2_); + + public static final native void GrpcServerConfig_setEnableHealthCheck( + long jarg1, GrpcServerConfig jarg1_, boolean jarg2); + + public static final native long GrpcServerConfig_endPoint(long jarg1, GrpcServerConfig jarg1_); + + public static final native long GrpcServerConfig_mutableEndPoint( + long jarg1, GrpcServerConfig jarg1_); + + public static final native boolean GrpcServerConfig_enableHealthCheck( + long jarg1, GrpcServerConfig jarg1_); + + public static final native void delete_GrpcServerConfig(long jarg1); + + public static final native long new_GrpcConfig(); + + public static final native void delete_GrpcConfig(long jarg1); + + public static final native String GrpcConfig_loadBalancePolicy(long jarg1, GrpcConfig jarg1_); + + public static final native void GrpcConfig_setLoadBalancePolicy( + long jarg1, GrpcConfig jarg1_, String jarg2); + + public static final native boolean GrpcConfig_enableHealthCheck(long jarg1, GrpcConfig jarg1_); + + public static final native void GrpcConfig_setEnableHealthCheck( + long jarg1, GrpcConfig jarg1_, boolean jarg2); + + public static final native void GrpcConfig_setEnableDnslookup( + long jarg1, GrpcConfig jarg1_, boolean jarg2); + + public static final native boolean GrpcConfig_enableDnslookup(long jarg1, GrpcConfig jarg1_); + + public static final native void delete_MessageOptionalHeader(long jarg1); + + public static final native void MessageOptionalHeader_encode( + long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_); + + public static final native long MessageOptionalHeader_decode( + long jarg1, MessageOptionalHeader jarg1_, long jarg2, java.math.BigInteger jarg3); + + public static final native String MessageOptionalHeader_componentType( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setComponentType( + long jarg1, MessageOptionalHeader jarg1_, String jarg2); + + public static final native long MessageOptionalHeader_srcNode( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native byte[] MessageOptionalHeader_srcNodeBuffer( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setSrcNode__SWIG_0( + long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_); + + public static final native void MessageOptionalHeader_setSrcNode__SWIG_1( + long jarg1, MessageOptionalHeader jarg1_, byte[] jarg2, java.math.BigInteger jarg3); + + public static final native long MessageOptionalHeader_dstNode( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native byte[] MessageOptionalHeader_dstNodeBuffer( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setDstNode__SWIG_0( + long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_); + + public static final native void MessageOptionalHeader_setDstNode__SWIG_1( + long jarg1, MessageOptionalHeader jarg1_, byte[] jarg2, java.math.BigInteger jarg3); + + public static final native String MessageOptionalHeader_dstInst( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setDstInst( + long jarg1, MessageOptionalHeader jarg1_, String jarg2); + + public static final native String MessageOptionalHeader_topic( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setTopic__SWIG_0( + long jarg1, MessageOptionalHeader jarg1_, long jarg2); + + public static final native void MessageOptionalHeader_setTopic__SWIG_1( + long jarg1, MessageOptionalHeader jarg1_, String jarg2); + + public static final native String MessageOptionalHeader_srcInst( + long jarg1, MessageOptionalHeader jarg1_); + + public static final native void MessageOptionalHeader_setSrcInst( + long jarg1, MessageOptionalHeader jarg1_, String jarg2); + + public static final native void delete_MessageHeader(long jarg1); + + public static final native void MessageHeader_encode( + long jarg1, MessageHeader jarg1_, long jarg2, ubytes jarg2_); + + public static final native long MessageHeader_decode( + long jarg1, MessageHeader jarg1_, long jarg2); + + public static final native short MessageHeader_version(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setVersion( + long jarg1, MessageHeader jarg1_, int jarg2); + + public static final native String MessageHeader_traceID(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setTraceID( + long jarg1, MessageHeader jarg1_, String jarg2); + + public static final native String MessageHeader_srcGwNode(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setSrcGwNode( + long jarg1, MessageHeader jarg1_, String jarg2); + + public static final native String MessageHeader_dstGwNode(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setDstGwNode( + long jarg1, MessageHeader jarg1_, String jarg2); + + public static final native int MessageHeader_packetType(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setPacketType( + long jarg1, MessageHeader jarg1_, int jarg2); + + public static final native short MessageHeader_ttl(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setTTL( + long jarg1, MessageHeader jarg1_, int jarg2); + + public static final native int MessageHeader_ext(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setExt( + long jarg1, MessageHeader jarg1_, int jarg2); + + public static final native long MessageHeader_optionalField(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setOptionalField( + long jarg1, MessageHeader jarg1_, long jarg2, MessageOptionalHeader jarg2_); + + public static final native int MessageHeader_length(long jarg1, MessageHeader jarg1_); + + public static final native boolean MessageHeader_isRespPacket(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setRespPacket(long jarg1, MessageHeader jarg1_); + + public static final native long MessageHeader_srcP2PNodeIDView( + long jarg1, MessageHeader jarg1_); + + public static final native long MessageHeader_dstP2PNodeIDView( + long jarg1, MessageHeader jarg1_); + + public static final native int MessageHeader_routeType(long jarg1, MessageHeader jarg1_); + + public static final native void MessageHeader_setRouteType( + long jarg1, MessageHeader jarg1_, long jarg2); + + public static final native boolean MessageHeader_hasOptionalField( + long jarg1, MessageHeader jarg1_); + + public static final native void delete_Message(long jarg1); + + public static final native long Message_header(long jarg1, Message jarg1_); + + public static final native void Message_setHeader( + long jarg1, Message jarg1_, long jarg2, MessageHeader jarg2_); + + public static final native int Message_version(long jarg1, Message jarg1_); + + public static final native void Message_setVersion(long jarg1, Message jarg1_, int jarg2); + + public static final native int Message_packetType(long jarg1, Message jarg1_); + + public static final native void Message_setPacketType(long jarg1, Message jarg1_, int jarg2); + + public static final native String Message_seq(long jarg1, Message jarg1_); + + public static final native void Message_setSeq(long jarg1, Message jarg1_, String jarg2); + + public static final native int Message_ext(long jarg1, Message jarg1_); + + public static final native void Message_setExt(long jarg1, Message jarg1_, int jarg2); + + public static final native boolean Message_isRespPacket(long jarg1, Message jarg1_); + + public static final native void Message_setRespPacket(long jarg1, Message jarg1_); + + public static final native long Message_length(long jarg1, Message jarg1_); + + public static final native long Message_payload(long jarg1, Message jarg1_); + + public static final native byte[] Message_payloadBuffer(long jarg1, Message jarg1_); + + public static final native void Message_setPayload( + long jarg1, Message jarg1_, long jarg2, ubytes jarg2_); + + public static final native void Message_setFrontMessage( + long jarg1, Message jarg1_, long jarg2, MessagePayload jarg2_); + + public static final native long Message_frontMessage(long jarg1, Message jarg1_); + + public static final native boolean Message_encode__SWIG_0( + long jarg1, Message jarg1_, long jarg2, ubytes jarg2_); + + public static final native boolean Message_encode__SWIG_1( + long jarg1, Message jarg1_, long jarg2); + + public static final native long Message_decode(long jarg1, Message jarg1_, long jarg2); + + public static final native void delete_MessageHeaderBuilder(long jarg1); + + public static final native long MessageHeaderBuilder_build__SWIG_0( + long jarg1, MessageHeaderBuilder jarg1_, long jarg2); + + public static final native long MessageHeaderBuilder_build__SWIG_1( + long jarg1, MessageHeaderBuilder jarg1_); + + public static final native long MessageHeaderBuilder_build__SWIG_2( + long jarg1, MessageHeaderBuilder jarg1_, long jarg2, MessageOptionalHeader jarg2_); + + public static final native void delete_MessageBuilder(long jarg1); + + public static final native long MessageBuilder_build__SWIG_0(long jarg1, MessageBuilder jarg1_); + + public static final native long MessageBuilder_build__SWIG_1( + long jarg1, MessageBuilder jarg1_, long jarg2); + + public static final native long MessageBuilder_build__SWIG_2( + long jarg1, + MessageBuilder jarg1_, + long jarg2, + long jarg3, + MessageOptionalHeader jarg3_, + long jarg4, + ubytes jarg4_); + + public static final native void delete_MessageOptionalHeaderBuilder(long jarg1); + + public static final native long MessageOptionalHeaderBuilder_build__SWIG_0( + long jarg1, + MessageOptionalHeaderBuilder jarg1_, + long jarg2, + MessageOptionalHeader jarg2_); + + public static final native long MessageOptionalHeaderBuilder_build__SWIG_1( + long jarg1, MessageOptionalHeaderBuilder jarg1_); + + public static final native String printOptionalField(long jarg1, MessageOptionalHeader jarg1_); + + public static final native String printMessage(long jarg1, Message jarg1_); + + public static final native String printWsMessage(long jarg1); + + public static final native int FrontMsgExtFlag_Response_get(); + + public static final native void delete_MessagePayload(long jarg1); + + public static final native long MessagePayload_encode( + long jarg1, MessagePayload jarg1_, long jarg2, ubytes jarg2_); + + public static final native long MessagePayload_decode( + long jarg1, MessagePayload jarg1_, long jarg2); + + public static final native short MessagePayload_version(long jarg1, MessagePayload jarg1_); + + public static final native void MessagePayload_setVersion( + long jarg1, MessagePayload jarg1_, short jarg2); + + public static final native long MessagePayload_data(long jarg1, MessagePayload jarg1_); + + public static final native byte[] MessagePayload_dataBuffer(long jarg1, MessagePayload jarg1_); + + public static final native void MessagePayload_setData__SWIG_0( + long jarg1, MessagePayload jarg1_, long jarg2, ubytes jarg2_); + + public static final native int MessagePayload_seq(long jarg1, MessagePayload jarg1_); + + public static final native void MessagePayload_setSeq( + long jarg1, MessagePayload jarg1_, int jarg2); + + public static final native long MessagePayload_length(long jarg1, MessagePayload jarg1_); + + public static final native String MessagePayload_traceID(long jarg1, MessagePayload jarg1_); + + public static final native void MessagePayload_setTraceID( + long jarg1, MessagePayload jarg1_, String jarg2); + + public static final native int MessagePayload_ext(long jarg1, MessagePayload jarg1_); + + public static final native void MessagePayload_setExt( + long jarg1, MessagePayload jarg1_, int jarg2); + + public static final native void MessagePayload_setRespPacket(long jarg1, MessagePayload jarg1_); + + public static final native boolean MessagePayload_isRespPacket( + long jarg1, MessagePayload jarg1_); + + public static final native void delete_MessagePayloadBuilder(long jarg1); + + public static final native long MessagePayloadBuilder_build__SWIG_0( + long jarg1, MessagePayloadBuilder jarg1_); + + public static final native long MessagePayloadBuilder_build__SWIG_1( + long jarg1, MessagePayloadBuilder jarg1_, long jarg2); + + public static final native void delete_IFrontClient(long jarg1); + + public static final native void IFrontClient_onReceiveMessage( + long jarg1, IFrontClient jarg1_, long jarg2, Message jarg2_, long jarg3); + + public static final native long new_ErrorCallback(); + + public static final native void delete_ErrorCallback(long jarg1); + + public static final native void ErrorCallback_onError( + long jarg1, ErrorCallback jarg1_, long jarg2, Error jarg2_); + + public static final native void ErrorCallback_director_connect( + ErrorCallback obj, long cptr, boolean mem_own, boolean weak_global); + + public static final native void ErrorCallback_change_ownership( + ErrorCallback obj, long cptr, boolean take_or_release); + + public static final native long new_MessageDispatcherHandler(); + + public static final native void delete_MessageDispatcherHandler(long jarg1); + + public static final native void MessageDispatcherHandler_onMessage( + long jarg1, MessageDispatcherHandler jarg1_, long jarg2, Message jarg2_); + + public static final native void MessageDispatcherHandler_director_connect( + MessageDispatcherHandler obj, long cptr, boolean mem_own, boolean weak_global); + + public static final native void MessageDispatcherHandler_change_ownership( + MessageDispatcherHandler obj, long cptr, boolean take_or_release); + + public static final native long new_SendResponseHandler(long jarg1); + + public static final native void delete_SendResponseHandler(long jarg1); + + public static final native void SendResponseHandler_sendResponse( + long jarg1, SendResponseHandler jarg1_, long jarg2); + + public static final native long new_IMessageHandler(); + + public static final native void delete_IMessageHandler(long jarg1); + + public static final native void IMessageHandler_onMessage( + long jarg1, + IMessageHandler jarg1_, + long jarg2, + Error jarg2_, + long jarg3, + Message jarg3_, + long jarg4, + SendResponseHandler jarg4_); + + public static final native void IMessageHandler_director_connect( + IMessageHandler obj, long cptr, boolean mem_own, boolean weak_global); + + public static final native void IMessageHandler_change_ownership( + IMessageHandler obj, long cptr, boolean take_or_release); + + public static final native void delete_IFront(long jarg1); + + public static final native void IFront_start(long jarg1, IFront jarg1_); + + public static final native void IFront_stop(long jarg1, IFront jarg1_); + + public static final native void IFront_registerTopicHandler__SWIG_0( + long jarg1, IFront jarg1_, String jarg2, long jarg3); + + public static final native void IFront_registerTopicHandler__SWIG_1( + long jarg1, IFront jarg1_, String jarg2, long jarg3, MessageDispatcherHandler jarg3_); + + public static final native void IFront_registerMessageHandler__SWIG_0( + long jarg1, IFront jarg1_, String jarg2, long jarg3); + + public static final native void IFront_registerMessageHandler__SWIG_1( + long jarg1, IFront jarg1_, String jarg2, long jarg3, MessageDispatcherHandler jarg3_); + + public static final native void IFront_asyncSendMessage__SWIG_0( + long jarg1, + IFront jarg1_, + int jarg2, + long jarg3, + MessageOptionalHeader jarg3_, + long jarg4, + ubytes jarg4_, + int jarg5, + int jarg6, + long jarg7, + long jarg8); + + public static final native void IFront_asyncSendMessage__SWIG_1( + long jarg1, + IFront jarg1_, + int jarg2, + long jarg3, + MessageOptionalHeader jarg3_, + byte[] jarg4, + java.math.BigInteger jarg5, + int jarg6, + int jarg7, + long jarg8, + ErrorCallback jarg8_, + long jarg9, + IMessageHandler jarg9_); + + public static final native void IFront_asyncSendResponse__SWIG_0( + long jarg1, + IFront jarg1_, + long jarg2, + ubytes jarg2_, + String jarg3, + long jarg4, + ubytes jarg4_, + int jarg5, + long jarg6); + + public static final native void IFront_asyncSendResponse__SWIG_1( + long jarg1, + IFront jarg1_, + long jarg2, + ubytes jarg2_, + String jarg3, + long jarg4, + ubytes jarg4_, + int jarg5, + long jarg6, + ErrorCallback jarg6_); + + public static final native long IFront_push__SWIG_0( + long jarg1, + IFront jarg1_, + int jarg2, + long jarg3, + MessageOptionalHeader jarg3_, + long jarg4, + ubytes jarg4_, + int jarg5, + int jarg6); + + public static final native long IFront_push__SWIG_1( + long jarg1, + IFront jarg1_, + int jarg2, + long jarg3, + MessageOptionalHeader jarg3_, + byte[] jarg4, + java.math.BigInteger jarg5, + int jarg6, + int jarg7); + + public static final native long IFront_pop(long jarg1, IFront jarg1_, String jarg2, int jarg3); + + public static final native long IFront_peek(long jarg1, IFront jarg1_, String jarg2); + + public static final native void IFront_asyncGetAgencies(long jarg1, IFront jarg1_, long jarg2); + + public static final native long IFront_registerNodeInfo(long jarg1, IFront jarg1_, long jarg2); + + public static final native long IFront_unRegisterNodeInfo(long jarg1, IFront jarg1_); + + public static final native long IFront_registerTopic(long jarg1, IFront jarg1_, String jarg2); + + public static final native long IFront_unRegisterTopic(long jarg1, IFront jarg1_, String jarg2); + + public static final native void delete_IFrontBuilder(long jarg1); + + public static final native long IFrontBuilder_buildClient( + long jarg1, IFrontBuilder jarg1_, String jarg2, long jarg3, boolean jarg4); + + public static final native int SDKMode_AIR_get(); + + public static final native int SDKMode_PRO_get(); + + public static final native long new_TransportBuilder(); + + public static final native void delete_TransportBuilder(long jarg1); + + public static final native long TransportBuilder_build( + long jarg1, + TransportBuilder jarg1_, + int jarg2, + long jarg3, + FrontConfig jarg3_, + long jarg4); + + public static final native long TransportBuilder_buildProTransport( + long jarg1, TransportBuilder jarg1_, long jarg2, FrontConfig jarg2_); + + public static final native long TransportBuilder_buildConfig( + long jarg1, TransportBuilder jarg1_, int jarg2, String jarg3); + + public static final native long TransportBuilder_frontConfigBuilder( + long jarg1, TransportBuilder jarg1_); + + public static final native long new_Transport(long jarg1, FrontConfig jarg1_); + + public static final native void delete_Transport(long jarg1); + + public static final native void Transport_start(long jarg1, Transport jarg1_); + + public static final native void Transport_stop(long jarg1, Transport jarg1_); + + public static final native long Transport_getFront(long jarg1, Transport jarg1_); + + public static final native long Transport_gateway(long jarg1, Transport jarg1_); + + public static final native long Transport_msgPayloadBuilder(long jarg1, Transport jarg1_); + + public static final native long Transport_msgBuilder(long jarg1, Transport jarg1_); + + public static final native long Transport_msgHeaderBuilder(long jarg1, Transport jarg1_); + + public static final native long Transport_routeInfoBuilder(long jarg1, Transport jarg1_); + + public static final native long Transport_getConfig(long jarg1, Transport jarg1_); + + public static final native long IFront_SWIGSmartPtrUpcast(long jarg1); + + public static void SwigDirector_ErrorCallback_onError(ErrorCallback jself, long error) { + jself.onError((error == 0) ? null : new Error(error, true)); + } + + public static void SwigDirector_MessageDispatcherHandler_onMessage( + MessageDispatcherHandler jself, long msg) { + jself.onMessage((msg == 0) ? null : new Message(msg, true)); + } + + public static void SwigDirector_IMessageHandler_onMessage( + IMessageHandler jself, long e, long msg, long sendResponseHandler) { + jself.onMessage( + (e == 0) ? null : new Error(e, true), + (msg == 0) ? null : new Message(msg, true), + new SendResponseHandler(sendResponseHandler, true)); + } + + private static final native void swig_module_init(); + + static { + swig_module_init(); + } +} From 97726a1ee0f8b51c0b078db303f134ef2e66412a Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 13 Sep 2024 11:23:35 +0800 Subject: [PATCH 037/120] implement transport based on the generated swig (#34) --- .../webank/wedpr/sdk/jni/common/Common.java | 36 ++ .../webank/wedpr/sdk/jni/common/Constant.java | 24 ++ .../sdk/jni/common/WeDPRSDKException.java | 41 +++ .../wedpr/sdk/jni/demo/TransportDemo.java | 121 +++++++ .../wedpr/sdk/jni/transport/IMessage.java | 58 +++ .../sdk/jni/transport/IMessageBuilder.java | 25 ++ .../sdk/jni/transport/TransportConfig.java | 127 +++++++ .../sdk/jni/transport/WeDPRTransport.java | 152 ++++++++ .../transport/handlers/MessageCallback.java | 33 ++ .../handlers/MessageDispatcherCallback.java | 30 ++ .../sdk/jni/transport/impl/MessageImpl.java | 154 ++++++++ .../sdk/jni/transport/impl/RouteType.java | 23 ++ .../sdk/jni/transport/impl/TransportImpl.java | 329 ++++++++++++++++++ .../transport/model/TransportEndPoint.java | 75 ++++ .../transport/model/TransportGrpcConfig.java | 62 ++++ 15 files changed, 1290 insertions(+) create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/WeDPRSDKException.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessage.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/RouteType.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportEndPoint.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportGrpcConfig.java diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java new file mode 100644 index 00000000..64bb8321 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java @@ -0,0 +1,36 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.common; + +import com.webank.wedpr.sdk.jni.generated.*; +import com.webank.wedpr.sdk.jni.generated.Error; + +public class Common { + public static void requireNotNull(String objectName, Object object) throws WeDPRSDKException { + if (object == null) { + throw new WeDPRSDKException("The object " + objectName + " must be not null!"); + } + } + + public static void checkResult(String interfaceName, Error error) throws WeDPRSDKException { + if (error == null || error.errorCode() == 0) { + return; + } + throw new WeDPRSDKException( + error.errorCode(), + "call " + interfaceName + " failed for: " + error.errorMessage()); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java new file mode 100644 index 00000000..df0ec8e2 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.common; + +public class Constant { + + public static int SUCCESS = 0; + public static int FAILED = -1; + + public static String SUCCESS_MSG = "success"; +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/WeDPRSDKException.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/WeDPRSDKException.java new file mode 100644 index 00000000..29515660 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/WeDPRSDKException.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.common; + +import lombok.Getter; + +@Getter +public class WeDPRSDKException extends Exception { + + private long code; + + public WeDPRSDKException(long code, String message) { + super(message); + this.code = code; + } + + public WeDPRSDKException(String message) { + super(message); + } + + public WeDPRSDKException(Throwable cause) { + super(cause); + } + + public WeDPRSDKException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java new file mode 100644 index 00000000..b753e076 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -0,0 +1,121 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.demo; + +import com.webank.wedpr.sdk.jni.generated.*; +import com.webank.wedpr.sdk.jni.generated.Error; +import com.webank.wedpr.sdk.jni.transport.IMessage; +import com.webank.wedpr.sdk.jni.transport.TransportConfig; +import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; +import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl; +import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint; + +public class TransportDemo { + public static class MessageDispatcherCallbackImpl extends MessageDispatcherCallback { + private final String nodeID; + + public MessageDispatcherCallbackImpl(String nodeID) { + this.nodeID = nodeID; + } + + @Override + public void onMessage(IMessage message) { + System.out.println( + "##### Node: " + + nodeID + + " receiveMessage, detail: " + + message.toString() + + ", "); + } + } + + public static class MessageErrorCallback extends ErrorCallback { + private final String nodeID; + + public MessageErrorCallback(String nodeID) { + this.nodeID = nodeID; + } + + @Override + public void onError(Error error) { + System.out.println( + "##### Node: " + + nodeID + + " MessageErrorCallback, result: " + + error.errorMessage() + + ", code:" + + error.errorCode()); + } + } + + public static void main(String[] args) throws Exception { + String nodeID = "testNode"; + if (args.length > 1) { + nodeID = args[0]; + } + TransportConfig transportConfig = new TransportConfig(2, nodeID); + String hostIp = "127.0.0.1"; + if (args.length > 2) { + hostIp = args[1]; + } + int listenPort = 9020; + if (args.length > 3) { + listenPort = Integer.valueOf(args[2]); + } + String listenIp = "0.0.0.0"; + TransportEndPoint endPoint = new TransportEndPoint(hostIp, listenIp, listenPort); + transportConfig.setSelfEndPoint(endPoint); + String grpcTarget = "ipv4:127.0.0.1:40600,127.0.0.1:40601"; + if (args.length > 4) { + grpcTarget = args[3]; + } + transportConfig.setGatewayTargets(grpcTarget); + System.out.println("####### transportConfig: " + transportConfig.toString()); + // build the gatewayTarget + WeDPRTransport transport = TransportImpl.build(transportConfig); + + System.out.println("####### before start the transport"); + transport.start(); + System.out.println("####### start the transport success"); + + // send Message to the gateway + String topic = "testTopic"; + transport.registerTopicHandler(topic, new MessageDispatcherCallbackImpl(nodeID)); + System.out.println("##### register topic success"); + String dstNode = "agency2Node"; + if (args.length > 5) { + dstNode = args[5]; + } + + byte[] dstNodeBytes = dstNode.getBytes(); + // every 2s send a message + Integer i = 0; + while (true) { + String payLoad = "testPayload" + i; + transport.asyncSendMessageByNodeID( + topic, + dstNodeBytes, + payLoad.getBytes(), + 0, + 10000, + new MessageErrorCallback(nodeID), + null); + i++; + Thread.sleep(2000); + } + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessage.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessage.java new file mode 100644 index 00000000..36e04600 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessage.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport; + +public interface IMessage { + public interface IMessageHeader { + int getVersion(); + + String getTraceID(); + + String getSrcGwNode(); + + String getDstGwNode(); + + int getPacketType(); + + int getTTL(); + + int getExt(); + + boolean isRespPacket(); + + int getRouteType(); + + String getComponentType(); + + byte[] getSrcNode(); + + byte[] getDstNode(); + + String getDstInst(); + + String getSrcInst(); + + String getTopic(); + } + + IMessageHeader getHeader(); + + Integer getSeq(); + + boolean isResponsePacket(); + + byte[] getPayload(); +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java new file mode 100644 index 00000000..c97bef67 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport; + +import com.webank.wedpr.sdk.jni.generated.Message; +import com.webank.wedpr.sdk.jni.transport.impl.MessageImpl; + +public class IMessageBuilder { + public static IMessage build(Message msg) { + return new MessageImpl(msg); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java new file mode 100644 index 00000000..e56d78bc --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java @@ -0,0 +1,127 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport; + +import com.webank.wedpr.sdk.jni.common.JniLibLoader; +import com.webank.wedpr.sdk.jni.generated.FrontConfig; +import com.webank.wedpr.sdk.jni.generated.TransportBuilder; +import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint; +import com.webank.wedpr.sdk.jni.transport.model.TransportGrpcConfig; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** the transport config used to build the transport */ +public class TransportConfig { + private static final Logger logger = LoggerFactory.getLogger(TransportConfig.class); + private static TransportBuilder transportBuilder; + + static { + // load the jni + JniLibLoader.loadJniLibrary(); + logger.info("loadJniLibrary success"); + createTransportBuilder(); + logger.info("createTransportBuilder success"); + } + + private static synchronized void createTransportBuilder() { + logger.info("init transportBuilder"); + transportBuilder = new TransportBuilder(); + logger.info("init transportBuilder success"); + } + + public static TransportBuilder getTransportBuilder() { + return transportBuilder; + } + + private final FrontConfig frontConfig; + private List components; + private TransportEndPoint selfEndPoint; + + public TransportConfig(Integer threadPoolSize, String nodeID) { + this.frontConfig = transportBuilder.buildConfig(threadPoolSize, nodeID); + // set default grpcConfig + setGrpcConfig(new TransportGrpcConfig()); + } + + public void setSelfEndPoint(TransportEndPoint endPoint) { + if (endPoint == null) { + return; + } + this.selfEndPoint = endPoint; + this.frontConfig.setSelfEndPoint(endPoint.getEndPoint()); + } + + public void setGatewayTargets(String gatewayTargets) { + this.frontConfig.setGatewayGrpcTarget(gatewayTargets); + } + + public void setGrpcConfig(TransportGrpcConfig grpcConfig) { + this.frontConfig.setGrpcConfig(grpcConfig.getGrpcConfig()); + } + + public void setComponents(List components) { + if (components == null || components.isEmpty()) { + return; + } + this.components = components; + for (String component : components) { + this.frontConfig.addComponent(component); + } + } + + public FrontConfig getFrontConfig() { + return frontConfig; + } + + public List getComponents() { + return components; + } + + public TransportEndPoint getSelfEndPoint() { + return selfEndPoint; + } + + public Integer getThreadPoolSize() { + return frontConfig.threadPoolSize(); + } + + public String getNodeID() { + return frontConfig.nodeID(); + } + + public String getGatewayTargets() { + return frontConfig.gatewayGrpcTarget(); + } + + @Override + public String toString() { + return "TransportConfig{" + + "components=" + + components + + ", selfEndPoint=" + + selfEndPoint + + ", threadPoolSize=" + + getThreadPoolSize() + + ", nodeID='" + + getNodeID() + + '\'' + + ", gatewayTargets='" + + getGatewayTargets() + + '\'' + + '}'; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java new file mode 100644 index 00000000..f8431244 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java @@ -0,0 +1,152 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport; + +import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; +import com.webank.wedpr.sdk.jni.generated.ErrorCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; + +public interface WeDPRTransport { + // start the transport + void start(); + // stop the transport + void stop(); + /** + * register the component + * + * @param component the component used to router + * @throws Exception failed case + */ + void registerComponent(String component) throws Exception; + + /** + * unregister the component + * + * @param component the component used to route + * @throws Exception failed case + */ + void unRegisterComponent(String component) throws Exception; + + /** + * register the topic + * + * @param topic the topic used to route + * @throws Exception failed case + */ + void registerTopic(String topic) throws Exception; + + /** + * unRegister the topic + * + * @param topic the topic used to route + * @throws Exception failed case + */ + void unRegisterTopic(String topic) throws Exception; + + //// the async interfaces + /** + * register the message handler + * + * @param topic the topic + * @param messageHandler the message handler + */ + void registerTopicHandler(String topic, MessageDispatcherCallback messageHandler); + + /** + * async send message by the nodeID + * + * @param topic the topic + * @param dstNode the dstNode + * @param payload the payload + * @param seq the seq of the payload + * @param timeout the timeout setting + * @param errorCallback the handler called after receive the message related to the topic + */ + void asyncSendMessageByNodeID( + String topic, + byte[] dstNode, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback); + + /** + * send message by the agency + * + * @param topic the topic + * @param agency the agency + * @param payload the payload + * @param seq the seq + * @param timeout the timeout + * @param errorCallback the handler called after receive the message related to the topic + */ + void asyncSendMessageByAgency( + String topic, + String agency, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback); + + void asyncSendMessageByComponent( + String topic, + String dstInst, + String component, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback); + + /** + * send message by the topic(will register firstly) + * + * @param topic the topic(used to route too + * @param payload the payload(the payload) + * @param seq the seq(the seq) + * @param timeout the timeout + * @param errorCallback the handler + */ + void asyncSendMessageByTopic( + String topic, + String dstInst, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback); + + /** @param topic the topic to remove */ + void removeTopic(String topic) throws WeDPRSDKException; + + //// the sync interfaces + void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload, int timeout) + throws WeDPRSDKException; + + void pushByComponent( + String topic, String dstInst, String component, int seq, byte[] payload, int timeout) + throws WeDPRSDKException; + + void pushByInst(String topic, String dstInst, int seq, byte[] payload, int timeout) + throws WeDPRSDKException; + + IMessage pop(String topic, int timeout) throws WeDPRSDKException; + + IMessage peek(String topic); +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java new file mode 100644 index 00000000..20473f7c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.handlers; + +import com.webank.wedpr.sdk.jni.generated.Error; +import com.webank.wedpr.sdk.jni.generated.IMessageHandler; +import com.webank.wedpr.sdk.jni.generated.Message; +import com.webank.wedpr.sdk.jni.generated.SendResponseHandler; +import com.webank.wedpr.sdk.jni.transport.IMessage; +import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; + +public abstract class MessageCallback extends IMessageHandler { + public abstract void onMessage( + Error error, IMessage message, SendResponseHandler sendResponseHandler); + + @Override + public void onMessage(Error e, Message msg, SendResponseHandler sendResponseHandler) { + onMessage(e, IMessageBuilder.build(msg), sendResponseHandler); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java new file mode 100644 index 00000000..1f02bf5c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.handlers; + +import com.webank.wedpr.sdk.jni.generated.Message; +import com.webank.wedpr.sdk.jni.generated.MessageDispatcherHandler; +import com.webank.wedpr.sdk.jni.transport.IMessage; +import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; + +public abstract class MessageDispatcherCallback extends MessageDispatcherHandler { + public abstract void onMessage(IMessage message); + + @Override + public void onMessage(Message msg) { + onMessage(IMessageBuilder.build(msg)); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java new file mode 100644 index 00000000..ebbaed1d --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java @@ -0,0 +1,154 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.impl; + +import com.webank.wedpr.sdk.jni.generated.Message; +import com.webank.wedpr.sdk.jni.generated.MessageHeader; +import com.webank.wedpr.sdk.jni.transport.IMessage; + +public class MessageImpl implements IMessage { + public class MessageHeaderImpl implements IMessageHeader { + private final MessageHeader messageHeader; + + protected MessageHeaderImpl(MessageHeader messageHeader) { + this.messageHeader = messageHeader; + } + + @Override + public int getVersion() { + return this.messageHeader.version(); + } + + @Override + public String getTraceID() { + return this.messageHeader.traceID(); + } + + @Override + public String getSrcGwNode() { + return this.messageHeader.srcGwNode(); + } + + @Override + public String getDstGwNode() { + return this.messageHeader.dstGwNode(); + } + + @Override + public int getPacketType() { + return this.messageHeader.packetType(); + } + + @Override + public int getTTL() { + return this.messageHeader.ttl(); + } + + @Override + public int getExt() { + return this.messageHeader.ext(); + } + + @Override + public boolean isRespPacket() { + return this.messageHeader.isRespPacket(); + } + + @Override + public int getRouteType() { + return this.messageHeader.routeType(); + } + + @Override + public String getComponentType() { + return this.messageHeader.optionalField().componentType(); + } + + @Override + public byte[] getSrcNode() { + return this.messageHeader.optionalField().srcNodeBuffer(); + } + + @Override + public byte[] getDstNode() { + return this.messageHeader.optionalField().dstNodeBuffer(); + } + + @Override + public String getDstInst() { + return this.messageHeader.optionalField().dstInst(); + } + + @Override + public String getSrcInst() { + return this.messageHeader.optionalField().srcInst(); + } + + @Override + public String getTopic() { + return this.messageHeader.optionalField().topic(); + } + + @Override + public String toString() { + return "MessageHeaderImpl{" + + "traceID=" + + getTraceID() + + ", srcInst=" + + getSrcInst() + + ", dstInst=" + + getDstInst() + + ", topic=" + + getTopic() + + ", response=" + + isResponsePacket() + + '}'; + } + } + + private final MessageHeaderImpl messageHeader; + private final Message rawMessage; + // v1, v2 + public MessageImpl(Message rawMessage) { + this.rawMessage = rawMessage; + this.messageHeader = new MessageHeaderImpl(rawMessage.header()); + } + + @Override + public IMessageHeader getHeader() { + return this.messageHeader; + } + + @Override + public Integer getSeq() { + return this.rawMessage.frontMessage().seq(); + } + + @Override + public byte[] getPayload() { + return this.rawMessage.frontMessage().dataBuffer(); + } + + @Override + public boolean isResponsePacket() { + return this.rawMessage.frontMessage().isRespPacket(); + } + + @Override + public String toString() { + return "MessageImpl{" + "messageHeader=" + messageHeader + '}'; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/RouteType.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/RouteType.java new file mode 100644 index 00000000..ab6ccb94 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/RouteType.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.impl; + +public enum RouteType { + ROUTE_THROUGH_NODEID, + ROUTE_THROUGH_COMPONENT, + ROUTE_THROUGH_AGENCY, + ROUTE_THROUGH_TOPIC; +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java new file mode 100644 index 00000000..142cf225 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -0,0 +1,329 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.impl; + +import com.webank.wedpr.sdk.jni.common.Common; +import com.webank.wedpr.sdk.jni.common.Constant; +import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; +import com.webank.wedpr.sdk.jni.generated.*; +import com.webank.wedpr.sdk.jni.generated.Error; +import com.webank.wedpr.sdk.jni.transport.IMessage; +import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; +import com.webank.wedpr.sdk.jni.transport.TransportConfig; +import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; +import java.math.BigInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TransportImpl implements WeDPRTransport { + private static Logger logger = LoggerFactory.getLogger(TransportImpl.class); + // the created transport + private final Transport transport; + private final TransportConfig transportConfig; + + public static WeDPRTransport build(TransportConfig transportConfig) { + return new TransportImpl( + TransportConfig.getTransportBuilder() + .buildProTransport(transportConfig.getFrontConfig()), + transportConfig); + } + + protected TransportImpl(Transport transport, TransportConfig transportConfig) { + logger.info("Build Transport, config: {}", transportConfig.toString()); + this.transport = transport; + this.transportConfig = transportConfig; + } + + @Override + public void start() { + logger.info("start the transport"); + this.transport.start(); + } + + @Override + public void stop() { + logger.info("stop the transport"); + this.transport.stop(); + } + + /** + * TODO: register the component + * + * @param component the component used to router + * @throws Exception failed case + */ + @Override + public void registerComponent(String component) throws Exception { + // Note: the front must exist after Transport created + } + + /** + * TODO: unregister the component + * + * @param component the component used to route + * @throws Exception failed case + */ + @Override + public void unRegisterComponent(String component) throws Exception {} + + /** + * register the topic + * + * @param topic the topic used to route + * @throws Exception failed case + */ + @Override + public void registerTopic(String topic) throws Exception { + this.transport.getFront().registerTopic(topic); + } + + /** + * unRegister the topic + * + * @param topic the topic used to route + * @throws Exception failed case + */ + @Override + public void unRegisterTopic(String topic) throws Exception { + this.transport.getFront().unRegisterTopic(topic); + } + + @Override + public void registerTopicHandler(String topic, MessageDispatcherCallback messageHandler) { + this.transport.getFront().registerTopicHandler(topic, messageHandler); + } + + /** + * async send message by the nodeID + * + * @param topic the topic + * @param dstNode the dstNode + * @param payload the payload + * @param seq the seq of the payload + * @param timeout the timeout setting + * @param errorCallback the handler called after receive the message related to the topic + */ + @Override + public void asyncSendMessageByNodeID( + String topic, + byte[] dstNode, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback) { + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setDstNode(dstNode, BigInteger.valueOf(dstNode.length)); + routeInfo.setTopic(topic); + this.transport + .getFront() + .asyncSendMessage( + RouteType.ROUTE_THROUGH_NODEID.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout, + errorCallback, + msgCallback); + } + + /** + * send message by the agency + * + * @param topic the topic + * @param agency the agency + * @param payload the payload + * @param seq the seq + * @param timeout the timeout + * @param errorCallback the handler called after receive the message related to the topic + */ + @Override + public void asyncSendMessageByAgency( + String topic, + String agency, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback) { + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstInst(agency); + this.transport + .getFront() + .asyncSendMessage( + RouteType.ROUTE_THROUGH_AGENCY.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout, + errorCallback, + msgCallback); + } + + @Override + public void asyncSendMessageByComponent( + String topic, + String dstInst, + String component, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback) { + // set the routeInfo + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstInst(dstInst); + routeInfo.setComponentType(component); + this.transport + .getFront() + .asyncSendMessage( + RouteType.ROUTE_THROUGH_COMPONENT.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout, + errorCallback, + msgCallback); + } + + /** + * send message by the topic(will register firstly) + * + * @param topic the topic(used to route too + * @param payload the payload(the payload) + * @param seq the seq(the seq) + * @param timeout the timeout + * @param errorCallback the handler + */ + @Override + public void asyncSendMessageByTopic( + String topic, + String dstInst, + byte[] payload, + int seq, + int timeout, + ErrorCallback errorCallback, + MessageCallback msgCallback) { + // set the routeInfo + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstInst(dstInst); + this.transport + .getFront() + .asyncSendMessage( + RouteType.ROUTE_THROUGH_TOPIC.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout, + errorCallback, + msgCallback); + } + + /** @param topic the topic to remove */ + @Override + public void removeTopic(String topic) throws WeDPRSDKException { + Error result = this.transport.getFront().unRegisterTopic(topic); + Common.checkResult("removeTopic", result); + } + + //// the sync interfaces + @Override + public void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload, int timeout) + throws WeDPRSDKException { + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstNode(dstNodeID, BigInteger.valueOf(dstNodeID.length)); + Error result = + this.transport + .getFront() + .push( + RouteType.ROUTE_THROUGH_NODEID.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout); + Common.checkResult("pushByNodeID", result); + } + + @Override + public void pushByComponent( + String topic, String dstInst, String component, int seq, byte[] payload, int timeout) + throws WeDPRSDKException { + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstInst(dstInst); + routeInfo.setComponentType(component); + Error result = + this.transport + .getFront() + .push( + RouteType.ROUTE_THROUGH_COMPONENT.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout); + Common.checkResult("pushByComponent", result); + } + + @Override + public void pushByInst(String topic, String dstInst, int seq, byte[] payload, int timeout) + throws WeDPRSDKException { + MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + routeInfo.setTopic(topic); + routeInfo.setDstInst(dstInst); + Error result = + this.transport + .getFront() + .push( + RouteType.ROUTE_THROUGH_TOPIC.ordinal(), + routeInfo, + payload, + BigInteger.valueOf(payload.length), + seq, + timeout); + Common.checkResult("pushByInst", result); + } + + @Override + public IMessage pop(String topic, int timeout) throws WeDPRSDKException { + Message msg = this.transport.getFront().pop(topic, timeout); + if (msg == null) { + throw new WeDPRSDKException( + Constant.FAILED, "Try to receive msg with topic " + topic + " timeout!"); + } + return IMessageBuilder.build(msg); + } + + @Override + public IMessage peek(String topic) { + Message msg = this.transport.getFront().peek(topic); + if (msg == null) { + return null; + } + return IMessageBuilder.build(msg); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportEndPoint.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportEndPoint.java new file mode 100644 index 00000000..b155e22d --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportEndPoint.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.model; + +import com.webank.wedpr.sdk.jni.generated.EndPoint; + +public class TransportEndPoint { + private final EndPoint endPoint; + + public TransportEndPoint(String hostIp, String listenIp, int listenPort) { + this.endPoint = new EndPoint(); + setHostIP(hostIp); + setListenIP(listenIp); + setListenPort(listenPort); + } + + public String getHostIP() { + return this.endPoint.host(); + } + + protected void setHostIP(String hostIP) { + this.endPoint.setHost(hostIP); + } + + public String getListenIP() { + return this.endPoint.listenIp(); + } + + protected void setListenIP(String listenIP) { + this.endPoint.setListenIp(listenIP); + } + + public Integer getListenPort() { + return this.endPoint.port(); + } + + protected void setListenPort(Integer listenPort) { + this.endPoint.setPort(listenPort); + } + + public EndPoint getEndPoint() { + return endPoint; + } + + public EndPoint getNativeEndPoint() { + return endPoint; + } + + @Override + public String toString() { + return "TransportEndPoint{" + + "hostIP='" + + getHostIP() + + '\'' + + ", listenIP='" + + getListenIP() + + '\'' + + ", listenPort=" + + getListenPort() + + '}'; + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportGrpcConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportGrpcConfig.java new file mode 100644 index 00000000..2c5060cf --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/TransportGrpcConfig.java @@ -0,0 +1,62 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.model; + +import com.webank.wedpr.sdk.jni.generated.*; + +public class TransportGrpcConfig { + private final GrpcConfig grpcConfig = new GrpcConfig(); + public static final String defaultLoadBalancePolicy = "round_robin"; + + public TransportGrpcConfig() { + grpcConfig.setEnableHealthCheck(true); + grpcConfig.setEnableDnslookup(false); + grpcConfig.setLoadBalancePolicy(defaultLoadBalancePolicy); + } + + public void setDefaultLoadBalancePolicy(String loadBalancePolicy) { + grpcConfig.setLoadBalancePolicy(loadBalancePolicy); + } + + public boolean isEnableHealthCheck() { + return grpcConfig.enableHealthCheck(); + } + + public boolean isEnableDnslookup() { + return grpcConfig.enableDnslookup(); + } + + public String getLoadBalancePolicy() { + return grpcConfig.loadBalancePolicy(); + } + + public GrpcConfig getGrpcConfig() { + return grpcConfig; + } + + @Override + public String toString() { + return "TransportGrpcConfig{" + + "enableHealthCheck=" + + isEnableHealthCheck() + + ", enableDnslookup=" + + isEnableDnslookup() + + ", loadBalancePolicy='" + + getLoadBalancePolicy() + + '\'' + + '}'; + } +} From af6f5d98eb81513d2560d96e0b6c1cf8326f8379 Mon Sep 17 00:00:00 2001 From: zachma <382847508@qq.com> Date: Fri, 13 Sep 2024 12:59:17 +0800 Subject: [PATCH 038/120] =?UTF-8?q?=E6=9B=B4=E6=96=B0pir=E7=AE=97=E6=B3=95?= =?UTF-8?q?=20(#35)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 更新PIR算法 * 更新PIR算法,隐藏部分敏感信息 * 清理ide自动编译class文件 * 清理所有.class * 更新PIR算法 * 更新PIR算法,隐藏部分敏感信息 * 清理ide自动编译class文件 * 清理所有.class --------- Co-authored-by: zachma(马晨) --- .gitignore | 3 + java/ppc-pir-services/build.gradle | 8 + .../main/java/com/wedpr/pir/demo/Demo.java | 20 +- .../java/com/wedpr/pir/sdk/PirClient.java | 40 +++- .../sdk/entity/body/ServiceConfigBody.java | 14 ++ .../pir/sdk/entity/param/PirJobParam.java | 30 +-- .../sdk/entity/request/ClientAuthRequest.java | 13 ++ .../entity/request/ClientDirectRequest.java | 16 ++ .../sdk/entity/request/PirBaseRequest.java | 13 +- .../sdk/entity/request/ServerJobRequest.java | 2 + .../sdk/entity/request/ServerOTRequest.java | 1 + .../entity/response/ClientAuthResponse.java | 15 ++ .../entity/response/PirResultResponse.java | 1 - .../com/wedpr/pir/sdk/enums/ParamEnum.java | 10 +- .../pir/sdk/exception/WedprStatusEnum.java | 1 + .../wedpr/pir/sdk/gateway/GatewayClient.java | 4 + .../com/wedpr/pir/sdk/helper/Constant.java | 17 ++ .../pir/sdk/helper/CryptoOperatorHelper.java | 5 +- .../wedpr/pir/sdk/service/PirJobService.java | 1 - .../webank/wedpr/pir/WedprPirApplication.java | 4 + .../wedpr/pir/crypto/entity/Dataset.java | 16 ++ .../pir/crypto/entity/DatasetStoragePath.java | 13 ++ .../pir/crypto/entity/WedprAuthTable.java | 17 ++ .../pir/crypto/service/PirTableService.java | 6 +- .../crypto/service/impl/PirOTServiceImpl.java | 10 +- .../service/impl/PirTableServiceImpl.java | 37 ++- .../wedpr/pir/http/common/CSVFileParser.java | 220 ++++++++++++++++++ .../wedpr/pir/http/common/Constant.java | 5 + .../wedpr/pir/http/config/HadoopConfig.java | 33 +++ .../pir/http/controller/PirController.java | 62 ++++- .../mapper/WedprServiceAuthTableMapper.java | 47 ++++ .../wedpr/pir/http/service/PirAppService.java | 4 +- .../pir/http/service/PirAuthService.java | 50 ++++ .../pir/http/service/PirDataService.java | 90 +++++++ .../main/resources/application-dev.properties | 7 +- .../resources/application-mysql.properties | 6 +- 36 files changed, 756 insertions(+), 85 deletions(-) create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java create mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java create mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java diff --git a/.gitignore b/.gitignore index 15029804..190391e3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ cmake-build-debug *.exe *.out *.app +**/*.class build/** GPATH @@ -43,6 +44,7 @@ GRTAGS GTAGS deps deps/** +**/bin d1-encrypted d2-encrypted @@ -137,6 +139,7 @@ ENV/ *.crt */bin */logs +*/bin *.csv */*/logs */start.out diff --git a/java/ppc-pir-services/build.gradle b/java/ppc-pir-services/build.gradle index ce80f537..095c7cd8 100644 --- a/java/ppc-pir-services/build.gradle +++ b/java/ppc-pir-services/build.gradle @@ -26,8 +26,16 @@ dependencies { implementation 'org.mybatis.spring.boot:mybatis-spring-boot-starter:2.2.2' implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.19.0' implementation 'commons-codec:commons-codec:1.15' + implementation("org.apache.hadoop:hadoop-common:2.7.5"){ + exclude group: 'javax.servlet', module: 'servlet-api' + } + implementation("org.apache.hadoop:hadoop-client:2.7.5"){ + exclude group: 'javax.servlet', module: 'servlet-api' + } + implementation 'com.opencsv:opencsv:5.9' implementation 'com.squareup.okhttp3:okhttp:4.9.1' implementation project(":ppc-pir-sdk") + implementation 'com.google.code.gson:gson:2.8.8' compileOnly 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java index f88a121b..fd1ad363 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java @@ -19,14 +19,12 @@ public static void main(String[] args) throws WedprException { private static void testIdFilter() throws WedprException { PirJobParam pirJobParam = new PirJobParam(); - pirJobParam.setJobId("j-12345678912"); + pirJobParam.setServiceId("s-123456789"); + pirJobParam.setAccessKeyId("a-123456789"); pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idFilter.getValue()); - pirJobParam.setParticipateAgencyId("1002"); - pirJobParam.setDatasetId("t_login_token"); - pirJobParam.setJobCreator("1001"); - pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); - pirJobParam.setSearchIdList(Arrays.asList("1", "20", "3", "456")); + pirJobParam.setGatewayUrl("localhost:5831"); + pirJobParam.setSearchIdList(Arrays.asList("1", "2", "3")); PirClient pirClient = new PirClient(pirJobParam); PirResultResponse pirResultResponse = pirClient.executePirJob(); @@ -35,14 +33,12 @@ private static void testIdFilter() throws WedprException { private static void testIdObfuscation() throws WedprException { PirJobParam pirJobParam = new PirJobParam(); - pirJobParam.setJobId("j-12345678912"); + pirJobParam.setServiceId("s-123456789"); + pirJobParam.setAccessKeyId("a-123456789"); pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idObfuscation.getValue()); - pirJobParam.setParticipateAgencyId("1002"); - pirJobParam.setDatasetId("t_login_token"); - pirJobParam.setJobCreator("1001"); - pirJobParam.setGatewayUrl("http://localhost:5831/api/pir/v3/server"); - pirJobParam.setSearchIdList(Arrays.asList("1", "20", "3", "456")); + pirJobParam.setGatewayUrl("localhost:5831"); + pirJobParam.setSearchIdList(Arrays.asList("3", "2", "1")); PirClient pirClient = new PirClient(pirJobParam); PirResultResponse pirResultResponse = pirClient.executePirJob(); diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java index ecf6ba75..41da2bd7 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java @@ -1,14 +1,19 @@ package com.wedpr.pir.sdk; import com.fasterxml.jackson.databind.ObjectMapper; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; import com.wedpr.pir.sdk.entity.body.SimpleEntity; import com.wedpr.pir.sdk.entity.param.PirJobParam; +import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; import com.wedpr.pir.sdk.entity.request.ServerJobRequest; +import com.wedpr.pir.sdk.entity.response.ClientAuthResponse; import com.wedpr.pir.sdk.entity.response.ClientOTResponse; import com.wedpr.pir.sdk.entity.response.PirResultResponse; +import com.wedpr.pir.sdk.enums.ParamEnum; import com.wedpr.pir.sdk.exception.WedprException; import com.wedpr.pir.sdk.exception.WedprStatusEnum; import com.wedpr.pir.sdk.gateway.GatewayClient; +import com.wedpr.pir.sdk.helper.Constant; import com.wedpr.pir.sdk.service.PirJobService; import java.util.Objects; @@ -26,13 +31,33 @@ public PirClient(PirJobParam pirJobParam) throws WedprException { public PirResultResponse executePirJob() throws WedprException { try { - ClientOTResponse otParamResponse = pirJobService.requesterOtCipher(); + ObjectMapper objectMapper = new ObjectMapper(); + // 认证 PirJobParam pirJobParam = pirJobService.getPirJobParam(); + ServiceConfigBody serviceConfigBody = pirJobParam.getServiceConfigBody(); + Integer invokeType = pirJobParam.getPirInvokeType(); + if (invokeType.equals(ParamEnum.JobMode.SDKMode.getValue())) { + ClientAuthRequest clientAuthRequest = new ClientAuthRequest(); + clientAuthRequest.setAccessKeyId(pirJobParam.getAccessKeyId()); + clientAuthRequest.setServiceId(pirJobParam.getServiceId()); + String authUrl = pirJobParam.getGatewayUrl() + Constant.PERMISSION; + String body = objectMapper.writeValueAsString(clientAuthRequest); + ClientAuthResponse authResult = + GatewayClient.sendPostRequestWithRetry( + authUrl, body, ClientAuthResponse.class); + if (!authResult.getCode().equals(WedprStatusEnum.SUCCESS.getCode())) { + throw new WedprException( + WedprStatusEnum.CALL_PSI_ERROR, "认证失败：" + authResult.getMessage()); + } + pirJobParam.setServiceConfigBody(authResult.getData()); + } + + // 执行任务 + ClientOTResponse otParamResponse = pirJobService.requesterOtCipher(); ServerJobRequest serverJobRequest = generatePirJobRequestFromJobAndOTParam(pirJobParam, otParamResponse); - String pirUrl = pirJobParam.getGatewayUrl(); - ObjectMapper objectMapper = new ObjectMapper(); + String pirUrl = pirJobParam.getGatewayUrl() + Constant.SERVER; String body = objectMapper.writeValueAsString(serverJobRequest); SimpleEntity otResult = GatewayClient.sendPostRequestWithRetry(pirUrl, body, SimpleEntity.class); @@ -42,7 +67,7 @@ public PirResultResponse executePirJob() throws WedprException { } return pirJobService.requesterOtRecover(otParamResponse, pirJobParam, otResult); } catch (Exception e) { - throw new WedprException(WedprStatusEnum.CALL_PSI_ERROR); + throw new WedprException(WedprStatusEnum.CALL_PSI_ERROR, e.getMessage()); } } @@ -50,10 +75,9 @@ public PirResultResponse executePirJob() throws WedprException { private ServerJobRequest generatePirJobRequestFromJobAndOTParam( PirJobParam pirJobParam, ClientOTResponse otParamResponse) { ServerJobRequest serverJobRequest = new ServerJobRequest(); - serverJobRequest.setJobId(pirJobParam.getJobId()); - serverJobRequest.setParticipateAgencyId(pirJobParam.getParticipateAgencyId()); - serverJobRequest.setDatasetId(pirJobParam.getDatasetId()); - serverJobRequest.setJobCreator(pirJobParam.getJobCreator()); + serverJobRequest.setServiceId(pirJobParam.getServiceId()); + serverJobRequest.setServiceConfigBody(pirJobParam.getServiceConfigBody()); + serverJobRequest.setAccessKeyId(pirJobParam.getAccessKeyId()); serverJobRequest.setJobAlgorithmType(pirJobParam.getJobAlgorithmType()); serverJobRequest.setX(otParamResponse.getX()); serverJobRequest.setY(otParamResponse.getY()); diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java new file mode 100644 index 00000000..5ffbd332 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java @@ -0,0 +1,14 @@ +package com.wedpr.pir.sdk.entity.body; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Data +public class ServiceConfigBody { + private String datasetId; + private String[] exists; + private String[] values; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java index 1812d9c4..4095dafc 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java @@ -1,5 +1,6 @@ package com.wedpr.pir.sdk.entity.param; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; import com.wedpr.pir.sdk.entity.request.PirBaseRequest; import com.wedpr.pir.sdk.enums.ParamEnum; import com.wedpr.pir.sdk.exception.WedprException; @@ -7,6 +8,7 @@ import com.wedpr.pir.sdk.helper.BasicTypeHelper; import java.util.Arrays; import java.util.List; +import java.util.Objects; import lombok.Data; import lombok.EqualsAndHashCode; @@ -24,13 +26,21 @@ public class PirJobParam extends PirBaseRequest { private List searchIdList; + private ServiceConfigBody serviceConfigBody; + public void check() throws WedprException { if (BasicTypeHelper.isStringEmpty(gatewayUrl)) { throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "访问gatewayUrl不能为空"); } - if (BasicTypeHelper.isStringEmpty(getJobId())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobId 不能为空"); + if (getPirInvokeType().equals(ParamEnum.JobMode.SDKMode.getValue()) + && BasicTypeHelper.isStringEmpty(getServiceId())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "serviceId 不能为空"); + } + + if (getPirInvokeType().equals(ParamEnum.JobMode.SDKMode.getValue()) + && BasicTypeHelper.isStringEmpty(getAccessKeyId())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "accessKeyId 不能为空"); } if (Arrays.stream(ParamEnum.JobType.values()) @@ -38,28 +48,20 @@ public void check() throws WedprException { throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobType输入错误"); } - if (BasicTypeHelper.isStringEmpty(getParticipateAgencyId())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "数据机构Id 不能为空"); - } - - if (BasicTypeHelper.isStringEmpty(getDatasetId())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "datasetId 不能为空"); - } - if (Arrays.stream(ParamEnum.AlgorithmType.values()) .noneMatch(enumValue -> enumValue.getValue().equals(getJobAlgorithmType()))) { throw new WedprException( WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobAlgorithmType输入错误"); } - if (searchIdList == null || searchIdList.size() == 0) { + if (Objects.isNull(searchIdList) || searchIdList.size() == 0) { throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "searchId列表不能为空"); } - if (getJobAlgorithmType().equals(ParamEnum.AlgorithmType.idObfuscation.getValue()) - && getObfuscationOrder() == -1) { + if (getPirInvokeType().equals(ParamEnum.JobMode.DirectorMode.getValue()) + && Objects.isNull(serviceConfigBody)) { throw new WedprException( - WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "哈希披露算法中, obfuscationOrder未设置"); + WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "向导模式下serviceConfigBody不能为空"); } } } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java new file mode 100644 index 00000000..0841ba6c --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java @@ -0,0 +1,13 @@ +package com.wedpr.pir.sdk.entity.request; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Data +public class ClientAuthRequest { + String serviceId; + String accessKeyId; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java new file mode 100644 index 00000000..9079ceb5 --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java @@ -0,0 +1,16 @@ +package com.wedpr.pir.sdk.entity.request; + +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; +import java.util.List; +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/5 + */ +@Data +public class ClientDirectRequest { + private String algorithmType; + private List searchIds; + private ServiceConfigBody serviceConfigBody; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java index 028db290..f38200b0 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java @@ -1,21 +1,22 @@ package com.wedpr.pir.sdk.entity.request; +import com.wedpr.pir.sdk.enums.ParamEnum; import lombok.Data; @Data public class PirBaseRequest { // 任务ID - String jobId; // 任务类型(0: 查询存在性, 1: 查询对应值) + String serviceId; + + String accessKeyId; + String jobType; // 数据方机构id - String participateAgencyId; - // 数据集id - String datasetId; - // 发起方用户名 - String jobCreator; // 匿踪算法类型(0: hash披露算法, 1: hash混淆算法) String jobAlgorithmType; // 查询范围 Integer obfuscationOrder = 9; + + Integer pirInvokeType = ParamEnum.JobMode.SDKMode.getValue(); } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java index 0f91aa94..296b00c1 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java @@ -1,6 +1,7 @@ package com.wedpr.pir.sdk.entity.request; import com.wedpr.pir.sdk.entity.body.PirDataBody; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; import java.math.BigInteger; import java.util.List; import lombok.Data; @@ -16,4 +17,5 @@ public class ServerJobRequest extends PirBaseRequest { BigInteger x; BigInteger y; List dataBodyList; + ServiceConfigBody serviceConfigBody; } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java index a1ad74de..99ebce12 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java @@ -13,6 +13,7 @@ public class ServerOTRequest { String jobType; String datasetId; + String[] params; BigInteger x; BigInteger y; List dataBodyList; diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java new file mode 100644 index 00000000..b2743f9a --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java @@ -0,0 +1,15 @@ +package com.wedpr.pir.sdk.entity.response; + +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Data +public class ClientAuthResponse { + String code; + String message; + ServiceConfigBody data; +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java index fd78b0df..1b250263 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java @@ -8,7 +8,6 @@ */ @Data public class PirResultResponse { - String jobId; String jobType; // List detail; ClientDecryptResponse detail; diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java index f3a2b321..27cdfa5f 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java @@ -29,7 +29,15 @@ public enum AlgorithmType { @AllArgsConstructor public enum HttpType { HttpTimeout(180), - HttpMaxRetries(4); + HttpMaxRetries(1); + private Integer value; + } + + @Getter + @AllArgsConstructor + public enum JobMode { + SDKMode(0), + DirectorMode(1); private Integer value; } } diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java index f1070a74..2313402d 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java @@ -34,6 +34,7 @@ public enum WedprStatusEnum { INVALID_TRANS_TYPE("2009", "不合法的交易类型"), INVALID_FILE_PATH("2010", "不合法交集文件路径"), INVALID_ROLE_VALUE("2011", "pir.Role设置错误"), + CSV_ERROR("2012", "csv解析是错误"), // 3000 db错误 DB_ERROR("3000", "DB查询失败"), diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java index ce58e8e2..c9cfaceb 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java @@ -47,6 +47,9 @@ private static String sendGetRequest(String url) throws IOException, WedprExcept } private static Response sendPostRequest(String url, String jsonBody) throws IOException { + if (!url.startsWith("http")) { + url = "http://" + url; + } OkHttpClient httpClient = new OkHttpClient().newBuilder().build(); RequestBody body = RequestBody.create(jsonBody, JSON_MEDIA_TYPE); Request request = new Request.Builder().post(body).url(url).build(); @@ -69,6 +72,7 @@ public static T sendPostRequestWithRetry(String url, String body, Class r } } catch (Exception e) { + e.printStackTrace(); retries++; if (retries > maxRetries) { throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java new file mode 100644 index 00000000..976252fd --- /dev/null +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java @@ -0,0 +1,17 @@ +package com.wedpr.pir.sdk.helper; +/** + * @author zachma + * @date 2024/9/3 + */ +public class Constant { + + public static final String SERVER = "/api/pir/v3/server"; + + public static final String PERMISSION = "/api/pir/v3/auth"; + + public static String PIR_TEMP_TABLE_PREFIX = "wedpr_pir_"; + + public static String datasetId2tableId(String datasetId) { + return PIR_TEMP_TABLE_PREFIX + datasetId.substring(2); + } +} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java index 51c89168..2bec5edf 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java @@ -2,7 +2,6 @@ import com.wedpr.pir.sdk.exception.WedprException; import com.wedpr.pir.sdk.exception.WedprStatusEnum; - import java.math.BigInteger; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -57,9 +56,7 @@ public static BigInteger OTMul(BigInteger a, BigInteger b) { return a.multiply(b).mod(DEFAULT_N); } - /** - * 批量获取searchID的Hash值 - * */ + /** 批量获取searchID的Hash值 */ public static List getIdHashVec(int obfuscationOrder, Integer idIndex, String searchId) throws WedprException { diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java index 5d6c0e1d..bdd97265 100644 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java +++ b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java @@ -56,7 +56,6 @@ public PirResultResponse requesterOtRecover( clientDecryptService.runDecryptOTparam(clientDecryptRequest); PirResultResponse pirResultResponse = new PirResultResponse(); - pirResultResponse.setJobId(clientJobRequest.getJobId()); pirResultResponse.setJobType(clientJobRequest.getJobType()); pirResultResponse.setDetail(clientDecryptResponse); return pirResultResponse; diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java index 9d016653..f2c23ed7 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java @@ -1,8 +1,10 @@ package com.webank.wedpr.pir; +import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.domain.EntityScan; /** * @author zachma @@ -10,6 +12,8 @@ */ @SpringBootApplication @EnableAutoConfiguration +@MapperScan("com.webank.wedpr.pir.http.mapper") +@EntityScan(basePackages = {"cn.webank.wedpr.http", "cn.webank.wedpr.pir"}) public class WedprPirApplication { public static void main(String[] args) { SpringApplication.run(WedprPirApplication.class, args); diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java new file mode 100644 index 00000000..4cb4f463 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java @@ -0,0 +1,16 @@ +package com.webank.wedpr.pir.crypto.entity; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/4 + */ +@Data +public class Dataset { + private String datasetId; + + private String datasetFields; + + private String datasetStoragePath; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java new file mode 100644 index 00000000..b6173658 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java @@ -0,0 +1,13 @@ +package com.webank.wedpr.pir.crypto.entity; + +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/4 + */ +@Data +public class DatasetStoragePath { + private String storageType; + private String filePath; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java new file mode 100644 index 00000000..85a347ec --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java @@ -0,0 +1,17 @@ +package com.webank.wedpr.pir.crypto.entity; + +import java.time.LocalDateTime; +import javax.persistence.Entity; +import lombok.Data; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Entity +@Data +public class WedprAuthTable { + private String serviceId; + private String accessKeyId; + private LocalDateTime expireTime; +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java index a70db535..a337c02d 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java @@ -9,11 +9,7 @@ */ public interface PirTableService { /** 根据id查询 */ - List idFilterTable(String datasetId, String filter); - - List idFilterTable(String datasetId, String filter, String[] params); - - List idObfuscationTable(String datasetId, String searchId); + List idFilterTable(String datasetId, String filter, String[] param); List idObfuscationTable(String datasetId, String searchId, String[] params); diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java index a61fa4c3..56a54e82 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java @@ -9,6 +9,7 @@ import com.wedpr.pir.sdk.entity.response.ServerOTResponse; import com.wedpr.pir.sdk.exception.WedprException; import com.wedpr.pir.sdk.helper.AESHelper; +import com.wedpr.pir.sdk.helper.Constant; import com.wedpr.pir.sdk.helper.ConvertTypeHelper; import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; import java.math.BigInteger; @@ -33,13 +34,15 @@ public ServerOTResponse runServerOTParam(ServerOTRequest serverOTRequest) throws WedprException { logger.info("Server start runServerOTParam."); String datasetId = serverOTRequest.getDatasetId(); + String tableId = Constant.datasetId2tableId(datasetId); BigInteger x = serverOTRequest.getX(); BigInteger y = serverOTRequest.getY(); + String[] params = serverOTRequest.getParams(); List serverResultlist = new ArrayList<>(); for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); String filter = serverOTRequest.getDataBodyList().get(i).getFilter(); - List objects = pirTableService.idFilterTable(datasetId, filter); + List objects = pirTableService.idFilterTable(tableId, filter, params); List pirTables = pirTableService.objectsToPirTableList(objects); List serverResultBodyTempList = new ArrayList<>(); for (PirTable pirTable : pirTables) { @@ -59,8 +62,9 @@ public ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) String datasetId = serverOTRequest.getDatasetId(); BigInteger x = serverOTRequest.getX(); BigInteger y = serverOTRequest.getY(); - + String tableId = Constant.datasetId2tableId(datasetId); List serverResultlist = new ArrayList<>(); + String[] params = serverOTRequest.getParams(); for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); List idHashList = serverOTRequest.getDataBodyList().get(i).getIdHashList(); @@ -68,7 +72,7 @@ public ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) for (int j = 0; j < idHashList.size(); j++) { List objects = - pirTableService.idObfuscationTable(datasetId, idHashList.get(j)); + pirTableService.idObfuscationTable(tableId, idHashList.get(j), params); List pirTables = pirTableService.objectsToPirTableList(objects); for (PirTable pirTable : pirTables) { ServerResultBody serverResultBody = diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java index 7d319b5f..ef38830a 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java @@ -19,42 +19,28 @@ public class PirTableServiceImpl implements PirTableService { @PersistenceContext private EntityManager entityManager; - @Override - public List idFilterTable(String datasetId, String filter) { - String sqlFormat = "SELECT * FROM %s WHERE id LIKE CONCAT(%s, '%%')"; - String sql = String.format(sqlFormat, datasetId, filter); - Query query = entityManager.createNativeQuery(sql); - return query.getResultList(); - } - @Override public List idFilterTable(String datasetId, String filter, String[] params) { - String sqlFormat = "SELECT %s FROM %s WHERE id LIKE CONCAT(%s, '%%')"; - String sql = String.format(sqlFormat, String.join(", ", params), datasetId, filter); - Query query = entityManager.createNativeQuery(sql); - return query.getResultList(); - } - - @Override - public List idObfuscationTable(String datasetId, String searchId) { - String sqlFormat = "SELECT * FROM %s WHERE id = %s"; - String sql = String.format(sqlFormat, datasetId, searchId); + String param = processParams(params); + String sqlFormat = "SELECT t.id as t_id, %s FROM %s t WHERE t.id LIKE CONCAT(%s, '%%')"; + String sql = String.format(sqlFormat, param, datasetId, filter); Query query = entityManager.createNativeQuery(sql); return query.getResultList(); } @Override public List idObfuscationTable(String datasetId, String searchId, String[] params) { - String sqlFormat = "SELECT %s FROM %s WHERE id = %s"; - String sql = String.format(sqlFormat, String.join(", ", params), datasetId, searchId); + String param = processParams(params); + String sqlFormat = "SELECT t.id as t_id, %s FROM %s t WHERE t.id = %s"; + String sql = String.format(sqlFormat, param, datasetId, searchId); Query query = entityManager.createNativeQuery(sql); return query.getResultList(); } - public List objectsToPirTableList(List params) { + public List objectsToPirTableList(List values) { List result = new LinkedList<>(); - for (int i = 0; i < params.size(); i++) { - Object[] temp = (Object[]) params.get(i); + for (int i = 0; i < values.size(); i++) { + Object[] temp = (Object[]) values.get(i); PirTable pirTable = new PirTable(); pirTable.setId(i + 1); pirTable.setPirKey(String.valueOf(temp[0])); @@ -64,4 +50,9 @@ public List objectsToPirTableList(List params) { } return result; } + + private String processParams(String[] params) { + String[] temp = Arrays.stream(params).map(s -> "t." + s).toArray(String[]::new); + return String.join(",", temp); + } } diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java new file mode 100644 index 00000000..4fd4c8e4 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java @@ -0,0 +1,220 @@ +package com.webank.wedpr.pir.http.common; + +import com.opencsv.CSVReaderHeaderAware; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CSVFileParser { + private static final Logger logger = LoggerFactory.getLogger(CSVFileParser.class); + + private interface ParseHandler { + Object call(CSVReaderHeaderAware reader) throws Exception; + } + + private static Object loadCSVFile(String filePath, int chunkSize, ParseHandler handler) + throws Exception { + try (Reader fileReader = new BufferedReader(new FileReader(filePath), chunkSize); + CSVReaderHeaderAware reader = new CSVReaderHeaderAware(fileReader)) { + if (handler != null) { + return handler.call(reader); + } + return null; + } catch (Exception e) { + logger.warn("CSVFileParser exception, filePath: {}, error: ", filePath, e); + throw new WedprException( + WedprStatusEnum.CSV_ERROR, "loadCSVFile exception for " + e.getMessage()); + } + } + + public static List> processCsv2SqlMap(String[] tableFields, String csvFilePath) + throws Exception { + return (List>) + loadCSVFile( + csvFilePath, + Constant.DEFAULT_READ_TRUNK_SIZE, + reader -> { + List> resultValue = new ArrayList<>(); + Map row; + while ((row = reader.readMap()) != null) { + List rowValue = new ArrayList<>(); + for (String field : tableFields) { + if (!row.keySet().contains(field.trim())) { + String errorMsg = + "extractFields failed for the field " + + field + + " not existed in the file " + + tableFields.toString(); + logger.warn(errorMsg); + throw new WedprException( + WedprStatusEnum.CSV_ERROR, errorMsg); + } + rowValue.add(row.get(field)); + } + resultValue.add(rowValue); + } + return resultValue; + }); + } + + public static Set getFields(String filePath) throws Exception { + return (Set) + (loadCSVFile( + filePath, + Constant.DEFAULT_READ_TRUNK_SIZE, + new ParseHandler() { + @Override + public Object call(CSVReaderHeaderAware reader) throws Exception { + return reader.readMap().keySet(); + } + })); + } + + public static class ExtractConfig { + private String originalFilePath; + private List extractFields; + private String extractFilePath; + private String fieldSplitter = ","; + private Integer writeChunkSize = Constant.DEFAULT_WRITE_TRUNK_SIZE; + private Integer readChunkSize = Constant.DEFAULT_READ_TRUNK_SIZE; + + public ExtractConfig() {} + + public ExtractConfig( + String originalFilePath, List extractFields, String extractFilePath) { + this.originalFilePath = originalFilePath; + this.extractFields = extractFields; + this.extractFilePath = extractFilePath; + } + + public String getOriginalFilePath() { + return originalFilePath; + } + + public void setOriginalFilePath(String originalFilePath) { + this.originalFilePath = originalFilePath; + } + + public List getExtractFields() { + return extractFields; + } + + public void setExtractFields(List extractFields) { + this.extractFields = extractFields; + } + + public String getExtractFilePath() { + return extractFilePath; + } + + public void setExtractFilePath(String extractFilePath) { + this.extractFilePath = extractFilePath; + } + + public String getFieldSplitter() { + return fieldSplitter; + } + + public void setFieldSplitter(String fieldSplitter) { + this.fieldSplitter = fieldSplitter; + } + + public Integer getWriteChunkSize() { + return writeChunkSize; + } + + public void setWriteChunkSize(Integer writeChunkSize) { + this.writeChunkSize = writeChunkSize; + } + + public Integer getReadChunkSize() { + return readChunkSize; + } + + public void setReadChunkSize(Integer readChunkSize) { + this.readChunkSize = readChunkSize; + } + + @Override + public String toString() { + return "ExtractConfig{" + + "originalFilePath='" + + originalFilePath + + '\'' + + ", extractFields=" + + extractFields + + ", extractFilePath='" + + extractFilePath + + '\'' + + ", fieldSplitter='" + + fieldSplitter + + '\'' + + ", writeChunkSize=" + + writeChunkSize + + ", readChunkSize=" + + readChunkSize + + '}'; + } + } + + public static void extractFields(ExtractConfig extractConfig) throws Exception { + loadCSVFile( + extractConfig.getOriginalFilePath(), + extractConfig.getReadChunkSize(), + new ParseHandler() { + @Override + public Object call(CSVReaderHeaderAware reader) throws Exception { + // check the fields + Map headerInfo = reader.readMap(); + Set fields = headerInfo.keySet(); + for (String field : extractConfig.getExtractFields()) { + if (!fields.contains(field.trim())) { + String errorMsg = + "extractFields failed for the field " + + field + + " not existed in the file " + + extractConfig.getOriginalFilePath(); + logger.warn(errorMsg); + throw new WedprException(WedprStatusEnum.CSV_ERROR, errorMsg); + } + } + Map row; + try (Writer writer = + new BufferedWriter( + new FileWriter(extractConfig.getExtractFilePath()), + extractConfig.getWriteChunkSize())) { + // write the data(Note: here no need to write the header) + while ((row = reader.readMap()) != null) { + int column = 0; + for (String field : extractConfig.getExtractFields()) { + writer.write(row.get(field)); + if (column < extractConfig.getExtractFields().size() - 1) { + writer.write(extractConfig.getFieldSplitter()); + } + column++; + } + writer.write(Constant.DEFAULT_LINE_SPLITTER); + } + } catch (Exception e) { + logger.warn( + "extractFields exception, config: {}, error", + extractConfig.toString(), + e); + throw new WedprException( + WedprStatusEnum.CSV_ERROR, + "extractFields exception, detail: " + + extractConfig.toString() + + ", error: " + + e.getMessage()); + } + return null; + } + }); + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java index 9ddfd9bb..f967d323 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java @@ -7,4 +7,9 @@ public class Constant { public static final String PIR_API_PREFIX = "/api/pir/v3"; + + public static final Integer DEFAULT_READ_TRUNK_SIZE = 1024 * 1024; + public static final Integer DEFAULT_WRITE_TRUNK_SIZE = 1024 * 1024; + + public static final String DEFAULT_LINE_SPLITTER = "\n"; } diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java new file mode 100644 index 00000000..def233a0 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java @@ -0,0 +1,33 @@ +package com.webank.wedpr.pir.http.config; + +import java.io.IOException; +import java.net.URI; +import org.apache.hadoop.fs.FileSystem; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author zachma + * @date 2024/9/4 + */ +@Configuration +public class HadoopConfig { + @Value("${hdfs.uri}") + private String hdfsUri; + + @Value("${hdfs.user}") + private String hdfsUser; + + @Value("${hdfs.local}") + private String hdfsLocal; + + @Bean + public FileSystem fileSystem() throws IOException, InterruptedException { + org.apache.hadoop.conf.Configuration configuration = + new org.apache.hadoop.conf.Configuration(); + configuration.set("fs.defaultFS", hdfsUri); + FileSystem fileSystem = FileSystem.get(URI.create(hdfsUri), configuration, hdfsUser); + return fileSystem; + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java index 361c63b0..e876b9d2 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java @@ -3,9 +3,18 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.webank.wedpr.pir.http.common.Constant; import com.webank.wedpr.pir.http.service.PirAppService; +import com.webank.wedpr.pir.http.service.PirAuthService; +import com.webank.wedpr.pir.http.service.PirDataService; +import com.wedpr.pir.sdk.PirClient; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; +import com.wedpr.pir.sdk.entity.param.PirJobParam; +import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; +import com.wedpr.pir.sdk.entity.request.ClientDirectRequest; import com.wedpr.pir.sdk.entity.request.ServerJobRequest; import com.wedpr.pir.sdk.entity.response.ClientJobResponse; +import com.wedpr.pir.sdk.entity.response.PirResultResponse; import com.wedpr.pir.sdk.entity.response.ServerOTResponse; +import com.wedpr.pir.sdk.enums.ParamEnum; import com.wedpr.pir.sdk.exception.WedprStatusEnum; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; @@ -26,10 +35,58 @@ public class PirController { @Autowired private ObjectMapper objectMapper; @Autowired private PirAppService pirAppService; + @Autowired private PirAuthService pirAuthService; + @Autowired private PirDataService pirDataService; + + @PostMapping(Constant.PIR_API_PREFIX + "/auth") + public ClientJobResponse pirAuthController(@RequestBody ClientAuthRequest clientAuthRequest) { + try { + logger.info( + "ClientAuthRequest: clientAuthRequest: {}.", + objectMapper.writeValueAsString(clientAuthRequest)); + // 1. 发起认证auth + ServiceConfigBody serviceConfigBody = + pirAuthService.checkServiceAuth(clientAuthRequest); + return new ClientJobResponse(WedprStatusEnum.SUCCESS, serviceConfigBody); + } catch (Exception e) { + logger.warn( + "匿踪任务认证失败, serviceId: {}, response: {}", + clientAuthRequest.getServiceId(), + WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); + return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, null); + } + } + + @PostMapping(Constant.PIR_API_PREFIX + "/client") + public PirResultResponse pirClientController( + @RequestBody ClientDirectRequest clientDirectRequest) { + try { + logger.info( + "ClientDirectRequest: clientDirectRequest: {}.", + objectMapper.writeValueAsString(clientDirectRequest)); + PirJobParam pirJobParam = new PirJobParam(); + pirJobParam.setPirInvokeType(ParamEnum.JobMode.DirectorMode.getValue()); + pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); + pirJobParam.setJobAlgorithmType(clientDirectRequest.getAlgorithmType()); + + // todo:对接网关 + pirJobParam.setGatewayUrl("localhost:5831"); + + pirJobParam.setSearchIdList(clientDirectRequest.getSearchIds()); + pirJobParam.setServiceConfigBody(clientDirectRequest.getServiceConfigBody()); + + PirClient pirClient = new PirClient(pirJobParam); + return pirClient.executePirJob(); + } catch (Exception e) { + logger.warn("匿踪任务失败, response: {}", WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); + return null; + } + } @PostMapping(Constant.PIR_API_PREFIX + "/server") public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serverJobRequest) { try { + pirDataService.processPirDataset(serverJobRequest.getServiceConfigBody()); logger.info( "WedprServerjob: serverJobRequest: {}.", objectMapper.writeValueAsString(serverJobRequest)); @@ -37,9 +94,10 @@ public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serve ServerOTResponse serverOTResponse = pirAppService.providerOtCipher(serverJobRequest); return new ClientJobResponse(WedprStatusEnum.SUCCESS, serverOTResponse); } catch (Exception e) { + e.printStackTrace(); logger.warn( - "匿踪任务失败, jobID: {}, response: {}", - serverJobRequest.getJobId(), + "匿踪任务失败, serviceId: {}, response: {}", + serverJobRequest.getServiceId(), WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, e.getMessage()); } diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java new file mode 100644 index 00000000..061fbe60 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java @@ -0,0 +1,47 @@ +package com.webank.wedpr.pir.http.mapper; + +import com.webank.wedpr.pir.crypto.entity.Dataset; +import com.webank.wedpr.pir.crypto.entity.WedprAuthTable; +import java.util.List; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Mapper +public interface WedprServiceAuthTableMapper { + + @Select( + "SELECT service_id, access_key_id, expire_time FROM wedpr_service_auth_table WHERE service_id = #{serviceId} AND access_key_id = #{accessKeyId}") + @Results({ + @Result(property = "serviceId", column = "service_id"), + @Result(property = "accessKeyId", column = "access_key_id"), + @Result(property = "expireTime", column = "expire_time") + }) + WedprAuthTable searchWedprAuthTable( + @Param("serviceId") String serviceId, @Param("accessKeyId") String accessKeyId); + + @Select("SELECT service_config FROM wedpr_published_service WHERE service_id = #{serviceId}") + String searchWedprServicePublishTable(@Param("serviceId") String serviceId); + + @Select("SHOW TABLES") + List showAllTables(); + + @Select( + "SELECT dataset_id, dataset_fields, dataset_storage_path FROM wedpr_dataset WHERE dataset_id = #{datasetId}") + @Results({ + @Result(property = "datasetId", column = "dataset_id"), + @Result(property = "datasetFields", column = "dataset_fields"), + @Result(property = "datasetStoragePath", column = "dataset_storage_path") + }) + Dataset searchWedprDatasetId(@Param("datasetId") String datasetId); + + @Update(value = "${nativeSql}") + void executeTableByNativeSql(@Param("nativeSql") String nativeSql); +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java index 4d0db22f..1852c523 100644 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java @@ -24,7 +24,7 @@ public ServerOTResponse providerOtCipher(ServerJobRequest serverJobRequest) // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 ServerOTRequest serverOTRequest = new ServerOTRequest(); serverOTRequest.setJobType(serverJobRequest.getJobType()); - serverOTRequest.setDatasetId(serverJobRequest.getDatasetId()); + serverOTRequest.setDatasetId(serverJobRequest.getServiceConfigBody().getDatasetId()); serverOTRequest.setX(serverJobRequest.getX()); serverOTRequest.setY(serverJobRequest.getY()); serverOTRequest.setDataBodyList(serverJobRequest.getDataBodyList()); @@ -33,10 +33,12 @@ public ServerOTResponse providerOtCipher(ServerJobRequest serverJobRequest) if (serverJobRequest .getJobAlgorithmType() .equals(ParamEnum.AlgorithmType.idFilter.getValue())) { + serverOTRequest.setParams(serverJobRequest.getServiceConfigBody().getValues()); otResultResponse = pirOTService.runServerOTParam(serverOTRequest); } else if (serverJobRequest .getJobAlgorithmType() .equals(ParamEnum.AlgorithmType.idObfuscation.getValue())) { + serverOTRequest.setParams(serverJobRequest.getServiceConfigBody().getExists()); otResultResponse = pirOTService.runServerOTCipher(serverOTRequest); } else { throw new WedprException(WedprStatusEnum.INVALID_INPUT_VALUE); diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java new file mode 100644 index 00000000..ca0d979e --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java @@ -0,0 +1,50 @@ +package com.webank.wedpr.pir.http.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.wedpr.pir.crypto.entity.WedprAuthTable; +import com.webank.wedpr.pir.http.mapper.WedprServiceAuthTableMapper; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; +import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; +import com.wedpr.pir.sdk.exception.WedprException; +import com.wedpr.pir.sdk.exception.WedprStatusEnum; +import java.time.LocalDateTime; +import java.util.Objects; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @author zachma + * @date 2024/9/3 + */ +@Service +public class PirAuthService { + + @Autowired private WedprServiceAuthTableMapper wedprServiceAuthTableMapper; + + public ServiceConfigBody checkServiceAuth(ClientAuthRequest clientAuthRequest) + throws WedprException { + try { + WedprAuthTable wedprAuthTable = + wedprServiceAuthTableMapper.searchWedprAuthTable( + clientAuthRequest.getServiceId(), clientAuthRequest.getAccessKeyId()); + if (Objects.isNull(wedprAuthTable)) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "用户未申请认证"); + } + + if (LocalDateTime.now().isAfter(wedprAuthTable.getExpireTime())) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "认证已经过期"); + } + + String serviceConfigStr = + wedprServiceAuthTableMapper.searchWedprServicePublishTable( + clientAuthRequest.getServiceId()); + ObjectMapper objectMapper = new ObjectMapper(); + ServiceConfigBody serviceConfigBody = + objectMapper.readValue(serviceConfigStr, ServiceConfigBody.class); + + return serviceConfigBody; + } catch (Exception e) { + throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, e.getMessage()); + } + } +} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java new file mode 100644 index 00000000..25946eb8 --- /dev/null +++ b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java @@ -0,0 +1,90 @@ +package com.webank.wedpr.pir.http.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.wedpr.pir.crypto.entity.Dataset; +import com.webank.wedpr.pir.crypto.entity.DatasetStoragePath; +import com.webank.wedpr.pir.http.common.CSVFileParser; +import com.webank.wedpr.pir.http.mapper.WedprServiceAuthTableMapper; +import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; +import com.wedpr.pir.sdk.helper.Constant; +import java.util.Arrays; +import java.util.List; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @author zachma + * @date 2024/9/4 + */ +@Service +public class PirDataService { + + @Autowired private WedprServiceAuthTableMapper wedprServiceAuthTableMapper; + + @Autowired private FileSystem fileSystem; + + @PersistenceContext private EntityManager entityManager; + + public void processPirDataset(ServiceConfigBody serviceConfigBody) throws Exception { + // 数据预处理 + List allTables = wedprServiceAuthTableMapper.showAllTables(); + String tableId = Constant.datasetId2tableId(serviceConfigBody.getDatasetId()); + if (allTables.contains(tableId)) { + return; + } + Dataset dataset = + wedprServiceAuthTableMapper.searchWedprDatasetId(serviceConfigBody.getDatasetId()); + ObjectMapper objectMapper = new ObjectMapper(); + DatasetStoragePath datasetStoragePath = + objectMapper.readValue(dataset.getDatasetStoragePath(), DatasetStoragePath.class); + + String csvFilePath = "./" + serviceConfigBody.getDatasetId(); + fileSystem.copyToLocalFile( + new Path(datasetStoragePath.getFilePath()), new Path(csvFilePath)); + + String[] datasetFields = + Arrays.stream(dataset.getDatasetFields().trim().split(",")) + .map(String::trim) + .toArray(String[]::new); + processCsv2Db(serviceConfigBody.getDatasetId(), datasetFields, csvFilePath); + } + + private void processCsv2Db(String datasetId, String[] datasetFields, String csvFilePath) + throws Exception { + List> sqlValues = CSVFileParser.processCsv2SqlMap(datasetFields, csvFilePath); + if (sqlValues.size() == 0) { + throw new Exception("数据集为空,请退回"); + } + + String tableId = Constant.datasetId2tableId(datasetId); + String createSqlFormat = "CREATE TABLE %s ( %s , PRIMARY KEY (`id`) USING BTREE )"; + + String[] fieldsWithType = new String[datasetFields.length]; + for (int i = 0; i < datasetFields.length; i++) { + if ("id".equalsIgnoreCase(datasetFields[i])) { + fieldsWithType[i] = datasetFields[i] + " VARCHAR(64)"; + } else { + fieldsWithType[i] = datasetFields[i] + " TEXT"; + } + } + String sql = String.format(createSqlFormat, tableId, String.join(",", fieldsWithType)); + wedprServiceAuthTableMapper.executeTableByNativeSql(sql); + + StringBuilder sb = new StringBuilder(); + for (List values : sqlValues) { + sb.append("(").append(String.join(",", values)).append("), "); + } + String insertValues = sb.toString(); + insertValues = insertValues.substring(0, insertValues.length() - 2); + + String insertSqlFormat = "INSERT INTO %s (%s) VALUES %s "; + sql = + String.format( + insertSqlFormat, tableId, String.join(",", datasetFields), insertValues); + wedprServiceAuthTableMapper.executeTableByNativeSql(sql); + } +} diff --git a/java/ppc-pir-services/src/main/resources/application-dev.properties b/java/ppc-pir-services/src/main/resources/application-dev.properties index 86233b92..ac03ea13 100644 --- a/java/ppc-pir-services/src/main/resources/application-dev.properties +++ b/java/ppc-pir-services/src/main/resources/application-dev.properties @@ -6,4 +6,9 @@ banner.charset=UTF-8 spring.jackson.local-date-time-format=yyyy-MM-dd HH:mm:ss spring.jackson.local-date-format=yyyy-MM-dd spring.jackson.local-time-format=HH:mm:ss -spring.jackson.time-zone=GMT+8 \ No newline at end of file +spring.jackson.time-zone=GMT+8 +spring.jackson.open-in-view=false + +hdfs.uri=hdfs://139.159.202.235:9900 +hdfs.user=ppc +hdfs.local=./temp \ No newline at end of file diff --git a/java/ppc-pir-services/src/main/resources/application-mysql.properties b/java/ppc-pir-services/src/main/resources/application-mysql.properties index 4c82652f..3f2227a1 100644 --- a/java/ppc-pir-services/src/main/resources/application-mysql.properties +++ b/java/ppc-pir-services/src/main/resources/application-mysql.properties @@ -1,6 +1,6 @@ -spring.datasource.url=jdbc:mysql://139.159.202.235:3306/ppc1?serverTimezone=Asia/Shanghai&useUnicode=true&autoReconnect=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&useFractionalSeconds=true&useUnbufferedInput=false&useSSL=false -spring.datasource.username=root -spring.datasource.password=Wedpr2023 +spring.datasource.url=jdbc:mysql://xxxxxxx:3306/xxxxx?serverTimezone=Asia/Shanghai&useUnicode=true&autoReconnect=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&useFractionalSeconds=true&useUnbufferedInput=false&useSSL=false +spring.datasource.username=xxxx +spring.datasource.password=xxxx spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5InnoDBDialect spring.jpa.database=mysql From 4de5c76ac723e19cdb8504c167920a36c9e478e6 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 13 Sep 2024 14:38:44 +0800 Subject: [PATCH 039/120] add cpp demo for transport (#36) --- cpp/wedpr-initializer/Initializer.cpp | 2 +- cpp/wedpr-initializer/Initializer.h | 2 +- .../ppc-front/ppc-front/CallbackManager.cpp | 2 +- .../sdk-wrapper/java/bindings/build.gradle | 2 +- .../wedpr/sdk/jni/demo/TransportDemo.java | 65 ++++++--- .../java/swig/wedpr_java_transport.i | 9 +- .../python/swig/wedpr_python_transport.i | 8 +- cpp/wedpr-transport/sdk/CMakeLists.txt | 16 +-- cpp/wedpr-transport/sdk/demo/CMakeLists.txt | 7 + .../sdk/demo/transport_client.cpp | 132 ++++++++++++++++++ cpp/wedpr-transport/sdk/src/CMakeLists.txt | 8 ++ cpp/wedpr-transport/sdk/{ => src}/Common.h | 0 .../sdk/{ => src}/ProTransportImpl.cpp | 0 .../sdk/{ => src}/ProTransportImpl.h | 0 .../sdk/{ => src}/Transport.cpp | 0 cpp/wedpr-transport/sdk/{ => src}/Transport.h | 0 .../sdk/{ => src}/TransportBuilder.cpp | 0 .../sdk/{ => src}/TransportBuilder.h | 0 .../sdk/{ => src}/TransportImpl.h | 0 19 files changed, 211 insertions(+), 42 deletions(-) create mode 100644 cpp/wedpr-transport/sdk/demo/CMakeLists.txt create mode 100644 cpp/wedpr-transport/sdk/demo/transport_client.cpp create mode 100644 cpp/wedpr-transport/sdk/src/CMakeLists.txt rename cpp/wedpr-transport/sdk/{ => src}/Common.h (100%) rename cpp/wedpr-transport/sdk/{ => src}/ProTransportImpl.cpp (100%) rename cpp/wedpr-transport/sdk/{ => src}/ProTransportImpl.h (100%) rename cpp/wedpr-transport/sdk/{ => src}/Transport.cpp (100%) rename cpp/wedpr-transport/sdk/{ => src}/Transport.h (100%) rename cpp/wedpr-transport/sdk/{ => src}/TransportBuilder.cpp (100%) rename cpp/wedpr-transport/sdk/{ => src}/TransportBuilder.h (100%) rename cpp/wedpr-transport/sdk/{ => src}/TransportImpl.h (100%) diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp index 26b120d9..d1e1acec 100644 --- a/cpp/wedpr-initializer/Initializer.cpp +++ b/cpp/wedpr-initializer/Initializer.cpp @@ -27,7 +27,7 @@ #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h" #include "ppc-psi/src/cm2020-psi/CM2020PSIFactory.h" #include "protocol/src/PPCMessage.h" -#include "wedpr-transport/sdk/TransportBuilder.h" +#include "wedpr-transport/sdk/src/TransportBuilder.h" #if 0 //TODO: optimize here #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h" diff --git a/cpp/wedpr-initializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h index 4a49702b..348d1040 100644 --- a/cpp/wedpr-initializer/Initializer.h +++ b/cpp/wedpr-initializer/Initializer.h @@ -24,7 +24,7 @@ #include "ppc-framework/rpc/RpcInterface.h" #include "ppc-framework/rpc/RpcTypeDef.h" #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h" -#include "wedpr-transport/sdk/Transport.h" +#include "wedpr-transport/sdk/src/Transport.h" #include #include diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp index 3f500292..efddec1c 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp @@ -167,7 +167,7 @@ void CallbackManager::onReceiveMessage(std::string const& topic, Message::Ptr ms } if (!callback) { - FRONT_LOG(DEBUG) << LOG_DESC("onReceiveMessage: not find the handler, put into the buffer") + FRONT_LOG(TRACE) << LOG_DESC("onReceiveMessage: not find the handler, put into the buffer") << LOG_KV("topic", topic); addMsgCache(topic, msg); return; diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index bbceb950..42409146 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -88,7 +88,7 @@ dependencies { archivesBaseName = 'wedpr-java-transport-jni' group = 'com.webank.wedpr' -version = '1.0.0' +version = '1.0.0-SNAPSHOT' jar { exclude '**/*.xml' diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java index b753e076..757d597a 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -39,7 +39,9 @@ public void onMessage(IMessage message) { + nodeID + " receiveMessage, detail: " + message.toString() - + ", "); + + ", payload: " + + new String(message.getPayload()) + + "#######"); } } @@ -58,37 +60,42 @@ public void onError(Error error) { + " MessageErrorCallback, result: " + error.errorMessage() + ", code:" - + error.errorCode()); + + error.errorCode() + + "######"); } } public static void main(String[] args) throws Exception { String nodeID = "testNode"; - if (args.length > 1) { + if (args.length > 0) { nodeID = args[0]; } TransportConfig transportConfig = new TransportConfig(2, nodeID); String hostIp = "127.0.0.1"; - if (args.length > 2) { + if (args.length > 1) { hostIp = args[1]; } int listenPort = 9020; - if (args.length > 3) { + if (args.length > 2) { listenPort = Integer.valueOf(args[2]); } String listenIp = "0.0.0.0"; TransportEndPoint endPoint = new TransportEndPoint(hostIp, listenIp, listenPort); transportConfig.setSelfEndPoint(endPoint); String grpcTarget = "ipv4:127.0.0.1:40600,127.0.0.1:40601"; - if (args.length > 4) { + if (args.length > 3) { grpcTarget = args[3]; } transportConfig.setGatewayTargets(grpcTarget); + String dstNode = "agency2Node"; + if (args.length > 4) { + dstNode = args[4]; + } System.out.println("####### transportConfig: " + transportConfig.toString()); + System.out.println("####### dstNode: " + dstNode); // build the gatewayTarget WeDPRTransport transport = TransportImpl.build(transportConfig); - System.out.println("####### before start the transport"); transport.start(); System.out.println("####### start the transport success"); @@ -96,25 +103,41 @@ public static void main(String[] args) throws Exception { String topic = "testTopic"; transport.registerTopicHandler(topic, new MessageDispatcherCallbackImpl(nodeID)); System.out.println("##### register topic success"); - String dstNode = "agency2Node"; - if (args.length > 5) { - dstNode = args[5]; - } byte[] dstNodeBytes = dstNode.getBytes(); // every 2s send a message Integer i = 0; + String syncTopic = "sync_" + topic; while (true) { - String payLoad = "testPayload" + i; - transport.asyncSendMessageByNodeID( - topic, - dstNodeBytes, - payLoad.getBytes(), - 0, - 10000, - new MessageErrorCallback(nodeID), - null); - i++; + try { + String payLoad = "testPayload" + i; + // send Message by nodeID + transport.asyncSendMessageByNodeID( + topic, + dstNodeBytes, + payLoad.getBytes(), + 0, + 10000, + new MessageErrorCallback(nodeID), + null); + + // push + String syncPayload = "sync_" + payLoad; + transport.pushByNodeID(syncTopic, dstNodeBytes, 0, syncPayload.getBytes(), 2000); + // pop + IMessage msg = transport.pop(syncTopic, 2000); + System.out.println( + "##### receive msg from " + + syncTopic + + ", detail: " + + msg.toString() + + ", payload: " + + new String(msg.getPayload()) + + "####"); + i++; + } catch (Exception e) { + System.out.println("#### exception: " + e.getMessage()); + } Thread.sleep(2000); } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index 35dfdd9b..c5cf2638 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -62,8 +62,8 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long); #include #include #include -#include "wedpr-transport/sdk/TransportBuilder.h" -#include "wedpr-transport/sdk/Transport.h" +#include "wedpr-transport/sdk/src/TransportBuilder.h" +#include "wedpr-transport/sdk/src/Transport.h" #include "ppc-framework/libwrapper/Buffer.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/RouteType.h" @@ -197,6 +197,5 @@ namespace bcos{ %include "ppc-framework/front/IFront.h" -%include "wedpr-transport/sdk/TransportBuilder.h" -%include "wedpr-transport/sdk/Transport.h" - +%include "wedpr-transport/sdk/src/TransportBuilder.h" +%include "wedpr-transport/sdk/src/Transport.h" \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i index ef073d01..16515cba 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i @@ -16,8 +16,8 @@ %{ #define SWIG_FILE_WITH_INIT #include -#include "wedpr-transport/sdk/TransportBuilder.h" -#include "wedpr-transport/sdk/Transport.h" +#include "wedpr-transport/sdk/src/TransportBuilder.h" +#include "wedpr-transport/sdk/src/Transport.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/RouteType.h" #include "ppc-framework/front/FrontConfig.h" @@ -44,6 +44,6 @@ namespace ppc::sdk{ %template(SharedRouteInfo) std::shared_ptr; %template(SharedTransport) std::shared_ptr; -%include "wedpr-transport/sdk/TransportBuilder.h" -%include "wedpr-transport/sdk/Transport.h" +%include "wedpr-transport/sdk/src/TransportBuilder.h" +%include "wedpr-transport/sdk/src/Transport.h" %include "ppc-framework/front/IFront.h" \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/CMakeLists.txt b/cpp/wedpr-transport/sdk/CMakeLists.txt index 1231ca68..4606f2dd 100644 --- a/cpp/wedpr-transport/sdk/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk/CMakeLists.txt @@ -1,8 +1,8 @@ -cmake_minimum_required(VERSION 3.14) -project(ppc-transport-sdk VERSION ${VERSION}) - -file(GLOB_RECURSE SRCS *.cpp) - -add_library(${WEDPR_TRANSPORT_SDK_TARGET} ${SRCS}) -target_link_libraries(${WEDPR_TRANSPORT_SDK_TARGET} PUBLIC - ${FRONT_TARGET} ${PB_PROTOCOL_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${CPU_FEATURES_LIB}) \ No newline at end of file +cmake_minimum_required(VERSION 3.14) + +add_subdirectory(src) + +if (DEMO) + add_subdirectory(demo) + enable_testing() +endif () \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/demo/CMakeLists.txt b/cpp/wedpr-transport/sdk/demo/CMakeLists.txt new file mode 100644 index 00000000..4ee3354c --- /dev/null +++ b/cpp/wedpr-transport/sdk/demo/CMakeLists.txt @@ -0,0 +1,7 @@ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# cmake settings +set(BINARY_NAME transport_sdk_demo) +add_executable(${BINARY_NAME} transport_client.cpp) +target_link_libraries(${BINARY_NAME} ${WEDPR_TRANSPORT_SDK_TARGET}) + +unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/demo/transport_client.cpp b/cpp/wedpr-transport/sdk/demo/transport_client.cpp new file mode 100644 index 00000000..b9e59d30 --- /dev/null +++ b/cpp/wedpr-transport/sdk/demo/transport_client.cpp @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2022 WeDPR. + * SPDX-License-Identifier: Apache-2.0 + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @file transport_bench.cpp + * @desc: bench for transport + * @author: yujiechen + * @date 2024-09-13 + */ +#include "ppc-framework/protocol/RouteType.h" +#include "wedpr-transport/sdk/src/TransportBuilder.h" +#include + +using namespace ppc::front; +using namespace ppc::sdk; +using namespace ppc::protocol; + +void Usage(std::string const& _appName) +{ + std::cout << _appName << " [hostIp] [listenPort] [gatewayTargets] [nodeID] [dstNode]" + << std::endl; + std::cout << "example:" << std::endl; + std::cout << _appName + << " 127.0.0.1 9020 ipv4:127.0.0.1:40600,127.0.0.1:40601 agency0Node agency1Node" + << std::endl; + std::cout << _appName + << " 127.0.0.1 9021 ipv4:127.0.0.1:40620,127.0.0.1:40621 agency1Node agency0Node" + << std::endl; +} + +int main(int argc, char* argv[]) +{ + if (argc < 6) + { + Usage(argv[0]); + return -1; + } + // the hostIp + std::string hostIp = argv[1]; + // the clientPort + int port = atoi(argv[2]); + std::string listenIp = "0.0.0.0"; + // the gatewayTargets + std::string gatewayTargets = argv[3]; + // the nodeID + std::string nodeID = argv[4]; + // the dstNode + std::string dstNode = argv[5]; + + auto transportBuilder = std::make_shared(); + auto frontConfig = transportBuilder->buildConfig(2, nodeID); + frontConfig->setGatewayGrpcTarget(gatewayTargets); + EndPoint endPoint(hostIp, port); + frontConfig->setSelfEndPoint(endPoint); + auto transport = transportBuilder->buildProTransport(frontConfig); + + // start the transport + std::cout << "#### start the front, detail: " << printFrontDesc(frontConfig); + transport->start(); + auto routeInfo = transport->routeInfoBuilder()->build(); + std::string topic = "sync_testDemo"; + routeInfo->setDstNode(bcos::bytes(dstNode.begin(), dstNode.end())); + routeInfo->setTopic(topic); + transport->getFront()->registerTopicHandler(topic, [](Message::Ptr msg) { + std::cout << "=== async receive message: " << printMessage(msg) << "====" << std::endl; + }); + std::string syncTopic = "sync__" + topic; + auto syncRouteInfo = transport->routeInfoBuilder()->build(); + syncRouteInfo->setDstNode(bcos::bytes(dstNode.begin(), dstNode.end())); + syncRouteInfo->setTopic(syncTopic); + // sendMessage test + long i = 0; + while (true) + { + try + { + std::string payload = "payload+++" + std::to_string(i); + bcos::bytes payloadBytes = bcos::bytes(payload.begin(), payload.end()); + // async test + transport->getFront()->asyncSendMessage((uint16_t)RouteType::ROUTE_THROUGH_NODEID, + routeInfo, std::move(bcos::bytes(payload.begin(), payload.end())), 0, 10000, + [](bcos::Error::Ptr error) { + if (error && error->errorCode() != 0) + { + std::cout << "!**** send message failed for: " << error->errorMessage() + << "***!" << std::endl; + } + }, + nullptr); + + // push + auto error = transport->getFront()->push((uint16_t)RouteType::ROUTE_THROUGH_NODEID, + syncRouteInfo, std::move(payloadBytes), 0, 10000); + if (!error && error->errorCode() != 0) + { + std::cout << "!**** send message failed for: " << error->errorMessage() << "***!" + << std::endl; + } + // pop + auto msg = transport->getFront()->pop(syncTopic, 10000); + if (msg == nullptr) + { + std::cout << "try to receive message timeout" << std::endl; + } + else + { + std::cout << "=== sync receive message: " << printMessage(msg) + << "====" << std::endl; + } + i++; + } + catch (std::exception const& e) + { + std::cout << "!**** exception: " << boost::diagnostic_information(e) << "****!" + << std::endl; + } + // wait for 2s + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + }; + return 0; +} diff --git a/cpp/wedpr-transport/sdk/src/CMakeLists.txt b/cpp/wedpr-transport/sdk/src/CMakeLists.txt new file mode 100644 index 00000000..1231ca68 --- /dev/null +++ b/cpp/wedpr-transport/sdk/src/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.14) +project(ppc-transport-sdk VERSION ${VERSION}) + +file(GLOB_RECURSE SRCS *.cpp) + +add_library(${WEDPR_TRANSPORT_SDK_TARGET} ${SRCS}) +target_link_libraries(${WEDPR_TRANSPORT_SDK_TARGET} PUBLIC + ${FRONT_TARGET} ${PB_PROTOCOL_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk/Common.h b/cpp/wedpr-transport/sdk/src/Common.h similarity index 100% rename from cpp/wedpr-transport/sdk/Common.h rename to cpp/wedpr-transport/sdk/src/Common.h diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp similarity index 100% rename from cpp/wedpr-transport/sdk/ProTransportImpl.cpp rename to cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp diff --git a/cpp/wedpr-transport/sdk/ProTransportImpl.h b/cpp/wedpr-transport/sdk/src/ProTransportImpl.h similarity index 100% rename from cpp/wedpr-transport/sdk/ProTransportImpl.h rename to cpp/wedpr-transport/sdk/src/ProTransportImpl.h diff --git a/cpp/wedpr-transport/sdk/Transport.cpp b/cpp/wedpr-transport/sdk/src/Transport.cpp similarity index 100% rename from cpp/wedpr-transport/sdk/Transport.cpp rename to cpp/wedpr-transport/sdk/src/Transport.cpp diff --git a/cpp/wedpr-transport/sdk/Transport.h b/cpp/wedpr-transport/sdk/src/Transport.h similarity index 100% rename from cpp/wedpr-transport/sdk/Transport.h rename to cpp/wedpr-transport/sdk/src/Transport.h diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp similarity index 100% rename from cpp/wedpr-transport/sdk/TransportBuilder.cpp rename to cpp/wedpr-transport/sdk/src/TransportBuilder.cpp diff --git a/cpp/wedpr-transport/sdk/TransportBuilder.h b/cpp/wedpr-transport/sdk/src/TransportBuilder.h similarity index 100% rename from cpp/wedpr-transport/sdk/TransportBuilder.h rename to cpp/wedpr-transport/sdk/src/TransportBuilder.h diff --git a/cpp/wedpr-transport/sdk/TransportImpl.h b/cpp/wedpr-transport/sdk/src/TransportImpl.h similarity index 100% rename from cpp/wedpr-transport/sdk/TransportImpl.h rename to cpp/wedpr-transport/sdk/src/TransportImpl.h From 09d4a3ead07b581f5f59b0119997d48107a14208 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sat, 14 Sep 2024 11:31:23 +0800 Subject: [PATCH 040/120] fix transport sdk crashed caused by jvm release the c-managed object (#37) * fix transport sdk crashed caused by jvm release the c-managed object * fix hdfs compile failed when disable sse on x86 machines * upgrade uplad-artifact * remove some swig interfaces * fix macos coverage failed --- .github/workflows/cpp_workflow.yml | 27 +++-- cpp/cmake/BuildInfo.cmake | 2 +- cpp/cmake/CompilerSettings.cmake | 2 +- cpp/cmake/Coverage.cmake | 4 +- cpp/cmake/Installlibhdfs3.cmake | 2 +- cpp/cmake/Options.cmake | 15 ++- cpp/ppc-framework/front/IFront.h | 8 +- cpp/tools/install_depends.sh | 6 +- cpp/vcpkg-configuration.json | 6 +- cpp/vcpkg.json | 26 +++- cpp/wedpr-main/common/NodeStarter.h | 1 + .../wedpr/sdk/jni/demo/TransportDemo.java | 15 ++- .../wedpr/sdk/jni/generated/FrontConfig.java | 4 + .../wedpr/sdk/jni/generated/IFront.java | 114 ++---------------- .../wedpr/sdk/jni/generated/Message.java | 4 + .../sdk/jni/generated/MessageBuilder.java | 4 + .../sdk/jni/generated/MessageHeader.java | 4 + .../jni/generated/MessageHeaderBuilder.java | 4 + .../jni/generated/MessageOptionalHeader.java | 4 + .../MessageOptionalHeaderBuilder.java | 4 + .../sdk/jni/generated/MessagePayload.java | 4 + .../jni/generated/MessagePayloadBuilder.java | 4 + ...s__Error_t_std__setT_std__string_tF_t.java | 37 ------ ...d__shared_ptrT_bcos__bytes_t_RRF_tF_t.java | 37 ------ ...ared_ptrT_ppc__protocol__Message_tF_t.java | 32 ----- .../wedpr/sdk/jni/generated/Transport.java | 4 + .../sdk/jni/generated/TransportBuilder.java | 13 -- .../generated/wedpr_java_transportJNI.java | 48 +------- .../sdk/jni/transport/IMessageBuilder.java | 11 ++ .../sdk/jni/transport/TransportConfig.java | 1 + .../sdk/jni/transport/WeDPRTransport.java | 10 +- .../transport/handlers/MessageCallback.java | 6 + .../handlers/MessageDispatcherCallback.java | 7 ++ .../handlers/MessageErrorCallback.java | 28 +++++ .../sdk/jni/transport/impl/MessageImpl.java | 3 + .../sdk/jni/transport/impl/TransportImpl.java | 50 ++++---- .../java/swig/wedpr_java_transport.i | 38 ++++++ 37 files changed, 249 insertions(+), 340 deletions(-) delete mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java delete mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java delete mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml index ce682ad2..2e50590e 100644 --- a/.github/workflows/cpp_workflow.yml +++ b/.github/workflows/cpp_workflow.yml @@ -75,7 +75,6 @@ jobs: if: runner.os != 'Windows' run: | cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest - make coverage build_sdk: name: build sdk @@ -129,20 +128,24 @@ jobs: - name: Build for macos if: runner.os == 'macOS' run: | + brew install lcov bash cpp/tools/install_depends.sh -o macos - mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ + mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=OFF -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ make -j3 - name: Test - if: runner.os != 'Windows' + if: runner.os == 'macOS' run: | cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test - make coverage - - uses: actions/upload-artifact@v2 + - name: Test and upload coverage + if: runner.os == 'Linux' + run: | + cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test + - uses: actions/upload-artifact@v4 if: runner.os == 'macos' with: name: libppc-crypto-sdk-jni.dylib path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: runner.os == 'Windows' with: name: libppc-crypto-sdk-jni.dylib @@ -185,7 +188,7 @@ jobs: - name: Prepare vcpkg if: runner.os != 'Windows' uses: friendlyanon/setup-vcpkg@v1 - with: { committish: 7e3dcf74e37034eea358934a90a11d618520e139 } + with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 } - uses: actions-rs/toolchain@v1 with: toolchain: nightly-2022-07-28 @@ -206,19 +209,19 @@ jobs: run: | export OMP_NUM_THREADS=1 cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ppc-air-node-centos-x64 path: ./cpp/build/bin/ppc-air-node - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ppc-pro-node-centos-x64 path: ./cpp/build/bin/ppc-pro-node - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ppc-gateway-service-centos-x64 path: ./cpp/build/bin/ppc-gateway-service - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: libppc-crypto-sdk-jni.so - path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so \ No newline at end of file + path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so diff --git a/cpp/cmake/BuildInfo.cmake b/cpp/cmake/BuildInfo.cmake index 7ef9778c..b100a7f5 100644 --- a/cpp/cmake/BuildInfo.cmake +++ b/cpp/cmake/BuildInfo.cmake @@ -1,6 +1,6 @@ function(create_build_info) # Set build platform; to be written to BuildInfo.h - set(PPC_BUILD_OS "${CMAKE_SYSTEM_NAME}") + set(PPC_BUILD_OS "${CMAKE_HOST_SYSTEM_NAME}") if (CMAKE_COMPILER_IS_MINGW) set(PPC_BUILD_COMPILER "mingw") diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake index 5e1140e3..8a068996 100644 --- a/cpp/cmake/CompilerSettings.cmake +++ b/cpp/cmake/CompilerSettings.cmake @@ -101,7 +101,7 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA add_compile_options(-fstack-protector) add_compile_options(-Winconsistent-missing-override) # Some Linux-specific Clang settings. We don't want these for OS X. - if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") + if ("${CMAKE_HOST_SYSTEM_NAME}" MATCHES "Linux") # Tell Boost that we're using Clang's libc++. Not sure exactly why we need to do. add_definitions(-DBOOST_ASIO_HAS_CLANG_LIBCXX) # Use fancy colors in the compiler diagnostics diff --git a/cpp/cmake/Coverage.cmake b/cpp/cmake/Coverage.cmake index b76a97e2..51c2f313 100644 --- a/cpp/cmake/Coverage.cmake +++ b/cpp/cmake/Coverage.cmake @@ -26,12 +26,12 @@ function(config_coverage TARGET REMOVE_FILE_PATTERN) message(STATUS "coverage REMOVE_FILE_PATTERN: " ${REMOVE_FILE_PATTERN}) if (APPLE) add_custom_target(${TARGET} - COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/ + COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -d ${CMAKE_BINARY_DIR}/ COMMAND ${LCOV_TOOL} -keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info COMMAND genhtml --keep-going --ignore-errors inconsistent,unmapped,source --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info) else() add_custom_target(${TARGET} - COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -i -d ${CMAKE_BINARY_DIR}/ + COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -o ${CMAKE_BINARY_DIR}/coverage.info.in -c -d ${CMAKE_BINARY_DIR}/ COMMAND ${LCOV_TOOL} --rc lcov_branch_coverage=1 -r ${CMAKE_BINARY_DIR}/coverage.info.in '*MacOS*' '/usr*' '.*vcpkg_installed*' '.*boost/*' '*test*' '*build*' '*deps*' ${REMOVE_FILE_PATTERN} -o ${CMAKE_BINARY_DIR}/coverage.info COMMAND genhtml --rc lcov_branch_coverage=1 -q -o ${CMAKE_BINARY_DIR}/CodeCoverage ${CMAKE_BINARY_DIR}/coverage.info) endif() diff --git a/cpp/cmake/Installlibhdfs3.cmake b/cpp/cmake/Installlibhdfs3.cmake index bcfc5d81..6b1f6c59 100644 --- a/cpp/cmake/Installlibhdfs3.cmake +++ b/cpp/cmake/Installlibhdfs3.cmake @@ -1,6 +1,6 @@ find_package(libxml2 REQUIRED) find_package(protobuf REQUIRED) -if ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") +if ("${CMAKE_HOST_SYSTEM_NAME}" MATCHES "Linux") find_package(unofficial-libuuid REQUIRED) endif() find_package(libhdfs3 REQUIRED) \ No newline at end of file diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index ab319fb3..0293ad6b 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -82,9 +82,10 @@ macro(configure_project) if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*SSE4.2.*") set(ENABLE_SSE ON) endif () - elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + elseif(NOT ${CMAKE_HOST_SYSTEM_NAME} MATCHES "Windows") # detect sse4_2 FILE(READ "/proc/cpuinfo" SUPPORTED_INSTRUCTIONS) + message("* Linux SUPPORTED_INSTRUCTIONS: ${SUPPORTED_INSTRUCTIONS}") if (${SUPPORTED_INSTRUCTIONS} MATCHES ".*sse4_2.*") set(ENABLE_SSE ON) endif () @@ -137,15 +138,17 @@ macro(configure_project) if (BUILD_ALL) # install all dependencies list(APPEND VCPKG_MANIFEST_FEATURES "all") + if(ENABLE_SSE) + # enable sse for libhdfs3 + list(APPEND VCPKG_MANIFEST_FEATURES "sse-libhdfs3") + else() + list(APPEND VCPKG_MANIFEST_FEATURES "default-libhdfs3") + endif() endif() if (BUILD_WEDPR_TOOLKIT) # install wedpr dependencies list(APPEND VCPKG_MANIFEST_FEATURES "toolkit") endif() - if(ENABLE_SSE) - # enable sse for libhdfs3 - list(APPEND VCPKG_MANIFEST_FEATURES "sse") - endif() # cpp_features if(ENABLE_CPU_FEATURES) list(APPEND VCPKG_MANIFEST_FEATURES "cpufeatures") @@ -164,7 +167,7 @@ macro(print_config NAME) message("-- CMAKE_BUILD_TYPE Build type ${CMAKE_BUILD_TYPE}") message("-- VCPKG_MANIFEST_FEATURES VCPKG manifest features ${VCPKG_MANIFEST_FEATURES}") message("-- CMAKE_TOOLCHAIN_FILE Cmake toolchain file ${CMAKE_TOOLCHAIN_FILE}") - message("-- TARGET_PLATFORM Target platform ${CMAKE_SYSTEM_NAME} ${ARCHITECTURE}") + message("-- TARGET_PLATFORM Target platform ${CMAKE_HOST_SYSTEM_NAME} ${ARCHITECTURE}") message("-- BUILD_STATIC Build static ${BUILD_STATIC}") message("-- COVERAGE Build code coverage ${COVERAGE}") message("-- TESTS Build tests ${TESTS}") diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index 1c0c49ea..da46c575 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -125,7 +125,7 @@ class IFront : virtual public IFrontClient std::string const& topic, ppc::protocol::MessageDispatcherCallback callback) = 0; /////// to simplify SDK wrapper //// - virtual void registerTopicHandler( + virtual void register_topic_handler( std::string const& topic, MessageDispatcherHandler::Ptr callback) { registerTopicHandler(topic, populateMessageDispatcherCallback(callback)); @@ -135,7 +135,7 @@ class IFront : virtual public IFrontClient std::string const& componentType, ppc::protocol::MessageDispatcherCallback callback) = 0; /////// to simplify SDK wrapper //// - virtual void registerMessageHandler( + virtual void register_msg_handler( std::string const& componentType, MessageDispatcherHandler::Ptr callback) { registerMessageHandler(componentType, populateMessageDispatcherCallback(callback)); @@ -165,7 +165,7 @@ class IFront : virtual public IFrontClient // !!! Note: the 'payload' type(char*) should not been changed, since it used to pass-in java // byte[] data - virtual void asyncSendMessage(uint16_t routeType, + virtual void async_send_message(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, uint64_t payloadSize, int seq, long timeout, ErrorCallback::Ptr errorCallback, IMessageHandler::Ptr msgHandler) @@ -183,7 +183,7 @@ class IFront : virtual public IFrontClient // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java // byte[] data - virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, + virtual void async_send_response(bcos::bytes const& dstNode, std::string const& traceID, bcos::bytes&& payload, int seq, ErrorCallback::Ptr errorCallback) { asyncSendResponse( diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh index f254e7cf..b3e3f555 100644 --- a/cpp/tools/install_depends.sh +++ b/cpp/tools/install_depends.sh @@ -67,7 +67,7 @@ install_centos_depends() { LOG_INFO "install depends for centos ..." # install the basic package - sudo yum install -y bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf + sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf LOG_INFO "install basic-package-depends for centos success..." LOG_INFO "install gsasl ... " @@ -85,7 +85,7 @@ install_ubuntu_depends() { LOG_INFO "install depends for ubuntu ..." # install the basic package - sudo apt install -y bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf + sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf LOG_INFO "install basic-package-depends for ubuntu success..." LOG_INFO "install gsasl ... " @@ -106,7 +106,7 @@ install_ubuntu_depends() install_macos_depends() { LOG_INFO "install depends for macos ..." - brew install clang cmake autoconf gsasl nasm + brew install clang cmake autoconf gsasl nasm lcov LOG_INFO "install depends for macos success ..." } diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json index a6334850..81446362 100644 --- a/cpp/vcpkg-configuration.json +++ b/cpp/vcpkg-configuration.json @@ -2,8 +2,8 @@ "registries": [ { "kind": "git", - "repository": "https://github.com/cyjseagull/registry", - "baseline": "6160f4167d4ab801f16a18bdd842a75edf7b069e", + "repository": "https://github.com/FISCO-BCOS/registry", + "baseline": "b483a0287c6ee7017983942af540e3ee672b0994", "packages": [ "openssl", "bcos-utilities", @@ -13,4 +13,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 83449a54..7ff9a24e 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -38,7 +38,7 @@ }, { "name": "libhdfs3", - "version": "2024-04-27" + "version": "2024-04-27#1" }, { "name": "tbb", @@ -58,8 +58,25 @@ } ], "features": { - "sse": { - "description": "Enable SSE4.2 for libhdfs3" + "sse-libhdfs3": { + "description": "Enable SSE4.2 for libhdfs3", + "dependencies":[ + { + "name": "libhdfs3", + "version>=": "2024-04-27#1", + "features": ["sse"] + } + ] + }, + "default-libhdfs3": { + "description": "Disable SSE4.2 for libhdfs3", + "dependencies":[ + { + "name": "libhdfs3", + "version>=": "2024-04-27#1", + "features": [] + } + ] }, "cpufeatures":{ "description": "Enable cpu features", @@ -98,7 +115,6 @@ "name": "grpc", "version>=": "1.51.1" }, - "libhdfs3", "tarscpp", "tbb", "libxml2" @@ -121,4 +137,4 @@ ] } } -} \ No newline at end of file +} diff --git a/cpp/wedpr-main/common/NodeStarter.h b/cpp/wedpr-main/common/NodeStarter.h index e33ad7ef..71dff545 100644 --- a/cpp/wedpr-main/common/NodeStarter.h +++ b/cpp/wedpr-main/common/NodeStarter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java index 757d597a..7c26c4e0 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -21,6 +21,7 @@ import com.webank.wedpr.sdk.jni.transport.TransportConfig; import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl; import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint; @@ -28,6 +29,10 @@ public class TransportDemo { public static class MessageDispatcherCallbackImpl extends MessageDispatcherCallback { private final String nodeID; + // java -cp 'conf/:lib/*:apps/*' com.webank.wedpr.sdk.jni.demo.TransportDemo agency0Node + // "127.0.0.1" 9020 "ipv4:127.0.0.1:40600,127.0.0.1:40601" "agency1Node" + // java -cp 'conf/:lib/*:apps/*' com.webank.wedpr.sdk.jni.demo.TransportDemo agency1Node + // "127.0.0.1" 9021 "ipv4:127.0.0.1:40620,127.0.0.1:40621" "agency0Node" public MessageDispatcherCallbackImpl(String nodeID) { this.nodeID = nodeID; } @@ -45,10 +50,10 @@ public void onMessage(IMessage message) { } } - public static class MessageErrorCallback extends ErrorCallback { + public static class MessageErrorCallbackImpl extends MessageErrorCallback { private final String nodeID; - public MessageErrorCallback(String nodeID) { + public MessageErrorCallbackImpl(String nodeID) { this.nodeID = nodeID; } @@ -101,7 +106,9 @@ public static void main(String[] args) throws Exception { // send Message to the gateway String topic = "testTopic"; - transport.registerTopicHandler(topic, new MessageDispatcherCallbackImpl(nodeID)); + MessageDispatcherCallback messageDispatcherCallback = + new MessageDispatcherCallbackImpl(nodeID); + transport.registerTopicHandler(topic, messageDispatcherCallback); System.out.println("##### register topic success"); byte[] dstNodeBytes = dstNode.getBytes(); @@ -118,7 +125,7 @@ public static void main(String[] args) throws Exception { payLoad.getBytes(), 0, 10000, - new MessageErrorCallback(nodeID), + new MessageErrorCallbackImpl(nodeID), null); // push diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java index c7de68ff..32fb72fc 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java @@ -112,4 +112,8 @@ public SWIGTYPE_p_std__vectorT_std__string_t mutableComponents() { return new SWIGTYPE_p_std__vectorT_std__string_t( wedpr_java_transportJNI.FrontConfig_mutableComponents(swigCPtr, this), false); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java index 75164877..ce4d2352 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java @@ -59,41 +59,13 @@ public void stop() { wedpr_java_transportJNI.IFront_stop(swigCPtr, this); } - /** - *
- * - * @param topic the topic
- * @param callback the callback called when receive specified topic - */ - public void registerTopicHandler( - String topic, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t callback) { - wedpr_java_transportJNI.IFront_registerTopicHandler__SWIG_0( - swigCPtr, - this, - topic, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t - .getCPtr(callback)); - } - - public void registerTopicHandler(String topic, MessageDispatcherHandler callback) { - wedpr_java_transportJNI.IFront_registerTopicHandler__SWIG_1( + public void register_topic_handler(String topic, MessageDispatcherHandler callback) { + wedpr_java_transportJNI.IFront_register_topic_handler( swigCPtr, this, topic, MessageDispatcherHandler.getCPtr(callback), callback); } - public void registerMessageHandler( - String componentType, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t callback) { - wedpr_java_transportJNI.IFront_registerMessageHandler__SWIG_0( - swigCPtr, - this, - componentType, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t - .getCPtr(callback)); - } - - public void registerMessageHandler(String componentType, MessageDispatcherHandler callback) { - wedpr_java_transportJNI.IFront_registerMessageHandler__SWIG_1( + public void register_msg_handler(String componentType, MessageDispatcherHandler callback) { + wedpr_java_transportJNI.IFront_register_msg_handler( swigCPtr, this, componentType, @@ -101,48 +73,7 @@ public void registerMessageHandler(String componentType, MessageDispatcherHandle callback); } - /** - * async send message
- *
- * - * @param routeType the route type
- * @param routeInfo the route info, include
- * - topic the topic
- * - dstInst the dst agency(must set when 'route by agency' and 'route by
- * component')
- * - dstNodeID the dst nodeID(must set when 'route by nodeID')
- * - componentType the componentType(must set when 'route by component')
- * @param payload the payload to send
- * @param seq the message seq
- * @param timeout timeout
- * @param callback callback - */ - public void asyncSendMessage( - int routeType, - MessageOptionalHeader routeInfo, - ubytes payload, - int seq, - int timeout, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t errorCallback, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t - callback) { - wedpr_java_transportJNI.IFront_asyncSendMessage__SWIG_0( - swigCPtr, - this, - routeType, - MessageOptionalHeader.getCPtr(routeInfo), - routeInfo, - ubytes.swigRelease(payload), - payload, - seq, - timeout, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.getCPtr( - errorCallback), - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t - .getCPtr(callback)); - } - - public void asyncSendMessage( + public void async_send_message( int routeType, MessageOptionalHeader routeInfo, byte[] payload, @@ -151,7 +82,7 @@ public void asyncSendMessage( int timeout, ErrorCallback errorCallback, IMessageHandler msgHandler) { - wedpr_java_transportJNI.IFront_asyncSendMessage__SWIG_1( + wedpr_java_transportJNI.IFront_async_send_message( swigCPtr, this, routeType, @@ -167,28 +98,9 @@ public void asyncSendMessage( msgHandler); } - public void asyncSendResponse( - ubytes dstNode, - String traceID, - ubytes payload, - int seq, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t errorCallback) { - wedpr_java_transportJNI.IFront_asyncSendResponse__SWIG_0( - swigCPtr, - this, - ubytes.getCPtr(dstNode), - dstNode, - traceID, - ubytes.swigRelease(payload), - payload, - seq, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t.getCPtr( - errorCallback)); - } - - public void asyncSendResponse( + public void async_send_response( ubytes dstNode, String traceID, ubytes payload, int seq, ErrorCallback errorCallback) { - wedpr_java_transportJNI.IFront_asyncSendResponse__SWIG_1( + wedpr_java_transportJNI.IFront_async_send_response( swigCPtr, this, ubytes.getCPtr(dstNode), @@ -248,16 +160,6 @@ public Message peek(String topic) { return (cPtr == 0) ? null : new Message(cPtr, true); } - public void asyncGetAgencies( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t - callback) { - wedpr_java_transportJNI.IFront_asyncGetAgencies( - swigCPtr, - this, - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t - .getCPtr(callback)); - } - /** * register the nodeInfo to the gateway
* diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java index 68091c7c..32ddfb70 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java @@ -133,4 +133,8 @@ public long decode(SWIGTYPE_p_bcos__bytesConstRef _buffer) { return wedpr_java_transportJNI.Message_decode( swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(_buffer)); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java index 769d0dae..d2f7ecaf 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageBuilder.java @@ -67,4 +67,8 @@ public Message build( payload); return (cPtr == 0) ? null : new Message(cPtr, true); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java index f8c215a4..12dbd933 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeader.java @@ -150,4 +150,8 @@ public void setRouteType(SWIGTYPE_p_ppc__protocol__RouteType type) { public boolean hasOptionalField() { return wedpr_java_transportJNI.MessageHeader_hasOptionalField(swigCPtr, this); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java index 4b3db731..71d876b4 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageHeaderBuilder.java @@ -61,4 +61,8 @@ public MessageOptionalHeader build(MessageOptionalHeader optionalHeader) { optionalHeader); return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java index 8ad6a360..2e09fca4 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java @@ -126,4 +126,8 @@ public String srcInst() { public void setSrcInst(String srcInst) { wedpr_java_transportJNI.MessageOptionalHeader_setSrcInst(swigCPtr, this, srcInst); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java index 16187d11..0078ee10 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeaderBuilder.java @@ -55,4 +55,8 @@ public MessageOptionalHeader build() { wedpr_java_transportJNI.MessageOptionalHeaderBuilder_build__SWIG_1(swigCPtr, this); return (cPtr == 0) ? null : new MessageOptionalHeader(cPtr, true); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java index 8c62f8aa..082e6f3d 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java @@ -106,4 +106,8 @@ public void setRespPacket() { public boolean isRespPacket() { return wedpr_java_transportJNI.MessagePayload_isRespPacket(swigCPtr, this); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java index c4d1f674..064a2c95 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayloadBuilder.java @@ -51,4 +51,8 @@ public MessagePayload build(SWIGTYPE_p_bcos__bytesConstRef buffer) { swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(buffer)); return (cPtr == 0) ? null : new MessagePayload(cPtr, true); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java deleted file mode 100644 index a0f6b854..00000000 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t.java +++ /dev/null @@ -1,37 +0,0 @@ -/* ---------------------------------------------------------------------------- - * This file was automatically generated by SWIG (https://www.swig.org). - * Version 4.2.1 - * - * Do not make changes to this file unless you know what you are doing - modify - * the SWIG interface file instead. - * ----------------------------------------------------------------------------- */ - -package com.webank.wedpr.sdk.jni.generated; - -public -class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t { - private transient long swigCPtr; - - protected - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t( - long cPtr, @SuppressWarnings("unused") boolean futureUse) { - swigCPtr = cPtr; - } - - protected - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t() { - swigCPtr = 0; - } - - protected static long getCPtr( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t - obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } - - protected static long swigRelease( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__setT_std__string_tF_t - obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } -} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java deleted file mode 100644 index 8b62167d..00000000 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t.java +++ /dev/null @@ -1,37 +0,0 @@ -/* ---------------------------------------------------------------------------- - * This file was automatically generated by SWIG (https://www.swig.org). - * Version 4.2.1 - * - * Do not make changes to this file unless you know what you are doing - modify - * the SWIG interface file instead. - * ----------------------------------------------------------------------------- */ - -package com.webank.wedpr.sdk.jni.generated; - -public -class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t { - private transient long swigCPtr; - - protected - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t( - long cPtr, @SuppressWarnings("unused") boolean futureUse) { - swigCPtr = cPtr; - } - - protected - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t() { - swigCPtr = 0; - } - - protected static long getCPtr( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t - obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } - - protected static long swigRelease( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t - obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } -} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java deleted file mode 100644 index 073a71ab..00000000 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t.java +++ /dev/null @@ -1,32 +0,0 @@ -/* ---------------------------------------------------------------------------- - * This file was automatically generated by SWIG (https://www.swig.org). - * Version 4.2.1 - * - * Do not make changes to this file unless you know what you are doing - modify - * the SWIG interface file instead. - * ----------------------------------------------------------------------------- */ - -package com.webank.wedpr.sdk.jni.generated; - -public class SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t { - private transient long swigCPtr; - - protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t( - long cPtr, @SuppressWarnings("unused") boolean futureUse) { - swigCPtr = cPtr; - } - - protected SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t() { - swigCPtr = 0; - } - - protected static long getCPtr( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } - - protected static long swigRelease( - SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t obj) { - return (obj == null) ? 0 : obj.swigCPtr; - } -} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java index 285e1c55..170e3992 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Transport.java @@ -86,4 +86,8 @@ public FrontConfig getConfig() { long cPtr = wedpr_java_transportJNI.Transport_getConfig(swigCPtr, this); return (cPtr == 0) ? null : new FrontConfig(cPtr, true); } + + public void disOwnMemory() { + swigSetCMemOwn(false); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java index ed489a57..5a77b230 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java @@ -52,19 +52,6 @@ public TransportBuilder() { this(wedpr_java_transportJNI.new_TransportBuilder(), true); } - public Transport build( - SDKMode mode, FrontConfig config, SWIGTYPE_p_ppc__gateway__IGateway__Ptr gateway) { - long cPtr = - wedpr_java_transportJNI.TransportBuilder_build( - swigCPtr, - this, - mode.swigValue(), - FrontConfig.getCPtr(config), - config, - SWIGTYPE_p_ppc__gateway__IGateway__Ptr.getCPtr(gateway)); - return (cPtr == 0) ? null : new Transport(cPtr, true); - } - public Transport buildProTransport(FrontConfig config) { long cPtr = wedpr_java_transportJNI.TransportBuilder_buildProTransport( diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java index debebac1..121320fd 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java @@ -653,32 +653,13 @@ public static final native void IMessageHandler_change_ownership( public static final native void IFront_stop(long jarg1, IFront jarg1_); - public static final native void IFront_registerTopicHandler__SWIG_0( - long jarg1, IFront jarg1_, String jarg2, long jarg3); - - public static final native void IFront_registerTopicHandler__SWIG_1( + public static final native void IFront_register_topic_handler( long jarg1, IFront jarg1_, String jarg2, long jarg3, MessageDispatcherHandler jarg3_); - public static final native void IFront_registerMessageHandler__SWIG_0( - long jarg1, IFront jarg1_, String jarg2, long jarg3); - - public static final native void IFront_registerMessageHandler__SWIG_1( + public static final native void IFront_register_msg_handler( long jarg1, IFront jarg1_, String jarg2, long jarg3, MessageDispatcherHandler jarg3_); - public static final native void IFront_asyncSendMessage__SWIG_0( - long jarg1, - IFront jarg1_, - int jarg2, - long jarg3, - MessageOptionalHeader jarg3_, - long jarg4, - ubytes jarg4_, - int jarg5, - int jarg6, - long jarg7, - long jarg8); - - public static final native void IFront_asyncSendMessage__SWIG_1( + public static final native void IFront_async_send_message( long jarg1, IFront jarg1_, int jarg2, @@ -693,18 +674,7 @@ public static final native void IFront_asyncSendMessage__SWIG_1( long jarg9, IMessageHandler jarg9_); - public static final native void IFront_asyncSendResponse__SWIG_0( - long jarg1, - IFront jarg1_, - long jarg2, - ubytes jarg2_, - String jarg3, - long jarg4, - ubytes jarg4_, - int jarg5, - long jarg6); - - public static final native void IFront_asyncSendResponse__SWIG_1( + public static final native void IFront_async_send_response( long jarg1, IFront jarg1_, long jarg2, @@ -742,8 +712,6 @@ public static final native long IFront_push__SWIG_1( public static final native long IFront_peek(long jarg1, IFront jarg1_, String jarg2); - public static final native void IFront_asyncGetAgencies(long jarg1, IFront jarg1_, long jarg2); - public static final native long IFront_registerNodeInfo(long jarg1, IFront jarg1_, long jarg2); public static final native long IFront_unRegisterNodeInfo(long jarg1, IFront jarg1_); @@ -765,14 +733,6 @@ public static final native long IFrontBuilder_buildClient( public static final native void delete_TransportBuilder(long jarg1); - public static final native long TransportBuilder_build( - long jarg1, - TransportBuilder jarg1_, - int jarg2, - long jarg3, - FrontConfig jarg3_, - long jarg4); - public static final native long TransportBuilder_buildProTransport( long jarg1, TransportBuilder jarg1_, long jarg2, FrontConfig jarg2_); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java index c97bef67..6b61f213 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java @@ -16,10 +16,21 @@ package com.webank.wedpr.sdk.jni.transport; import com.webank.wedpr.sdk.jni.generated.Message; +import com.webank.wedpr.sdk.jni.generated.MessageOptionalHeader; +import com.webank.wedpr.sdk.jni.generated.MessageOptionalHeaderBuilder; import com.webank.wedpr.sdk.jni.transport.impl.MessageImpl; public class IMessageBuilder { public static IMessage build(Message msg) { return new MessageImpl(msg); } + + public static MessageOptionalHeader buildRouteInfo( + MessageOptionalHeaderBuilder routeInfoBuilder, String topic) { + // return the ownership to cpp, since it is created by cpp + MessageOptionalHeader routeInfo = routeInfoBuilder.build(); + routeInfo.setTopic(topic); + routeInfo.disOwnMemory(); + return routeInfo; + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java index e56d78bc..68f8232a 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java @@ -53,6 +53,7 @@ public static TransportBuilder getTransportBuilder() { public TransportConfig(Integer threadPoolSize, String nodeID) { this.frontConfig = transportBuilder.buildConfig(threadPoolSize, nodeID); + this.frontConfig.disOwnMemory(); // set default grpcConfig setGrpcConfig(new TransportGrpcConfig()); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java index f8431244..97146fde 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java @@ -16,9 +16,9 @@ package com.webank.wedpr.sdk.jni.transport; import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; -import com.webank.wedpr.sdk.jni.generated.ErrorCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; public interface WeDPRTransport { // start the transport @@ -82,7 +82,7 @@ void asyncSendMessageByNodeID( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback); /** @@ -101,7 +101,7 @@ void asyncSendMessageByAgency( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback); void asyncSendMessageByComponent( @@ -111,7 +111,7 @@ void asyncSendMessageByComponent( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback); /** @@ -129,7 +129,7 @@ void asyncSendMessageByTopic( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback); /** @param topic the topic to remove */ diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java index 20473f7c..9b2b67d0 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java @@ -26,6 +26,12 @@ public abstract class MessageCallback extends IMessageHandler { public abstract void onMessage( Error error, IMessage message, SendResponseHandler sendResponseHandler); + // release the ownership to c++, in case of it's released by the jvm + protected void finalize() { + swigReleaseOwnership(); + delete(); + } + @Override public void onMessage(Error e, Message msg, SendResponseHandler sendResponseHandler) { onMessage(e, IMessageBuilder.build(msg), sendResponseHandler); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java index 1f02bf5c..eda2ef65 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java @@ -23,6 +23,13 @@ public abstract class MessageDispatcherCallback extends MessageDispatcherHandler { public abstract void onMessage(IMessage message); + // TODO: check this will cause memory leak or not + // release the ownership to c++, in case of it's released by the jvm + protected void finalize() { + swigReleaseOwnership(); + delete(); + } + @Override public void onMessage(Message msg) { onMessage(IMessageBuilder.build(msg)); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java new file mode 100644 index 00000000..0d85b06a --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java @@ -0,0 +1,28 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.handlers; + +import com.webank.wedpr.sdk.jni.generated.ErrorCallback; + +public abstract class MessageErrorCallback extends ErrorCallback { + + // release the ownership to c++, in case of it's released by the jvm + @Override + protected void finalize() { + swigReleaseOwnership(); + delete(); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java index ebbaed1d..8edcc1b6 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/MessageImpl.java @@ -25,6 +25,8 @@ public class MessageHeaderImpl implements IMessageHeader { protected MessageHeaderImpl(MessageHeader messageHeader) { this.messageHeader = messageHeader; + this.messageHeader.optionalField().disOwnMemory(); + this.messageHeader.disOwnMemory(); } @Override @@ -124,6 +126,7 @@ public String toString() { // v1, v2 public MessageImpl(Message rawMessage) { this.rawMessage = rawMessage; + this.rawMessage.disOwnMemory(); this.messageHeader = new MessageHeaderImpl(rawMessage.header()); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java index 142cf225..ebc36f26 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -26,12 +26,13 @@ import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; import java.math.BigInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TransportImpl implements WeDPRTransport { - private static Logger logger = LoggerFactory.getLogger(TransportImpl.class); + private static final Logger logger = LoggerFactory.getLogger(TransportImpl.class); // the created transport private final Transport transport; private final TransportConfig transportConfig; @@ -46,6 +47,7 @@ public static WeDPRTransport build(TransportConfig transportConfig) { protected TransportImpl(Transport transport, TransportConfig transportConfig) { logger.info("Build Transport, config: {}", transportConfig.toString()); this.transport = transport; + this.transport.disOwnMemory(); this.transportConfig = transportConfig; } @@ -105,7 +107,7 @@ public void unRegisterTopic(String topic) throws Exception { @Override public void registerTopicHandler(String topic, MessageDispatcherCallback messageHandler) { - this.transport.getFront().registerTopicHandler(topic, messageHandler); + this.transport.getFront().register_topic_handler(topic, messageHandler); } /** @@ -125,14 +127,14 @@ public void asyncSendMessageByNodeID( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback) { - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstNode(dstNode, BigInteger.valueOf(dstNode.length)); - routeInfo.setTopic(topic); this.transport .getFront() - .asyncSendMessage( + .async_send_message( RouteType.ROUTE_THROUGH_NODEID.ordinal(), routeInfo, payload, @@ -160,14 +162,14 @@ public void asyncSendMessageByAgency( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback) { - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstInst(agency); this.transport .getFront() - .asyncSendMessage( + .async_send_message( RouteType.ROUTE_THROUGH_AGENCY.ordinal(), routeInfo, payload, @@ -186,16 +188,16 @@ public void asyncSendMessageByComponent( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback) { // set the routeInfo - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstInst(dstInst); routeInfo.setComponentType(component); this.transport .getFront() - .asyncSendMessage( + .async_send_message( RouteType.ROUTE_THROUGH_COMPONENT.ordinal(), routeInfo, payload, @@ -222,15 +224,15 @@ public void asyncSendMessageByTopic( byte[] payload, int seq, int timeout, - ErrorCallback errorCallback, + MessageErrorCallback errorCallback, MessageCallback msgCallback) { // set the routeInfo - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstInst(dstInst); this.transport .getFront() - .asyncSendMessage( + .async_send_message( RouteType.ROUTE_THROUGH_TOPIC.ordinal(), routeInfo, payload, @@ -252,8 +254,8 @@ public void removeTopic(String topic) throws WeDPRSDKException { @Override public void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload, int timeout) throws WeDPRSDKException { - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstNode(dstNodeID, BigInteger.valueOf(dstNodeID.length)); Error result = this.transport @@ -272,8 +274,8 @@ public void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload public void pushByComponent( String topic, String dstInst, String component, int seq, byte[] payload, int timeout) throws WeDPRSDKException { - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstInst(dstInst); routeInfo.setComponentType(component); Error result = @@ -292,8 +294,8 @@ public void pushByComponent( @Override public void pushByInst(String topic, String dstInst, int seq, byte[] payload, int timeout) throws WeDPRSDKException { - MessageOptionalHeader routeInfo = this.transport.routeInfoBuilder().build(); - routeInfo.setTopic(topic); + MessageOptionalHeader routeInfo = + IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); routeInfo.setDstInst(dstInst); Error result = this.transport diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index c5cf2638..2c656f17 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -179,6 +179,44 @@ namespace bcos{ %typemap(javain) OutputBuffer "$javainput" %typemap(javaout) OutputBuffer { return $jnicall; } + + +// refer to: https://stackoverflow.com/questions/12103206/is-it-possible-to-add-text-to-an-existing-typemap-in-swig +%define WRAP(CLASS) +%extend CLASS { +%proxycode %{ + public void disOwnMemory() { + swigSetCMemOwn(false); + } +%} +} +%enddef + +// Note: these object is created from cpp, and maintained with shared_ptr, +// the java code should disOwnMemory in case of released multiple times +WRAP(ppc::front::FrontConfig) +WRAP(ppc::protocol::Message) +WRAP(ppc::protocol::MessageOptionalHeader) +WRAP(ppc::protocol::MessageHeader) +WRAP(ppc::protocol::MessagePayload) +WRAP(ppc::protocol::MessageBuilder) +WRAP(ppc::protocol::MessageHeaderBuilder) +WRAP(ppc::protocol::MessagePayloadBuilder) +WRAP(ppc::protocol::MessageOptionalHeaderBuilder) +WRAP(ppc::sdk::Transport) + +// the method no need to wrapper +%ignore ppc::sdk::TransportBuilder::build; +%ignore ppc::front::IFront::onReceiveMessage; +%ignore ppc::front::IFront::asyncSendMessage; +%ignore ppc::front::IFront::asyncGetAgencies; +%ignore ppc::front::IFront::registerTopicHandler; +%ignore ppc::front::IFront::registerMessageHandler; +%ignore ppc::front::IFront::asyncSendResponse; +%ignore ppc::front::IFront::populateErrorCallback; +%ignore ppc::front::IFront::populateMessageDispatcherCallback; +%ignore ppc::front::IFront::populateMsgCallback; + /* ///// tests /// %inline { From a0c25384a88cc59cc091d92a520cd4cc12088938 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sat, 14 Sep 2024 17:54:05 +0800 Subject: [PATCH 041/120] fix uint16_t not initilized caused bug (#38) * fix uint16_t not initilized caused bug * fix message encode not return bug * remove databatch dependency to tbb --- cpp/ppc-framework/io/DataBatch.h | 13 +------------ cpp/ppc-framework/protocol/Message.h | 8 +++++--- cpp/ppc-framework/protocol/MessagePayload.h | 7 ++++--- .../ppc-psi/src/cm2020-psi/Common.h | 15 ++++++++++++++- .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 2 +- .../src/cm2020-psi/core/CM2020PSISender.cpp | 2 +- .../ecdh-multi-psi/core/EcdhMultiPSICalculator.h | 1 + .../src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h | 1 + .../protocol/src/v1/MessageHeaderImpl.cpp | 8 ++++---- .../protocol/src/v1/MessageImpl.cpp | 1 + cpp/wedpr-protocol/protocol/tests/MessageTest.cpp | 5 ++--- cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt | 2 +- cpp/wedpr-storage/ppc-io/src/CMakeLists.txt | 1 + cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt | 2 +- .../ppc-front/ppc-front/FrontImpl.cpp | 5 +++-- .../ppc-gateway/gateway/router/LocalRouter.cpp | 2 +- .../sdk-wrapper/java/bindings/build.gradle | 2 +- 17 files changed, 43 insertions(+), 34 deletions(-) diff --git a/cpp/ppc-framework/io/DataBatch.h b/cpp/ppc-framework/io/DataBatch.h index 46ace021..b0da8d2e 100644 --- a/cpp/ppc-framework/io/DataBatch.h +++ b/cpp/ppc-framework/io/DataBatch.h @@ -20,7 +20,6 @@ #pragma once #include "../Common.h" #include -#include #include #include #include @@ -199,18 +198,8 @@ class DataBatch DataSchema getDataSchema() const { return m_dataSchema; } - uint32_t dedup() - { - if (!m_data || m_data->empty()) - { - return 0; - } - tbb::parallel_sort(m_data->begin(), m_data->end()); - auto unique_end = std::unique(m_data->begin(), m_data->end()); - m_data->erase(unique_end, m_data->end()); - return m_data->size(); - } + std::shared_ptr>& mutableData() { return m_data; } private: std::shared_ptr> m_data; diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 2988fc4b..291b1d2e 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -159,6 +159,7 @@ class MessageHeader virtual bool hasOptionalField() const = 0; protected: + // Note: must init here to 0, otherwise, it will be unexpected value in some other platform // the msg version, used to support compatibility uint8_t m_version = 0; // the traceID @@ -168,11 +169,11 @@ class MessageHeader // the dstGwNode std::string m_dstGwNode; // the packetType - uint16_t m_packetType; + uint16_t m_packetType = 0; // the ttl int16_t m_ttl = 0; // the ext(contains the router policy and response flag) - uint16_t m_ext; + uint16_t m_ext = 0; //// the optional field(used to route between components and nodes) MessageOptionalHeader::Ptr m_optionalField; uint16_t mutable m_length; @@ -324,7 +325,8 @@ inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg) } std::ostringstream stringstream; stringstream << LOG_KV("rsp", _msg->isRespPacket()) << LOG_KV("traceID", _msg->seq()) - << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length()); + << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length()) + << LOG_KV("ext", _msg->ext()); return stringstream.str(); } diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h index a8e6a57c..0a612a97 100644 --- a/cpp/ppc-framework/protocol/MessagePayload.h +++ b/cpp/ppc-framework/protocol/MessagePayload.h @@ -69,13 +69,14 @@ class MessagePayload protected: // the front payload version, used to support compatibility - uint8_t m_version; + // Note: must init here to 0, otherwise, it will be unexpected value in some other platform + uint8_t m_version = 0; // the seq - uint16_t m_seq; + uint16_t m_seq = 0; // the traceID std::string m_traceID; bcos::bytes m_data; - uint16_t m_ext; + uint16_t m_ext = 0; int64_t mutable m_length; }; diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h index 237f1a65..ccfffdbe 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h @@ -21,9 +21,9 @@ #include "ppc-framework/Common.h" #include "ppc-framework/protocol/PPCMessageFace.h" - #include #include +#include namespace ppc::psi @@ -65,4 +65,17 @@ enum class CM2020PSIRetCode : int INVALID_TASK_PARAM = -3002 }; +inline uint32_t dedupDataBatch(ppc::io::DataBatch::Ptr dataBatch) +{ + if (!dataBatch || dataBatch->mutableData() == nullptr || dataBatch->mutableData()->empty()) + { + return 0; + } + auto& data = dataBatch->mutableData(); + tbb::parallel_sort(data->begin(), data->end()); + auto unique_end = std::unique(data->begin(), data->end()); + data->erase(unique_end, data->end()); + return data->size(); +} + } // namespace ppc::psi diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp index 2e4f46b0..0f4c92d1 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp @@ -908,7 +908,7 @@ void CM2020PSIReceiver::saveResults() } CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("originCount", finalResults->size()); - m_resultCount = finalResults->dedup(); + m_resultCount = dedupDataBatch(finalResults); CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("resultCount", m_resultCount); m_taskState->writeLines(finalResults, DataSchema::Bytes); diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp index 860f8bfd..d4018dc4 100644 --- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp +++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp @@ -808,7 +808,7 @@ void CM2020PSISender::saveResults() } CM2020_PSI_LOG(INFO) << LOG_BADGE("before dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("originCount", finalResults->size()); - m_resultCount = finalResults->dedup(); + m_resultCount = dedupDataBatch(finalResults); CM2020_PSI_LOG(INFO) << LOG_BADGE("after dedup") << LOG_KV("taskID", m_taskID) << LOG_KV("resultCount", m_resultCount); m_taskState->writeLines(finalResults, DataSchema::Bytes); diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h index ea2459ad..d024b335 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h @@ -2,6 +2,7 @@ #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" #include "ppc-psi/src/psi-framework/TaskState.h" +#include namespace ppc::psi { diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h index 4a3076a6..30b57041 100644 --- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h +++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h @@ -2,6 +2,7 @@ #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h" #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h" #include "ppc-psi/src/psi-framework/TaskState.h" +#include namespace ppc::psi { diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp index 6690dd07..8dc17520 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp @@ -167,13 +167,13 @@ int64_t MessageHeaderImpl::decode(bcos::bytesConstRef data) uint16_t MessageHeaderImpl::routeType() const { - if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByComponent) + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByNodeID) { - return (uint16_t)RouteType::ROUTE_THROUGH_COMPONENT; + return (uint16_t)RouteType::ROUTE_THROUGH_NODEID; } - if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByAgency) + if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByComponent) { - return (uint16_t)RouteType::ROUTE_THROUGH_AGENCY; + return (uint16_t)RouteType::ROUTE_THROUGH_COMPONENT; } if (m_ext & (uint16_t)ppc::gateway::GatewayMsgExtFlag::RouteByAgency) { diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp index 361029e6..9b156941 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp @@ -33,6 +33,7 @@ bool MessageImpl::encode(bcos::bytes& _buffer) { _buffer.insert(_buffer.end(), m_payload->begin(), m_payload->end()); } + return true; } bool MessageImpl::encode(bcos::boostssl::EncodedMsg& encodedMsg) diff --git a/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp index cf7bfce5..ea262f9d 100644 --- a/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp +++ b/cpp/wedpr-protocol/protocol/tests/MessageTest.cpp @@ -110,15 +110,14 @@ BOOST_AUTO_TEST_CASE(testMessage) bcos::bytes dstNode; std::string srcInst; std::string dstInst; - std::shared_ptr payload; + std::shared_ptr payload = nullptr; auto msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, componentType, srcNode, srcInst, dstNode, dstInst, payload); checkEncodeDecode(msgBuilder, msg); // with payload - payload = std::make_shared(); std::string payloadData = "payloadf@#$@#$sdfs234"; - *payload = bcos::bytes(payloadData.begin(), payloadData.end()); + payload = std::make_shared(payloadData.begin(), payloadData.end()); msg = fakeMsg(msgBuilder, version, traceID, srcGwNode, dstGwNode, packetType, ttl, ext, topic, componentType, srcNode, srcInst, dstNode, dstInst, payload); checkEncodeDecode(msgBuilder, msg); diff --git a/cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt index 6daccc8c..0b3611d8 100644 --- a/cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt +++ b/cpp/wedpr-storage/ppc-io/demo/CMakeLists.txt @@ -7,5 +7,5 @@ set(BINARY_NAME io-demo) add_executable(${BINARY_NAME} ${SOURCES}) target_include_directories(${BINARY_NAME} PRIVATE .) -target_link_libraries(${BINARY_NAME} ${IO_TARGET} TBB::tbb) +target_link_libraries(${BINARY_NAME} ${IO_TARGET}) unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt index 4be7d7a8..d26ddab5 100644 --- a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt +++ b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt @@ -1,3 +1,4 @@ file(GLOB_RECURSE SRCS *.cpp) add_library(${IO_TARGET} ${SRCS}) +# Note: the DataBatch depends on tbb target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB}) \ No newline at end of file diff --git a/cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt index 993e8bc0..549e861c 100644 --- a/cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt +++ b/cpp/wedpr-storage/ppc-io/tests/CMakeLists.txt @@ -6,5 +6,5 @@ set(TEST_BINARY_NAME test-ppc-io) add_executable(${TEST_BINARY_NAME} ${SOURCES}) target_include_directories(${TEST_BINARY_NAME} PRIVATE .) -target_link_libraries(${TEST_BINARY_NAME} ${IO_TARGET} ${BOOST_UNIT_TEST} TBB::tbb) +target_link_libraries(${TEST_BINARY_NAME} ${IO_TARGET} ${BOOST_UNIT_TEST}) add_test(NAME test-io WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME}) diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index 58aae243..ca774092 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -224,8 +224,9 @@ void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::P routeInfo->setSrcNode(m_nodeID); auto payload = std::make_shared(); frontMessage->encode(*payload); - FRONT_LOG(TRACE) << LOG_DESC("asyncSendMessageToGateway") << LOG_KV("traceID", traceID) - << LOG_KV("route", printOptionalField(routeInfo)); + FRONT_LOG(TRACE) << LOG_DESC("asyncSendMessageToGateway") << LOG_KV("routeType", routeType) + << LOG_KV("traceID", traceID) << printOptionalField(routeInfo) + << LOG_KV("payloadSize", frontMessage->length()); m_gatewayClient->asyncSendMessage( routeType, routeInfo, traceID, std::move(*payload), timeout, callback); } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp index 513f6b5f..9b910408 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp @@ -98,7 +98,7 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call return; } LOCAL_ROUTER_LOG(WARNING) << LOG_DESC("dispatcherMessage to front failed") - << LOG_KV("code", error->errorMessage()) + << LOG_KV("code", error->errorCode()) << LOG_KV("msg", error->errorMessage()); }); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index 42409146..67e5940e 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -86,7 +86,7 @@ dependencies { annotationProcessor("org.projectlombok:lombok:${lombokVersion}") } -archivesBaseName = 'wedpr-java-transport-jni' +archivesBaseName = 'wedpr-gateway-sdk' group = 'com.webank.wedpr' version = '1.0.0-SNAPSHOT' From bd22f9f581bf89d05d47217b7f8798023b71ffcf Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 18 Sep 2024 10:05:27 +0800 Subject: [PATCH 042/120] add generated cxx wrapper to support multi-platform compile (#39) * add generated cxx wrapper * update dylib postfix --- cpp/CMakeLists.txt | 11 - .../sdk-wrapper/CMakeLists.txt | 4 +- .../sdk-wrapper/java/CMakeLists.txt | 45 +- .../sdk-wrapper/java/bindings/build.gradle | 14 +- .../sdk-wrapper/java/bindings/settings.gradle | 2 +- .../wedpr/sdk/jni/common/JniLibLoader.java | 4 +- .../wedpr/sdk/jni/generated/IFrontClient.java | 2 - .../src/wedpr_java_transportJAVA_wrap.cxx | 6763 +++++++++++++++++ .../java/src/wedpr_java_transportJAVA_wrap.h | 58 + 9 files changed, 6865 insertions(+), 38 deletions(-) create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 45d13348..67650d29 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -89,17 +89,6 @@ set(ALL_SOURCE_LIST wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE} wedpr-initializer wedpr-main) -if(BUILD_WEDPR_TOOLKIT) - # fetch the python dependencies - option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON) - message(STATUS "Python fetch dependencies: ${FETCH_PYTHON_DEPS}") - if(WIN32) - message(STATUS "Getting SWIG for Windows: ...") - include(swig) - message(STATUS "Getting SWIG for Windows: ...DONE") - endif() -endif() - if(BUILD_ALL) add_sources("${ALL_SOURCE_LIST}") elseif(BUILD_UDF) diff --git a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt index 08c97539..e1d6cf3d 100644 --- a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt @@ -6,6 +6,9 @@ include(${SWIG_USE_FILE}) # Add subdirectories for each language if desired option(BUILD_PYTHON "Build Python SWIG module" ON) if(BUILD_PYTHON) + # fetch the python dependencies + option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON) + message(STATUS "Python fetch dependencies: ${FETCH_PYTHON_DEPS}") include(python) add_subdirectory(python) endif() @@ -13,6 +16,5 @@ endif() # Add subdirectories for each language if desired option(BUILD_JAVA "Build JAVA SWIG module" ON) if(BUILD_JAVA) - include(java) add_subdirectory(java) endif() diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt index 9276c9e6..a184200f 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt @@ -1,19 +1,32 @@ -file(GLOB_RECURSE SRCS wedpr_java_transport.i) -set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) - -set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.generated") - -set_property(SOURCE swig/wedpr_java_transport.i PROPERTY COMPILE_OPTIONS - -package ${WEDPR_TRANSPORT_PACKAGE}) - -file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_DIR}) -file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_LIB_DIR}) - -swig_add_library(${WEDPR_JAVA_TRANSPORT} - TYPE MODULE - LANGUAGE java - OUTPUT_DIR ${WEDPR_JAVA_TRANSPORT_DIR} - SOURCES ${SRCS}) +option(AUTO_GENERATE "Auto generate the targes" OFF) +if(AUTO_GENERATE) + if(WIN32) + message(STATUS "Getting SWIG for Windows: ...") + include(swig) + message(STATUS "Getting SWIG for Windows: ...DONE") + endif() + # only auto-generate should find swig + include(java) + file(GLOB_RECURSE SRCS wedpr_java_transport.i) + set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) + + set(WEDPR_TRANSPORT_PACKAGE "com.webank.wedpr.sdk.jni.generated") + + set_property(SOURCE swig/wedpr_java_transport.i PROPERTY COMPILE_OPTIONS + -package ${WEDPR_TRANSPORT_PACKAGE}) + + file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_DIR}) + file(MAKE_DIRECTORY ${WEDPR_JAVA_TRANSPORT_LIB_DIR}) + + swig_add_library(${WEDPR_JAVA_TRANSPORT} + TYPE MODULE + LANGUAGE java + OUTPUT_DIR ${WEDPR_JAVA_TRANSPORT_DIR} + SOURCES ${SRCS}) +else() + file(GLOB_RECURSE SRCS src/*.cxx) + add_library(${WEDPR_JAVA_TRANSPORT} SHARED ${SRCS}) +endif() SET(LIBRARY_OUTPUT_PATH ${WEDPR_JAVA_TRANSPORT_LIB_DIR}) message("LIBRARY_OUTPUT_PATH: ${WEDPR_JAVA_TRANSPORT_LIB_DIR}") diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index 67e5940e..362b0ba3 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -7,6 +7,9 @@ plugins { id "de.undercouch.download" version "4.1.2" id 'com.github.sherter.google-java-format' version '0.9' } +apply plugin: 'maven-publish' +apply plugin: 'java' +apply plugin: 'signing' // Additional attribute definition ext { @@ -93,7 +96,7 @@ version = '1.0.0-SNAPSHOT' jar { exclude '**/*.xml' exclude '**/*.properties' - /*manifest { + manifest { try { def repo = grgit.open(currentDir: project.rootDir) if (repo != null) { @@ -109,7 +112,7 @@ jar { } catch (Exception e) { e.printStackTrace() } - } from sourceSets.main.output*/ + } from sourceSets.main.output doLast { copy { @@ -164,7 +167,7 @@ publishing { pom { name = 'wedpr' - description = 'wedpr java transport jni' + description = 'wedpr java gateway sdk' url = 'http://www.fisco-bcos.org' licenses { @@ -200,7 +203,8 @@ publishing { password 'deployment123' } } + signing { + sign publishing.publications.mavenJava + } } - } - diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle index 5c90d6b7..4a6f5d4e 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/settings.gradle @@ -1,2 +1,2 @@ -rootProject.name = 'wedpr-java-transport-jni' +rootProject.name = 'wedpr-gateway-sdk' diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java index f30e9165..ff34a1de 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/JniLibLoader.java @@ -101,9 +101,9 @@ public static String getLibName(String baseName) { } else if (osName.contains(MAC)) { String arch = getArch(); if ("arm".equals(arch)) { - return "lib" + baseName + "-aarch64" + ".jnilib"; + return "lib" + baseName + "-aarch64" + ".dylib"; } - return "lib" + baseName + ".jnilib"; + return "lib" + baseName + ".dylib"; } else { throw new RuntimeException("unrecognized OS: " + osName); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java index ac777ec8..a9d97546 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java @@ -43,8 +43,6 @@ public synchronized void delete() { /** * : receive message from gateway, call by gateway
*
- * - * @return void */ public void onReceiveMessage( Message _msg, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx new file mode 100644 index 00000000..66ff9162 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx @@ -0,0 +1,6763 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + + +#define SWIG_VERSION 0x040201 +#define SWIGJAVA +#define SWIG_DIRECTORS + +/* ----------------------------------------------------------------------------- + * This section contains generic SWIG labels for method/variable + * declarations/attributes, and other compiler dependent labels. + * ----------------------------------------------------------------------------- */ + +/* template workaround for compilers that cannot correctly implement the C++ standard */ +#ifndef SWIGTEMPLATEDISAMBIGUATOR +# if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x560) +# define SWIGTEMPLATEDISAMBIGUATOR template +# elif defined(__HP_aCC) +/* Needed even with `aCC -AA' when `aCC -V' reports HP ANSI C++ B3910B A.03.55 */ +/* If we find a maximum version that requires this, the test would be __HP_aCC <= 35500 for A.03.55 */ +# define SWIGTEMPLATEDISAMBIGUATOR template +# else +# define SWIGTEMPLATEDISAMBIGUATOR +# endif +#endif + +/* inline attribute */ +#ifndef SWIGINLINE +# if defined(__cplusplus) || (defined(__GNUC__) && !defined(__STRICT_ANSI__)) +# define SWIGINLINE inline +# else +# define SWIGINLINE +# endif +#endif + +/* attribute recognised by some compilers to avoid 'unused' warnings */ +#ifndef SWIGUNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +# elif defined(__ICC) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +#endif + +#ifndef SWIG_MSC_UNSUPPRESS_4505 +# if defined(_MSC_VER) +# pragma warning(disable : 4505) /* unreferenced local function has been removed */ +# endif +#endif + +#ifndef SWIGUNUSEDPARM +# ifdef __cplusplus +# define SWIGUNUSEDPARM(p) +# else +# define SWIGUNUSEDPARM(p) p SWIGUNUSED +# endif +#endif + +/* internal SWIG method */ +#ifndef SWIGINTERN +# define SWIGINTERN static SWIGUNUSED +#endif + +/* internal inline SWIG method */ +#ifndef SWIGINTERNINLINE +# define SWIGINTERNINLINE SWIGINTERN SWIGINLINE +#endif + +/* exporting methods */ +#if defined(__GNUC__) +# if (__GNUC__ >= 4) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) +# ifndef GCC_HASCLASSVISIBILITY +# define GCC_HASCLASSVISIBILITY +# endif +# endif +#endif + +#ifndef SWIGEXPORT +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# if defined(STATIC_LINKED) +# define SWIGEXPORT +# else +# define SWIGEXPORT __declspec(dllexport) +# endif +# else +# if defined(__GNUC__) && defined(GCC_HASCLASSVISIBILITY) +# define SWIGEXPORT __attribute__ ((visibility("default"))) +# else +# define SWIGEXPORT +# endif +# endif +#endif + +/* calling conventions for Windows */ +#ifndef SWIGSTDCALL +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# define SWIGSTDCALL __stdcall +# else +# define SWIGSTDCALL +# endif +#endif + +/* Deal with Microsoft's attempt at deprecating C standard runtime functions */ +#if !defined(SWIG_NO_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_CRT_SECURE_NO_DEPRECATE) +# define _CRT_SECURE_NO_DEPRECATE +#endif + +/* Deal with Microsoft's attempt at deprecating methods in the standard C++ library */ +#if !defined(SWIG_NO_SCL_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_SCL_SECURE_NO_DEPRECATE) +# define _SCL_SECURE_NO_DEPRECATE +#endif + +/* Deal with Apple's deprecated 'AssertMacros.h' from Carbon-framework */ +#if defined(__APPLE__) && !defined(__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES) +# define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 +#endif + +/* Intel's compiler complains if a variable which was never initialised is + * cast to void, which is a common idiom which we use to indicate that we + * are aware a variable isn't used. So we just silence that warning. + * See: https://github.com/swig/swig/issues/192 for more discussion. + */ +#ifdef __INTEL_COMPILER +# pragma warning disable 592 +#endif + +#if defined(__cplusplus) && __cplusplus >=201103L +# define SWIG_NULLPTR nullptr +#else +# define SWIG_NULLPTR NULL +#endif + +/* ----------------------------------------------------------------------------- + * swigcompat.swg + * + * Macros to provide support compatibility with older C and C++ standards. + * ----------------------------------------------------------------------------- */ + +/* C99 and C++11 should provide snprintf, but define SWIG_NO_SNPRINTF + * if you're missing it. + */ +#if ((defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L) || \ + (defined __cplusplus && __cplusplus >= 201103L) || \ + defined SWIG_HAVE_SNPRINTF) && \ + !defined SWIG_NO_SNPRINTF +# define SWIG_snprintf(O,S,F,A) snprintf(O,S,F,A) +# define SWIG_snprintf2(O,S,F,A,B) snprintf(O,S,F,A,B) +#else +/* Fallback versions ignore the buffer size, but most of our uses either have a + * fixed maximum possible size or dynamically allocate a buffer that's large + * enough. + */ +# define SWIG_snprintf(O,S,F,A) sprintf(O,F,A) +# define SWIG_snprintf2(O,S,F,A,B) sprintf(O,F,A,B) +#endif + + +#include +#include +#include + + +/* Support for throwing Java exceptions */ +typedef enum { + SWIG_JavaOutOfMemoryError = 1, + SWIG_JavaIOException, + SWIG_JavaRuntimeException, + SWIG_JavaIndexOutOfBoundsException, + SWIG_JavaArithmeticException, + SWIG_JavaIllegalArgumentException, + SWIG_JavaNullPointerException, + SWIG_JavaDirectorPureVirtual, + SWIG_JavaUnknownError, + SWIG_JavaIllegalStateException, +} SWIG_JavaExceptionCodes; + +typedef struct { + SWIG_JavaExceptionCodes code; + const char *java_exception; +} SWIG_JavaExceptions_t; + + +static void SWIGUNUSED SWIG_JavaThrowException(JNIEnv *jenv, SWIG_JavaExceptionCodes code, const char *msg) { + jclass excep; + static const SWIG_JavaExceptions_t java_exceptions[] = { + { SWIG_JavaOutOfMemoryError, "java/lang/OutOfMemoryError" }, + { SWIG_JavaIOException, "java/io/IOException" }, + { SWIG_JavaRuntimeException, "java/lang/RuntimeException" }, + { SWIG_JavaIndexOutOfBoundsException, "java/lang/IndexOutOfBoundsException" }, + { SWIG_JavaArithmeticException, "java/lang/ArithmeticException" }, + { SWIG_JavaIllegalArgumentException, "java/lang/IllegalArgumentException" }, + { SWIG_JavaNullPointerException, "java/lang/NullPointerException" }, + { SWIG_JavaDirectorPureVirtual, "java/lang/RuntimeException" }, + { SWIG_JavaUnknownError, "java/lang/UnknownError" }, + { SWIG_JavaIllegalStateException, "java/lang/IllegalStateException" }, + { (SWIG_JavaExceptionCodes)0, "java/lang/UnknownError" } + }; + const SWIG_JavaExceptions_t *except_ptr = java_exceptions; + + while (except_ptr->code != code && except_ptr->code) + except_ptr++; + + jenv->ExceptionClear(); + excep = jenv->FindClass(except_ptr->java_exception); + if (excep) + jenv->ThrowNew(excep, msg); +} + + +/* Contract support */ + +#define SWIG_contract_assert(nullreturn, expr, msg) do { if (!(expr)) {SWIG_JavaThrowException(jenv, SWIG_JavaIllegalArgumentException, msg); return nullreturn; } } while (0) + +/* ----------------------------------------------------------------------------- + * director_common.swg + * + * This file contains support for director classes which is common between + * languages. + * ----------------------------------------------------------------------------- */ + +/* + Use -DSWIG_DIRECTOR_STATIC if you prefer to avoid the use of the + 'Swig' namespace. This could be useful for multi-modules projects. +*/ +#ifdef SWIG_DIRECTOR_STATIC +/* Force anonymous (static) namespace */ +#define Swig +#endif +/* ----------------------------------------------------------------------------- + * director.swg + * + * This file contains support for director classes so that Java proxy + * methods can be called from C++. + * ----------------------------------------------------------------------------- */ + +#if defined(DEBUG_DIRECTOR_OWNED) || defined(DEBUG_DIRECTOR_EXCEPTION) || defined(DEBUG_DIRECTOR_THREAD_NAME) +#include +#endif + +#include + +#if defined(SWIG_JAVA_USE_THREAD_NAME) + +#if !defined(SWIG_JAVA_GET_THREAD_NAME) +namespace Swig { + SWIGINTERN int GetThreadName(char *name, size_t len); +} + +#if defined(__linux__) + +#include +SWIGINTERN int Swig::GetThreadName(char *name, size_t len) { + (void)len; +#if defined(PR_GET_NAME) + return prctl(PR_GET_NAME, (unsigned long)name, 0, 0, 0); +#else + (void)name; + return 1; +#endif +} + +#elif defined(__unix__) || defined(__APPLE__) + +#include +SWIGINTERN int Swig::GetThreadName(char *name, size_t len) { + return pthread_getname_np(pthread_self(), name, len); +} + +#else + +SWIGINTERN int Swig::GetThreadName(char *name, size_t len) { + (void)len; + (void)name; + return 1; +} +#endif + +#endif + +#endif + +#if defined(SWIG_JAVA_DETACH_ON_THREAD_END) +#include +#endif + +namespace Swig { + + /* Java object wrapper */ + class JObjectWrapper { + public: + JObjectWrapper() : jthis_(SWIG_NULLPTR), weak_global_(true) { + } + + ~JObjectWrapper() { + jthis_ = SWIG_NULLPTR; + weak_global_ = true; + } + + bool set(JNIEnv *jenv, jobject jobj, bool mem_own, bool weak_global) { + if (!jthis_) { + weak_global_ = weak_global || !mem_own; // hold as weak global if explicitly requested or not owned + if (jobj) + jthis_ = weak_global_ ? jenv->NewWeakGlobalRef(jobj) : jenv->NewGlobalRef(jobj); +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "JObjectWrapper::set(" << jobj << ", " << (weak_global ? "weak_global" : "global_ref") << ") -> " << jthis_ << std::endl; +#endif + return true; + } else { +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "JObjectWrapper::set(" << jobj << ", " << (weak_global ? "weak_global" : "global_ref") << ") -> already set" << std::endl; +#endif + return false; + } + } + + jobject get(JNIEnv *jenv) const { +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "JObjectWrapper::get("; + if (jthis_) + std::cout << jthis_; + else + std::cout << "null"; + std::cout << ") -> return new local ref" << std::endl; +#endif + return (jthis_ ? jenv->NewLocalRef(jthis_) : jthis_); + } + + void release(JNIEnv *jenv) { +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "JObjectWrapper::release(" << jthis_ << "): " << (weak_global_ ? "weak global ref" : "global ref") << std::endl; +#endif + if (jthis_) { + if (weak_global_) { + if (jenv->IsSameObject(jthis_, SWIG_NULLPTR) == JNI_FALSE) + jenv->DeleteWeakGlobalRef((jweak)jthis_); + } else + jenv->DeleteGlobalRef(jthis_); + } + + jthis_ = SWIG_NULLPTR; + weak_global_ = true; + } + + /* Only call peek if you know what you are doing wrt to weak/global references */ + jobject peek() { + return jthis_; + } + + /* Java proxy releases ownership of C++ object, C++ object is now + responsible for destruction (creates NewGlobalRef to pin Java proxy) */ + void java_change_ownership(JNIEnv *jenv, jobject jself, bool take_or_release) { + if (take_or_release) { /* Java takes ownership of C++ object's lifetime. */ + if (!weak_global_) { + jenv->DeleteGlobalRef(jthis_); + jthis_ = jenv->NewWeakGlobalRef(jself); + weak_global_ = true; + } + } else { + /* Java releases ownership of C++ object's lifetime */ + if (weak_global_) { + jenv->DeleteWeakGlobalRef((jweak)jthis_); + jthis_ = jenv->NewGlobalRef(jself); + weak_global_ = false; + } + } + } + +#if defined(SWIG_JAVA_DETACH_ON_THREAD_END) + static void detach(void *jvm) { + static_cast(jvm)->DetachCurrentThread(); + } + + static void make_detach_key() { + pthread_key_create(&detach_key_, detach); + } + + /* thread-local key to register a destructor */ + static pthread_key_t detach_key_; +#endif + + private: + /* pointer to Java object */ + jobject jthis_; + /* Local or global reference flag */ + bool weak_global_; + }; + +#if defined(SWIG_JAVA_DETACH_ON_THREAD_END) + pthread_key_t JObjectWrapper::detach_key_; +#endif + + /* Local JNI reference deleter */ + class LocalRefGuard { + JNIEnv *jenv_; + jobject jobj_; + + // non-copyable + LocalRefGuard(const LocalRefGuard &); + LocalRefGuard &operator=(const LocalRefGuard &); + public: + LocalRefGuard(JNIEnv *jenv, jobject jobj): jenv_(jenv), jobj_(jobj) {} + ~LocalRefGuard() { + if (jobj_) + jenv_->DeleteLocalRef(jobj_); + } + }; + + /* director base class */ + class Director { + /* pointer to Java virtual machine */ + JavaVM *swig_jvm_; + + protected: +#if defined (_MSC_VER) && (_MSC_VER<1300) + class JNIEnvWrapper; + friend class JNIEnvWrapper; +#endif + /* Utility class for managing the JNI environment */ + class JNIEnvWrapper { + const Director *director_; + JNIEnv *jenv_; + int env_status; + public: + JNIEnvWrapper(const Director *director) : director_(director), jenv_(SWIG_NULLPTR), env_status(0) { +#if defined(__ANDROID__) + JNIEnv **jenv = &jenv_; +#else + void **jenv = (void **)&jenv_; +#endif + env_status = director_->swig_jvm_->GetEnv((void **)&jenv_, JNI_VERSION_1_2); + JavaVMAttachArgs args; + args.version = JNI_VERSION_1_2; + args.group = SWIG_NULLPTR; + args.name = SWIG_NULLPTR; +#if defined(SWIG_JAVA_USE_THREAD_NAME) + char thread_name[64]; // MAX_TASK_COMM_LEN=16 is hard-coded in the Linux kernel and MacOS has MAXTHREADNAMESIZE=64. + if (Swig::GetThreadName(thread_name, sizeof(thread_name)) == 0) { + args.name = thread_name; +#if defined(DEBUG_DIRECTOR_THREAD_NAME) + std::cout << "JNIEnvWrapper: thread name: " << thread_name << std::endl; + } else { + std::cout << "JNIEnvWrapper: Couldn't set Java thread name" << std::endl; +#endif + } +#endif +#if defined(SWIG_JAVA_ATTACH_CURRENT_THREAD_AS_DAEMON) + // Attach a daemon thread to the JVM. Useful when the JVM should not wait for + // the thread to exit upon shutdown. Only for jdk-1.4 and later. + director_->swig_jvm_->AttachCurrentThreadAsDaemon(jenv, &args); +#else + director_->swig_jvm_->AttachCurrentThread(jenv, &args); +#endif + +#if defined(SWIG_JAVA_DETACH_ON_THREAD_END) + // At least on Android 6, detaching after every call causes a memory leak. + // Instead, register a thread desructor and detach only when the thread ends. + // See https://developer.android.com/training/articles/perf-jni#threads + static pthread_once_t once = PTHREAD_ONCE_INIT; + + pthread_once(&once, JObjectWrapper::make_detach_key); + pthread_setspecific(JObjectWrapper::detach_key_, director->swig_jvm_); +#endif + } + ~JNIEnvWrapper() { +#if !defined(SWIG_JAVA_DETACH_ON_THREAD_END) && !defined(SWIG_JAVA_NO_DETACH_CURRENT_THREAD) + // Some JVMs, eg jdk-1.4.2 and lower on Solaris have a bug and crash with the DetachCurrentThread call. + // However, without this call, the JVM hangs on exit when the thread was not created by the JVM and creates a memory leak. + if (env_status == JNI_EDETACHED) + director_->swig_jvm_->DetachCurrentThread(); +#endif + } + JNIEnv *getJNIEnv() const { + return jenv_; + } + }; + + struct SwigDirectorMethod { + const char *name; + const char *desc; + jmethodID methid; + SwigDirectorMethod(JNIEnv *jenv, jclass baseclass, const char *name, const char *desc) : name(name), desc(desc) { + methid = jenv->GetMethodID(baseclass, name, desc); + } + }; + + /* Java object wrapper */ + JObjectWrapper swig_self_; + + /* Disconnect director from Java object */ + void swig_disconnect_director_self(const char *disconn_method) { + JNIEnvWrapper jnienv(this) ; + JNIEnv *jenv = jnienv.getJNIEnv() ; + jobject jobj = swig_self_.get(jenv); + LocalRefGuard ref_deleter(jenv, jobj); +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "Swig::Director::disconnect_director_self(" << jobj << ")" << std::endl; +#endif + if (jobj && jenv->IsSameObject(jobj, SWIG_NULLPTR) == JNI_FALSE) { + jmethodID disconn_meth = jenv->GetMethodID(jenv->GetObjectClass(jobj), disconn_method, "()V"); + if (disconn_meth) { +#if defined(DEBUG_DIRECTOR_OWNED) + std::cout << "Swig::Director::disconnect_director_self upcall to " << disconn_method << std::endl; +#endif + jenv->CallVoidMethod(jobj, disconn_meth); + } + } + } + + jclass swig_new_global_ref(JNIEnv *jenv, const char *classname) { + jclass clz = jenv->FindClass(classname); + return clz ? (jclass)jenv->NewGlobalRef(clz) : SWIG_NULLPTR; + } + + public: + Director(JNIEnv *jenv) : swig_jvm_((JavaVM *) SWIG_NULLPTR), swig_self_() { + /* Acquire the Java VM pointer */ + jenv->GetJavaVM(&swig_jvm_); + } + + virtual ~Director() { + JNIEnvWrapper jnienv(this) ; + JNIEnv *jenv = jnienv.getJNIEnv() ; + swig_self_.release(jenv); + } + + bool swig_set_self(JNIEnv *jenv, jobject jself, bool mem_own, bool weak_global) { + return swig_self_.set(jenv, jself, mem_own, weak_global); + } + + jobject swig_get_self(JNIEnv *jenv) const { + return swig_self_.get(jenv); + } + + // Change C++ object's ownership, relative to Java + void swig_java_change_ownership(JNIEnv *jenv, jobject jself, bool take_or_release) { + swig_self_.java_change_ownership(jenv, jself, take_or_release); + } + }; + + // Zero initialized bool array + template class BoolArray { + bool array_[N]; + public: + BoolArray() { + memset(array_, 0, sizeof(array_)); + } + bool& operator[](size_t n) { + return array_[n]; + } + bool operator[](size_t n) const { + return array_[n]; + } + }; + + // Utility classes and functions for exception handling. + + // Simple holder for a Java string during exception handling, providing access to a c-style string + class JavaString { + public: + JavaString(JNIEnv *jenv, jstring jstr) : jenv_(jenv), jstr_(jstr), cstr_(SWIG_NULLPTR) { + if (jenv_ && jstr_) + cstr_ = (const char *) jenv_->GetStringUTFChars(jstr_, SWIG_NULLPTR); + } + + ~JavaString() { + if (jenv_ && jstr_ && cstr_) + jenv_->ReleaseStringUTFChars(jstr_, cstr_); + } + + const char *c_str(const char *null_string = "null JavaString") const { + return cstr_ ? cstr_ : null_string; + } + + private: + // non-copyable + JavaString(const JavaString &); + JavaString &operator=(const JavaString &); + + JNIEnv *jenv_; + jstring jstr_; + const char *cstr_; + }; + + // Helper class to extract the exception message from a Java throwable + class JavaExceptionMessage { + public: + JavaExceptionMessage(JNIEnv *jenv, jthrowable throwable) : message_(jenv, exceptionMessageFromThrowable(jenv, throwable)) { + } + + // Return a C string of the exception message in the jthrowable passed in the constructor + // If no message is available, null_string is return instead + const char *message(const char *null_string = "Could not get exception message in JavaExceptionMessage") const { + return message_.c_str(null_string); + } + + private: + // non-copyable + JavaExceptionMessage(const JavaExceptionMessage &); + JavaExceptionMessage &operator=(const JavaExceptionMessage &); + + // Get exception message by calling Java method Throwable.getMessage() + static jstring exceptionMessageFromThrowable(JNIEnv *jenv, jthrowable throwable) { + jstring jmsg = SWIG_NULLPTR; + if (jenv && throwable) { + jenv->ExceptionClear(); // Cannot invoke methods with any pending exceptions + jclass throwclz = jenv->GetObjectClass(throwable); + if (throwclz) { + // All Throwable classes have a getMessage() method, so call it to extract the exception message + jmethodID getMessageMethodID = jenv->GetMethodID(throwclz, "getMessage", "()Ljava/lang/String;"); + if (getMessageMethodID) + jmsg = (jstring)jenv->CallObjectMethod(throwable, getMessageMethodID); + } + if (jmsg == SWIG_NULLPTR && jenv->ExceptionCheck()) + jenv->ExceptionClear(); + } + return jmsg; + } + + JavaString message_; + }; + + // C++ Exception class for handling Java exceptions thrown during a director method Java upcall + class DirectorException : public std::exception { + public: + + // Construct exception from a Java throwable + DirectorException(JNIEnv *jenv, jthrowable throwable) : jenv_(jenv), throwable_(throwable), classname_(SWIG_NULLPTR), msg_(SWIG_NULLPTR) { + + // Call Java method Object.getClass().getName() to obtain the throwable's class name (delimited by '/') + if (jenv && throwable) { + jenv->ExceptionClear(); // Cannot invoke methods with any pending exceptions + jclass throwclz = jenv->GetObjectClass(throwable); + if (throwclz) { + jclass clzclz = jenv->GetObjectClass(throwclz); + if (clzclz) { + jmethodID getNameMethodID = jenv->GetMethodID(clzclz, "getName", "()Ljava/lang/String;"); + if (getNameMethodID) { + jstring jstr_classname = (jstring)(jenv->CallObjectMethod(throwclz, getNameMethodID)); + // Copy strings, since there is no guarantee that jenv will be active when handled + if (jstr_classname) { + JavaString jsclassname(jenv, jstr_classname); + const char *classname = jsclassname.c_str(SWIG_NULLPTR); + if (classname) + classname_ = copypath(classname); + } + } + } + } + } + + JavaExceptionMessage exceptionmsg(jenv, throwable); + msg_ = copystr(exceptionmsg.message(SWIG_NULLPTR)); + } + + // More general constructor for handling as a java.lang.RuntimeException + DirectorException(const char *msg) : jenv_(SWIG_NULLPTR), throwable_(SWIG_NULLPTR), classname_(SWIG_NULLPTR), msg_(msg ? copystr(msg) : SWIG_NULLPTR) { + } + + ~DirectorException() throw() { + delete[] classname_; + delete[] msg_; + } + + const char *what() const throw() { + return msg_ ? msg_ : "Unspecified DirectorException message"; + } + + // Reconstruct and raise/throw the Java Exception that caused the DirectorException + // Note that any error in the JNI exception handling results in a Java RuntimeException + void throwException(JNIEnv *jenv) const { + if (jenv) { + if (jenv == jenv_ && throwable_) { + // Throw original exception if not already pending + jthrowable throwable = jenv->ExceptionOccurred(); + if (throwable && jenv->IsSameObject(throwable, throwable_) == JNI_FALSE) { + jenv->ExceptionClear(); + throwable = SWIG_NULLPTR; + } + if (!throwable) + jenv->Throw(throwable_); + } else { + // Try and reconstruct original exception, but original stacktrace is not reconstructed + jenv->ExceptionClear(); + + jmethodID ctorMethodID = SWIG_NULLPTR; + jclass throwableclass = SWIG_NULLPTR; + if (classname_) { + throwableclass = jenv->FindClass(classname_); + if (throwableclass) + ctorMethodID = jenv->GetMethodID(throwableclass, "", "(Ljava/lang/String;)V"); + } + + if (ctorMethodID) { + jenv->ThrowNew(throwableclass, what()); + } else { + SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, what()); + } + } + } + } + + // Deprecated - use throwException + void raiseJavaException(JNIEnv *jenv) const { + throwException(jenv); + } + + // Create and throw the DirectorException + static void raise(JNIEnv *jenv, jthrowable throwable) { + throw DirectorException(jenv, throwable); + } + + private: + static char *copypath(const char *srcmsg) { + char *target = copystr(srcmsg); + for (char *c=target; *c; ++c) { + if ('.' == *c) + *c = '/'; + } + return target; + } + + static char *copystr(const char *srcmsg) { + char *target = SWIG_NULLPTR; + if (srcmsg) { + size_t msglen = strlen(srcmsg) + 1; + target = new char[msglen]; + strncpy(target, srcmsg, msglen); + } + return target; + } + + JNIEnv *jenv_; + jthrowable throwable_; + const char *classname_; + const char *msg_; + }; + + // Helper method to determine if a Java throwable matches a particular Java class type + // Note side effect of clearing any pending exceptions + SWIGINTERN bool ExceptionMatches(JNIEnv *jenv, jthrowable throwable, const char *classname) { + bool matches = false; + + if (throwable && jenv && classname) { + // Exceptions need to be cleared for correct behavior. + // The caller of ExceptionMatches should restore pending exceptions if desired - + // the caller already has the throwable. + jenv->ExceptionClear(); + + jclass clz = jenv->FindClass(classname); + if (clz) { + jclass classclz = jenv->GetObjectClass(clz); + jmethodID isInstanceMethodID = jenv->GetMethodID(classclz, "isInstance", "(Ljava/lang/Object;)Z"); + if (isInstanceMethodID) { + matches = jenv->CallBooleanMethod(clz, isInstanceMethodID, throwable) != 0; + } + } + +#if defined(DEBUG_DIRECTOR_EXCEPTION) + if (jenv->ExceptionCheck()) { + // Typically occurs when an invalid classname argument is passed resulting in a ClassNotFoundException + JavaExceptionMessage exc(jenv, jenv->ExceptionOccurred()); + std::cout << "Error: ExceptionMatches: class '" << classname << "' : " << exc.message() << std::endl; + } +#endif + } + return matches; + } +} + +namespace Swig { + namespace { + jclass jclass_wedpr_java_transportJNI = NULL; + jmethodID director_method_ids[3]; + } +} + +#ifdef __cplusplus +#include +/* SwigValueWrapper is described in swig.swg */ +template class SwigValueWrapper { + struct SwigSmartPointer { + T *ptr; + SwigSmartPointer(T *p) : ptr(p) { } + ~SwigSmartPointer() { delete ptr; } + SwigSmartPointer& operator=(SwigSmartPointer& rhs) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = rhs.ptr; rhs.ptr = 0; return *this; } + void reset(T *p) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = p; } + } pointer; + SwigValueWrapper& operator=(const SwigValueWrapper& rhs); + SwigValueWrapper(const SwigValueWrapper& rhs); +public: + SwigValueWrapper() : pointer(0) { } + SwigValueWrapper& operator=(const T& t) { SwigSmartPointer tmp(new T(t)); pointer = tmp; return *this; } +#if __cplusplus >=201103L + SwigValueWrapper& operator=(T&& t) { SwigSmartPointer tmp(new T(std::move(t))); pointer = tmp; return *this; } + operator T&&() const { return std::move(*pointer.ptr); } +#else + operator T&() const { return *pointer.ptr; } +#endif + T *operator&() const { return pointer.ptr; } + static void reset(SwigValueWrapper& t, T *p) { t.pointer.reset(p); } +}; + +/* + * SwigValueInit() is a generic initialisation solution as the following approach: + * + * T c_result = T(); + * + * doesn't compile for all types for example: + * + * unsigned int c_result = unsigned int(); + */ +template T SwigValueInit() { + return T(); +} + +#if __cplusplus >=201103L +# define SWIG_STD_MOVE(OBJ) std::move(OBJ) +#else +# define SWIG_STD_MOVE(OBJ) OBJ +#endif + +#endif + + +#include // Use the C99 official header + + +#include +#include + + +#include +#include + + +#include + + +#define SWIG_FILE_WITH_INIT +#include +#include +#include +#include "wedpr-transport/sdk/src/TransportBuilder.h" +#include "wedpr-transport/sdk/src/Transport.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/RouteType.h" +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/GrpcConfig.h" +#include +#include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/MessagePayload.h" + + +/* Check for overflow converting to Java int (always signed 32-bit) from (unsigned variable-bit) size_t */ +SWIGINTERN jint SWIG_JavaIntFromSize_t(size_t size) { + static const jint JINT_MAX = 0x7FFFFFFF; + return (size > (size_t)JINT_MAX) ? -1 : (jint)size; +} + + +SWIGINTERN jint SWIG_VectorSize(size_t size) { + jint sz = SWIG_JavaIntFromSize_t(size); + if (sz == -1) + throw std::out_of_range("vector size is too large to fit into a Java int"); + return sz; +} + +SWIGINTERN std::vector< uint8_t > *new_std_vector_Sl_uint8_t_Sg___SWIG_2(jint count,unsigned char const &value){ + if (count < 0) + throw std::out_of_range("vector count must be positive"); + return new std::vector< unsigned char >(static_cast::size_type>(count), value); + } +SWIGINTERN jint std_vector_Sl_uint8_t_Sg__doCapacity(std::vector< uint8_t > *self){ + return SWIG_VectorSize(self->capacity()); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__doReserve(std::vector< uint8_t > *self,jint n){ + if (n < 0) + throw std::out_of_range("vector reserve size must be positive"); + self->reserve(n); + } +SWIGINTERN jint std_vector_Sl_uint8_t_Sg__doSize(std::vector< uint8_t > const *self){ + return SWIG_VectorSize(self->size()); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__doAdd__SWIG_0(std::vector< uint8_t > *self,std::vector< unsigned char >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__doAdd__SWIG_1(std::vector< uint8_t > *self,jint index,std::vector< unsigned char >::value_type const &x){ + jint size = static_cast(self->size()); + if (0 <= index && index <= size) { + self->insert(self->begin() + index, x); + } else { + throw std::out_of_range("vector index out of range"); + } + } +SWIGINTERN std::vector< unsigned char >::value_type std_vector_Sl_uint8_t_Sg__doRemove(std::vector< uint8_t > *self,jint index){ + jint size = static_cast(self->size()); + if (0 <= index && index < size) { + unsigned char const old_value = (*self)[index]; + self->erase(self->begin() + index); + return old_value; + } else { + throw std::out_of_range("vector index out of range"); + } + } +SWIGINTERN std::vector< unsigned char >::value_type const &std_vector_Sl_uint8_t_Sg__doGet(std::vector< uint8_t > *self,jint index){ + jint size = static_cast(self->size()); + if (index >= 0 && index < size) + return (*self)[index]; + else + throw std::out_of_range("vector index out of range"); + } +SWIGINTERN std::vector< unsigned char >::value_type std_vector_Sl_uint8_t_Sg__doSet(std::vector< uint8_t > *self,jint index,std::vector< unsigned char >::value_type const &val){ + jint size = static_cast(self->size()); + if (index >= 0 && index < size) { + unsigned char const old_value = (*self)[index]; + (*self)[index] = val; + return old_value; + } + else + throw std::out_of_range("vector index out of range"); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__doRemoveRange(std::vector< uint8_t > *self,jint fromIndex,jint toIndex){ + jint size = static_cast(self->size()); + if (0 <= fromIndex && fromIndex <= toIndex && toIndex <= size) { + self->erase(self->begin() + fromIndex, self->begin() + toIndex); + } else { + throw std::out_of_range("vector index out of range"); + } + } +SWIGINTERN std::vector< int8_t > *new_std_vector_Sl_int8_t_Sg___SWIG_2(jint count,signed char const &value){ + if (count < 0) + throw std::out_of_range("vector count must be positive"); + return new std::vector< signed char >(static_cast::size_type>(count), value); + } +SWIGINTERN jint std_vector_Sl_int8_t_Sg__doCapacity(std::vector< int8_t > *self){ + return SWIG_VectorSize(self->capacity()); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg__doReserve(std::vector< int8_t > *self,jint n){ + if (n < 0) + throw std::out_of_range("vector reserve size must be positive"); + self->reserve(n); + } +SWIGINTERN jint std_vector_Sl_int8_t_Sg__doSize(std::vector< int8_t > const *self){ + return SWIG_VectorSize(self->size()); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg__doAdd__SWIG_0(std::vector< int8_t > *self,std::vector< signed char >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg__doAdd__SWIG_1(std::vector< int8_t > *self,jint index,std::vector< signed char >::value_type const &x){ + jint size = static_cast(self->size()); + if (0 <= index && index <= size) { + self->insert(self->begin() + index, x); + } else { + throw std::out_of_range("vector index out of range"); + } + } +SWIGINTERN std::vector< signed char >::value_type std_vector_Sl_int8_t_Sg__doRemove(std::vector< int8_t > *self,jint index){ + jint size = static_cast(self->size()); + if (0 <= index && index < size) { + signed char const old_value = (*self)[index]; + self->erase(self->begin() + index); + return old_value; + } else { + throw std::out_of_range("vector index out of range"); + } + } +SWIGINTERN std::vector< signed char >::value_type const &std_vector_Sl_int8_t_Sg__doGet(std::vector< int8_t > *self,jint index){ + jint size = static_cast(self->size()); + if (index >= 0 && index < size) + return (*self)[index]; + else + throw std::out_of_range("vector index out of range"); + } +SWIGINTERN std::vector< signed char >::value_type std_vector_Sl_int8_t_Sg__doSet(std::vector< int8_t > *self,jint index,std::vector< signed char >::value_type const &val){ + jint size = static_cast(self->size()); + if (index >= 0 && index < size) { + signed char const old_value = (*self)[index]; + (*self)[index] = val; + return old_value; + } + else + throw std::out_of_range("vector index out of range"); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg__doRemoveRange(std::vector< int8_t > *self,jint fromIndex,jint toIndex){ + jint size = static_cast(self->size()); + if (0 <= fromIndex && fromIndex <= toIndex && toIndex <= size) { + self->erase(self->begin() + fromIndex, self->begin() + toIndex); + } else { + throw std::out_of_range("vector index out of range"); + } + } + +struct SWIG_null_deleter { + void operator() (void const *) const { + } +}; +#define SWIG_NO_NULL_DELETER_0 , SWIG_null_deleter() +#define SWIG_NO_NULL_DELETER_1 +#define SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW +#define SWIG_NO_NULL_DELETER_SWIG_POINTER_OWN + + +#include + + + +/* --------------------------------------------------- + * C++ director class methods + * --------------------------------------------------- */ + +#include "wedpr_java_transportJAVA_wrap.h" + +SwigDirector_ErrorCallback::SwigDirector_ErrorCallback(JNIEnv *jenv) : ppc::front::ErrorCallback(), Swig::Director(jenv) { +} + +SwigDirector_ErrorCallback::~SwigDirector_ErrorCallback() { + swig_disconnect_director_self("swigDirectorDisconnect"); +} + + +void SwigDirector_ErrorCallback::onError(bcos::Error::Ptr error) { + JNIEnvWrapper swigjnienv(this) ; + JNIEnv * jenv = swigjnienv.getJNIEnv() ; + jobject swigjobj = (jobject) NULL ; + jlong jerror ; + + if (!swig_override[0]) { + SWIG_JavaThrowException(JNIEnvWrapper(this).getJNIEnv(), SWIG_JavaDirectorPureVirtual, "Attempted to invoke pure virtual method ppc::front::ErrorCallback::onError."); + return; + } + swigjobj = swig_get_self(jenv); + if (swigjobj && jenv->IsSameObject(swigjobj, NULL) == JNI_FALSE) { + jerror = 0; + if (error) { + *((std::shared_ptr< bcos::Error > **)&jerror) = new std::shared_ptr< bcos::Error >(error); + } + jenv->CallStaticVoidMethod(Swig::jclass_wedpr_java_transportJNI, Swig::director_method_ids[0], swigjobj, jerror); + jthrowable swigerror = jenv->ExceptionOccurred(); + if (swigerror) { + Swig::DirectorException::raise(jenv, swigerror); + } + + } else { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null upcall object in ppc::front::ErrorCallback::onError "); + } + if (swigjobj) jenv->DeleteLocalRef(swigjobj); +} + +void SwigDirector_ErrorCallback::swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global) { + static jclass baseclass = swig_new_global_ref(jenv, "com/webank/wedpr/sdk/jni/generated/ErrorCallback"); + if (!baseclass) return; + static SwigDirectorMethod methods[] = { + SwigDirectorMethod(jenv, baseclass, "onError", "(Lcom/webank/wedpr/sdk/jni/generated/Error;)V") + }; + + if (swig_set_self(jenv, jself, swig_mem_own, weak_global)) { + bool derived = (jenv->IsSameObject(baseclass, jcls) ? false : true); + for (int i = 0; i < 1; ++i) { + swig_override[i] = false; + if (derived) { + jmethodID methid = jenv->GetMethodID(jcls, methods[i].name, methods[i].desc); + swig_override[i] = methods[i].methid && (methid != methods[i].methid); + jenv->ExceptionClear(); + } + } + } +} + + +SwigDirector_MessageDispatcherHandler::SwigDirector_MessageDispatcherHandler(JNIEnv *jenv) : ppc::front::MessageDispatcherHandler(), Swig::Director(jenv) { +} + +SwigDirector_MessageDispatcherHandler::~SwigDirector_MessageDispatcherHandler() { + swig_disconnect_director_self("swigDirectorDisconnect"); +} + + +void SwigDirector_MessageDispatcherHandler::onMessage(ppc::protocol::Message::Ptr msg) { + JNIEnvWrapper swigjnienv(this) ; + JNIEnv * jenv = swigjnienv.getJNIEnv() ; + jobject swigjobj = (jobject) NULL ; + jlong jmsg ; + + if (!swig_override[0]) { + SWIG_JavaThrowException(JNIEnvWrapper(this).getJNIEnv(), SWIG_JavaDirectorPureVirtual, "Attempted to invoke pure virtual method ppc::front::MessageDispatcherHandler::onMessage."); + return; + } + swigjobj = swig_get_self(jenv); + if (swigjobj && jenv->IsSameObject(swigjobj, NULL) == JNI_FALSE) { + jmsg = 0; + if (msg) { + *((std::shared_ptr< ppc::protocol::Message > **)&jmsg) = new std::shared_ptr< ppc::protocol::Message >(msg); + } + jenv->CallStaticVoidMethod(Swig::jclass_wedpr_java_transportJNI, Swig::director_method_ids[1], swigjobj, jmsg); + jthrowable swigerror = jenv->ExceptionOccurred(); + if (swigerror) { + Swig::DirectorException::raise(jenv, swigerror); + } + + } else { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null upcall object in ppc::front::MessageDispatcherHandler::onMessage "); + } + if (swigjobj) jenv->DeleteLocalRef(swigjobj); +} + +void SwigDirector_MessageDispatcherHandler::swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global) { + static jclass baseclass = swig_new_global_ref(jenv, "com/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler"); + if (!baseclass) return; + static SwigDirectorMethod methods[] = { + SwigDirectorMethod(jenv, baseclass, "onMessage", "(Lcom/webank/wedpr/sdk/jni/generated/Message;)V") + }; + + if (swig_set_self(jenv, jself, swig_mem_own, weak_global)) { + bool derived = (jenv->IsSameObject(baseclass, jcls) ? false : true); + for (int i = 0; i < 1; ++i) { + swig_override[i] = false; + if (derived) { + jmethodID methid = jenv->GetMethodID(jcls, methods[i].name, methods[i].desc); + swig_override[i] = methods[i].methid && (methid != methods[i].methid); + jenv->ExceptionClear(); + } + } + } +} + + +SwigDirector_IMessageHandler::SwigDirector_IMessageHandler(JNIEnv *jenv) : ppc::front::IMessageHandler(), Swig::Director(jenv) { +} + +SwigDirector_IMessageHandler::~SwigDirector_IMessageHandler() { + swig_disconnect_director_self("swigDirectorDisconnect"); +} + + +void SwigDirector_IMessageHandler::onMessage(bcos::Error::Ptr e,ppc::protocol::Message::Ptr msg,ppc::front::SendResponseHandler sendResponseHandler) { + JNIEnvWrapper swigjnienv(this) ; + JNIEnv * jenv = swigjnienv.getJNIEnv() ; + jobject swigjobj = (jobject) NULL ; + jlong je ; + jlong jmsg ; + jlong jsendResponseHandler ; + + if (!swig_override[0]) { + SWIG_JavaThrowException(JNIEnvWrapper(this).getJNIEnv(), SWIG_JavaDirectorPureVirtual, "Attempted to invoke pure virtual method ppc::front::IMessageHandler::onMessage."); + return; + } + swigjobj = swig_get_self(jenv); + if (swigjobj && jenv->IsSameObject(swigjobj, NULL) == JNI_FALSE) { + je = 0; + if (e) { + *((std::shared_ptr< bcos::Error > **)&je) = new std::shared_ptr< bcos::Error >(e); + } + jmsg = 0; + if (msg) { + *((std::shared_ptr< ppc::protocol::Message > **)&jmsg) = new std::shared_ptr< ppc::protocol::Message >(msg); + } + jsendResponseHandler = 0; + *((ppc::front::SendResponseHandler **)&jsendResponseHandler) = new ppc::front::SendResponseHandler(SWIG_STD_MOVE(sendResponseHandler)); + jenv->CallStaticVoidMethod(Swig::jclass_wedpr_java_transportJNI, Swig::director_method_ids[2], swigjobj, je, jmsg, jsendResponseHandler); + jthrowable swigerror = jenv->ExceptionOccurred(); + if (swigerror) { + Swig::DirectorException::raise(jenv, swigerror); + } + + } else { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null upcall object in ppc::front::IMessageHandler::onMessage "); + } + if (swigjobj) jenv->DeleteLocalRef(swigjobj); +} + +void SwigDirector_IMessageHandler::swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global) { + static jclass baseclass = swig_new_global_ref(jenv, "com/webank/wedpr/sdk/jni/generated/IMessageHandler"); + if (!baseclass) return; + static SwigDirectorMethod methods[] = { + SwigDirectorMethod(jenv, baseclass, "onMessage", "(Lcom/webank/wedpr/sdk/jni/generated/Error;Lcom/webank/wedpr/sdk/jni/generated/Message;Lcom/webank/wedpr/sdk/jni/generated/SendResponseHandler;)V") + }; + + if (swig_set_self(jenv, jself, swig_mem_own, weak_global)) { + bool derived = (jenv->IsSameObject(baseclass, jcls) ? false : true); + for (int i = 0; i < 1; ++i) { + swig_override[i] = false; + if (derived) { + jmethodID methid = jenv->GetMethodID(jcls, methods[i].name, methods[i].desc); + swig_override[i] = methods[i].methid && (methid != methods[i].methid); + jenv->ExceptionClear(); + } + } + } +} + + + +#ifdef __cplusplus +extern "C" { +#endif + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedBcosError(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< bcos::Error > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >(); + *(std::shared_ptr< bcos::Error > **)&jresult = (result && *result) ? new std::shared_ptr< bcos::Error >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedBcosError(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< bcos::Error > *arg1 = (std::shared_ptr< bcos::Error > *) 0 ; + std::shared_ptr< bcos::Error > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< bcos::Error > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedFrontConfig(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >(); + *(std::shared_ptr< ppc::front::FrontConfig > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::FrontConfig >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedFrontConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::FrontConfig > *arg1 = (std::shared_ptr< ppc::front::FrontConfig > *) 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedGrpcConfig(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >(); + *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::GrpcConfig >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedGrpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::GrpcConfig > *arg1 = (std::shared_ptr< ppc::protocol::GrpcConfig > *) 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedFront(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::IFront > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >(); + *(std::shared_ptr< ppc::front::IFront > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IFront >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedFront(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::IFront > *arg1 = (std::shared_ptr< ppc::front::IFront > *) 0 ; + std::shared_ptr< ppc::front::IFront > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IFront > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedFrontClient(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::IFrontClient > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >(); + *(std::shared_ptr< ppc::front::IFrontClient > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IFrontClient >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedFrontClient(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::IFrontClient > *arg1 = (std::shared_ptr< ppc::front::IFrontClient > *) 0 ; + std::shared_ptr< ppc::front::IFrontClient > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IFrontClient > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedErrorCallback(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::ErrorCallback > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >(); + *(std::shared_ptr< ppc::front::ErrorCallback > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::ErrorCallback >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedErrorCallback(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::ErrorCallback > *arg1 = (std::shared_ptr< ppc::front::ErrorCallback > *) 0 ; + std::shared_ptr< ppc::front::ErrorCallback > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::ErrorCallback > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessageDispatcherHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >(); + *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::MessageDispatcherHandler >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessageDispatcherHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::MessageDispatcherHandler > *arg1 = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *) 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedIMessageHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::IMessageHandler > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >(); + *(std::shared_ptr< ppc::front::IMessageHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IMessageHandler >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedIMessageHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::IMessageHandler > *arg1 = (std::shared_ptr< ppc::front::IMessageHandler > *) 0 ; + std::shared_ptr< ppc::front::IMessageHandler > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IMessageHandler > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedGateway(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::gateway::IGateway > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >(); + *(std::shared_ptr< ppc::gateway::IGateway > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::gateway::IGateway >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedGateway(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::gateway::IGateway > *arg1 = (std::shared_ptr< ppc::gateway::IGateway > *) 0 ; + std::shared_ptr< ppc::gateway::IGateway > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::gateway::IGateway > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessage(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::Message > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >(); + *(std::shared_ptr< ppc::protocol::Message > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::Message >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessage(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::Message > *arg1 = (std::shared_ptr< ppc::protocol::Message > *) 0 ; + std::shared_ptr< ppc::protocol::Message > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::Message > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessageHeader(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >(); + *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeader >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessageHeader(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessageHeader > *arg1 = (std::shared_ptr< ppc::protocol::MessageHeader > *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessagePayload(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >(); + *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayload >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessagePayload(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessagePayload > *arg1 = (std::shared_ptr< ppc::protocol::MessagePayload > *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedRouteInfo(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(); + *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedRouteInfo(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *arg1 = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessageBuilder(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >(); + *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageBuilder >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessageBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessageBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageBuilder > *) 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessageHeaderBuilder(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(); + *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessageHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedMessagePayloadBuilder(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(); + *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedMessagePayloadBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedRouteInfoBuilder(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(); + *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedRouteInfoBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1 : &tempnull1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ubytes_1_1SWIG_10(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::vector< uint8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::vector< uint8_t > *)new std::vector< uint8_t >(); + *(std::vector< uint8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ubytes_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + std::vector< uint8_t > *arg1 = 0 ; + std::vector< uint8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + if (!arg1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< uint8_t > const & is null"); + return 0; + } + result = (std::vector< uint8_t > *)new std::vector< uint8_t >((std::vector< uint8_t > const &)*arg1); + *(std::vector< uint8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1isEmpty(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + result = (bool)((std::vector< uint8_t > const *)arg1)->empty(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1clear(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + (arg1)->clear(); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ubytes_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jint jarg1, jshort jarg2) { + jlong jresult = 0 ; + jint arg1 ; + unsigned char *arg2 = 0 ; + unsigned char temp2 ; + std::vector< uint8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1; + temp2 = (unsigned char)jarg2; + arg2 = &temp2; + try { + result = (std::vector< uint8_t > *)new_std_vector_Sl_uint8_t_Sg___SWIG_2(SWIG_STD_MOVE(arg1),(unsigned char const &)*arg2); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + *(std::vector< uint8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doCapacity(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + try { + result = std_vector_Sl_uint8_t_Sg__doCapacity(arg1); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doReserve(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + try { + std_vector_Sl_uint8_t_Sg__doReserve(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::length_error &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + try { + result = std_vector_Sl_uint8_t_Sg__doSize((std::vector< unsigned char > const *)arg1); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doAdd_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jshort jarg2) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::value_type *arg2 = 0 ; + std::vector< unsigned char >::value_type temp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + temp2 = (std::vector< unsigned char >::value_type)jarg2; + arg2 = &temp2; + std_vector_Sl_uint8_t_Sg__doAdd__SWIG_0(arg1,(unsigned char const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doAdd_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jshort jarg3) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + std::vector< unsigned char >::value_type temp3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + temp3 = (std::vector< unsigned char >::value_type)jarg3; + arg3 = &temp3; + try { + std_vector_Sl_uint8_t_Sg__doAdd__SWIG_1(arg1,SWIG_STD_MOVE(arg2),(unsigned char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doRemove(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + jshort jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + std::vector< unsigned char >::value_type result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + try { + result = (std::vector< unsigned char >::value_type)std_vector_Sl_uint8_t_Sg__doRemove(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jshort)result; + return jresult; +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doGet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + jshort jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + std::vector< unsigned char >::value_type *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + try { + result = (std::vector< unsigned char >::value_type *) &std_vector_Sl_uint8_t_Sg__doGet(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jshort)*result; + return jresult; +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doSet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jshort jarg3) { + jshort jresult = 0 ; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + std::vector< unsigned char >::value_type temp3 ; + std::vector< unsigned char >::value_type result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + temp3 = (std::vector< unsigned char >::value_type)jarg3; + arg3 = &temp3; + try { + result = (std::vector< unsigned char >::value_type)std_vector_Sl_uint8_t_Sg__doSet(arg1,SWIG_STD_MOVE(arg2),(unsigned char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jshort)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ubytes_1doRemoveRange(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jint jarg3) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + jint arg2 ; + jint arg3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< uint8_t > **)&jarg1; + arg2 = jarg2; + arg3 = jarg3; + try { + std_vector_Sl_uint8_t_Sg__doRemoveRange(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1ubytes(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(std::vector< uint8_t > **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ibytes_1_1SWIG_10(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::vector< int8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::vector< int8_t > *)new std::vector< int8_t >(); + *(std::vector< int8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ibytes_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + std::vector< int8_t > *arg1 = 0 ; + std::vector< int8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + if (!arg1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< int8_t > const & is null"); + return 0; + } + result = (std::vector< int8_t > *)new std::vector< int8_t >((std::vector< int8_t > const &)*arg1); + *(std::vector< int8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1isEmpty(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + result = (bool)((std::vector< int8_t > const *)arg1)->empty(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1clear(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + (arg1)->clear(); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ibytes_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jint jarg1, jbyte jarg2) { + jlong jresult = 0 ; + jint arg1 ; + signed char *arg2 = 0 ; + signed char temp2 ; + std::vector< int8_t > *result = 0 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1; + temp2 = (signed char)jarg2; + arg2 = &temp2; + try { + result = (std::vector< int8_t > *)new_std_vector_Sl_int8_t_Sg___SWIG_2(SWIG_STD_MOVE(arg1),(signed char const &)*arg2); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + *(std::vector< int8_t > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doCapacity(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + try { + result = std_vector_Sl_int8_t_Sg__doCapacity(arg1); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doReserve(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + try { + std_vector_Sl_int8_t_Sg__doReserve(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::length_error &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + try { + result = std_vector_Sl_int8_t_Sg__doSize((std::vector< signed char > const *)arg1); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doAdd_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyte jarg2) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::value_type *arg2 = 0 ; + std::vector< signed char >::value_type temp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + temp2 = (std::vector< signed char >::value_type)jarg2; + arg2 = &temp2; + std_vector_Sl_int8_t_Sg__doAdd__SWIG_0(arg1,(signed char const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doAdd_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jbyte jarg3) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + std::vector< signed char >::value_type temp3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + temp3 = (std::vector< signed char >::value_type)jarg3; + arg3 = &temp3; + try { + std_vector_Sl_int8_t_Sg__doAdd__SWIG_1(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT jbyte JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doRemove(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + jbyte jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + std::vector< signed char >::value_type result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + try { + result = (std::vector< signed char >::value_type)std_vector_Sl_int8_t_Sg__doRemove(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jbyte)result; + return jresult; +} + + +SWIGEXPORT jbyte JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doGet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + jbyte jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + std::vector< signed char >::value_type *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + try { + result = (std::vector< signed char >::value_type *) &std_vector_Sl_int8_t_Sg__doGet(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jbyte)*result; + return jresult; +} + + +SWIGEXPORT jbyte JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doSet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jbyte jarg3) { + jbyte jresult = 0 ; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + std::vector< signed char >::value_type temp3 ; + std::vector< signed char >::value_type result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + temp3 = (std::vector< signed char >::value_type)jarg3; + arg3 = &temp3; + try { + result = (std::vector< signed char >::value_type)std_vector_Sl_int8_t_Sg__doSet(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return 0; + } + jresult = (jbyte)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ibytes_1doRemoveRange(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jint jarg3) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + jint arg2 ; + jint arg3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(std::vector< int8_t > **)&jarg1; + arg2 = jarg2; + arg3 = jarg3; + try { + std_vector_Sl_int8_t_Sg__doRemoveRange(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what()); + return ; + } +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1ibytes(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(std::vector< int8_t > **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1buildError_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jbyteArray jarg1, jbyteArray jarg2, jint jarg3, jint jarg4, jstring jarg5) { + jlong jresult = 0 ; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + bcos::Error result; + + (void)jenv; + (void)jcls; + { + arg1 = (char *) jenv->GetByteArrayElements(jarg1, 0); + } + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + arg3 = (int)jarg3; + arg4 = (int32_t)jarg4; + if(!jarg5) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg5_pstr = (const char *)jenv->GetStringUTFChars(jarg5, 0); + if (!arg5_pstr) return 0; + std::string arg5_str(arg5_pstr); + arg5 = &arg5_str; + jenv->ReleaseStringUTFChars(jarg5, arg5_pstr); + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5); + *(std::shared_ptr< bcos::Error > **)&jresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + { + jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); + } + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + + + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1buildError_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jbyteArray jarg1, jbyteArray jarg2, jint jarg3, jint jarg4, jstring jarg5, jlong jarg6, jobject jarg6_) { + jlong jresult = 0 ; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + bcos::Error *arg6 = 0 ; + bcos::Error result; + + (void)jenv; + (void)jcls; + (void)jarg6_; + { + arg1 = (char *) jenv->GetByteArrayElements(jarg1, 0); + } + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + arg3 = (int)jarg3; + arg4 = (int32_t)jarg4; + if(!jarg5) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg5_pstr = (const char *)jenv->GetStringUTFChars(jarg5, 0); + if (!arg5_pstr) return 0; + std::string arg5_str(arg5_pstr); + arg5 = &arg5_str; + jenv->ReleaseStringUTFChars(jarg5, arg5_pstr); + + arg6 = (bcos::Error *)((*(std::shared_ptr< const bcos::Error > **)&jarg6) ? (*(std::shared_ptr< const bcos::Error > **)&jarg6)->get() : 0); + if (!arg6) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::Error const & reference is null"); + return 0; + } + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6); + *(std::shared_ptr< bcos::Error > **)&jresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + { + jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); + } + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + + + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1buildError_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jbyteArray jarg1, jbyteArray jarg2, jint jarg3, jint jarg4, jstring jarg5, jlong jarg6) { + jlong jresult = 0 ; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + std::exception *arg6 = 0 ; + bcos::Error result; + + (void)jenv; + (void)jcls; + { + arg1 = (char *) jenv->GetByteArrayElements(jarg1, 0); + } + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + arg3 = (int)jarg3; + arg4 = (int32_t)jarg4; + if(!jarg5) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg5_pstr = (const char *)jenv->GetStringUTFChars(jarg5, 0); + if (!arg5_pstr) return 0; + std::string arg5_str(arg5_pstr); + arg5 = &arg5_str; + jenv->ReleaseStringUTFChars(jarg5, arg5_pstr); + arg6 = *(std::exception **)&jarg6; + if (!arg6) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::exception const & is null"); + return 0; + } + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6); + *(std::shared_ptr< bcos::Error > **)&jresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + { + jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); + } + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + + + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1Error_1_1SWIG_10(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + bcos::Error *result = 0 ; + + (void)jenv; + (void)jcls; + result = (bcos::Error *)new bcos::Error(); + + *(std::shared_ptr< bcos::Error > **)&jresult = result ? new std::shared_ptr< bcos::Error >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1Error_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jstring jarg2) { + jlong jresult = 0 ; + int64_t arg1 ; + std::string arg2 ; + bcos::Error *result = 0 ; + + (void)jenv; + (void)jcls; + arg1 = (int64_t)jarg1; + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + result = (bcos::Error *)new bcos::Error(arg1,arg2); + + *(std::shared_ptr< bcos::Error > **)&jresult = result ? new std::shared_ptr< bcos::Error >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1Error(JNIEnv *jenv, jclass jcls, jlong jarg1) { + bcos::Error *arg1 = (bcos::Error *) 0 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< bcos::Error > **)&jarg1; + arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1errorCode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + bcos::Error *arg1 = (bcos::Error *) 0 ; + std::shared_ptr< bcos::Error const > *smartarg1 = 0 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const bcos::Error > **)&jarg1; + arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); + result = (int64_t)((bcos::Error const *)arg1)->errorCode(); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1errorMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + bcos::Error *arg1 = (bcos::Error *) 0 ; + std::shared_ptr< bcos::Error const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const bcos::Error > **)&jarg1; + arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((bcos::Error const *)arg1)->errorMessage(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1setErrorCode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + bcos::Error *arg1 = (bcos::Error *) 0 ; + int64_t arg2 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< bcos::Error > **)&jarg1; + arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (int64_t)jarg2; + (arg1)->setErrorCode(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1setErrorMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + bcos::Error *arg1 = (bcos::Error *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< bcos::Error > **)&jarg1; + arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setErrorMessage((std::string const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_InputBuffer_1data_1set(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + InputBuffer *arg1 = (InputBuffer *) 0 ; + unsigned char *arg2 = (unsigned char *) 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(InputBuffer **)&jarg1; + arg2 = *(unsigned char **)&jarg2; + if (arg1) (arg1)->data = (unsigned char const *)arg2; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_InputBuffer_1data_1get(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + InputBuffer *arg1 = (InputBuffer *) 0 ; + unsigned char *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(InputBuffer **)&jarg1; + result = (unsigned char *) ((arg1)->data); + *(unsigned char **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_InputBuffer_1len_1set(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) { + InputBuffer *arg1 = (InputBuffer *) 0 ; + uint64_t arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(InputBuffer **)&jarg1; + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg2); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg2 = 0; + if (sz > 0) { + arg2 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + if (arg1) (arg1)->len = arg2; +} + + +SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_InputBuffer_1len_1get(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jobject jresult = 0 ; + InputBuffer *arg1 = (InputBuffer *) 0 ; + uint64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(InputBuffer **)&jarg1; + result = (uint64_t) ((arg1)->len); + { + jbyteArray ba = jenv->NewByteArray(9); + jbyte* bae = jenv->GetByteArrayElements(ba, 0); + jclass clazz = jenv->FindClass("java/math/BigInteger"); + jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V"); + jobject bigint; + int i; + + bae[0] = 0; + for(i=1; i<9; i++ ) { + bae[i] = (jbyte)(result>>8*(8-i)); + } + + jenv->ReleaseByteArrayElements(ba, bae, 0); + bigint = jenv->NewObject(clazz, mid, ba); + jenv->DeleteLocalRef(ba); + jresult = bigint; + } + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1InputBuffer(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + InputBuffer *result = 0 ; + + (void)jenv; + (void)jcls; + result = (InputBuffer *)new InputBuffer(); + *(InputBuffer **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1InputBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1) { + InputBuffer *arg1 = (InputBuffer *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(InputBuffer **)&jarg1; + delete arg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_OutputBuffer_1data_1set(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + unsigned char *arg2 = (unsigned char *) 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(OutputBuffer **)&jarg1; + arg2 = *(unsigned char **)&jarg2; + if (arg1) (arg1)->data = arg2; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_OutputBuffer_1data_1get(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + unsigned char *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(OutputBuffer **)&jarg1; + result = (unsigned char *) ((arg1)->data); + *(unsigned char **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_OutputBuffer_1len_1set(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) { + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + uint64_t arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(OutputBuffer **)&jarg1; + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg2); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg2 = 0; + if (sz > 0) { + arg2 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + if (arg1) (arg1)->len = arg2; +} + + +SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_OutputBuffer_1len_1get(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jobject jresult = 0 ; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + uint64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(OutputBuffer **)&jarg1; + result = (uint64_t) ((arg1)->len); + { + jbyteArray ba = jenv->NewByteArray(9); + jbyte* bae = jenv->GetByteArrayElements(ba, 0); + jclass clazz = jenv->FindClass("java/math/BigInteger"); + jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V"); + jobject bigint; + int i; + + bae[0] = 0; + for(i=1; i<9; i++ ) { + bae[i] = (jbyte)(result>>8*(8-i)); + } + + jenv->ReleaseByteArrayElements(ba, bae, 0); + bigint = jenv->NewObject(clazz, mid, ba); + jenv->DeleteLocalRef(ba); + jresult = bigint; + } + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1OutputBuffer(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + OutputBuffer *result = 0 ; + + (void)jenv; + (void)jcls; + result = (OutputBuffer *)new OutputBuffer(); + *(OutputBuffer **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1OutputBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1) { + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(OutputBuffer **)&jarg1; + delete arg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1FrontConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1threadPoolSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + int result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setThreadPoolSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + int arg2 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (int)jarg2; + (arg1)->setThreadPoolSize(arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1nodeID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setNodeID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setNodeID((std::string const &)*arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1selfEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint(); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1mutableSelfEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint(); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setSelfEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + ppc::protocol::EndPoint *arg2 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(ppc::protocol::EndPoint **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "ppc::protocol::EndPoint const & is null"); + return ; + } + (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setGatewayGrpcTarget(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setGatewayGrpcTarget((std::string const &)*arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1gatewayGrpcTarget(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setGrpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + ppc::protocol::GrpcConfig::Ptr arg2 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + ppc::protocol::GrpcConfig::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::GrpcConfig::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->setGrpcConfig(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1grpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + ppc::protocol::GrpcConfig::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig(); + *(ppc::protocol::GrpcConfig::Ptr **)&jresult = *result ? new ppc::protocol::GrpcConfig::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1generateNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + ppc::protocol::INodeInfo::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo(); + *(ppc::protocol::INodeInfo::Ptr **)&jresult = new ppc::protocol::INodeInfo::Ptr(result); + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1getComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + std::vector< std::string > *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (std::vector< std::string > *) &((ppc::front::FrontConfig const *)arg1)->getComponents(); + *(std::vector< std::string > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::vector< std::string > *arg2 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(std::vector< std::string > **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< std::string > const & is null"); + return ; + } + (arg1)->setComponents((std::vector< std::string > const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1addComponent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->addComponent((std::string const &)*arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1mutableComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + std::vector< std::string > *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1; + arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (std::vector< std::string > *) &(arg1)->mutableComponents(); + *(std::vector< std::string > **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1FrontConfigBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::front::FrontConfigBuilder **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfigBuilder_1build_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + ppc::front::FrontConfig::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::front::FrontConfigBuilder **)&jarg1; + result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(); + *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfigBuilder_1build_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jstring jarg3) { + jlong jresult = 0 ; + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + int arg2 ; + std::string arg3 ; + ppc::front::FrontConfig::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::front::FrontConfigBuilder **)&jarg1; + arg2 = (int)jarg2; + if(!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); + if (!arg3_pstr) return 0; + (&arg3)->assign(arg3_pstr); + jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); + result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3); + *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printFrontDesc(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::front::FrontConfig::Ptr *arg1 = 0 ; + ppc::front::FrontConfig::Ptr tempnull1 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = jarg1 ? *(ppc::front::FrontConfig::Ptr **)&jarg1 : &tempnull1; + result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1EndPoint_1_1SWIG_10(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint(); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1EndPoint_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jstring jarg1, jint jarg2) { + jlong jresult = 0 ; + std::string *arg1 = 0 ; + uint16_t arg2 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + if(!jarg1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg1_pstr = (const char *)jenv->GetStringUTFChars(jarg1, 0); + if (!arg1_pstr) return 0; + std::string arg1_str(arg1_pstr); + arg1 = &arg1_str; + jenv->ReleaseStringUTFChars(jarg1, arg1_pstr); + arg2 = (uint16_t)jarg2; + result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1EndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1host(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1port(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1setHost(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setHost(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1setPort(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + uint16_t arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + arg2 = (uint16_t)jarg2; + (arg1)->setPort(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1setListenIp(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string *arg2 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setListenIp((std::string const &)*arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1entryPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint(); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1listenEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint(); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1listenIp(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::EndPoint **)&jarg1; + result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_10(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::protocol::GrpcServerConfig *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(); + *(ppc::protocol::GrpcServerConfig **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) { + jlong jresult = 0 ; + ppc::protocol::EndPoint arg1 ; + bool arg2 ; + ppc::protocol::EndPoint *argp1 ; + ppc::protocol::GrpcServerConfig *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + argp1 = *(ppc::protocol::EndPoint **)&jarg1; + if (!argp1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint"); + return 0; + } + arg1 = *argp1; + arg2 = jarg2 ? true : false; + result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2); + *(ppc::protocol::GrpcServerConfig **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1listenEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint(); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + ppc::protocol::EndPoint arg2 ; + ppc::protocol::EndPoint *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + argp2 = *(ppc::protocol::EndPoint **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint"); + return ; + } + arg2 = *argp2; + (arg1)->setEndPoint(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) { + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + bool arg2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + arg2 = jarg2 ? true : false; + (arg1)->setEnableHealthCheck(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1endPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint(); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1mutableEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint(); + *(ppc::protocol::EndPoint **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1GrpcServerConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcConfig(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::protocol::GrpcConfig *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig(); + + *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jresult = result ? new std::shared_ptr< ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1GrpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1loadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setLoadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setLoadBalancePolicy((std::string const &)*arg2); +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) { + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + bool arg2 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + arg2 = jarg2 ? true : false; + (arg1)->setEnableHealthCheck(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) { + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + bool arg2 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + arg2 = jarg2 ? true : false; + (arg1)->setEnableDnslookup(arg2); +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1; + arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageOptionalHeader(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1encode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null"); + return ; + } + ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg3) { + jlong jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytesConstRef arg2 ; + uint64_t arg3 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return 0; + } + clazz = jenv->GetObjectClass(jarg3); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg3, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg3 = 0; + if (sz > 0) { + arg3 = (uint64_t const)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + result = (int64_t)(arg1)->decode(arg2,arg3); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1componentType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType(); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setComponentType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string arg2 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setComponentType(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1srcNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + bcos::bytes *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNode(); + *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + return jresult; +} + + +SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1srcNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jbyteArray jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + OutputBuffer result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer(); + { + jresult = jenv->NewByteArray((&result)->len); + jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); + delete[] (&result)->data; + } + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcNode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null"); + return ; + } + (arg1)->setSrcNode((bcos::bytes const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcNode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg3); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg3, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg3 = 0; + if (sz > 0) { + arg3 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + (arg1)->setSrcNode(arg2,arg3); + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1dstNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + bcos::bytes *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNode(); + *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + return jresult; +} + + +SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1dstNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jbyteArray jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + OutputBuffer result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer(); + { + jresult = jenv->NewByteArray((&result)->len); + jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); + delete[] (&result)->data; + } + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null"); + return ; + } + (arg1)->setDstNode((bcos::bytes const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg3); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg3, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg3 = 0; + if (sz > 0) { + arg3 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + (arg1)->setDstNode(arg2,arg3); + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1dstInst(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstInst(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setDstInst((std::string const &)*arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1topic(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setTopic_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + std::unique_ptr< std::string > rvrdeleter2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(std::string **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::string && is null"); + return ; + } + rvrdeleter2.reset(arg2); + (arg1)->setTopic((std::string &&)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setTopic_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setTopic((std::string const &)*arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1srcInst(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst(); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcInst(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setSrcInst((std::string const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageHeader(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1encode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null"); + return ; + } + ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (int64_t)(arg1)->decode(arg2); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1version(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jshort jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + uint8_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version(); + jresult = (jshort)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setVersion(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setVersion(arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1traceID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setTraceID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setTraceID(arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1srcGwNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setSrcGwNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setSrcGwNode((std::string const &)*arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1dstGwNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setDstGwNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setDstGwNode((std::string const &)*arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1packetType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setPacketType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setPacketType(arg2); +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1ttl(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jshort jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + int16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl(); + jresult = (jshort)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setTTL(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setTTL(arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1ext(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setExt(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setExt(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1optionalField(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField(); + *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setOptionalField(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->setOptionalField(arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1length(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1isRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->setRespPacket(); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1srcP2PNodeIDView(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + std::string_view result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView(); + *(std::string_view **)&jresult = new std::string_view(result); + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1dstP2PNodeIDView(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + std::string_view result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView(); + *(std::string_view **)&jresult = new std::string_view(result); + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1routeType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1setRouteType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + ppc::protocol::RouteType arg2 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + ppc::protocol::RouteType *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::RouteType **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::RouteType"); + return ; + } + arg2 = *argp2; + (arg1)->setRouteType(arg2); +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeader_1hasOptionalField(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1; + arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1Message(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1header(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + ppc::protocol::MessageHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::Message const *)arg1)->header(); + *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setHeader(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + ppc::protocol::MessageHeader::Ptr arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + ppc::protocol::MessageHeader::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::MessageHeader::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->setHeader(arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1version(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::Message const *)arg1)->version(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setVersion(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setVersion(arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1packetType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setPacketType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setPacketType(arg2); +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1seq(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setSeq(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::string arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setSeq(arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1ext(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setExt(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setExt(arg2); +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1isRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->setRespPacket(); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1length(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + uint32_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (uint32_t)((ppc::protocol::Message const *)arg1)->length(); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1payload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + std::shared_ptr< bcos::bytes > result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::Message const *)arg1)->payload(); + *(std::shared_ptr< bcos::bytes > **)&jresult = result ? new std::shared_ptr< bcos::bytes >(result) : 0; + return jresult; +} + + +SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1payloadBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jbyteArray jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + OutputBuffer result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::Message const *)arg1)->payloadBuffer(); + { + jresult = jenv->NewByteArray((&result)->len); + jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); + delete[] (&result)->data; + } + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setPayload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< bcos::bytes > arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + std::shared_ptr< bcos::bytes > *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(std::shared_ptr< bcos::bytes > **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->setPayload(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setFrontMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + ppc::protocol::MessagePayload::Ptr arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + ppc::protocol::MessagePayload::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::MessagePayload::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->setFrontMessage(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1frontMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + ppc::protocol::MessagePayload::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage(); + *(ppc::protocol::MessagePayload::Ptr **)&jresult = *result ? new ppc::protocol::MessagePayload::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1encode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + jboolean jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null"); + return 0; + } + result = (bool)(arg1)->encode(*arg2); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1encode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jboolean jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::boostssl::EncodedMsg *arg2 = 0 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(bcos::boostssl::EncodedMsg **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::boostssl::EncodedMsg & is null"); + return 0; + } + result = (bool)(arg1)->encode(*arg2); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::Message > **)&jarg1; + arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (int64_t)(arg1)->decode(arg2); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeaderBuilder_1build_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + ppc::protocol::MessageHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (arg1)->build(arg2); + *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeaderBuilder_1build_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + result = (arg1)->build(); + *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageHeaderBuilder_1build_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + jlong jresult = 0 ; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull2 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + arg2 = jarg2 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2 : &tempnull2; + result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2); + *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageBuilder_1build_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); + result = (arg1)->build(); + *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageBuilder_1build_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + ppc::protocol::Message::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (arg1)->build(arg2); + *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageBuilder_1build_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_) { + jlong jresult = 0 ; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + ppc::protocol::RouteType arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + bcos::bytes *arg4 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + ppc::protocol::RouteType *argp2 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ; + std::unique_ptr< bcos::bytes > rvrdeleter4 ; + ppc::protocol::Message::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + (void)jarg4_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::RouteType **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::RouteType"); + return 0; + } + arg2 = *argp2; + arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; + arg4 = *(bcos::bytes **)&jarg4; + if (!arg4) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null"); + return 0; + } + rvrdeleter4.reset(arg4); + result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4); + *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageOptionalHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeaderBuilder_1build_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + jlong jresult = 0 ; + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull2 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + arg2 = jarg2 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2 : &tempnull2; + result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2); + *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeaderBuilder_1build_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); + result = (arg1)->build(); + *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printOptionalField(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr arg1 ; + ppc::protocol::MessageOptionalHeader::Ptr *argp1 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + argp1 = *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg1; + if (argp1) arg1 = *argp1; + result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1)); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::Message::Ptr *arg1 = 0 ; + ppc::protocol::Message::Ptr tempnull1 ; + std::string result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = jarg1 ? *(ppc::protocol::Message::Ptr **)&jarg1 : &tempnull1; + result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printWsMessage(JNIEnv *jenv, jclass jcls, jlong jarg1) { + jstring jresult = 0 ; + bcos::boostssl::MessageFace::Ptr *arg1 = 0 ; + std::string result; + + (void)jenv; + (void)jcls; + arg1 = *(bcos::boostssl::MessageFace::Ptr **)&jarg1; + if (!arg1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::boostssl::MessageFace::Ptr const & is null"); + return 0; + } + result = ppc::protocol::printWsMessage((bcos::boostssl::MessageFace::Ptr const &)*arg1); + jresult = jenv->NewStringUTF((&result)->c_str()); + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontMsgExtFlag_1Response_1get(JNIEnv *jenv, jclass jcls) { + jint jresult = 0 ; + ppc::protocol::FrontMsgExtFlag result; + + (void)jenv; + (void)jcls; + result = (ppc::protocol::FrontMsgExtFlag)ppc::protocol::FrontMsgExtFlag::Response; + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessagePayload(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1encode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + jlong jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null"); + return 0; + } + result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (int64_t)(arg1)->decode(arg2); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1version(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jshort jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + uint8_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version(); + jresult = (jshort)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setVersion(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jshort jarg2) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint8_t arg2 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint8_t)jarg2; + (arg1)->setVersion(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1data(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + bcos::bytes *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (bcos::bytes *) &((ppc::protocol::MessagePayload const *)arg1)->data(); + *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + return jresult; +} + + +SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1dataBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jbyteArray jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + OutputBuffer result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer(); + { + jresult = jenv->NewByteArray((&result)->len); + jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); + delete[] (&result)->data; + } + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setData_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytes *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + std::unique_ptr< bcos::bytes > rvrdeleter2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(bcos::bytes **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null"); + return ; + } + rvrdeleter2.reset(arg2); + (arg1)->setData((bcos::bytes &&)*arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1seq(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setSeq(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setSeq(arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1length(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + int64_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length(); + jresult = (jlong)result; + return jresult; +} + + +SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1traceID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jstring jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + std::string *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID(); + jresult = jenv->NewStringUTF(result->c_str()); + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setTraceID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->setTraceID((std::string const &)*arg2); +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1ext(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jint jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + uint16_t result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext(); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setExt(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint16_t arg2 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + (arg1)->setExt(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->setRespPacket(); +} + + +SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1isRespPacket(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jboolean jresult = 0 ; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + bool result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1; + arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); + result = (bool)(arg1)->isRespPacket(); + jresult = (jboolean)result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessagePayloadBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessagePayloadBuilder *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayloadBuilder_1build_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + ppc::protocol::MessagePayload::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessagePayloadBuilder *)(smartarg1 ? smartarg1->get() : 0); + result = (arg1)->build(); + *(ppc::protocol::MessagePayload::Ptr **)&jresult = result ? new ppc::protocol::MessagePayload::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayloadBuilder_1build_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + bcos::bytesConstRef *argp2 ; + ppc::protocol::MessagePayload::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jarg1; + arg1 = (ppc::protocol::MessagePayloadBuilder *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::bytesConstRef **)&jarg2; + if (!argp2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef"); + return 0; + } + arg2 = *argp2; + result = (arg1)->build(arg2); + *(ppc::protocol::MessagePayload::Ptr **)&jresult = result ? new ppc::protocol::MessagePayload::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontClient(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ; + std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::IFrontClient > **)&jarg1; + arg1 = (ppc::front::IFrontClient *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFrontClient_1onReceiveMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_, jlong jarg3) { + ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ; + ppc::protocol::Message::Ptr *arg2 = 0 ; + SwigValueWrapper< std::function< void (std::shared_ptr< bcos::Error >) > > arg3 ; + std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr tempnull2 ; + ppc::protocol::ReceiveMsgFunc *argp3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFrontClient > **)&jarg1; + arg1 = (ppc::front::IFrontClient *)(smartarg1 ? smartarg1->get() : 0); + arg2 = jarg2 ? *(ppc::protocol::Message::Ptr **)&jarg2 : &tempnull2; + argp3 = *(ppc::protocol::ReceiveMsgFunc **)&jarg3; + if (!argp3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::ReceiveMsgFunc"); + return ; + } + arg3 = *argp3; + (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ErrorCallback(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::front::ErrorCallback *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(jenv); + + *(std::shared_ptr< ppc::front::ErrorCallback > **)&jresult = result ? new std::shared_ptr< ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1ErrorCallback(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ; + std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::ErrorCallback > **)&jarg1; + arg1 = (ppc::front::ErrorCallback *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ErrorCallback_1onError(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ; + bcos::Error::Ptr arg2 ; + std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ; + bcos::Error::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::ErrorCallback > **)&jarg1; + arg1 = (ppc::front::ErrorCallback *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::Error::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->onError(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ErrorCallback_1director_1connect(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jswig_mem_own, jboolean jweak_global) { + std::shared_ptr< ppc::front::ErrorCallback > *obj = *((std::shared_ptr< ppc::front::ErrorCallback > **)&objarg); + (void)jcls; + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_ErrorCallback *director = static_cast(obj->operator->()); + director->swig_connect_director(jenv, jself, jenv->GetObjectClass(jself), (jswig_mem_own == JNI_TRUE), (jweak_global == JNI_TRUE)); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_ErrorCallback_1change_1ownership(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jtake_or_release) { + std::shared_ptr< ppc::front::ErrorCallback > *obj = *((std::shared_ptr< ppc::front::ErrorCallback > **)&objarg); + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_ErrorCallback *director = dynamic_cast(obj->operator->()); + (void)jcls; + if (director) { + director->swig_java_change_ownership(jenv, jself, jtake_or_release ? true : false); + } +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1MessageDispatcherHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::front::MessageDispatcherHandler *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(jenv); + + *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jresult = result ? new std::shared_ptr< ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageDispatcherHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jarg1; + arg1 = (ppc::front::MessageDispatcherHandler *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageDispatcherHandler_1onMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ; + ppc::protocol::Message::Ptr arg2 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jarg1; + arg1 = (ppc::front::MessageDispatcherHandler *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::protocol::Message::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->onMessage(arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageDispatcherHandler_1director_1connect(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jswig_mem_own, jboolean jweak_global) { + std::shared_ptr< ppc::front::MessageDispatcherHandler > *obj = *((std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&objarg); + (void)jcls; + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_MessageDispatcherHandler *director = static_cast(obj->operator->()); + director->swig_connect_director(jenv, jself, jenv->GetObjectClass(jself), (jswig_mem_own == JNI_TRUE), (jweak_global == JNI_TRUE)); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageDispatcherHandler_1change_1ownership(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jtake_or_release) { + std::shared_ptr< ppc::front::MessageDispatcherHandler > *obj = *((std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&objarg); + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_MessageDispatcherHandler *director = dynamic_cast(obj->operator->()); + (void)jcls; + if (director) { + director->swig_java_change_ownership(jenv, jself, jtake_or_release ? true : false); + } +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SendResponseHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + jlong jresult = 0 ; + SwigValueWrapper< std::function< void (std::shared_ptr< bcos::bytes > &&) > > arg1 ; + ppc::protocol::SendResponseFunction *argp1 ; + ppc::front::SendResponseHandler *result = 0 ; + + (void)jenv; + (void)jcls; + argp1 = *(ppc::protocol::SendResponseFunction **)&jarg1; + if (!argp1) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::SendResponseFunction"); + return 0; + } + arg1 = *argp1; + result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1); + *(ppc::front::SendResponseHandler **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SendResponseHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::front::SendResponseHandler **)&jarg1; + delete arg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_SendResponseHandler_1sendResponse(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ; + std::shared_ptr< bcos::bytes > *arg2 = 0 ; + std::unique_ptr< std::shared_ptr< bcos::bytes > > rvrdeleter2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::front::SendResponseHandler **)&jarg1; + arg2 = *(std::shared_ptr< bcos::bytes > **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::shared_ptr< bcos::bytes > && is null"); + return ; + } + rvrdeleter2.reset(arg2); + (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1IMessageHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::front::IMessageHandler *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(jenv); + + *(std::shared_ptr< ppc::front::IMessageHandler > **)&jresult = result ? new std::shared_ptr< ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IMessageHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ; + std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::IMessageHandler > **)&jarg1; + arg1 = (ppc::front::IMessageHandler *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IMessageHandler_1onMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_) { + ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ; + bcos::Error::Ptr arg2 ; + ppc::protocol::Message::Ptr arg3 ; + SwigValueWrapper< ppc::front::SendResponseHandler > arg4 ; + std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ; + bcos::Error::Ptr *argp2 ; + ppc::protocol::Message::Ptr *argp3 ; + ppc::front::SendResponseHandler *argp4 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + (void)jarg3_; + (void)jarg4_; + + smartarg1 = *(std::shared_ptr< ppc::front::IMessageHandler > **)&jarg1; + arg1 = (ppc::front::IMessageHandler *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::Error::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + argp3 = *(ppc::protocol::Message::Ptr **)&jarg3; + if (argp3) arg3 = *argp3; + argp4 = *(ppc::front::SendResponseHandler **)&jarg4; + if (!argp4) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::front::SendResponseHandler"); + return ; + } + arg4 = *argp4; + (arg1)->onMessage(arg2,arg3,arg4); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IMessageHandler_1director_1connect(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jswig_mem_own, jboolean jweak_global) { + std::shared_ptr< ppc::front::IMessageHandler > *obj = *((std::shared_ptr< ppc::front::IMessageHandler > **)&objarg); + (void)jcls; + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_IMessageHandler *director = static_cast(obj->operator->()); + director->swig_connect_director(jenv, jself, jenv->GetObjectClass(jself), (jswig_mem_own == JNI_TRUE), (jweak_global == JNI_TRUE)); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IMessageHandler_1change_1ownership(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jtake_or_release) { + std::shared_ptr< ppc::front::IMessageHandler > *obj = *((std::shared_ptr< ppc::front::IMessageHandler > **)&objarg); + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_IMessageHandler *director = dynamic_cast(obj->operator->()); + (void)jcls; + if (director) { + director->swig_java_change_ownership(jenv, jself, jtake_or_release ? true : false); + } +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFront(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1start(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->start(); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1stop(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->stop(); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1register_1topic_1handler(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2, jlong jarg3, jobject jarg3_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr arg3 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr *argp3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + argp3 = *(ppc::front::MessageDispatcherHandler::Ptr **)&jarg3; + if (argp3) arg3 = *argp3; + (arg1)->register_topic_handler((std::string const &)*arg2,arg3); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1register_1msg_1handler(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2, jlong jarg3, jobject jarg3_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr arg3 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr *argp3 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + argp3 = *(ppc::front::MessageDispatcherHandler::Ptr **)&jarg3; + if (argp3) arg3 = *argp3; + (arg1)->register_msg_handler((std::string const &)*arg2,arg3); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1async_1send_1message(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jbyteArray jarg4, jobject jarg5, jint jarg6, jint jarg7, jlong jarg8, jobject jarg8_, jlong jarg9, jobject jarg9_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + char *arg4 = (char *) 0 ; + uint64_t arg5 ; + int arg6 ; + long arg7 ; + ppc::front::ErrorCallback::Ptr arg8 ; + ppc::front::IMessageHandler::Ptr arg9 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ; + ppc::front::ErrorCallback::Ptr *argp8 ; + ppc::front::IMessageHandler::Ptr *argp9 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + (void)jarg8_; + (void)jarg9_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; + { + arg4 = (char *) jenv->GetByteArrayElements(jarg4, 0); + } + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg5) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg5); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg5, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg5 = 0; + if (sz > 0) { + arg5 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + arg6 = (int)jarg6; + arg7 = (long)jarg7; + argp8 = *(ppc::front::ErrorCallback::Ptr **)&jarg8; + if (argp8) arg8 = *argp8; + argp9 = *(ppc::front::IMessageHandler::Ptr **)&jarg9; + if (argp9) arg9 = *argp9; + (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9); + { + jenv->ReleaseByteArrayElements(jarg4, (jbyte *) arg4, 0); + } + +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1async_1send_1response(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_, jstring jarg3, jlong jarg4, jobject jarg4_, jint jarg5, jlong jarg6, jobject jarg6_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + bcos::bytes *arg2 = 0 ; + std::string *arg3 = 0 ; + bcos::bytes *arg4 = 0 ; + int arg5 ; + ppc::front::ErrorCallback::Ptr arg6 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + std::unique_ptr< bcos::bytes > rvrdeleter4 ; + ppc::front::ErrorCallback::Ptr *argp6 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + (void)jarg4_; + (void)jarg6_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + + arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0); + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null"); + return ; + } + if(!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); + if (!arg3_pstr) return ; + std::string arg3_str(arg3_pstr); + arg3 = &arg3_str; + jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); + arg4 = *(bcos::bytes **)&jarg4; + if (!arg4) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null"); + return ; + } + rvrdeleter4.reset(arg4); + arg5 = (int)jarg5; + argp6 = *(ppc::front::ErrorCallback::Ptr **)&jarg6; + if (argp6) arg6 = *argp6; + (arg1)->async_send_response((bcos::bytes const &)*arg2,(std::string const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_, jint jarg5, jint jarg6) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + bcos::bytes *arg4 = 0 ; + int arg5 ; + long arg6 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ; + std::unique_ptr< bcos::bytes > rvrdeleter4 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + (void)jarg4_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; + arg4 = *(bcos::bytes **)&jarg4; + if (!arg4) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null"); + return 0; + } + rvrdeleter4.reset(arg4); + arg5 = (int)jarg5; + arg6 = (long)jarg6; + result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jbyteArray jarg4, jobject jarg5, jint jarg6, jint jarg7) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + char *arg4 = (char *) 0 ; + uint64_t arg5 ; + int arg6 ; + long arg7 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg3_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + arg2 = (uint16_t)jarg2; + arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; + { + arg4 = (char *) jenv->GetByteArrayElements(jarg4, 0); + } + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg5) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return 0; + } + clazz = jenv->GetObjectClass(jarg5); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg5, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg5 = 0; + if (sz > 0) { + arg5 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + arg6 = (int)jarg6; + arg7 = (long)jarg7; + result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + { + jenv->ReleaseByteArrayElements(jarg4, (jbyte *) arg4, 0); + } + + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1pop(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2, jint jarg3) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + long arg3 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + arg3 = (long)jarg3; + result = (arg1)->pop((std::string const &)*arg2,arg3); + *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1peek(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + result = (arg1)->peek((std::string const &)*arg2); + *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + ppc::protocol::INodeInfo::Ptr *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + arg2 = *(ppc::protocol::INodeInfo::Ptr **)&jarg2; + if (!arg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "ppc::protocol::INodeInfo::Ptr const & is null"); + return 0; + } + result = (arg1)->registerNodeInfo((ppc::protocol::INodeInfo::Ptr const &)*arg2); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1unRegisterNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + result = (arg1)->unRegisterNodeInfo(); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerTopic(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + result = (arg1)->registerTopic((std::string const &)*arg2); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1unRegisterTopic(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + bcos::Error::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + result = (arg1)->unRegisterTopic((std::string const &)*arg2); + *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::front::IFrontBuilder **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFrontBuilder_1buildClient(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2, jlong jarg3, jboolean jarg4) { + jlong jresult = 0 ; + ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ; + std::string arg2 ; + std::function< void () > arg3 ; + bool arg4 ; + std::function< void () > *argp3 ; + ppc::front::IFrontClient::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::front::IFrontBuilder **)&jarg1; + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return 0; + (&arg2)->assign(arg2_pstr); + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + argp3 = *(std::function< void () > **)&jarg3; + if (!argp3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null std::function< void () >"); + return 0; + } + arg3 = *argp3; + arg4 = jarg4 ? true : false; + result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4); + *(ppc::front::IFrontClient::Ptr **)&jresult = result ? new ppc::front::IFrontClient::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_SDKMode_1AIR_1get(JNIEnv *jenv, jclass jcls) { + jint jresult = 0 ; + ppc::sdk::SDKMode result; + + (void)jenv; + (void)jcls; + result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::AIR; + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_SDKMode_1PRO_1get(JNIEnv *jenv, jclass jcls) { + jint jresult = 0 ; + ppc::sdk::SDKMode result; + + (void)jenv; + (void)jcls; + result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::PRO; + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1TransportBuilder(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::sdk::TransportBuilder *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder(); + *(ppc::sdk::TransportBuilder **)&jresult = result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1TransportBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + + (void)jenv; + (void)jcls; + arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; + delete arg1; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1buildProTransport(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + jlong jresult = 0 ; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + ppc::front::FrontConfig::Ptr arg2 ; + ppc::front::FrontConfig::Ptr *argp2 ; + ppc::sdk::Transport::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; + argp2 = *(ppc::front::FrontConfig::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + result = (arg1)->buildProTransport(arg2); + *(ppc::sdk::Transport::Ptr **)&jresult = result ? new ppc::sdk::Transport::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1buildConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jstring jarg3) { + jlong jresult = 0 ; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + int arg2 ; + std::string arg3 ; + ppc::front::FrontConfig::Ptr result; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; + arg2 = (int)jarg2; + if(!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return 0; + } + const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); + if (!arg3_pstr) return 0; + (&arg3)->assign(arg3_pstr); + jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); + result = (arg1)->buildConfig(arg2,arg3); + *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1frontConfigBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + ppc::front::FrontConfigBuilder::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; + result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder(); + *(ppc::front::FrontConfigBuilder::Ptr **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1Transport(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::FrontConfig::Ptr arg1 ; + ppc::front::FrontConfig::Ptr *argp1 ; + ppc::sdk::Transport *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + argp1 = *(ppc::front::FrontConfig::Ptr **)&jarg1; + if (argp1) arg1 = *argp1; + result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1); + + *(std::shared_ptr< ppc::sdk::Transport > **)&jresult = result ? new std::shared_ptr< ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1Transport(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1start(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->start(); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1stop(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + (arg1)->stop(); +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1getFront(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::front::IFront::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront(); + *(ppc::front::IFront::Ptr **)&jresult = *result ? new ppc::front::IFront::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1gateway(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::gateway::IGateway::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway(); + *(ppc::gateway::IGateway::Ptr **)&jresult = result; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1msgPayloadBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::protocol::MessagePayloadBuilder::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder(); + *(ppc::protocol::MessagePayloadBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessagePayloadBuilder::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1msgBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::protocol::MessageBuilder::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder(); + *(ppc::protocol::MessageBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageBuilder::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1msgHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::protocol::MessageHeaderBuilder::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder(); + *(ppc::protocol::MessageHeaderBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageHeaderBuilder::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1routeInfoBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeaderBuilder::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder(); + *(ppc::protocol::MessageOptionalHeaderBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageOptionalHeaderBuilder::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Transport_1getConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + ppc::front::FrontConfig::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1; + arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig(); + *(ppc::front::FrontConfig::Ptr **)&jresult = *result ? new ppc::front::FrontConfig::Ptr(*result) : 0; + return jresult; +} + + +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1SWIGSmartPtrUpcast(JNIEnv *jenv, jclass jcls, jlong jarg1) { + jlong baseptr = 0; + std::shared_ptr< ppc::front::IFront > *argp1; + (void)jenv; + (void)jcls; + argp1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + *(std::shared_ptr< ppc::front::IFrontClient > **)&baseptr = argp1 ? new std::shared_ptr< ppc::front::IFrontClient >(*argp1) : 0; + return baseptr; +} + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_swig_1module_1init(JNIEnv *jenv, jclass jcls) { + int i; + + static struct { + const char *method; + const char *signature; + } methods[3] = { + { + "SwigDirector_ErrorCallback_onError", "(Lcom/webank/wedpr/sdk/jni/generated/ErrorCallback;J)V" + }, + { + "SwigDirector_MessageDispatcherHandler_onMessage", "(Lcom/webank/wedpr/sdk/jni/generated/MessageDispatcherHandler;J)V" + }, + { + "SwigDirector_IMessageHandler_onMessage", "(Lcom/webank/wedpr/sdk/jni/generated/IMessageHandler;JJJ)V" + } + }; + Swig::jclass_wedpr_java_transportJNI = (jclass) jenv->NewGlobalRef(jcls); + if (!Swig::jclass_wedpr_java_transportJNI) return; + for (i = 0; i < (int) (sizeof(methods)/sizeof(methods[0])); ++i) { + Swig::director_method_ids[i] = jenv->GetStaticMethodID(jcls, methods[i].method, methods[i].signature); + if (!Swig::director_method_ids[i]) return; + } +} + + +#ifdef __cplusplus +} +#endif + diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h new file mode 100644 index 00000000..89b6596b --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h @@ -0,0 +1,58 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +#ifndef SWIG_wedpr_java_transport_WRAP_H_ +#define SWIG_wedpr_java_transport_WRAP_H_ + +class SwigDirector_ErrorCallback : public ppc::front::ErrorCallback, public Swig::Director { + +public: + void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global); + SwigDirector_ErrorCallback(JNIEnv *jenv); + virtual ~SwigDirector_ErrorCallback(); + virtual void onError(bcos::Error::Ptr error); +public: + bool swig_overrides(int n) { + return (n < 1 ? swig_override[n] : false); + } +protected: + Swig::BoolArray<1> swig_override; +}; + +class SwigDirector_MessageDispatcherHandler : public ppc::front::MessageDispatcherHandler, public Swig::Director { + +public: + void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global); + SwigDirector_MessageDispatcherHandler(JNIEnv *jenv); + virtual ~SwigDirector_MessageDispatcherHandler(); + virtual void onMessage(ppc::protocol::Message::Ptr msg); +public: + bool swig_overrides(int n) { + return (n < 1 ? swig_override[n] : false); + } +protected: + Swig::BoolArray<1> swig_override; +}; + +class SwigDirector_IMessageHandler : public ppc::front::IMessageHandler, public Swig::Director { + +public: + void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global); + SwigDirector_IMessageHandler(JNIEnv *jenv); + virtual ~SwigDirector_IMessageHandler(); + virtual void onMessage(bcos::Error::Ptr e,ppc::protocol::Message::Ptr msg,ppc::front::SendResponseHandler sendResponseHandler); +public: + bool swig_overrides(int n) { + return (n < 1 ? swig_override[n] : false); + } +protected: + Swig::BoolArray<1> swig_override; +}; + + +#endif From 299f946516fb5ce309516115897fba3e005d3ba1 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 18 Sep 2024 12:48:00 +0800 Subject: [PATCH 043/120] update async_send_response interface (#40) * update build.gradle * update async_send_response interface --- cpp/ppc-framework/front/IFront.h | 9 +- .../sdk-wrapper/java/bindings/build.gradle | 5 +- .../wedpr/sdk/jni/generated/IFront.java | 12 +- .../wedpr/sdk/jni/generated/IFrontClient.java | 2 + .../generated/wedpr_java_transportJNI.java | 16 +-- .../sdk/jni/transport/WeDPRTransport.java | 16 +++ .../sdk/jni/transport/impl/TransportImpl.java | 19 +++ .../src/wedpr_java_transportJAVA_wrap.cxx | 117 +++++++++++++----- 8 files changed, 148 insertions(+), 48 deletions(-) diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index da46c575..fd0c10cc 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -183,11 +183,14 @@ class IFront : virtual public IFrontClient // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java // byte[] data - virtual void async_send_response(bcos::bytes const& dstNode, std::string const& traceID, - bcos::bytes&& payload, int seq, ErrorCallback::Ptr errorCallback) + virtual void async_send_response(char* dstNode, uint64_t dstNodeSize, std::string const& traceID, + char* payload, uint64_t payloadSize, int seq, ErrorCallback::Ptr errorCallback) { + // TODO: optimize here + bcos::bytes copiedDstNode(dstNode, dstNode + dstNodeSize); + bcos::bytes copyedPayload(payload, payload + payloadSize); asyncSendResponse( - dstNode, traceID, std::move(payload), seq, populateErrorCallback(errorCallback)); + copiedDstNode, traceID, std::move(copyedPayload), seq, populateErrorCallback(errorCallback)); } // the sync interface for async_send_message diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index 362b0ba3..c703444d 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -24,6 +24,7 @@ ext { junitVersion = "4.13.2" log4jVersion = "2.23.0" jmhVersion = "1.36" + println("*ossrhUsername: " + ossrhUsername) } println("Notice: current gradle version is " + gradle.gradleVersion) @@ -199,8 +200,8 @@ publishing { url = !version.endsWith("SNAPSHOT") ? releasesRepoURL : snapshotsRepoURL credentials { - username 'deployment' - password 'deployment123' + username ossrhUsername + password ossrhPassword } } signing { diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java index ce4d2352..bc5aac7c 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java @@ -99,15 +99,21 @@ public void async_send_message( } public void async_send_response( - ubytes dstNode, String traceID, ubytes payload, int seq, ErrorCallback errorCallback) { + byte[] dstNode, + java.math.BigInteger dstNodeSize, + String traceID, + byte[] payload, + java.math.BigInteger payloadSize, + int seq, + ErrorCallback errorCallback) { wedpr_java_transportJNI.IFront_async_send_response( swigCPtr, this, - ubytes.getCPtr(dstNode), dstNode, + dstNodeSize, traceID, - ubytes.swigRelease(payload), payload, + payloadSize, seq, ErrorCallback.getCPtr(errorCallback), errorCallback); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java index a9d97546..ac777ec8 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFrontClient.java @@ -43,6 +43,8 @@ public synchronized void delete() { /** * : receive message from gateway, call by gateway
*
+ * + * @return void */ public void onReceiveMessage( Message _msg, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java index 121320fd..8418644a 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java @@ -677,14 +677,14 @@ public static final native void IFront_async_send_message( public static final native void IFront_async_send_response( long jarg1, IFront jarg1_, - long jarg2, - ubytes jarg2_, - String jarg3, - long jarg4, - ubytes jarg4_, - int jarg5, - long jarg6, - ErrorCallback jarg6_); + byte[] jarg2, + java.math.BigInteger jarg3, + String jarg4, + byte[] jarg5, + java.math.BigInteger jarg6, + int jarg7, + long jarg8, + ErrorCallback jarg8_); public static final native long IFront_push__SWIG_0( long jarg1, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java index 97146fde..5a716f6b 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java @@ -132,6 +132,22 @@ void asyncSendMessageByTopic( MessageErrorCallback errorCallback, MessageCallback msgCallback); + /** + * send response to given node + * + * @param dstNode the node that should receive the response + * @param traceID the traceID of the response + * @param payload the payload + * @param seq the seq + * @param errorCallback the error callback + */ + void asyncSendResponse( + byte[] dstNode, + String traceID, + byte[] payload, + int seq, + MessageErrorCallback errorCallback); + /** @param topic the topic to remove */ void removeTopic(String topic) throws WeDPRSDKException; diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java index ebc36f26..46420d94 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -243,6 +243,25 @@ public void asyncSendMessageByTopic( msgCallback); } + @Override + public void asyncSendResponse( + byte[] dstNode, + String traceID, + byte[] payload, + int seq, + MessageErrorCallback errorCallback) { + this.transport + .getFront() + .async_send_response( + dstNode, + BigInteger.valueOf(dstNode.length), + traceID, + payload, + BigInteger.valueOf(payload.length), + seq, + errorCallback); + } + /** @param topic the topic to remove */ @Override public void removeTopic(String topic) throws WeDPRSDKException { diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx index 66ff9162..e103bfd8 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx @@ -6080,51 +6080,104 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } -SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1async_1send_1response(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_, jstring jarg3, jlong jarg4, jobject jarg4_, jint jarg5, jlong jarg6, jobject jarg6_) { +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1async_1send_1response(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3, jstring jarg4, jbyteArray jarg5, jobject jarg6, jint jarg7, jlong jarg8, jobject jarg8_) { ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; - bcos::bytes *arg2 = 0 ; - std::string *arg3 = 0 ; - bcos::bytes *arg4 = 0 ; - int arg5 ; - ppc::front::ErrorCallback::Ptr arg6 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + std::string *arg4 = 0 ; + char *arg5 = (char *) 0 ; + uint64_t arg6 ; + int arg7 ; + ppc::front::ErrorCallback::Ptr arg8 ; std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; - std::unique_ptr< bcos::bytes > rvrdeleter4 ; - ppc::front::ErrorCallback::Ptr *argp6 ; + ppc::front::ErrorCallback::Ptr *argp8 ; (void)jenv; (void)jcls; (void)jarg1_; - (void)jarg2_; - (void)jarg4_; - (void)jarg6_; + (void)jarg8_; smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); - - arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0); - if (!arg2) { - SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null"); - return ; - } - if(!jarg3) { + { + arg2 = (char *) jenv->GetByteArrayElements(jarg2, 0); + } + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg3); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg3, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg3 = 0; + if (sz > 0) { + arg3 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + if(!jarg4) { SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); return ; } - const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); - if (!arg3_pstr) return ; - std::string arg3_str(arg3_pstr); - arg3 = &arg3_str; - jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); - arg4 = *(bcos::bytes **)&jarg4; - if (!arg4) { - SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null"); - return ; + const char *arg4_pstr = (const char *)jenv->GetStringUTFChars(jarg4, 0); + if (!arg4_pstr) return ; + std::string arg4_str(arg4_pstr); + arg4 = &arg4_str; + jenv->ReleaseStringUTFChars(jarg4, arg4_pstr); + { + arg5 = (char *) jenv->GetByteArrayElements(jarg5, 0); } - rvrdeleter4.reset(arg4); - arg5 = (int)jarg5; - argp6 = *(ppc::front::ErrorCallback::Ptr **)&jarg6; - if (argp6) arg6 = *argp6; - (arg1)->async_send_response((bcos::bytes const &)*arg2,(std::string const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6); + { + jclass clazz; + jmethodID mid; + jbyteArray ba; + jbyte* bae; + jsize sz; + int i; + + if (!jarg6) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null"); + return ; + } + clazz = jenv->GetObjectClass(jarg6); + mid = jenv->GetMethodID(clazz, "toByteArray", "()[B"); + ba = (jbyteArray)jenv->CallObjectMethod(jarg6, mid); + bae = jenv->GetByteArrayElements(ba, 0); + sz = jenv->GetArrayLength(ba); + arg6 = 0; + if (sz > 0) { + arg6 = (uint64_t)(signed char)bae[0]; + for(i=1; iReleaseByteArrayElements(ba, bae, 0); + } + arg7 = (int)jarg7; + argp8 = *(ppc::front::ErrorCallback::Ptr **)&jarg8; + if (argp8) arg8 = *argp8; + (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8); + { + jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); + } + { + jenv->ReleaseByteArrayElements(jarg5, (jbyte *) arg5, 0); + } + + } From 9d69fe3252b6507205fe5674923c342c44d306f4 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 18 Sep 2024 21:29:13 +0800 Subject: [PATCH 044/120] add asyncGetPeers && registerComponent && unRegisterComponent (#41) --- cpp/cmake/Options.cmake | 1 + cpp/ppc-framework/front/IFront.h | 28 +- cpp/ppc-framework/gateway/IGateway.h | 2 +- cpp/ppc-framework/protocol/INodeInfo.h | 5 +- .../src/config/ppc_deploy_config.py | 6 +- .../grpc/client/GatewayClient.cpp | 8 +- .../grpc/client/GatewayClient.h | 2 +- .../grpc/server/GatewayServer.cpp | 11 +- .../grpc/server/GatewayServer.h | 2 +- cpp/wedpr-protocol/proto/pb/Service.proto | 5 +- .../protobuf/src/NodeInfoImpl.cpp | 14 +- .../protobuf/src/NodeInfoImpl.h | 48 +++- .../ppc-front/ppc-front/Front.cpp | 52 ++-- .../ppc-front/ppc-front/FrontImpl.cpp | 24 ++ .../ppc-front/ppc-front/FrontImpl.h | 11 +- .../ppc-gateway/gateway/GatewayImpl.cpp | 5 +- .../ppc-gateway/gateway/GatewayImpl.h | 2 +- .../gateway/router/GatewayNodeInfo.h | 1 + .../gateway/router/GatewayNodeInfoImpl.cpp | 15 ++ .../gateway/router/GatewayNodeInfoImpl.h | 2 + .../ppc-gateway/gateway/router/LocalRouter.h | 1 - .../gateway/router/PeerRouterTable.cpp | 21 +- .../gateway/router/PeerRouterTable.h | 2 +- .../sdk-wrapper/CMakeLists.txt | 2 +- .../jni/generated/GetPeersInfoHandler.java | 67 +++++ .../wedpr/sdk/jni/generated/IFront.java | 18 ++ .../generated/SharedGetPeersInfoHandler.java | 54 ++++ .../generated/wedpr_java_transportJNI.java | 33 +++ .../sdk/jni/transport/WeDPRTransport.java | 17 ++ .../transport/handlers/GetPeersCallback.java | 31 +++ .../sdk/jni/transport/impl/TransportImpl.java | 37 ++- .../src/wedpr_java_transportJAVA_wrap.cxx | 254 +++++++++++++++++- .../java/src/wedpr_java_transportJAVA_wrap.h | 15 ++ .../java/swig/wedpr_java_transport.i | 3 + .../sdk/src/ProTransportImpl.cpp | 2 +- 35 files changed, 739 insertions(+), 62 deletions(-) create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGetPeersInfoHandler.java create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake index 0293ad6b..cdc6eace 100644 --- a/cpp/cmake/Options.cmake +++ b/cpp/cmake/Options.cmake @@ -181,6 +181,7 @@ macro(print_config NAME) message("-- BUILD_SDK BUILD SDK ${BUILD_SDK}") message("-- BUILD_UDF BUILD UDF ${BUILD_UDF}") message("-- BUILD_WEDPR_TOOLKIT BUILD_WEDPR_TOOLKIT ${BUILD_WEDPR_TOOLKIT}") + message("-- AUTO_GENERATE AUTO_GENERATE ${AUTO_GENERATE}") message("-- DEBUG ${DEBUG}") message("------------------------------------------------------------------------") message("") diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index fd0c10cc..dfa8e95f 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -63,6 +63,7 @@ class MessageDispatcherHandler virtual void onMessage(ppc::protocol::Message::Ptr msg) = 0; }; + class SendResponseHandler { public: @@ -92,6 +93,16 @@ class IMessageHandler SendResponseHandler sendResponseHandler) = 0; }; +class GetPeersInfoHandler +{ +public: + using Ptr = std::shared_ptr; + GetPeersInfoHandler() = default; + virtual ~GetPeersInfoHandler() {} + + virtual void onPeersInfo(bcos::Error::Ptr e, std::string const& peersInfo) = 0; +}; + ///////// the callback definition for sdk wrapper ///////// class IFront : virtual public IFrontClient @@ -183,14 +194,15 @@ class IFront : virtual public IFrontClient // !!! Note: the 'payload ' type(char*) should not been changed, since it used to pass-in java // byte[] data - virtual void async_send_response(char* dstNode, uint64_t dstNodeSize, std::string const& traceID, - char* payload, uint64_t payloadSize, int seq, ErrorCallback::Ptr errorCallback) + virtual void async_send_response(char* dstNode, uint64_t dstNodeSize, + std::string const& traceID, char* payload, uint64_t payloadSize, int seq, + ErrorCallback::Ptr errorCallback) { // TODO: optimize here bcos::bytes copiedDstNode(dstNode, dstNode + dstNodeSize); bcos::bytes copyedPayload(payload, payload + payloadSize); - asyncSendResponse( - copiedDstNode, traceID, std::move(copyedPayload), seq, populateErrorCallback(errorCallback)); + asyncSendResponse(copiedDstNode, traceID, std::move(copyedPayload), seq, + populateErrorCallback(errorCallback)); } // the sync interface for async_send_message @@ -209,9 +221,11 @@ class IFront : virtual public IFrontClient virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0; virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0; - virtual void asyncGetAgencies( + virtual void asyncGetAgencies(std::vector const& components, std::function)> callback) = 0; + virtual void asyncGetPeers(GetPeersInfoHandler::Ptr getPeersCallback) = 0; + /** * @brief register the nodeInfo to the gateway * @param nodeInfo the nodeInfo @@ -223,6 +237,8 @@ class IFront : virtual public IFrontClient */ virtual bcos::Error::Ptr unRegisterNodeInfo() = 0; + virtual ppc::protocol::INodeInfo::Ptr const& nodeInfo() = 0; + /** * @brief register the topic * @@ -237,6 +253,8 @@ class IFront : virtual public IFrontClient */ virtual bcos::Error::Ptr unRegisterTopic(std::string const& topic) = 0; + virtual void registerComponent(std::string const& component) = 0; + virtual void unRegisterComponent(std::string const& component) = 0; private: ppc::protocol::ReceiveMsgFunc populateErrorCallback(ErrorCallback::Ptr errorCallback) diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h index 7ccc53c5..2b5431b0 100644 --- a/cpp/ppc-framework/gateway/IGateway.h +++ b/cpp/ppc-framework/gateway/IGateway.h @@ -66,7 +66,7 @@ class IGateway bcos::bytes&& payload) = 0; virtual void asyncGetPeers(std::function callback) = 0; - virtual void asyncGetAgencies( + virtual void asyncGetAgencies(std::vector const& components, std::function)> callback) = 0; virtual bcos::Error::Ptr registerNodeInfo(ppc::protocol::INodeInfo::Ptr const& nodeInfo) = 0; diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h index c949e8ed..37f8b772 100644 --- a/cpp/ppc-framework/protocol/INodeInfo.h +++ b/cpp/ppc-framework/protocol/INodeInfo.h @@ -47,7 +47,10 @@ class INodeInfo // components virtual void setComponents(std::set const& components) = 0; + virtual bool addComponent(std::string const& component) = 0; + virtual bool eraseComponent(std::string const& component) = 0; virtual std::set const& components() const = 0; + virtual std::vector copiedComponents() const = 0; virtual void encode(bcos::bytes& data) const = 0; virtual void decode(bcos::bytesConstRef data) = 0; @@ -58,7 +61,7 @@ class INodeInfo virtual bool equal(INodeInfo::Ptr const& info) { return (nodeID().toBytes() == info->nodeID().toBytes()) && - (components() == info->components()); + (copiedComponents() == info->copiedComponents()); } virtual void toJson(Json::Value& jsonObject) const = 0; diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py index bdaffb85..88bae36f 100644 --- a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py +++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py @@ -279,9 +279,13 @@ def __init__(self, config, gateway_must_exists, node_must_exists): # parse the node config utilities.log_debug("load the node config") node_config_section_name = "[[agency.node]]" + # Note: the node is not required to exist node_config_list = utilities.get_item_value( - self.config, "node", None, node_must_exists, node_config_section_name) + self.config, "node", None, False, node_config_section_name) self.node_list = {} + # the case without node + if node_config_list is None: + return # TODO: check the node-name for node_object in node_config_list: node_config = NodeConfig( diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp index 9a71005b..b8108dbe 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp @@ -65,12 +65,16 @@ void GatewayClient::asyncGetPeers(std::function const& components, std::function)> callback) { auto response = std::make_shared(); auto context = std::make_shared(); - auto request = std::make_shared(); + auto request = std::make_shared(); + for (auto const& it : components) + { + request->add_components(it); + } // lambda keeps the lifecycle for clientContext m_stub->async()->asyncGetAgencies( context.get(), request.get(), response.get(), [context, callback, response](Status status) { diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h index 2206e000..2594890e 100644 --- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h +++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h @@ -54,7 +54,7 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient bcos::bytes&& payload, long timeout, ppc::protocol::ReceiveMsgFunc callback) override; void asyncGetPeers(std::function callback) override; - void asyncGetAgencies( + void asyncGetAgencies(std::vector const& components, std::function)> callback) override; void asyncSendbroadcastMessage(ppc::protocol::RouteType routeType, diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp index e25fcb14..f8e701ed 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp @@ -75,14 +75,19 @@ grpc::ServerUnaryReactor* GatewayServer::asyncGetPeers( return reactor; } -grpc::ServerUnaryReactor* GatewayServer::asyncGetAgencies( - grpc::CallbackServerContext* context, const ppc::proto::Empty*, ppc::proto::AgenciesInfo* reply) +grpc::ServerUnaryReactor* GatewayServer::asyncGetAgencies(grpc::CallbackServerContext* context, + const ppc::proto::Condition* condition, ppc::proto::AgenciesInfo* reply) { ServerUnaryReactor* reactor(context->DefaultReactor()); try { + std::vector components; + for (int i = 0; i < condition->components_size(); i++) + { + components.emplace_back(condition->components(i)); + } m_gateway->asyncGetAgencies( - [reactor, reply](bcos::Error::Ptr error, std::set agencies) { + components, [reactor, reply](bcos::Error::Ptr error, std::set agencies) { toSerializedError(reply->mutable_error(), error); for (auto const& it : agencies) { diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.h b/cpp/wedpr-protocol/grpc/server/GatewayServer.h index 7c62048d..7de12ed7 100644 --- a/cpp/wedpr-protocol/grpc/server/GatewayServer.h +++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.h @@ -42,7 +42,7 @@ class GatewayServer : public ppc::proto::Gateway::CallbackService grpc::ServerUnaryReactor* asyncGetPeers(grpc::CallbackServerContext* context, const ppc::proto::Empty* request, ppc::proto::PeersInfo* reply) override; grpc::ServerUnaryReactor* asyncGetAgencies(grpc::CallbackServerContext* context, - const ppc::proto::Empty* request, ppc::proto::AgenciesInfo* reply) override; + const ppc::proto::Condition* request, ppc::proto::AgenciesInfo* reply) override; grpc::ServerUnaryReactor* registerNodeInfo(grpc::CallbackServerContext* context, diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto index f531f08f..2f633dba 100644 --- a/cpp/wedpr-protocol/proto/pb/Service.proto +++ b/cpp/wedpr-protocol/proto/pb/Service.proto @@ -41,7 +41,10 @@ message PeersInfo{ string peersInfo = 2; }; message Empty{ +}; +message Condition{ + repeated string components = 1; }; service Front { @@ -50,7 +53,7 @@ service Front { service Gateway{ rpc asyncSendMessage(SendedMessageRequest) returns(Error){} rpc asyncGetPeers(Empty)returns(PeersInfo){} - rpc asyncGetAgencies(Empty)returns(AgenciesInfo){} + rpc asyncGetAgencies(Condition)returns(AgenciesInfo){} rpc registerNodeInfo(NodeInfo) returns(Error){} rpc unRegisterNodeInfo(NodeInfo)returns(Error){} rpc registerTopic(NodeInfo) returns(Error){} diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp index 59cf149e..2ecde528 100644 --- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp @@ -25,16 +25,28 @@ using namespace ppc::protocol; void NodeInfoImpl::encode(bcos::bytes& data) const { + encodeFields(); + encodePBObject(data, m_rawNodeInfo); +} + +void NodeInfoImpl::encodeFields() const +{ + bcos::ReadGuard l(x_components); // set the components for (auto const& component : m_components) { m_rawNodeInfo->add_components(component); } - encodePBObject(data, m_rawNodeInfo); } void NodeInfoImpl::decode(bcos::bytesConstRef data) { decodePBObject(m_rawNodeInfo, data); + decodeFields(); +} + +void NodeInfoImpl::decodeFields() +{ + bcos::WriteGuard l(x_components); m_components = std::set( m_rawNodeInfo->components().begin(), m_rawNodeInfo->components().end()); } diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h index 02f8b31d..2871074a 100644 --- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h +++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h @@ -32,7 +32,9 @@ class NodeInfoImpl : public INodeInfo NodeInfoImpl() { m_rawNodeInfo = std::make_shared(); } explicit NodeInfoImpl(std::shared_ptr rawNodeInfo) : m_rawNodeInfo(rawNodeInfo) - {} + { + decodeFields(); + } NodeInfoImpl(bcos::bytesConstRef const& data) : NodeInfoImpl() { decode(data); } NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) : NodeInfoImpl() @@ -54,9 +56,45 @@ class NodeInfoImpl : public INodeInfo void setComponents(std::set const& components) override { + bcos::WriteGuard l(x_components); m_components = components; } - std::set const& components() const override { return m_components; } + + std::set const& components() const override + { + bcos::ReadGuard l(x_components); + return m_components; + } + + std::vector copiedComponents() const override + { + bcos::ReadGuard l(x_components); + return std::vector(m_components.begin(), m_components.end()); + } + + bool addComponent(std::string const& component) override + { + bcos::UpgradableGuard l(x_components); + if (m_components.count(component)) + { + return false; + } + bcos::UpgradeGuard ul(l); + m_components.insert(component); + return true; + } + + bool eraseComponent(std::string const& component) override + { + bcos::UpgradableGuard l(x_components); + if (!m_components.count(component)) + { + return false; + } + bcos::UpgradeGuard ul(l); + m_components.erase(component); + return true; + } std::string const& endPoint() const override { return m_rawNodeInfo->endpoint(); } @@ -78,9 +116,15 @@ class NodeInfoImpl : public INodeInfo void toJson(Json::Value& jsonObject) const override; + virtual void encodeFields() const; + +protected: + virtual void decodeFields(); + private: std::shared_ptr m_front; std::set m_components; + mutable bcos::SharedMutex x_components; std::shared_ptr m_rawNodeInfo; }; diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp index 7a205b7b..0fa9c7f5 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp @@ -29,7 +29,7 @@ using namespace ppc::front; Front::Front(ppc::front::PPCMessageFaceFactory::Ptr ppcMsgFactory, IFront::Ptr front) : m_messageFactory(std::move(ppcMsgFactory)), m_front(std::move(front)) { - m_fetcher = std::make_shared(60 * 1000, "metaFetcher"); + m_fetcher = std::make_shared(10 * 1000, "metaFetcher"); m_fetcher->registerTimeoutHandler([this]() { try { @@ -58,30 +58,31 @@ void Front::stop() void Front::fetchGatewayMetaInfo() { auto self = weak_from_this(); - m_front->asyncGetAgencies([self](bcos::Error::Ptr error, std::set agencies) { - auto front = self.lock(); - if (!front) - { - return; - } - if (error && error->errorCode() != 0) - { - FRONT_LOG(WARNING) << LOG_DESC("asyncGetAgencies failed") - << LOG_KV("code", error->errorCode()) - << LOG_KV("msg", error->errorMessage()); - return; - } - std::vector agencyList(agencies.begin(), agencies.end()); - bcos::UpgradableGuard l(front->x_agencyList); - if (front->m_agencyList == agencyList) - { - return; - } - bcos::UpgradeGuard ul(l); - front->m_agencyList = agencyList; - FRONT_LOG(INFO) << LOG_DESC("Update agencies information") - << LOG_KV("agencies", printVector(agencyList)); - }); + m_front->asyncGetAgencies(m_front->nodeInfo()->copiedComponents(), + [self](bcos::Error::Ptr error, std::set agencies) { + auto front = self.lock(); + if (!front) + { + return; + } + if (error && error->errorCode() != 0) + { + FRONT_LOG(WARNING) + << LOG_DESC("asyncGetAgencies failed") << LOG_KV("code", error->errorCode()) + << LOG_KV("msg", error->errorMessage()); + return; + } + std::vector agencyList(agencies.begin(), agencies.end()); + bcos::UpgradableGuard l(front->x_agencyList); + if (front->m_agencyList == agencyList) + { + return; + } + bcos::UpgradeGuard ul(l); + front->m_agencyList = agencyList; + FRONT_LOG(INFO) << LOG_DESC("Update agencies information") + << LOG_KV("agencies", printVector(agencyList)); + }); m_fetcher->restart(); } @@ -192,4 +193,5 @@ void Front::registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType, << LOG_KV("error", boost::diagnostic_information(e)); } }); + m_front->registerComponent(std::to_string(type)); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp index ca774092..96a4b00b 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp @@ -275,4 +275,28 @@ bcos::Error::Ptr FrontImpl::push(uint16_t routeType, MessageOptionalHeader::Ptr routeType, routeInfo, std::move(payload), seq, timeout, [promise](bcos::Error::Ptr error) { promise->set_value(error); }, nullptr); return promise->get_future().get(); +} + +void FrontImpl::asyncGetPeers(GetPeersInfoHandler::Ptr getPeersCallback) +{ + m_gatewayClient->asyncGetPeers( + [getPeersCallback](bcos::Error::Ptr error, std::string peersInfo) { + getPeersCallback->onPeersInfo(error, peersInfo); + }); +} + +void FrontImpl::registerComponent(std::string const& component) +{ + // Note: the node will report the latest components + auto ret = m_nodeInfo->addComponent(component); + FRONT_LOG(INFO) << LOG_DESC("registerComponent") << LOG_KV("component", component) + << LOG_KV("insert", ret); +} + +void FrontImpl::unRegisterComponent(std::string const& component) +{ + // Note: the node will report the latest components + auto ret = m_nodeInfo->eraseComponent(component); + FRONT_LOG(INFO) << LOG_DESC("unRegisterComponent") << LOG_KV("component", component) + << LOG_KV("erase", ret); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h index 84000656..13fb00d9 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h +++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h @@ -141,12 +141,14 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ return m_gatewayClient->registerTopic(bcos::ref(m_nodeID), topic); } - void asyncGetAgencies( + void asyncGetAgencies(std::vector const& components, std::function)> callback) override { - m_gatewayClient->asyncGetAgencies(callback); + m_gatewayClient->asyncGetAgencies(components, callback); } + void asyncGetPeers(GetPeersInfoHandler::Ptr getPeersCallback) override; + /** * @brief unRegister the topic * @@ -170,6 +172,11 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_ void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID, bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) override; + ppc::protocol::INodeInfo::Ptr const& nodeInfo() override { return m_nodeInfo; } + + void registerComponent(std::string const& component) override; + void unRegisterComponent(std::string const& component) override; + private: void asyncSendMessageToGateway(bool responsePacket, ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType, diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp index fe2becda..d8bc8fea 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp @@ -322,13 +322,14 @@ void GatewayImpl::asyncGetPeers(std::function cal } } -void GatewayImpl::asyncGetAgencies(std::function)> callback) +void GatewayImpl::asyncGetAgencies(std::vector const& components, + std::function)> callback) { if (!callback) { return; } - auto agencies = m_peerRouter->agencies(); + auto agencies = m_peerRouter->agencies(components); agencies.insert(m_agency); callback(nullptr, agencies); } \ No newline at end of file diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h index 87e10fa5..fd83cb25 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h @@ -68,7 +68,7 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this callback) override; - void asyncGetAgencies( + void asyncGetAgencies(std::vector const& components, std::function)> callback) override; protected: diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h index cfa4362e..6014a6e2 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h @@ -59,6 +59,7 @@ class GatewayNodeInfo virtual void unRegisterTopic(bcos::bytes const& nodeID, std::string const& topic) = 0; virtual std::map nodeList() const = 0; + virtual bool existComponent(std::string const& component) const = 0; virtual uint16_t nodeSize() const = 0; virtual void toJson(Json::Value& jsonObject) const = 0; }; diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp index 0dda811e..1c52fa8a 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp @@ -59,6 +59,19 @@ INodeInfo::Ptr GatewayNodeInfoImpl::nodeInfo(bcos::bytes const& nodeID) const return nullptr; } +bool GatewayNodeInfoImpl::existComponent(std::string const& component) const +{ + bcos::ReadGuard l(x_nodeList); + for (auto const& it : m_nodeList) + { + if (it.second->components().count(component)) + { + return true; + } + } + return false; +} + void GatewayNodeInfoImpl::updateNodeList() { // Note: can't use clear_nodelist here, for clear_nodelist will destroy the allocated nodelist, @@ -68,11 +81,13 @@ void GatewayNodeInfoImpl::updateNodeList() for (auto const& it : m_nodeList) { auto nodeInfo = std::dynamic_pointer_cast(it.second); + nodeInfo->encodeFields(); m_rawGatewayInfo->mutable_nodelist()->UnsafeArenaAddAllocated( nodeInfo->rawNodeInfo().get()); } } +// Note: this is wrappered with lock bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info) { auto nodeID = info->nodeID().toBytes(); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h index 553af157..aeab2c1d 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.h @@ -85,6 +85,8 @@ class GatewayNodeInfoImpl : public GatewayNodeInfo void toJson(Json::Value& jsonObject) const override; + bool existComponent(std::string const& component) const override; + private: void updateNodeList(); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h index a3b33f2f..d1fc9040 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h @@ -53,7 +53,6 @@ class LocalRouter virtual std::vector chooseReceiver( ppc::protocol::Message::Ptr const& msg); - // TODO: register component virtual bool dispatcherMessage(ppc::protocol::Message::Ptr const& msg, ppc::protocol::ReceiveMsgFunc callback, bool holding = true); diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index 6063e355..9685e82c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -116,13 +116,30 @@ void PeerRouterTable::removeP2PID(std::string const& p2pNode) removeP2PNodeIDFromAgencyInfos(p2pNode); } -std::set PeerRouterTable::agencies() const +std::set PeerRouterTable::agencies(std::vector const& components) const { std::set agencies; bcos::ReadGuard l(x_mutex); for (auto const& it : m_agency2GatewayInfos) { - agencies.insert(it.first); + // get all agencies + if (components.empty()) + { + agencies.insert(it.first); + continue; + } + // get agencies according to component + for (auto const& gatewayInfo : it.second) + { + for (auto const& component : components) + { + if (gatewayInfo->existComponent(component)) + { + agencies.insert(it.first); + break; + } + } + } } return agencies; } diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h index d9819420..fdb7bc6c 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h @@ -43,7 +43,7 @@ class PeerRouterTable virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const; - std::set agencies() const; + std::set agencies(std::vector const& components) const; std::map gatewayInfos() const { diff --git a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt index e1d6cf3d..ab69fa97 100644 --- a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt @@ -4,7 +4,7 @@ find_package(SWIG REQUIRED) include(${SWIG_USE_FILE}) # Add subdirectories for each language if desired -option(BUILD_PYTHON "Build Python SWIG module" ON) +option(BUILD_PYTHON "Build Python SWIG module" OFF) if(BUILD_PYTHON) # fetch the python dependencies option(FETCH_PYTHON_DEPS "Install python required modules if not available" ON) diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler.java new file mode 100644 index 00000000..5a73e448 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler.java @@ -0,0 +1,67 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class GetPeersInfoHandler { + private transient long swigCPtr; + private transient boolean swigCMemOwn; + + protected GetPeersInfoHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(GetPeersInfoHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected void swigSetCMemOwn(boolean own) { + swigCMemOwn = own; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_GetPeersInfoHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + protected void swigDirectorDisconnect() { + swigSetCMemOwn(false); + delete(); + } + + public void swigReleaseOwnership() { + swigSetCMemOwn(false); + wedpr_java_transportJNI.GetPeersInfoHandler_change_ownership(this, swigCPtr, false); + } + + public void swigTakeOwnership() { + swigSetCMemOwn(true); + wedpr_java_transportJNI.GetPeersInfoHandler_change_ownership(this, swigCPtr, true); + } + + public GetPeersInfoHandler() { + this(wedpr_java_transportJNI.new_GetPeersInfoHandler(), true); + wedpr_java_transportJNI.GetPeersInfoHandler_director_connect(this, swigCPtr, true, true); + } + + public void onPeersInfo(Error e, String peersInfo) { + wedpr_java_transportJNI.GetPeersInfoHandler_onPeersInfo( + swigCPtr, this, Error.getCPtr(e), e, peersInfo); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java index bc5aac7c..1fcf80c1 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java @@ -166,6 +166,11 @@ public Message peek(String topic) { return (cPtr == 0) ? null : new Message(cPtr, true); } + public void asyncGetPeers(GetPeersInfoHandler getPeersCallback) { + wedpr_java_transportJNI.IFront_asyncGetPeers( + swigCPtr, this, GetPeersInfoHandler.getCPtr(getPeersCallback), getPeersCallback); + } + /** * register the nodeInfo to the gateway
* @@ -184,6 +189,11 @@ public Error unRegisterNodeInfo() { return (cPtr == 0) ? null : new Error(cPtr, true); } + public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr nodeInfo() { + return new SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr( + wedpr_java_transportJNI.IFront_nodeInfo(swigCPtr, this), false); + } + /** * register the topic
*
@@ -205,4 +215,12 @@ public Error unRegisterTopic(String topic) { long cPtr = wedpr_java_transportJNI.IFront_unRegisterTopic(swigCPtr, this, topic); return (cPtr == 0) ? null : new Error(cPtr, true); } + + public void registerComponent(String component) { + wedpr_java_transportJNI.IFront_registerComponent(swigCPtr, this, component); + } + + public void unRegisterComponent(String component) { + wedpr_java_transportJNI.IFront_unRegisterComponent(swigCPtr, this, component); + } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGetPeersInfoHandler.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGetPeersInfoHandler.java new file mode 100644 index 00000000..a97f2ab9 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedGetPeersInfoHandler.java @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +package com.webank.wedpr.sdk.jni.generated; + +public class SharedGetPeersInfoHandler { + private transient long swigCPtr; + protected transient boolean swigCMemOwn; + + protected SharedGetPeersInfoHandler(long cPtr, boolean cMemoryOwn) { + swigCMemOwn = cMemoryOwn; + swigCPtr = cPtr; + } + + protected static long getCPtr(SharedGetPeersInfoHandler obj) { + return (obj == null) ? 0 : obj.swigCPtr; + } + + protected static long swigRelease(SharedGetPeersInfoHandler obj) { + long ptr = 0; + if (obj != null) { + if (!obj.swigCMemOwn) + throw new RuntimeException("Cannot release ownership as memory is not owned"); + ptr = obj.swigCPtr; + obj.swigCMemOwn = false; + obj.delete(); + } + return ptr; + } + + @SuppressWarnings({"deprecation", "removal"}) + protected void finalize() { + delete(); + } + + public synchronized void delete() { + if (swigCPtr != 0) { + if (swigCMemOwn) { + swigCMemOwn = false; + wedpr_java_transportJNI.delete_SharedGetPeersInfoHandler(swigCPtr); + } + swigCPtr = 0; + } + } + + public SharedGetPeersInfoHandler() { + this(wedpr_java_transportJNI.new_SharedGetPeersInfoHandler(), true); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java index 8418644a..311ec4fe 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java @@ -41,6 +41,10 @@ public class wedpr_java_transportJNI { public static final native void delete_SharedIMessageHandler(long jarg1); + public static final native long new_SharedGetPeersInfoHandler(); + + public static final native void delete_SharedGetPeersInfoHandler(long jarg1); + public static final native long new_SharedGateway(); public static final native void delete_SharedGateway(long jarg1); @@ -647,6 +651,19 @@ public static final native void IMessageHandler_director_connect( public static final native void IMessageHandler_change_ownership( IMessageHandler obj, long cptr, boolean take_or_release); + public static final native long new_GetPeersInfoHandler(); + + public static final native void delete_GetPeersInfoHandler(long jarg1); + + public static final native void GetPeersInfoHandler_onPeersInfo( + long jarg1, GetPeersInfoHandler jarg1_, long jarg2, Error jarg2_, String jarg3); + + public static final native void GetPeersInfoHandler_director_connect( + GetPeersInfoHandler obj, long cptr, boolean mem_own, boolean weak_global); + + public static final native void GetPeersInfoHandler_change_ownership( + GetPeersInfoHandler obj, long cptr, boolean take_or_release); + public static final native void delete_IFront(long jarg1); public static final native void IFront_start(long jarg1, IFront jarg1_); @@ -712,14 +729,25 @@ public static final native long IFront_push__SWIG_1( public static final native long IFront_peek(long jarg1, IFront jarg1_, String jarg2); + public static final native void IFront_asyncGetPeers( + long jarg1, IFront jarg1_, long jarg2, GetPeersInfoHandler jarg2_); + public static final native long IFront_registerNodeInfo(long jarg1, IFront jarg1_, long jarg2); public static final native long IFront_unRegisterNodeInfo(long jarg1, IFront jarg1_); + public static final native long IFront_nodeInfo(long jarg1, IFront jarg1_); + public static final native long IFront_registerTopic(long jarg1, IFront jarg1_, String jarg2); public static final native long IFront_unRegisterTopic(long jarg1, IFront jarg1_, String jarg2); + public static final native void IFront_registerComponent( + long jarg1, IFront jarg1_, String jarg2); + + public static final native void IFront_unRegisterComponent( + long jarg1, IFront jarg1_, String jarg2); + public static final native void delete_IFrontBuilder(long jarg1); public static final native long IFrontBuilder_buildClient( @@ -783,6 +811,11 @@ public static void SwigDirector_IMessageHandler_onMessage( new SendResponseHandler(sendResponseHandler, true)); } + public static void SwigDirector_GetPeersInfoHandler_onPeersInfo( + GetPeersInfoHandler jself, long e, String peersInfo) { + jself.onPeersInfo((e == 0) ? null : new Error(e, true), peersInfo); + } + private static final native void swig_module_init(); static { diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java index 5a716f6b..5e0545e4 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java @@ -16,6 +16,7 @@ package com.webank.wedpr.sdk.jni.transport; import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; +import com.webank.wedpr.sdk.jni.transport.handlers.GetPeersCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; @@ -57,6 +58,22 @@ public interface WeDPRTransport { */ void unRegisterTopic(String topic) throws Exception; + /** + * register handlers according to component + * + * @param component the component of the message should handled by the given callback + * @param messageDispatcherCallback the message callback + */ + void registerComponentHandler( + String component, MessageDispatcherCallback messageDispatcherCallback); + + /** + * async get peers information + * + * @param handler the handler that handle the peersInfo + */ + void asyncGetPeers(GetPeersCallback handler); + //// the async interfaces /** * register the message handler diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java new file mode 100644 index 00000000..17e783d5 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.sdk.jni.transport.handlers; + +import com.webank.wedpr.sdk.jni.generated.Error; +import com.webank.wedpr.sdk.jni.generated.GetPeersInfoHandler; + +public abstract class GetPeersCallback extends GetPeersInfoHandler { + + public abstract void onPeersInfo(Error e, String peersInfo); + + // release the ownership to c++, in case of it's released by the jvm + @Override + protected void finalize() { + swigReleaseOwnership(); + delete(); + } +} diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java index 46420d94..d2d497b6 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -24,6 +24,7 @@ import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; import com.webank.wedpr.sdk.jni.transport.TransportConfig; import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; +import com.webank.wedpr.sdk.jni.transport.handlers.GetPeersCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; @@ -70,8 +71,8 @@ public void stop() { * @throws Exception failed case */ @Override - public void registerComponent(String component) throws Exception { - // Note: the front must exist after Transport created + public void registerComponent(String component) { + this.transport.getFront().registerComponent(component); } /** @@ -81,7 +82,9 @@ public void registerComponent(String component) throws Exception { * @throws Exception failed case */ @Override - public void unRegisterComponent(String component) throws Exception {} + public void unRegisterComponent(String component) { + this.transport.getFront().unRegisterComponent(component); + } /** * register the topic @@ -91,7 +94,8 @@ public void unRegisterComponent(String component) throws Exception {} */ @Override public void registerTopic(String topic) throws Exception { - this.transport.getFront().registerTopic(topic); + Error result = this.transport.getFront().registerTopic(topic); + Common.checkResult("registerTopic", result); } /** @@ -102,7 +106,30 @@ public void registerTopic(String topic) throws Exception { */ @Override public void unRegisterTopic(String topic) throws Exception { - this.transport.getFront().unRegisterTopic(topic); + Error result = this.transport.getFront().unRegisterTopic(topic); + Common.checkResult("unRegisterTopic", result); + } + + /** + * register handlers according to component + * + * @param component the component of the message should handled by the given callback + * @param messageDispatcherCallback the message callback + */ + @Override + public void registerComponentHandler( + String component, MessageDispatcherCallback messageDispatcherCallback) { + this.transport.getFront().register_msg_handler(component, messageDispatcherCallback); + } + + /** + * async get peers information + * + * @param handler the handler that handle the peersInfo + */ + @Override + public void asyncGetPeers(GetPeersCallback handler) { + this.transport.getFront().asyncGetPeers(handler); } @Override diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx index e103bfd8..c5bc735c 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx @@ -781,7 +781,7 @@ namespace Swig { namespace Swig { namespace { jclass jclass_wedpr_java_transportJNI = NULL; - jmethodID director_method_ids[3]; + jmethodID director_method_ids[4]; } } @@ -1202,6 +1202,66 @@ void SwigDirector_IMessageHandler::swig_connect_director(JNIEnv *jenv, jobject j } +SwigDirector_GetPeersInfoHandler::SwigDirector_GetPeersInfoHandler(JNIEnv *jenv) : ppc::front::GetPeersInfoHandler(), Swig::Director(jenv) { +} + +SwigDirector_GetPeersInfoHandler::~SwigDirector_GetPeersInfoHandler() { + swig_disconnect_director_self("swigDirectorDisconnect"); +} + + +void SwigDirector_GetPeersInfoHandler::onPeersInfo(bcos::Error::Ptr e,std::string const &peersInfo) { + JNIEnvWrapper swigjnienv(this) ; + JNIEnv * jenv = swigjnienv.getJNIEnv() ; + jobject swigjobj = (jobject) NULL ; + jlong je ; + jstring jpeersInfo = 0 ; + + if (!swig_override[0]) { + SWIG_JavaThrowException(JNIEnvWrapper(this).getJNIEnv(), SWIG_JavaDirectorPureVirtual, "Attempted to invoke pure virtual method ppc::front::GetPeersInfoHandler::onPeersInfo."); + return; + } + swigjobj = swig_get_self(jenv); + if (swigjobj && jenv->IsSameObject(swigjobj, NULL) == JNI_FALSE) { + je = 0; + if (e) { + *((std::shared_ptr< bcos::Error > **)&je) = new std::shared_ptr< bcos::Error >(e); + } + jpeersInfo = jenv->NewStringUTF((&peersInfo)->c_str()); + Swig::LocalRefGuard peersInfo_refguard(jenv, jpeersInfo); + jenv->CallStaticVoidMethod(Swig::jclass_wedpr_java_transportJNI, Swig::director_method_ids[3], swigjobj, je, jpeersInfo); + jthrowable swigerror = jenv->ExceptionOccurred(); + if (swigerror) { + Swig::DirectorException::raise(jenv, swigerror); + } + + } else { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null upcall object in ppc::front::GetPeersInfoHandler::onPeersInfo "); + } + if (swigjobj) jenv->DeleteLocalRef(swigjobj); +} + +void SwigDirector_GetPeersInfoHandler::swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global) { + static jclass baseclass = swig_new_global_ref(jenv, "com/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler"); + if (!baseclass) return; + static SwigDirectorMethod methods[] = { + SwigDirectorMethod(jenv, baseclass, "onPeersInfo", "(Lcom/webank/wedpr/sdk/jni/generated/Error;Ljava/lang/String;)V") + }; + + if (swig_set_self(jenv, jself, swig_mem_own, weak_global)) { + bool derived = (jenv->IsSameObject(baseclass, jcls) ? false : true); + for (int i = 0; i < 1; ++i) { + swig_override[i] = false; + if (derived) { + jmethodID methid = jenv->GetMethodID(jcls, methods[i].name, methods[i].desc); + swig_override[i] = methods[i].methid && (methid != methods[i].methid); + jenv->ExceptionClear(); + } + } + } +} + + #ifdef __cplusplus extern "C" { @@ -1399,6 +1459,30 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedGetPeersInfoHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *result = 0 ; + + (void)jenv; + (void)jcls; + result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >(); + *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::GetPeersInfoHandler >(*result) : 0; + if (1) delete result; + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedGetPeersInfoHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + std::shared_ptr< ppc::front::GetPeersInfoHandler > *arg1 = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *) 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > tempnull1 ; + + (void)jenv; + (void)jcls; + arg1 = jarg1 ? *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jarg1 : &tempnull1; + delete arg1; +} + + SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedGateway(JNIEnv *jenv, jclass jcls) { jlong jresult = 0 ; std::shared_ptr< ppc::gateway::IGateway > *result = 0 ; @@ -5912,6 +5996,84 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GetPeersInfoHandler(JNIEnv *jenv, jclass jcls) { + jlong jresult = 0 ; + ppc::front::GetPeersInfoHandler *result = 0 ; + + (void)jenv; + (void)jcls; + result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(jenv); + + *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jresult = result ? new std::shared_ptr< ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_1) : 0; + + return jresult; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1GetPeersInfoHandler(JNIEnv *jenv, jclass jcls, jlong jarg1) { + ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + + smartarg1 = *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jarg1; + arg1 = (ppc::front::GetPeersInfoHandler *)(smartarg1 ? smartarg1->get() : 0); + (void)arg1; delete smartarg1; +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GetPeersInfoHandler_1onPeersInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_, jstring jarg3) { + ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ; + bcos::Error::Ptr arg2 ; + std::string *arg3 = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ; + bcos::Error::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jarg1; + arg1 = (ppc::front::GetPeersInfoHandler *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(bcos::Error::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + if(!jarg3) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); + if (!arg3_pstr) return ; + std::string arg3_str(arg3_pstr); + arg3 = &arg3_str; + jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); + (arg1)->onPeersInfo(arg2,(std::string const &)*arg3); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GetPeersInfoHandler_1director_1connect(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jswig_mem_own, jboolean jweak_global) { + std::shared_ptr< ppc::front::GetPeersInfoHandler > *obj = *((std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&objarg); + (void)jcls; + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_GetPeersInfoHandler *director = static_cast(obj->operator->()); + director->swig_connect_director(jenv, jself, jenv->GetObjectClass(jself), (jswig_mem_own == JNI_TRUE), (jweak_global == JNI_TRUE)); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GetPeersInfoHandler_1change_1ownership(JNIEnv *jenv, jclass jcls, jobject jself, jlong objarg, jboolean jtake_or_release) { + std::shared_ptr< ppc::front::GetPeersInfoHandler > *obj = *((std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&objarg); + // Keep a local instance of the smart pointer around while we are using the raw pointer + // Avoids using smart pointer specific API. + SwigDirector_GetPeersInfoHandler *director = dynamic_cast(obj->operator->()); + (void)jcls; + if (director) { + director->swig_java_change_ownership(jenv, jself, jtake_or_release ? true : false); + } +} + + SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFront(JNIEnv *jenv, jclass jcls, jlong jarg1) { ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; @@ -6339,6 +6501,25 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr } +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1asyncGetPeers(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + ppc::front::GetPeersInfoHandler::Ptr arg2 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::front::GetPeersInfoHandler::Ptr *argp2 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + (void)jarg2_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + argp2 = *(ppc::front::GetPeersInfoHandler::Ptr **)&jarg2; + if (argp2) arg2 = *argp2; + (arg1)->asyncGetPeers(arg2); +} + + SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) { jlong jresult = 0 ; ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; @@ -6381,6 +6562,24 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr } +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1nodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) { + jlong jresult = 0 ; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + ppc::protocol::INodeInfo::Ptr *result = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo(); + *(ppc::protocol::INodeInfo::Ptr **)&jresult = result; + return jresult; +} + + SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerTopic(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { jlong jresult = 0 ; ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; @@ -6437,6 +6636,54 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr } +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerComponent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->registerComponent((std::string const &)*arg2); +} + + +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1unRegisterComponent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) { + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + + (void)jenv; + (void)jcls; + (void)jarg1_; + + smartarg1 = *(std::shared_ptr< ppc::front::IFront > **)&jarg1; + arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); + if(!jarg2) { + SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string"); + return ; + } + const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); + if (!arg2_pstr) return ; + std::string arg2_str(arg2_pstr); + arg2 = &arg2_str; + jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); + (arg1)->unRegisterComponent((std::string const &)*arg2); +} + + SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) { ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ; @@ -6790,7 +7037,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra static struct { const char *method; const char *signature; - } methods[3] = { + } methods[4] = { { "SwigDirector_ErrorCallback_onError", "(Lcom/webank/wedpr/sdk/jni/generated/ErrorCallback;J)V" }, @@ -6799,6 +7046,9 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra }, { "SwigDirector_IMessageHandler_onMessage", "(Lcom/webank/wedpr/sdk/jni/generated/IMessageHandler;JJJ)V" + }, + { + "SwigDirector_GetPeersInfoHandler_onPeersInfo", "(Lcom/webank/wedpr/sdk/jni/generated/GetPeersInfoHandler;JLjava/lang/String;)V" } }; Swig::jclass_wedpr_java_transportJNI = (jclass) jenv->NewGlobalRef(jcls); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h index 89b6596b..2a61714c 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.h @@ -54,5 +54,20 @@ class SwigDirector_IMessageHandler : public ppc::front::IMessageHandler, public Swig::BoolArray<1> swig_override; }; +class SwigDirector_GetPeersInfoHandler : public ppc::front::GetPeersInfoHandler, public Swig::Director { + +public: + void swig_connect_director(JNIEnv *jenv, jobject jself, jclass jcls, bool swig_mem_own, bool weak_global); + SwigDirector_GetPeersInfoHandler(JNIEnv *jenv); + virtual ~SwigDirector_GetPeersInfoHandler(); + virtual void onPeersInfo(bcos::Error::Ptr e,std::string const &peersInfo); +public: + bool swig_overrides(int n) { + return (n < 1 ? swig_override[n] : false); + } +protected: + Swig::BoolArray<1> swig_override; +}; + #endif diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index 2c656f17..212ffdd9 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -41,6 +41,7 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long); %shared_ptr(ppc::front::ErrorCallback); %shared_ptr(ppc::front::MessageDispatcherHandler); %shared_ptr(ppc::front::IMessageHandler); +%shared_ptr(ppc::front::GetPeersInfoHandler); %shared_ptr(ppc::gateway::IGateway); %shared_ptr(bcos::Error); @@ -133,6 +134,7 @@ namespace bcos{ %template(SharedErrorCallback) std::shared_ptr; %template(SharedMessageDispatcherHandler) std::shared_ptr; %template(SharedIMessageHandler) std::shared_ptr; +%template(SharedGetPeersInfoHandler) std::shared_ptr; %template(SharedGateway) std::shared_ptr; @@ -153,6 +155,7 @@ namespace bcos{ %feature("director") ppc::front::ErrorCallback; %feature("director") ppc::front::MessageDispatcherHandler; %feature("director") ppc::front::IMessageHandler; +%feature("director") ppc::front::GetPeersInfoHandler; // Note: the field data should equal to the fieldMap of class or the function %include various.i diff --git a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp index b2a50563..3f401309 100644 --- a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp +++ b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp @@ -77,7 +77,7 @@ void ProTransportImpl::keepAlive() { try { - m_gateway->registerNodeInfo(m_config->generateNodeInfo()); + m_gateway->registerNodeInfo(m_front->nodeInfo()); } catch (std::exception const& e) { From 84bbab6b7d130074192047618d6082c909fb447a Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 24 Sep 2024 07:46:15 +0800 Subject: [PATCH 045/120] fix the java-callback throw exception caused coredump (#42) --- .../ppc-front/ppc-front/CallbackManager.cpp | 13 +++++++++++-- .../wedpr/sdk/jni/demo/TransportDemo.java | 6 +++++- .../transport/handlers/GetPeersCallback.java | 17 ++++++++++++++++- .../jni/transport/handlers/MessageCallback.java | 10 +++++++++- .../handlers/MessageDispatcherCallback.java | 10 +++++++++- .../handlers/MessageErrorCallback.java | 13 +++++++++++++ 6 files changed, 63 insertions(+), 6 deletions(-) diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp index efddec1c..1da25b26 100644 --- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp +++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp @@ -116,8 +116,17 @@ void CallbackManager::handleCallback(bcos::Error::Ptr const& error, std::string { return; } - m_threadPool->enqueue( - [error, callback, message, resFunc] { callback->msgCallback(error, message, resFunc); }); + m_threadPool->enqueue([error, callback, message, resFunc] { + try + { + callback->msgCallback(error, message, resFunc); + } + catch (std::exception const& e) + { + FRONT_LOG(WARNING) << LOG_DESC("handleCallback exception") + << LOG_KV("error", boost::diagnostic_information(e)); + } + }); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java index 7c26c4e0..f9ef4550 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -15,6 +15,7 @@ package com.webank.wedpr.sdk.jni.demo; +import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; import com.webank.wedpr.sdk.jni.generated.*; import com.webank.wedpr.sdk.jni.generated.Error; import com.webank.wedpr.sdk.jni.transport.IMessage; @@ -24,6 +25,7 @@ import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl; import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint; +import lombok.SneakyThrows; public class TransportDemo { public static class MessageDispatcherCallbackImpl extends MessageDispatcherCallback { @@ -37,6 +39,7 @@ public MessageDispatcherCallbackImpl(String nodeID) { this.nodeID = nodeID; } + @SneakyThrows(Exception.class) @Override public void onMessage(IMessage message) { System.out.println( @@ -47,6 +50,7 @@ public void onMessage(IMessage message) { + ", payload: " + new String(message.getPayload()) + "#######"); + throw new WeDPRSDKException("Exception test"); } } @@ -58,7 +62,7 @@ public MessageErrorCallbackImpl(String nodeID) { } @Override - public void onError(Error error) { + public void onErrorResult(Error error) { System.out.println( "##### Node: " + nodeID diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java index 17e783d5..3f02e7f1 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/GetPeersCallback.java @@ -17,10 +17,25 @@ import com.webank.wedpr.sdk.jni.generated.Error; import com.webank.wedpr.sdk.jni.generated.GetPeersInfoHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class GetPeersCallback extends GetPeersInfoHandler { + private static final Logger logger = LoggerFactory.getLogger(GetPeersCallback.class); - public abstract void onPeersInfo(Error e, String peersInfo); + public abstract void onPeers(Error e, String peersInfo); + + public void onPeersInfo(Error e, String peersInfo) { + try { + onPeers(e, peersInfo); + } catch (Exception exception) { + logger.warn( + "onPeersInfo exception, result: {}, peersInfo: {}, e: ", + e.toString(), + peersInfo, + exception); + } + } // release the ownership to c++, in case of it's released by the jvm @Override diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java index 9b2b67d0..8e12f8e5 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageCallback.java @@ -21,8 +21,12 @@ import com.webank.wedpr.sdk.jni.generated.SendResponseHandler; import com.webank.wedpr.sdk.jni.transport.IMessage; import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class MessageCallback extends IMessageHandler { + private static final Logger logger = LoggerFactory.getLogger(MessageCallback.class); + public abstract void onMessage( Error error, IMessage message, SendResponseHandler sendResponseHandler); @@ -34,6 +38,10 @@ protected void finalize() { @Override public void onMessage(Error e, Message msg, SendResponseHandler sendResponseHandler) { - onMessage(e, IMessageBuilder.build(msg), sendResponseHandler); + try { + onMessage(e, IMessageBuilder.build(msg), sendResponseHandler); + } catch (Exception exception) { + logger.warn("onMessage exception, msg: {}, e:", msg.toString(), exception); + } } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java index eda2ef65..7eb0eec5 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageDispatcherCallback.java @@ -19,8 +19,12 @@ import com.webank.wedpr.sdk.jni.generated.MessageDispatcherHandler; import com.webank.wedpr.sdk.jni.transport.IMessage; import com.webank.wedpr.sdk.jni.transport.IMessageBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class MessageDispatcherCallback extends MessageDispatcherHandler { + private static final Logger logger = LoggerFactory.getLogger(MessageDispatcherCallback.class); + public abstract void onMessage(IMessage message); // TODO: check this will cause memory leak or not @@ -32,6 +36,10 @@ protected void finalize() { @Override public void onMessage(Message msg) { - onMessage(IMessageBuilder.build(msg)); + try { + onMessage(IMessageBuilder.build(msg)); + } catch (Exception e) { + logger.warn("onMessage exception, msg: {}, e:", msg.toString(), e); + } } } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java index 0d85b06a..0eb22a13 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/handlers/MessageErrorCallback.java @@ -15,10 +15,23 @@ package com.webank.wedpr.sdk.jni.transport.handlers; +import com.webank.wedpr.sdk.jni.generated.Error; import com.webank.wedpr.sdk.jni.generated.ErrorCallback; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class MessageErrorCallback extends ErrorCallback { + private static final Logger logger = LoggerFactory.getLogger(MessageErrorCallback.class); + public abstract void onErrorResult(Error error); + + public void onError(Error error) { + try { + onErrorResult(error); + } catch (Exception e) { + logger.warn("onError exception, error: {}, e:", error.toString(), e); + } + } // release the ownership to c++, in case of it's released by the jvm @Override protected void finalize() { From 7556ad5febe6bb986fc0f06ea82a4e889e919d8f Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 24 Sep 2024 23:19:09 +0800 Subject: [PATCH 046/120] fix buildMessage from null Message caused coredump (#43) * fix buildMessage from null Message caused coredump * fix the buffer released after convert to java byte-array --- .../wedpr/sdk/jni/demo/TransportDemo.java | 43 ++++++++++++-- .../sdk/jni/transport/IMessageBuilder.java | 3 + .../src/wedpr_java_transportJAVA_wrap.cxx | 58 +++++++++++++++++-- .../java/swig/wedpr_java_transport.i | 4 +- 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java index f9ef4550..6c1c2c02 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -15,12 +15,12 @@ package com.webank.wedpr.sdk.jni.demo; -import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; import com.webank.wedpr.sdk.jni.generated.*; import com.webank.wedpr.sdk.jni.generated.Error; import com.webank.wedpr.sdk.jni.transport.IMessage; import com.webank.wedpr.sdk.jni.transport.TransportConfig; import com.webank.wedpr.sdk.jni.transport.WeDPRTransport; +import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl; @@ -30,13 +30,15 @@ public class TransportDemo { public static class MessageDispatcherCallbackImpl extends MessageDispatcherCallback { private final String nodeID; + private final WeDPRTransport weDPRTransport; // java -cp 'conf/:lib/*:apps/*' com.webank.wedpr.sdk.jni.demo.TransportDemo agency0Node // "127.0.0.1" 9020 "ipv4:127.0.0.1:40600,127.0.0.1:40601" "agency1Node" // java -cp 'conf/:lib/*:apps/*' com.webank.wedpr.sdk.jni.demo.TransportDemo agency1Node // "127.0.0.1" 9021 "ipv4:127.0.0.1:40620,127.0.0.1:40621" "agency0Node" - public MessageDispatcherCallbackImpl(String nodeID) { + public MessageDispatcherCallbackImpl(String nodeID, WeDPRTransport transport) { this.nodeID = nodeID; + this.weDPRTransport = transport; } @SneakyThrows(Exception.class) @@ -50,7 +52,38 @@ public void onMessage(IMessage message) { + ", payload: " + new String(message.getPayload()) + "#######"); - throw new WeDPRSDKException("Exception test"); + + String response = "#### hello, response!"; + System.out.println( + "#### sendResponse: " + + response + + ", traceID: " + + message.getHeader().getTraceID() + + ",srcNode: " + + new String(message.getHeader().getSrcNode())); + this.weDPRTransport.asyncSendResponse( + message.getHeader().getSrcNode(), + message.getHeader().getTraceID(), + response.getBytes(), + 0, + new MessageErrorCallbackImpl(this.nodeID)); + } + } + + public static class MessageCallbackImpl extends MessageCallback { + + @Override + public void onMessage( + Error error, IMessage message, SendResponseHandler sendResponseHandler) { + if (error != null && error.errorCode() != 0) { + System.out.println("#### MessageCallbackImpl error: " + error.errorMessage()); + return; + } + System.out.println( + "#### receive response: " + + message + + ", payload: " + + new String(message.getPayload())); } } @@ -111,7 +144,7 @@ public static void main(String[] args) throws Exception { // send Message to the gateway String topic = "testTopic"; MessageDispatcherCallback messageDispatcherCallback = - new MessageDispatcherCallbackImpl(nodeID); + new MessageDispatcherCallbackImpl(nodeID, transport); transport.registerTopicHandler(topic, messageDispatcherCallback); System.out.println("##### register topic success"); @@ -130,7 +163,7 @@ public static void main(String[] args) throws Exception { 0, 10000, new MessageErrorCallbackImpl(nodeID), - null); + new MessageCallbackImpl()); // push String syncPayload = "sync_" + payLoad; diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java index 6b61f213..d63c99e9 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java @@ -22,6 +22,9 @@ public class IMessageBuilder { public static IMessage build(Message msg) { + if (msg == null) { + return null; + } return new MessageImpl(msg); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx index c5bc735c..5624a7c7 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx @@ -222,6 +222,22 @@ static void SWIGUNUSED SWIG_JavaThrowException(JNIEnv *jenv, SWIG_JavaExceptionC #define SWIG_contract_assert(nullreturn, expr, msg) do { if (!(expr)) {SWIG_JavaThrowException(jenv, SWIG_JavaIllegalArgumentException, msg); return nullreturn; } } while (0) +/* SWIG Errors applicable to all language modules, values are reserved from -1 to -99 */ +#define SWIG_UnknownError -1 +#define SWIG_IOError -2 +#define SWIG_RuntimeError -3 +#define SWIG_IndexError -4 +#define SWIG_TypeError -5 +#define SWIG_DivisionByZero -6 +#define SWIG_OverflowError -7 +#define SWIG_SyntaxError -8 +#define SWIG_ValueError -9 +#define SWIG_SystemError -10 +#define SWIG_AttributeError -11 +#define SWIG_MemoryError -12 +#define SWIG_NullReferenceError -13 + + /* ----------------------------------------------------------------------------- * director_common.swg * @@ -836,6 +852,44 @@ template T SwigValueInit() { #include // Use the C99 official header +SWIGINTERN void SWIG_JavaException(JNIEnv *jenv, int code, const char *msg) { + SWIG_JavaExceptionCodes exception_code = SWIG_JavaUnknownError; + switch(code) { + case SWIG_MemoryError: + exception_code = SWIG_JavaOutOfMemoryError; + break; + case SWIG_IOError: + exception_code = SWIG_JavaIOException; + break; + case SWIG_SystemError: + case SWIG_RuntimeError: + exception_code = SWIG_JavaRuntimeException; + break; + case SWIG_OverflowError: + case SWIG_IndexError: + exception_code = SWIG_JavaIndexOutOfBoundsException; + break; + case SWIG_DivisionByZero: + exception_code = SWIG_JavaArithmeticException; + break; + case SWIG_SyntaxError: + case SWIG_ValueError: + case SWIG_TypeError: + exception_code = SWIG_JavaIllegalArgumentException; + break; + case SWIG_UnknownError: + default: + exception_code = SWIG_JavaUnknownError; + break; + } + SWIG_JavaThrowException(jenv, exception_code, msg); +} + + +#include +#include + + #include #include @@ -3766,7 +3820,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav { jresult = jenv->NewByteArray((&result)->len); jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); - delete[] (&result)->data; } return jresult; } @@ -3877,7 +3930,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav { jresult = jenv->NewByteArray((&result)->len); jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); - delete[] (&result)->data; } return jresult; } @@ -4882,7 +4934,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav { jresult = jenv->NewByteArray((&result)->len); jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); - delete[] (&result)->data; } return jresult; } @@ -5444,7 +5495,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav { jresult = jenv->NewByteArray((&result)->len); jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data); - delete[] (&result)->data; } return jresult; } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i index 212ffdd9..fd0a6d1c 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i @@ -2,6 +2,7 @@ %include "stdint.i" %include "typemaps.i" +%include #if defined(SWIGJAVA) #if defined(SWIGWORDSIZE64) @@ -173,11 +174,10 @@ namespace bcos{ %typemap(argout) OutputBuffer { JCALL3(ReleaseByteArrayElements, jenv, $input, (jbyte *) $1.data, 0); } -// Note: will cause copy herer +// Note: will cause copy here %typemap(out) OutputBuffer { $result = JCALL1(NewByteArray, jenv, $1.len); JCALL4(SetByteArrayRegion, jenv, $result, 0, $1.len, (jbyte *) $1.data); - delete[] $1.data; } %typemap(javain) OutputBuffer "$javainput" %typemap(javaout) OutputBuffer { return $jnicall; } From b25d9437cc43fa7ce4deb7068610b36679e2e6f5 Mon Sep 17 00:00:00 2001 From: octopus <912554887@qq.com> Date: Tue, 3 Sep 2024 17:02:50 +0800 Subject: [PATCH 047/120] remove dataset --- python/ppc_common/ppc_dataset/__init__.py | 0 .../ppc_common/ppc_dataset/dataset_helper.py | 60 --------- .../ppc_dataset/dataset_helper_factory.py | 61 --------- python/ppc_common/ppc_initialize/__init__.py | 0 .../dataset_handler_initialize.py | 30 ----- .../tests/dataset_initializer_test.py | 126 ------------------ python/ppc_model_gateway/config.py | 2 - python/ppc_scheduler/common/initializer.py | 13 +- .../workflow/worker/engine/mpc_engine.py | 35 ++--- .../workflow/worker/engine/psi_engine.py | 16 +-- 10 files changed, 32 insertions(+), 311 deletions(-) delete mode 100644 python/ppc_common/ppc_dataset/__init__.py delete mode 100644 python/ppc_common/ppc_dataset/dataset_helper.py delete mode 100644 python/ppc_common/ppc_dataset/dataset_helper_factory.py delete mode 100644 python/ppc_common/ppc_initialize/__init__.py delete mode 100644 python/ppc_common/ppc_initialize/dataset_handler_initialize.py delete mode 100644 python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py diff --git a/python/ppc_common/ppc_dataset/__init__.py b/python/ppc_common/ppc_dataset/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/ppc_common/ppc_dataset/dataset_helper.py b/python/ppc_common/ppc_dataset/dataset_helper.py deleted file mode 100644 index 718134a0..00000000 --- a/python/ppc_common/ppc_dataset/dataset_helper.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -from ppc_common.deps_services.sharding_file_object import ShardingFileObject -from ppc_common.deps_services.file_object import SplitMode -from ppc_common.deps_services.storage_api import StorageType -import os - - -class DataSetHelper: - # default split granularity is 1G - def __init__(self, dataset_user, dataset_id, dataset_local_path, - sql_storage, remote_storage, chunk_config, logger): - self._split_mode = SplitMode.NONE - self._chunk_config = chunk_config - self._chunk_size = self._chunk_config.file_object_chunk_size - self._dataset_user = dataset_user - remote_path = os.path.join(dataset_user, dataset_id) - self._file_object = ShardingFileObject( - dataset_local_path, remote_path, remote_storage, sql_storage, logger) - - @property - def file_object(self): - return self._file_object - - def download_dataset(self, enforce_flush=False): - """download the dataset - """ - self._file_object.download(enforce_flush=enforce_flush) - - def upload_dataset(self): - """upload the dataset - """ - self._file_object.upload(self._split_mode, self._chunk_size) - - def save_data(self, data): - """save the dataset - """ - self._file_object.save_data( - data=data, split_mode=self._split_mode, granularity=self._chunk_size) - - def update_data(self, data): - return self._file_object.update_data( - updated_data=data, split_mode=self._split_mode, granularity=self._chunk_size) - - def file_rename(self, new_storage_path, with_user_home=False): - if with_user_home: - self._file_object.rename(os.path.join( - self._dataset_user, new_storage_path)) - else: - self._file_object.rename(new_storage_path) - - def get_local_path(self): - """get the local path - """ - return self._file_object.get_local_path() - - def get_remote_path(self): - """get the remote path - """ - return self._file_object.get_remote_path() diff --git a/python/ppc_common/ppc_dataset/dataset_helper_factory.py b/python/ppc_common/ppc_dataset/dataset_helper_factory.py deleted file mode 100644 index 4fc0a2b0..00000000 --- a/python/ppc_common/ppc_dataset/dataset_helper_factory.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -from ppc_common.ppc_dataset.dataset_helper import DataSetHelper - - -class DataSetHelperFactory: - def __init__(self, sql_storage, remote_storage, chunk_config, logger): - self._sql_storage = sql_storage - self._remote_storage = remote_storage - self._chunk_config = chunk_config - self._logger = logger - - def create(self, dataset_user, dataset_id, dataset_local_path): - return DataSetHelper(dataset_user=dataset_user, dataset_id=dataset_id, - dataset_local_path=dataset_local_path, sql_storage=self._sql_storage, - chunk_config=self._chunk_config, logger=self._logger, - remote_storage=self._remote_storage) - - -def upload_dataset(dataset_helper_factory, dataset_user, dataset_id, dataset_local_path, logger, log_keyword): - dataset_helper = dataset_helper_factory.create( - dataset_user, dataset_id, dataset_local_path) - logger.info( - f"{log_keyword}: Upload dataset: {dataset_helper.get_local_path()} => {dataset_helper.get_remote_path()}") - dataset_helper.upload_dataset() - logger.info( - f"{log_keyword}: Upload dataset success: {dataset_helper.get_local_path()} => {dataset_helper.get_remote_path()}") - return dataset_helper - - -def delete_dataset(dataset_helper_factory, dataset_user, dataset_id, logger, log_keyword): - dataset_helper = dataset_helper_factory.create( - dataset_user=dataset_user, dataset_id=dataset_id, dataset_local_path=None) - logger.info( - f"{log_keyword}: Delete dataset: {dataset_helper.get_remote_path()}") - dataset_helper.file_object.delete() - logger.info( - f"{log_keyword}: Delete dataset success: {dataset_helper.get_remote_path()}") - return dataset_helper - - -def download_dataset(dataset_helper_factory, dataset_user, dataset_id, dataset_local_path, logger, log_keyword, enforce_flush=False): - dataset_helper = dataset_helper_factory.create( - dataset_user, dataset_id, dataset_local_path) - logger.info( - f"{log_keyword}: Download dataset: {dataset_helper.get_remote_path()}=>{dataset_helper.get_local_path()}") - dataset_helper.download_dataset(enforce_flush) - logger.info( - f"{log_keyword} dataset success: {dataset_helper.get_remote_path()}=>{dataset_helper.get_local_path()}") - return dataset_helper - - -def get_dataset(dataset_helper_factory, dataset_user, dataset_id, logger, log_keyword): - dataset_helper = dataset_helper_factory.create( - dataset_user, dataset_id, None) - logger.info( - f"{log_keyword}: Get dataset from: {dataset_helper.get_remote_path()}") - data = dataset_helper.file_object.get_data() - logger.info( - f"{log_keyword}: Get dataset success from: {dataset_helper.get_remote_path()}") - return data diff --git a/python/ppc_common/ppc_initialize/__init__.py b/python/ppc_common/ppc_initialize/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/ppc_common/ppc_initialize/dataset_handler_initialize.py b/python/ppc_common/ppc_initialize/dataset_handler_initialize.py deleted file mode 100644 index 3abe855e..00000000 --- a/python/ppc_common/ppc_initialize/dataset_handler_initialize.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader -from ppc_common.ppc_config.file_chunk_config import FileChunkConfig -from ppc_common.ppc_dataset.dataset_helper_factory import DataSetHelperFactory -from ppc_common.deps_services.mysql_storage import MySQLStorage -from ppc_common.deps_services import storage_loader - - -class DataSetHandlerInitialize: - def __init__(self, config, logger): - self._config = config - self._logger = logger - # self._init_sql_storage() - self._init_remote_storage() - self._init_dataset_factory() - - def _init_sql_storage(self): - self.sql_storage = MySQLStorage( - SQLStorageConfigLoader.load(self._config)) - - def _init_remote_storage(self): - self.storage_client = storage_loader.load(self._config, self._logger) - - def _init_dataset_factory(self): - self.file_chunk_config = FileChunkConfig(self._config) - self.dataset_helper_factory = DataSetHelperFactory( - sql_storage=self.sql_storage, - remote_storage=self.storage_client, - chunk_config=self.file_chunk_config, - logger=self._logger) diff --git a/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py b/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py deleted file mode 100644 index af3018e2..00000000 --- a/python/ppc_common/ppc_initialize/tests/dataset_initializer_test.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -import logging -from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader -from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize -from ppc_common.ppc_config.file_chunk_config import FileChunkConfig -from ppc_common.ppc_dataset import dataset_helper_factory -import sys -from ppc_common.ppc_utils import utils - - -class DataSetInitializerWrapper: - def __init__(self, ut_obj, file_chunk_size): - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - self.logger = logging.getLogger(__name__) - self.config = {} - self.config[SQLStorageConfigLoader.DATABASE_URL_KEY] = "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8mb4" - self.config[FileChunkConfig.ENABLE_ALL_CHUNCK_FILE_MGR_KEY] = True - self.config["STORAGE_TYPE"] = "HDFS" - self.config["HDFS_USER"] = "chenyujie" - self.config["HDFS_ENDPOINT"] = "http://127.0.0.1:9870" - self.config[FileChunkConfig.FILE_CHUNK_SIZE_MB_KEY] = file_chunk_size - self.dataset_handler_initializer = DataSetHandlerInitialize( - self.config, self.logger) - self.ut_obj = ut_obj - - def _download_and_check(self, dataset_user, dataset_id, dataset_local_path, expected_md5=None): - # download - download_local_path = dataset_local_path + ".download" - dataset_helper_factory.download_dataset( - dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, - dataset_id=dataset_id, dataset_user=dataset_user, logger=self.logger, - dataset_local_path=download_local_path, - log_keyword="testDownload", enforce_flush=True - ) - # check md5 - downloaded_file_md5 = utils.calculate_md5(download_local_path) - self.ut_obj.assertEquals(expected_md5, downloaded_file_md5) - # get the dataset - data = dataset_helper_factory.get_dataset( - dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, - dataset_id=dataset_id, dataset_user=dataset_user, logger=self.logger, - log_keyword="testGetDataSet" - ) - get_data_md5sum = utils.md5sum(data) - self.ut_obj.assertEquals(expected_md5, get_data_md5sum) - - def test_dataset_ops(self, dataset_user, dataset_id, dataset_local_path, updated_data): - # upload the dataset - dataset_helper_factory.upload_dataset( - dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, - dataset_id=dataset_id, dataset_user=dataset_user, - dataset_local_path=dataset_local_path, - logger=self.logger, log_keyword="testUpload") - origin_md5 = utils.calculate_md5(dataset_local_path) - self._download_and_check(dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=dataset_local_path, - expected_md5=origin_md5) - # update dataset - dataset_helper = self.dataset_handler_initializer.dataset_helper_factory.create( - dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=None) - dataset_helper.update_data(updated_data) - # check - self._download_and_check(dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=dataset_local_path, - expected_md5=utils.md5sum(updated_data)) - # rename - rename_path = dataset_id + ".rename" - dataset_helper.file_rename(rename_path, True) - # check - self._download_and_check(dataset_user=dataset_user, - dataset_id=rename_path, - dataset_local_path=dataset_local_path + ".rename", - expected_md5=utils.md5sum(updated_data)) - # delete - dataset_helper = dataset_helper_factory.delete_dataset( - dataset_helper_factory=self.dataset_handler_initializer.dataset_helper_factory, - dataset_id=rename_path, dataset_user=dataset_user, - logger=self.logger, log_keyword="testDelete") - self.ut_obj.assertEquals(dataset_helper.file_object.existed(), False) - - -class DatasetInitializerTest(unittest.TestCase): - def testNoFileSplit(self): - wrapper = DataSetInitializerWrapper(self, 100) - dataset_user = "yujiechen" - dataset_id = "test_file_no_split" - dataset_local_path = "tools/test" - uploaded_data = "abcdsfsdfsdfssdfsd" - wrapper.test_dataset_ops(dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=dataset_local_path, - updated_data=uploaded_data) - - def testFileSplit(self): - wrapper = DataSetInitializerWrapper(self, 1) - dataset_user = "yujiechen" - dataset_id = "test_file_split_file" - dataset_local_path = "tools/test" - uploaded_data = "abcdsfsdfsdfssdfsd" - wrapper.test_dataset_ops(dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=dataset_local_path, - updated_data=uploaded_data) - - def testFileSplitAndUpdateSplit(self): - wrapper = DataSetInitializerWrapper(self, 1) - dataset_user = "yujiechen" - dataset_id = "test_large_file_split_file" - dataset_local_path = "tools/test" - updated_file = "tools/test2" - uploaded_data = None - with open(updated_file, "rb") as fp: - uploaded_data = fp.read() - wrapper.test_dataset_ops(dataset_user=dataset_user, - dataset_id=dataset_id, - dataset_local_path=dataset_local_path, - updated_data=uploaded_data) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/ppc_model_gateway/config.py b/python/ppc_model_gateway/config.py index 1b9edb23..0e36903d 100644 --- a/python/ppc_model_gateway/config.py +++ b/python/ppc_model_gateway/config.py @@ -5,8 +5,6 @@ import yaml -from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize - path = os.getcwd() log_dir = os.sep.join([path, 'logs']) chain_log_dir = os.sep.join([path, 'bin', 'logs']) diff --git a/python/ppc_scheduler/common/initializer.py b/python/ppc_scheduler/common/initializer.py index 6ea92305..291dccc3 100644 --- a/python/ppc_scheduler/common/initializer.py +++ b/python/ppc_scheduler/common/initializer.py @@ -4,12 +4,12 @@ from contextlib import contextmanager import yaml +from python.ppc_common.ppc_config.file_chunk_config import FileChunkConfig from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from ppc_common.deps_services import storage_loader from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager -from ppc_common.ppc_initialize.dataset_handler_initialize import DataSetHandlerInitialize from ppc_common.ppc_utils import common_func from ppc_scheduler.job.job_manager import JobManager @@ -25,9 +25,9 @@ def __init__(self, log_config_path, config_path): self.sql_session = None self.sql_engine = None self.storage_client = None + self.file_chunk_config = None # 只用于测试 self.mock_logger = None - self.dataset_handler_initializer = None def init_all(self): self.init_log() @@ -36,7 +36,7 @@ def init_all(self): self.init_job_manager() self.init_sql_client() self.init_storage_client() - self.init_others() + self.init_file_chunk() def init_log(self): logging.config.fileConfig(self.log_config_path) @@ -79,10 +79,9 @@ def create_sql_session(self): def init_storage_client(self): self.storage_client = storage_loader.load( self.config_data, self.logger()) - - def init_others(self): - self.dataset_handler_initializer = DataSetHandlerInitialize( - self.config_data, self.logger()) + + def init_file_chunk(self): + self.file_chunk_config = FileChunkConfig(self.config_data) def logger(self, name=None): if self.mock_logger is None: diff --git a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py index b159c7b2..5011e970 100644 --- a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py @@ -3,7 +3,6 @@ import pandas as pd -from ppc_common.ppc_dataset import dataset_helper_factory from ppc_common.ppc_utils import utils from ppc_scheduler.mpc_generator.generator import CodeGenerator from ppc_scheduler.workflow.common.job_context import JobContext @@ -121,7 +120,7 @@ def _get_dataset_column_count(self): def _make_dataset_to_mpc_data_plus_psi_data(self, my_dataset_number): chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, - chunksize=self.components.dataset_handler_initializer.file_chunk_config.read_chunk_size) + chunksize=self.components.file_chunk_config.read_chunk_size) psi_data = pd.read_csv(self.job_context.psi_result_path, delimiter=utils.CSV_SEP) for chunk in chunk_list: self._make_dataset_field_normalized(chunk) @@ -161,13 +160,14 @@ def _prepare_mpc_after_psi(self): self.log.info(f"start prepare_mpc_after_psi, job_id={job_id}") my_dataset_number = self._get_dataset_column_count() - dataset_helper_factory.download_dataset( - dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, - dataset_user=self.job_context.user_name, - dataset_id=self.job_context.dataset_id, - dataset_local_path=self.job_context.dataset_file_path, - log_keyword="prepare_mpc_after_psi", - logger=self.log) + # TODO: 下载数据 + # dataset_helper_factory.download_dataset( + # dataset_helper_factory=None, + # dataset_user=self.job_context.user_name, + # dataset_id=self.job_context.dataset_id, + # dataset_local_path=self.job_context.dataset_file_path, + # log_keyword="prepare_mpc_after_psi", + # logger=self.log) if not utils.file_exists(self.job_context.psi_result_path): self.log.info( @@ -186,7 +186,7 @@ def _prepare_mpc_after_psi(self): def _make_dataset_to_mpc_data_direct(self, my_dataset_number): chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, - chunksize=self.components.dataset_handler_initializer.file_chunk_config.read_chunk_size) + chunksize=self.components.file_chunk_config.read_chunk_size) for chunk in chunk_list: self._make_dataset_field_normalized(chunk) self._save_selected_column_data(my_dataset_number, chunk, self.job_context.mpc_prepare_path) @@ -195,13 +195,14 @@ def _prepare_mpc_without_psi(self): job_id = self.job_context.job_id self.log.info(f"start prepare_mpc_without_psi, job_id={job_id}") my_dataset_number = self._get_dataset_column_count() - dataset_helper_factory.download_dataset( - dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, - dataset_user=self.job_context.user_name, - dataset_id=self.job_context.dataset_id, - dataset_local_path=self.job_context.dataset_file_path, - log_keyword="prepare_mpc_without_psi", - logger=self.log) + # TODO: 下载数据 + # dataset_helper_factory.download_dataset( + # dataset_helper_factory=None, + # dataset_user=self.job_context.user_name, + # dataset_id=self.job_context.dataset_id, + # dataset_local_path=self.job_context.dataset_file_path, + # log_keyword="prepare_mpc_without_psi", + # logger=self.log) self._make_dataset_to_mpc_data_direct(my_dataset_number) diff --git a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py index acc5ed29..cefd49dc 100644 --- a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py @@ -3,7 +3,6 @@ import os import time -from ppc_common.ppc_dataset import dataset_helper_factory from ppc_common.ppc_utils import utils, common_func from ppc_scheduler.workflow.common.job_context import JobContext @@ -106,13 +105,14 @@ def _run_multi_party_psi(self): self.log.info(f"call psi service successfully, job_id={job_id}, result: {psi_result}") def origin_dataset_to_psi_inputs(self): - dataset_helper_factory.download_dataset( - dataset_helper_factory=self.components.dataset_handler_initializer.dataset_helper_factory, - dataset_user=self.job_context.user_name, - dataset_id=self.job_context.dataset_id, - dataset_local_path=self.job_context.dataset_file_path, - log_keyword="prepare_dataset", - logger=self.log) + # TODO: 下载数据 + # dataset_helper_factory.download_dataset( + # dataset_helper_factory=None, + # dataset_user=self.job_context.user_name, + # dataset_id=self.job_context.dataset_id, + # dataset_local_path=self.job_context.dataset_file_path, + # log_keyword="prepare_dataset", + # logger=self.log) field = (self.job_context.psi_fields.split(utils.CSV_SEP)[self.job_context.my_index]).lower() if field == '': From 7591f69fddd911a1f2ab78ab186d7e788d31ef71 Mon Sep 17 00:00:00 2001 From: octopus <912554887@qq.com> Date: Wed, 4 Sep 2024 14:25:43 +0800 Subject: [PATCH 048/120] update wedpr scheduler --- .gitignore | 2 + python/ppc_common/db_models/computing_node.py | 3 +- python/ppc_common/db_models/config.sql | 42 ++- .../ppc_common/db_models/job_worker_record.py | 10 +- .../deps_services/storage_loader.py | 2 +- .../async_thread_executor.py | 2 +- python/ppc_common/ppc_utils/exception.py | 2 + python/ppc_model/conf/application-sample.yml | 4 +- python/ppc_scheduler/common/global_context.py | 7 +- .../common/global_job_manager.py | 15 + python/ppc_scheduler/common/initializer.py | 64 ++-- python/ppc_scheduler/common/log_utils.py | 15 +- .../ppc_scheduler/conf/application-sample.yml | 15 +- python/ppc_scheduler/conf/logging.conf | 4 +- .../database/computing_node_mapper.py | 67 +++-- .../database/job_worker_mapper.py | 26 +- .../default_flow_config_sample.py} | 55 ++-- .../demo/prefect_workflow_sample.py | 79 +++++ python/ppc_scheduler/demo/web_submit_job.json | 134 +++++++++ .../demo/workflow_args_sample.json | 73 +++++ .../ppc_scheduler/endpoints/job_controller.py | 38 ++- python/ppc_scheduler/job/job_manager.py | 59 ++-- python/ppc_scheduler/job/job_status.py | 3 +- python/ppc_scheduler/job/job_type.py | 4 + .../model_node_client.py | 22 +- .../{psi_node_client.py => psi_client.py} | 38 ++- python/ppc_scheduler/ppc_scheduler_app.py | 123 ++++++-- python/ppc_scheduler/scripts/start.sh | 69 +++++ python/ppc_scheduler/scripts/stop.sh | 42 +++ .../workflow/builder/__init__.py | 0 .../workflow/builder/flow_builder.py | 86 ++++++ python/ppc_scheduler/workflow/common/codec.py | 58 +--- .../workflow/common/flow_utils.py | 4 +- .../workflow/common/job_context.py | 129 +++----- .../workflow/common/worker_type.py | 18 +- python/ppc_scheduler/workflow/constructor.py | 82 ----- .../workflow/scheduler/__init__.py | 0 .../workflow/{ => scheduler}/scheduler.py | 53 ++-- .../workflow/scheduler/scheduler_api.py | 8 + .../workflow/worker/api_worker.py | 4 +- .../workflow/worker/default_worker.py | 37 +++ .../workflow/worker/engine/model_engine.py | 128 +++----- .../workflow/worker/engine/mpc_engine.py | 282 +----------------- .../workflow/worker/engine/psi_engine.py | 155 +--------- .../workflow/worker/engine/shell_engine.py | 16 + .../workflow/worker/engine/work_engine.py | 6 + .../workflow/worker/exit_worker.py | 33 -- .../workflow/worker/model_worker.py | 10 +- .../workflow/worker/mpc_worker.py | 6 +- .../workflow/worker/psi_worker.py | 18 +- .../workflow/worker/python_worker.py | 4 +- .../workflow/worker/shell_worker.py | 14 +- .../ppc_scheduler/workflow/worker/worker.py | 53 +++- .../workflow/worker/worker_factory.py | 22 +- 54 files changed, 1193 insertions(+), 1052 deletions(-) create mode 100644 python/ppc_scheduler/common/global_job_manager.py rename python/ppc_scheduler/{workflow/common/default_flow_config.py => demo/default_flow_config_sample.py} (80%) create mode 100644 python/ppc_scheduler/demo/prefect_workflow_sample.py create mode 100644 python/ppc_scheduler/demo/web_submit_job.json create mode 100644 python/ppc_scheduler/demo/workflow_args_sample.json rename python/ppc_scheduler/node/computing_node_client/{psi_node_client.py => psi_client.py} (66%) create mode 100755 python/ppc_scheduler/scripts/start.sh create mode 100755 python/ppc_scheduler/scripts/stop.sh create mode 100644 python/ppc_scheduler/workflow/builder/__init__.py create mode 100644 python/ppc_scheduler/workflow/builder/flow_builder.py delete mode 100644 python/ppc_scheduler/workflow/constructor.py create mode 100644 python/ppc_scheduler/workflow/scheduler/__init__.py rename python/ppc_scheduler/workflow/{ => scheduler}/scheduler.py (68%) create mode 100644 python/ppc_scheduler/workflow/scheduler/scheduler_api.py create mode 100644 python/ppc_scheduler/workflow/worker/default_worker.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/shell_engine.py create mode 100644 python/ppc_scheduler/workflow/worker/engine/work_engine.py delete mode 100644 python/ppc_scheduler/workflow/worker/exit_worker.py diff --git a/.gitignore b/.gitignore index 190391e3..c4556f6c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.venv + # Prerequisites *.d diff --git a/python/ppc_common/db_models/computing_node.py b/python/ppc_common/db_models/computing_node.py index bc0b988d..e8fa5543 100644 --- a/python/ppc_common/db_models/computing_node.py +++ b/python/ppc_common/db_models/computing_node.py @@ -3,8 +3,9 @@ class ComputingNodeRecord(db.Model): - __tablename__ = 't_computing_node' + __tablename__ = 'wedpr_computing_node' id = db.Column(db.String(255), primary_key=True) url = db.Column(db.String(255)) type = db.Column(db.String(255)) + token = db.Column(db.String(255)) loading = db.Column(db.Integer) diff --git a/python/ppc_common/db_models/config.sql b/python/ppc_common/db_models/config.sql index 295f5820..36d8406e 100644 --- a/python/ppc_common/db_models/config.sql +++ b/python/ppc_common/db_models/config.sql @@ -1,26 +1,40 @@ -CREATE TABLE t_job_worker ( - worker_id VARCHAR(100) PRIMARY KEY, - job_id VARCHAR(255) INDEX, +CREATE TABLE wedpr_scheduler_job_worker_table ( + worker_id VARCHAR(100), + job_id VARCHAR(255), type VARCHAR(255), status VARCHAR(255), + args TEXT, upstreams TEXT, inputs_statement TEXT, outputs TEXT, - create_time BIGINT, - update_time BIGINT -)ENGINE=InnoDB default charset=utf8mb4 default collate=utf8mb4_unicode_ci; + create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (worker_id), + INDEX job_id_idx (job_id) +)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC; -CREATE TABLE t_computing_node ( - id VARCHAR(255) PRIMARY KEY, +CREATE TABLE wedpr_scheduler_job_table ( + job_id VARCHAR(255), + request TEXT, + status TEXT, + create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (job_id) +)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC; + +CREATE TABLE wedpr_computing_node ( + id VARCHAR(255), url VARCHAR(255), type VARCHAR(255), - loading INT -)ENGINE=InnoDB default charset=utf8mb4 default collate=utf8mb4_unicode_ci; + loading INT, + token VARCHAR(255) DEFAULT '', + PRIMARY KEY (id) +)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC; -INSERT INTO t_computing_node (id, url, type, loading) +INSERT INTO wedpr_computing_node (id, url, type, loading, token) VALUES - ("001", '127.0.0.1:10200', 'PSI', 0), - ("002", '127.0.0.1:10201', 'MPC', 0), - ("003", '127.0.0.1:10202', 'MODEL', 0); + ("001", '127.0.0.1:10200', 'PSI', 0, ''), + ("002", '127.0.0.1:10201', 'MPC', 0, ''), + ("003", '127.0.0.1:10202', 'MODEL', 0, ''); diff --git a/python/ppc_common/db_models/job_worker_record.py b/python/ppc_common/db_models/job_worker_record.py index 29f94f61..af9b4c26 100644 --- a/python/ppc_common/db_models/job_worker_record.py +++ b/python/ppc_common/db_models/job_worker_record.py @@ -1,15 +1,17 @@ +from datetime import datetime from ppc_common.db_models import db class JobWorkerRecord(db.Model): - __tablename__ = 't_job_worker' - worker_id = db.Column(db.String(100), primary_key=True) + __tablename__ = 'wedpr_scheduler_job_worker_table' + worker_id = db.Column(db.String(127), primary_key=True) job_id = db.Column(db.String(255), index=True) type = db.Column(db.String(255)) status = db.Column(db.String(255)) upstreams = db.Column(db.Text) inputs_statement = db.Column(db.Text) + args = db.Column(db.Text) outputs = db.Column(db.Text) - create_time = db.Column(db.BigInteger) - update_time = db.Column(db.BigInteger) + create_time = db.Column(db.DateTime, default=datetime.now) + update_time = db.Column(db.DateTime, onupdate=datetime.now) diff --git a/python/ppc_common/deps_services/storage_loader.py b/python/ppc_common/deps_services/storage_loader.py index ba64fdac..0688648a 100644 --- a/python/ppc_common/deps_services/storage_loader.py +++ b/python/ppc_common/deps_services/storage_loader.py @@ -9,6 +9,6 @@ def load(config: dict, logger): 'HDFS_USER', None, config, False) hdfs_home = common_func.get_config_value( "HDFS_HOME", None, config, False) - return HdfsStorage(config['HDFS_ENDPOINT'], hdfs_user, hdfs_home) + return HdfsStorage(config['HDFS_URL'], hdfs_user, hdfs_home) else: raise Exception('unsupported storage type') diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py index 15b1f5ca..fe012878 100644 --- a/python/ppc_common/ppc_async_executor/async_thread_executor.py +++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py @@ -69,7 +69,7 @@ def _cleanup_finished_threads(self): for target_id in finished_threads: with self.lock: del self.threads[target_id] - self.logger.info(f"Cleanup finished thread {target_id}") + self.logger.info(f"cleanup finished thread {target_id}") def __del__(self): self.kill_all() diff --git a/python/ppc_common/ppc_utils/exception.py b/python/ppc_common/ppc_utils/exception.py index 212b3f71..c5ae1ea2 100644 --- a/python/ppc_common/ppc_utils/exception.py +++ b/python/ppc_common/ppc_utils/exception.py @@ -110,6 +110,8 @@ class PpcErrorCode(Enum): TASK_IS_KILLED = {11002: "the task is killed!"} ROLE_TYPE_ERROR = {12000: "role type is illegal."} + + UNSUPPORTED_WORK_TYPE = {13000: "unsupported work type."} def get_code(self): return list(self.value.keys())[0] diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml index 2fecc8eb..b7458d70 100644 --- a/python/ppc_model/conf/application-sample.yml +++ b/python/ppc_model/conf/application-sample.yml @@ -23,10 +23,10 @@ DB_TYPE: "mysql" SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4" # interagency services -HDFS_ENDPOINT: "http://127.0.0.1:50070" +HDFS_URL: "http://127.0.0.1:50070" # HDFS, STORAGE_TYPE: "HDFS" -HDFS_ENDPOINT: "http://127.0.0.1:9870" +HDFS_URL: "http://127.0.0.1:9870" HDFS_USER: "ppc" HDFS_HOME: "/user/ppc/model/webank" diff --git a/python/ppc_scheduler/common/global_context.py b/python/ppc_scheduler/common/global_context.py index 7a090c83..0d96ed01 100644 --- a/python/ppc_scheduler/common/global_context.py +++ b/python/ppc_scheduler/common/global_context.py @@ -1,8 +1,3 @@ -import os - from ppc_scheduler.common.initializer import Initializer -dirName, _ = os.path.split(os.path.abspath(__file__)) -config_path = "application.yml" - -components = Initializer(log_config_path='logging.conf', config_path=config_path) +components = Initializer() \ No newline at end of file diff --git a/python/ppc_scheduler/common/global_job_manager.py b/python/ppc_scheduler/common/global_job_manager.py new file mode 100644 index 00000000..14d91556 --- /dev/null +++ b/python/ppc_scheduler/common/global_job_manager.py @@ -0,0 +1,15 @@ + + +from ppc_scheduler.job.job_manager import JobManager + + +global_job_manager: JobManager = None + +def update_job_manager(job_manager): + global global_job_manager + global_job_manager = job_manager + +def get_job_manager(): + global global_job_manager + return global_job_manager + diff --git a/python/ppc_scheduler/common/initializer.py b/python/ppc_scheduler/common/initializer.py index 291dccc3..162feebf 100644 --- a/python/ppc_scheduler/common/initializer.py +++ b/python/ppc_scheduler/common/initializer.py @@ -4,42 +4,39 @@ from contextlib import contextmanager import yaml -from python.ppc_common.ppc_config.file_chunk_config import FileChunkConfig from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from ppc_common.deps_services import storage_loader -from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager from ppc_common.ppc_utils import common_func -from ppc_scheduler.job.job_manager import JobManager - class Initializer: - def __init__(self, log_config_path, config_path): + def __init__(self): + self.config_data = None self.job_cache_dir = None - self.log_config_path = log_config_path - self.config_path = config_path self.config_data = None - self.job_manager = None - self.thread_event_manager = None self.sql_session = None self.sql_engine = None self.storage_client = None - self.file_chunk_config = None - # 只用于测试 - self.mock_logger = None - def init_all(self): - self.init_log() + def init_all(self, config_data): + self.config_data = config_data + + # self.init_log() + # self.init_config() self.init_cache() - self.init_config() - self.init_job_manager() self.init_sql_client() self.init_storage_client() - self.init_file_chunk() - - def init_log(self): - logging.config.fileConfig(self.log_config_path) + + @staticmethod + def init_config(config_path: str): + with open(config_path, 'rb') as f: + config_data = yaml.safe_load(f.read()) + return config_data + + @staticmethod + def init_log(log_config_path: str): + logging.config.fileConfig(log_config_path) def init_cache(self): self.job_cache_dir = common_func.get_config_value( @@ -47,19 +44,6 @@ def init_cache(self): if not os.path.exists(self.job_cache_dir): os.makedirs(self.job_cache_dir) - def init_config(self): - with open(self.config_path, 'rb') as f: - self.config_data = yaml.safe_load(f.read()) - - def init_job_manager(self): - self.thread_event_manager = ThreadEventManager() - self.job_manager = JobManager( - logger=self.logger(), - thread_event_manager=self.thread_event_manager, - workspace=self.config_data['WORKSPACE'], - job_timeout_h=self.config_data['JOB_TIMEOUT_H'] - ) - def init_sql_client(self): self.sql_engine = create_engine(self.config_data['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True) self.sql_session = sessionmaker(bind=self.sql_engine) @@ -79,12 +63,10 @@ def create_sql_session(self): def init_storage_client(self): self.storage_client = storage_loader.load( self.config_data, self.logger()) - - def init_file_chunk(self): - self.file_chunk_config = FileChunkConfig(self.config_data) - def logger(self, name=None): - if self.mock_logger is None: - return logging.getLogger(name) - else: - return self.mock_logger + def update_thread_event_manager(self, thread_event_manager): + self.thread_event_manager = thread_event_manager + + @staticmethod + def logger(name=None): + return logging.getLogger(name) diff --git a/python/ppc_scheduler/common/log_utils.py b/python/ppc_scheduler/common/log_utils.py index 5e675d2f..159b9e5c 100644 --- a/python/ppc_scheduler/common/log_utils.py +++ b/python/ppc_scheduler/common/log_utils.py @@ -6,12 +6,16 @@ def job_start_log_info(job_id): - return f"=====================start_{job_id}=====================" - + return f"=====================start_job_{job_id}=====================" def job_end_log_info(job_id): - return f"======================end_{job_id}======================" + return f"======================end_job_{job_id}======================" + +def worker_start_log_info(work_id): + return f"=====================start_work_{work_id}=====================" +def worker_end_log_info(work_id): + return f"======================end_work_{work_id}======================" def upload_job_log(storage_client, job_id, extra=None): job_log_path = None @@ -26,8 +30,11 @@ def upload_job_log(storage_client, job_id, extra=None): job_log.write(extra) job_log.close() storage_client.upload_file(job_log_path, job_id + os.sep + utils.LOG_NAME) + except (Exception) as e: + print(e) finally: - os.remove(job_log_path) + if os.path.exists(job_log_path): + os.remove(job_log_path) def read_line_inverse(file): diff --git a/python/ppc_scheduler/conf/application-sample.yml b/python/ppc_scheduler/conf/application-sample.yml index 0be7408c..6d43e9b3 100644 --- a/python/ppc_scheduler/conf/application-sample.yml +++ b/python/ppc_scheduler/conf/application-sample.yml @@ -1,20 +1,13 @@ -HOST: "0.0.0.0" +HTTP_HOST: "0.0.0.0" HTTP_PORT: 43471 +HTTP_THREAD_NUM: 8 JOB_TIMEOUT_H: 1800 - -AGENCY_ID: "1001" -PPCS_RPC_TOKEN: "ppcs_psi_apikey" - WORKSPACE: "/data/app/files/job" -HDFS_ENDPOINT: "http://127.0.0.1:50070" +STORAGE_TYPE: "HDFS" +HDFS_URL: "http://127.0.0.1:50070" # mysql or dm DB_TYPE: "mysql" SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4" # SQLALCHEMY_DATABASE_URI: "dm+dmPython://ppcv16:ppc12345678@127.0.0.1:5236" - -SQLALCHEMY_TRACK_MODIFICATIONS: False - -MPC_NODE_DIRECT_PORT: 5899 -IS_MALICIOUS: False \ No newline at end of file diff --git a/python/ppc_scheduler/conf/logging.conf b/python/ppc_scheduler/conf/logging.conf index bff66b16..b472e2ab 100644 --- a/python/ppc_scheduler/conf/logging.conf +++ b/python/ppc_scheduler/conf/logging.conf @@ -16,13 +16,13 @@ keys=fileHandler,consoleHandler,accessHandler [handler_accessHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/wedpr/appmonitor.log', 'D', 1, 30, 'utf-8') +args=('./logs/appmonitor.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter [handler_fileHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/wedpr/scheduler.log', 'D', 1, 30, 'utf-8') +args=('./logs/scheduler.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter diff --git a/python/ppc_scheduler/database/computing_node_mapper.py b/python/ppc_scheduler/database/computing_node_mapper.py index 0a95ad79..5b6417a3 100644 --- a/python/ppc_scheduler/database/computing_node_mapper.py +++ b/python/ppc_scheduler/database/computing_node_mapper.py @@ -3,54 +3,59 @@ from ppc_common.db_models.computing_node import ComputingNodeRecord -def insert_computing_node(session, node_id: str, url: str, node_type: str, loading: int): - new_node = ComputingNodeRecord( - id=node_id, - url=url, - type=node_type, - loading=loading - ) +# def insert_computing_node(session, node_id: str, url: str, node_type: str, loading: int): +# new_node = ComputingNodeRecord( +# id=node_id, +# url=url, +# type=node_type, +# loading=loading +# ) - session.add(new_node) +# session.add(new_node) -def delete_computing_node(session, url: str, node_type: str): - stmt = ( - delete(ComputingNodeRecord).where( - and_( - ComputingNodeRecord.url == url, - ComputingNodeRecord.type == node_type - ) - ) - ) +# def delete_computing_node(session, url: str, node_type: str): +# stmt = ( +# delete(ComputingNodeRecord).where( +# and_( +# ComputingNodeRecord.url == url, +# ComputingNodeRecord.type == node_type +# ) +# ) +# ) - result = session.execute(stmt) +# result = session.execute(stmt) - return result.rowcount > 0 +# return result.rowcount > 0 def get_and_update_min_loading_url(session, node_type): - subquery = ( - select([ComputingNodeRecord.id]).where( - and_( - ComputingNodeRecord.type == node_type - ) - ).order_by(ComputingNodeRecord.loading.asc()).limit(1) - ).scalar_subquery() + + # select min_loading node + min_loading_node_id = session.query(ComputingNodeRecord.id).filter( + ComputingNodeRecord.type == node_type + ).order_by(ComputingNodeRecord.loading.asc()).first() + + # update loading stmt = ( update(ComputingNodeRecord).where( and_( - ComputingNodeRecord.id == subquery + ComputingNodeRecord.id == min_loading_node_id.id, + ComputingNodeRecord.type == node_type ) ).values( loading=ComputingNodeRecord.loading + 1 - ).returning(ComputingNodeRecord.url) + ) ) + session.execute(stmt) - result = session.execute(stmt) - return result.scalar() - + # select min_loading node + record = session.query(ComputingNodeRecord.url, ComputingNodeRecord.token).filter( + ComputingNodeRecord.id == min_loading_node_id.id + ).order_by(ComputingNodeRecord.loading.asc()).first() + + return record def release_loading(session, url: str, node_type: str): stmt = ( diff --git a/python/ppc_scheduler/database/job_worker_mapper.py b/python/ppc_scheduler/database/job_worker_mapper.py index a4ee3699..0d8a7e8b 100644 --- a/python/ppc_scheduler/database/job_worker_mapper.py +++ b/python/ppc_scheduler/database/job_worker_mapper.py @@ -1,10 +1,31 @@ -from sqlalchemy import and_, update +import datetime +import json +# from sqlalchemy.dialects.mysql import insert +from sqlalchemy import and_, update, insert from sqlalchemy.exc import NoResultFound from ppc_common.db_models.job_worker_record import JobWorkerRecord from ppc_common.ppc_utils import utils from ppc_scheduler.workflow.common import codec +from ppc_scheduler.workflow.common.worker_status import WorkerStatus +def insert_job_worker(session, worker): + upstreams_str = json.dumps(worker['upstreams']) + inputs_statement_str = json.dumps(worker['inputs_statement']) + args_str=json.dumps(worker['args']) + + stmt = insert(JobWorkerRecord).prefix_with(" IGNORE").values( + worker_id=worker['worker_id'], + job_id=worker['job_id'], + type=worker['type'], + status=WorkerStatus.PENDING, + args=args_str, + upstreams=upstreams_str, + inputs_statement=inputs_statement_str + ) + + result = session.execute(stmt) + return result.rowcount > 0 def query_job_worker(session, job_id, worker_id): try: @@ -24,8 +45,7 @@ def update_job_worker(session, job_id, worker_id, status, outputs): ) ).values( status=status, - outputs=codec.serialize_worker_outputs_for_db(outputs), - update_time=utils.make_timestamp() + outputs=codec.serialize_worker_outputs(outputs) ) ) result = session.execute(stmt) diff --git a/python/ppc_scheduler/workflow/common/default_flow_config.py b/python/ppc_scheduler/demo/default_flow_config_sample.py similarity index 80% rename from python/ppc_scheduler/workflow/common/default_flow_config.py rename to python/ppc_scheduler/demo/default_flow_config_sample.py index 6797bbc4..4cf0b5de 100644 --- a/python/ppc_scheduler/workflow/common/default_flow_config.py +++ b/python/ppc_scheduler/demo/default_flow_config_sample.py @@ -1,35 +1,34 @@ flow_dict = { + + "SHELL": [ + { + "index": 1, + "type": "T_SHELL" + }, + { + "index": 2, + "type": "T_SHELL", + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "T_SHELL", + "upstreams": [ + { + "index": 2 + } + ] + } + ], + "PSI": [ { "index": 1, - "type": "T_PSI", - "isParamsProvided": False, - "params": { - "type": 0, - "algorithm": 0, - "syncResult": True, - "parties": [ - { - "index": "", - "partyIndex": 1 - }, - { - "index": "", - "partyIndex": 0, - "data": { - "index": "", - "input": { - "type": 2, - "path": "" - }, - "output": { - "type": 2, - "path": "" - } - } - } - ] - } + "type": "T_PSI" } ], diff --git a/python/ppc_scheduler/demo/prefect_workflow_sample.py b/python/ppc_scheduler/demo/prefect_workflow_sample.py new file mode 100644 index 00000000..9a146260 --- /dev/null +++ b/python/ppc_scheduler/demo/prefect_workflow_sample.py @@ -0,0 +1,79 @@ +import time +from prefect import Task, Flow +from prefect.triggers import all_successful, any_failed +from prefect import Flow +from prefect.executors import LocalDaskExecutor + +class SuccessTask(Task): + def run(self): + print(" ===>>>> Success Task Ran") + +class FailTask(Task): + def run(self): + print(" ===>>> Fail Task Ran") + +class JobTask(Task): + def __init__(self, name): + super().__init__(name=name) + self.name = name + + def run(self, x, y): + print(" ==> " + self.name + " is running, " + str(x) + ", " + str(y)) + return x + y + +# with Flow("My Flow") as f: +# t1 = a(1, 2) # t1 != a +# t2 = a(5, 7) # t2 != a + +with Flow("example_flow") as flow: + + task1 = JobTask("t1") + task2 = JobTask("t2") + task3 = JobTask("t3") + task4 = JobTask("t4") + + # task1=t1(1,2) + # task2=t2(3,4) + # task3=t3(4,5) + # task4=t4(6,7) + task1.bind(x = 1, y = 2) + task2.bind(x = 3, y = 4) + task3.bind(x = 5, y = 6) + task4.bind(x = 7, y = 8) + + task3.set_upstream(task1, flow=flow) + task3.set_upstream(task2, flow=flow) + task4.set_upstream(task3, flow=flow) + + # 定义成功任务，仅当所有上游任务成功时才运行 + success_task = SuccessTask(name="Success Task") + success_task.trigger = all_successful + + # 定义失败任务，只要任何一个上游任务失败就运行 + fail_task = FailTask(name="Fail Task") + fail_task.trigger = any_failed + + # 设置依赖关系 + task1.set_downstream(success_task, flow=flow) + task2.set_downstream(success_task, flow=flow) + task3.set_downstream(success_task, flow=flow) + task4.set_downstream(success_task, flow=flow) + + task1.set_downstream(fail_task, flow=flow) + task2.set_downstream(fail_task, flow=flow) + task3.set_downstream(fail_task, flow=flow) + task4.set_downstream(fail_task, flow=flow) + + +# flow.executor = LocalDaskExecutor() + +start_time = time.time() +# 运行工作流 +flow_state=flow.run() + +end_time = time.time() +print(f" ## costs: {end_time - start_time}") + +print(flow_state.result) + +flow.visualize(flow_state, "./my_flow", 'svg') \ No newline at end of file diff --git a/python/ppc_scheduler/demo/web_submit_job.json b/python/ppc_scheduler/demo/web_submit_job.json new file mode 100644 index 00000000..f5b6d9cd --- /dev/null +++ b/python/ppc_scheduler/demo/web_submit_job.json @@ -0,0 +1,134 @@ +[ + { + "job": { + "jobType": "PSI", + "projectName": "octo-test", + "param": "{\"dataSetList\":[{\"idFields\":[\"x200\"],\"dataset\":{\"owner\":\"octo1\",\"ownerAgency\":\"WeBank\",\"path\":\"/user/ppc/webank/octo1/d-9323009025845253\",\"storageTypeStr\":\"HDFS\",\"datasetID\":\"d-9323009025845253\"},\"receiveResult\":false},{\"idFields\":[\" x106\"],\"dataset\":{\"owner\":\"octopuswang1\",\"ownerAgency\":\"SGD\",\"path\":\"/user/ppc/sgd/octopuswang1/d-9322966927878149\",\"storageTypeStr\":\"HDFS\",\"datasetID\":\"d-9322966927878149\"},\"receiveResult\":true}]}" + }, + "taskParties": [ + { + "userName": "octo1", + "agency": "WeBank" + }, + { + "userName": "octopuswang1", + "agency": "SGD" + } + ], + "datasetList": [ + "d-xxx", + "d-yyy" + ] + }, + { + "dataSetList": [ + { + "idFields": [ + "x200" + ], + "dataset": { + "owner": "octo1", + "ownerAgency": "WeBank", + "path": "/user/ppc/webank/octo1/d-9323009025845253", + "storageTypeStr": "HDFS", + "datasetID": "d-9323009025845253" + }, + "receiveResult": false + }, + { + "idFields": [ + " x106" + ], + "dataset": { + "owner": "octopuswang1", + "ownerAgency": "SGD", + "path": "/user/ppc/sgd/octopuswang1/d-9322966927878149", + "storageTypeStr": "HDFS", + "datasetID": "d-9322966927878149" + }, + "receiveResult": true + } + ] + } + ] + + { + "jsonrpc": "1", + "method": "asyncRunTask", + "token": "", + "id": 61480, + "params": { + "taskID": "9441012273129477-PSI", + "type": 0, + "algorithm": 0, + "syncResult": true, + "lowBandwidth": false, + "parties": [ + { + "id": "WeBank", + "partyIndex": 0, + "data": { + "id": "9441012273129477", + "input": { + "datasetID": "d-9431392664627205", + "type": 2, + "storageTypeStr": "HDFS", + "path": "/user/ppc/webank/octo1/d-9323009025845253", + "owner": "octo1", + "ownerAgency": "WeBank", + "storagePath": { + "storageType": "HDFS", + "filePath": "/user/ppc/webank/octo1/d-9323009025845253" + } + }, + "output": { + "datasetID": null, + "type": 2, + "storageTypeStr": "HDFS", + "path": "/user/wedpr//home/cache/octo1/9441012273129477/psi_result.csv", + "owner": "octo1", + "ownerAgency": "WeBank", + "storagePath": { + "storageType": "HDFS", + "filePath": "/user/wedpr/home/cache/octo1/9441012273129477/psi_result.csv" + } + } + } + }, + { + "id": "SGD", + "partyIndex": 0, + "data": { + "id": "9441012273129477", + "input": { + "datasetID": "d-9431392714827781", + "type": 2, + "storageTypeStr": "HDFS", + "path": "/user/ppc/webank/octo1/d-9323009025845253", + "owner": "octopuswang1", + "ownerAgency": "SGD", + "storagePath": { + "storageType": "HDFS", + "filePath": "/user/ppc/webank/octo1/d-9323009025845253" + } + }, + "output": { + "datasetID": null, + "type": 2, + "storageTypeStr": "HDFS", + "path": "/user/wedpr//home/cache/octopuswang1/9441012273129477/psi_result.csv", + "owner": "octopuswang1", + "ownerAgency": "SGD", + "storagePath": { + "storageType": "HDFS", + "filePath": "/user/wedpr/home/cache/octopuswang1/9441012273129477/psi_result.csv" + } + } + } + } + ], + "receiverList": [ + "SGD" + ] + } + } \ No newline at end of file diff --git a/python/ppc_scheduler/demo/workflow_args_sample.json b/python/ppc_scheduler/demo/workflow_args_sample.json new file mode 100644 index 00000000..d77a9c1c --- /dev/null +++ b/python/ppc_scheduler/demo/workflow_args_sample.json @@ -0,0 +1,73 @@ +{ + "jobId": "x12345678", + "agency": "WeBank", + "workflow": [ + { + "index": 1, + "type": "SHELL", + "args": [ + "sleep 10" + ] + }, + { + "index": 2, + "type": "SHELL", + "args": [ + "sleep 10" + ], + "upstreams": [ + { + "index": 1 + } + ] + }, + { + "index": 3, + "type": "SHELL", + "args": [ + "sleep 10" + ], + "upstreams": [ + { + "index": 2 + } + ] + }, + { + "index": 4, + "type": "SHELL", + "args": [ + "sleep 10" + ], + "upstreams": [ + { + "index": 3 + } + ] + }, + { + "index": 5, + "type": "SHELL", + "args": [ + "sleep 10" + ], + "upstreams": [ + { + "index": 4 + } + ] + }, + { + "index": 6, + "type": "SHELL", + "args": [ + "sleep 10" + ], + "upstreams": [ + { + "index": 5 + } + ] + } + ] +} \ No newline at end of file diff --git a/python/ppc_scheduler/endpoints/job_controller.py b/python/ppc_scheduler/endpoints/job_controller.py index 883fbca8..dbf8d9cb 100644 --- a/python/ppc_scheduler/endpoints/job_controller.py +++ b/python/ppc_scheduler/endpoints/job_controller.py @@ -4,25 +4,42 @@ from flask_restx import Resource from ppc_common.ppc_utils import utils +from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException from ppc_scheduler.common.global_context import components from ppc_scheduler.endpoints.body_schema import response_job_status, response_base from ppc_scheduler.endpoints.restx import api +from ppc_scheduler.common.global_job_manager import get_job_manager -ns = api.namespace('ppc-scheduler/job', +ns = api.namespace('/wedpr/v3/scheduler/job', description='Operations related to run job') @ns.route('/') class JobCollection(Resource): - @api.response(201, 'Job started successfully.', response_base) + @api.response(200, 'Job started successfully.', response_base) def post(self, job_id): """ Run a specific job by job_id. """ - args = request.get_json() - components.logger().info(f"run job request, job_id: {job_id}, args: {args}") - components.job_manager.run_task(job_id, (args,)) + request_body = request.get_json() + + if 'jobId' not in request_body: + raise PpcException( + PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(), + f"Missing 'jobId' in request") + # if 'agency' not in request_body: + # raise PpcException( + # PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(), + # f"Missing 'agency' in request") + if 'workflow' not in request_body: + raise PpcException( + PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(), + f"Missing 'workflow' in request") + + components.logger().info(f"Recv run job request, job_id: {job_id}, request: {request_body}") + + get_job_manager().run_task(job_id, request_body) return utils.BASE_RESPONSE @api.response(200, 'Job status retrieved successfully.', response_job_status) @@ -31,7 +48,10 @@ def get(self, job_id): Get the status of a specific job by job_id. """ response = utils.BASE_RESPONSE - status, time_costs = components.job_manager.status(job_id) + + components.logger().info(f"Recv query job request, job_id: {job_id}") + + status, time_costs = get_job_manager().status(job_id) response['data'] = { 'status': status, 'time_costs': time_costs @@ -43,6 +63,8 @@ def delete(self, job_id): """ Kill a specific job by job_id. """ - components.logger().info(f"kill request, job_id: {job_id}") - components.job_manager.kill_job(job_id) + + components.logger().info(f"receive kill job request, job_id: {job_id}") + + get_job_manager().kill_job(job_id) return utils.BASE_RESPONSE diff --git a/python/ppc_scheduler/job/job_manager.py b/python/ppc_scheduler/job/job_manager.py index 95afa15a..29cc5b38 100644 --- a/python/ppc_scheduler/job/job_manager.py +++ b/python/ppc_scheduler/job/job_manager.py @@ -10,46 +10,61 @@ from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode from ppc_scheduler.common import log_utils from ppc_scheduler.job.job_status import JobStatus -from ppc_scheduler.workflow.scheduler import Scheduler +from ppc_scheduler.workflow.scheduler.scheduler import Scheduler +from ppc_scheduler.workflow.builder.flow_builder import FlowBuilder +from ppc_scheduler.workflow.common.job_context import JobContext class JobManager: def __init__(self, logger, + scheduler: Scheduler, thread_event_manager: ThreadEventManager, - workspace, + workspace: str, job_timeout_h: Union[int, float]): self.logger = logger self._thread_event_manager = thread_event_manager + self._scheduler = scheduler self._workspace = workspace self._job_timeout_s = job_timeout_h * 3600 self._rw_lock = rwlock.RWLockWrite() self._jobs: dict[str, list] = {} + self._flow_builder = FlowBuilder(logger=logger) self._async_executor = AsyncThreadExecutor( event_manager=self._thread_event_manager, logger=logger) - self._cleanup_thread = threading.Thread(target=self._loop_cleanup) + self._cleanup_thread = threading.Thread(target=self._loop_action) self._cleanup_thread.daemon = True self._cleanup_thread.start() - self.scheduler = Scheduler(self._workspace) - def run_task(self, job_id, args=()): + def run_task(self, job_id, request_body): """ - 发起任务 - param args: 任务参数 + run task + param job_id: job id + param request: job params """ + # TODO: The database persists job information with self._rw_lock.gen_wlock(): if job_id in self._jobs: - self.logger.info(f"Task already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}") + self.logger.info(f"Job already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}") return self._jobs[job_id] = [JobStatus.RUNNING, datetime.datetime.now(), 0] self.logger.info(log_utils.job_start_log_info(job_id)) - self._async_executor.execute(job_id, self._run_job_flow, self._on_task_finish, args) - - def _run_job_flow(self, args): + + # Create job context + job_context = JobContext.create_job_context(request_body, self._workspace) + # Build job workflow + flow_context = self._flow_builder.build_flow_context(job_id=job_context.job_id, workflow_configs=job_context.workflow_configs) + # Persistent workflow + self._flow_builder.save_flow_context(job_context.job_id, flow_context) + # Run workflow + self._async_executor.execute(job_id, self._run_job_flow, self._on_task_finish, (job_context, flow_context)) + + def _run_job_flow(self, job_context, flow_context): """ - 运行任务流 - + run job flow """ - self.scheduler.schedule_job_flow(args) + + # the scheduler module starts scheduling tasks + self._scheduler.run(job_context, flow_context) def kill_job(self, job_id: str): with self._rw_lock.gen_rlock(): @@ -62,9 +77,10 @@ def kill_job(self, job_id: str): with self._rw_lock.gen_wlock(): self._jobs[job_id][0] = JobStatus.FAILURE - def status(self, job_id: str) -> [str, float]: + def status(self, job_id: str) -> tuple[str, float]: """ - 返回: 任务状态, 执行耗时(s) + query task status + return: task status and task cost(s) """ with self._rw_lock.gen_rlock(): if job_id not in self._jobs: @@ -88,11 +104,12 @@ def _on_task_finish(self, job_id: str, is_succeeded: bool, e: Exception = None): self.logger.warn(f"Job {job_id} failed, time_costs: {time_costs}s, error: {e}") self.logger.info(log_utils.job_end_log_info(job_id)) - def _loop_cleanup(self): + def _loop_action(self): while True: + time.sleep(20) self._terminate_timeout_jobs() self._cleanup_finished_jobs() - time.sleep(5) + self._report_jobs() def _terminate_timeout_jobs(self): jobs_to_kill = [] @@ -115,9 +132,15 @@ def _cleanup_finished_jobs(self): value[1]).total_seconds() if alive_time >= self._job_timeout_s + 3600: jobs_to_cleanup.append((job_id, value[3])) + self.logger.info(f"Job is finished, job_id: {job_id}") with self._rw_lock.gen_wlock(): for job_id, job_id in jobs_to_cleanup: if job_id in self._jobs: del self._jobs[job_id] self._thread_event_manager.remove_event(job_id) self.logger.info(f"Cleanup job cache, job_id: {job_id}") + + def _report_jobs(self): + with self._rw_lock.gen_rlock(): + job_count = len(self._jobs) + self.logger.info(f" ## Report job count: {job_count}") diff --git a/python/ppc_scheduler/job/job_status.py b/python/ppc_scheduler/job/job_status.py index a0a530ad..c9cd8fa2 100644 --- a/python/ppc_scheduler/job/job_status.py +++ b/python/ppc_scheduler/job/job_status.py @@ -1,6 +1,5 @@ class JobStatus: RUNNING = 'RUNNING' SUCCESS = 'SUCCESS' - FAILURE = 'FAILURE' - TIMEOUT = 'TIMEOUT' + FAILURE = 'FAILED' KILLED = 'KILLED' diff --git a/python/ppc_scheduler/job/job_type.py b/python/ppc_scheduler/job/job_type.py index d734639f..22e67f02 100644 --- a/python/ppc_scheduler/job/job_type.py +++ b/python/ppc_scheduler/job/job_type.py @@ -6,3 +6,7 @@ class JobType: FEATURE_ENGINEERING = 'FEATURE_ENGINEERING' TRAINING = 'TRAINING' PREDICTION = 'PREDICTION' + + SHELL = "SHELL" + PYTHON = "PYTHON" + API = "API" diff --git a/python/ppc_scheduler/node/computing_node_client/model_node_client.py b/python/ppc_scheduler/node/computing_node_client/model_node_client.py index 03456c9b..64a14120 100644 --- a/python/ppc_scheduler/node/computing_node_client/model_node_client.py +++ b/python/ppc_scheduler/node/computing_node_client/model_node_client.py @@ -8,8 +8,8 @@ class ModelClient: - def __init__(self, log, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): - self.log = log + def __init__(self, logger, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): + self.logger = logger self.endpoint = endpoint self.polling_interval_s = polling_interval_s self.max_retries = max_retries @@ -20,7 +20,7 @@ def __init__(self, log, endpoint, polling_interval_s: int = 5, max_retries: int def run(self, args): task_id = args['task_id'] try: - self.log.info(f"ModelApi: begin to run model task {task_id}") + self.logger.info(f"ModelApi: begin to run model task {task_id}") response = self._send_request_with_retry(http_utils.send_post_request, endpoint=self.endpoint, uri=RUN_MODEL_API_PREFIX + task_id, @@ -28,20 +28,20 @@ def run(self, args): check_response(response) return self._poll_task_status(task_id) except Exception as e: - self.log.error(f"ModelApi: run model task error, task: {task_id}, error: {e}") + self.logger.error(f"ModelApi: run model task error, task: {task_id}, error: {e}") raise e def kill(self, job_id): try: - self.log.info(f"ModelApi: begin to kill model task {job_id}") + self.logger.info(f"ModelApi: begin to kill model task {job_id}") response = self._send_request_with_retry(http_utils.send_delete_request, endpoint=self.endpoint, uri=RUN_MODEL_API_PREFIX + job_id) check_response(response) - self.log.info(f"ModelApi: model task {job_id} was killed") + self.logger.info(f"ModelApi: model task {job_id} was killed") return response except Exception as e: - self.log.warn(f"ModelApi: kill model task {job_id} failed, error: {e}") + self.logger.warn(f"ModelApi: kill model task {job_id} failed, error: {e}") raise e def _poll_task_status(self, task_id): @@ -51,10 +51,10 @@ def _poll_task_status(self, task_id): uri=RUN_MODEL_API_PREFIX + task_id) check_response(response) if response['data']['status'] == self._completed_status: - self.log.info(f"task {task_id} completed, response: {response['data']}") + self.logger.info(f"task {task_id} completed, response: {response['data']}") return response elif response['data']['status'] == self._failed_status: - self.log.warn(f"task {task_id} failed, response: {response['data']}") + self.logger.warn(f"task {task_id} failed, response: {response['data']}") raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) else: time.sleep(self.polling_interval_s) @@ -73,12 +73,12 @@ def _send_request_with_retry(self, request_func, *args, **kwargs): response = request_func(*args, **kwargs) return response except Exception as e: - self.log.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") + self.logger.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") attempt += 1 if attempt < self.max_retries: time.sleep(self.retry_delay_s) else: - self.log.warn(f"Request failed after {self.max_retries} attempts") + self.logger.warn(f"Request failed after {self.max_retries} attempts") raise e diff --git a/python/ppc_scheduler/node/computing_node_client/psi_node_client.py b/python/ppc_scheduler/node/computing_node_client/psi_client.py similarity index 66% rename from python/ppc_scheduler/node/computing_node_client/psi_node_client.py rename to python/ppc_scheduler/node/computing_node_client/psi_client.py index a4506a4c..43fbd5f1 100644 --- a/python/ppc_scheduler/node/computing_node_client/psi_node_client.py +++ b/python/ppc_scheduler/node/computing_node_client/psi_client.py @@ -1,3 +1,4 @@ +import json import random import time @@ -7,35 +8,40 @@ class PsiClient: - def __init__(self, log, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): - self.log = log + def __init__(self, logger, endpoint, token, polling_interval_s: int = 3, max_retries: int = 3, retry_delay_s: int = 3): + self.logger = logger self.endpoint = endpoint + self.token = token self.polling_interval_s = polling_interval_s self.max_retries = max_retries self.retry_delay_s = retry_delay_s - self._async_run_task_method = 'asyncRunTask' self._get_task_status_method = 'getTaskStatus' self._completed_status = 'COMPLETED' self._failed_status = 'FAILED' - def run(self, job_info, token): - params = { + def run(self, *args): + + params = args[0] + if type(params) == str: + params = json.loads(params) + + json_rpc_request = { 'jsonrpc': '1', - 'method': self._async_run_task_method, - 'token': token, + 'method': 'asyncRunTask', + 'token': self.token, 'id': random.randint(1, 65535), - 'params': job_info + 'params': params } - response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, json_rpc_request) check_privacy_service_response(response) - return self._poll_task_status(job_info['taskID'], token) + return self._poll_task_status(params['taskID']) - def _poll_task_status(self, task_id, token): + def _poll_task_status(self, task_id): while True: params = { 'jsonrpc': '1', - 'method': self._get_task_status_method, - 'token': token, + 'method': 'getTaskStatus', + 'token': self.token, 'id': random.randint(1, 65535), 'params': { 'taskID': task_id, @@ -46,7 +52,7 @@ def _poll_task_status(self, task_id, token): if response['result']['status'] == self._completed_status: return response['result'] elif response['result']['status'] == self._failed_status: - self.log.warn(f"task {task_id} failed, response: {response['data']}") + self.logger.warn(f"task {task_id} failed, response: {response['data']}") raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) time.sleep(self.polling_interval_s) @@ -57,10 +63,10 @@ def _send_request_with_retry(self, request_func, *args, **kwargs): response = request_func(*args, **kwargs) return response except Exception as e: - self.log.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") + self.logger.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}") attempt += 1 if attempt < self.max_retries: time.sleep(self.retry_delay_s) else: - self.log.warn(f"Request failed after {self.max_retries} attempts") + self.logger.warn(f"Request failed after {self.max_retries} attempts") raise e diff --git a/python/ppc_scheduler/ppc_scheduler_app.py b/python/ppc_scheduler/ppc_scheduler_app.py index fb44f888..0195236f 100644 --- a/python/ppc_scheduler/ppc_scheduler_app.py +++ b/python/ppc_scheduler/ppc_scheduler_app.py @@ -1,48 +1,125 @@ # Note: here can't be refactored by autopep +import os +import sys +import argparse + +current_file_path = os.path.abspath(__file__) +current_file_real_path = os.path.realpath(current_file_path) + + +current_dir = os.path.dirname(current_file_real_path) +parent_dir = os.path.dirname(current_dir) + +sys.path.append(current_dir) +sys.path.append(parent_dir) +print(sys.path) + from ppc_scheduler.endpoints.restx import api from ppc_scheduler.endpoints.job_controller import ns as job_namespace +from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager +from ppc_scheduler.job.job_manager import JobManager +from ppc_scheduler.workflow.scheduler.scheduler import Scheduler +from ppc_scheduler.common.global_job_manager import update_job_manager from ppc_scheduler.common.global_context import components +from ppc_scheduler.workflow.scheduler.scheduler_api import SchedulerApi from paste.translogger import TransLogger from flask import Flask, Blueprint from cheroot.wsgi import Server as WSGIServer -from cheroot.ssl.builtin import BuiltinSSLAdapter -import os -import sys -sys.path.append("../") - app = Flask(__name__) +def init_thread_event_manager(): + thread_event_manager = ThreadEventManager() + return thread_event_manager + +def init_scheduler(config_data, workspace: str, logger): + scheduler_api = Scheduler(workspace, logger=logger) + return scheduler_api + +def init_job_manager(config_data, workspace: str, thread_event_manager: ThreadEventManager, scheduler: SchedulerApi, logger): + + job_timeout_h = config_data['JOB_TIMEOUT_H'] + + job_manager = JobManager( + logger=logger, + scheduler=scheduler, + thread_event_manager=thread_event_manager, + workspace=workspace, + job_timeout_h=job_timeout_h + ) + + logger.info("Initialize job manager, job_timeout_h: %s", job_timeout_h) + + return job_manager -def initialize_app(app): - # 初始化应用功能组件 - components.init_all() +def initialize_app(app, config_path, log_config_path): + # init log first + components.init_log(log_config_path=log_config_path) + logger = components.logger() + + # init config + config_data = components.init_config(config_path=config_path) + # workspaces + workspace = config_data['WORKSPACE'] + logger.info(f" ==> Initialize workspace: {workspace}") + + # event manager + thread_event_manager = init_thread_event_manager() + + # scheduler + scheduler = init_scheduler(config_data=config_data, workspace=workspace, logger=logger) + + # job manager + job_manager = init_job_manager(config_data=config_data, workspace=workspace, thread_event_manager=thread_event_manager, scheduler=scheduler, logger=logger) + + update_job_manager(job_manager) + + # initialize application components + components.init_all(config_data=config_data) + components.update_thread_event_manager(thread_event_manager) - app.config.update(components.config_data) + app.config.update(config_data) + blueprint = Blueprint('api', __name__, url_prefix='/api') api.init_app(blueprint) api.add_namespace(job_namespace) app.register_blueprint(blueprint) + + components.logger().info(app.url_map) - -if __name__ == '__main__': - initialize_app(app) +def main(config_path, log_config_path): + + print(f"Using config: {config_path}") + print(f"Using logging config: {log_config_path}") + + initialize_app(app, config_path, log_config_path) app.config['SECRET_KEY'] = os.urandom(24) - server = WSGIServer((app.config['HOST'], app.config['HTTP_PORT']), - TransLogger(app, setup_console_handler=False), numthreads=2) + + listen_ip = app.config['HTTP_HOST'] + listen_port = app.config['HTTP_PORT'] + thread_num = app.config['HTTP_THREAD_NUM'] + + server = WSGIServer((listen_ip, listen_port), + TransLogger(app, setup_console_handler=False), numthreads=thread_num) - ssl_switch = app.config['SSL_SWITCH'] protocol = 'http' - if ssl_switch == 1: - protocol = 'https' - server.ssl_adapter = BuiltinSSLAdapter( - certificate=app.config['SSL_CRT'], - private_key=app.config['SSL_KEY'], - certificate_chain=app.config['CA_CRT']) - - message = f"Starting ppc scheduler server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']}" + message = f"Starting wedpr scheduler server at {protocol}://{listen_ip}:{listen_port}" print(message) components.logger().info(message) server.start() + +if __name__ == "__main__": + # Create ArgumentParser + parser = argparse.ArgumentParser(description='wedpr scheduler') + # Add argument + parser.add_argument('--config', default='./conf/application.yml', + help='Path to the configuration file') + parser.add_argument('--log_config', default='./conf/logging.conf', + help='Path to the logging configuration file') + # Parser argument + args = parser.parse_args() + + # Run main program + main(config_path=args.config, log_config_path=args.log_config) diff --git a/python/ppc_scheduler/scripts/start.sh b/python/ppc_scheduler/scripts/start.sh new file mode 100755 index 00000000..0dbd65b6 --- /dev/null +++ b/python/ppc_scheduler/scripts/start.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +LANG=zh_CN.UTF-8 +############################################################################## +## +## Wedpr scheduler service start up script for UN*X +## +############################################################################## + +# @function: output log with red color (error log) +# @param: content: error message + +function LOG_ERROR() { + local content=${1} + echo -e "\033[31m"${content}"\033[0m" +} + +# @function: output information log +# @param: content: information message +function LOG_INFO() { + local content=${1} + echo -e "\033[32m"${content}"\033[0m" +} + +function Usage() { + LOG_INFO "Usage:start the console" + LOG_INFO "./start_scheduler.sh" +} + +function check_python() { + # check python + if ! command -v python >/dev/null 2>&1; then + echo "Python is not installed." + exit 1 + fi + + # check python version, 3.9 support + python_version=$(python --version) + if [[ $python_version =~ Python\ 3.9.* ]]; then + echo "Python version is 3.9, as required." + else + echo "Python version is not 3.9. Installed version is: $python_version" + exit 2 + fi +} + +# source venv/bin/activate + +pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}') +if [ ! -z ${pid} ]; then + echo " the scheduler has started, pid is ${pid}." + exit 0 +fi + +nohup python $dirpath/ppc_scheduler/ppc_scheduler_app.py --config $dirpath/conf/application.yml --log_config $dirpath/conf/logging.conf >/dev/null 2>&1 & + +sleep 3 + +pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}') +if [ ! -z ${pid} ]; then + echo " start scheduler successfully, pid is ${pid}." + exit 0 +else + echo " start scheduler failed." + exit 1 +fi diff --git a/python/ppc_scheduler/scripts/stop.sh b/python/ppc_scheduler/scripts/stop.sh new file mode 100755 index 00000000..686122f6 --- /dev/null +++ b/python/ppc_scheduler/scripts/stop.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +dirpath="$(cd "$(dirname "$0")" && pwd)" +cd $dirpath + +LANG=zh_CN.UTF-8 +############################################################################## +## +## Wedpr scheduler service start up script for UN*X +## +############################################################################## + +# @function: output log with red color (error log) +# @param: content: error message + +function LOG_ERROR() { + local content=${1} + echo -e "\033[31m"${content}"\033[0m" +} + +# @function: output information log +# @param: content: information message +function LOG_INFO() { + local content=${1} + echo -e "\033[32m"${content}"\033[0m" +} + +function Usage() { + LOG_INFO "Usage:start the console" + LOG_INFO "./start_scheduler.sh" +} + +pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}') + +if [ ! -z ${pid} ]; then + kill -9 ${pid} + echo " scheduler is running, pid is ${pid}, kill it." + exit 0 +else + echo " scheduler is not running." + exit 1 +fi diff --git a/python/ppc_scheduler/workflow/builder/__init__.py b/python/ppc_scheduler/workflow/builder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/builder/flow_builder.py b/python/ppc_scheduler/workflow/builder/flow_builder.py new file mode 100644 index 00000000..207ff7fe --- /dev/null +++ b/python/ppc_scheduler/workflow/builder/flow_builder.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +import json +from ppc_common.db_models.job_worker_record import JobWorkerRecord +from ppc_scheduler.common.global_context import components +from ppc_scheduler.database import job_worker_mapper +from ppc_scheduler.workflow.common import flow_utils +from ppc_scheduler.workflow.common.worker_status import WorkerStatus + + +class FlowBuilder: + def __init__(self, logger): + self.logger = logger + + def build_flow_context(self, job_id, workflow_configs): + self.logger.info(f"## start build_flow_context, job_id: {job_id}") + flow_context = {} + index_type_map = {} + for workflow_config in workflow_configs: + workflow_type = workflow_config['type'] + workflow_index = workflow_config['index'] + workflow_args = workflow_config['args'] + index_type_map[workflow_index] = workflow_type + + for workflow_config in workflow_configs: + workflow_type = workflow_config['type'] + workflow_index = workflow_config['index'] + workflow_args = workflow_config['args'] + worker_id = flow_utils.cat_worker_id(job_id, workflow_index, workflow_type) + upstreams = [] + inputs_statement = [] + inputs_statement_tuple = [] + if 'upstreams' in workflow_config: + for upstream_config in workflow_config["upstreams"]: + index = upstream_config['index'] + upstream_id = flow_utils.cat_worker_id(job_id, index, index_type_map[index]) + upstreams.append(upstream_id), + if 'output_input_map' in upstream_config: + for mapping in upstream_config.get("output_input_map", []): + output_index, input_index = mapping.split(":") + inputs_statement_tuple.append((upstream_id, int(output_index), int(input_index))) + + inputs_statement_tuple.sort(key=lambda x: x[2]) + for upstream_id, output_index, _ in inputs_statement_tuple: + inputs_statement.append( + { + 'output_index': output_index, + 'upstream': upstream_id + } + ) + + worker_context = { + 'job_id': job_id, + 'worker_id': worker_id, + 'type': workflow_type, + 'status': WorkerStatus.PENDING, + 'args': workflow_args, + 'upstreams': upstreams, + 'inputs_statement': inputs_statement + } + + self.logger.debug(f"## mid build_flow_context, work_context:\n{worker_context}") + + flow_context[worker_id] = worker_context + self.logger.info(f"## end build_flow_context, flow_context:\n{flow_context}") + + return flow_context + + def save_flow_context(self, job_id, flow_context): + self.logger.info(f"## start save flow context, job_id: {job_id}") + with components.create_sql_session() as session: + for worker_id in flow_context: + worker_context = flow_context[worker_id] + insert_success = job_worker_mapper.insert_job_worker(session, worker_context) + if insert_success: + # insert + self.logger.info(f"## Save worker context successfully, job_id: {job_id}, worker_id: {worker_id}, work_context:\n{worker_context}") + continue + # worker already exist + worker_record = job_worker_mapper.query_job_worker(session,job_id, worker_id) + worker_context['status'] = worker_record.status + worker_context['args'] = json.loads(worker_record.args) + worker_context['upstreams'] = json.loads(worker_record.upstreams) + worker_context['inputs_statement'] = json.loads(worker_record.inputs_statement) + self.logger.info(f"Load worker context successfully, job_id: {job_id}, worker_id: {worker_id}, work_context:\n{worker_context}") + self.logger.info(f"## end save flow context, job_id: {job_id}") \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/common/codec.py b/python/ppc_scheduler/workflow/common/codec.py index 3fde280a..45fa3e7d 100644 --- a/python/ppc_scheduler/workflow/common/codec.py +++ b/python/ppc_scheduler/workflow/common/codec.py @@ -1,57 +1,19 @@ -from ppc_common.ppc_protos.generated.ppc_pb2 import JobWorkerOutputs, JobWorkerUpstreams, \ - InputStatement, JobWorkerInputsStatement -from ppc_common.ppc_utils import utils - +import json def deserialize_worker_outputs(outputs_str): - outputs = [] - outputs_pb = JobWorkerOutputs() - utils.str_to_pb(outputs_pb, outputs_str) - for output in outputs_pb.outputs: - outputs.append(output) - return outputs - - -def serialize_worker_outputs_for_db(outputs): - outputs_pb = JobWorkerOutputs() - for output in outputs: - outputs_pb.outputs.append(output) - return utils.pb_to_str(outputs_pb) + return json.loads(outputs_str) +def serialize_worker_outputs(outputs): + return json.dumps(outputs) def deserialize_upstreams(upstreams_str): - upstreams = [] - upstream_pb = JobWorkerUpstreams() - utils.str_to_pb(upstream_pb, upstreams_str) - for upstream in upstream_pb.upstreams: - upstreams.append(upstream) - return upstreams - - -def serialize_upstreams_for_db(upstreams): - upstreams_pb = JobWorkerUpstreams() - for upstream in upstreams: - upstreams_pb.upstreams.append(upstream) - return utils.pb_to_str(upstreams_pb) + return json.loads(upstreams_str) +def serialize_upstreams(upstreams): + return json.dumps(upstreams) def deserialize_inputs_statement(inputs_statement_str): - inputs_statement = [] - inputs_statement_pb = JobWorkerInputsStatement() - utils.str_to_pb(inputs_statement_pb, inputs_statement_str) - for input_statement_pb in inputs_statement_pb.inputs_statement: - inputs_statement.append({ - 'upstream': input_statement_pb.upstream, - 'output_index': input_statement_pb.output_index - }) - return inputs_statement - + return json.loads(inputs_statement_str) -def serialize_inputs_statement_for_db(inputs_statement): - inputs_statement_pb = JobWorkerInputsStatement() - for input_statement in inputs_statement: - input_statement_pb = InputStatement() - input_statement_pb.upstream = input_statement['upstream'] - input_statement_pb.output_index = input_statement['output_index'] - inputs_statement_pb.inputs_statement.append(input_statement_pb) - return utils.pb_to_str(inputs_statement_pb) +def serialize_inputs_statement(inputs_statement): + return json.dumps(inputs_statement) \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/common/flow_utils.py b/python/ppc_scheduler/workflow/common/flow_utils.py index 3d05e11e..e2486e4b 100644 --- a/python/ppc_scheduler/workflow/common/flow_utils.py +++ b/python/ppc_scheduler/workflow/common/flow_utils.py @@ -26,9 +26,9 @@ def to_worker_inputs(job_workers, inputs_statement): worker_inputs = [] for each in inputs_statement: output_index = each['output_index'] - upstream_unit = each['upstream_unit'] + upstream = each['upstream'] worker_inputs.append({ 'output_index': output_index, - 'upstream_outputs': job_workers[upstream_unit] + 'upstream_outputs': job_workers[upstream] }) return worker_inputs diff --git a/python/ppc_scheduler/workflow/common/job_context.py b/python/ppc_scheduler/workflow/common/job_context.py index 0a6c6feb..387af114 100644 --- a/python/ppc_scheduler/workflow/common/job_context.py +++ b/python/ppc_scheduler/workflow/common/job_context.py @@ -2,7 +2,6 @@ from ppc_scheduler.job.job_type import JobType from ppc_scheduler.mpc_generator.generator import CodeGenerator -from ppc_scheduler.workflow.common.default_flow_config import flow_dict from ppc_scheduler.common.global_context import components @@ -15,37 +14,13 @@ class JobContext: MPC_OUTPUT_FILE = "mpc_output.txt" HDFS_STORAGE_PATH = "/user/ppc/" - def __init__(self, args, workspace): - self.args = args - #todo: 确保java服务给过来的任务信息包含如下字段，如果是建模相关的任务，还需要job_context.model_config_dict = args['model_config'] - # 如果是mpc任务，还需要有args['sql']，或者args['mpc_content'] - self.job_id: str = args['job_id'] - self.user_name: str = args['user_name'] - self.dataset_id: str = args['dataset_id'] - - self.psi_fields: str = args['psi_fields'] - - self.result_receiver_list: list = args['result_receiver_list'] - self.participant_id_list: list = args['participant_id_list'] - self.job_type = args['job_type'] - self.dataset_record_count = args['dataset_record_count'] - - self.my_index = None - self.need_run_psi = False - self.need_run_fe = False - self.mpc_content = None - - self.model_config_dict: dict = {} - self.tag_provider_agency_id = None - self.job_subtype = None - self.predict_algorithm = None + def __init__(self, job_id, workspace): + self.job_id = job_id + self.workspace = workspace - self.worker_configs: list = [] self.workflow_view_path = 'workflow_view' - - self.workspace = workspace self.job_cache_dir = "{}{}{}".format(self.workspace, os.sep, self.job_id) - self.dataset_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.dataset_id) + # self.dataset_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.dataset_id) self.psi_prepare_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_PREPARE_FILE) self.psi_result_index_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_INDEX_FILE) self.psi_result_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_FILE) @@ -57,59 +32,43 @@ def __init__(self, args, workspace): self.mpc_output_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_OUTPUT_FILE) @staticmethod - def load_from_args(args, workspace): - job_context = JobContext(args, workspace) - job_context.my_index = job_context.participant_id_list.index(components.config_data['AGENCY_ID']) - - if job_context.job_type == JobType.PREPROCESSING or \ - job_context.job_type == JobType.TRAINING or \ - job_context.job_type == JobType.PREDICTION or \ - job_context.job_type == JobType.FEATURE_ENGINEERING: - - job_context.model_config_dict = args['model_config'] - job_context.tag_provider_agency_id = job_context.participant_id_list[0] - if 'job_subtype' in job_context.model_config_dict: - job_context.job_subtype = job_context.model_config_dict['job_subtype'] - if 'predict_algorithm' in job_context.model_config_dict: - job_context.predict_algorithm = job_context.model_config_dict['predict_algorithm'] - if 'use_psi' in job_context.model_config_dict: - job_context.need_run_psi = job_context.model_config_dict['use_psi'] == 1 - if 'use_iv' in job_context.model_config_dict: - job_context.need_run_fe = job_context.model_config_dict['use_iv'] == 1 - - if job_context.job_type == JobType.PSI: - job_context.worker_configs = flow_dict['PSI'] - elif job_context.job_type == JobType.MPC: - if 'sql' in job_context.args: - job_context.sql = args['sql'] - job_context.mpc_content = CodeGenerator(job_context.sql) - else: - job_context.mpc_content = job_context.args['mpc_content'] - if "PSI_OPTION=True" in job_context.mpc_content: - job_context.need_run_psi = True - job_context.worker_configs = flow_dict['PSI_MPC'] - else: - job_context.worker_configs = flow_dict['MPC'] - elif job_context.job_type == JobType.PREPROCESSING: - job_context.worker_configs = flow_dict['PREPROCESSING'] - elif job_context.job_type == JobType.FEATURE_ENGINEERING: - if job_context.need_run_psi: - job_context.worker_configs = flow_dict['PSI_FEATURE_ENGINEERING'] - else: - job_context.worker_configs = flow_dict['FEATURE_ENGINEERING'] - elif job_context.job_type == JobType.TRAINING: - if job_context.need_run_psi: - if job_context.need_run_fe: - job_context.worker_configs = flow_dict['PSI_FEATURE_ENGINEERING_TRAINING'] - else: - job_context.worker_configs = flow_dict['PSI_TRAINING'] - else: - if job_context.need_run_fe: - job_context.worker_configs = flow_dict['FEATURE_ENGINEERING_TRAINING'] - else: - job_context.worker_configs = flow_dict['TRAINING'] - elif job_context.job_type == JobType.PREDICTION: - job_context.worker_configs = flow_dict['PREDICTION'] - else: - raise Exception("Unsupported job type {}".format(job_context.job_type)) - return job_context + def create_job_context(request: object, workspace: str): + job_context = JobContext(request, workspace) + + """ + request format + { + "jobId": "job_id", + "agency": "WeBank", + "workflow": [ + { + "index": 1, + "type": "WorkerType1", + "args": [ + "arg1", + "arg2" + ] + }, + { + "index": 2, + "type": "WorkerType1", + "args": [ + "arg1", + "arg2" + ], + "upstreams": [ + { + "index": 1 + } + ] + } + ] + } + """ + + job_context.request = request + job_context.job_id = request['jobId'] + job_context.agency = request['agency'] + job_context.workflow_configs = request['workflow'] + + return job_context \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/common/worker_type.py b/python/ppc_scheduler/workflow/common/worker_type.py index d10f1f8f..54ccd245 100644 --- a/python/ppc_scheduler/workflow/common/worker_type.py +++ b/python/ppc_scheduler/workflow/common/worker_type.py @@ -1,16 +1,16 @@ class WorkerType: # generic job worker - T_API = 'T_API' - T_PYTHON = 'T_PYTHON' - T_SHELL = 'T_SHELL' + T_API = 'API' + T_PYTHON = 'PYTHON' + T_SHELL = 'SHELL' # specific job worker - T_PSI = 'T_PSI' - T_MPC = 'T_MPC' - T_PREPROCESSING = 'T_PREPROCESSING' - T_FEATURE_ENGINEERING = 'T_FEATURE_ENGINEERING' - T_TRAINING = 'T_TRAINING' - T_PREDICTION = 'T_PREDICTION' + T_PSI = 'PSI' + T_MPC = 'MPC' + T_PREPROCESSING = 'PREPROCESSING' + T_FEATURE_ENGINEERING = 'FEATURE_ENGINEERING' + T_TRAINING = 'XGB_TRAINING' + T_PREDICTION = 'XGB_PREDICTING' # finish job T_ON_SUCCESS = 'T_ON_SUCCESS' diff --git a/python/ppc_scheduler/workflow/constructor.py b/python/ppc_scheduler/workflow/constructor.py deleted file mode 100644 index ca6cf22b..00000000 --- a/python/ppc_scheduler/workflow/constructor.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -from ppc_common.db_models.job_worker_record import JobWorkerRecord -from ppc_common.ppc_utils import utils -from ppc_scheduler.common.global_context import components -from ppc_scheduler.database import job_worker_mapper -from ppc_scheduler.workflow.common import codec, flow_utils -from ppc_scheduler.workflow.common.job_context import JobContext -from ppc_scheduler.workflow.common.worker_status import WorkerStatus - - -class Constructor: - def __init__(self): - self.log = components.logger() - - def build_flow_context(self, job_context: JobContext): - self.log.info(f"start build_flow_context, job_id: {job_context.job_id}") - job_id = job_context.job_id - flow_context = {} - index_type_map = {} - for worker_config in job_context.worker_configs: - index_type_map[worker_config['index']] = worker_config['type'] - - for worker_config in job_context.worker_configs: - worker_type = worker_config['type'] - worker_id = flow_utils.cat_worker_id(job_id, worker_config['index'], worker_type) - upstreams = [] - inputs_statement = [] - inputs_statement_tuple = [] - if 'upstream' in worker_config: - for upstream_config in worker_config["upstreams"]: - index = upstream_config['index'] - upstream_id = flow_utils.cat_worker_id(job_id, index, index_type_map[index]) - upstreams.append(upstream_id), - if 'output_input_map' in upstream_config: - for mapping in upstream_config.get("output_input_map", []): - output_index, input_index = mapping.split(":") - inputs_statement_tuple.append((upstream_id, int(output_index), int(input_index))) - - inputs_statement_tuple.sort(key=lambda x: x[2]) - for upstream_id, output_index, _ in inputs_statement_tuple: - inputs_statement.append( - { - 'output_index': output_index, - 'upstream': upstream_id - } - ) - worker_context = self._construct_context(job_context, worker_id, worker_type, - upstreams, inputs_statement) - flow_context[worker_id] = worker_context - self.log.info(f"end build_flow_context, flow_context:\n{flow_context}") - - def _construct_context(self, job_context, worker_id, worker_type, upstreams, inputs_statement): - context = { - 'type': worker_type, - 'status': WorkerStatus.PENDING, - 'upstreams': upstreams, - 'inputs_statement': inputs_statement - } - - with components.create_sql_session() as session: - worker_record = job_worker_mapper.query_job_worker(components.create_sql_session, - job_context.job_id, worker_id) - if worker_record is None: - worker_record = JobWorkerRecord( - worker_id=worker_id, - job_id=job_context.job_id, - type=worker_type, - status=WorkerStatus.PENDING, - upstreams=codec.serialize_upstreams_for_db(upstreams), - inputs_statement=codec.serialize_inputs_statement_for_db(inputs_statement), - create_time=utils.make_timestamp(), - update_time=utils.make_timestamp() - ) - session.add(worker_record) - session.commit() - else: - context['status'] = worker_record.status - context['upstreams'] = codec.deserialize_upstreams(worker_record.upstreams) - context['inputs_statement'] = codec.deserialize_inputs_statement(worker_record.inputs_statement) - - self.log.debug(f"Load worker_context successfully, worker_id: {worker_id}, context:\n{context}") - return context diff --git a/python/ppc_scheduler/workflow/scheduler/__init__.py b/python/ppc_scheduler/workflow/scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_scheduler/workflow/scheduler.py b/python/ppc_scheduler/workflow/scheduler/scheduler.py similarity index 68% rename from python/ppc_scheduler/workflow/scheduler.py rename to python/ppc_scheduler/workflow/scheduler/scheduler.py index ed2e1fc4..82f6262d 100644 --- a/python/ppc_scheduler/workflow/scheduler.py +++ b/python/ppc_scheduler/workflow/scheduler/scheduler.py @@ -1,3 +1,4 @@ +import time from prefect import Flow from prefect.executors import LocalDaskExecutor from prefect.triggers import all_successful, any_failed @@ -6,31 +7,27 @@ from ppc_scheduler.workflow.common.job_context import JobContext from ppc_scheduler.workflow.common.worker_status import WorkerStatus from ppc_scheduler.workflow.common.worker_type import WorkerType -from ppc_scheduler.workflow.constructor import Constructor +from ppc_scheduler.workflow.builder.flow_builder import FlowBuilder from ppc_scheduler.workflow.worker.worker_factory import WorkerFactory +from ppc_scheduler.workflow.scheduler.scheduler_api import SchedulerApi -class Scheduler: - def __init__(self, workspace): +class Scheduler(SchedulerApi): + def __init__(self, workspace, logger): self.workspace = workspace - self.constructor = Constructor() + self.logger = logger - def schedule_job_flow(self, args): - job_context = JobContext.load_from_args(args, self.workspace) - flow_context = self.constructor.build_flow_context(job_context) - self._run(job_context, flow_context) - - @staticmethod - def _run(job_context, flow_context): - job_workers = {} + def run(self, job_context: JobContext, flow_context: dict): + job_id = job_context.job_id job_flow = Flow(f"job_flow_{job_id}") - # create a final job worker to handle success finish_job_on_success = WorkerFactory.build_worker( job_context, flow_utils.success_id(job_id), - WorkerType.T_ON_SUCCESS) + WorkerType.T_ON_SUCCESS, + None + ) finish_job_on_success.trigger = all_successful finish_job_on_success.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow) @@ -43,17 +40,22 @@ def _run(job_context, flow_context): finish_job_on_failure = WorkerFactory.build_worker( job_context, flow_utils.failure_id(job_id), - WorkerType.T_ON_FAILURE) + WorkerType.T_ON_FAILURE, + None + ) # do finish_job_on_failure while any job worker failed finish_job_on_failure.trigger = any_failed finish_job_on_failure.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow) job_flow.add_task(finish_job_on_failure) - + + job_workers = {} # create main job workers for worker_id in flow_context: worker_type = flow_context[worker_id]['type'] - job_worker = WorkerFactory.build_worker(job_context, worker_id, worker_type) + worker_args = flow_context[worker_id]['args'] + + job_worker = WorkerFactory.build_worker(job_context=job_context, worker_id=worker_id, worker_type=worker_type, worker_args=worker_args) job_flow.add_task(job_worker) job_workers[worker_id] = job_worker @@ -65,23 +67,34 @@ def _run(job_context, flow_context): for worker_id in flow_context: # set upstream upstreams = flow_context[worker_id]['upstreams'] + status = flow_context[worker_id]['status'] for upstream in upstreams: if upstream not in job_workers: - raise Exception(-1, f"upstream job worker not found: {upstream}, " + raise Exception(-1, f"Not found upstream job worker : {upstream}, " f"job_id: {job_context.job_id}") job_workers[worker_id].set_upstream(job_workers[upstream], flow=job_flow) - + # bind worker inputs inputs_statement = flow_context[worker_id]['inputs_statement'] worker_inputs = flow_utils.to_worker_inputs(job_workers, inputs_statement) - job_workers[worker_id].bind(worker_status=flow_context[worker_id]['status'], + job_workers[worker_id].bind(worker_status=status, worker_inputs=worker_inputs, flow=job_flow) # enable parallel execution job_flow.executor = LocalDaskExecutor() + # + start_time = time.time() + # run dag workflow job_flow_state = job_flow.run() + + end_time = time.time() + + self.logger.info(f" ## Job worker result, job: {job_id}, success: {job_flow_state.is_successful()}, costs: {end_time - start_time}, flow_state: {job_flow_state}") # save workflow view as file job_flow.visualize(job_flow_state, job_context.workflow_view_path, 'svg') + + if not job_flow_state.is_successful(): + raise Exception(-1, f"Job run failed, job_id: {job_id}") diff --git a/python/ppc_scheduler/workflow/scheduler/scheduler_api.py b/python/ppc_scheduler/workflow/scheduler/scheduler_api.py new file mode 100644 index 00000000..9ef5a25f --- /dev/null +++ b/python/ppc_scheduler/workflow/scheduler/scheduler_api.py @@ -0,0 +1,8 @@ +from abc import ABC, abstractmethod + +from ppc_scheduler.workflow.common.job_context import JobContext + +class SchedulerApi(ABC): + @abstractmethod + def run(self, job_context: JobContext, flow_context: dict): + pass \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/worker/api_worker.py b/python/ppc_scheduler/workflow/worker/api_worker.py index 2698133b..0160caf5 100644 --- a/python/ppc_scheduler/workflow/worker/api_worker.py +++ b/python/ppc_scheduler/workflow/worker/api_worker.py @@ -3,8 +3,8 @@ class ApiWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs): ... diff --git a/python/ppc_scheduler/workflow/worker/default_worker.py b/python/ppc_scheduler/workflow/worker/default_worker.py new file mode 100644 index 00000000..02d2cf5f --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/default_worker.py @@ -0,0 +1,37 @@ +import os +import time + +from ppc_common.ppc_utils import utils +from ppc_scheduler.common import log_utils +from ppc_scheduler.workflow.common.worker_type import WorkerType +from ppc_scheduler.workflow.worker.worker import Worker + + +class DefaultWorker(Worker): + + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) + + def engine_run(self, worker_inputs): + # log_utils.upload_job_log(self.components.storage_client, self.job_context.job_id) + # self._save_workflow_view_file() + if self.worker_type == WorkerType.T_ON_FAILURE: + # notice job manager that this job has failed + raise Exception("Job run failed, job_id: " + self.job_context.job_id) + + # TODO: + # def _save_workflow_view_file(self): + # file = f"{self.job_context.workflow_view_path}.svg" + + # if not utils.file_exists(file): + # return + + # try_count = 10 + # while try_count > 0: + # if utils.file_exists(file): + # break + # time.sleep(1) + # try_count -= 1 + # self.components.storage_client.upload_file(file, + # self.job_context.job_id + os.sep + + # self.job_context.workflow_view_path) diff --git a/python/ppc_scheduler/workflow/worker/engine/model_engine.py b/python/ppc_scheduler/workflow/worker/engine/model_engine.py index 2542c611..68084783 100644 --- a/python/ppc_scheduler/workflow/worker/engine/model_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/model_engine.py @@ -3,99 +3,51 @@ from ppc_scheduler.workflow.common.job_context import JobContext from ppc_scheduler.workflow.common.worker_type import WorkerType +from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine - -class ModelWorkerEngine: - def __init__(self, model_client, worker_type, components, job_context: JobContext): +class ModelWorkerEngine(WorkerEngine): + def __init__(self, model_client, worker_type, worker_id, components, job_context: JobContext): self.model_client = model_client self.worker_type = worker_type + self.worker_id = worker_id self.components = components self.job_context = job_context - self.log = self.components.logger() - - def run(self) -> list: - if self.worker_type == WorkerType.T_PREPROCESSING: - self._run_preprocessing() - elif self.worker_type == WorkerType.T_FEATURE_ENGINEERING: - self._run_feature_engineering() - elif self.worker_type == WorkerType.T_TRAINING: - self._run_training() - elif self.worker_type == WorkerType.T_PREDICTION: - self._run_prediction() + self.logger = self.components.logger() + + def run(self, *args) -> list: + if self.worker_type == WorkerType.T_PREPROCESSING\ + or self.worker_type == WorkerType.T_FEATURE_ENGINEERING\ + or self.worker_type == WorkerType.T_TRAINING\ + or self.worker_type == WorkerType.T_PREDICTION: + pass else: raise ValueError(f"Unsupported worker type: {self.worker_type}") - return [] - - def _run_preprocessing(self): - start = time.time() - job_id = self.job_context.job_id - task_id = job_id + '_d' - user_name = self.job_context.user_name - dataset_storage_path = os.path.join(user_name, self.job_context.dataset_id) - args = { - 'job_id': job_id, - 'task_id': task_id, - 'task_type': 'PREPROCESSING', - 'dataset_id': self.job_context.dataset_id, - 'dataset_storage_path': dataset_storage_path, - 'job_algorithm_type': self.job_context.job_type, - 'need_run_psi': self.job_context.need_run_psi, - 'model_dict': self.job_context.model_config_dict - } - self.log.info(f"start prepare_xgb, job_id: {job_id}, task_id: {task_id}, args: {args}") - self.model_client.run(args) - self.log.info( - f"call compute_xgb_job service success, job: {job_id}, " - f"task_id: {task_id}, timecost: {time.time() - start}") - - def _run_feature_engineering(self): - start = time.time() + job_id = self.job_context.job_id - task_id = job_id + '_f' - args = { - 'job_id': job_id, - 'task_id': task_id, - 'task_type': 'FEATURE_ENGINEERING', - 'is_label_holder': self.job_context.tag_provider_agency_id == self.components.config_data['AGENCY_ID'], - 'result_receiver_id_list': self.job_context.result_receiver_list, - 'participant_id_list': self.job_context.participant_id_list, - 'model_dict': self.job_context.model_config_dict - } - self.log.info(f"start feature_engineering, job_id: {job_id}, task_id: {task_id}, args: {args}") - self.model_client.run(args) - self.log.info( - f"call compute_xgb_job service success, job: {job_id}, " - f"task_id: {task_id}, timecost: {time.time() - start}") - - def _run_training(self): - # todo 支持LR - task_id = self.job_context.job_id + '_t' - task_type = 'XGB_TRAINING' - xgb_predict_algorithm = '' - self._run_model(task_id, task_type, xgb_predict_algorithm) - - def _run_prediction(self): - # todo 支持LR - task_id = self.job_context.job_id + '_p' - task_type = 'XGB_PREDICTING' - xgb_predict_algorithm = self.job_context.predict_algorithm - self._run_model(task_id, task_type, xgb_predict_algorithm) - - def _run_model(self, task_id, task_type, model_algorithm): - job_id = self.job_context.job_id - args = { - "job_id": job_id, - 'task_id': task_id, - 'task_type': task_type, - 'is_label_holder': self.job_context.tag_provider_agency_id == self.components.config_data['AGENCY_ID'], - 'result_receiver_id_list': self.job_context.result_receiver_list, - 'participant_id_list': self.job_context.participant_id_list, - 'model_predict_algorithm': model_algorithm, - "algorithm_type": self.job_context.job_type, - "algorithm_subtype": self.job_context.job_subtype, - "model_dict": self.job_context.model_config_dict - } - self.log.info(f"start run xgb task, job_id, job: {job_id}, " - f"task_id: {task_id}, task_type: {task_type}, args: {args}") - self.model_client.run(args) - self.log.info(f"call compute_xgb_job service success, job: {job_id}") + start_time = time.time() + + self.logger.info(f"## model engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}") + + # send job request to model node and wait for the job to finish + # self.psi_client.run(*args) + + time_costs = time.time() - start_time + self.logger.info(f"## model engine run finished, job_id={job_id}, timecost: {time_costs}s") + + # args = { + # 'job_id': job_id, + # 'task_id': task_id, + # 'task_type': 'PREPROCESSING', + # 'dataset_id': self.job_context.dataset_id, + # 'dataset_storage_path': dataset_storage_path, + # 'job_algorithm_type': self.job_context.job_type, + # 'need_run_psi': self.job_context.need_run_psi, + # 'model_dict': self.job_context.model_config_dict + # } + # self.log.info(f"start prepare_xgb, job_id: {job_id}, task_id: {task_id}, args: {args}") + # self.model_client.run(args) + # self.log.info( + # f"call compute_xgb_job service success, job: {job_id}, " + # f"task_id: {task_id}, timecost: {time.time() - start}") + + return [] diff --git a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py index 5011e970..f55d091e 100644 --- a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py @@ -1,285 +1,31 @@ # -*- coding: utf-8 -*- import os +from time import time import pandas as pd from ppc_common.ppc_utils import utils from ppc_scheduler.mpc_generator.generator import CodeGenerator from ppc_scheduler.workflow.common.job_context import JobContext +from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine -class MpcWorkerEngine: +class MpcWorkerEngine(WorkerEngine): def __init__(self, mpc_client, worker_type, components, job_context: JobContext): self.mpc_client = mpc_client self.worker_type = worker_type self.components = components self.job_context = job_context - self.log = self.components.logger() + self.logger = self.components.logger() - def run(self) -> list: - bit_length = self._prepare_mpc_file() - if self.job_context.need_run_psi: - self._prepare_mpc_after_psi() - else: - self._prepare_mpc_without_psi() - self._run_mpc_job(bit_length) - self._finish_mpc_job() - return [self.job_context.mpc_output_path] - - def _prepare_mpc_file(self): - # if self.job_context.sql is not None: - # # compile sql to mpc content - # mpc_content = CodeGenerator(self.job_context.sql) - # else: - # mpc_content = self.job_context.mpc_content - utils.write_content_to_file(self.job_context.mpc_content, self.job_context.mpc_file_path) - - self.components.storage_client.upload_file(self.job_context.mpc_file_path, - self.job_context.job_id + os.sep + self.job_context.mpc_file_name) - return self._get_share_bytes_length(self.job_context.mpc_content) - - def _run_mpc_job(self, bit_length): + def run(self, *args) -> list: + job_id = self.job_context.job_id - mpc_record = 0 - self.log.info(f"start compute_mpc_job, job id: {job_id}") - if self.job_context.mpc_prepare_path: - mpc_record = sum(1 for _ in open(self.job_context.mpc_prepare_path)) - self.log.info(f"compute_mpc_job, mpc record: {mpc_record}") - utils.replace(self.job_context.mpc_file_path, mpc_record) - self._replace_mpc_field_holder() - self.components.storage_client.upload_file(self.job_context.mpc_file_path, - job_id + os.sep + self.job_context.mpc_file_name) - job_info = { - "jobId": job_id, - "mpcNodeUseGateway": False, - "receiverNodeIp": "", - "mpcNodeDirectPort": self.components.config_data["MPC_NODE_DIRECT_PORT"], - "participantCount": len(self.job_context.participant_id_list), - "selfIndex": self.job_context.my_index, - "isMalicious": self.components.config_data["IS_MALICIOUS"], - "bitLength": bit_length, - "inputFileName": "{}-P{}-0".format(JobContext.MPC_PREPARE_FILE, self.job_context.my_index), - "outputFileName": JobContext.MPC_OUTPUT_FILE - } - - self.log.info(f"call compute_mpc_job service, model run, params: {job_info}") - self.mpc_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) - self.components.storage_client.download_file(job_id + os.sep + JobContext.MPC_OUTPUT_FILE, - self.job_context.mpc_output_path) - self.log.info(f"call compute_mpc_job service success") - - def _finish_mpc_job(self): - job_id = self.job_context.job_id - index_file = None - if self.components.config_data['AGENCY_ID'] in self.job_context.result_receiver_list: - if self.job_context.need_run_psi: - if not utils.file_exists(self.job_context.psi_result_path): - self.log.info( - f"download finish_mpc_job psi_result_path, job_id={job_id}, " - f"download {job_id + os.sep + JobContext.PSI_RESULT_FILE}") - self.components.storage_client.download_file(job_id + os.sep + JobContext.PSI_RESULT_FILE, - self.job_context.psi_result_path) - self._order_psi_csv() - index_file = self.job_context.psi_result_path - if not utils.file_exists(self.job_context.mpc_output_path): - self.log.info( - f"download finish_mpc_job mpc_output_path, job_id={job_id}, " - f"download {job_id + os.sep + JobContext.MPC_OUTPUT_FILE}") - self.components.storage_client.download_file(job_id + os.sep + JobContext.MPC_OUTPUT_FILE, - self.job_context.mpc_output_path) - - self.log.info( - f"finish_mpc_job mpc_output_path, job_id={job_id}") - self._parse_and_write_final_mpc_result(self.job_context.mpc_output_path, index_file) - self.components.storage_client.upload_file(self.job_context.mpc_result_path, - job_id + os.sep + JobContext.MPC_RESULT_FILE) - self.log.info(f"finish_mpc_job success, job_id={job_id}") - - def _get_share_bytes_length(self, algorithm_content): - target = '# BIT_LENGTH = ' - if target in algorithm_content: - start = algorithm_content.find(target) - end = algorithm_content.find('\n', start + len(target)) - bit_length = int(algorithm_content[start + len(target): end].strip()) - self.log.info(f"OUTPUT_BIT_LENGTH = {bit_length}") - return bit_length - else: - self.log.info(f"OUTPUT_BIT_LENGTH = 64") - return 64 - - def _get_dataset_column_count(self): - with open(self.job_context.mpc_file_path, "r") as file: - mpc_str = file.read() - - lines = mpc_str.split('\n') - for line in lines: - if f"source{self.job_context.my_index}_column_count =" in line or \ - f"source{self.job_context.my_index}_column_count=" in line: - index = line.find('=') - return int(line[index + 1:].strip('\n').strip()) - - def _make_dataset_to_mpc_data_plus_psi_data(self, my_dataset_number): - chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, - chunksize=self.components.file_chunk_config.read_chunk_size) - psi_data = pd.read_csv(self.job_context.psi_result_path, delimiter=utils.CSV_SEP) - for chunk in chunk_list: - self._make_dataset_field_normalized(chunk) - mpc_data_df = pd.merge(chunk, psi_data, on=['id']).sort_values( - by='id', ascending=True) - self._save_selected_column_data( - my_dataset_number, mpc_data_df, self.job_context.mpc_prepare_path) - - @staticmethod - def _make_dataset_field_normalized(dataset_df): - data_field = dataset_df.columns.values - if 'id' in data_field: - data_field_normalized_names = ['id'] - size = len(data_field) - for i in range(size - 1): - data_field_normalized_names.append( - utils.NORMALIZED_NAMES.format(i)) - else: - data_field_normalized_names = [] - size = len(data_field) - for i in range(size): - data_field_normalized_names.append( - utils.NORMALIZED_NAMES.format(i)) - dataset_df.columns = data_field_normalized_names - - @staticmethod - def _save_selected_column_data(my_dataset_number, data_df, mpc_prepare_path): - column_list = [] - for i in range(0, int(my_dataset_number)): - column_list.append(utils.NORMALIZED_NAMES.format(i)) - result_new = pd.DataFrame(data_df, columns=column_list) - # sep must be space (ppc-mpc inputs) - result_new.to_csv(mpc_prepare_path, sep=' ', mode='a', header=False, index=None) - - def _prepare_mpc_after_psi(self): - job_id = self.job_context.job_id - self.log.info(f"start prepare_mpc_after_psi, job_id={job_id}") - my_dataset_number = self._get_dataset_column_count() - - # TODO: 下载数据 - # dataset_helper_factory.download_dataset( - # dataset_helper_factory=None, - # dataset_user=self.job_context.user_name, - # dataset_id=self.job_context.dataset_id, - # dataset_local_path=self.job_context.dataset_file_path, - # log_keyword="prepare_mpc_after_psi", - # logger=self.log) - - if not utils.file_exists(self.job_context.psi_result_path): - self.log.info( - f"prepare_mpc_after_psi, download psi_result_path ,job_id={job_id}, " - f"download {job_id + os.sep + JobContext.PSI_RESULT_FILE}") - - self.components.storage_client.download_file(job_id + os.sep + JobContext.PSI_RESULT_FILE, - self.job_context.psi_result_path) - - self._make_dataset_to_mpc_data_plus_psi_data(my_dataset_number) - - hdfs_mpc_prepare_path = "{}-P{}-0".format(job_id + os.sep + JobContext.MPC_PREPARE_FILE, - self.job_context.my_index) - self.components.storage_client.upload_file(self.job_context.mpc_prepare_path, hdfs_mpc_prepare_path) - self.log.info(f"call prepare_mpc_after_psi success: job_id={job_id}") - - def _make_dataset_to_mpc_data_direct(self, my_dataset_number): - chunk_list = pd.read_csv(self.job_context.dataset_file_path, delimiter=utils.CSV_SEP, - chunksize=self.components.file_chunk_config.read_chunk_size) - for chunk in chunk_list: - self._make_dataset_field_normalized(chunk) - self._save_selected_column_data(my_dataset_number, chunk, self.job_context.mpc_prepare_path) - - def _prepare_mpc_without_psi(self): - job_id = self.job_context.job_id - self.log.info(f"start prepare_mpc_without_psi, job_id={job_id}") - my_dataset_number = self._get_dataset_column_count() - # TODO: 下载数据 - # dataset_helper_factory.download_dataset( - # dataset_helper_factory=None, - # dataset_user=self.job_context.user_name, - # dataset_id=self.job_context.dataset_id, - # dataset_local_path=self.job_context.dataset_file_path, - # log_keyword="prepare_mpc_without_psi", - # logger=self.log) - - self._make_dataset_to_mpc_data_direct(my_dataset_number) - - hdfs_mpc_prepare_path = "{}-P{}-0".format(job_id + os.sep + JobContext.MPC_PREPARE_FILE, - self.job_context.my_index) - self.components.storage_client.upload_file(self.job_context.mpc_prepare_path, hdfs_mpc_prepare_path) - self.log.info(f"call prepare_mpc_without_psi success: job_id={job_id}") - - def _order_psi_csv(self): - """ - order_psi_csv - """ - data = pd.read_csv(self.job_context.psi_result_path, delimiter=utils.CSV_SEP) - data.sort_values(by="id").to_csv(self.job_context.psi_result_path, - sep=utils.CSV_SEP, header=True, index=None) - - def _parse_and_write_final_mpc_result(self, mpc_output_path, index_file): - self.log.info("run parse_and_write_final_mpc_result") - final_result_fields = 'id' - need_add_fields = True - column_count = 0 - for row_data in open(mpc_output_path): - if row_data.__contains__(utils.PPC_RESULT_FIELDS_FLAG): - need_add_fields = False - final_result_fields += ',' + \ - row_data[row_data.find( - '=') + 1:].strip().replace(utils.BLANK_SEP, utils.CSV_SEP) - elif row_data.__contains__(utils.PPC_RESULT_VALUES_FLAG): - column_count = len(row_data.split( - '=')[1].strip().split(utils.BLANK_SEP)) - break - - if need_add_fields: - for i in range(column_count): - final_result_fields += ',' + 'result' + str(i) - - id_list = [] - if self.job_context.need_run_psi: - df = pd.read_csv(index_file, delimiter=utils.CSV_SEP) - for result_id in df["id"]: - id_list.append(result_id) - - with open(self.job_context.mpc_result_path, "w", encoding='utf-8') as file: - file.write(final_result_fields + '\n') - row_count = 0 - for row_data in open(mpc_output_path): - if row_data.__contains__(utils.PPC_RESULT_VALUES_FLAG): - values = row_data.split('=')[1].strip().split(utils.BLANK_SEP) - if self.job_context.need_run_psi: - if row_count >= len(id_list): - row = str(id_list[-1] + row_count - len(id_list) + 1) - else: - row = str(id_list[row_count]) - else: - row = str(row_count) - - for value in values: - try: - row += (',' + value) - except: - row += (',%s' % value) - file.write(row + '\n') - row_count += 1 - file.close() - - self.log.info("finish parse_and_write_final_mpc_result") - - def _replace_mpc_field_holder(self): - party_count = len(self.job_context.participant_id_list) - if self.job_context.need_run_psi: - dataset_record_count = 0 - if self.job_context.mpc_prepare_path: - dataset_record_count = sum(1 for _ in open(self.job_context.mpc_prepare_path)) - for i in range(party_count): - utils.replace(self.job_context.mpc_file_path, - dataset_record_count, f'$(source{i}_record_count)') - else: - for i in range(party_count): - utils.replace(self.job_context.mpc_file_path, - self.job_context.dataset_record_count, f'$(source{i}_record_count)') + start_time = time.time() + + self.logger.info(f"computing mpc begin, job_id={job_id}") + + time_costs = time.time() - start_time + self.logger.info(f"computing mpc finished, job_id={job_id}, timecost: {time_costs}s") + + return [self.job_context.mpc_output_path] \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py index cefd49dc..b6e735cc 100644 --- a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py @@ -1,158 +1,31 @@ # -*- coding: utf-8 -*- -import codecs import os import time from ppc_common.ppc_utils import utils, common_func from ppc_scheduler.workflow.common.job_context import JobContext +from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine -class PsiWorkerEngine: - def __init__(self, psi_client, worker_type, components, job_context: JobContext): +class PsiWorkerEngine(WorkerEngine): + def __init__(self, psi_client, worker_id, worker_type, components, job_context: JobContext): self.psi_client = psi_client + self.worker_id = worker_id self.worker_type = worker_type self.components = components self.job_context = job_context - self.log = self.components.logger() + self.logger = self.components.logger() - def run(self) -> list: + def run(self, *args) -> list: job_id = self.job_context.job_id start_time = time.time() - self.origin_dataset_to_psi_inputs() - self.log.info(f"compute two party psi, job_id={job_id}") - if len(self.job_context.participant_id_list) == 2: - self._run_two_party_psi() - else: - self._run_multi_party_psi() + self.logger.info(f"## psi engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}") + + # send job request to psi node and wait for the job to finish + self.psi_client.run(*args) + time_costs = time.time() - start_time - self.log.info(f"computing psi finished, job_id={job_id}, timecost: {time_costs}s") - return [JobContext.HDFS_STORAGE_PATH + job_id + os.sep + self.job_context.PSI_RESULT_INDEX_FILE] - - def _run_two_party_psi(self): - job_id = self.job_context.job_id - agency_id = self.components.config_data['AGENCY_ID'] - job_info = { - "taskID": job_id, - "type": 0, - "algorithm": 0, - "syncResult": True, - "lowBandwidth": False, - "parties": [ - { - "id": self.job_context.participant_id_list[1 - self.job_context.my_index], - "partyIndex": 1 - self.job_context.my_index - }, - { - "id": agency_id, - "partyIndex": self.job_context.my_index, - "data": - { - "id": self.job_context.job_id, - "input": { - "type": 2, - "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_PREPARE_FILE - }, - "output": { - "type": 2, - "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_RESULT_FILE - } - } - } - ] - } - psi_result = self.psi_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) - self.log.info(f"call psi service successfully, job_id={job_id}, result: {psi_result}") - - def _run_multi_party_psi(self): - job_id = self.job_context.job_id - participant_number = len(self.job_context.participant_id_list) - participant_list = [] - # parties_index = [0, 1 ··· 1, 2] - parties_index = participant_number * [1] # role: partner - parties_index[0] = 0 # role: calculator - parties_index[-1] = 2 # role: master - for index, agency_id in enumerate(self.job_context.participant_id_list): - party_map = {} - if self.job_context.my_index == index: - party_map["id"] = agency_id - party_map["partyIndex"] = parties_index[index] - party_map["data"] = { - "id": self.job_context.job_id, - "input": { - "type": 2, - "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_PREPARE_FILE - }, - "output": { - "type": 2, - "path": self.job_context.HDFS_STORAGE_PATH + job_id + os.sep + JobContext.PSI_RESULT_FILE - } - } - else: - party_map["id"] = agency_id - party_map["partyIndex"] = parties_index[index] - participant_list.append(party_map) - - job_info = { - "taskID": job_id, - "type": 0, - "algorithm": 4, - "syncResult": True, - "receiverList": self.job_context.result_receiver_list, - "parties": participant_list - } - psi_result = self.psi_client.run(job_info, self.components.config_data['PPCS_RPC_TOKEN']) - self.log.info(f"call psi service successfully, job_id={job_id}, result: {psi_result}") - - def origin_dataset_to_psi_inputs(self): - # TODO: 下载数据 - # dataset_helper_factory.download_dataset( - # dataset_helper_factory=None, - # dataset_user=self.job_context.user_name, - # dataset_id=self.job_context.dataset_id, - # dataset_local_path=self.job_context.dataset_file_path, - # log_keyword="prepare_dataset", - # logger=self.log) - - field = (self.job_context.psi_fields.split(utils.CSV_SEP)[self.job_context.my_index]).lower() - if field == '': - field = 'id' - prepare_file = open(self.job_context.psi_prepare_path, 'w') - psi_split_reg = "===" - file_encoding = common_func.get_file_encoding(self.job_context.dataset_file_path) - if psi_split_reg in field: - field_multi = field.split(psi_split_reg) - with codecs.open(self.job_context.dataset_file_path, "r", file_encoding) as dataset: - fields = next(dataset).lower() - fields_list = fields.strip().split(utils.CSV_SEP) - id_idx_list = [] - for filed_idx in field_multi: - id_idx_list.append(fields_list.index(filed_idx)) - for line in dataset: - if line.strip() == "": - continue - final_str = "" - for id_idx_multi in id_idx_list: - if len(line.strip().split(utils.CSV_SEP, id_idx_multi + 1)) < id_idx_multi + 1: - continue - final_str = "{}-{}".format(final_str, line.strip().split( - utils.CSV_SEP, id_idx_multi + 1)[id_idx_multi]).strip("\r\n") - print(final_str, file=prepare_file) - prepare_file.close() - else: - with codecs.open(self.job_context.dataset_file_path, "r", file_encoding) as dataset: - # ignore lower/upper case - fields = next(dataset).lower() - fields_list = fields.strip().split(utils.CSV_SEP) - id_idx = fields_list.index(field) - for line in dataset: - if line.strip() == "": - continue - if len(line.strip().split(utils.CSV_SEP, id_idx + 1)) < id_idx + 1: - continue - print(line.strip().split(utils.CSV_SEP, id_idx + 1) - [id_idx], file=prepare_file) - prepare_file.close() - self.components.storage_client.upload_file(self.job_context.psi_prepare_path, - self.job_context.job_id + os.sep + JobContext.PSI_PREPARE_FILE) - utils.delete_file(self.job_context.psi_prepare_path) + self.logger.info(f"## psi engine run finished, job_id={job_id}, timecost: {time_costs}s") + + return [JobContext.HDFS_STORAGE_PATH + job_id + os.sep + self.job_context.PSI_RESULT_INDEX_FILE] \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/worker/engine/shell_engine.py b/python/ppc_scheduler/workflow/worker/engine/shell_engine.py new file mode 100644 index 00000000..a406f8a7 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/engine/shell_engine.py @@ -0,0 +1,16 @@ +import os +from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine +from ppc_scheduler.common.global_context import components + +class ShellEngine(WorkerEngine): + + def __init__(self, cmd: str) -> None: + super().__init__() + self.cmd = cmd + + def run(self, *args): + # print("shell engine is processing.") + components.logger().info(f"shell engine is processing, cmd: {self.cmd}") + + result = os.system(self.cmd) + return list(str(result)) \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/worker/engine/work_engine.py b/python/ppc_scheduler/workflow/worker/engine/work_engine.py new file mode 100644 index 00000000..6cff7e23 --- /dev/null +++ b/python/ppc_scheduler/workflow/worker/engine/work_engine.py @@ -0,0 +1,6 @@ +from abc import ABC, abstractmethod + +class WorkerEngine(ABC): + @abstractmethod + def run(self, *args): + pass \ No newline at end of file diff --git a/python/ppc_scheduler/workflow/worker/exit_worker.py b/python/ppc_scheduler/workflow/worker/exit_worker.py deleted file mode 100644 index 9cff596b..00000000 --- a/python/ppc_scheduler/workflow/worker/exit_worker.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import time - -from ppc_common.ppc_utils import utils -from ppc_scheduler.common import log_utils -from ppc_scheduler.workflow.common.worker_type import WorkerType -from ppc_scheduler.workflow.worker.worker import Worker - - -class ExitWorker(Worker): - - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) - - def engine_run(self, worker_inputs): - log_utils.upload_job_log(self.components.storage_client, self.job_context.job_id) - self._save_workflow_view_file() - if self.worker_type == WorkerType.T_ON_FAILURE: - # notice job manager that this job has failed - raise Exception() - - def _save_workflow_view_file(self): - file = f"{self.job_context.workflow_view_path}.svg" - try_count = 10 - while try_count > 0: - if utils.file_exists(file): - break - time.sleep(1) - try_count -= 1 - - self.components.storage_client.upload_file(file, - self.job_context.job_id + os.sep + - self.job_context.workflow_view_path) diff --git a/python/ppc_scheduler/workflow/worker/model_worker.py b/python/ppc_scheduler/workflow/worker/model_worker.py index e843bd08..6be85fe4 100644 --- a/python/ppc_scheduler/workflow/worker/model_worker.py +++ b/python/ppc_scheduler/workflow/worker/model_worker.py @@ -1,19 +1,19 @@ -from ppc_scheduler.node.computing_node_client import ModelClient +from ppc_scheduler.node.computing_node_client.model_node_client import ModelClient from ppc_scheduler.workflow.worker.engine.model_engine import ModelWorkerEngine from ppc_scheduler.workflow.worker.worker import Worker class ModelWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs): node_endpoint = self.node_manager.get_node(self.worker_type) model_client = ModelClient(self.components.logger(), node_endpoint) - model_engine = ModelWorkerEngine(model_client, self.worker_type, self.components, self.job_context) + model_engine = ModelWorkerEngine(model_client, self.worker_type, self.worker_id, self.components, self.job_context) try: - outputs = model_engine.run() + outputs = model_engine.run(*self.worker_args) return outputs finally: self.node_manager.release_node(node_endpoint, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/mpc_worker.py b/python/ppc_scheduler/workflow/worker/mpc_worker.py index c5a98c5f..c8fc64eb 100644 --- a/python/ppc_scheduler/workflow/worker/mpc_worker.py +++ b/python/ppc_scheduler/workflow/worker/mpc_worker.py @@ -1,12 +1,12 @@ -from ppc_scheduler.node.computing_node_client import MpcClient +from ppc_scheduler.node.computing_node_client.mpc_node_client import MpcClient from ppc_scheduler.workflow.worker.engine.mpc_engine import MpcWorkerEngine from ppc_scheduler.workflow.worker.worker import Worker class MpcWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs) -> list: node_endpoint = self.node_manager.get_node(self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/psi_worker.py b/python/ppc_scheduler/workflow/worker/psi_worker.py index e8f18cca..bf596268 100644 --- a/python/ppc_scheduler/workflow/worker/psi_worker.py +++ b/python/ppc_scheduler/workflow/worker/psi_worker.py @@ -1,19 +1,21 @@ -from ppc_scheduler.node.computing_node_client.psi_node_client import PsiClient +from ppc_scheduler.node.computing_node_client.psi_client import PsiClient from ppc_scheduler.workflow.worker.engine.psi_engine import PsiWorkerEngine from ppc_scheduler.workflow.worker.worker import Worker class PsiWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs) -> list: - node_endpoint = self.node_manager.get_node(self.worker_type) - psi_client = PsiClient(self.components.logger(), node_endpoint) - psi_engine = PsiWorkerEngine(psi_client, self.worker_type, self.components, self.job_context) + psi_client_node = self.node_manager.get_node(self.worker_type) + logger = self.components.logger() + logger.info(f"## getting psi client : {psi_client_node}") + psi_client = PsiClient(logger, psi_client_node[0], psi_client_node[1]) + psi_engine = PsiWorkerEngine(psi_client, self.worker_id, self.worker_type, self.components, self.job_context) try: - outputs = psi_engine.run() + outputs = psi_engine.run(*self.worker_args) return outputs finally: - self.node_manager.release_node(node_endpoint, self.worker_type) + self.node_manager.release_node(psi_client_node, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/python_worker.py b/python/ppc_scheduler/workflow/worker/python_worker.py index a6a36013..9d5e295b 100644 --- a/python/ppc_scheduler/workflow/worker/python_worker.py +++ b/python/ppc_scheduler/workflow/worker/python_worker.py @@ -3,8 +3,8 @@ class PythonWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs): ... diff --git a/python/ppc_scheduler/workflow/worker/shell_worker.py b/python/ppc_scheduler/workflow/worker/shell_worker.py index 832b8794..3317593d 100644 --- a/python/ppc_scheduler/workflow/worker/shell_worker.py +++ b/python/ppc_scheduler/workflow/worker/shell_worker.py @@ -1,10 +1,16 @@ from ppc_scheduler.workflow.worker.worker import Worker +from ppc_scheduler.workflow.worker.engine.shell_engine import ShellEngine class ShellWorker(Worker): - def __init__(self, components, job_context, worker_id, worker_type, *args, **kwargs): - super().__init__(components, job_context, worker_id, worker_type, *args, **kwargs) + def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs): + super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) - def engine_run(self, worker_inputs): - ... + def engine_run(self, worker_inputs): + + # self.log_worker() + + mpc_engine = ShellEngine(cmd=self.worker_args[0]) + outputs = mpc_engine.run() + return outputs diff --git a/python/ppc_scheduler/workflow/worker/worker.py b/python/ppc_scheduler/workflow/worker/worker.py index b9823ce2..290179f6 100644 --- a/python/ppc_scheduler/workflow/worker/worker.py +++ b/python/ppc_scheduler/workflow/worker/worker.py @@ -4,6 +4,7 @@ from prefect import Task from prefect.engine import signals +from ppc_scheduler.common import log_utils from ppc_scheduler.database import job_worker_mapper from ppc_scheduler.node.node_manager import ComputingNodeManager from ppc_scheduler.workflow.common import codec, flow_utils @@ -12,73 +13,93 @@ class Worker(Task): - def __init__(self, components, job_context, worker_id, worker_type, retries=0, retry_delay_s=0, *args, **kwargs): + def __init__(self, components, job_context, worker_id, worker_type, worker_args, retries=0, retry_delay_s=0, *args, **kwargs): super().__init__(*args, **kwargs) self.components = components self.node_manager = ComputingNodeManager(components) - self.log = components.logger() + self.logger = components.logger() self.job_context = job_context self.worker_id = worker_id self.worker_type = worker_type + self.worker_args = worker_args self.retries = retries self.retry_delay_s = retry_delay_s + def log_worker(self): + self.components.logger().info(" ## View worker, job_id = %s, work_id = %s", self.job_context.job_id, self.worker_id) + def engine_run(self, worker_inputs) -> list: # this func should be implemented by subclass ... def run(self, worker_status, worker_inputs): + self.components.logger().info("Worker begin run, job_id = %s, work_id = %s, work_status = %s", self.job_context.job_id, self.worker_id, worker_status) + start_time = time.time() try: # job is killed if self.components.thread_event_manager.event_status(self.job_context.job_id): self._write_failed_status(WorkerStatus.KILLED) - self.log.warn( - f"worker was killed, job_id: {self.job_context.job_id}, worker: {self.worker_id}") + self.logger.warn( + f" worker was killed, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}") raise signals.FAIL(message='killed!') if worker_status == WorkerStatus.SUCCESS: - # return outputs saved in db directly while current worker has been finished + self.logger.info( + f" Worker has been executed successfully, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}") return self._load_output_from_db() inputs = [] if self.worker_type != WorkerType.T_ON_SUCCESS \ and self.worker_type != WorkerType.T_ON_FAILURE: inputs = flow_utils.to_origin_inputs(worker_inputs) - + outputs = self._try_run_task(inputs) - self._save_worker_result(outputs) + if outputs is not None: + self._save_worker_result(outputs) except FunctionTimedOut: - self._write_failed_status(WorkerStatus.TIMEOUT) - self.log.error( - f"worker was timeout, job_id: {self.job_context.job_id}, worker: {self.worker_id}") + self._write_failed_status(WorkerStatus.FAILURE) + end_time = time.time() + self.logger.error( + f"[OnError] job worker was timeout, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}, elapsed time: {end_time - start_time}") raise signals.FAIL(message='timeout!') except BaseException as be: self._write_failed_status(WorkerStatus.FAILURE) - self.log.error(f"[OnError]job worker failed, job_id: {self.job_context.job_id}, worker: {self.worker_id}") - self.log.exception(be) + self.logger.error(f"[OnError] job worker failed, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}") + self.logger.exception(be) raise signals.FAIL(message='failed!') def _try_run_task(self, inputs): - self.log.info(f"job_id: {self.job_context.job_id}, worker: {self.worker_id}, inputs: {inputs}") + start_time = time.time() + # self.logger.info(f"Worker try run task, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}, inputs: {inputs}") # parse inputs for worker if self.retries: attempt = 0 while attempt <= self.retries: try: + self.logger.info(log_utils.worker_start_log_info(self.worker_id)) outputs = self.engine_run(inputs) + self.logger.info(log_utils.worker_end_log_info(self.worker_id)) return outputs except Exception as e: attempt += 1 if attempt > self.retries: - self.log.warn( + self.logger.error( f"worker failed after {self.retries} attempts, " - f"job_id: {self.job_context.job_id}, worker: {self.worker_id}") + f"job_id: {self.job_context.job_id}, worker_id: {self.worker_id}") raise e else: + self.logger.warn( + f"worker failed, attempts: {attempt}" + f"job_id: {self.job_context.job_id}, worker_id: {self.worker_id}") + self.logger.exception(e) time.sleep(self.retry_delay_s) else: + # outputs = self.engine_run(inputs) + self.logger.info(log_utils.worker_start_log_info(self.worker_id)) outputs = self.engine_run(inputs) - self.log.info(f"job_id: {self.job_context.job_id}, worker: {self.worker_id}, outputs: {outputs}") + self.logger.info(log_utils.worker_end_log_info(self.worker_id)) + end_time = time.time() + # self.logger.info(f"Worker try run task end, job_id: {self.job_context.job_id}, worker: {self.worker_id}, elapsed time: {end_time - start_time}, outputs: {outputs}") return outputs def _load_output_from_db(self): diff --git a/python/ppc_scheduler/workflow/worker/worker_factory.py b/python/ppc_scheduler/workflow/worker/worker_factory.py index 016ce9b4..2518a89b 100644 --- a/python/ppc_scheduler/workflow/worker/worker_factory.py +++ b/python/ppc_scheduler/workflow/worker/worker_factory.py @@ -1,7 +1,8 @@ +from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException from ppc_scheduler.common.global_context import components from ppc_scheduler.workflow.common.worker_type import WorkerType from ppc_scheduler.workflow.worker.api_worker import ApiWorker -from ppc_scheduler.workflow.worker.exit_worker import ExitWorker +from ppc_scheduler.workflow.worker.default_worker import DefaultWorker from ppc_scheduler.workflow.worker.model_worker import ModelWorker from ppc_scheduler.workflow.worker.mpc_worker import MpcWorker from ppc_scheduler.workflow.worker.psi_worker import PsiWorker @@ -12,21 +13,24 @@ class WorkerFactory: @staticmethod - def build_worker(job_context, worker_id, worker_type): + def build_worker(job_context, worker_id, worker_type, worker_args, *args, **kwargs): if worker_type == WorkerType.T_API: - return ApiWorker(components, job_context, worker_id, worker_type) + return ApiWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs) elif worker_type == WorkerType.T_PYTHON: - return PythonWorker(components, job_context, worker_id, worker_type) + return PythonWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs) elif worker_type == WorkerType.T_SHELL: - return ShellWorker(components, job_context, worker_id, worker_type) + return ShellWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_PSI: - return PsiWorker(components, job_context, worker_id, worker_type) + return PsiWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_MPC: - return MpcWorker(components, job_context, worker_id, worker_type) + return MpcWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_PREPROCESSING or \ worker_type == WorkerType.T_FEATURE_ENGINEERING or \ worker_type == WorkerType.T_TRAINING or \ worker_type == WorkerType.T_PREDICTION: - return ModelWorker(components, job_context, worker_id, worker_type) + return ModelWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) + elif worker_type == WorkerType.T_ON_SUCCESS or \ + worker_type == WorkerType.T_ON_FAILURE: + return DefaultWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) else: - return ExitWorker(components, job_context, worker_id, worker_type) + raise PpcException(PpcErrorCode.UNSUPPORTED_WORK_TYPE, f"Unsupported worker type: {worker_type}") From cd7f1229083ae62b877307f753dabe3772e63250 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 27 Sep 2024 09:35:09 +0800 Subject: [PATCH 049/120] add wedpr-python-gateway-sdk (#46) --- cpp/cmake/TargetSettings.cmake | 3 +- cpp/cmake/python.cmake | 8 +- .../sdk-wrapper/java/CMakeLists.txt | 1 + .../sdk-wrapper/python/CMakeLists.txt | 58 ++++--- .../sdk-wrapper/python/bindings/setup.py | 62 ++++++++ .../python/bindings/transport/__init__.py | 0 .../python/bindings/utils/__init__.py | 0 .../python/bindings/utils/lib_loader.py | 89 +++++++++++ .../bindings/utils/tests/test_lib_loader.py | 12 ++ .../sdk-wrapper/python/setup.py.in | 35 ----- .../python/swig/wedpr_python_transport.i | 141 ++++++++++++++++-- 11 files changed, 331 insertions(+), 78 deletions(-) create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py delete mode 100644 cpp/wedpr-transport/sdk-wrapper/python/setup.py.in diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake index 47573b12..9a2ebfab 100644 --- a/cpp/cmake/TargetSettings.cmake +++ b/cpp/cmake/TargetSettings.cmake @@ -138,7 +138,8 @@ set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk) # ==== the swig wrapper ===== set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport") -set(WEDPR_PYTHON_TRANSPORT_DIR ${PROJECT_BINARY_DIR}/python/${WEDPR_PYTHON_TRANSPORT}) +set(WEDPR_PYTHON_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python/generated/${WEDPR_PYTHON_TRANSPORT}) +set(WEDPR_PYTHON_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/python/bindings/libs) set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport_jni") set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/generated/${WEDPR_JAVA_TRANSPORT}) diff --git a/cpp/cmake/python.cmake b/cpp/cmake/python.cmake index 8adbf35d..e941adfb 100644 --- a/cpp/cmake/python.cmake +++ b/cpp/cmake/python.cmake @@ -19,7 +19,13 @@ if(UNIX AND NOT APPLE) endif() # Find Python 3 -find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) +find_package(Python REQUIRED COMPONENTS Interpreter Development) +message("Python_FOUND:${Python_FOUND}") +message("Python_VERSION:${Python_VERSION}") +message("Python_Development_FOUND:${Python_Development_FOUND}") +message("Python_LIBRARIES:${Python_LIBRARIES}") +message("Python_INCLUDE_DIRS:${Python_INCLUDE_DIRS}") + list(APPEND CMAKE_SWIG_FLAGS "-py3" "-DPY3") function(search_python_module) diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt index a184200f..c41394e7 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt @@ -1,4 +1,5 @@ option(AUTO_GENERATE "Auto generate the targes" OFF) +# Note: use SWIG Version 4.2.1 if(AUTO_GENERATE) if(WIN32) message(STATUS "Getting SWIG for Windows: ...") diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt index f925ec3a..e695da02 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt +++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt @@ -1,54 +1,50 @@ -file(GLOB_RECURSE SRCS *.i) -set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) +option(AUTO_GENERATE "Auto generate the targes" OFF) +# Note: use SWIG Version 4.2.1 +if(AUTO_GENERATE) + file(GLOB_RECURSE SRCS *.i) + set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON) -swig_add_library( - ${WEDPR_PYTHON_TRANSPORT} - TYPE MODULE - LANGUAGE python - OUTPUT_DIR ${WEDPR_PYTHON_TRANSPORT_DIR} - SOURCES ${SRCS} -) - -message("#### Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}") -target_include_directories( - ${WEDPR_PYTHON_TRANSPORT} - PRIVATE - ../include - ${Python3_INCLUDE_DIRS} - ) -set_property(TARGET ${WEDPR_PYTHON_TRANSPORT} PROPERTY SWIG_USE_TARGET_INCLUDE_DIRECTORIES ON) + file(MAKE_DIRECTORY ${WEDPR_PYTHON_TRANSPORT_DIR}) + file(MAKE_DIRECTORY ${WEDPR_PYTHON_TRANSPORT_LIB_DIR}) + + swig_add_library( + ${WEDPR_PYTHON_TRANSPORT} + TYPE MODULE + LANGUAGE python + OUTPUT_DIR ${WEDPR_PYTHON_TRANSPORT_DIR} + SOURCES ${SRCS} + ) + set_property(TARGET ${WEDPR_PYTHON_TRANSPORT} PROPERTY SWIG_USE_TARGET_INCLUDE_DIRECTORIES ON) +else() + file(GLOB_RECURSE SRCS src/*.cxx) + add_library(${WEDPR_PYTHON_TRANSPORT} SHARED ${SRCS}) +endif() + +target_include_directories(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${Python_INCLUDE_DIRS}) # note: macOS is APPLE and also UNIX ! if(APPLE) set_property(TARGET ${WEDPR_PYTHON_TRANSPORT} APPEND PROPERTY LINK_FLAGS "-flat_namespace -undefined suppress" ) - set_target_properties(${WEDPR_PYTHON_TRANSPORT} PROPERTIES - SUFFIX ".so" - INSTALL_RPATH "@loader_path;@loader_path/../../${WEDPR_PYTHON_TRANSPORT}/.libs" + set_target_properties(${WEDPR_PYTHON_TRANSPORT} PROPERTIES + INSTALL_RPATH "@loader_path;@loader_path/../../${WEDPR_PYTHON_TRANSPORT}/.libs" ) elseif(UNIX) set_target_properties(${WEDPR_PYTHON_TRANSPORT} PROPERTIES INSTALL_RPATH "$ORIGIN:$ORIGIN/../../${WEDPR_PYTHON_TRANSPORT}/.libs" ) endif() -target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET}) - -SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_DIR}/) - # Variable PYTHON_LIBRARIES can contains keyword `optimized` # which won't be interpreted inside a generator expression. # i.e. we can't use: $<$:${PYTHON_LIBRARIES}> # see: https://cmake.org/cmake/help/git-stage/command/target_link_libraries.html#command:target_link_libraries if(MSVC) target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${Python3_LIBRARIES}) +else() + target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${Python_LIBRARIES}) endif() - -# Configure setup.py and copy to output directory -file(GENERATE OUTPUT ${WEDPR_PYTHON_TRANSPORT_DIR}/__init__.py CONTENT "__version__ = \"${PYTHON_TOOLKIT_VERSION}\"\n") -set(SETUP_PY_IN ${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in) -set(SETUP_PY_OUT ${WEDPR_PYTHON_TRANSPORT_DIR}/setup.py) -configure_file(${SETUP_PY_IN} ${SETUP_PY_OUT}) +SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_LIB_DIR}) message(STATUS "CMAKE_INSTALL_INCLUDEDIR => ${CMAKE_INSTALL_INCLUDEDIR}") message(STATUS "CMAKE_CURRENT_SOURCE_DIR => ${CMAKE_CURRENT_SOURCE_DIR}") \ No newline at end of file diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py new file mode 100644 index 00000000..e60894f8 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py @@ -0,0 +1,62 @@ +import sys + +from setuptools import find_packages, setup +from setuptools.command.bdist_egg import bdist_egg + + +class bdist_egg_disabled(bdist_egg): + """Disabled version of bdist_egg + + Prevents setup.py install from performing setuptools' default easy_install, + which it should never ever do. + """ + + def run(self): + sys.exit( + "Aborting implicit building of eggs. Use `pip install .` to install from source." + ) + + +setup_args = dict( + name='wedpr-python-gateway-sdk', + packages=find_packages(), + version="1.0.0-rc1", + description="wedpr-python-gateway-sdk: The gateway sdk for WeDPR", + long_description_content_type="text/markdown", + author="WeDPR Development Team", + author_email="wedpr@webank.com", + url="https://github.com/WeBankBlockchain/WeDPR-Component", + license="Apache-2.0", + platforms="Linux, Mac OS X", + keywords=['Interactive', 'Interpreter', 'Shell', 'Web'], + python_requires=">=3.8", + include_package_data=True, + classifiers=[ + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + ], +) + +setup_args['cmdclass'] = { + 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, +} + +setup_args['install_requires'] = install_requires = [] +with open('requirements.txt') as f: + for line in f.readlines(): + req = line.strip() + if not req or req.startswith(('-e', '#')): + continue + install_requires.append(req) + + +def main(): + setup(**setup_args) + + +if __name__ == '__main__': + main() diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py new file mode 100644 index 00000000..dbc9a78b --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +import platform +import os +from enum import Enum +import sys +import ctypes + + +class OSType(Enum): + MACOS_X64 = 1 + MACOS_ARM64 = 2 + WINDOWS = 3 + LINUX_X64 = 4 + LINUX_ARM64 = 5 + + +class ArchType(Enum): + ARM64 = 1 + X86_64 = 2 + + +class LibLoader: + # the library information + TRANSPORT_LIB_PATH = "libs" + TRANSPORT_LIB_NAME = "wedpr_python_transport" + LIB_PREFIX = "lib" + MACOS_LIB_POSTFIX = "dylib" + WINDOWS_LIB_POSTFIX = "dll" + LINUX_LIB_PORSTFIX = "so" + # the OS name + MACOS_OS = "darwin" + WINDOWS_OS = "windows" + LINUX_OS = "linux" + + AARCH64 = "aarch64" + ARM64 = "arm64" + X86_64 = "x86_64" + + @staticmethod + def get_arch_type(): + arch_type = platform.machine().lower() + if LibLoader.AARCH64 in arch_type or LibLoader.ARM64 in arch_type: + return ArchType.ARM64 + if LibLoader.X86_64 in arch_type: + return ArchType.X86_64 + raise Exception(f"Unsupported arch type: {arch_type}") + + @staticmethod + def get_os_type(): + os_type = platform.system().lower() + arch_type = LibLoader.get_arch_type() + if LibLoader.MACOS_OS in os_type: + if arch_type == ArchType.ARM64: + return OSType.MACOS_ARM64 + else: + return OSType.MACOS_X64 + if LibLoader.LINUX_OS in os_type: + if arch_type == ArchType.ARM64: + return OSType.LINUX_ARM64 + else: + return OSType.LINUX_X64 + if LibLoader.WINDOWS_OS in os_type: + return OSType.WINDOWS + raise Exception(f"Unsupported os {os_type}, arch: {arch_type}") + + @staticmethod + def get_lib_name(): + os_type = LibLoader.get_os_type() + if os_type == OSType.WINDOWS: + return "{}{}.{}".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.WINDOWS_LIB_POSTFIX) + if os_type == OSType.MACOS_ARM64: + return "{}{}-arm64.{}".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.MACOS_LIB_POSTFIX) + if os_type == OSType.MACOS_X64: + return "{}{}.{}".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.MACOS_LIB_POSTFIX) + if os_type == OSType.LINUX_ARM64: + return "{}{}.{}-arm64".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.LINUX_LIB_PORSTFIX) + if os_type == OSType.LINUX_X64: + return "{}{}.{}".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.LINUX_LIB_PORSTFIX) + raise Exception( + f"get_lib_name failed for not support the os_type: {os_type}") + + @staticmethod + def load_lib(): + lib_path = os.path.join( + LibLoader.TRANSPORT_LIB_PATH, LibLoader.get_lib_name()) + print(f"#### lib_path: {lib_path}") + lib = ctypes.cdll.LoadLibrary(lib_path) + return lib diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py new file mode 100644 index 00000000..2f287a85 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +import unittest + + +class TestLibLoader(unittest.TestCase): + def test_load_lib(self): + from utils.lib_loader import LibLoader + _wedpr_python_transport = LibLoader.load_lib() + + +if __name__ == '__main__': + unittest.main() diff --git a/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in b/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in deleted file mode 100644 index b2ec003f..00000000 --- a/cpp/wedpr-transport/sdk-wrapper/python/setup.py.in +++ /dev/null @@ -1,35 +0,0 @@ -import setuptools.command.install -import shutil -from distutils.sysconfig import get_python_lib - - -class CompiledLibInstall(setuptools.command.install.install): - """ - Specialized install to install to python libs - """ - - def run(self): - """ - Run method called by setup - :return: - """ - # Get filenames from CMake variable - filenames = '${PYTHON_INSTALL_FILES}'.split(';') - - # Directory to install to - install_dir = get_python_lib() - - # Install files - [shutil.copy(filename, install_dir) for filename in filenames] - - -if __name__ == '__main__': - setuptools.setup( - name='wedpr-python-transport', - version='1.0.0-rc1', - packages=['wedpr-python-transport'], - license='Apache License 2.0', - author='wedpr', - author_email='wedpr@webank.com', - cmdclass={'install': CompiledLibInstall} - ) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i index 16515cba..60e4f350 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i @@ -1,4 +1,11 @@ -%module wedpr_python_transport +%define MODULEIMPORT +" +from utils.lib_loader import LibLoader +_wedpr_python_transport = LibLoader.load_lib() +" +%enddef + +%module(moduleimport=MODULEIMPORT) wedpr_python_transport %include %include @@ -7,43 +14,157 @@ %include +// shared_ptr definition %shared_ptr(ppc::front::FrontConfig); +%shared_ptr(ppc::front::IFront); +%shared_ptr(ppc::front::IFrontClient); +// the callbacks +%shared_ptr(ppc::front::ErrorCallback); +%shared_ptr(ppc::front::MessageDispatcherHandler); +%shared_ptr(ppc::front::IMessageHandler); +%shared_ptr(ppc::front::GetPeersInfoHandler); + +%shared_ptr(ppc::gateway::IGateway); %shared_ptr(bcos::Error); +%shared_ptr(bcos::bytes); %shared_ptr(ppc::protocol::Message); -%shared_ptr(ppc::protocol::MessageOptionalHeader) +%shared_ptr(ppc::protocol::MessageOptionalHeader); +%shared_ptr(ppc::protocol::MessageHeader); +%shared_ptr(ppc::protocol::MessagePayload); +%shared_ptr(ppc::protocol::MessageBuilder); +%shared_ptr(ppc::protocol::MessageHeaderBuilder); +%shared_ptr(ppc::protocol::MessagePayloadBuilder); +%shared_ptr(ppc::protocol::MessageOptionalHeaderBuilder); +%shared_ptr(ppc::protocol::GrpcConfig); %shared_ptr(ppc::sdk::Transport); %{ #define SWIG_FILE_WITH_INIT +#include +#include #include #include "wedpr-transport/sdk/src/TransportBuilder.h" #include "wedpr-transport/sdk/src/Transport.h" +#include "ppc-framework/libwrapper/Buffer.h" #include "ppc-framework/front/IFront.h" #include "ppc-framework/protocol/RouteType.h" #include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/GrpcConfig.h" #include +#include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/MessagePayload.h" %} +namespace ppc::sdk{ + class Transport; + class TransportBuilder; +} + +namespace ppc::gateway{ + class IGateway; +} + +namespace ppc::protocol{ + class Message; + class MessageHeader; + class MessagePayload; + class MessageOptionalHeader; + class MessageBuilder; + class MessageHeaderBuilder; + class MessagePayloadBuilder; + class MessageOptionalHeaderBuilder; + class EndPoint; + class GrpcConfig; + class RouteType; +} namespace ppc::front{ class FrontConfig; class IFront; + class IFrontClient; class FrontImpl; class FrontBuilderImpl; - class RouteType; class GatewayEndPoint; + class ErrorCallback; + class MessageDispatcherHandler; + class IMessageHandler; + class SendResponseHandler; } -namespace ppc::sdk{ - class Transport; - class TransportBuilder; +namespace std{ + class vector; } -%template(SharedFrontConfig) std::shared_ptr; +namespace bcos{ + using byte = uint8_t; + using bytes = std::vector; + class Error; +} + +// define shared_ptr objects %template(SharedBcosError) std::shared_ptr; + +%template(SharedFrontConfig) std::shared_ptr; +%template(SharedGrpcConfig) std::shared_ptr; + +%template(SharedFront) std::shared_ptr; +%template(SharedFrontClient) std::shared_ptr; + +%template(SharedErrorCallback) std::shared_ptr; +%template(SharedMessageDispatcherHandler) std::shared_ptr; +%template(SharedIMessageHandler) std::shared_ptr; +%template(SharedGetPeersInfoHandler) std::shared_ptr; + +%template(SharedGateway) std::shared_ptr; + %template(SharedMessage) std::shared_ptr; +%template(SharedMessageHeader) std::shared_ptr; +%template(SharedMessagePayload) std::shared_ptr; %template(SharedRouteInfo) std::shared_ptr; -%template(SharedTransport) std::shared_ptr; + +%template(SharedMessageBuilder) std::shared_ptr; +%template(SharedMessageHeaderBuilder) std::shared_ptr; +%template(SharedMessagePayloadBuilder) std::shared_ptr; +%template(SharedRouteInfoBuilder) std::shared_ptr; + +%template(ubytes) std::vector; +%template(ibytes) std::vector; + +/// callbacks +%feature("director") ppc::front::ErrorCallback; +%feature("director") ppc::front::MessageDispatcherHandler; +%feature("director") ppc::front::IMessageHandler; +%feature("director") ppc::front::GetPeersInfoHandler; + +// the method no need to wrapper +%ignore ppc::sdk::TransportBuilder::build; +%ignore ppc::front::IFront::onReceiveMessage; +%ignore ppc::front::IFront::asyncSendMessage; +%ignore ppc::front::IFront::asyncGetAgencies; +%ignore ppc::front::IFront::registerTopicHandler; +%ignore ppc::front::IFront::registerMessageHandler; +%ignore ppc::front::IFront::asyncSendResponse; +%ignore ppc::front::IFront::populateErrorCallback; +%ignore ppc::front::IFront::populateMessageDispatcherCallback; +%ignore ppc::front::IFront::populateMsgCallback; + +/* +///// tests /// +%inline { +} +///// tests /// +*/ + +// define the interface should been exposed +%include "bcos-utilities/Error.h" +%include "ppc-framework/libwrapper/Buffer.h" +%include "ppc-framework/front/FrontConfig.h" +%include "ppc-framework/protocol/EndPoint.h" +%include "ppc-framework/protocol/GrpcConfig.h" +%include "ppc-framework/protocol/Message.h" +%include "ppc-framework/protocol/MessagePayload.h" + +%include "ppc-framework/front/IFront.h" %include "wedpr-transport/sdk/src/TransportBuilder.h" -%include "wedpr-transport/sdk/src/Transport.h" -%include "ppc-framework/front/IFront.h" \ No newline at end of file +%include "wedpr-transport/sdk/src/Transport.h" \ No newline at end of file From 83d2fe884326d16e91814a3f92f6a50e6a65128b Mon Sep 17 00:00:00 2001 From: octopus <912554887@qq.com> Date: Fri, 27 Sep 2024 09:46:39 +0800 Subject: [PATCH 050/120] fix model client bug --- .../async_thread_executor.py | 2 +- .../model_node_client.py | 38 +++++++++++-------- .../node/computing_node_client/psi_client.py | 6 ++- python/ppc_scheduler/node/node_manager.py | 1 + .../workflow/common/worker_type.py | 1 + .../workflow/worker/engine/model_engine.py | 3 +- .../workflow/worker/model_worker.py | 6 +-- .../workflow/worker/worker_factory.py | 3 +- 8 files changed, 36 insertions(+), 24 deletions(-) diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py index fe012878..15b1f5ca 100644 --- a/python/ppc_common/ppc_async_executor/async_thread_executor.py +++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py @@ -69,7 +69,7 @@ def _cleanup_finished_threads(self): for target_id in finished_threads: with self.lock: del self.threads[target_id] - self.logger.info(f"cleanup finished thread {target_id}") + self.logger.info(f"Cleanup finished thread {target_id}") def __del__(self): self.kill_all() diff --git a/python/ppc_scheduler/node/computing_node_client/model_node_client.py b/python/ppc_scheduler/node/computing_node_client/model_node_client.py index 64a14120..0bb9d9fe 100644 --- a/python/ppc_scheduler/node/computing_node_client/model_node_client.py +++ b/python/ppc_scheduler/node/computing_node_client/model_node_client.py @@ -1,3 +1,4 @@ +import json import time from ppc_common.ppc_utils import http_utils @@ -8,40 +9,47 @@ class ModelClient: - def __init__(self, logger, endpoint, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): + def __init__(self, logger, endpoint, token, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): self.logger = logger self.endpoint = endpoint + self.token = token self.polling_interval_s = polling_interval_s self.max_retries = max_retries self.retry_delay_s = retry_delay_s self._completed_status = 'COMPLETED' self._failed_status = 'FAILED' - def run(self, args): - task_id = args['task_id'] + def run(self, *args): + + params = args[0] + if type(params) == str: + params = json.loads(params) + + task_id = params['task_id'] + try: - self.logger.info(f"ModelApi: begin to run model task {task_id}") + self.logger.info(f"model client begin to run model task {task_id}") response = self._send_request_with_retry(http_utils.send_post_request, endpoint=self.endpoint, uri=RUN_MODEL_API_PREFIX + task_id, - params=args) + params=params) check_response(response) return self._poll_task_status(task_id) except Exception as e: - self.logger.error(f"ModelApi: run model task error, task: {task_id}, error: {e}") + self.logger.error(f"model client run model task exception, task: {task_id}, e: {e}") raise e - def kill(self, job_id): + def kill(self, task_id): try: - self.logger.info(f"ModelApi: begin to kill model task {job_id}") + self.logger.info(f"model client begin to kill model task {task_id}") response = self._send_request_with_retry(http_utils.send_delete_request, endpoint=self.endpoint, - uri=RUN_MODEL_API_PREFIX + job_id) + uri=RUN_MODEL_API_PREFIX + task_id) check_response(response) - self.logger.info(f"ModelApi: model task {job_id} was killed") + self.logger.info(f"model client task {task_id} was killed") return response except Exception as e: - self.logger.warn(f"ModelApi: kill model task {job_id} failed, error: {e}") + self.logger.warn(f"model client kill task {task_id} exception, e: {e}") raise e def _poll_task_status(self, task_id): @@ -51,18 +59,18 @@ def _poll_task_status(self, task_id): uri=RUN_MODEL_API_PREFIX + task_id) check_response(response) if response['data']['status'] == self._completed_status: - self.logger.info(f"task {task_id} completed, response: {response['data']}") + self.logger.info(f"model client task {task_id} completed, response: {response['data']}") return response elif response['data']['status'] == self._failed_status: - self.logger.warn(f"task {task_id} failed, response: {response['data']}") + self.logger.warn(f"model client task {task_id} failed, response: {response['data']}") raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) else: time.sleep(self.polling_interval_s) - def get_remote_log(self, job_id): + def get_remote_log(self, remote_id): response = self._send_request_with_retry(http_utils.send_get_request, endpoint=self.endpoint, - uri=GET_MODEL_LOG_API_PREFIX + job_id) + uri=GET_MODEL_LOG_API_PREFIX + remote_id) check_response(response) return response['data'] diff --git a/python/ppc_scheduler/node/computing_node_client/psi_client.py b/python/ppc_scheduler/node/computing_node_client/psi_client.py index 43fbd5f1..7dcd5ba3 100644 --- a/python/ppc_scheduler/node/computing_node_client/psi_client.py +++ b/python/ppc_scheduler/node/computing_node_client/psi_client.py @@ -25,6 +25,8 @@ def run(self, *args): if type(params) == str: params = json.loads(params) + task_id = params['taskID'] + json_rpc_request = { 'jsonrpc': '1', 'method': 'asyncRunTask', @@ -34,9 +36,9 @@ def run(self, *args): } response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, json_rpc_request) check_privacy_service_response(response) - return self._poll_task_status(params['taskID']) + return self._poll_task_status(task_id) - def _poll_task_status(self, task_id): + def _poll_task_status(self, task_id: str): while True: params = { 'jsonrpc': '1', diff --git a/python/ppc_scheduler/node/node_manager.py b/python/ppc_scheduler/node/node_manager.py index fa2b2247..cadf3e29 100644 --- a/python/ppc_scheduler/node/node_manager.py +++ b/python/ppc_scheduler/node/node_manager.py @@ -5,6 +5,7 @@ class ComputingNodeManager: type_map = { WorkerType.T_PSI: 'PSI', + WorkerType.T_ML_PSI: 'PSI', WorkerType.T_MPC: 'MPC', WorkerType.T_PREPROCESSING: 'MODEL', WorkerType.T_FEATURE_ENGINEERING: 'MODEL', diff --git a/python/ppc_scheduler/workflow/common/worker_type.py b/python/ppc_scheduler/workflow/common/worker_type.py index 54ccd245..6f430978 100644 --- a/python/ppc_scheduler/workflow/common/worker_type.py +++ b/python/ppc_scheduler/workflow/common/worker_type.py @@ -6,6 +6,7 @@ class WorkerType: # specific job worker T_PSI = 'PSI' + T_ML_PSI = 'ML_PSI' T_MPC = 'MPC' T_PREPROCESSING = 'PREPROCESSING' T_FEATURE_ENGINEERING = 'FEATURE_ENGINEERING' diff --git a/python/ppc_scheduler/workflow/worker/engine/model_engine.py b/python/ppc_scheduler/workflow/worker/engine/model_engine.py index 68084783..8f2b24ba 100644 --- a/python/ppc_scheduler/workflow/worker/engine/model_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/model_engine.py @@ -1,4 +1,3 @@ -import os import time from ppc_scheduler.workflow.common.job_context import JobContext @@ -29,7 +28,7 @@ def run(self, *args) -> list: self.logger.info(f"## model engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}") # send job request to model node and wait for the job to finish - # self.psi_client.run(*args) + self.model_client.run(*args) time_costs = time.time() - start_time self.logger.info(f"## model engine run finished, job_id={job_id}, timecost: {time_costs}s") diff --git a/python/ppc_scheduler/workflow/worker/model_worker.py b/python/ppc_scheduler/workflow/worker/model_worker.py index 6be85fe4..0268e705 100644 --- a/python/ppc_scheduler/workflow/worker/model_worker.py +++ b/python/ppc_scheduler/workflow/worker/model_worker.py @@ -9,11 +9,11 @@ def __init__(self, components, job_context, worker_id, worker_type, worker_args, super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) def engine_run(self, worker_inputs): - node_endpoint = self.node_manager.get_node(self.worker_type) - model_client = ModelClient(self.components.logger(), node_endpoint) + model_client_node = self.node_manager.get_node(self.worker_type) + model_client = ModelClient(self.components.logger(), model_client_node[0], model_client_node[1]) model_engine = ModelWorkerEngine(model_client, self.worker_type, self.worker_id, self.components, self.job_context) try: outputs = model_engine.run(*self.worker_args) return outputs finally: - self.node_manager.release_node(node_endpoint, self.worker_type) + self.node_manager.release_node(model_client_node, self.worker_type) diff --git a/python/ppc_scheduler/workflow/worker/worker_factory.py b/python/ppc_scheduler/workflow/worker/worker_factory.py index 2518a89b..9b15db5d 100644 --- a/python/ppc_scheduler/workflow/worker/worker_factory.py +++ b/python/ppc_scheduler/workflow/worker/worker_factory.py @@ -20,7 +20,8 @@ def build_worker(job_context, worker_id, worker_type, worker_args, *args, **kwar return PythonWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs) elif worker_type == WorkerType.T_SHELL: return ShellWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) - elif worker_type == WorkerType.T_PSI: + elif worker_type == WorkerType.T_PSI or \ + worker_type == WorkerType.T_ML_PSI: return PsiWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_MPC: return MpcWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) From acb9355be862ecc993547fe4985addbcbbedcdd2 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Sun, 29 Sep 2024 10:08:53 +0800 Subject: [PATCH 051/120] implement wedpr-python-gateway-sdk (#48) * implement wedpr-python-gateway-sdk * add generated cxx file --- cpp/ppc-framework/front/IFront.h | 6 +- cpp/ppc-framework/protocol/Message.h | 6 +- .../protocol/src/v1/MessagePayloadImpl.cpp | 1 + .../wedpr/sdk/jni/generated/IFront.java | 6 +- .../jni/generated/MessageOptionalHeader.java | 6 +- .../generated/wedpr_java_transportJNI.java | 8 +- .../sdk/jni/transport/impl/TransportImpl.java | 10 +- .../src/wedpr_java_transportJAVA_wrap.cxx | 12 +- .../sdk-wrapper/python/bindings/__init__.py | 1 + .../python/bindings/libs/__init__.py | 0 .../bindings/libs/_wedpr_python_transport.py | 19 + .../python/bindings/transport/api/__init__.py | 0 .../bindings/transport/api/message_api.py | 84 + .../bindings/transport/api/transport_api.py | 50 + .../bindings/transport/demo/__init__.py | 0 .../bindings/transport/demo/transport_demo.py | 65 + .../bindings/transport/generated/__init__.py | 0 .../generated/wedpr_python_transport.py | 1595 ++ .../bindings/transport/impl/__init__.py | 0 .../transport/impl/message_factory.py | 11 + .../bindings/transport/impl/message_impl.py | 87 + .../transport/impl/route_info_builder.py | 20 + .../bindings/transport/impl/transport.py | 100 + .../transport/impl/transport_config.py | 49 + .../transport/impl/transport_loader.py | 15 + .../python/bindings/utils/lib_loader.py | 8 - .../bindings/utils/tests/test_lib_loader.py | 3 +- .../src/wedpr_python_transportPYTHON_wrap.cxx | 23571 ++++++++++++++++ .../src/wedpr_python_transportPYTHON_wrap.h | 184 + .../python/swig/wedpr_python_transport.i | 17 +- 30 files changed, 25898 insertions(+), 36 deletions(-) create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/__init__.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.h diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h index dfa8e95f..5e0a249a 100644 --- a/cpp/ppc-framework/front/IFront.h +++ b/cpp/ppc-framework/front/IFront.h @@ -211,7 +211,11 @@ class IFront : virtual public IFrontClient long timeout) = 0; // TODO: optmize here - virtual bcos::Error::Ptr push(uint16_t routeType, + // Note: the python not support function overload, for different interfaces with the same + // functionality, it is best to define methods with different names the 'payload', 'payloadSize' + // should not been changed any more, since the swig has defined by the name to convert python + // bytes to cpp (char*, uint64_t) %pybuffer_binary(char* payload, uint64_t payloadSize) + virtual bcos::Error::Ptr push_msg(uint16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload, uint64_t payloadSize, int seq, long timeout) { diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h index 291b1d2e..55a73ddd 100644 --- a/cpp/ppc-framework/protocol/Message.h +++ b/cpp/ppc-framework/protocol/Message.h @@ -73,7 +73,11 @@ class MessageOptionalHeader // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in // java byte[] into c bytes - virtual void setDstNode(char* data, uint64_t length) { m_dstNode.assign(data, data + length); } + // Note: the python not support function override + virtual void setDstNodeBuffer(char* data, uint64_t length) + { + m_dstNode.assign(data, data + length); + } // the target agency that need receive the message virtual std::string const& dstInst() const { return m_dstInst; } diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp index 07c4327f..e43124d8 100644 --- a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp +++ b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp @@ -58,6 +58,7 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer) BOOST_THROW_EXCEPTION( WeDPRException() << errinfo_comment("Malform payload for too small!")); } + m_length = buffer.size(); auto pointer = buffer.data(); // the version m_version = boost::asio::detail::socket_ops::network_to_host_short(*((uint16_t*)pointer)); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java index 1fcf80c1..935dd715 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java @@ -122,7 +122,7 @@ public void async_send_response( public Error push( int routeType, MessageOptionalHeader routeInfo, ubytes payload, int seq, int timeout) { long cPtr = - wedpr_java_transportJNI.IFront_push__SWIG_0( + wedpr_java_transportJNI.IFront_push( swigCPtr, this, routeType, @@ -135,7 +135,7 @@ public Error push( return (cPtr == 0) ? null : new Error(cPtr, true); } - public Error push( + public Error push_msg( int routeType, MessageOptionalHeader routeInfo, byte[] payload, @@ -143,7 +143,7 @@ public Error push( int seq, int timeout) { long cPtr = - wedpr_java_transportJNI.IFront_push__SWIG_1( + wedpr_java_transportJNI.IFront_push_msg( swigCPtr, this, routeType, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java index 2e09fca4..892922bb 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java @@ -89,12 +89,12 @@ public byte[] dstNodeBuffer() { } public void setDstNode(ubytes dstNode) { - wedpr_java_transportJNI.MessageOptionalHeader_setDstNode__SWIG_0( + wedpr_java_transportJNI.MessageOptionalHeader_setDstNode( swigCPtr, this, ubytes.getCPtr(dstNode), dstNode); } - public void setDstNode(byte[] data, java.math.BigInteger length) { - wedpr_java_transportJNI.MessageOptionalHeader_setDstNode__SWIG_1( + public void setDstNodeBuffer(byte[] data, java.math.BigInteger length) { + wedpr_java_transportJNI.MessageOptionalHeader_setDstNodeBuffer( swigCPtr, this, data, length); } diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java index 311ec4fe..81c85a3c 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java @@ -353,10 +353,10 @@ public static final native long MessageOptionalHeader_dstNode( public static final native byte[] MessageOptionalHeader_dstNodeBuffer( long jarg1, MessageOptionalHeader jarg1_); - public static final native void MessageOptionalHeader_setDstNode__SWIG_0( + public static final native void MessageOptionalHeader_setDstNode( long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_); - public static final native void MessageOptionalHeader_setDstNode__SWIG_1( + public static final native void MessageOptionalHeader_setDstNodeBuffer( long jarg1, MessageOptionalHeader jarg1_, byte[] jarg2, java.math.BigInteger jarg3); public static final native String MessageOptionalHeader_dstInst( @@ -703,7 +703,7 @@ public static final native void IFront_async_send_response( long jarg8, ErrorCallback jarg8_); - public static final native long IFront_push__SWIG_0( + public static final native long IFront_push( long jarg1, IFront jarg1_, int jarg2, @@ -714,7 +714,7 @@ public static final native long IFront_push__SWIG_0( int jarg5, int jarg6); - public static final native long IFront_push__SWIG_1( + public static final native long IFront_push_msg( long jarg1, IFront jarg1_, int jarg2, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java index d2d497b6..c1e4eb48 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -158,7 +158,7 @@ public void asyncSendMessageByNodeID( MessageCallback msgCallback) { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); - routeInfo.setDstNode(dstNode, BigInteger.valueOf(dstNode.length)); + routeInfo.setDstNodeBuffer(dstNode, BigInteger.valueOf(dstNode.length)); this.transport .getFront() .async_send_message( @@ -302,11 +302,11 @@ public void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload throws WeDPRSDKException { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); - routeInfo.setDstNode(dstNodeID, BigInteger.valueOf(dstNodeID.length)); + routeInfo.setDstNodeBuffer(dstNodeID, BigInteger.valueOf(dstNodeID.length)); Error result = this.transport .getFront() - .push( + .push_msg( RouteType.ROUTE_THROUGH_NODEID.ordinal(), routeInfo, payload, @@ -327,7 +327,7 @@ public void pushByComponent( Error result = this.transport .getFront() - .push( + .push_msg( RouteType.ROUTE_THROUGH_COMPONENT.ordinal(), routeInfo, payload, @@ -346,7 +346,7 @@ public void pushByInst(String topic, String dstInst, int seq, byte[] payload, in Error result = this.transport .getFront() - .push( + .push_msg( RouteType.ROUTE_THROUGH_TOPIC.ordinal(), routeInfo, payload, diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx index 5624a7c7..36134a4e 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx +++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx @@ -3935,7 +3935,7 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav } -SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) { ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; bcos::bytes *arg2 = 0 ; std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; @@ -3957,7 +3957,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } -SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) { +SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) { ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; char *arg2 = (char *) 0 ; uint64_t arg3 ; @@ -3998,7 +3998,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } jenv->ReleaseByteArrayElements(ba, bae, 0); } - (arg1)->setDstNode(arg2,arg3); + (arg1)->setDstNodeBuffer(arg2,arg3); { jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); } @@ -6393,7 +6393,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra } -SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_, jint jarg5, jint jarg6) { +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_, jint jarg5, jint jarg6) { jlong jresult = 0 ; ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; uint16_t arg2 ; @@ -6430,7 +6430,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr } -SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jbyteArray jarg4, jobject jarg5, jint jarg6, jint jarg7) { +SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1msg(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jbyteArray jarg4, jobject jarg5, jint jarg6, jint jarg7) { jlong jresult = 0 ; ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; uint16_t arg2 ; @@ -6483,7 +6483,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr } arg6 = (int)jarg6; arg7 = (long)jarg7; - result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7); + result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7); *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; { jenv->ReleaseByteArrayElements(jarg4, (jbyte *) arg4, 0); diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py new file mode 100644 index 00000000..5becc17c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py new file mode 100644 index 00000000..1605caa8 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import shutil +import pkg_resources +from utils.lib_loader import LibLoader + + +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys + import pkg_resources + import imp + __file__ = pkg_resources.resource_filename( + __name__, LibLoader.get_lib_name()) + __loader__ = None + del __bootstrap__, __loader__ + imp.load_dynamic(__name__, __file__) + + +__bootstrap__() diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py new file mode 100644 index 00000000..036c0069 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +from abc import ABC, abstractmethod + + +class MessageHeaderAPI(ABC): + @abstractmethod + def get_version(self) -> str: + pass + + @abstractmethod + def get_trace_id(self) -> str: + pass + + @abstractmethod + def get_src_gw_node(self) -> str: + pass + + @abstractmethod + def get_dst_gw_node(self) -> str: + pass + + @abstractmethod + def get_packet_type(self) -> int: + pass + + @abstractmethod + def get_ttl(self) -> int: + pass + + @abstractmethod + def get_ext(self) -> int: + pass + + @abstractmethod + def is_resp_packet(self) -> bool: + pass + + @abstractmethod + def get_route_type(self) -> int: + pass + + @abstractmethod + def get_component_type(self) -> str: + pass + + @abstractmethod + def get_src_node(self) -> bytes: + pass + + @abstractmethod + def get_dst_node(self) -> bytes: + pass + + @abstractmethod + def get_dst_inst(self) -> str: + pass + + @abstractmethod + def get_src_inst(self) -> str: + pass + + @abstractmethod + def get_topic(self) -> str: + pass + + +class MessageAPI(ABC): + + @abstractmethod + def get_header(self) -> MessageHeaderAPI: + pass + + @abstractmethod + def get_seq(self) -> int: + pass + + @abstractmethod + def get_payload(self) -> bytes: + pass + + @abstractmethod + def get_length(self) -> int: + pass diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py new file mode 100644 index 00000000..8167f87c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +from abc import ABC, abstractmethod +from transport.api.message_api import MessageAPI + + +class TransportAPI(ABC): + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def push_by_nodeid(topic: str, dstNode: bytes, seq: int, payload: bytes, timeout: int): + pass + + @abstractmethod + def push_by_inst(topic: str, dstInst: str, seq: int, payload: bytes, timeout: int): + pass + + @abstractmethod + def push_by_component(topic: str, dstInst: str, component: str, seq: int, payload: bytes, timeout: int): + pass + + @abstractmethod + def pop(self, topic, timeoutMs) -> MessageAPI: + pass + + @abstractmethod + def peek(self, topic) -> MessageAPI: + pass + + @abstractmethod + def register_topic(self, topic): + pass + + @abstractmethod + def unregister_topic(self, topic): + pass + + @abstractmethod + def register_component(self, component): + pass + + @abstractmethod + def unregister_component(self, component): + pass diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py new file mode 100644 index 00000000..55571d59 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Note: here can't be refactored by autopep +import os +import sys +root_path = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.join(root_path, "../..")) + + +import argparse +from transport.impl.transport import Transport +from transport.impl.transport_config import TransportConfig +from transport.impl.transport_loader import TransportLoader +import time + + +def parse_args(): + parser = argparse.ArgumentParser(prog=sys.argv[0]) + parser.add_argument("-t", '--threadpool_size', + help='the threadpool size', default=4, required=True) + parser.add_argument("-n", '--node_id', + help='the nodeID', required=False) + parser.add_argument("-g", '--gateway_targets', + help='the gateway targets, e.g: ipv4:127.0.0.1:40620,127.0.0.1:40621', required=True) + parser.add_argument("-i", '--host_ip', + help='the host ip, e.g.: 127.0.0.1', required=True) + parser.add_argument("-p", "--listen_port", + help="the listen port", required=True) + parser.add_argument("-d", "--dst_node", + help="the dst node", required=True) + args = parser.parse_args() + return args + + +def message_event_loop(args): + transport_config = TransportConfig( + int(args.threadpool_size), args.node_id, args.gateway_targets) + transport_config.set_self_endpoint( + args.host_ip, int(args.listen_port), "0.0.0.0") + transport = TransportLoader.load(transport_config) + print(f"Create transport success, config: {transport_config.desc()}") + transport.start() + print(f"Start transport success") + test_topic = "sync_message_event_loop_test" + while Transport.should_exit is False: + try: + payload = b"test" + transport.push_by_nodeid(topic=test_topic, dstNode=bytes(args.dst_node, encoding="utf-8"), + seq=0, payload=payload, timeout=6000) + msg = transport.pop(test_topic, timeout_ms=6000) + if msg is None: + print("Receive message timeout") + continue + print( + f"Receive message: {msg.detail()}, buffer: {str(msg.get_payload())}") + except Exception as e: + print(f"exception: {e}") + time.sleep(2) + print(f"stop the transport") + transport.stop() + print(f"stop the transport successfully") + + +if __name__ == "__main__": + args = parse_args() + message_event_loop(args) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py new file mode 100644 index 00000000..16d734a2 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py @@ -0,0 +1,1595 @@ +# This file was automatically generated by SWIG (https://www.swig.org). +# Version 4.2.1 +# +# Do not make changes to this file unless you know what you are doing - modify +# the SWIG interface file instead. + +import weakref +from sys import version_info as _swig_python_version_info + +# Import the low-level C/C++ module +from libs import _wedpr_python_transport + + +try: + import builtins as __builtin__ +except ImportError: + import __builtin__ + + +def _swig_repr(self): + try: + strthis = "proxy of " + self.this.__repr__() + except __builtin__.Exception: + strthis = "" + return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) + + +def _swig_setattr_nondynamic_instance_variable(set): + def set_instance_attr(self, name, value): + if name == "this": + set(self, name, value) + elif name == "thisown": + self.this.own(value) + elif hasattr(self, name) and isinstance(getattr(type(self), name), property): + set(self, name, value) + else: + raise AttributeError( + "You cannot add instance attributes to %s" % self) + return set_instance_attr + + +def _swig_setattr_nondynamic_class_variable(set): + def set_class_attr(cls, name, value): + if hasattr(cls, name) and not isinstance(getattr(cls, name), property): + set(cls, name, value) + else: + raise AttributeError("You cannot add class attributes to %s" % cls) + return set_class_attr + + +def _swig_add_metaclass(metaclass): + """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass""" + def wrapper(cls): + return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy()) + return wrapper + + +class _SwigNonDynamicMeta(type): + """Meta class to enforce nondynamic attributes (no new attributes) for a class""" + __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__) + + +class SwigPyIterator(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_SwigPyIterator + + def value(self): + return _wedpr_python_transport.SwigPyIterator_value(self) + + def incr(self, n=1): + return _wedpr_python_transport.SwigPyIterator_incr(self, n) + + def decr(self, n=1): + return _wedpr_python_transport.SwigPyIterator_decr(self, n) + + def distance(self, x): + return _wedpr_python_transport.SwigPyIterator_distance(self, x) + + def equal(self, x): + return _wedpr_python_transport.SwigPyIterator_equal(self, x) + + def copy(self): + return _wedpr_python_transport.SwigPyIterator_copy(self) + + def next(self): + return _wedpr_python_transport.SwigPyIterator_next(self) + + def __next__(self): + return _wedpr_python_transport.SwigPyIterator___next__(self) + + def previous(self): + return _wedpr_python_transport.SwigPyIterator_previous(self) + + def advance(self, n): + return _wedpr_python_transport.SwigPyIterator_advance(self, n) + + def __eq__(self, x): + return _wedpr_python_transport.SwigPyIterator___eq__(self, x) + + def __ne__(self, x): + return _wedpr_python_transport.SwigPyIterator___ne__(self, x) + + def __iadd__(self, n): + return _wedpr_python_transport.SwigPyIterator___iadd__(self, n) + + def __isub__(self, n): + return _wedpr_python_transport.SwigPyIterator___isub__(self, n) + + def __add__(self, n): + return _wedpr_python_transport.SwigPyIterator___add__(self, n) + + def __sub__(self, *args): + return _wedpr_python_transport.SwigPyIterator___sub__(self, *args) + + def __iter__(self): + return self + + +# Register SwigPyIterator in _wedpr_python_transport: +_wedpr_python_transport.SwigPyIterator_swigregister(SwigPyIterator) +SHARED_PTR_DISOWN = _wedpr_python_transport.SHARED_PTR_DISOWN + + +class SharedBcosError(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedBcosError_swiginit( + self, _wedpr_python_transport.new_SharedBcosError()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedBcosError + + +# Register SharedBcosError in _wedpr_python_transport: +_wedpr_python_transport.SharedBcosError_swigregister(SharedBcosError) + + +class SharedFrontConfig(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedFrontConfig_swiginit( + self, _wedpr_python_transport.new_SharedFrontConfig()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedFrontConfig + + +# Register SharedFrontConfig in _wedpr_python_transport: +_wedpr_python_transport.SharedFrontConfig_swigregister(SharedFrontConfig) + + +class SharedGrpcConfig(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedGrpcConfig_swiginit( + self, _wedpr_python_transport.new_SharedGrpcConfig()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedGrpcConfig + + +# Register SharedGrpcConfig in _wedpr_python_transport: +_wedpr_python_transport.SharedGrpcConfig_swigregister(SharedGrpcConfig) + + +class SharedFront(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedFront_swiginit( + self, _wedpr_python_transport.new_SharedFront()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedFront + + +# Register SharedFront in _wedpr_python_transport: +_wedpr_python_transport.SharedFront_swigregister(SharedFront) + + +class SharedFrontClient(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedFrontClient_swiginit( + self, _wedpr_python_transport.new_SharedFrontClient()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedFrontClient + + +# Register SharedFrontClient in _wedpr_python_transport: +_wedpr_python_transport.SharedFrontClient_swigregister(SharedFrontClient) + + +class SharedErrorCallback(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedErrorCallback_swiginit( + self, _wedpr_python_transport.new_SharedErrorCallback()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedErrorCallback + + +# Register SharedErrorCallback in _wedpr_python_transport: +_wedpr_python_transport.SharedErrorCallback_swigregister(SharedErrorCallback) + + +class SharedMessageDispatcherHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessageDispatcherHandler_swiginit( + self, _wedpr_python_transport.new_SharedMessageDispatcherHandler()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessageDispatcherHandler + + +# Register SharedMessageDispatcherHandler in _wedpr_python_transport: +_wedpr_python_transport.SharedMessageDispatcherHandler_swigregister( + SharedMessageDispatcherHandler) + + +class SharedIMessageHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedIMessageHandler_swiginit( + self, _wedpr_python_transport.new_SharedIMessageHandler()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedIMessageHandler + + +# Register SharedIMessageHandler in _wedpr_python_transport: +_wedpr_python_transport.SharedIMessageHandler_swigregister( + SharedIMessageHandler) + + +class SharedGetPeersInfoHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedGetPeersInfoHandler_swiginit( + self, _wedpr_python_transport.new_SharedGetPeersInfoHandler()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedGetPeersInfoHandler + + +# Register SharedGetPeersInfoHandler in _wedpr_python_transport: +_wedpr_python_transport.SharedGetPeersInfoHandler_swigregister( + SharedGetPeersInfoHandler) + + +class SharedGateway(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedGateway_swiginit( + self, _wedpr_python_transport.new_SharedGateway()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedGateway + + +# Register SharedGateway in _wedpr_python_transport: +_wedpr_python_transport.SharedGateway_swigregister(SharedGateway) + + +class SharedMessage(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessage_swiginit( + self, _wedpr_python_transport.new_SharedMessage()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessage + + +# Register SharedMessage in _wedpr_python_transport: +_wedpr_python_transport.SharedMessage_swigregister(SharedMessage) + + +class SharedMessageHeader(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessageHeader_swiginit( + self, _wedpr_python_transport.new_SharedMessageHeader()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessageHeader + + +# Register SharedMessageHeader in _wedpr_python_transport: +_wedpr_python_transport.SharedMessageHeader_swigregister(SharedMessageHeader) + + +class SharedMessagePayload(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessagePayload_swiginit( + self, _wedpr_python_transport.new_SharedMessagePayload()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessagePayload + + +# Register SharedMessagePayload in _wedpr_python_transport: +_wedpr_python_transport.SharedMessagePayload_swigregister(SharedMessagePayload) + + +class SharedRouteInfo(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedRouteInfo_swiginit( + self, _wedpr_python_transport.new_SharedRouteInfo()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedRouteInfo + + +# Register SharedRouteInfo in _wedpr_python_transport: +_wedpr_python_transport.SharedRouteInfo_swigregister(SharedRouteInfo) + + +class SharedMessageBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessageBuilder_swiginit( + self, _wedpr_python_transport.new_SharedMessageBuilder()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessageBuilder + + +# Register SharedMessageBuilder in _wedpr_python_transport: +_wedpr_python_transport.SharedMessageBuilder_swigregister(SharedMessageBuilder) + + +class SharedMessageHeaderBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessageHeaderBuilder_swiginit( + self, _wedpr_python_transport.new_SharedMessageHeaderBuilder()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessageHeaderBuilder + + +# Register SharedMessageHeaderBuilder in _wedpr_python_transport: +_wedpr_python_transport.SharedMessageHeaderBuilder_swigregister( + SharedMessageHeaderBuilder) + + +class SharedMessagePayloadBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedMessagePayloadBuilder_swiginit( + self, _wedpr_python_transport.new_SharedMessagePayloadBuilder()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedMessagePayloadBuilder + + +# Register SharedMessagePayloadBuilder in _wedpr_python_transport: +_wedpr_python_transport.SharedMessagePayloadBuilder_swigregister( + SharedMessagePayloadBuilder) + + +class SharedRouteInfoBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.SharedRouteInfoBuilder_swiginit( + self, _wedpr_python_transport.new_SharedRouteInfoBuilder()) + __swig_destroy__ = _wedpr_python_transport.delete_SharedRouteInfoBuilder + + +# Register SharedRouteInfoBuilder in _wedpr_python_transport: +_wedpr_python_transport.SharedRouteInfoBuilder_swigregister( + SharedRouteInfoBuilder) + + +class ubytes(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def iterator(self): + return _wedpr_python_transport.ubytes_iterator(self) + + def __iter__(self): + return self.iterator() + + def __nonzero__(self): + return _wedpr_python_transport.ubytes___nonzero__(self) + + def __bool__(self): + return _wedpr_python_transport.ubytes___bool__(self) + + def __len__(self): + return _wedpr_python_transport.ubytes___len__(self) + + def __getslice__(self, i, j): + return _wedpr_python_transport.ubytes___getslice__(self, i, j) + + def __setslice__(self, *args): + return _wedpr_python_transport.ubytes___setslice__(self, *args) + + def __delslice__(self, i, j): + return _wedpr_python_transport.ubytes___delslice__(self, i, j) + + def __delitem__(self, *args): + return _wedpr_python_transport.ubytes___delitem__(self, *args) + + def __getitem__(self, *args): + return _wedpr_python_transport.ubytes___getitem__(self, *args) + + def __setitem__(self, *args): + return _wedpr_python_transport.ubytes___setitem__(self, *args) + + def pop(self): + return _wedpr_python_transport.ubytes_pop(self) + + def append(self, x): + return _wedpr_python_transport.ubytes_append(self, x) + + def empty(self): + return _wedpr_python_transport.ubytes_empty(self) + + def size(self): + return _wedpr_python_transport.ubytes_size(self) + + def swap(self, v): + return _wedpr_python_transport.ubytes_swap(self, v) + + def begin(self): + return _wedpr_python_transport.ubytes_begin(self) + + def end(self): + return _wedpr_python_transport.ubytes_end(self) + + def rbegin(self): + return _wedpr_python_transport.ubytes_rbegin(self) + + def rend(self): + return _wedpr_python_transport.ubytes_rend(self) + + def clear(self): + return _wedpr_python_transport.ubytes_clear(self) + + def get_allocator(self): + return _wedpr_python_transport.ubytes_get_allocator(self) + + def pop_back(self): + return _wedpr_python_transport.ubytes_pop_back(self) + + def erase(self, *args): + return _wedpr_python_transport.ubytes_erase(self, *args) + + def __init__(self, *args): + _wedpr_python_transport.ubytes_swiginit( + self, _wedpr_python_transport.new_ubytes(*args)) + + def push_back(self, x): + return _wedpr_python_transport.ubytes_push_back(self, x) + + def front(self): + return _wedpr_python_transport.ubytes_front(self) + + def back(self): + return _wedpr_python_transport.ubytes_back(self) + + def assign(self, n, x): + return _wedpr_python_transport.ubytes_assign(self, n, x) + + def resize(self, *args): + return _wedpr_python_transport.ubytes_resize(self, *args) + + def insert(self, *args): + return _wedpr_python_transport.ubytes_insert(self, *args) + + def reserve(self, n): + return _wedpr_python_transport.ubytes_reserve(self, n) + + def capacity(self): + return _wedpr_python_transport.ubytes_capacity(self) + __swig_destroy__ = _wedpr_python_transport.delete_ubytes + + +# Register ubytes in _wedpr_python_transport: +_wedpr_python_transport.ubytes_swigregister(ubytes) + + +class ibytes(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def iterator(self): + return _wedpr_python_transport.ibytes_iterator(self) + + def __iter__(self): + return self.iterator() + + def __nonzero__(self): + return _wedpr_python_transport.ibytes___nonzero__(self) + + def __bool__(self): + return _wedpr_python_transport.ibytes___bool__(self) + + def __len__(self): + return _wedpr_python_transport.ibytes___len__(self) + + def __getslice__(self, i, j): + return _wedpr_python_transport.ibytes___getslice__(self, i, j) + + def __setslice__(self, *args): + return _wedpr_python_transport.ibytes___setslice__(self, *args) + + def __delslice__(self, i, j): + return _wedpr_python_transport.ibytes___delslice__(self, i, j) + + def __delitem__(self, *args): + return _wedpr_python_transport.ibytes___delitem__(self, *args) + + def __getitem__(self, *args): + return _wedpr_python_transport.ibytes___getitem__(self, *args) + + def __setitem__(self, *args): + return _wedpr_python_transport.ibytes___setitem__(self, *args) + + def pop(self): + return _wedpr_python_transport.ibytes_pop(self) + + def append(self, x): + return _wedpr_python_transport.ibytes_append(self, x) + + def empty(self): + return _wedpr_python_transport.ibytes_empty(self) + + def size(self): + return _wedpr_python_transport.ibytes_size(self) + + def swap(self, v): + return _wedpr_python_transport.ibytes_swap(self, v) + + def begin(self): + return _wedpr_python_transport.ibytes_begin(self) + + def end(self): + return _wedpr_python_transport.ibytes_end(self) + + def rbegin(self): + return _wedpr_python_transport.ibytes_rbegin(self) + + def rend(self): + return _wedpr_python_transport.ibytes_rend(self) + + def clear(self): + return _wedpr_python_transport.ibytes_clear(self) + + def get_allocator(self): + return _wedpr_python_transport.ibytes_get_allocator(self) + + def pop_back(self): + return _wedpr_python_transport.ibytes_pop_back(self) + + def erase(self, *args): + return _wedpr_python_transport.ibytes_erase(self, *args) + + def __init__(self, *args): + _wedpr_python_transport.ibytes_swiginit( + self, _wedpr_python_transport.new_ibytes(*args)) + + def push_back(self, x): + return _wedpr_python_transport.ibytes_push_back(self, x) + + def front(self): + return _wedpr_python_transport.ibytes_front(self) + + def back(self): + return _wedpr_python_transport.ibytes_back(self) + + def assign(self, n, x): + return _wedpr_python_transport.ibytes_assign(self, n, x) + + def resize(self, *args): + return _wedpr_python_transport.ibytes_resize(self, *args) + + def insert(self, *args): + return _wedpr_python_transport.ibytes_insert(self, *args) + + def reserve(self, n): + return _wedpr_python_transport.ibytes_reserve(self, n) + + def capacity(self): + return _wedpr_python_transport.ibytes_capacity(self) + __swig_destroy__ = _wedpr_python_transport.delete_ibytes + + +# Register ibytes in _wedpr_python_transport: +_wedpr_python_transport.ibytes_swigregister(ibytes) + + +class Error(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + @staticmethod + def buildError(*args): + return _wedpr_python_transport.Error_buildError(*args) + + def __init__(self, *args): + _wedpr_python_transport.Error_swiginit( + self, _wedpr_python_transport.new_Error(*args)) + __swig_destroy__ = _wedpr_python_transport.delete_Error + + def errorCode(self): + return _wedpr_python_transport.Error_errorCode(self) + + def errorMessage(self): + return _wedpr_python_transport.Error_errorMessage(self) + + def setErrorCode(self, _errorCode): + return _wedpr_python_transport.Error_setErrorCode(self, _errorCode) + + def setErrorMessage(self, _errorMessage): + return _wedpr_python_transport.Error_setErrorMessage(self, _errorMessage) + + +# Register Error in _wedpr_python_transport: +_wedpr_python_transport.Error_swigregister(Error) + + +class InputBuffer(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + data = property(_wedpr_python_transport.InputBuffer_data_get, + _wedpr_python_transport.InputBuffer_data_set) + len = property(_wedpr_python_transport.InputBuffer_len_get, + _wedpr_python_transport.InputBuffer_len_set) + + def __init__(self): + _wedpr_python_transport.InputBuffer_swiginit( + self, _wedpr_python_transport.new_InputBuffer()) + __swig_destroy__ = _wedpr_python_transport.delete_InputBuffer + + +# Register InputBuffer in _wedpr_python_transport: +_wedpr_python_transport.InputBuffer_swigregister(InputBuffer) + + +class OutputBuffer(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + data = property(_wedpr_python_transport.OutputBuffer_data_get, + _wedpr_python_transport.OutputBuffer_data_set) + len = property(_wedpr_python_transport.OutputBuffer_len_get, + _wedpr_python_transport.OutputBuffer_len_set) + + def __init__(self): + _wedpr_python_transport.OutputBuffer_swiginit( + self, _wedpr_python_transport.new_OutputBuffer()) + __swig_destroy__ = _wedpr_python_transport.delete_OutputBuffer + + +# Register OutputBuffer in _wedpr_python_transport: +_wedpr_python_transport.OutputBuffer_swigregister(OutputBuffer) + + +class FrontConfig(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_FrontConfig + + def threadPoolSize(self): + return _wedpr_python_transport.FrontConfig_threadPoolSize(self) + + def setThreadPoolSize(self, threadPoolSize): + return _wedpr_python_transport.FrontConfig_setThreadPoolSize(self, threadPoolSize) + + def nodeID(self): + return _wedpr_python_transport.FrontConfig_nodeID(self) + + def setNodeID(self, nodeID): + return _wedpr_python_transport.FrontConfig_setNodeID(self, nodeID) + + def selfEndPoint(self): + return _wedpr_python_transport.FrontConfig_selfEndPoint(self) + + def mutableSelfEndPoint(self): + return _wedpr_python_transport.FrontConfig_mutableSelfEndPoint(self) + + def setSelfEndPoint(self, endPoint): + return _wedpr_python_transport.FrontConfig_setSelfEndPoint(self, endPoint) + + def setGatewayGrpcTarget(self, gatewayGrpcTarget): + return _wedpr_python_transport.FrontConfig_setGatewayGrpcTarget(self, gatewayGrpcTarget) + + def gatewayGrpcTarget(self): + return _wedpr_python_transport.FrontConfig_gatewayGrpcTarget(self) + + def setGrpcConfig(self, grpcConfig): + return _wedpr_python_transport.FrontConfig_setGrpcConfig(self, grpcConfig) + + def grpcConfig(self): + return _wedpr_python_transport.FrontConfig_grpcConfig(self) + + def generateNodeInfo(self): + return _wedpr_python_transport.FrontConfig_generateNodeInfo(self) + + def getComponents(self): + return _wedpr_python_transport.FrontConfig_getComponents(self) + + def setComponents(self, components): + return _wedpr_python_transport.FrontConfig_setComponents(self, components) + + def addComponent(self, component): + return _wedpr_python_transport.FrontConfig_addComponent(self, component) + + def mutableComponents(self): + return _wedpr_python_transport.FrontConfig_mutableComponents(self) + + +# Register FrontConfig in _wedpr_python_transport: +_wedpr_python_transport.FrontConfig_swigregister(FrontConfig) + + +class FrontConfigBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_FrontConfigBuilder + + def build(self, *args): + return _wedpr_python_transport.FrontConfigBuilder_build(self, *args) + + +# Register FrontConfigBuilder in _wedpr_python_transport: +_wedpr_python_transport.FrontConfigBuilder_swigregister(FrontConfigBuilder) + + +def printFrontDesc(config): + return _wedpr_python_transport.printFrontDesc(config) + + +class EndPoint(object): + r"""the endpoint""" + + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self, *args): + _wedpr_python_transport.EndPoint_swiginit( + self, _wedpr_python_transport.new_EndPoint(*args)) + __swig_destroy__ = _wedpr_python_transport.delete_EndPoint + + def host(self): + return _wedpr_python_transport.EndPoint_host(self) + + def port(self): + return _wedpr_python_transport.EndPoint_port(self) + + def setHost(self, host): + return _wedpr_python_transport.EndPoint_setHost(self, host) + + def setPort(self, port): + return _wedpr_python_transport.EndPoint_setPort(self, port) + + def setListenIp(self, listenIp): + return _wedpr_python_transport.EndPoint_setListenIp(self, listenIp) + + def entryPoint(self): + return _wedpr_python_transport.EndPoint_entryPoint(self) + + def listenEndPoint(self): + return _wedpr_python_transport.EndPoint_listenEndPoint(self) + + def listenIp(self): + return _wedpr_python_transport.EndPoint_listenIp(self) + + +# Register EndPoint in _wedpr_python_transport: +_wedpr_python_transport.EndPoint_swigregister(EndPoint) + + +class GrpcServerConfig(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self, *args): + _wedpr_python_transport.GrpcServerConfig_swiginit( + self, _wedpr_python_transport.new_GrpcServerConfig(*args)) + + def listenEndPoint(self): + return _wedpr_python_transport.GrpcServerConfig_listenEndPoint(self) + + def setEndPoint(self, endPoint): + return _wedpr_python_transport.GrpcServerConfig_setEndPoint(self, endPoint) + + def setEnableHealthCheck(self, enableHealthCheck): + return _wedpr_python_transport.GrpcServerConfig_setEnableHealthCheck(self, enableHealthCheck) + + def endPoint(self): + return _wedpr_python_transport.GrpcServerConfig_endPoint(self) + + def mutableEndPoint(self): + return _wedpr_python_transport.GrpcServerConfig_mutableEndPoint(self) + + def enableHealthCheck(self): + return _wedpr_python_transport.GrpcServerConfig_enableHealthCheck(self) + __swig_destroy__ = _wedpr_python_transport.delete_GrpcServerConfig + + +# Register GrpcServerConfig in _wedpr_python_transport: +_wedpr_python_transport.GrpcServerConfig_swigregister(GrpcServerConfig) + + +class GrpcConfig(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.GrpcConfig_swiginit( + self, _wedpr_python_transport.new_GrpcConfig()) + __swig_destroy__ = _wedpr_python_transport.delete_GrpcConfig + + def loadBalancePolicy(self): + return _wedpr_python_transport.GrpcConfig_loadBalancePolicy(self) + + def setLoadBalancePolicy(self, loadBalancePolicy): + return _wedpr_python_transport.GrpcConfig_setLoadBalancePolicy(self, loadBalancePolicy) + + def enableHealthCheck(self): + return _wedpr_python_transport.GrpcConfig_enableHealthCheck(self) + + def setEnableHealthCheck(self, enableHealthCheck): + return _wedpr_python_transport.GrpcConfig_setEnableHealthCheck(self, enableHealthCheck) + + def setEnableDnslookup(self, enableDnslookup): + return _wedpr_python_transport.GrpcConfig_setEnableDnslookup(self, enableDnslookup) + + def enableDnslookup(self): + return _wedpr_python_transport.GrpcConfig_enableDnslookup(self) + + +# Register GrpcConfig in _wedpr_python_transport: +_wedpr_python_transport.GrpcConfig_swigregister(GrpcConfig) + + +class MessageOptionalHeader(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessageOptionalHeader + + def encode(self, buffer): + return _wedpr_python_transport.MessageOptionalHeader_encode(self, buffer) + + def decode(self, data, _offset): + return _wedpr_python_transport.MessageOptionalHeader_decode(self, data, _offset) + + def componentType(self): + return _wedpr_python_transport.MessageOptionalHeader_componentType(self) + + def setComponentType(self, componentType): + return _wedpr_python_transport.MessageOptionalHeader_setComponentType(self, componentType) + + def srcNode(self): + return _wedpr_python_transport.MessageOptionalHeader_srcNode(self) + + def srcNodeBuffer(self): + r""" for swig-wrapper(pass the binary data)""" + return _wedpr_python_transport.MessageOptionalHeader_srcNodeBuffer(self) + + def setSrcNode(self, *args): + return _wedpr_python_transport.MessageOptionalHeader_setSrcNode(self, *args) + + def dstNode(self): + return _wedpr_python_transport.MessageOptionalHeader_dstNode(self) + + def dstNodeBuffer(self): + return _wedpr_python_transport.MessageOptionalHeader_dstNodeBuffer(self) + + def setDstNode(self, dstNode): + return _wedpr_python_transport.MessageOptionalHeader_setDstNode(self, dstNode) + + def setDstNodeBuffer(self, data): + return _wedpr_python_transport.MessageOptionalHeader_setDstNodeBuffer(self, data) + + def dstInst(self): + return _wedpr_python_transport.MessageOptionalHeader_dstInst(self) + + def setDstInst(self, dstInst): + return _wedpr_python_transport.MessageOptionalHeader_setDstInst(self, dstInst) + + def topic(self): + return _wedpr_python_transport.MessageOptionalHeader_topic(self) + + def setTopic(self, *args): + return _wedpr_python_transport.MessageOptionalHeader_setTopic(self, *args) + + def srcInst(self): + return _wedpr_python_transport.MessageOptionalHeader_srcInst(self) + + def setSrcInst(self, srcInst): + return _wedpr_python_transport.MessageOptionalHeader_setSrcInst(self, srcInst) + + +# Register MessageOptionalHeader in _wedpr_python_transport: +_wedpr_python_transport.MessageOptionalHeader_swigregister( + MessageOptionalHeader) + + +class MessageHeader(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessageHeader + + def encode(self, buffer): + return _wedpr_python_transport.MessageHeader_encode(self, buffer) + + def decode(self, data): + return _wedpr_python_transport.MessageHeader_decode(self, data) + + def version(self): + return _wedpr_python_transport.MessageHeader_version(self) + + def setVersion(self, version): + return _wedpr_python_transport.MessageHeader_setVersion(self, version) + + def traceID(self): + return _wedpr_python_transport.MessageHeader_traceID(self) + + def setTraceID(self, traceID): + return _wedpr_python_transport.MessageHeader_setTraceID(self, traceID) + + def srcGwNode(self): + return _wedpr_python_transport.MessageHeader_srcGwNode(self) + + def setSrcGwNode(self, srcGwNode): + return _wedpr_python_transport.MessageHeader_setSrcGwNode(self, srcGwNode) + + def dstGwNode(self): + return _wedpr_python_transport.MessageHeader_dstGwNode(self) + + def setDstGwNode(self, dstGwNode): + return _wedpr_python_transport.MessageHeader_setDstGwNode(self, dstGwNode) + + def packetType(self): + return _wedpr_python_transport.MessageHeader_packetType(self) + + def setPacketType(self, packetType): + return _wedpr_python_transport.MessageHeader_setPacketType(self, packetType) + + def ttl(self): + return _wedpr_python_transport.MessageHeader_ttl(self) + + def setTTL(self, ttl): + return _wedpr_python_transport.MessageHeader_setTTL(self, ttl) + + def ext(self): + return _wedpr_python_transport.MessageHeader_ext(self) + + def setExt(self, ext): + return _wedpr_python_transport.MessageHeader_setExt(self, ext) + + def optionalField(self): + return _wedpr_python_transport.MessageHeader_optionalField(self) + + def setOptionalField(self, optionalField): + return _wedpr_python_transport.MessageHeader_setOptionalField(self, optionalField) + + def length(self): + return _wedpr_python_transport.MessageHeader_length(self) + + def isRespPacket(self): + return _wedpr_python_transport.MessageHeader_isRespPacket(self) + + def setRespPacket(self): + return _wedpr_python_transport.MessageHeader_setRespPacket(self) + + def srcP2PNodeIDView(self): + return _wedpr_python_transport.MessageHeader_srcP2PNodeIDView(self) + + def dstP2PNodeIDView(self): + return _wedpr_python_transport.MessageHeader_dstP2PNodeIDView(self) + + def routeType(self): + return _wedpr_python_transport.MessageHeader_routeType(self) + + def setRouteType(self, type): + return _wedpr_python_transport.MessageHeader_setRouteType(self, type) + + def hasOptionalField(self): + return _wedpr_python_transport.MessageHeader_hasOptionalField(self) + + +# Register MessageHeader in _wedpr_python_transport: +_wedpr_python_transport.MessageHeader_swigregister(MessageHeader) + + +class Message(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_Message + + def header(self): + return _wedpr_python_transport.Message_header(self) + + def setHeader(self, header): + return _wedpr_python_transport.Message_setHeader(self, header) + + def version(self): + r""" the overloaed implementation ===""" + return _wedpr_python_transport.Message_version(self) + + def setVersion(self, version): + return _wedpr_python_transport.Message_setVersion(self, version) + + def packetType(self): + return _wedpr_python_transport.Message_packetType(self) + + def setPacketType(self, packetType): + return _wedpr_python_transport.Message_setPacketType(self, packetType) + + def seq(self): + return _wedpr_python_transport.Message_seq(self) + + def setSeq(self, traceID): + return _wedpr_python_transport.Message_setSeq(self, traceID) + + def ext(self): + return _wedpr_python_transport.Message_ext(self) + + def setExt(self, ext): + return _wedpr_python_transport.Message_setExt(self, ext) + + def isRespPacket(self): + return _wedpr_python_transport.Message_isRespPacket(self) + + def setRespPacket(self): + return _wedpr_python_transport.Message_setRespPacket(self) + + def length(self): + return _wedpr_python_transport.Message_length(self) + + def payload(self): + return _wedpr_python_transport.Message_payload(self) + + def payloadBuffer(self): + return _wedpr_python_transport.Message_payloadBuffer(self) + + def setPayload(self, _payload): + return _wedpr_python_transport.Message_setPayload(self, _payload) + + def setFrontMessage(self, frontMessage): + return _wedpr_python_transport.Message_setFrontMessage(self, frontMessage) + + def frontMessage(self): + return _wedpr_python_transport.Message_frontMessage(self) + + def encode(self, *args): + return _wedpr_python_transport.Message_encode(self, *args) + + def decode(self, _buffer): + return _wedpr_python_transport.Message_decode(self, _buffer) + + +# Register Message in _wedpr_python_transport: +_wedpr_python_transport.Message_swigregister(Message) + + +class MessageHeaderBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessageHeaderBuilder + + def build(self, *args): + return _wedpr_python_transport.MessageHeaderBuilder_build(self, *args) + + +# Register MessageHeaderBuilder in _wedpr_python_transport: +_wedpr_python_transport.MessageHeaderBuilder_swigregister(MessageHeaderBuilder) + + +class MessageBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessageBuilder + + def build(self, *args): + return _wedpr_python_transport.MessageBuilder_build(self, *args) + + +# Register MessageBuilder in _wedpr_python_transport: +_wedpr_python_transport.MessageBuilder_swigregister(MessageBuilder) + + +class MessageOptionalHeaderBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessageOptionalHeaderBuilder + + def build(self, *args): + return _wedpr_python_transport.MessageOptionalHeaderBuilder_build(self, *args) + + +# Register MessageOptionalHeaderBuilder in _wedpr_python_transport: +_wedpr_python_transport.MessageOptionalHeaderBuilder_swigregister( + MessageOptionalHeaderBuilder) + + +def printOptionalField(optionalHeader): + return _wedpr_python_transport.printOptionalField(optionalHeader) + + +def printMessage(_msg): + return _wedpr_python_transport.printMessage(_msg) + + +def printWsMessage(_msg): + return _wedpr_python_transport.printWsMessage(_msg) + + +FrontMsgExtFlag_Response = _wedpr_python_transport.FrontMsgExtFlag_Response + + +class MessagePayload(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessagePayload + + def encode(self, buffer): + return _wedpr_python_transport.MessagePayload_encode(self, buffer) + + def decode(self, data): + return _wedpr_python_transport.MessagePayload_decode(self, data) + + def version(self): + return _wedpr_python_transport.MessagePayload_version(self) + + def setVersion(self, version): + return _wedpr_python_transport.MessagePayload_setVersion(self, version) + + def data(self): + return _wedpr_python_transport.MessagePayload_data(self) + + def dataBuffer(self): + return _wedpr_python_transport.MessagePayload_dataBuffer(self) + + def setData(self, *args): + return _wedpr_python_transport.MessagePayload_setData(self, *args) + + def seq(self): + return _wedpr_python_transport.MessagePayload_seq(self) + + def setSeq(self, seq): + return _wedpr_python_transport.MessagePayload_setSeq(self, seq) + + def length(self): + return _wedpr_python_transport.MessagePayload_length(self) + + def traceID(self): + return _wedpr_python_transport.MessagePayload_traceID(self) + + def setTraceID(self, traceID): + return _wedpr_python_transport.MessagePayload_setTraceID(self, traceID) + + def ext(self): + return _wedpr_python_transport.MessagePayload_ext(self) + + def setExt(self, ext): + return _wedpr_python_transport.MessagePayload_setExt(self, ext) + + def setRespPacket(self): + return _wedpr_python_transport.MessagePayload_setRespPacket(self) + + def isRespPacket(self): + return _wedpr_python_transport.MessagePayload_isRespPacket(self) + + +# Register MessagePayload in _wedpr_python_transport: +_wedpr_python_transport.MessagePayload_swigregister(MessagePayload) + + +class MessagePayloadBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_MessagePayloadBuilder + + def build(self, *args): + return _wedpr_python_transport.MessagePayloadBuilder_build(self, *args) + + +# Register MessagePayloadBuilder in _wedpr_python_transport: +_wedpr_python_transport.MessagePayloadBuilder_swigregister( + MessagePayloadBuilder) + + +class IFrontClient(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_IFrontClient + + def onReceiveMessage(self, _msg, _callback): + r""" + : receive message from gateway, call by gateway + :param _message:: received ppc message + :rtype: void + :return: void + """ + return _wedpr_python_transport.IFrontClient_onReceiveMessage(self, _msg, _callback) + + +# Register IFrontClient in _wedpr_python_transport: +_wedpr_python_transport.IFrontClient_swigregister(IFrontClient) + + +class ErrorCallback(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + if self.__class__ == ErrorCallback: + _self = None + else: + _self = self + _wedpr_python_transport.ErrorCallback_swiginit( + self, _wedpr_python_transport.new_ErrorCallback(_self, )) + __swig_destroy__ = _wedpr_python_transport.delete_ErrorCallback + + def onError(self, error): + return _wedpr_python_transport.ErrorCallback_onError(self, error) + + def __disown__(self): + self.this.disown() + _wedpr_python_transport.disown_ErrorCallback(self) + return weakref.proxy(self) + + +# Register ErrorCallback in _wedpr_python_transport: +_wedpr_python_transport.ErrorCallback_swigregister(ErrorCallback) + + +class MessageDispatcherHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + if self.__class__ == MessageDispatcherHandler: + _self = None + else: + _self = self + _wedpr_python_transport.MessageDispatcherHandler_swiginit( + self, _wedpr_python_transport.new_MessageDispatcherHandler(_self, )) + __swig_destroy__ = _wedpr_python_transport.delete_MessageDispatcherHandler + + def onMessage(self, msg): + return _wedpr_python_transport.MessageDispatcherHandler_onMessage(self, msg) + + def __disown__(self): + self.this.disown() + _wedpr_python_transport.disown_MessageDispatcherHandler(self) + return weakref.proxy(self) + + +# Register MessageDispatcherHandler in _wedpr_python_transport: +_wedpr_python_transport.MessageDispatcherHandler_swigregister( + MessageDispatcherHandler) + + +class SendResponseHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self, responseFunc): + _wedpr_python_transport.SendResponseHandler_swiginit( + self, _wedpr_python_transport.new_SendResponseHandler(responseFunc)) + __swig_destroy__ = _wedpr_python_transport.delete_SendResponseHandler + + def sendResponse(self, payload): + return _wedpr_python_transport.SendResponseHandler_sendResponse(self, payload) + + +# Register SendResponseHandler in _wedpr_python_transport: +_wedpr_python_transport.SendResponseHandler_swigregister(SendResponseHandler) + + +class IMessageHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + if self.__class__ == IMessageHandler: + _self = None + else: + _self = self + _wedpr_python_transport.IMessageHandler_swiginit( + self, _wedpr_python_transport.new_IMessageHandler(_self, )) + __swig_destroy__ = _wedpr_python_transport.delete_IMessageHandler + + def onMessage(self, e, msg, sendResponseHandler): + return _wedpr_python_transport.IMessageHandler_onMessage(self, e, msg, sendResponseHandler) + + def __disown__(self): + self.this.disown() + _wedpr_python_transport.disown_IMessageHandler(self) + return weakref.proxy(self) + + +# Register IMessageHandler in _wedpr_python_transport: +_wedpr_python_transport.IMessageHandler_swigregister(IMessageHandler) + + +class GetPeersInfoHandler(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + if self.__class__ == GetPeersInfoHandler: + _self = None + else: + _self = self + _wedpr_python_transport.GetPeersInfoHandler_swiginit( + self, _wedpr_python_transport.new_GetPeersInfoHandler(_self, )) + __swig_destroy__ = _wedpr_python_transport.delete_GetPeersInfoHandler + + def onPeersInfo(self, e, peersInfo): + return _wedpr_python_transport.GetPeersInfoHandler_onPeersInfo(self, e, peersInfo) + + def __disown__(self): + self.this.disown() + _wedpr_python_transport.disown_GetPeersInfoHandler(self) + return weakref.proxy(self) + + +# Register GetPeersInfoHandler in _wedpr_python_transport: +_wedpr_python_transport.GetPeersInfoHandler_swigregister(GetPeersInfoHandler) + + +class IFront(IFrontClient): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_IFront + + def start(self): + r""" + start the IFront + + :param front: the IFront to start + """ + return _wedpr_python_transport.IFront_start(self) + + def stop(self): + r""" + stop the IFront + + :param front: the IFront to stop + """ + return _wedpr_python_transport.IFront_stop(self) + + def register_topic_handler(self, topic, callback): + return _wedpr_python_transport.IFront_register_topic_handler(self, topic, callback) + + def register_msg_handler(self, componentType, callback): + return _wedpr_python_transport.IFront_register_msg_handler(self, componentType, callback) + + def async_send_message(self, routeType, routeInfo, payload, seq, timeout, errorCallback, msgHandler): + return _wedpr_python_transport.IFront_async_send_message(self, routeType, routeInfo, payload, seq, timeout, errorCallback, msgHandler) + + def async_send_response(self, dstNode, dstNodeSize, traceID, payload, seq, errorCallback): + return _wedpr_python_transport.IFront_async_send_response(self, dstNode, dstNodeSize, traceID, payload, seq, errorCallback) + + def push(self, routeType, routeInfo, payload, seq, timeout): + return _wedpr_python_transport.IFront_push(self, routeType, routeInfo, payload, seq, timeout) + + def push_msg(self, routeType, routeInfo, payload, seq, timeout): + return _wedpr_python_transport.IFront_push_msg(self, routeType, routeInfo, payload, seq, timeout) + + def pop(self, topic, timeoutMs): + return _wedpr_python_transport.IFront_pop(self, topic, timeoutMs) + + def peek(self, topic): + return _wedpr_python_transport.IFront_peek(self, topic) + + def asyncGetPeers(self, getPeersCallback): + return _wedpr_python_transport.IFront_asyncGetPeers(self, getPeersCallback) + + def registerNodeInfo(self, nodeInfo): + r""" + register the nodeInfo to the gateway + :type nodeInfo: ppc::protocol::INodeInfo::Ptr + :param nodeInfo: the nodeInfo + """ + return _wedpr_python_transport.IFront_registerNodeInfo(self, nodeInfo) + + def unRegisterNodeInfo(self): + r"""unRegister the nodeInfo to the gateway""" + return _wedpr_python_transport.IFront_unRegisterNodeInfo(self) + + def nodeInfo(self): + return _wedpr_python_transport.IFront_nodeInfo(self) + + def registerTopic(self, topic): + r""" + register the topic + + :type topic: string + :param topic: the topic to register + """ + return _wedpr_python_transport.IFront_registerTopic(self, topic) + + def unRegisterTopic(self, topic): + r""" + unRegister the topic + + :type topic: string + :param topic: the topic to unregister + """ + return _wedpr_python_transport.IFront_unRegisterTopic(self, topic) + + def registerComponent(self, component): + return _wedpr_python_transport.IFront_registerComponent(self, component) + + def unRegisterComponent(self, component): + return _wedpr_python_transport.IFront_unRegisterComponent(self, component) + + +# Register IFront in _wedpr_python_transport: +_wedpr_python_transport.IFront_swigregister(IFront) + + +class IFrontBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined - class is abstract") + __repr__ = _swig_repr + __swig_destroy__ = _wedpr_python_transport.delete_IFrontBuilder + + def buildClient(self, endPoint, onUnHealthHandler, removeHandlerOnUnhealth): + return _wedpr_python_transport.IFrontBuilder_buildClient(self, endPoint, onUnHealthHandler, removeHandlerOnUnhealth) + + +# Register IFrontBuilder in _wedpr_python_transport: +_wedpr_python_transport.IFrontBuilder_swigregister(IFrontBuilder) +SDKMode_AIR = _wedpr_python_transport.SDKMode_AIR +SDKMode_PRO = _wedpr_python_transport.SDKMode_PRO + + +class TransportBuilder(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _wedpr_python_transport.TransportBuilder_swiginit( + self, _wedpr_python_transport.new_TransportBuilder()) + __swig_destroy__ = _wedpr_python_transport.delete_TransportBuilder + + def buildProTransport(self, config): + return _wedpr_python_transport.TransportBuilder_buildProTransport(self, config) + + def buildConfig(self, threadPoolSize, nodeID): + return _wedpr_python_transport.TransportBuilder_buildConfig(self, threadPoolSize, nodeID) + + def frontConfigBuilder(self): + return _wedpr_python_transport.TransportBuilder_frontConfigBuilder(self) + + +# Register TransportBuilder in _wedpr_python_transport: +_wedpr_python_transport.TransportBuilder_swigregister(TransportBuilder) + + +class Transport(object): + thisown = property(lambda x: x.this.own(), lambda x, + v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self, config): + _wedpr_python_transport.Transport_swiginit( + self, _wedpr_python_transport.new_Transport(config)) + __swig_destroy__ = _wedpr_python_transport.delete_Transport + + def start(self): + return _wedpr_python_transport.Transport_start(self) + + def stop(self): + return _wedpr_python_transport.Transport_stop(self) + + def getFront(self): + return _wedpr_python_transport.Transport_getFront(self) + + def gateway(self): + return _wedpr_python_transport.Transport_gateway(self) + + def msgPayloadBuilder(self): + return _wedpr_python_transport.Transport_msgPayloadBuilder(self) + + def msgBuilder(self): + return _wedpr_python_transport.Transport_msgBuilder(self) + + def msgHeaderBuilder(self): + return _wedpr_python_transport.Transport_msgHeaderBuilder(self) + + def routeInfoBuilder(self): + return _wedpr_python_transport.Transport_routeInfoBuilder(self) + + def getConfig(self): + return _wedpr_python_transport.Transport_getConfig(self) + + +# Register Transport in _wedpr_python_transport: +_wedpr_python_transport.Transport_swigregister(Transport) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py new file mode 100644 index 00000000..59f5ef68 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py @@ -0,0 +1,11 @@ +from transport.impl.message_impl import MessageImpl +from transport.api.message_api import MessageAPI +from transport.generated.wedpr_python_transport import Message + + +class MessageFactory: + @staticmethod + def build(message: Message) -> MessageAPI: + if message is None: + return None + return MessageImpl(message) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py new file mode 100644 index 00000000..218462ba --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +from transport.api.message_api import MessageHeaderAPI +from transport.api.message_api import MessageAPI +from transport.generated.wedpr_python_transport import Message +from transport.generated.wedpr_python_transport import MessageHeader + + +class MessageHeaderImpl(MessageHeaderAPI): + + def __init__(self, header: MessageHeader): + if header is None: + raise Exception( + "Build MessageHeader failed for empty header object passed in") + self.__header = header + + def get_version(self) -> str: + return self.__header.version() + + def get_trace_id(self) -> str: + return self.__header.traceID() + + def get_src_gw_node(self) -> str: + return self.__header.srcGwNode() + + def get_dst_gw_node(self) -> str: + return self.__header.dstGwNode() + + def get_packet_type(self) -> int: + return self.__header.packetType() + + def get_ttl(self) -> int: + return self.__header.ttl() + + def get_ext(self) -> int: + return self.__header.exit() + + def is_resp_packet(self) -> bool: + return self.__header.isRespPacket() + + def get_route_type(self) -> int: + return self.__header.routeType() + + def get_component_type(self) -> str: + return self.__header.optionalField().componentType() + + def get_src_node(self) -> bytes: + return self.__header.optionalField().srcNodeBuffer() + + def get_dst_node(self) -> bytes: + return self.__header.optionalField().dstNodeBuffer() + + def get_dst_inst(self) -> str: + return self.__header.optionalField().dstInst() + + def get_src_inst(self) -> str: + return self.__header.optionalField().srcInst() + + def get_topic(self) -> str: + return self.__header.optionalField().topic() + + def detail(self) -> str: + return f"version: {self.get_version()}, topic: {self.get_topic()}, src_inst: {self.get_src_inst()},src_node: {str(self.get_src_node())}, dst_inst: {self.get_dst_inst()}, dst_node: {str(self.get_dst_node())}" + + +class MessageImpl(MessageAPI): + def __init__(self, message: Message): + if message is None: + raise Exception( + "Build Message failed for empty message object passed in") + self.__message = message + self.__message_header = MessageHeaderImpl(self.__message.header()) + + def get_header(self) -> MessageHeaderAPI: + return self.__message_header + + def get_seq(self) -> int: + return self.__message.frontMessage().seq() + + def get_payload(self) -> bytes: + return self.__message.frontMessage().dataBuffer() + + def get_length(self) -> int: + return self.__message.frontMessage().length() + + def detail(self) -> str: + return f"seq: {self.get_seq()}, header: {self.get_header().detail()}, length: {self.get_length()}" diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py new file mode 100644 index 00000000..d7fe8037 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from transport.generated.wedpr_python_transport import MessageOptionalHeaderBuilder +from transport.generated.wedpr_python_transport import MessageOptionalHeader + + +class RouteInfoBuilder: + def __init__(self, builder: MessageOptionalHeaderBuilder): + self.__route_info_builder = builder + + def build(self, topic: str, dst_node: bytes, dst_inst: str, component: str) -> MessageOptionalHeader: + routeInfo = self.__route_info_builder.build() + if topic is not None: + routeInfo.setTopic(topic) + if dst_node is not None: + routeInfo.setDstNodeBuffer(dst_node) + if dst_inst is not None: + routeInfo.setDstInst(dst_inst) + if component is not None: + routeInfo.setComponentType(component) + return routeInfo diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py new file mode 100644 index 00000000..b6239b6c --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +from transport.generated.wedpr_python_transport import TransportBuilder +from transport.generated.wedpr_python_transport import Transport +from transport.generated.wedpr_python_transport import Error +from transport.api.message_api import MessageAPI +from transport.impl.route_info_builder import RouteInfoBuilder +from transport.impl.message_factory import MessageFactory +from transport.generated.wedpr_python_transport import MessageOptionalHeader +from transport.api.transport_api import TransportAPI + +from enum import Enum +import signal + + +class RouteType(Enum): + ROUTE_THROUGH_NODEID = 0 + ROUTE_THROUGH_COMPONENT = 1 + ROUTE_THROUGH_AGENCY = 2 + ROUTE_THROUGH_TOPIC = 3 + + +class Transport(TransportAPI): + should_exit = False + + def __init__(self, transport: Transport): + self.__transport = transport + self.__route_info_builder = RouteInfoBuilder( + self.__transport.routeInfoBuilder()) + + def start(self): + self.__transport.start() + + def stop(self): + self.__transport.stop() + + def _push_msg(self, route_type: int, route_info: MessageOptionalHeader, payload: bytes, seq: int, timeout: int): + try: + return self.__transport.getFront().push_msg(route_type, route_info, payload, seq, timeout) + except Exception as e: + raise e + + def push_by_nodeid(self, topic: str, dstNode: bytes, seq: int, payload: bytes, timeout: int): + route_info = self.__route_info_builder.build( + topic=topic, dst_node=dstNode, dst_inst=None, component=None) + result = self._push_msg( + RouteType.ROUTE_THROUGH_NODEID.value, route_info, payload, seq, timeout) + Transport.check_result("push_by_nodeid", result) + + def push_by_inst(self, topic: str, dstInst: str, seq: int, payload: bytes, timeout: int): + route_info = self.__route_info_builder.build( + topic=topic, dst_node=None, dst_inst=dstInst, component=None) + result = self._push_msg( + RouteType.ROUTE_THROUGH_AGENCY.value, route_info, payload, len(payload), seq, timeout) + Transport.check_result("push_by_inst", result) + + def push_by_component(self, topic: str, dstInst: str, component: str, seq: int, payload: bytes, timeout: int): + route_info = self.__route_info_builder.build( + topic=topic, dst_node=None, dst_inst=dstInst, component=component) + result = self._push_msg( + RouteType.ROUTE_THROUGH_COMPONENT.value, route_info, payload, len(payload), seq, timeout) + Transport.check_result("push_by_component", result) + + def pop(self, topic, timeout_ms) -> MessageAPI: + return MessageFactory.build(self.__transport.getFront().pop(topic, timeout_ms)) + + def peek(self, topic): + return self.__transport.peek(topic) + + @staticmethod + def check_result(method: str, result: Error): + if result is None: + return + if result.errorCode() != 0: + raise Exception( + f"call {method} failed for {result.errorMessage()}, code: {result.errorCode()}") + + def register_topic(self, topic): + result = self.__transport.getFront().registerTopic(topic) + Transport.check_result("register_topic", result) + + def unregister_topic(self, topic): + result = self.__transport.getFront().unRegisterTopic(topic) + Transport.check_result("unregister_topic", result) + + def register_component(self, component): + result = self.__transport.getFront().registerComponent(component) + Transport.check_result("register_component", result) + + def unregister_component(self, component): + result = self.__transport.getFront().unRegisterComponent(component) + Transport.check_result("unregister_component", result) + + +def signal_handler(sig, frame): + print('You pressed Ctrl+C! Exiting gracefully.') + Transport.should_exit = True + + +signal.signal(signal.SIGINT, signal_handler) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py new file mode 100644 index 00000000..9b7d7487 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +from transport.generated.wedpr_python_transport import FrontConfig +from transport.generated.wedpr_python_transport import TransportBuilder +from transport.generated.wedpr_python_transport import EndPoint +from transport.generated.wedpr_python_transport import GrpcConfig + + +class TransportConfig: + """ + the transport config + """ + + def __init__(self, threadpool_size: int, nodeID: str, gateway_targets: str): + self.__transport_builder = TransportBuilder() + self.__front_config = self.__transport_builder.buildConfig( + threadpool_size, nodeID) + self.__front_config.setGatewayGrpcTarget(gateway_targets) + + def get_front_config(self) -> FrontConfig: + return self.__front_config + + def set_self_endpoint(self, host: str, port: int, listen_ip: str = None): + endPoint = EndPoint(host, port) + if listen_ip is None: + listen_ip = "0.0.0.0" + endPoint.setListenIp(listen_ip) + self.__front_config.setSelfEndPoint(endPoint) + + def set_grpc_config(self, grpc_config: GrpcConfig): + self.__front_config.set_grpc_config(grpc_config) + + def get_thread_pool_size(self) -> int: + return self.__front_config.threadPoolSize() + + def get_node_id(self) -> str: + return self.__front_config.nodeID() + + def get_gateway_targets(self) -> str: + return self.__front_config.gatewayGrpcTarget() + + def get_self_endpoint(self) -> EndPoint: + return self.__front_config.selfEndPoint() + + def desc(self): + return f"thread_pool_size: {self.get_thread_pool_size()}, \ + nodeID: {self.get_node_id()}, \ + gatewayTargets: {self.get_gateway_targets()}, \ + endPoint: {self.get_self_endpoint().entryPoint()}" diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py new file mode 100644 index 00000000..02b0c547 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from transport.impl.transport_config import TransportConfig +from transport.generated.wedpr_python_transport import TransportBuilder +from transport.impl.transport import Transport +import signal + + +class TransportLoader: + transport_builder = TransportBuilder() + + @staticmethod + def load(transport_config: TransportConfig) -> Transport: + transport = TransportLoader.transport_builder.buildProTransport( + transport_config.get_front_config()) + return Transport(transport) diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py index dbc9a78b..e469aca7 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py @@ -79,11 +79,3 @@ def get_lib_name(): return "{}{}.{}".format(LibLoader.LIB_PREFIX, LibLoader.TRANSPORT_LIB_NAME, LibLoader.LINUX_LIB_PORSTFIX) raise Exception( f"get_lib_name failed for not support the os_type: {os_type}") - - @staticmethod - def load_lib(): - lib_path = os.path.join( - LibLoader.TRANSPORT_LIB_PATH, LibLoader.get_lib_name()) - print(f"#### lib_path: {lib_path}") - lib = ctypes.cdll.LoadLibrary(lib_path) - return lib diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py index 2f287a85..8de1e1cd 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py +++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py @@ -4,8 +4,7 @@ class TestLibLoader(unittest.TestCase): def test_load_lib(self): - from utils.lib_loader import LibLoader - _wedpr_python_transport = LibLoader.load_lib() + LibLoader.get_lib_name() if __name__ == '__main__': diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx new file mode 100644 index 00000000..12d60c03 --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx @@ -0,0 +1,23571 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + + +#define SWIG_VERSION 0x040201 +#define SWIGPYTHON +#define SWIG_DIRECTORS +#define SWIG_PYTHON_DIRECTOR_NO_VTABLE + +/* ----------------------------------------------------------------------------- + * This section contains generic SWIG labels for method/variable + * declarations/attributes, and other compiler dependent labels. + * ----------------------------------------------------------------------------- */ + +/* template workaround for compilers that cannot correctly implement the C++ standard */ +#ifndef SWIGTEMPLATEDISAMBIGUATOR +# if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x560) +# define SWIGTEMPLATEDISAMBIGUATOR template +# elif defined(__HP_aCC) +/* Needed even with `aCC -AA' when `aCC -V' reports HP ANSI C++ B3910B A.03.55 */ +/* If we find a maximum version that requires this, the test would be __HP_aCC <= 35500 for A.03.55 */ +# define SWIGTEMPLATEDISAMBIGUATOR template +# else +# define SWIGTEMPLATEDISAMBIGUATOR +# endif +#endif + +/* inline attribute */ +#ifndef SWIGINLINE +# if defined(__cplusplus) || (defined(__GNUC__) && !defined(__STRICT_ANSI__)) +# define SWIGINLINE inline +# else +# define SWIGINLINE +# endif +#endif + +/* attribute recognised by some compilers to avoid 'unused' warnings */ +#ifndef SWIGUNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +# elif defined(__ICC) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +#endif + +#ifndef SWIG_MSC_UNSUPPRESS_4505 +# if defined(_MSC_VER) +# pragma warning(disable : 4505) /* unreferenced local function has been removed */ +# endif +#endif + +#ifndef SWIGUNUSEDPARM +# ifdef __cplusplus +# define SWIGUNUSEDPARM(p) +# else +# define SWIGUNUSEDPARM(p) p SWIGUNUSED +# endif +#endif + +/* internal SWIG method */ +#ifndef SWIGINTERN +# define SWIGINTERN static SWIGUNUSED +#endif + +/* internal inline SWIG method */ +#ifndef SWIGINTERNINLINE +# define SWIGINTERNINLINE SWIGINTERN SWIGINLINE +#endif + +/* exporting methods */ +#if defined(__GNUC__) +# if (__GNUC__ >= 4) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) +# ifndef GCC_HASCLASSVISIBILITY +# define GCC_HASCLASSVISIBILITY +# endif +# endif +#endif + +#ifndef SWIGEXPORT +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# if defined(STATIC_LINKED) +# define SWIGEXPORT +# else +# define SWIGEXPORT __declspec(dllexport) +# endif +# else +# if defined(__GNUC__) && defined(GCC_HASCLASSVISIBILITY) +# define SWIGEXPORT __attribute__ ((visibility("default"))) +# else +# define SWIGEXPORT +# endif +# endif +#endif + +/* calling conventions for Windows */ +#ifndef SWIGSTDCALL +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# define SWIGSTDCALL __stdcall +# else +# define SWIGSTDCALL +# endif +#endif + +/* Deal with Microsoft's attempt at deprecating C standard runtime functions */ +#if !defined(SWIG_NO_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_CRT_SECURE_NO_DEPRECATE) +# define _CRT_SECURE_NO_DEPRECATE +#endif + +/* Deal with Microsoft's attempt at deprecating methods in the standard C++ library */ +#if !defined(SWIG_NO_SCL_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_SCL_SECURE_NO_DEPRECATE) +# define _SCL_SECURE_NO_DEPRECATE +#endif + +/* Deal with Apple's deprecated 'AssertMacros.h' from Carbon-framework */ +#if defined(__APPLE__) && !defined(__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES) +# define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 +#endif + +/* Intel's compiler complains if a variable which was never initialised is + * cast to void, which is a common idiom which we use to indicate that we + * are aware a variable isn't used. So we just silence that warning. + * See: https://github.com/swig/swig/issues/192 for more discussion. + */ +#ifdef __INTEL_COMPILER +# pragma warning disable 592 +#endif + +#if defined(__cplusplus) && __cplusplus >=201103L +# define SWIG_NULLPTR nullptr +#else +# define SWIG_NULLPTR NULL +#endif + +/* ----------------------------------------------------------------------------- + * swigcompat.swg + * + * Macros to provide support compatibility with older C and C++ standards. + * ----------------------------------------------------------------------------- */ + +/* C99 and C++11 should provide snprintf, but define SWIG_NO_SNPRINTF + * if you're missing it. + */ +#if ((defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L) || \ + (defined __cplusplus && __cplusplus >= 201103L) || \ + defined SWIG_HAVE_SNPRINTF) && \ + !defined SWIG_NO_SNPRINTF +# define SWIG_snprintf(O,S,F,A) snprintf(O,S,F,A) +# define SWIG_snprintf2(O,S,F,A,B) snprintf(O,S,F,A,B) +#else +/* Fallback versions ignore the buffer size, but most of our uses either have a + * fixed maximum possible size or dynamically allocate a buffer that's large + * enough. + */ +# define SWIG_snprintf(O,S,F,A) sprintf(O,F,A) +# define SWIG_snprintf2(O,S,F,A,B) sprintf(O,F,A,B) +#endif + + +#if defined(__GNUC__) && defined(_WIN32) && !defined(SWIG_PYTHON_NO_HYPOT_WORKAROUND) +/* Workaround for '::hypot' has not been declared', see https://bugs.python.org/issue11566 */ +# include +#endif + +#if !defined(PY_SSIZE_T_CLEAN) && !defined(SWIG_NO_PY_SSIZE_T_CLEAN) +#define PY_SSIZE_T_CLEAN +#endif + +#if __GNUC__ >= 7 +#pragma GCC diagnostic push +#if defined(__cplusplus) && __cplusplus >=201703L +#pragma GCC diagnostic ignored "-Wregister" /* For python-2.7 headers that use register */ +#endif +#endif + +#if defined(_DEBUG) && defined(SWIG_PYTHON_INTERPRETER_NO_DEBUG) +/* Use debug wrappers with the Python release dll */ + +#if defined(_MSC_VER) && _MSC_VER >= 1929 +/* Workaround compilation errors when redefining _DEBUG in MSVC 2019 version 16.10 and later + * See https://github.com/swig/swig/issues/2090 */ +# include +#endif + +# undef _DEBUG +# include +# define _DEBUG 1 +#else +# include +#endif + +#if __GNUC__ >= 7 +#pragma GCC diagnostic pop +#endif + +#include + +/* ----------------------------------------------------------------------------- + * swigrun.swg + * + * This file contains generic C API SWIG runtime support for pointer + * type checking. + * ----------------------------------------------------------------------------- */ + +/* This should only be incremented when either the layout of swig_type_info changes, + or for whatever reason, the runtime changes incompatibly */ +#define SWIG_RUNTIME_VERSION "4" + +/* define SWIG_TYPE_TABLE_NAME as "SWIG_TYPE_TABLE" */ +#ifdef SWIG_TYPE_TABLE +# define SWIG_QUOTE_STRING(x) #x +# define SWIG_EXPAND_AND_QUOTE_STRING(x) SWIG_QUOTE_STRING(x) +# define SWIG_TYPE_TABLE_NAME SWIG_EXPAND_AND_QUOTE_STRING(SWIG_TYPE_TABLE) +#else +# define SWIG_TYPE_TABLE_NAME +#endif + +/* + You can use the SWIGRUNTIME and SWIGRUNTIMEINLINE macros for + creating a static or dynamic library from the SWIG runtime code. + In 99.9% of the cases, SWIG just needs to declare them as 'static'. + + But only do this if strictly necessary, ie, if you have problems + with your compiler or suchlike. +*/ + +#ifndef SWIGRUNTIME +# define SWIGRUNTIME SWIGINTERN +#endif + +#ifndef SWIGRUNTIMEINLINE +# define SWIGRUNTIMEINLINE SWIGRUNTIME SWIGINLINE +#endif + +/* Generic buffer size */ +#ifndef SWIG_BUFFER_SIZE +# define SWIG_BUFFER_SIZE 1024 +#endif + +/* Flags for pointer conversions */ +#define SWIG_POINTER_DISOWN 0x1 +#define SWIG_CAST_NEW_MEMORY 0x2 +#define SWIG_POINTER_NO_NULL 0x4 +#define SWIG_POINTER_CLEAR 0x8 +#define SWIG_POINTER_RELEASE (SWIG_POINTER_CLEAR | SWIG_POINTER_DISOWN) + +/* Flags for new pointer objects */ +#define SWIG_POINTER_OWN 0x1 + + +/* + Flags/methods for returning states. + + The SWIG conversion methods, as ConvertPtr, return an integer + that tells if the conversion was successful or not. And if not, + an error code can be returned (see swigerrors.swg for the codes). + + Use the following macros/flags to set or process the returning + states. + + In old versions of SWIG, code such as the following was usually written: + + if (SWIG_ConvertPtr(obj,vptr,ty.flags) != -1) { + // success code + } else { + //fail code + } + + Now you can be more explicit: + + int res = SWIG_ConvertPtr(obj,vptr,ty.flags); + if (SWIG_IsOK(res)) { + // success code + } else { + // fail code + } + + which is the same really, but now you can also do + + Type *ptr; + int res = SWIG_ConvertPtr(obj,(void **)(&ptr),ty.flags); + if (SWIG_IsOK(res)) { + // success code + if (SWIG_IsNewObj(res) { + ... + delete *ptr; + } else { + ... + } + } else { + // fail code + } + + I.e., now SWIG_ConvertPtr can return new objects and you can + identify the case and take care of the deallocation. Of course that + also requires SWIG_ConvertPtr to return new result values, such as + + int SWIG_ConvertPtr(obj, ptr,...) { + if () { + if () { + *ptr = ; + return SWIG_NEWOBJ; + } else { + *ptr = ; + return SWIG_OLDOBJ; + } + } else { + return SWIG_BADOBJ; + } + } + + Of course, returning the plain '0(success)/-1(fail)' still works, but you can be + more explicit by returning SWIG_BADOBJ, SWIG_ERROR or any of the + SWIG errors code. + + Finally, if the SWIG_CASTRANK_MODE is enabled, the result code + allows returning the 'cast rank', for example, if you have this + + int food(double) + int fooi(int); + + and you call + + food(1) // cast rank '1' (1 -> 1.0) + fooi(1) // cast rank '0' + + just use the SWIG_AddCast()/SWIG_CheckState() +*/ + +#define SWIG_OK (0) +/* Runtime errors are < 0 */ +#define SWIG_ERROR (-1) +/* Errors in range -1 to -99 are in swigerrors.swg (errors for all languages including those not using the runtime) */ +/* Errors in range -100 to -199 are language specific errors defined in *errors.swg */ +/* Errors < -200 are generic runtime specific errors */ +#define SWIG_ERROR_RELEASE_NOT_OWNED (-200) + +#define SWIG_IsOK(r) (r >= 0) +#define SWIG_ArgError(r) ((r != SWIG_ERROR) ? r : SWIG_TypeError) + +/* The CastRankLimit says how many bits are used for the cast rank */ +#define SWIG_CASTRANKLIMIT (1 << 8) +/* The NewMask denotes the object was created (using new/malloc) */ +#define SWIG_NEWOBJMASK (SWIG_CASTRANKLIMIT << 1) +/* The TmpMask is for in/out typemaps that use temporary objects */ +#define SWIG_TMPOBJMASK (SWIG_NEWOBJMASK << 1) +/* Simple returning values */ +#define SWIG_BADOBJ (SWIG_ERROR) +#define SWIG_OLDOBJ (SWIG_OK) +#define SWIG_NEWOBJ (SWIG_OK | SWIG_NEWOBJMASK) +#define SWIG_TMPOBJ (SWIG_OK | SWIG_TMPOBJMASK) +/* Check, add and del object mask methods */ +#define SWIG_AddNewMask(r) (SWIG_IsOK(r) ? (r | SWIG_NEWOBJMASK) : r) +#define SWIG_DelNewMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_NEWOBJMASK) : r) +#define SWIG_IsNewObj(r) (SWIG_IsOK(r) && (r & SWIG_NEWOBJMASK)) +#define SWIG_AddTmpMask(r) (SWIG_IsOK(r) ? (r | SWIG_TMPOBJMASK) : r) +#define SWIG_DelTmpMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_TMPOBJMASK) : r) +#define SWIG_IsTmpObj(r) (SWIG_IsOK(r) && (r & SWIG_TMPOBJMASK)) + +/* Cast-Rank Mode */ +#if defined(SWIG_CASTRANK_MODE) +# ifndef SWIG_TypeRank +# define SWIG_TypeRank unsigned long +# endif +# ifndef SWIG_MAXCASTRANK /* Default cast allowed */ +# define SWIG_MAXCASTRANK (2) +# endif +# define SWIG_CASTRANKMASK ((SWIG_CASTRANKLIMIT) -1) +# define SWIG_CastRank(r) (r & SWIG_CASTRANKMASK) +SWIGINTERNINLINE int SWIG_AddCast(int r) { + return SWIG_IsOK(r) ? ((SWIG_CastRank(r) < SWIG_MAXCASTRANK) ? (r + 1) : SWIG_ERROR) : r; +} +SWIGINTERNINLINE int SWIG_CheckState(int r) { + return SWIG_IsOK(r) ? SWIG_CastRank(r) + 1 : 0; +} +#else /* no cast-rank mode */ +# define SWIG_AddCast(r) (r) +# define SWIG_CheckState(r) (SWIG_IsOK(r) ? 1 : 0) +#endif + + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*swig_converter_func)(void *, int *); +typedef struct swig_type_info *(*swig_dycast_func)(void **); + +/* Structure to store information on one type */ +typedef struct swig_type_info { + const char *name; /* mangled name of this type */ + const char *str; /* human readable name of this type */ + swig_dycast_func dcast; /* dynamic cast function down a hierarchy */ + struct swig_cast_info *cast; /* linked list of types that can cast into this type */ + void *clientdata; /* language specific type data */ + int owndata; /* flag if the structure owns the clientdata */ +} swig_type_info; + +/* Structure to store a type and conversion function used for casting */ +typedef struct swig_cast_info { + swig_type_info *type; /* pointer to type that is equivalent to this type */ + swig_converter_func converter; /* function to cast the void pointers */ + struct swig_cast_info *next; /* pointer to next cast in linked list */ + struct swig_cast_info *prev; /* pointer to the previous cast */ +} swig_cast_info; + +/* Structure used to store module information + * Each module generates one structure like this, and the runtime collects + * all of these structures and stores them in a circularly linked list.*/ +typedef struct swig_module_info { + swig_type_info **types; /* Array of pointers to swig_type_info structures that are in this module */ + size_t size; /* Number of types in this module */ + struct swig_module_info *next; /* Pointer to next element in circularly linked list */ + swig_type_info **type_initial; /* Array of initially generated type structures */ + swig_cast_info **cast_initial; /* Array of initially generated casting structures */ + void *clientdata; /* Language specific module data */ +} swig_module_info; + +/* + Compare two type names skipping the space characters, therefore + "char*" == "char *" and "Class" == "Class", etc. + + Return 0 when the two name types are equivalent, as in + strncmp, but skipping ' '. +*/ +SWIGRUNTIME int +SWIG_TypeNameComp(const char *f1, const char *l1, + const char *f2, const char *l2) { + for (;(f1 != l1) && (f2 != l2); ++f1, ++f2) { + while ((*f1 == ' ') && (f1 != l1)) ++f1; + while ((*f2 == ' ') && (f2 != l2)) ++f2; + if (*f1 != *f2) return (*f1 > *f2) ? 1 : -1; + } + return (int)((l1 - f1) - (l2 - f2)); +} + +/* + Check type equivalence in a name list like ||... + Return 0 if equal, -1 if nb < tb, 1 if nb > tb +*/ +SWIGRUNTIME int +SWIG_TypeCmp(const char *nb, const char *tb) { + int equiv = 1; + const char* te = tb + strlen(tb); + const char* ne = nb; + while (equiv != 0 && *ne) { + for (nb = ne; *ne; ++ne) { + if (*ne == '|') break; + } + equiv = SWIG_TypeNameComp(nb, ne, tb, te); + if (*ne) ++ne; + } + return equiv; +} + +/* + Check type equivalence in a name list like ||... + Return 0 if not equal, 1 if equal +*/ +SWIGRUNTIME int +SWIG_TypeEquiv(const char *nb, const char *tb) { + return SWIG_TypeCmp(nb, tb) == 0 ? 1 : 0; +} + +/* + Check the typename +*/ +SWIGRUNTIME swig_cast_info * +SWIG_TypeCheck(const char *c, swig_type_info *ty) { + if (ty) { + swig_cast_info *iter = ty->cast; + while (iter) { + if (strcmp(iter->type->name, c) == 0) { + if (iter == ty->cast) + return iter; + /* Move iter to the top of the linked list */ + iter->prev->next = iter->next; + if (iter->next) + iter->next->prev = iter->prev; + iter->next = ty->cast; + iter->prev = 0; + if (ty->cast) ty->cast->prev = iter; + ty->cast = iter; + return iter; + } + iter = iter->next; + } + } + return 0; +} + +/* + Identical to SWIG_TypeCheck, except strcmp is replaced with a pointer comparison +*/ +SWIGRUNTIME swig_cast_info * +SWIG_TypeCheckStruct(const swig_type_info *from, swig_type_info *ty) { + if (ty) { + swig_cast_info *iter = ty->cast; + while (iter) { + if (iter->type == from) { + if (iter == ty->cast) + return iter; + /* Move iter to the top of the linked list */ + iter->prev->next = iter->next; + if (iter->next) + iter->next->prev = iter->prev; + iter->next = ty->cast; + iter->prev = 0; + if (ty->cast) ty->cast->prev = iter; + ty->cast = iter; + return iter; + } + iter = iter->next; + } + } + return 0; +} + +/* + Cast a pointer up an inheritance hierarchy +*/ +SWIGRUNTIMEINLINE void * +SWIG_TypeCast(swig_cast_info *ty, void *ptr, int *newmemory) { + return ((!ty) || (!ty->converter)) ? ptr : (*ty->converter)(ptr, newmemory); +} + +/* + Dynamic pointer casting. Down an inheritance hierarchy +*/ +SWIGRUNTIME swig_type_info * +SWIG_TypeDynamicCast(swig_type_info *ty, void **ptr) { + swig_type_info *lastty = ty; + if (!ty || !ty->dcast) return ty; + while (ty && (ty->dcast)) { + ty = (*ty->dcast)(ptr); + if (ty) lastty = ty; + } + return lastty; +} + +/* + Return the name associated with this type +*/ +SWIGRUNTIMEINLINE const char * +SWIG_TypeName(const swig_type_info *ty) { + return ty->name; +} + +/* + Return the pretty name associated with this type, + that is an unmangled type name in a form presentable to the user. +*/ +SWIGRUNTIME const char * +SWIG_TypePrettyName(const swig_type_info *type) { + /* The "str" field contains the equivalent pretty names of the + type, separated by vertical-bar characters. Choose the last + name. It should be the most specific; a fully resolved name + but not necessarily with default template parameters expanded. */ + if (!type) return NULL; + if (type->str != NULL) { + const char *last_name = type->str; + const char *s; + for (s = type->str; *s; s++) + if (*s == '|') last_name = s+1; + return last_name; + } + else + return type->name; +} + +/* + Set the clientdata field for a type +*/ +SWIGRUNTIME void +SWIG_TypeClientData(swig_type_info *ti, void *clientdata) { + swig_cast_info *cast = ti->cast; + /* if (ti->clientdata == clientdata) return; */ + ti->clientdata = clientdata; + + while (cast) { + if (!cast->converter) { + swig_type_info *tc = cast->type; + if (!tc->clientdata) { + SWIG_TypeClientData(tc, clientdata); + } + } + cast = cast->next; + } +} +SWIGRUNTIME void +SWIG_TypeNewClientData(swig_type_info *ti, void *clientdata) { + SWIG_TypeClientData(ti, clientdata); + ti->owndata = 1; +} + +/* + Search for a swig_type_info structure only by mangled name + Search is a O(log #types) + + We start searching at module start, and finish searching when start == end. + Note: if start == end at the beginning of the function, we go all the way around + the circular list. +*/ +SWIGRUNTIME swig_type_info * +SWIG_MangledTypeQueryModule(swig_module_info *start, + swig_module_info *end, + const char *name) { + swig_module_info *iter = start; + do { + if (iter->size) { + size_t l = 0; + size_t r = iter->size - 1; + do { + /* since l+r >= 0, we can (>> 1) instead (/ 2) */ + size_t i = (l + r) >> 1; + const char *iname = iter->types[i]->name; + if (iname) { + int compare = strcmp(name, iname); + if (compare == 0) { + return iter->types[i]; + } else if (compare < 0) { + if (i) { + r = i - 1; + } else { + break; + } + } else if (compare > 0) { + l = i + 1; + } + } else { + break; /* should never happen */ + } + } while (l <= r); + } + iter = iter->next; + } while (iter != end); + return 0; +} + +/* + Search for a swig_type_info structure for either a mangled name or a human readable name. + It first searches the mangled names of the types, which is a O(log #types) + If a type is not found it then searches the human readable names, which is O(#types). + + We start searching at module start, and finish searching when start == end. + Note: if start == end at the beginning of the function, we go all the way around + the circular list. +*/ +SWIGRUNTIME swig_type_info * +SWIG_TypeQueryModule(swig_module_info *start, + swig_module_info *end, + const char *name) { + /* STEP 1: Search the name field using binary search */ + swig_type_info *ret = SWIG_MangledTypeQueryModule(start, end, name); + if (ret) { + return ret; + } else { + /* STEP 2: If the type hasn't been found, do a complete search + of the str field (the human readable name) */ + swig_module_info *iter = start; + do { + size_t i = 0; + for (; i < iter->size; ++i) { + if (iter->types[i]->str && (SWIG_TypeEquiv(iter->types[i]->str, name))) + return iter->types[i]; + } + iter = iter->next; + } while (iter != end); + } + + /* neither found a match */ + return 0; +} + +/* + Pack binary data into a string +*/ +SWIGRUNTIME char * +SWIG_PackData(char *c, void *ptr, size_t sz) { + static const char hex[17] = "0123456789abcdef"; + const unsigned char *u = (unsigned char *) ptr; + const unsigned char *eu = u + sz; + for (; u != eu; ++u) { + unsigned char uu = *u; + *(c++) = hex[(uu & 0xf0) >> 4]; + *(c++) = hex[uu & 0xf]; + } + return c; +} + +/* + Unpack binary data from a string +*/ +SWIGRUNTIME const char * +SWIG_UnpackData(const char *c, void *ptr, size_t sz) { + unsigned char *u = (unsigned char *) ptr; + const unsigned char *eu = u + sz; + for (; u != eu; ++u) { + char d = *(c++); + unsigned char uu; + if ((d >= '0') && (d <= '9')) + uu = (unsigned char)((d - '0') << 4); + else if ((d >= 'a') && (d <= 'f')) + uu = (unsigned char)((d - ('a'-10)) << 4); + else + return (char *) 0; + d = *(c++); + if ((d >= '0') && (d <= '9')) + uu |= (unsigned char)(d - '0'); + else if ((d >= 'a') && (d <= 'f')) + uu |= (unsigned char)(d - ('a'-10)); + else + return (char *) 0; + *u = uu; + } + return c; +} + +/* + Pack 'void *' into a string buffer. +*/ +SWIGRUNTIME char * +SWIG_PackVoidPtr(char *buff, void *ptr, const char *name, size_t bsz) { + char *r = buff; + if ((2*sizeof(void *) + 2) > bsz) return 0; + *(r++) = '_'; + r = SWIG_PackData(r,&ptr,sizeof(void *)); + if (strlen(name) + 1 > (bsz - (r - buff))) return 0; + strcpy(r,name); + return buff; +} + +SWIGRUNTIME const char * +SWIG_UnpackVoidPtr(const char *c, void **ptr, const char *name) { + if (*c != '_') { + if (strcmp(c,"NULL") == 0) { + *ptr = (void *) 0; + return name; + } else { + return 0; + } + } + return SWIG_UnpackData(++c,ptr,sizeof(void *)); +} + +SWIGRUNTIME char * +SWIG_PackDataName(char *buff, void *ptr, size_t sz, const char *name, size_t bsz) { + char *r = buff; + size_t lname = (name ? strlen(name) : 0); + if ((2*sz + 2 + lname) > bsz) return 0; + *(r++) = '_'; + r = SWIG_PackData(r,ptr,sz); + if (lname) { + strncpy(r,name,lname+1); + } else { + *r = 0; + } + return buff; +} + +SWIGRUNTIME const char * +SWIG_UnpackDataName(const char *c, void *ptr, size_t sz, const char *name) { + if (*c != '_') { + if (strcmp(c,"NULL") == 0) { + memset(ptr,0,sz); + return name; + } else { + return 0; + } + } + return SWIG_UnpackData(++c,ptr,sz); +} + +#ifdef __cplusplus +} +#endif + +/* SWIG Errors applicable to all language modules, values are reserved from -1 to -99 */ +#define SWIG_UnknownError -1 +#define SWIG_IOError -2 +#define SWIG_RuntimeError -3 +#define SWIG_IndexError -4 +#define SWIG_TypeError -5 +#define SWIG_DivisionByZero -6 +#define SWIG_OverflowError -7 +#define SWIG_SyntaxError -8 +#define SWIG_ValueError -9 +#define SWIG_SystemError -10 +#define SWIG_AttributeError -11 +#define SWIG_MemoryError -12 +#define SWIG_NullReferenceError -13 + + +/* Compatibility macros for Python 3 */ +#if PY_VERSION_HEX >= 0x03000000 + +#define PyClass_Check(obj) PyObject_IsInstance(obj, (PyObject *)&PyType_Type) +#define PyInt_Check(x) PyLong_Check(x) +#define PyInt_AsLong(x) PyLong_AsLong(x) +#define PyInt_FromLong(x) PyLong_FromLong(x) +#define PyInt_FromSize_t(x) PyLong_FromSize_t(x) +#define PyString_Check(name) PyBytes_Check(name) +#define PyString_FromString(x) PyUnicode_FromString(x) +#define PyString_Format(fmt, args) PyUnicode_Format(fmt, args) +#define PyString_AsString(str) PyBytes_AsString(str) +#define PyString_Size(str) PyBytes_Size(str) +#define PyString_InternFromString(key) PyUnicode_InternFromString(key) +#define Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_BASETYPE +#define _PyLong_FromSsize_t(x) PyLong_FromSsize_t(x) + +#endif + +#ifndef Py_TYPE +# define Py_TYPE(op) ((op)->ob_type) +#endif + +/* SWIG APIs for compatibility of both Python 2 & 3 */ + +#if PY_VERSION_HEX >= 0x03000000 +# define SWIG_Python_str_FromFormat PyUnicode_FromFormat +#else +# define SWIG_Python_str_FromFormat PyString_FromFormat +#endif + + +/* Wrapper around PyUnicode_AsUTF8AndSize - call Py_XDECREF on the returned pbytes when finished with the returned string */ +SWIGINTERN const char * +SWIG_PyUnicode_AsUTF8AndSize(PyObject *str, Py_ssize_t *psize, PyObject **pbytes) +{ +#if PY_VERSION_HEX >= 0x03030000 +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 + *pbytes = NULL; + return PyUnicode_AsUTF8AndSize(str, psize); +# else + *pbytes = PyUnicode_AsUTF8String(str); + const char *chars = *pbytes ? PyBytes_AsString(*pbytes) : NULL; + if (chars && psize) + *psize = PyBytes_Size(*pbytes); + return chars; +# endif +#else + char *chars = NULL; + *pbytes = NULL; + PyString_AsStringAndSize(str, &chars, psize); + return chars; +#endif +} + +SWIGINTERN PyObject* +SWIG_Python_str_FromChar(const char *c) +{ +#if PY_VERSION_HEX >= 0x03000000 + return PyUnicode_FromString(c); +#else + return PyString_FromString(c); +#endif +} + +#ifndef PyObject_DEL +# define PyObject_DEL PyObject_Del +#endif + +/* SWIGPY_USE_CAPSULE is no longer used within SWIG itself, but some user interface files check for it. */ +# define SWIGPY_USE_CAPSULE +#ifdef SWIGPYTHON_BUILTIN +# define SWIGPY_CAPSULE_ATTR_NAME "type_pointer_capsule_builtin" SWIG_TYPE_TABLE_NAME +#else +# define SWIGPY_CAPSULE_ATTR_NAME "type_pointer_capsule" SWIG_TYPE_TABLE_NAME +#endif +# define SWIGPY_CAPSULE_NAME ("swig_runtime_data" SWIG_RUNTIME_VERSION "." SWIGPY_CAPSULE_ATTR_NAME) + +#if PY_VERSION_HEX < 0x03020000 +#define PyDescr_TYPE(x) (((PyDescrObject *)(x))->d_type) +#define PyDescr_NAME(x) (((PyDescrObject *)(x))->d_name) +#define Py_hash_t long +#endif + +#ifdef Py_LIMITED_API +# define PyTuple_GET_ITEM PyTuple_GetItem +/* Note that PyTuple_SetItem() has different semantics from PyTuple_SET_ITEM as it decref's the original tuple item, so in general they cannot be used + interchangeably. However in SWIG-generated code PyTuple_SET_ITEM is only used with newly initialized tuples without any items and for them this does work. */ +# define PyTuple_SET_ITEM PyTuple_SetItem +# define PyTuple_GET_SIZE PyTuple_Size +# define PyCFunction_GET_FLAGS PyCFunction_GetFlags +# define PyCFunction_GET_FUNCTION PyCFunction_GetFunction +# define PyCFunction_GET_SELF PyCFunction_GetSelf +# define PyList_GET_ITEM PyList_GetItem +# define PyList_SET_ITEM PyList_SetItem +# define PySliceObject PyObject +#endif + +/* ----------------------------------------------------------------------------- + * error manipulation + * ----------------------------------------------------------------------------- */ + +SWIGRUNTIME PyObject* +SWIG_Python_ErrorType(int code) { + PyObject* type = 0; + switch(code) { + case SWIG_MemoryError: + type = PyExc_MemoryError; + break; + case SWIG_IOError: + type = PyExc_IOError; + break; + case SWIG_RuntimeError: + type = PyExc_RuntimeError; + break; + case SWIG_IndexError: + type = PyExc_IndexError; + break; + case SWIG_TypeError: + type = PyExc_TypeError; + break; + case SWIG_DivisionByZero: + type = PyExc_ZeroDivisionError; + break; + case SWIG_OverflowError: + type = PyExc_OverflowError; + break; + case SWIG_SyntaxError: + type = PyExc_SyntaxError; + break; + case SWIG_ValueError: + type = PyExc_ValueError; + break; + case SWIG_SystemError: + type = PyExc_SystemError; + break; + case SWIG_AttributeError: + type = PyExc_AttributeError; + break; + default: + type = PyExc_RuntimeError; + } + return type; +} + + +SWIGRUNTIME void +SWIG_Python_AddErrorMsg(const char* mesg) +{ + PyObject *type = 0; + PyObject *value = 0; + PyObject *traceback = 0; + + if (PyErr_Occurred()) + PyErr_Fetch(&type, &value, &traceback); + if (value) { + PyObject *old_str = PyObject_Str(value); + PyObject *bytes = NULL; + const char *tmp = SWIG_PyUnicode_AsUTF8AndSize(old_str, NULL, &bytes); + PyErr_Clear(); + Py_XINCREF(type); + if (tmp) + PyErr_Format(type, "%s %s", tmp, mesg); + else + PyErr_Format(type, "%s", mesg); + Py_XDECREF(bytes); + Py_DECREF(old_str); + Py_DECREF(value); + } else { + PyErr_SetString(PyExc_RuntimeError, mesg); + } +} + +SWIGRUNTIME int +SWIG_Python_TypeErrorOccurred(PyObject *obj) +{ + PyObject *error; + if (obj) + return 0; + error = PyErr_Occurred(); + return error && PyErr_GivenExceptionMatches(error, PyExc_TypeError); +} + +SWIGRUNTIME void +SWIG_Python_RaiseOrModifyTypeError(const char *message) +{ + if (SWIG_Python_TypeErrorOccurred(NULL)) { + /* Use existing TypeError to preserve stacktrace and enhance with given message */ + PyObject *newvalue; + PyObject *type = NULL, *value = NULL, *traceback = NULL; + PyErr_Fetch(&type, &value, &traceback); +#if PY_VERSION_HEX >= 0x03000000 + newvalue = PyUnicode_FromFormat("%S\nAdditional information:\n%s", value, message); +#else + newvalue = PyString_FromFormat("%s\nAdditional information:\n%s", PyString_AsString(value), message); +#endif + if (newvalue) { + Py_XDECREF(value); + PyErr_Restore(type, newvalue, traceback); + } else { + PyErr_Restore(type, value, traceback); + } + } else { + /* Raise TypeError using given message */ + PyErr_SetString(PyExc_TypeError, message); + } +} + +#if defined(SWIG_PYTHON_NO_THREADS) +# if defined(SWIG_PYTHON_THREADS) +# undef SWIG_PYTHON_THREADS +# endif +#endif +#if defined(SWIG_PYTHON_THREADS) /* Threading support is enabled */ +# if !defined(SWIG_PYTHON_USE_GIL) && !defined(SWIG_PYTHON_NO_USE_GIL) +# define SWIG_PYTHON_USE_GIL +# endif +# if defined(SWIG_PYTHON_USE_GIL) /* Use PyGILState threads calls */ +# if !defined(SWIG_PYTHON_INITIALIZE_THREADS) +# if PY_VERSION_HEX < 0x03070000 +# define SWIG_PYTHON_INITIALIZE_THREADS PyEval_InitThreads() +# else +# define SWIG_PYTHON_INITIALIZE_THREADS +# endif +# endif +# ifdef __cplusplus /* C++ code */ + class SWIG_Python_Thread_Block { + bool status; + PyGILState_STATE state; + public: + void end() { if (status) { PyGILState_Release(state); status = false;} } + SWIG_Python_Thread_Block() : status(true), state(PyGILState_Ensure()) {} + ~SWIG_Python_Thread_Block() { end(); } + }; + class SWIG_Python_Thread_Allow { + bool status; + PyThreadState *save; + public: + void end() { if (status) { status = false; PyEval_RestoreThread(save); }} + SWIG_Python_Thread_Allow() : status(true), save(PyEval_SaveThread()) {} + ~SWIG_Python_Thread_Allow() { end(); } + }; +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK SWIG_Python_Thread_Block _swig_thread_block +# define SWIG_PYTHON_THREAD_END_BLOCK _swig_thread_block.end() +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW SWIG_Python_Thread_Allow _swig_thread_allow +# define SWIG_PYTHON_THREAD_END_ALLOW _swig_thread_allow.end() +# else /* C code */ +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK PyGILState_STATE _swig_thread_block = PyGILState_Ensure() +# define SWIG_PYTHON_THREAD_END_BLOCK PyGILState_Release(_swig_thread_block) +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW PyThreadState *_swig_thread_allow = PyEval_SaveThread() +# define SWIG_PYTHON_THREAD_END_ALLOW PyEval_RestoreThread(_swig_thread_allow) +# endif +# else /* Old thread way, not implemented, user must provide it */ +# if !defined(SWIG_PYTHON_INITIALIZE_THREADS) +# define SWIG_PYTHON_INITIALIZE_THREADS +# endif +# if !defined(SWIG_PYTHON_THREAD_BEGIN_BLOCK) +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK +# endif +# if !defined(SWIG_PYTHON_THREAD_END_BLOCK) +# define SWIG_PYTHON_THREAD_END_BLOCK +# endif +# if !defined(SWIG_PYTHON_THREAD_BEGIN_ALLOW) +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW +# endif +# if !defined(SWIG_PYTHON_THREAD_END_ALLOW) +# define SWIG_PYTHON_THREAD_END_ALLOW +# endif +# endif +#else /* No thread support */ +# define SWIG_PYTHON_INITIALIZE_THREADS +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK +# define SWIG_PYTHON_THREAD_END_BLOCK +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW +# define SWIG_PYTHON_THREAD_END_ALLOW +#endif + +/* ----------------------------------------------------------------------------- + * Python API portion that goes into the runtime + * ----------------------------------------------------------------------------- */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Constant declarations + * ----------------------------------------------------------------------------- */ + +/* Constant Types */ +#define SWIG_PY_POINTER 4 +#define SWIG_PY_BINARY 5 + +/* Constant information structure */ +typedef struct swig_const_info { + int type; + const char *name; + long lvalue; + double dvalue; + void *pvalue; + swig_type_info **ptype; +} swig_const_info; + +#ifdef __cplusplus +} +#endif + + +/* ----------------------------------------------------------------------------- + * pyrun.swg + * + * This file contains the runtime support for Python modules + * and includes code for managing global variables and pointer + * type checking. + * + * ----------------------------------------------------------------------------- */ + +#if PY_VERSION_HEX < 0x02070000 /* 2.7.0 */ +# error "This version of SWIG only supports Python >= 2.7" +#endif + +#if PY_VERSION_HEX >= 0x03000000 && PY_VERSION_HEX < 0x03030000 +# error "This version of SWIG only supports Python 3 >= 3.3" +#endif + +/* Common SWIG API */ + +/* for raw pointers */ +#define SWIG_Python_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, 0) +#define SWIG_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtr(obj, pptr, type, flags) +#define SWIG_ConvertPtrAndOwn(obj,pptr,type,flags,own) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, own) + +#ifdef SWIGPYTHON_BUILTIN +#define SWIG_NewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(self, ptr, type, flags) +#else +#define SWIG_NewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(NULL, ptr, type, flags) +#endif + +#define SWIG_InternalNewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(NULL, ptr, type, flags) + +#define SWIG_CheckImplicit(ty) SWIG_Python_CheckImplicit(ty) +#define SWIG_AcquirePtr(ptr, src) SWIG_Python_AcquirePtr(ptr, src) +#define swig_owntype int + +/* for raw packed data */ +#define SWIG_ConvertPacked(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) +#define SWIG_NewPackedObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) + +/* for class or struct pointers */ +#define SWIG_ConvertInstance(obj, pptr, type, flags) SWIG_ConvertPtr(obj, pptr, type, flags) +#define SWIG_NewInstanceObj(ptr, type, flags) SWIG_NewPointerObj(ptr, type, flags) + +/* for C or C++ function pointers */ +#define SWIG_ConvertFunctionPtr(obj, pptr, type) SWIG_Python_ConvertFunctionPtr(obj, pptr, type) +#define SWIG_NewFunctionPtrObj(ptr, type) SWIG_Python_NewPointerObj(NULL, ptr, type, 0) + +/* for C++ member pointers, ie, member methods */ +#define SWIG_ConvertMember(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) +#define SWIG_NewMemberObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) + + +/* Runtime API */ + +#define SWIG_GetModule(clientdata) SWIG_Python_GetModule(clientdata) +#define SWIG_SetModule(clientdata, pointer) SWIG_Python_SetModule(pointer) +#define SWIG_NewClientData(obj) SwigPyClientData_New(obj) + +#define SWIG_SetErrorObj SWIG_Python_SetErrorObj +#define SWIG_SetErrorMsg SWIG_Python_SetErrorMsg +#define SWIG_ErrorType(code) SWIG_Python_ErrorType(code) +#define SWIG_Error(code, msg) SWIG_Python_SetErrorMsg(SWIG_ErrorType(code), msg) +#define SWIG_fail goto fail + + +/* Runtime API implementation */ + +/* Error manipulation */ + +SWIGINTERN void +SWIG_Python_SetErrorObj(PyObject *errtype, PyObject *obj) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + PyErr_SetObject(errtype, obj); + Py_DECREF(obj); + SWIG_PYTHON_THREAD_END_BLOCK; +} + +SWIGINTERN void +SWIG_Python_SetErrorMsg(PyObject *errtype, const char *msg) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + PyErr_SetString(errtype, msg); + SWIG_PYTHON_THREAD_END_BLOCK; +} + +#define SWIG_Python_Raise(obj, type, desc) SWIG_Python_SetErrorObj(SWIG_Python_ExceptionType(desc), obj) + +/* Set a constant value */ + +#if defined(SWIGPYTHON_BUILTIN) + +SWIGINTERN void +SwigPyBuiltin_AddPublicSymbol(PyObject *seq, const char *key) { + PyObject *s = PyString_InternFromString(key); + PyList_Append(seq, s); + Py_DECREF(s); +} + +SWIGINTERN void +SWIG_Python_SetConstant(PyObject *d, PyObject *public_interface, const char *name, PyObject *obj) { + PyDict_SetItemString(d, name, obj); + Py_DECREF(obj); + if (public_interface) + SwigPyBuiltin_AddPublicSymbol(public_interface, name); +} + +#else + +SWIGINTERN void +SWIG_Python_SetConstant(PyObject *d, const char *name, PyObject *obj) { + PyDict_SetItemString(d, name, obj); + Py_DECREF(obj); +} + +#endif + +/* Append a value to the result obj */ + +SWIGINTERN PyObject* +SWIG_Python_AppendOutput(PyObject* result, PyObject* obj) { + if (!result) { + result = obj; + } else if (result == Py_None) { + Py_DECREF(result); + result = obj; + } else { + if (!PyList_Check(result)) { + PyObject *o2 = result; + result = PyList_New(1); + if (result) { + PyList_SET_ITEM(result, 0, o2); + } else { + Py_DECREF(obj); + return o2; + } + } + PyList_Append(result,obj); + Py_DECREF(obj); + } + return result; +} + +/* Unpack the argument tuple */ + +SWIGINTERN Py_ssize_t +SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, PyObject **objs) +{ + if (!args) { + if (!min && !max) { + return 1; + } else { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got none", + name, (min == max ? "" : "at least "), (int)min); + return 0; + } + } + if (!PyTuple_Check(args)) { + if (min <= 1 && max >= 1) { + Py_ssize_t i; + objs[0] = args; + for (i = 1; i < max; ++i) { + objs[i] = 0; + } + return 2; + } + PyErr_SetString(PyExc_SystemError, "UnpackTuple() argument list is not a tuple"); + return 0; + } else { + Py_ssize_t l = PyTuple_GET_SIZE(args); + if (l < min) { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", + name, (min == max ? "" : "at least "), (int)min, (int)l); + return 0; + } else if (l > max) { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", + name, (min == max ? "" : "at most "), (int)max, (int)l); + return 0; + } else { + Py_ssize_t i; + for (i = 0; i < l; ++i) { + objs[i] = PyTuple_GET_ITEM(args, i); + } + for (; l < max; ++l) { + objs[l] = 0; + } + return i + 1; + } + } +} + +SWIGINTERN int +SWIG_Python_CheckNoKeywords(PyObject *kwargs, const char *name) { + int no_kwargs = 1; + if (kwargs) { + assert(PyDict_Check(kwargs)); + if (PyDict_Size(kwargs) > 0) { + PyErr_Format(PyExc_TypeError, "%s() does not take keyword arguments", name); + no_kwargs = 0; + } + } + return no_kwargs; +} + +/* A functor is a function object with one single object argument */ +#define SWIG_Python_CallFunctor(functor, obj) PyObject_CallFunctionObjArgs(functor, obj, NULL); + +/* + Helper for static pointer initialization for both C and C++ code, for example + static PyObject *SWIG_STATIC_POINTER(MyVar) = NewSomething(...); +*/ +#ifdef __cplusplus +#define SWIG_STATIC_POINTER(var) var +#else +#define SWIG_STATIC_POINTER(var) var = 0; if (!var) var +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Python-specific SWIG API */ +#define SWIG_newvarlink() SWIG_Python_newvarlink() +#define SWIG_addvarlink(p, name, get_attr, set_attr) SWIG_Python_addvarlink(p, name, get_attr, set_attr) +#define SWIG_InstallConstants(d, constants) SWIG_Python_InstallConstants(d, constants) + +/* ----------------------------------------------------------------------------- + * global variable support code. + * ----------------------------------------------------------------------------- */ + +typedef struct swig_globalvar { + char *name; /* Name of global variable */ + PyObject *(*get_attr)(void); /* Return the current value */ + int (*set_attr)(PyObject *); /* Set the value */ + struct swig_globalvar *next; +} swig_globalvar; + +typedef struct swig_varlinkobject { + PyObject_HEAD + swig_globalvar *vars; +} swig_varlinkobject; + +SWIGINTERN PyObject * +swig_varlink_repr(PyObject *SWIGUNUSEDPARM(v)) { +#if PY_VERSION_HEX >= 0x03000000 + return PyUnicode_InternFromString(""); +#else + return PyString_FromString(""); +#endif +} + +SWIGINTERN PyObject * +swig_varlink_str(PyObject *o) { + swig_varlinkobject *v = (swig_varlinkobject *) o; +#if PY_VERSION_HEX >= 0x03000000 + PyObject *str = PyUnicode_InternFromString("("); + PyObject *tail; + PyObject *joined; + swig_globalvar *var; + for (var = v->vars; var; var=var->next) { + tail = PyUnicode_FromString(var->name); + joined = PyUnicode_Concat(str, tail); + Py_DecRef(str); + Py_DecRef(tail); + str = joined; + if (var->next) { + tail = PyUnicode_InternFromString(", "); + joined = PyUnicode_Concat(str, tail); + Py_DecRef(str); + Py_DecRef(tail); + str = joined; + } + } + tail = PyUnicode_InternFromString(")"); + joined = PyUnicode_Concat(str, tail); + Py_DecRef(str); + Py_DecRef(tail); + str = joined; +#else + PyObject *str = PyString_FromString("("); + swig_globalvar *var; + for (var = v->vars; var; var=var->next) { + PyString_ConcatAndDel(&str,PyString_FromString(var->name)); + if (var->next) PyString_ConcatAndDel(&str,PyString_FromString(", ")); + } + PyString_ConcatAndDel(&str,PyString_FromString(")")); +#endif + return str; +} + +SWIGINTERN void +swig_varlink_dealloc(PyObject *o) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + swig_globalvar *var = v->vars; + while (var) { + swig_globalvar *n = var->next; + free(var->name); + free(var); + var = n; + } +} + +SWIGINTERN PyObject * +swig_varlink_getattr(PyObject *o, char *n) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + PyObject *res = NULL; + swig_globalvar *var = v->vars; + while (var) { + if (strcmp(var->name,n) == 0) { + res = (*var->get_attr)(); + break; + } + var = var->next; + } + if (res == NULL && !PyErr_Occurred()) { + PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + } + return res; +} + +SWIGINTERN int +swig_varlink_setattr(PyObject *o, char *n, PyObject *p) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + int res = 1; + swig_globalvar *var = v->vars; + while (var) { + if (strcmp(var->name,n) == 0) { + res = (*var->set_attr)(p); + break; + } + var = var->next; + } + if (res == 1 && !PyErr_Occurred()) { + PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + } + return res; +} + +SWIGINTERN PyTypeObject* +swig_varlink_type(void) { + static char varlink__doc__[] = "Swig var link object"; +#ifndef Py_LIMITED_API + static PyTypeObject varlink_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX >= 0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "swigvarlink", /* tp_name */ + sizeof(swig_varlinkobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) swig_varlink_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /*tp_print*/ +#else + (Py_ssize_t)0, /*tp_vectorcall_offset*/ +#endif + (getattrfunc) swig_varlink_getattr, /* tp_getattr */ + (setattrfunc) swig_varlink_setattr, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc) swig_varlink_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc) swig_varlink_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + varlink__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* tp_iter -> tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030C0000 + 0, /* tp_watched */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + varlink_type = tmp; + type_init = 1; + if (PyType_Ready(&varlink_type) < 0) + return NULL; + } + return &varlink_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)swig_varlink_dealloc }, + { Py_tp_repr, (void *)swig_varlink_repr }, + { Py_tp_getattr, (void *)swig_varlink_getattr }, + { Py_tp_setattr, (void *)swig_varlink_setattr }, + { Py_tp_str, (void *)swig_varlink_str }, + { Py_tp_doc, (void *)varlink__doc__ }, + { 0, NULL } + }; + PyType_Spec spec = { + "swigvarlink", + sizeof(swig_varlinkobject), + 0, + Py_TPFLAGS_DEFAULT, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +/* Create a variable linking object for use later */ +SWIGINTERN PyObject * +SWIG_Python_newvarlink(void) { + swig_varlinkobject *result = PyObject_New(swig_varlinkobject, swig_varlink_type()); + if (result) { + result->vars = 0; + } + return ((PyObject*) result); +} + +SWIGINTERN void +SWIG_Python_addvarlink(PyObject *p, const char *name, PyObject *(*get_attr)(void), int (*set_attr)(PyObject *p)) { + swig_varlinkobject *v = (swig_varlinkobject *) p; + swig_globalvar *gv = (swig_globalvar *) malloc(sizeof(swig_globalvar)); + if (gv) { + size_t size = strlen(name)+1; + gv->name = (char *)malloc(size); + if (gv->name) { + memcpy(gv->name, name, size); + gv->get_attr = get_attr; + gv->set_attr = set_attr; + gv->next = v->vars; + } + } + v->vars = gv; +} + + +static PyObject *Swig_Globals_global = NULL; + +SWIGINTERN PyObject * +SWIG_globals(void) { + if (Swig_Globals_global == NULL) { + Swig_Globals_global = SWIG_newvarlink(); + } + return Swig_Globals_global; +} + +#ifdef __cplusplus +} +#endif + +/* ----------------------------------------------------------------------------- + * Pointer declarations + * ----------------------------------------------------------------------------- */ + +/* Flags for new pointer objects */ +#define SWIG_POINTER_NOSHADOW (SWIG_POINTER_OWN << 1) +#define SWIG_POINTER_NEW (SWIG_POINTER_NOSHADOW | SWIG_POINTER_OWN) + +#define SWIG_POINTER_IMPLICIT_CONV (SWIG_POINTER_DISOWN << 1) + +#define SWIG_BUILTIN_TP_INIT (SWIG_POINTER_OWN << 2) +#define SWIG_BUILTIN_INIT (SWIG_BUILTIN_TP_INIT | SWIG_POINTER_OWN) + +#ifdef __cplusplus +extern "C" { +#endif + +/* The python void return value */ + +SWIGRUNTIMEINLINE PyObject * +SWIG_Py_Void(void) +{ + PyObject *none = Py_None; + Py_INCREF(none); + return none; +} + +/* SwigPyClientData */ + +typedef struct { + PyObject *klass; + PyObject *newraw; + PyObject *newargs; + PyObject *destroy; + int delargs; + int implicitconv; + PyTypeObject *pytype; +} SwigPyClientData; + +SWIGRUNTIMEINLINE int +SWIG_Python_CheckImplicit(swig_type_info *ty) +{ + SwigPyClientData *data = (SwigPyClientData *)ty->clientdata; + int fail = data ? data->implicitconv : 0; + if (fail) + PyErr_SetString(PyExc_TypeError, "Implicit conversion is prohibited for explicit constructors."); + return fail; +} + +SWIGRUNTIMEINLINE PyObject * +SWIG_Python_ExceptionType(swig_type_info *desc) { + SwigPyClientData *data = desc ? (SwigPyClientData *) desc->clientdata : 0; + PyObject *klass = data ? data->klass : 0; + return (klass ? klass : PyExc_RuntimeError); +} + + +SWIGRUNTIME SwigPyClientData * +SwigPyClientData_New(PyObject* obj) +{ + if (!obj) { + return 0; + } else { + SwigPyClientData *data = (SwigPyClientData *)malloc(sizeof(SwigPyClientData)); + /* the klass element */ + data->klass = obj; + Py_INCREF(data->klass); + /* the newraw method and newargs arguments used to create a new raw instance */ + if (PyClass_Check(obj)) { + data->newraw = 0; + Py_INCREF(obj); + data->newargs = obj; + } else { + data->newraw = PyObject_GetAttrString(data->klass, "__new__"); + if (data->newraw) { + data->newargs = PyTuple_New(1); + if (data->newargs) { + Py_INCREF(obj); + PyTuple_SET_ITEM(data->newargs, 0, obj); + } else { + Py_DECREF(data->newraw); + Py_DECREF(data->klass); + free(data); + return 0; + } + } else { + Py_INCREF(obj); + data->newargs = obj; + } + } + /* the destroy method, aka as the C++ delete method */ + data->destroy = PyObject_GetAttrString(data->klass, "__swig_destroy__"); + if (PyErr_Occurred()) { + PyErr_Clear(); + data->destroy = 0; + } + if (data->destroy) { + data->delargs = !(PyCFunction_GET_FLAGS(data->destroy) & METH_O); + } else { + data->delargs = 0; + } + data->implicitconv = 0; + data->pytype = 0; + return data; + } +} + +SWIGRUNTIME void +SwigPyClientData_Del(SwigPyClientData *data) +{ + Py_XDECREF(data->klass); + Py_XDECREF(data->newraw); + Py_XDECREF(data->newargs); + Py_XDECREF(data->destroy); + free(data); +} + +/* =============== SwigPyObject =====================*/ + +typedef struct { + PyObject_HEAD + void *ptr; + swig_type_info *ty; + int own; + PyObject *next; +#ifdef SWIGPYTHON_BUILTIN + PyObject *dict; +#endif +} SwigPyObject; + + +#ifdef SWIGPYTHON_BUILTIN + +SWIGRUNTIME PyObject * +SwigPyObject_get___dict__(PyObject *v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + + if (!sobj->dict) + sobj->dict = PyDict_New(); + + Py_XINCREF(sobj->dict); + return sobj->dict; +} + +#endif + +SWIGRUNTIME PyObject * +SwigPyObject_long(SwigPyObject *v) +{ + return PyLong_FromVoidPtr(v->ptr); +} + +SWIGRUNTIME PyObject * +SwigPyObject_format(const char* fmt, SwigPyObject *v) +{ + PyObject *res = NULL; + PyObject *args = PyTuple_New(1); + if (args) { + PyObject *val = SwigPyObject_long(v); + if (val) { + PyObject *ofmt; + PyTuple_SET_ITEM(args, 0, val); + ofmt = SWIG_Python_str_FromChar(fmt); + if (ofmt) { +#if PY_VERSION_HEX >= 0x03000000 + res = PyUnicode_Format(ofmt,args); +#else + res = PyString_Format(ofmt,args); +#endif + Py_DECREF(ofmt); + } + } + Py_DECREF(args); + } + return res; +} + +SWIGRUNTIME PyObject * +SwigPyObject_oct(SwigPyObject *v) +{ + return SwigPyObject_format("%o",v); +} + +SWIGRUNTIME PyObject * +SwigPyObject_hex(SwigPyObject *v) +{ + return SwigPyObject_format("%x",v); +} + +SWIGRUNTIME PyObject * +SwigPyObject_repr(SwigPyObject *v) +{ + const char *name = SWIG_TypePrettyName(v->ty); + PyObject *repr = SWIG_Python_str_FromFormat("", (name ? name : "unknown"), (void *)v); + if (repr && v->next) { + PyObject *nrep = SwigPyObject_repr((SwigPyObject *)v->next); + if (nrep) { +# if PY_VERSION_HEX >= 0x03000000 + PyObject *joined = PyUnicode_Concat(repr, nrep); + Py_DecRef(repr); + Py_DecRef(nrep); + repr = joined; +# else + PyString_ConcatAndDel(&repr,nrep); +# endif + } else { + Py_DecRef(repr); + repr = NULL; + } + } + return repr; +} + +/* We need a version taking two PyObject* parameters so it's a valid + * PyCFunction to use in swigobject_methods[]. */ +SWIGRUNTIME PyObject * +SwigPyObject_repr2(PyObject *v, PyObject *SWIGUNUSEDPARM(args)) +{ + return SwigPyObject_repr((SwigPyObject*)v); +} + +SWIGRUNTIME int +SwigPyObject_compare(SwigPyObject *v, SwigPyObject *w) +{ + void *i = v->ptr; + void *j = w->ptr; + return (i < j) ? -1 : ((i > j) ? 1 : 0); +} + +/* Added for Python 3.x, would it also be useful for Python 2.x? */ +SWIGRUNTIME PyObject* +SwigPyObject_richcompare(SwigPyObject *v, SwigPyObject *w, int op) +{ + PyObject* res = NULL; + if (!PyErr_Occurred()) { + if (op != Py_EQ && op != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + res = PyBool_FromLong( (SwigPyObject_compare(v, w)==0) == (op == Py_EQ) ? 1 : 0); + } + return res; +} + + +SWIGRUNTIME PyTypeObject* SwigPyObject_TypeOnce(void); + +#ifdef SWIGPYTHON_BUILTIN +static swig_type_info *SwigPyObject_stype = 0; +SWIGRUNTIME PyTypeObject* +SwigPyObject_type(void) { + SwigPyClientData *cd; + assert(SwigPyObject_stype); + cd = (SwigPyClientData*) SwigPyObject_stype->clientdata; + assert(cd); + assert(cd->pytype); + return cd->pytype; +} +#else +SWIGRUNTIME PyTypeObject* +SwigPyObject_type(void) { + static PyTypeObject *SWIG_STATIC_POINTER(type) = SwigPyObject_TypeOnce(); + return type; +} +#endif + +SWIGRUNTIMEINLINE int +SwigPyObject_Check(PyObject *op) { + PyTypeObject *target_tp = SwigPyObject_type(); + PyTypeObject *op_type = Py_TYPE(op); +#ifdef SWIGPYTHON_BUILTIN + if (PyType_IsSubtype(op_type, target_tp)) + return 1; + return (strcmp(op_type->tp_name, "SwigPyObject") == 0); +#else + if (op_type == target_tp) + return 1; +# ifdef Py_LIMITED_API + int cmp; + PyObject *tp_name = PyObject_GetAttrString((PyObject *)op_type, "__name__"); + if (!tp_name) + return 0; + cmp = PyUnicode_CompareWithASCIIString(tp_name, "SwigPyObject"); + Py_DECREF(tp_name); + return cmp == 0; +# else + return (strcmp(op_type->tp_name, "SwigPyObject") == 0); +# endif +#endif +} + +SWIGRUNTIME PyObject * +SwigPyObject_New(void *ptr, swig_type_info *ty, int own); + +static PyObject* Swig_Capsule_global = NULL; + +SWIGRUNTIME void +SwigPyObject_dealloc(PyObject *v) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + PyObject *next = sobj->next; + if (sobj->own == SWIG_POINTER_OWN) { + swig_type_info *ty = sobj->ty; + SwigPyClientData *data = ty ? (SwigPyClientData *) ty->clientdata : 0; + PyObject *destroy = data ? data->destroy : 0; + if (destroy) { + /* destroy is always a VARARGS method */ + PyObject *res; + + /* PyObject_CallFunction() has the potential to silently drop + the active exception. In cases of unnamed temporary + variable or where we just finished iterating over a generator + StopIteration will be active right now, and this needs to + remain true upon return from SwigPyObject_dealloc. So save + and restore. */ + + PyObject *type = NULL, *value = NULL, *traceback = NULL; + PyErr_Fetch(&type, &value, &traceback); + + if (data->delargs) { + /* we need to create a temporary object to carry the destroy operation */ + PyObject *tmp = SwigPyObject_New(sobj->ptr, ty, 0); + if (tmp) { + res = SWIG_Python_CallFunctor(destroy, tmp); + } else { + res = 0; + } + Py_XDECREF(tmp); + } else { + PyCFunction meth = PyCFunction_GET_FUNCTION(destroy); + PyObject *mself = PyCFunction_GET_SELF(destroy); + res = ((*meth)(mself, v)); + } + if (!res) + PyErr_WriteUnraisable(destroy); + + PyErr_Restore(type, value, traceback); + + Py_XDECREF(res); + } +#if !defined(SWIG_PYTHON_SILENT_MEMLEAK) + else { + const char *name = SWIG_TypePrettyName(ty); + printf("swig/python detected a memory leak of type '%s', no destructor found.\n", (name ? name : "unknown")); + } +#endif + Py_XDECREF(Swig_Capsule_global); + } + Py_XDECREF(next); +#ifdef SWIGPYTHON_BUILTIN + Py_XDECREF(sobj->dict); +#endif + PyObject_DEL(v); +} + +SWIGRUNTIME PyObject* +SwigPyObject_append(PyObject* v, PyObject* next) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + if (!SwigPyObject_Check(next)) { + PyErr_SetString(PyExc_TypeError, "Attempt to append a non SwigPyObject"); + return NULL; + } + ((SwigPyObject *)next)->next = sobj->next; + sobj->next = next; + Py_INCREF(next); + return SWIG_Py_Void(); +} + +SWIGRUNTIME PyObject* +SwigPyObject_next(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + if (sobj->next) { + Py_INCREF(sobj->next); + return sobj->next; + } else { + return SWIG_Py_Void(); + } +} + +SWIGINTERN PyObject* +SwigPyObject_disown(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + sobj->own = 0; + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject* +SwigPyObject_acquire(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + sobj->own = SWIG_POINTER_OWN; + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject* +SwigPyObject_own(PyObject *v, PyObject *args) +{ + PyObject *val = 0; + if (!PyArg_UnpackTuple(args, "own", 0, 1, &val)) { + return NULL; + } else { + SwigPyObject *sobj = (SwigPyObject *)v; + PyObject *obj = PyBool_FromLong(sobj->own); + if (val) { + if (PyObject_IsTrue(val)) { + Py_DECREF(SwigPyObject_acquire(v,args)); + } else { + Py_DECREF(SwigPyObject_disown(v,args)); + } + } + return obj; + } +} + +static PyMethodDef +swigobject_methods[] = { + {"disown", SwigPyObject_disown, METH_NOARGS, "releases ownership of the pointer"}, + {"acquire", SwigPyObject_acquire, METH_NOARGS, "acquires ownership of the pointer"}, + {"own", SwigPyObject_own, METH_VARARGS, "returns/sets ownership of the pointer"}, + {"append", SwigPyObject_append, METH_O, "appends another 'this' object"}, + {"next", SwigPyObject_next, METH_NOARGS, "returns the next 'this' object"}, + {"__repr__",SwigPyObject_repr2, METH_NOARGS, "returns object representation"}, + {0, 0, 0, 0} +}; + +SWIGRUNTIME PyTypeObject* +SwigPyObject_TypeOnce(void) { + static char swigobject_doc[] = "Swig object carries a C/C++ instance pointer"; +#ifndef Py_LIMITED_API + static PyNumberMethods SwigPyObject_as_number = { + (binaryfunc)0, /*nb_add*/ + (binaryfunc)0, /*nb_subtract*/ + (binaryfunc)0, /*nb_multiply*/ + /* nb_divide removed in Python 3 */ +#if PY_VERSION_HEX < 0x03000000 + (binaryfunc)0, /*nb_divide*/ +#endif + (binaryfunc)0, /*nb_remainder*/ + (binaryfunc)0, /*nb_divmod*/ + (ternaryfunc)0,/*nb_power*/ + (unaryfunc)0, /*nb_negative*/ + (unaryfunc)0, /*nb_positive*/ + (unaryfunc)0, /*nb_absolute*/ + (inquiry)0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ +#if PY_VERSION_HEX < 0x03000000 + 0, /*nb_coerce*/ +#endif + (unaryfunc)SwigPyObject_long, /*nb_int*/ +#if PY_VERSION_HEX < 0x03000000 + (unaryfunc)SwigPyObject_long, /*nb_long*/ +#else + 0, /*nb_reserved*/ +#endif + (unaryfunc)0, /*nb_float*/ +#if PY_VERSION_HEX < 0x03000000 + (unaryfunc)SwigPyObject_oct, /*nb_oct*/ + (unaryfunc)SwigPyObject_hex, /*nb_hex*/ +#endif +#if PY_VERSION_HEX >= 0x03050000 /* 3.5 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_inplace_matrix_multiply */ +#elif PY_VERSION_HEX >= 0x03000000 /* 3.0 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_index, nb_inplace_divide removed */ +#else + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_index */ +#endif + }; + + static PyTypeObject swigpyobject_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX >= 0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "SwigPyObject", /* tp_name */ + sizeof(SwigPyObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)SwigPyObject_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /*tp_print*/ +#else + (Py_ssize_t)0, /*tp_vectorcall_offset*/ +#endif + (getattrfunc)0, /* tp_getattr */ + (setattrfunc)0, /* tp_setattr */ +#if PY_VERSION_HEX >= 0x03000000 + 0, /* tp_reserved in 3.0.1, tp_compare in 3.0.0 but not used */ +#else + (cmpfunc)SwigPyObject_compare, /* tp_compare */ +#endif + (reprfunc)SwigPyObject_repr, /* tp_repr */ + &SwigPyObject_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)0, /* tp_hash */ + (ternaryfunc)0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + swigobject_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)SwigPyObject_richcompare,/* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + swigobject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030C0000 + 0, /* tp_watched */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + swigpyobject_type = tmp; + type_init = 1; + if (PyType_Ready(&swigpyobject_type) != 0) + return NULL; + } + return &swigpyobject_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)SwigPyObject_dealloc }, + { Py_tp_repr, (void *)SwigPyObject_repr }, + { Py_tp_getattro, (void *)PyObject_GenericGetAttr }, + { Py_tp_doc, (void *)swigobject_doc }, + { Py_tp_richcompare, (void *)SwigPyObject_richcompare }, + { Py_tp_methods, (void *)swigobject_methods }, + { Py_nb_int, (void *)SwigPyObject_long }, + { 0, NULL } + }; + PyType_Spec spec = { + "SwigPyObject", + sizeof(SwigPyObject), + 0, + Py_TPFLAGS_DEFAULT, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +SWIGRUNTIME PyObject * +SwigPyObject_New(void *ptr, swig_type_info *ty, int own) +{ + SwigPyObject *sobj = PyObject_New(SwigPyObject, SwigPyObject_type()); + if (sobj) { + sobj->ptr = ptr; + sobj->ty = ty; + sobj->own = own; + sobj->next = 0; +#ifdef SWIGPYTHON_BUILTIN + sobj->dict = 0; +#endif + if (own == SWIG_POINTER_OWN) { + /* Obtain a reference to the Python capsule wrapping the module information, so that the + * module information is correctly destroyed after all SWIG python objects have been freed + * by the GC (and corresponding destructors invoked) */ + Py_XINCREF(Swig_Capsule_global); + } + } + return (PyObject *)sobj; +} + +/* ----------------------------------------------------------------------------- + * Implements a simple Swig Packed type, and use it instead of string + * ----------------------------------------------------------------------------- */ + +typedef struct { + PyObject_HEAD + void *pack; + swig_type_info *ty; + size_t size; +} SwigPyPacked; + +SWIGRUNTIME PyObject * +SwigPyPacked_repr(SwigPyPacked *v) +{ + char result[SWIG_BUFFER_SIZE]; + if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))) { + return SWIG_Python_str_FromFormat("", result, v->ty->name); + } else { + return SWIG_Python_str_FromFormat("", v->ty->name); + } +} + +SWIGRUNTIME PyObject * +SwigPyPacked_str(SwigPyPacked *v) +{ + char result[SWIG_BUFFER_SIZE]; + if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))){ + return SWIG_Python_str_FromFormat("%s%s", result, v->ty->name); + } else { + return SWIG_Python_str_FromChar(v->ty->name); + } +} + +SWIGRUNTIME int +SwigPyPacked_compare(SwigPyPacked *v, SwigPyPacked *w) +{ + size_t i = v->size; + size_t j = w->size; + int s = (i < j) ? -1 : ((i > j) ? 1 : 0); + return s ? s : strncmp((const char *)v->pack, (const char *)w->pack, 2*v->size); +} + +SWIGRUNTIME PyTypeObject* SwigPyPacked_TypeOnce(void); + +SWIGRUNTIME PyTypeObject* +SwigPyPacked_type(void) { + static PyTypeObject *SWIG_STATIC_POINTER(type) = SwigPyPacked_TypeOnce(); + return type; +} + +SWIGRUNTIMEINLINE int +SwigPyPacked_Check(PyObject *op) { + PyTypeObject* op_type = Py_TYPE(op); + if (op_type == SwigPyPacked_TypeOnce()) + return 1; +#ifdef Py_LIMITED_API + int cmp; + PyObject *tp_name = PyObject_GetAttrString((PyObject *)op_type, "__name__"); + if (!tp_name) + return 0; + cmp = PyUnicode_CompareWithASCIIString(tp_name, "SwigPyPacked"); + Py_DECREF(tp_name); + return cmp == 0; +#else + return (strcmp(op_type->tp_name, "SwigPyPacked") == 0); +#endif +} + +SWIGRUNTIME void +SwigPyPacked_dealloc(PyObject *v) +{ + if (SwigPyPacked_Check(v)) { + SwigPyPacked *sobj = (SwigPyPacked *) v; + free(sobj->pack); + } + PyObject_DEL(v); +} + +SWIGRUNTIME PyTypeObject* +SwigPyPacked_TypeOnce(void) { + static char swigpacked_doc[] = "Swig object carries a C/C++ instance pointer"; +#ifndef Py_LIMITED_API + static PyTypeObject swigpypacked_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX>=0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "SwigPyPacked", /* tp_name */ + sizeof(SwigPyPacked), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)SwigPyPacked_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /*tp_print*/ +#else + (Py_ssize_t)0, /*tp_vectorcall_offset*/ +#endif + (getattrfunc)0, /* tp_getattr */ + (setattrfunc)0, /* tp_setattr */ +#if PY_VERSION_HEX>=0x03000000 + 0, /* tp_reserved in 3.0.1 */ +#else + (cmpfunc)SwigPyPacked_compare, /* tp_compare */ +#endif + (reprfunc)SwigPyPacked_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)0, /* tp_hash */ + (ternaryfunc)0, /* tp_call */ + (reprfunc)SwigPyPacked_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + swigpacked_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030C0000 + 0, /* tp_watched */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + swigpypacked_type = tmp; + type_init = 1; + if (PyType_Ready(&swigpypacked_type) != 0) + return NULL; + } + return &swigpypacked_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)SwigPyPacked_dealloc }, + { Py_tp_repr, (void *)SwigPyPacked_repr }, + { Py_tp_str, (void *)SwigPyPacked_str }, + { Py_tp_getattro, (void *)PyObject_GenericGetAttr }, + { Py_tp_doc, (void *)swigpacked_doc }, + { 0, NULL } + }; + PyType_Spec spec = { + "SwigPyPacked", + sizeof(SwigPyPacked), + 0, + Py_TPFLAGS_DEFAULT, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +SWIGRUNTIME PyObject * +SwigPyPacked_New(void *ptr, size_t size, swig_type_info *ty) +{ + SwigPyPacked *sobj = PyObject_New(SwigPyPacked, SwigPyPacked_type()); + if (sobj) { + void *pack = malloc(size); + if (pack) { + memcpy(pack, ptr, size); + sobj->pack = pack; + sobj->ty = ty; + sobj->size = size; + } else { + PyObject_DEL((PyObject *) sobj); + sobj = 0; + } + } + return (PyObject *) sobj; +} + +SWIGRUNTIME swig_type_info * +SwigPyPacked_UnpackData(PyObject *obj, void *ptr, size_t size) +{ + if (SwigPyPacked_Check(obj)) { + SwigPyPacked *sobj = (SwigPyPacked *)obj; + if (sobj->size != size) return 0; + memcpy(ptr, sobj->pack, size); + return sobj->ty; + } else { + return 0; + } +} + +/* ----------------------------------------------------------------------------- + * pointers/data manipulation + * ----------------------------------------------------------------------------- */ + +static PyObject *Swig_This_global = NULL; + +SWIGRUNTIME PyObject * +SWIG_This(void) +{ + if (Swig_This_global == NULL) + Swig_This_global = SWIG_Python_str_FromChar("this"); + return Swig_This_global; +} + +/* #define SWIG_PYTHON_SLOW_GETSET_THIS */ + +/* TODO: I don't know how to implement the fast getset in Python 3 right now */ +#if PY_VERSION_HEX>=0x03000000 +#define SWIG_PYTHON_SLOW_GETSET_THIS +#endif + +SWIGRUNTIME SwigPyObject * +SWIG_Python_GetSwigThis(PyObject *pyobj) +{ + PyObject *obj; + + if (SwigPyObject_Check(pyobj)) + return (SwigPyObject *) pyobj; + +#ifdef SWIGPYTHON_BUILTIN + (void)obj; +# ifdef PyWeakref_CheckProxy + if (PyWeakref_CheckProxy(pyobj)) { + pyobj = PyWeakref_GET_OBJECT(pyobj); + if (pyobj && SwigPyObject_Check(pyobj)) + return (SwigPyObject*) pyobj; + } +# endif + return NULL; +#else + + obj = 0; + +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + if (PyInstance_Check(pyobj)) { + obj = _PyInstance_Lookup(pyobj, SWIG_This()); + } else { + PyObject **dictptr = _PyObject_GetDictPtr(pyobj); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + obj = dict ? PyDict_GetItem(dict, SWIG_This()) : 0; + } else { +#ifdef PyWeakref_CheckProxy + if (PyWeakref_CheckProxy(pyobj)) { + PyObject *wobj = PyWeakref_GET_OBJECT(pyobj); + return wobj ? SWIG_Python_GetSwigThis(wobj) : 0; + } +#endif + obj = PyObject_GetAttr(pyobj,SWIG_This()); + if (obj) { + Py_DECREF(obj); + } else { + if (PyErr_Occurred()) PyErr_Clear(); + return 0; + } + } + } +#else + obj = PyObject_GetAttr(pyobj,SWIG_This()); + if (obj) { + Py_DECREF(obj); + } else { + if (PyErr_Occurred()) PyErr_Clear(); + return 0; + } +#endif + if (obj && !SwigPyObject_Check(obj)) { + /* a PyObject is called 'this', try to get the 'real this' + SwigPyObject from it */ + return SWIG_Python_GetSwigThis(obj); + } + return (SwigPyObject *)obj; +#endif +} + +/* Acquire a pointer value */ + +SWIGRUNTIME int +SWIG_Python_AcquirePtr(PyObject *obj, int own) { + if (own == SWIG_POINTER_OWN) { + SwigPyObject *sobj = SWIG_Python_GetSwigThis(obj); + if (sobj) { + int oldown = sobj->own; + sobj->own = own; + return oldown; + } + } + return 0; +} + +/* Convert a pointer value */ + +SWIGRUNTIME int +SWIG_Python_ConvertPtrAndOwn(PyObject *obj, void **ptr, swig_type_info *ty, int flags, int *own) { + int res; + SwigPyObject *sobj; + int implicit_conv = (flags & SWIG_POINTER_IMPLICIT_CONV) != 0; + + if (!obj) + return SWIG_ERROR; + if (obj == Py_None && !implicit_conv) { + if (ptr) + *ptr = 0; + return (flags & SWIG_POINTER_NO_NULL) ? SWIG_NullReferenceError : SWIG_OK; + } + + res = SWIG_ERROR; + + sobj = SWIG_Python_GetSwigThis(obj); + if (own) + *own = 0; + while (sobj) { + void *vptr = sobj->ptr; + if (ty) { + swig_type_info *to = sobj->ty; + if (to == ty) { + /* no type cast needed */ + if (ptr) *ptr = vptr; + break; + } else { + swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); + if (!tc) { + sobj = (SwigPyObject *)sobj->next; + } else { + if (ptr) { + int newmemory = 0; + *ptr = SWIG_TypeCast(tc,vptr,&newmemory); + if (newmemory == SWIG_CAST_NEW_MEMORY) { + assert(own); /* badly formed typemap which will lead to a memory leak - it must set and use own to delete *ptr */ + if (own) + *own = *own | SWIG_CAST_NEW_MEMORY; + } + } + break; + } + } + } else { + if (ptr) *ptr = vptr; + break; + } + } + if (sobj) { + if (((flags & SWIG_POINTER_RELEASE) == SWIG_POINTER_RELEASE) && !sobj->own) { + res = SWIG_ERROR_RELEASE_NOT_OWNED; + } else { + if (own) + *own = *own | sobj->own; + if (flags & SWIG_POINTER_DISOWN) { + sobj->own = 0; + } + if (flags & SWIG_POINTER_CLEAR) { + sobj->ptr = 0; + } + res = SWIG_OK; + } + } else { + if (implicit_conv) { + SwigPyClientData *data = ty ? (SwigPyClientData *) ty->clientdata : 0; + if (data && !data->implicitconv) { + PyObject *klass = data->klass; + if (klass) { + PyObject *impconv; + data->implicitconv = 1; /* avoid recursion and call 'explicit' constructors*/ + impconv = SWIG_Python_CallFunctor(klass, obj); + data->implicitconv = 0; + if (PyErr_Occurred()) { + PyErr_Clear(); + impconv = 0; + } + if (impconv) { + SwigPyObject *iobj = SWIG_Python_GetSwigThis(impconv); + if (iobj) { + void *vptr; + res = SWIG_Python_ConvertPtrAndOwn((PyObject*)iobj, &vptr, ty, 0, 0); + if (SWIG_IsOK(res)) { + if (ptr) { + *ptr = vptr; + /* transfer the ownership to 'ptr' */ + iobj->own = 0; + res = SWIG_AddCast(res); + res = SWIG_AddNewMask(res); + } else { + res = SWIG_AddCast(res); + } + } + } + Py_DECREF(impconv); + } + } + } + if (!SWIG_IsOK(res) && obj == Py_None) { + if (ptr) + *ptr = 0; + if (PyErr_Occurred()) + PyErr_Clear(); + res = SWIG_OK; + } + } + } + return res; +} + +/* Convert a function ptr value */ + +SWIGRUNTIME int +SWIG_Python_ConvertFunctionPtr(PyObject *obj, void **ptr, swig_type_info *ty) { + if (!PyCFunction_Check(obj)) { + return SWIG_ConvertPtr(obj, ptr, ty, 0); + } else { + void *vptr = 0; + swig_cast_info *tc; + + /* here we get the method pointer for callbacks */ +#ifndef Py_LIMITED_API + const char *doc = (((PyCFunctionObject *)obj) -> m_ml -> ml_doc); +#else + PyObject* pystr_doc = PyObject_GetAttrString(obj, "__doc__"); + PyObject *bytes = NULL; + const char *doc = pystr_doc ? SWIG_PyUnicode_AsUTF8AndSize(pystr_doc, NULL, &bytes) : 0; +#endif + const char *desc = doc ? strstr(doc, "swig_ptr: ") : 0; + if (desc) + desc = ty ? SWIG_UnpackVoidPtr(desc + 10, &vptr, ty->name) : 0; +#ifdef Py_LIMITED_API + Py_XDECREF(bytes); + Py_XDECREF(pystr_doc); +#endif + if (!desc) + return SWIG_ERROR; + tc = SWIG_TypeCheck(desc,ty); + if (tc) { + int newmemory = 0; + *ptr = SWIG_TypeCast(tc,vptr,&newmemory); + assert(!newmemory); /* newmemory handling not yet implemented */ + } else { + return SWIG_ERROR; + } + return SWIG_OK; + } +} + +/* Convert a packed pointer value */ + +SWIGRUNTIME int +SWIG_Python_ConvertPacked(PyObject *obj, void *ptr, size_t sz, swig_type_info *ty) { + swig_type_info *to = SwigPyPacked_UnpackData(obj, ptr, sz); + if (!to) return SWIG_ERROR; + if (ty) { + if (to != ty) { + /* check type cast? */ + swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); + if (!tc) return SWIG_ERROR; + } + } + return SWIG_OK; +} + +/* ----------------------------------------------------------------------------- + * Create a new pointer object + * ----------------------------------------------------------------------------- */ + +/* + Create a new instance object, without calling __init__, and set the + 'this' attribute. +*/ + +SWIGRUNTIME PyObject* +SWIG_Python_NewShadowInstance(SwigPyClientData *data, PyObject *swig_this) +{ + PyObject *inst = 0; + PyObject *newraw = data->newraw; + if (newraw) { + inst = PyObject_Call(newraw, data->newargs, NULL); + if (inst) { +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + PyObject **dictptr = _PyObject_GetDictPtr(inst); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + if (dict == NULL) { + dict = PyDict_New(); + *dictptr = dict; + } + if (dict) { + PyDict_SetItem(dict, SWIG_This(), swig_this); + } else{ + Py_DECREF(inst); + inst = 0; + } + } +#else + if (PyObject_SetAttr(inst, SWIG_This(), swig_this) == -1) { + Py_DECREF(inst); + inst = 0; + } +#endif + } + } else { +#if PY_VERSION_HEX >= 0x03000000 + PyObject *empty_args = PyTuple_New(0); + if (empty_args) { + PyObject *empty_kwargs = PyDict_New(); + if (empty_kwargs) { +#ifndef Py_LIMITED_API + newfunc newfn = ((PyTypeObject *)data->newargs)->tp_new; +#else + newfunc newfn = (newfunc)PyType_GetSlot((PyTypeObject *)data->newargs, Py_tp_new); +#endif + inst = newfn((PyTypeObject *)data->newargs, empty_args, empty_kwargs); + Py_DECREF(empty_kwargs); + if (inst) { + if (PyObject_SetAttr(inst, SWIG_This(), swig_this) == -1) { + Py_DECREF(inst); + inst = 0; + } else { + PyType_Modified(Py_TYPE(inst)); + } + } + } + Py_DECREF(empty_args); + } +#else + PyObject *dict = PyDict_New(); + if (dict) { + PyDict_SetItem(dict, SWIG_This(), swig_this); + inst = PyInstance_NewRaw(data->newargs, dict); + Py_DECREF(dict); + } +#endif + } + return inst; +} + +SWIGRUNTIME int +SWIG_Python_SetSwigThis(PyObject *inst, PyObject *swig_this) +{ +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + PyObject **dictptr = _PyObject_GetDictPtr(inst); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + if (dict == NULL) { + dict = PyDict_New(); + *dictptr = dict; + } + if (dict) { + return PyDict_SetItem(dict, SWIG_This(), swig_this); + } else{ + return -1; + } + } +#endif + return PyObject_SetAttr(inst, SWIG_This(), swig_this); +} + + +SWIGINTERN PyObject * +SWIG_Python_InitShadowInstance(PyObject *args) { + PyObject *obj[2]; + if (!SWIG_Python_UnpackTuple(args, "swiginit", 2, 2, obj)) { + return NULL; + } else { + SwigPyObject *sthis = SWIG_Python_GetSwigThis(obj[0]); + if (sthis) { + Py_DECREF(SwigPyObject_append((PyObject*) sthis, obj[1])); + } else { + if (SWIG_Python_SetSwigThis(obj[0], obj[1]) != 0) + return NULL; + } + return SWIG_Py_Void(); + } +} + +/* Create a new pointer object */ + +SWIGRUNTIME PyObject * +SWIG_Python_NewPointerObj(PyObject *self, void *ptr, swig_type_info *type, int flags) { + SwigPyClientData *clientdata; + PyObject * robj; + int own; + + if (!ptr) + return SWIG_Py_Void(); + + clientdata = type ? (SwigPyClientData *)(type->clientdata) : 0; + own = (flags & SWIG_POINTER_OWN) ? SWIG_POINTER_OWN : 0; + if (clientdata && clientdata->pytype) { + SwigPyObject *newobj; + if (flags & SWIG_BUILTIN_TP_INIT) { + newobj = (SwigPyObject*) self; + if (newobj->ptr) { +#ifndef Py_LIMITED_API + allocfunc alloc = clientdata->pytype->tp_alloc; +#else + allocfunc alloc = (allocfunc)PyType_GetSlot(clientdata->pytype, Py_tp_alloc); +#endif + PyObject *next_self = alloc(clientdata->pytype, 0); + while (newobj->next) + newobj = (SwigPyObject *) newobj->next; + newobj->next = next_self; + newobj = (SwigPyObject *)next_self; +#ifdef SWIGPYTHON_BUILTIN + newobj->dict = 0; +#endif + } + } else { + newobj = PyObject_New(SwigPyObject, clientdata->pytype); +#ifdef SWIGPYTHON_BUILTIN + if (newobj) { + newobj->dict = 0; + } +#endif + } + if (newobj) { + newobj->ptr = ptr; + newobj->ty = type; + newobj->own = own; + newobj->next = 0; + return (PyObject*) newobj; + } + return SWIG_Py_Void(); + } + + assert(!(flags & SWIG_BUILTIN_TP_INIT)); + + robj = SwigPyObject_New(ptr, type, own); + if (robj && clientdata && !(flags & SWIG_POINTER_NOSHADOW)) { + PyObject *inst = SWIG_Python_NewShadowInstance(clientdata, robj); + Py_DECREF(robj); + robj = inst; + } + return robj; +} + +/* Create a new packed object */ + +SWIGRUNTIMEINLINE PyObject * +SWIG_Python_NewPackedObj(void *ptr, size_t sz, swig_type_info *type) { + return ptr ? SwigPyPacked_New((void *) ptr, sz, type) : SWIG_Py_Void(); +} + +/* -----------------------------------------------------------------------------* + * Get type list + * -----------------------------------------------------------------------------*/ + +#ifdef SWIG_LINK_RUNTIME +void *SWIG_ReturnGlobalTypeList(void *); +#endif + +static PyObject *Swig_TypeCache_global = NULL; + +/* The python cached type query */ +SWIGRUNTIME PyObject * +SWIG_Python_TypeCache(void) { + if (Swig_TypeCache_global == NULL) { + Swig_TypeCache_global = PyDict_New(); + } + return Swig_TypeCache_global; +} + +SWIGRUNTIME swig_module_info * +SWIG_Python_GetModule(void *SWIGUNUSEDPARM(clientdata)) { +#ifdef SWIG_LINK_RUNTIME + static void *type_pointer = (void *)0; + /* first check if module already created */ + if (!type_pointer) { + type_pointer = SWIG_ReturnGlobalTypeList((void *)0); + } +#else + void *type_pointer = PyCapsule_Import(SWIGPY_CAPSULE_NAME, 0); + if (PyErr_Occurred()) { + PyErr_Clear(); + type_pointer = (void *)0; + } +#endif + return (swig_module_info *) type_pointer; +} + + +static int interpreter_counter = 0; /* how many (sub-)interpreters are using swig_module's types */ + +SWIGRUNTIME void +SWIG_Python_DestroyModule(PyObject *obj) +{ + swig_module_info *swig_module = (swig_module_info *) PyCapsule_GetPointer(obj, SWIGPY_CAPSULE_NAME); + swig_type_info **types = swig_module->types; + size_t i; + if (--interpreter_counter != 0) /* another sub-interpreter may still be using the swig_module's types */ + return; + for (i =0; i < swig_module->size; ++i) { + swig_type_info *ty = types[i]; + if (ty->owndata) { + SwigPyClientData *data = (SwigPyClientData *) ty->clientdata; + ty->clientdata = 0; + if (data) SwigPyClientData_Del(data); + } + } + Py_DECREF(SWIG_This()); + Swig_This_global = NULL; + Py_DECREF(SWIG_globals()); + Swig_Globals_global = NULL; + Py_DECREF(SWIG_Python_TypeCache()); + Swig_TypeCache_global = NULL; + Swig_Capsule_global = NULL; +} + +SWIGRUNTIME void +SWIG_Python_SetModule(swig_module_info *swig_module) { +#if PY_VERSION_HEX >= 0x03000000 + /* Add a dummy module object into sys.modules */ + PyObject *module = PyImport_AddModule("swig_runtime_data" SWIG_RUNTIME_VERSION); +#else + static PyMethodDef swig_empty_runtime_method_table[] = { {NULL, NULL, 0, NULL} }; /* Sentinel */ + PyObject *module = Py_InitModule("swig_runtime_data" SWIG_RUNTIME_VERSION, swig_empty_runtime_method_table); +#endif + PyObject *pointer = PyCapsule_New((void *) swig_module, SWIGPY_CAPSULE_NAME, SWIG_Python_DestroyModule); + if (pointer && module) { + if (PyModule_AddObject(module, SWIGPY_CAPSULE_ATTR_NAME, pointer) == 0) { + ++interpreter_counter; + Swig_Capsule_global = pointer; + } else { + Py_DECREF(pointer); + } + } else { + Py_XDECREF(pointer); + } +} + +SWIGRUNTIME swig_type_info * +SWIG_Python_TypeQuery(const char *type) +{ + PyObject *cache = SWIG_Python_TypeCache(); + PyObject *key = SWIG_Python_str_FromChar(type); + PyObject *obj = PyDict_GetItem(cache, key); + swig_type_info *descriptor; + if (obj) { + descriptor = (swig_type_info *) PyCapsule_GetPointer(obj, NULL); + } else { + swig_module_info *swig_module = SWIG_GetModule(0); + descriptor = SWIG_TypeQueryModule(swig_module, swig_module, type); + if (descriptor) { + obj = PyCapsule_New((void*) descriptor, NULL, NULL); + if (obj) { + PyDict_SetItem(cache, key, obj); + Py_DECREF(obj); + } + } + } + Py_DECREF(key); + return descriptor; +} + +/* + For backward compatibility only +*/ +#define SWIG_POINTER_EXCEPTION 0 +#define SWIG_arg_fail(arg) SWIG_Python_ArgFail(arg) +#define SWIG_MustGetPtr(p, type, argnum, flags) SWIG_Python_MustGetPtr(p, type, argnum, flags) + +SWIGRUNTIME int +SWIG_Python_AddErrMesg(const char* mesg, int infront) +{ + if (PyErr_Occurred()) { + PyObject *type = 0; + PyObject *value = 0; + PyObject *traceback = 0; + PyErr_Fetch(&type, &value, &traceback); + if (value) { + PyObject *old_str = PyObject_Str(value); + PyObject *bytes = NULL; + const char *tmp = SWIG_PyUnicode_AsUTF8AndSize(old_str, NULL, &bytes); + const char *errmesg = tmp ? tmp : "Invalid error message"; + Py_XINCREF(type); + PyErr_Clear(); + if (infront) { + PyErr_Format(type, "%s %s", mesg, errmesg); + } else { + PyErr_Format(type, "%s %s", errmesg, mesg); + } + Py_XDECREF(bytes); + Py_DECREF(old_str); + } + return 1; + } else { + return 0; + } +} + +SWIGRUNTIME int +SWIG_Python_ArgFail(int argnum) +{ + if (PyErr_Occurred()) { + /* add information about failing argument */ + char mesg[256]; + PyOS_snprintf(mesg, sizeof(mesg), "argument number %d:", argnum); + return SWIG_Python_AddErrMesg(mesg, 1); + } else { + return 0; + } +} + +SWIGRUNTIMEINLINE const char * +SwigPyObject_GetDesc(PyObject *self) +{ + SwigPyObject *v = (SwigPyObject *)self; + swig_type_info *ty = v ? v->ty : 0; + return ty ? ty->str : ""; +} + +SWIGRUNTIME void +SWIG_Python_TypeError(const char *type, PyObject *obj) +{ + if (type) { +#if defined(SWIG_COBJECT_TYPES) + if (obj && SwigPyObject_Check(obj)) { + const char *otype = (const char *) SwigPyObject_GetDesc(obj); + if (otype) { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, 'SwigPyObject(%s)' is received", + type, otype); + return; + } + } else +#endif + { +#ifndef Py_LIMITED_API + /* tp_name is not accessible */ + const char *otype = (obj ? obj->ob_type->tp_name : 0); + if (otype) { + PyObject *str = PyObject_Str(obj); + PyObject *bytes = NULL; + const char *cstr = str ? SWIG_PyUnicode_AsUTF8AndSize(str, NULL, &bytes) : 0; + if (cstr) { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s(%s)' is received", + type, otype, cstr); + } else { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s' is received", + type, otype); + } + Py_XDECREF(bytes); + Py_XDECREF(str); + return; + } +#endif + } + PyErr_Format(PyExc_TypeError, "a '%s' is expected", type); + } else { + PyErr_Format(PyExc_TypeError, "unexpected type is received"); + } +} + + +/* Convert a pointer value, signal an exception on a type mismatch */ +SWIGRUNTIME void * +SWIG_Python_MustGetPtr(PyObject *obj, swig_type_info *ty, int SWIGUNUSEDPARM(argnum), int flags) { + void *result; + if (SWIG_Python_ConvertPtr(obj, &result, ty, flags) == -1) { + PyErr_Clear(); + } + return result; +} + +#ifdef SWIGPYTHON_BUILTIN +SWIGRUNTIME int +SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) { + PyTypeObject *tp = obj->ob_type; + PyObject *descr; + PyObject *encoded_name; + descrsetfunc f; + int res = -1; + +# ifdef Py_USING_UNICODE + if (PyString_Check(name)) { + name = PyUnicode_Decode(PyString_AsString(name), PyString_Size(name), NULL, NULL); + if (!name) + return -1; + } else if (!PyUnicode_Check(name)) +# else + if (!PyString_Check(name)) +# endif + { + PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", name->ob_type->tp_name); + return -1; + } else { + Py_INCREF(name); + } + + if (!tp->tp_dict) { + if (PyType_Ready(tp) != 0) + goto done; + } + + descr = _PyType_Lookup(tp, name); + f = NULL; + if (descr != NULL) + f = descr->ob_type->tp_descr_set; + if (!f) { + if (PyString_Check(name)) { + encoded_name = name; + Py_INCREF(name); + } else { + encoded_name = PyUnicode_AsUTF8String(name); + if (!encoded_name) + goto done; + } + PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.200s'", tp->tp_name, PyString_AsString(encoded_name)); + Py_DECREF(encoded_name); + } else { + res = f(descr, obj, value); + } + + done: + Py_DECREF(name); + return res; +} +#endif + + +#ifdef __cplusplus +} +#endif + + + +#define SWIG_exception_fail(code, msg) do { SWIG_Error(code, msg); SWIG_fail; } while(0) + +#define SWIG_contract_assert(expr, msg) do { if (!(expr)) { SWIG_Error(SWIG_RuntimeError, msg); SWIG_fail; } } while (0) + + + + #define SWIG_exception(code, msg) do { SWIG_Error(code, msg); SWIG_fail;; } while(0) + +/* ----------------------------------------------------------------------------- + * director_common.swg + * + * This file contains support for director classes which is common between + * languages. + * ----------------------------------------------------------------------------- */ + +/* + Use -DSWIG_DIRECTOR_STATIC if you prefer to avoid the use of the + 'Swig' namespace. This could be useful for multi-modules projects. +*/ +#ifdef SWIG_DIRECTOR_STATIC +/* Force anonymous (static) namespace */ +#define Swig +#endif +/* ----------------------------------------------------------------------------- + * director.swg + * + * This file contains support for director classes so that Python proxy + * methods can be called from C++. + * ----------------------------------------------------------------------------- */ + +#ifndef SWIG_DIRECTOR_PYTHON_HEADER_ +#define SWIG_DIRECTOR_PYTHON_HEADER_ + +#include +#include +#include +#include +#include + +#if defined(SWIG_PYTHON_THREADS) +/* __THREAD__ is the old macro to activate some thread support */ +# if !defined(__THREAD__) +# define __THREAD__ 1 +# endif +#endif + +#ifdef __THREAD__ +#ifndef Py_LIMITED_API +# include "pythread.h" +#else +# if defined(_WIN32) +# include +# else +# include +# endif +#endif +#endif + +/* + Use -DSWIG_PYTHON_DIRECTOR_NO_VTABLE if you don't want to generate a 'virtual + table', and avoid multiple GetAttr calls to retrieve the python + methods. +*/ + +#ifndef SWIG_PYTHON_DIRECTOR_NO_VTABLE +#ifndef SWIG_PYTHON_DIRECTOR_VTABLE +#define SWIG_PYTHON_DIRECTOR_VTABLE +#endif +#endif + + + +/* + Use -DSWIG_DIRECTOR_NO_UEH if you prefer to avoid the use of the + Undefined Exception Handler provided by swig. +*/ +#ifndef SWIG_DIRECTOR_NO_UEH +#ifndef SWIG_DIRECTOR_UEH +#define SWIG_DIRECTOR_UEH +#endif +#endif + + +/* + Use -DSWIG_DIRECTOR_NORTTI if you prefer to avoid the use of the + native C++ RTTI and dynamic_cast<>. But be aware that directors + could stop working when using this option. +*/ +#ifdef SWIG_DIRECTOR_NORTTI +/* + When we don't use the native C++ RTTI, we implement a minimal one + only for Directors. +*/ +# ifndef SWIG_DIRECTOR_RTDIR +# define SWIG_DIRECTOR_RTDIR + +namespace Swig { + class Director; + SWIGINTERN std::map& get_rtdir_map() { + static std::map rtdir_map; + return rtdir_map; + } + + SWIGINTERNINLINE void set_rtdir(void *vptr, Director *rtdir) { + get_rtdir_map()[vptr] = rtdir; + } + + SWIGINTERNINLINE Director *get_rtdir(void *vptr) { + std::map::const_iterator pos = get_rtdir_map().find(vptr); + Director *rtdir = (pos != get_rtdir_map().end()) ? pos->second : 0; + return rtdir; + } +} +# endif /* SWIG_DIRECTOR_RTDIR */ + +# define SWIG_DIRECTOR_CAST(ARG) Swig::get_rtdir(static_cast(ARG)) +# define SWIG_DIRECTOR_RGTR(ARG1, ARG2) Swig::set_rtdir(static_cast(ARG1), ARG2) + +#else + +# define SWIG_DIRECTOR_CAST(ARG) dynamic_cast(ARG) +# define SWIG_DIRECTOR_RGTR(ARG1, ARG2) + +#endif /* SWIG_DIRECTOR_NORTTI */ + +extern "C" { + struct swig_type_info; +} + +namespace Swig { + + /* memory handler */ + struct GCItem { + virtual ~GCItem() {} + + virtual int get_own() const { + return 0; + } + }; + + struct GCItem_var { + GCItem_var(GCItem *item = 0) : _item(item) { + } + + GCItem_var& operator=(GCItem *item) { + GCItem *tmp = _item; + _item = item; + delete tmp; + return *this; + } + + ~GCItem_var() { + delete _item; + } + + GCItem * operator->() const { + return _item; + } + + private: + GCItem *_item; + }; + + struct GCItem_Object : GCItem { + GCItem_Object(int own) : _own(own) { + } + + virtual ~GCItem_Object() { + } + + int get_own() const { + return _own; + } + + private: + int _own; + }; + + template + struct GCItem_T : GCItem { + GCItem_T(Type *ptr) : _ptr(ptr) { + } + + virtual ~GCItem_T() { + delete _ptr; + } + + private: + Type *_ptr; + }; + + template + struct GCArray_T : GCItem { + GCArray_T(Type *ptr) : _ptr(ptr) { + } + + virtual ~GCArray_T() { + delete[] _ptr; + } + + private: + Type *_ptr; + }; + + /* base class for director exceptions */ + class DirectorException : public std::exception { + protected: + std::string swig_msg; + public: + DirectorException(PyObject *error, const char *hdr ="", const char *msg ="") : swig_msg(hdr) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + if (msg[0]) { + swig_msg += " "; + swig_msg += msg; + } + if (!PyErr_Occurred()) { + PyErr_SetString(error, swig_msg.c_str()); + } + SWIG_PYTHON_THREAD_END_BLOCK; + } + + virtual ~DirectorException() throw() { + } + + /* Deprecated, use what() instead */ + const char *getMessage() const { + return what(); + } + + const char *what() const throw() { + return swig_msg.c_str(); + } + + static void raise(PyObject *error, const char *msg) { + throw DirectorException(error, msg); + } + + static void raise(const char *msg) { + raise(PyExc_RuntimeError, msg); + } + }; + + /* type mismatch in the return value from a python method call */ + class DirectorTypeMismatchException : public DirectorException { + public: + DirectorTypeMismatchException(PyObject *error, const char *msg="") + : DirectorException(error, "SWIG director type mismatch", msg) { + } + + DirectorTypeMismatchException(const char *msg="") + : DirectorException(PyExc_TypeError, "SWIG director type mismatch", msg) { + } + + static void raise(PyObject *error, const char *msg) { + throw DirectorTypeMismatchException(error, msg); + } + + static void raise(const char *msg) { + throw DirectorTypeMismatchException(msg); + } + }; + + /* any python exception that occurs during a director method call */ + class DirectorMethodException : public DirectorException { + public: + DirectorMethodException(const char *msg = "") + : DirectorException(PyExc_RuntimeError, "SWIG director method error.", msg) { + } + + static void raise(const char *msg) { + throw DirectorMethodException(msg); + } + }; + + /* attempt to call a pure virtual method via a director method */ + class DirectorPureVirtualException : public DirectorException { + public: + DirectorPureVirtualException(const char *msg = "") + : DirectorException(PyExc_RuntimeError, "SWIG director pure virtual method called", msg) { + } + + static void raise(const char *msg) { + throw DirectorPureVirtualException(msg); + } + }; + + +#ifdef __THREAD__ +#ifndef Py_LIMITED_API + class Mutex + { + public: + Mutex() { + mutex_ = PyThread_allocate_lock(); + } + + ~Mutex() { + PyThread_release_lock(mutex_); + } + + private: + void Lock() { + PyThread_acquire_lock(mutex_, WAIT_LOCK); + } + + void Unlock() { + PyThread_free_lock(mutex_); + } + + PyThread_type_lock mutex_; + + friend class Guard; + }; +#elif defined(_WIN32) + class Mutex : private CRITICAL_SECTION { + public: + Mutex() { + InitializeCriticalSection(this); + } + + ~Mutex() { + DeleteCriticalSection(this); + } + + private: + void Lock() { + EnterCriticalSection(this); + } + + void Unlock() { + LeaveCriticalSection(this); + } + + friend class Guard; + }; +#else + class Mutex { + public: + Mutex() { + pthread_mutex_init(&mutex_, NULL); + } + + ~Mutex() { + pthread_mutex_destroy(&mutex_); + } + + private: + void Lock() { + pthread_mutex_lock(&mutex_); + } + + void Unlock() { + pthread_mutex_unlock(&mutex_); + } + + friend class Guard; + + pthread_mutex_t mutex_; + }; +#endif + class Guard { + Mutex &mutex_; + + public: + Guard(Mutex & mutex) : mutex_(mutex) { + mutex_.Lock(); + } + + ~Guard() { + mutex_.Unlock(); + } + }; +# define SWIG_GUARD(mutex) Guard _guard(mutex) +#else +# define SWIG_GUARD(mutex) +#endif + + /* director base class */ + class Director { + private: + /* pointer to the wrapped python object */ + PyObject *swig_self; + /* flag indicating whether the object is owned by python or c++ */ + mutable bool swig_disown_flag; + + /* decrement the reference count of the wrapped python object */ + void swig_decref() const { + if (swig_disown_flag) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_DECREF(swig_self); + SWIG_PYTHON_THREAD_END_BLOCK; + } + } + + public: + /* wrap a python object. */ + Director(PyObject *self) : swig_self(self), swig_disown_flag(false) { + } + + /* discard our reference at destruction */ + virtual ~Director() { + swig_decref(); + } + + /* return a pointer to the wrapped python object */ + PyObject *swig_get_self() const { + return swig_self; + } + + /* acquire ownership of the wrapped python object (the sense of "disown" is from python) */ + void swig_disown() const { + if (!swig_disown_flag) { + swig_disown_flag=true; + swig_incref(); + } + } + + /* increase the reference count of the wrapped python object */ + void swig_incref() const { + if (swig_disown_flag) { + Py_INCREF(swig_self); + } + } + + /* methods to implement pseudo protected director members */ + virtual bool swig_get_inner(const char * /* swig_protected_method_name */) const { + return true; + } + + virtual void swig_set_inner(const char * /* swig_protected_method_name */, bool /* swig_val */) const { + } + + /* ownership management */ + private: + typedef std::map swig_ownership_map; + mutable swig_ownership_map swig_owner; +#ifdef __THREAD__ + static Mutex swig_mutex_own; +#endif + + public: + template + void swig_acquire_ownership_array(Type *vptr) const { + if (vptr) { + SWIG_GUARD(swig_mutex_own); + swig_owner[vptr] = new GCArray_T(vptr); + } + } + + template + void swig_acquire_ownership(Type *vptr) const { + if (vptr) { + SWIG_GUARD(swig_mutex_own); + swig_owner[vptr] = new GCItem_T(vptr); + } + } + + void swig_acquire_ownership_obj(void *vptr, int own) const { + if (vptr && own) { + SWIG_GUARD(swig_mutex_own); + swig_owner[vptr] = new GCItem_Object(own); + } + } + + int swig_release_ownership(void *vptr) const { + int own = 0; + if (vptr) { + SWIG_GUARD(swig_mutex_own); + swig_ownership_map::iterator iter = swig_owner.find(vptr); + if (iter != swig_owner.end()) { + own = iter->second->get_own(); + swig_owner.erase(iter); + } + } + return own; + } + + template + static PyObject *swig_pyobj_disown(PyObject *pyobj, PyObject *SWIGUNUSEDPARM(args)) { + SwigPyObject *sobj = (SwigPyObject *)pyobj; + sobj->own = 0; + Director *d = SWIG_DIRECTOR_CAST(reinterpret_cast(sobj->ptr)); + if (d) + d->swig_disown(); + return PyWeakref_NewProxy(pyobj, NULL); + } + }; + +#ifdef __THREAD__ + Mutex Director::swig_mutex_own; +#endif +} + +#endif + +/* -------- TYPES TABLE (BEGIN) -------- */ + +#define SWIGTYPE_p_ConstPtr swig_types[0] +#define SWIGTYPE_p_InputBuffer swig_types[1] +#define SWIGTYPE_p_OutputBuffer swig_types[2] +#define SWIGTYPE_p_PrevStdError swig_types[3] +#define SWIGTYPE_p_Ptr swig_types[4] +#define SWIGTYPE_p_UniqueConstPtr swig_types[5] +#define SWIGTYPE_p_UniquePtr swig_types[6] +#define SWIGTYPE_p_allocator_type swig_types[7] +#define SWIGTYPE_p_bcos__Error swig_types[8] +#define SWIGTYPE_p_bcos__boostssl__EncodedMsg swig_types[9] +#define SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr swig_types[10] +#define SWIGTYPE_p_bcos__bytesConstRef swig_types[11] +#define SWIGTYPE_p_char swig_types[12] +#define SWIGTYPE_p_difference_type swig_types[13] +#define SWIGTYPE_p_int swig_types[14] +#define SWIGTYPE_p_long_long swig_types[15] +#define SWIGTYPE_p_p_PyObject swig_types[16] +#define SWIGTYPE_p_ppc__front__ErrorCallback swig_types[17] +#define SWIGTYPE_p_ppc__front__FrontConfig swig_types[18] +#define SWIGTYPE_p_ppc__front__FrontConfigBuilder swig_types[19] +#define SWIGTYPE_p_ppc__front__GetPeersInfoHandler swig_types[20] +#define SWIGTYPE_p_ppc__front__IFront swig_types[21] +#define SWIGTYPE_p_ppc__front__IFrontBuilder swig_types[22] +#define SWIGTYPE_p_ppc__front__IFrontClient swig_types[23] +#define SWIGTYPE_p_ppc__front__IMessageHandler swig_types[24] +#define SWIGTYPE_p_ppc__front__MessageDispatcherHandler swig_types[25] +#define SWIGTYPE_p_ppc__front__SendResponseHandler swig_types[26] +#define SWIGTYPE_p_ppc__gateway__IGateway__Ptr swig_types[27] +#define SWIGTYPE_p_ppc__protocol__EndPoint swig_types[28] +#define SWIGTYPE_p_ppc__protocol__GrpcConfig swig_types[29] +#define SWIGTYPE_p_ppc__protocol__GrpcServerConfig swig_types[30] +#define SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr swig_types[31] +#define SWIGTYPE_p_ppc__protocol__Message swig_types[32] +#define SWIGTYPE_p_ppc__protocol__MessageBuilder swig_types[33] +#define SWIGTYPE_p_ppc__protocol__MessageHeader swig_types[34] +#define SWIGTYPE_p_ppc__protocol__MessageHeaderBuilder swig_types[35] +#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeader swig_types[36] +#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeaderBuilder swig_types[37] +#define SWIGTYPE_p_ppc__protocol__MessagePayload swig_types[38] +#define SWIGTYPE_p_ppc__protocol__MessagePayloadBuilder swig_types[39] +#define SWIGTYPE_p_ppc__protocol__RouteType swig_types[40] +#define SWIGTYPE_p_ppc__sdk__Transport swig_types[41] +#define SWIGTYPE_p_ppc__sdk__TransportBuilder swig_types[42] +#define SWIGTYPE_p_short swig_types[43] +#define SWIGTYPE_p_signed_char swig_types[44] +#define SWIGTYPE_p_size_type swig_types[45] +#define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[46] +#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[47] +#define SWIGTYPE_p_std__exception swig_types[48] +#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[49] +#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[50] +#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[51] +#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[52] +#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[53] +#define SWIGTYPE_p_std__invalid_argument swig_types[54] +#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[55] +#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[56] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[57] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[58] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[59] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[60] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[61] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[62] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[63] +#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[64] +#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[65] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[66] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[67] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[68] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[69] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[70] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[71] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[72] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[73] +#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[74] +#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[75] +#define SWIGTYPE_p_std__string swig_types[76] +#define SWIGTYPE_p_std__string_view swig_types[77] +#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[78] +#define SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t swig_types[79] +#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[80] +#define SWIGTYPE_p_swig__SwigPyIterator swig_types[81] +#define SWIGTYPE_p_unsigned_char swig_types[82] +#define SWIGTYPE_p_unsigned_int swig_types[83] +#define SWIGTYPE_p_unsigned_long_long swig_types[84] +#define SWIGTYPE_p_unsigned_short swig_types[85] +#define SWIGTYPE_p_value_type swig_types[86] +static swig_type_info *swig_types[88]; +static swig_module_info swig_module = {swig_types, 87, 0, 0, 0, 0}; +#define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) +#define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) + +/* -------- TYPES TABLE (END) -------- */ + +#ifdef SWIG_TypeQuery +# undef SWIG_TypeQuery +#endif +#define SWIG_TypeQuery SWIG_Python_TypeQuery + +/*----------------------------------------------- + @(target):= _wedpr_python_transport.so + ------------------------------------------------*/ +#if PY_VERSION_HEX >= 0x03000000 +# define SWIG_init PyInit__wedpr_python_transport + +#else +# define SWIG_init init_wedpr_python_transport + +#endif +#define SWIG_name "_wedpr_python_transport" + +#ifdef __cplusplus +#include +/* SwigValueWrapper is described in swig.swg */ +template class SwigValueWrapper { + struct SwigSmartPointer { + T *ptr; + SwigSmartPointer(T *p) : ptr(p) { } + ~SwigSmartPointer() { delete ptr; } + SwigSmartPointer& operator=(SwigSmartPointer& rhs) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = rhs.ptr; rhs.ptr = 0; return *this; } + void reset(T *p) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = p; } + } pointer; + SwigValueWrapper& operator=(const SwigValueWrapper& rhs); + SwigValueWrapper(const SwigValueWrapper& rhs); +public: + SwigValueWrapper() : pointer(0) { } + SwigValueWrapper& operator=(const T& t) { SwigSmartPointer tmp(new T(t)); pointer = tmp; return *this; } +#if __cplusplus >=201103L + SwigValueWrapper& operator=(T&& t) { SwigSmartPointer tmp(new T(std::move(t))); pointer = tmp; return *this; } + operator T&&() const { return std::move(*pointer.ptr); } +#else + operator T&() const { return *pointer.ptr; } +#endif + T *operator&() const { return pointer.ptr; } + static void reset(SwigValueWrapper& t, T *p) { t.pointer.reset(p); } +}; + +/* + * SwigValueInit() is a generic initialisation solution as the following approach: + * + * T c_result = T(); + * + * doesn't compile for all types for example: + * + * unsigned int c_result = unsigned int(); + */ +template T SwigValueInit() { + return T(); +} + +#if __cplusplus >=201103L +# define SWIG_STD_MOVE(OBJ) std::move(OBJ) +#else +# define SWIG_STD_MOVE(OBJ) OBJ +#endif + +#endif + + +#define SWIG_as_voidptr(a) const_cast< void * >(static_cast< const void * >(a)) +#define SWIG_as_voidptrptr(a) ((void)SWIG_as_voidptr(*a),reinterpret_cast< void** >(a)) + + +#include + + +namespace swig { + class SwigPtr_PyObject { + protected: + PyObject *_obj; + + public: + SwigPtr_PyObject() :_obj(0) + { + } + + SwigPtr_PyObject(const SwigPtr_PyObject& item) : _obj(item._obj) + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_XINCREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + + SwigPtr_PyObject(PyObject *obj, bool initial_ref = true) :_obj(obj) + { + if (initial_ref) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_XINCREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + } + + SwigPtr_PyObject & operator=(const SwigPtr_PyObject& item) + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_XINCREF(item._obj); + Py_XDECREF(_obj); + _obj = item._obj; + SWIG_PYTHON_THREAD_END_BLOCK; + return *this; + } + + ~SwigPtr_PyObject() + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + Py_XDECREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + + operator PyObject *() const + { + return _obj; + } + + PyObject *operator->() const + { + return _obj; + } + }; +} + + +namespace swig { + struct SwigVar_PyObject : SwigPtr_PyObject { + SwigVar_PyObject(PyObject* obj = 0) : SwigPtr_PyObject(obj, false) { } + + SwigVar_PyObject & operator = (PyObject* obj) + { + Py_XDECREF(_obj); + _obj = obj; + return *this; + } + }; +} + + +#include // Use the C99 official header + + +#include + +#if PY_VERSION_HEX >= 0x03020000 +# define SWIGPY_SLICEOBJECT PyObject +#else +# define SWIGPY_SLICEOBJECT PySliceObject +#endif + + +#include +#include + + +#if defined(__GNUC__) +# if __GNUC__ == 2 && __GNUC_MINOR <= 96 +# define SWIG_STD_NOMODERN_STL +# endif +#endif + + +#include + + +#include + + +namespace swig { + struct stop_iteration { + }; + + struct SwigPyIterator { + private: + SwigPtr_PyObject _seq; + + protected: + SwigPyIterator(PyObject *seq) : _seq(seq) + { + } + + public: + virtual ~SwigPyIterator() {} + + // Access iterator method, required by Python + virtual PyObject *value() const = 0; + + // Forward iterator method, required by Python + virtual SwigPyIterator *incr(size_t n = 1) = 0; + + // Backward iterator method, very common in C++, but not required in Python + virtual SwigPyIterator *decr(size_t /*n*/ = 1) + { + throw stop_iteration(); + } + + // Random access iterator methods, but not required in Python + virtual ptrdiff_t distance(const SwigPyIterator &/*x*/) const + { + throw std::invalid_argument("operation not supported"); + } + + virtual bool equal (const SwigPyIterator &/*x*/) const + { + throw std::invalid_argument("operation not supported"); + } + + // C++ common/needed methods + virtual SwigPyIterator *copy() const = 0; + + PyObject *next() + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; // disable threads + PyObject *obj = value(); + incr(); + SWIG_PYTHON_THREAD_END_BLOCK; // re-enable threads + return obj; + } + + /* Make an alias for Python 3.x */ + PyObject *__next__() + { + return next(); + } + + PyObject *previous() + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; // disable threads + decr(); + PyObject *obj = value(); + SWIG_PYTHON_THREAD_END_BLOCK; // re-enable threads + return obj; + } + + SwigPyIterator *advance(ptrdiff_t n) + { + return (n > 0) ? incr(n) : decr(-n); + } + + bool operator == (const SwigPyIterator& x) const + { + return equal(x); + } + + bool operator != (const SwigPyIterator& x) const + { + return ! operator==(x); + } + + SwigPyIterator& operator += (ptrdiff_t n) + { + return *advance(n); + } + + SwigPyIterator& operator -= (ptrdiff_t n) + { + return *advance(-n); + } + + SwigPyIterator* operator + (ptrdiff_t n) const + { + return copy()->advance(n); + } + + SwigPyIterator* operator - (ptrdiff_t n) const + { + return copy()->advance(-n); + } + + ptrdiff_t operator - (const SwigPyIterator& x) const + { + return x.distance(*this); + } + + static swig_type_info* descriptor() { + static int init = 0; + static swig_type_info* desc = 0; + if (!init) { + desc = SWIG_TypeQuery("swig::SwigPyIterator *"); + init = 1; + } + return desc; + } + }; + +#if defined(SWIGPYTHON_BUILTIN) + inline PyObject* make_output_iterator_builtin (PyObject *pyself) + { + Py_INCREF(pyself); + return pyself; + } +#endif +} + + +SWIGINTERN int +SWIG_AsVal_double (PyObject *obj, double *val) +{ + int res = SWIG_TypeError; + if (PyFloat_Check(obj)) { + if (val) *val = PyFloat_AsDouble(obj); + return SWIG_OK; +#if PY_VERSION_HEX < 0x03000000 + } else if (PyInt_Check(obj)) { + if (val) *val = (double) PyInt_AsLong(obj); + return SWIG_OK; +#endif + } else if (PyLong_Check(obj)) { + double v = PyLong_AsDouble(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + double d = PyFloat_AsDouble(obj); + if (!PyErr_Occurred()) { + if (val) *val = d; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + long v = PyLong_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_AddCast(SWIG_OK)); + } else { + PyErr_Clear(); + } + } + } +#endif + return res; +} + + +#include + + +#include + + +SWIGINTERNINLINE int +SWIG_CanCastAsInteger(double *d, double min, double max) { + double x = *d; + if ((min <= x && x <= max)) { + double fx, cx, rd; + errno = 0; + fx = floor(x); + cx = ceil(x); + rd = ((x - fx) < 0.5) ? fx : cx; /* simple rint */ + if ((errno == EDOM) || (errno == ERANGE)) { + errno = 0; + } else { + double summ, reps, diff; + if (rd < x) { + diff = x - rd; + } else if (rd > x) { + diff = rd - x; + } else { + return 1; + } + summ = rd + x; + reps = diff/summ; + if (reps < 8*DBL_EPSILON) { + *d = rd; + return 1; + } + } + } + return 0; +} + + +SWIGINTERN int +SWIG_AsVal_unsigned_SS_long (PyObject *obj, unsigned long *val) +{ +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(obj)) { + long v = PyInt_AsLong(obj); + if (v >= 0) { + if (val) *val = v; + return SWIG_OK; + } else { + return SWIG_OverflowError; + } + } else +#endif + if (PyLong_Check(obj)) { + unsigned long v = PyLong_AsUnsignedLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + return SWIG_OverflowError; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + unsigned long v = PyLong_AsUnsignedLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + double d; + int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); + // Largest double not larger than ULONG_MAX (not portably calculated easily) + // Note that double(ULONG_MAX) is stored in a double rounded up by one (for 64-bit unsigned long) + // 0xfffffffffffff800ULL == (uint64_t)std::nextafter(double(__uint128_t(ULONG_MAX)+1), double(0)) + const double ulong_max = sizeof(unsigned long) == 8 ? 0xfffffffffffff800ULL : ULONG_MAX; + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, ulong_max)) { + if (val) *val = (unsigned long)(d); + return res; + } + } + } +#endif + return SWIG_TypeError; +} + + +#include +#if !defined(SWIG_NO_LLONG_MAX) +# if !defined(LLONG_MAX) && defined(__GNUC__) && defined (__LONG_LONG_MAX__) +# define LLONG_MAX __LONG_LONG_MAX__ +# define LLONG_MIN (-LLONG_MAX - 1LL) +# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) +# endif +#endif + + +#if defined(LLONG_MAX) && !defined(SWIG_LONG_LONG_AVAILABLE) +# define SWIG_LONG_LONG_AVAILABLE +#endif + + +#ifdef SWIG_LONG_LONG_AVAILABLE +SWIGINTERN int +SWIG_AsVal_unsigned_SS_long_SS_long (PyObject *obj, unsigned long long *val) +{ + int res = SWIG_TypeError; + if (PyLong_Check(obj)) { + unsigned long long v = PyLong_AsUnsignedLongLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + res = SWIG_OverflowError; + } + } else { + unsigned long v; + res = SWIG_AsVal_unsigned_SS_long (obj,&v); + if (SWIG_IsOK(res)) { + if (val) *val = v; + return res; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + const double mant_max = 1LL << DBL_MANT_DIG; + double d; + res = SWIG_AsVal_double (obj,&d); + if (SWIG_IsOK(res) && !SWIG_CanCastAsInteger(&d, 0, mant_max)) + return SWIG_OverflowError; + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, mant_max)) { + if (val) *val = (unsigned long long)(d); + return SWIG_AddCast(res); + } + res = SWIG_TypeError; + } +#endif + return res; +} +#endif + + +SWIGINTERNINLINE int +SWIG_AsVal_size_t (PyObject * obj, size_t *val) +{ + int res = SWIG_TypeError; +#ifdef SWIG_LONG_LONG_AVAILABLE + if (sizeof(size_t) <= sizeof(unsigned long)) { +#endif + unsigned long v; + res = SWIG_AsVal_unsigned_SS_long (obj, val ? &v : 0); + if (SWIG_IsOK(res) && val) *val = static_cast< size_t >(v); +#ifdef SWIG_LONG_LONG_AVAILABLE + } else if (sizeof(size_t) <= sizeof(unsigned long long)) { + unsigned long long v; + res = SWIG_AsVal_unsigned_SS_long_SS_long (obj, val ? &v : 0); + if (SWIG_IsOK(res) && val) *val = static_cast< size_t >(v); + } +#endif + return res; +} + + + #define SWIG_From_long PyInt_FromLong + + +#ifdef SWIG_LONG_LONG_AVAILABLE +SWIGINTERNINLINE PyObject* +SWIG_From_long_SS_long (long long value) +{ + return ((value < LONG_MIN) || (value > LONG_MAX)) ? + PyLong_FromLongLong(value) : PyInt_FromLong(static_cast< long >(value)); +} +#endif + + +SWIGINTERNINLINE PyObject * +SWIG_From_ptrdiff_t (ptrdiff_t value) +{ +#ifdef SWIG_LONG_LONG_AVAILABLE + if (sizeof(ptrdiff_t) <= sizeof(long)) { +#endif + return SWIG_From_long (static_cast< long >(value)); +#ifdef SWIG_LONG_LONG_AVAILABLE + } else { + /* assume sizeof(ptrdiff_t) <= sizeof(long long) */ + return SWIG_From_long_SS_long (static_cast< long long >(value)); + } +#endif +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_bool (bool value) +{ + return PyBool_FromLong(value ? 1 : 0); +} + + +SWIGINTERN int +SWIG_AsVal_long (PyObject *obj, long* val) +{ +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(obj)) { + if (val) *val = PyInt_AsLong(obj); + return SWIG_OK; + } else +#endif + if (PyLong_Check(obj)) { + long v = PyLong_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + return SWIG_OverflowError; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + long v = PyInt_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + double d; + int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); + // Largest double not larger than LONG_MAX (not portably calculated easily) + // Note that double(LONG_MAX) is stored in a double rounded up by one (for 64-bit long) + // 0x7ffffffffffffc00LL == (int64_t)std::nextafter(double(__uint128_t(LONG_MAX)+1), double(0)) + const double long_max = sizeof(long) == 8 ? 0x7ffffffffffffc00LL : LONG_MAX; + // No equivalent needed for 64-bit double(LONG_MIN) is exactly LONG_MIN + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, LONG_MIN, long_max)) { + if (val) *val = (long)(d); + return res; + } + } + } +#endif + return SWIG_TypeError; +} + + +#ifdef SWIG_LONG_LONG_AVAILABLE +SWIGINTERN int +SWIG_AsVal_long_SS_long (PyObject *obj, long long *val) +{ + int res = SWIG_TypeError; + if (PyLong_Check(obj)) { + long long v = PyLong_AsLongLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + res = SWIG_OverflowError; + } + } else { + long v; + res = SWIG_AsVal_long (obj,&v); + if (SWIG_IsOK(res)) { + if (val) *val = v; + return res; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + const double mant_max = 1LL << DBL_MANT_DIG; + const double mant_min = -mant_max; + double d; + res = SWIG_AsVal_double (obj,&d); + if (SWIG_IsOK(res) && !SWIG_CanCastAsInteger(&d, mant_min, mant_max)) + return SWIG_OverflowError; + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, mant_min, mant_max)) { + if (val) *val = (long long)(d); + return SWIG_AddCast(res); + } + res = SWIG_TypeError; + } +#endif + return res; +} +#endif + + +SWIGINTERNINLINE int +SWIG_AsVal_ptrdiff_t (PyObject * obj, ptrdiff_t *val) +{ + int res = SWIG_TypeError; +#ifdef SWIG_LONG_LONG_AVAILABLE + if (sizeof(ptrdiff_t) <= sizeof(long)) { +#endif + long v; + res = SWIG_AsVal_long (obj, val ? &v : 0); + if (SWIG_IsOK(res) && val) *val = static_cast< ptrdiff_t >(v); +#ifdef SWIG_LONG_LONG_AVAILABLE + } else if (sizeof(ptrdiff_t) <= sizeof(long long)) { + long long v; + res = SWIG_AsVal_long_SS_long (obj, val ? &v : 0); + if (SWIG_IsOK(res) && val) *val = static_cast< ptrdiff_t >(v); + } +#endif + return res; +} + + +#include + + +#include + + +SWIGINTERNINLINE PyObject* + SWIG_From_int (int value) +{ + return PyInt_FromLong((long) value); +} + + +#define SWIG_FILE_WITH_INIT +#include +#include +#include +#include +#include +#include "wedpr-transport/sdk/src/TransportBuilder.h" +#include "wedpr-transport/sdk/src/Transport.h" +#include "ppc-framework/libwrapper/Buffer.h" +#include "ppc-framework/front/IFront.h" +#include "ppc-framework/protocol/RouteType.h" +#include "ppc-framework/front/FrontConfig.h" +#include "ppc-framework/protocol/GrpcConfig.h" +#include +#include "ppc-framework/protocol/EndPoint.h" +#include "ppc-framework/protocol/Message.h" +#include "ppc-framework/protocol/MessagePayload.h" + + +namespace swig { + template + struct noconst_traits { + typedef Type noconst_type; + }; + + template + struct noconst_traits { + typedef Type noconst_type; + }; + + /* + type categories + */ + struct pointer_category { }; + struct value_category { }; + + /* + General traits that provides type_name and type_info + */ + template struct traits { }; + + template + inline const char* type_name() { + return traits::noconst_type >::type_name(); + } + + template struct traits_info { + static swig_type_info *type_query(std::string name) { + name += " *"; + return SWIG_TypeQuery(name.c_str()); + } + static swig_type_info *type_info() { + static swig_type_info *info = type_query(type_name()); + return info; + } + }; + + /* + Partial specialization for pointers (traits_info) + */ + template struct traits_info { + static swig_type_info *type_query(std::string name) { + name += " *"; + return SWIG_TypeQuery(name.c_str()); + } + static swig_type_info *type_info() { + static swig_type_info *info = type_query(type_name()); + return info; + } + }; + + template + inline swig_type_info *type_info() { + return traits_info::type_info(); + } + + /* + Partial specialization for pointers (traits) + */ + template struct traits { + typedef pointer_category category; + static std::string make_ptr_name(const char* name) { + std::string ptrname = name; + ptrname += " *"; + return ptrname; + } + static const char* type_name() { + static std::string name = make_ptr_name(swig::type_name()); + return name.c_str(); + } + }; + + template + struct traits_as { }; + + template + struct traits_check { }; + +} + + +namespace swig { + /* + Traits that provides the from method + */ + template struct traits_from_ptr { + static PyObject *from(Type *val, int owner = 0) { + return SWIG_InternalNewPointerObj(val, type_info(), owner); + } + }; + + template struct traits_from { + static PyObject *from(const Type& val) { + return traits_from_ptr::from(new Type(val), 1); + } + }; + + template struct traits_from { + static PyObject *from(Type* val) { + return traits_from_ptr::from(val, 0); + } + }; + + template struct traits_from { + static PyObject *from(const Type* val) { + return traits_from_ptr::from(const_cast(val), 0); + } + }; + + + template + inline PyObject *from(const Type& val) { + return traits_from::from(val); + } + + template + inline PyObject *from_ptr(Type* val, int owner) { + return traits_from_ptr::from(val, owner); + } + + /* + Traits that provides the asval/as/check method + */ + template + struct traits_asptr { + static int asptr(PyObject *obj, Type **val) { + int res = SWIG_ERROR; + swig_type_info *descriptor = type_info(); + if (val) { + Type *p = 0; + int newmem = 0; + res = descriptor ? SWIG_ConvertPtrAndOwn(obj, (void **)&p, descriptor, 0, &newmem) : SWIG_ERROR; + if (SWIG_IsOK(res)) { + if (newmem & SWIG_CAST_NEW_MEMORY) { + res |= SWIG_NEWOBJMASK; + } + *val = p; + } + } else { + res = descriptor ? SWIG_ConvertPtr(obj, 0, descriptor, 0) : SWIG_ERROR; + } + return res; + } + }; + + template + inline int asptr(PyObject *obj, Type **vptr) { + return traits_asptr::asptr(obj, vptr); + } + + template + struct traits_asval { + static int asval(PyObject *obj, Type *val) { + if (val) { + Type *p = 0; + int res = traits_asptr::asptr(obj, &p); + if (!SWIG_IsOK(res)) return res; + if (p) { + typedef typename noconst_traits::noconst_type noconst_type; + *(const_cast(val)) = *p; + if (SWIG_IsNewObj(res)){ + delete p; + res = SWIG_DelNewMask(res); + } + return res; + } else { + return SWIG_ERROR; + } + } else { + return traits_asptr::asptr(obj, (Type **)(0)); + } + } + }; + + template struct traits_asval { + static int asval(PyObject *obj, Type **val) { + if (val) { + typedef typename noconst_traits::noconst_type noconst_type; + noconst_type *p = 0; + int res = traits_asptr::asptr(obj, &p); + if (SWIG_IsOK(res)) { + *(const_cast(val)) = p; + } + return res; + } else { + return traits_asptr::asptr(obj, (Type **)(0)); + } + } + }; + + template + inline int asval(PyObject *obj, Type *val) { + return traits_asval::asval(obj, val); + } + + template + struct traits_as { + static Type as(PyObject *obj) { + Type v; + int res = asval(obj, &v); + if (!obj || !SWIG_IsOK(res)) { + if (!PyErr_Occurred()) { + ::SWIG_Error(SWIG_TypeError, swig::type_name()); + } + throw std::invalid_argument("bad type"); + } + return v; + } + }; + + template + struct traits_as { + static Type as(PyObject *obj) { + Type *v = 0; + int res = (obj ? traits_asptr::asptr(obj, &v) : SWIG_ERROR); + if (SWIG_IsOK(res) && v) { + if (SWIG_IsNewObj(res)) { + Type r(*v); + delete v; + return r; + } else { + return *v; + } + } else { + if (!PyErr_Occurred()) { + SWIG_Error(SWIG_TypeError, swig::type_name()); + } + throw std::invalid_argument("bad type"); + } + } + }; + + template + struct traits_as { + static Type* as(PyObject *obj) { + Type *v = 0; + int res = (obj ? traits_asptr::asptr(obj, &v) : SWIG_ERROR); + if (SWIG_IsOK(res)) { + return v; + } else { + if (!PyErr_Occurred()) { + SWIG_Error(SWIG_TypeError, swig::type_name()); + } + throw std::invalid_argument("bad type"); + } + } + }; + + template + inline Type as(PyObject *obj) { + return traits_as::category>::as(obj); + } + + template + struct traits_check { + static bool check(PyObject *obj) { + int res = obj ? asval(obj, (Type *)(0)) : SWIG_ERROR; + return SWIG_IsOK(res) ? true : false; + } + }; + + template + struct traits_check { + static bool check(PyObject *obj) { + int res = obj ? asptr(obj, (Type **)(0)) : SWIG_ERROR; + return SWIG_IsOK(res) ? true : false; + } + }; + + template + inline bool check(PyObject *obj) { + return traits_check::category>::check(obj); + } +} + + +#if defined(__SUNPRO_CC) && defined(_RWSTD_VER) +# if !defined(SWIG_NO_STD_NOITERATOR_TRAITS_STL) +# define SWIG_STD_NOITERATOR_TRAITS_STL +# endif +#endif + +#if !defined(SWIG_STD_NOITERATOR_TRAITS_STL) +#include +#else +namespace std { + template + struct iterator_traits { + typedef ptrdiff_t difference_type; + typedef typename Iterator::value_type value_type; + }; + + template + struct iterator_traits<__reverse_bi_iterator > { + typedef Distance difference_type; + typedef T value_type; + }; + + template + struct iterator_traits { + typedef T value_type; + typedef ptrdiff_t difference_type; + }; + + template + inline typename iterator_traits<_InputIterator>::difference_type + distance(_InputIterator __first, _InputIterator __last) + { + typename iterator_traits<_InputIterator>::difference_type __n = 0; + while (__first != __last) { + ++__first; ++__n; + } + return __n; + } +} +#endif + + +namespace swig { + template + class SwigPyIterator_T : public SwigPyIterator + { + public: + typedef OutIterator out_iterator; + typedef typename std::iterator_traits::value_type value_type; + typedef SwigPyIterator_T self_type; + + SwigPyIterator_T(out_iterator curr, PyObject *seq) + : SwigPyIterator(seq), current(curr) + { + } + + const out_iterator& get_current() const + { + return current; + } + + + bool equal (const SwigPyIterator &iter) const + { + const self_type *iters = dynamic_cast(&iter); + if (iters) { + return (current == iters->get_current()); + } else { + throw std::invalid_argument("bad iterator type"); + } + } + + ptrdiff_t distance(const SwigPyIterator &iter) const + { + const self_type *iters = dynamic_cast(&iter); + if (iters) { + return std::distance(current, iters->get_current()); + } else { + throw std::invalid_argument("bad iterator type"); + } + } + + protected: + out_iterator current; + }; + + template + struct from_oper + { + typedef const ValueType& argument_type; + typedef PyObject *result_type; + result_type operator()(argument_type v) const + { + return swig::from(v); + } + }; + + template::value_type, + typename FromOper = from_oper > + class SwigPyForwardIteratorOpen_T : public SwigPyIterator_T + { + public: + FromOper from; + typedef OutIterator out_iterator; + typedef ValueType value_type; + typedef SwigPyIterator_T base; + typedef SwigPyForwardIteratorOpen_T self_type; + + SwigPyForwardIteratorOpen_T(out_iterator curr, PyObject *seq) + : SwigPyIterator_T(curr, seq) + { + } + + PyObject *value() const { + return from(static_cast(*(base::current))); + } + + SwigPyIterator *copy() const + { + return new self_type(*this); + } + + SwigPyIterator *incr(size_t n = 1) + { + while (n--) { + ++base::current; + } + return this; + } + + }; + + template::value_type, + typename FromOper = from_oper > + class SwigPyIteratorOpen_T : public SwigPyForwardIteratorOpen_T + { + public: + FromOper from; + typedef OutIterator out_iterator; + typedef ValueType value_type; + typedef SwigPyIterator_T base; + typedef SwigPyIteratorOpen_T self_type; + + SwigPyIteratorOpen_T(out_iterator curr, PyObject *seq) + : SwigPyForwardIteratorOpen_T(curr, seq) + { + } + + SwigPyIterator *decr(size_t n = 1) + { + while (n--) { + --base::current; + } + return this; + } + }; + + template::value_type, + typename FromOper = from_oper > + class SwigPyForwardIteratorClosed_T : public SwigPyIterator_T + { + public: + FromOper from; + typedef OutIterator out_iterator; + typedef ValueType value_type; + typedef SwigPyIterator_T base; + typedef SwigPyForwardIteratorClosed_T self_type; + + SwigPyForwardIteratorClosed_T(out_iterator curr, out_iterator first, out_iterator last, PyObject *seq) + : SwigPyIterator_T(curr, seq), begin(first), end(last) + { + } + + PyObject *value() const { + if (base::current == end) { + throw stop_iteration(); + } else { + return from(static_cast(*(base::current))); + } + } + + SwigPyIterator *copy() const + { + return new self_type(*this); + } + + SwigPyIterator *incr(size_t n = 1) + { + while (n--) { + if (base::current == end) { + throw stop_iteration(); + } else { + ++base::current; + } + } + return this; + } + + protected: + out_iterator begin; + out_iterator end; + }; + + template::value_type, + typename FromOper = from_oper > + class SwigPyIteratorClosed_T : public SwigPyForwardIteratorClosed_T + { + public: + FromOper from; + typedef OutIterator out_iterator; + typedef ValueType value_type; + typedef SwigPyIterator_T base; + typedef SwigPyForwardIteratorClosed_T base0; + typedef SwigPyIteratorClosed_T self_type; + + SwigPyIteratorClosed_T(out_iterator curr, out_iterator first, out_iterator last, PyObject *seq) + : SwigPyForwardIteratorClosed_T(curr, first, last, seq) + { + } + + SwigPyIterator *decr(size_t n = 1) + { + while (n--) { + if (base::current == base0::begin) { + throw stop_iteration(); + } else { + --base::current; + } + } + return this; + } + }; + + + template + inline SwigPyIterator* + make_output_forward_iterator(const OutIter& current, const OutIter& begin,const OutIter& end, PyObject *seq = 0) + { + return new SwigPyForwardIteratorClosed_T(current, begin, end, seq); + } + + template + inline SwigPyIterator* + make_output_iterator(const OutIter& current, const OutIter& begin,const OutIter& end, PyObject *seq = 0) + { + return new SwigPyIteratorClosed_T(current, begin, end, seq); + } + + template + inline SwigPyIterator* + make_output_forward_iterator(const OutIter& current, PyObject *seq = 0) + { + return new SwigPyForwardIteratorOpen_T(current, seq); + } + + template + inline SwigPyIterator* + make_output_iterator(const OutIter& current, PyObject *seq = 0) + { + return new SwigPyIteratorOpen_T(current, seq); + } + +} + + +SWIGINTERN int +SWIG_AsVal_unsigned_SS_char (PyObject * obj, unsigned char *val) +{ + unsigned long v; + int res = SWIG_AsVal_unsigned_SS_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v > UCHAR_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< unsigned char >(v); + } + } + return res; +} + + +SWIGINTERNINLINE PyObject* +SWIG_From_unsigned_SS_long (unsigned long value) +{ + return (value > LONG_MAX) ? + PyLong_FromUnsignedLong(value) : PyInt_FromLong(static_cast< long >(value)); +} + + +SWIGINTERNINLINE PyObject * +SWIG_From_unsigned_SS_char (unsigned char value) +{ + return SWIG_From_unsigned_SS_long (value); +} + + +namespace swig { + template <> struct traits< unsigned char > { + typedef value_category category; + static const char* type_name() { return"unsigned char"; } + }; + template <> struct traits_asval< unsigned char > { + typedef unsigned char value_type; + static int asval(PyObject *obj, value_type *val) { + return SWIG_AsVal_unsigned_SS_char (obj, val); + } + }; + template <> struct traits_from< unsigned char > { + typedef unsigned char value_type; + static PyObject *from(const value_type& val) { + return SWIG_From_unsigned_SS_char (val); + } + }; +} + + +#include + +namespace std { + template <> + struct less + { + bool + operator()(PyObject * v, PyObject *w) const + { + bool res; + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + res = PyObject_RichCompareBool(v, w, Py_LT) ? true : false; + /* This may fall into a case of inconsistent + eg. ObjA > ObjX > ObjB + but ObjA < ObjB + */ + if( PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_TypeError) ) + { + /* Objects can't be compared, this mostly occurred in Python 3.0 */ + /* Compare their ptr directly for a workaround */ + res = (v < w); + PyErr_Clear(); + } + SWIG_PYTHON_THREAD_END_BLOCK; + return res; + } + }; + + template <> + struct less + { + bool + operator()(const swig::SwigPtr_PyObject& v, const swig::SwigPtr_PyObject& w) const + { + return std::less()(v, w); + } + }; + + template <> + struct less + { + bool + operator()(const swig::SwigVar_PyObject& v, const swig::SwigVar_PyObject& w) const + { + return std::less()(v, w); + } + }; + +} + +namespace swig { + template <> struct traits { + typedef value_category category; + static const char* type_name() { return "PyObject *"; } + }; + + template <> struct traits_asval { + typedef PyObject * value_type; + static int asval(PyObject *obj, value_type *val) { + if (val) *val = obj; + return SWIG_OK; + } + }; + + template <> + struct traits_check { + static bool check(PyObject *) { + return true; + } + }; + + template <> struct traits_from { + typedef PyObject * value_type; + static PyObject *from(const value_type& val) { + Py_XINCREF(val); + return val; + } + }; + +} + +namespace swig { + template + inline size_t + check_index(Difference i, size_t size, bool insert = false) { + if ( i < 0 ) { + if ((size_t) (-i) <= size) + return (size_t) (i + size); + } else if ( (size_t) i < size ) { + return (size_t) i; + } else if (insert && ((size_t) i == size)) { + return size; + } + throw std::out_of_range("index out of range"); + } + + template + void + slice_adjust(Difference i, Difference j, Py_ssize_t step, size_t size, Difference &ii, Difference &jj, bool insert = false) { + if (step == 0) { + throw std::invalid_argument("slice step cannot be zero"); + } else if (step > 0) { + // Required range: 0 <= i < size, 0 <= j < size, i <= j + if (i < 0) { + ii = 0; + } else if (i < (Difference)size) { + ii = i; + } else if (insert && (i >= (Difference)size)) { + ii = (Difference)size; + } + if (j < 0) { + jj = 0; + } else { + jj = (j < (Difference)size) ? j : (Difference)size; + } + if (jj < ii) + jj = ii; + } else { + // Required range: -1 <= i < size-1, -1 <= j < size-1, i >= j + if (i < -1) { + ii = -1; + } else if (i < (Difference) size) { + ii = i; + } else if (i >= (Difference)(size-1)) { + ii = (Difference)(size-1); + } + if (j < -1) { + jj = -1; + } else { + jj = (j < (Difference)size ) ? j : (Difference)(size-1); + } + if (ii < jj) + ii = jj; + } + } + + template + inline typename Sequence::iterator + getpos(Sequence* self, Difference i) { + typename Sequence::iterator pos = self->begin(); + std::advance(pos, check_index(i,self->size())); + return pos; + } + + template + inline typename Sequence::const_iterator + cgetpos(const Sequence* self, Difference i) { + typename Sequence::const_iterator pos = self->begin(); + std::advance(pos, check_index(i,self->size())); + return pos; + } + + template + inline void + erase(Sequence* seq, const typename Sequence::iterator& position) { + seq->erase(position); + } + + template + struct traits_reserve { + static void reserve(Sequence & /*seq*/, typename Sequence::size_type /*n*/) { + // This should be specialized for types that support reserve + } + }; + + template + inline Sequence* + getslice(const Sequence* self, Difference i, Difference j, Py_ssize_t step) { + typename Sequence::size_type size = self->size(); + Difference ii = 0; + Difference jj = 0; + swig::slice_adjust(i, j, step, size, ii, jj); + + if (step > 0) { + typename Sequence::const_iterator sb = self->begin(); + typename Sequence::const_iterator se = self->begin(); + std::advance(sb,ii); + std::advance(se,jj); + if (step == 1) { + return new Sequence(sb, se); + } else { + Sequence *sequence = new Sequence(); + swig::traits_reserve::reserve(*sequence, (jj - ii + step - 1) / step); + typename Sequence::const_iterator it = sb; + while (it!=se) { + sequence->push_back(*it); + for (Py_ssize_t c=0; c::reserve(*sequence, (ii - jj - step - 1) / -step); + typename Sequence::const_reverse_iterator sb = self->rbegin(); + typename Sequence::const_reverse_iterator se = self->rbegin(); + std::advance(sb,size-ii-1); + std::advance(se,size-jj-1); + typename Sequence::const_reverse_iterator it = sb; + while (it!=se) { + sequence->push_back(*it); + for (Py_ssize_t c=0; c<-step && it!=se; ++c) + it++; + } + return sequence; + } + } + + template + inline void + setslice(Sequence* self, Difference i, Difference j, Py_ssize_t step, const InputSeq& is = InputSeq()) { + typename Sequence::size_type size = self->size(); + Difference ii = 0; + Difference jj = 0; + swig::slice_adjust(i, j, step, size, ii, jj, true); + if (step > 0) { + if (step == 1) { + size_t ssize = jj - ii; + if (ssize <= is.size()) { + // expanding/staying the same size + swig::traits_reserve::reserve(*self, self->size() - ssize + is.size()); + typename Sequence::iterator sb = self->begin(); + typename InputSeq::const_iterator isit = is.begin(); + std::advance(sb,ii); + std::advance(isit, jj - ii); + self->insert(std::copy(is.begin(), isit, sb), isit, is.end()); + } else { + // shrinking + typename Sequence::iterator sb = self->begin(); + typename Sequence::iterator se = self->begin(); + std::advance(sb,ii); + std::advance(se,jj); + self->erase(sb,se); + sb = self->begin(); + std::advance(sb,ii); + self->insert(sb, is.begin(), is.end()); + } + } else { + size_t replacecount = (jj - ii + step - 1) / step; + if (is.size() != replacecount) { + char msg[1024]; + PyOS_snprintf(msg, sizeof(msg), "attempt to assign sequence of size %lu to extended slice of size %lu", (unsigned long)is.size(), (unsigned long)replacecount); + throw std::invalid_argument(msg); + } + typename Sequence::const_iterator isit = is.begin(); + typename Sequence::iterator it = self->begin(); + std::advance(it,ii); + for (size_t rc=0; rcend(); ++rc) { + *it++ = *isit++; + for (Py_ssize_t c=0; c<(step-1) && it != self->end(); ++c) + it++; + } + } + } else { + size_t replacecount = (ii - jj - step - 1) / -step; + if (is.size() != replacecount) { + char msg[1024]; + PyOS_snprintf(msg, sizeof(msg), "attempt to assign sequence of size %lu to extended slice of size %lu", (unsigned long)is.size(), (unsigned long)replacecount); + throw std::invalid_argument(msg); + } + typename Sequence::const_iterator isit = is.begin(); + typename Sequence::reverse_iterator it = self->rbegin(); + std::advance(it,size-ii-1); + for (size_t rc=0; rcrend(); ++rc) { + *it++ = *isit++; + for (Py_ssize_t c=0; c<(-step-1) && it != self->rend(); ++c) + it++; + } + } + } + + template + inline void + delslice(Sequence* self, Difference i, Difference j, Py_ssize_t step) { + typename Sequence::size_type size = self->size(); + Difference ii = 0; + Difference jj = 0; + swig::slice_adjust(i, j, step, size, ii, jj, true); + if (step > 0) { + typename Sequence::iterator sb = self->begin(); + std::advance(sb,ii); + if (step == 1) { + typename Sequence::iterator se = self->begin(); + std::advance(se,jj); + self->erase(sb,se); + } else { + typename Sequence::iterator it = sb; + size_t delcount = (jj - ii + step - 1) / step; + while (delcount) { + it = self->erase(it); + for (Py_ssize_t c=0; c<(step-1) && it != self->end(); ++c) + it++; + delcount--; + } + } + } else { + typename Sequence::reverse_iterator sb = self->rbegin(); + std::advance(sb,size-ii-1); + typename Sequence::reverse_iterator it = sb; + size_t delcount = (ii - jj - step - 1) / -step; + while (delcount) { + it = typename Sequence::reverse_iterator(self->erase((++it).base())); + for (Py_ssize_t c=0; c<(-step-1) && it != self->rend(); ++c) + it++; + delcount--; + } + } + } +} + + +namespace swig { + template + struct IteratorProtocol { + static void assign(PyObject *obj, Seq *seq) { + SwigVar_PyObject iter = PyObject_GetIter(obj); + if (iter) { + SwigVar_PyObject item = PyIter_Next(iter); + while (item) { + seq->insert(seq->end(), swig::as(item)); + item = PyIter_Next(iter); + } + } + } + + static bool check(PyObject *obj) { + bool ret = false; + SwigVar_PyObject iter = PyObject_GetIter(obj); + if (iter) { + SwigVar_PyObject item = PyIter_Next(iter); + ret = true; + while (item) { + ret = swig::check(item); + item = ret ? PyIter_Next(iter) : 0; + } + } + return ret; + } + }; + + template + struct traits_asptr_stdseq { + typedef Seq sequence; + typedef T value_type; + + static bool is_iterable(PyObject *obj) { + SwigVar_PyObject iter = PyObject_GetIter(obj); + PyErr_Clear(); + return iter != 0; + } + + static int asptr(PyObject *obj, sequence **seq) { + int ret = SWIG_ERROR; + if (obj == Py_None || SWIG_Python_GetSwigThis(obj)) { + sequence *p; + swig_type_info *descriptor = swig::type_info(); + if (descriptor && SWIG_IsOK(::SWIG_ConvertPtr(obj, (void **)&p, descriptor, 0))) { + if (seq) *seq = p; + return SWIG_OLDOBJ; + } + } else if (is_iterable(obj)) { + try { + if (seq) { + *seq = new sequence(); + IteratorProtocol::assign(obj, *seq); + if (!PyErr_Occurred()) + return SWIG_NEWOBJ; + } else { + return IteratorProtocol::check(obj) ? SWIG_OK : SWIG_ERROR; + } + } catch (std::exception& e) { + if (seq && !PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, e.what()); + } + if (seq) + delete *seq; + return SWIG_ERROR; + } + return ret; + } + }; + + template + struct traits_from_stdseq { + typedef Seq sequence; + typedef T value_type; + typedef typename Seq::size_type size_type; + typedef typename sequence::const_iterator const_iterator; + + static PyObject *from(const sequence& seq) { +#ifdef SWIG_PYTHON_EXTRA_NATIVE_CONTAINERS + swig_type_info *desc = swig::type_info(); + if (desc && desc->clientdata) { + return SWIG_InternalNewPointerObj(new sequence(seq), desc, SWIG_POINTER_OWN); + } +#endif + size_type size = seq.size(); + if (size <= (size_type)INT_MAX) { + PyObject *obj = PyTuple_New((Py_ssize_t)size); + Py_ssize_t i = 0; + for (const_iterator it = seq.begin(); it != seq.end(); ++it, ++i) { + PyTuple_SetItem(obj,i,swig::from(*it)); + } + return obj; + } else { + PyErr_SetString(PyExc_OverflowError,"sequence size not valid in python"); + return NULL; + } + } + }; +} + + + namespace swig { + template + struct traits_reserve > { + static void reserve(std::vector &seq, typename std::vector::size_type n) { + seq.reserve(n); + } + }; + + template + struct traits_asptr > { + static int asptr(PyObject *obj, std::vector **vec) { + return traits_asptr_stdseq >::asptr(obj, vec); + } + }; + + template + struct traits_from > { + static PyObject *from(const std::vector& vec) { + return traits_from_stdseq >::from(vec); + } + }; + } + + + namespace swig { + template <> struct traits > > { + typedef pointer_category category; + static const char* type_name() { + return "std::vector<" "unsigned char" "," "std::allocator< unsigned char >" " >"; + } + }; + } + +SWIGINTERN swig::SwigPyIterator *std_vector_Sl_uint8_t_Sg__iterator(std::vector< uint8_t > *self,PyObject **PYTHON_SELF){ + return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF); + } +SWIGINTERN bool std_vector_Sl_uint8_t_Sg____nonzero__(std::vector< uint8_t > const *self){ + return !(self->empty()); + } +SWIGINTERN bool std_vector_Sl_uint8_t_Sg____bool__(std::vector< uint8_t > const *self){ + return !(self->empty()); + } +SWIGINTERN std::vector< unsigned char >::size_type std_vector_Sl_uint8_t_Sg____len__(std::vector< uint8_t > const *self){ + return self->size(); + } + +#ifdef SWIG_LONG_LONG_AVAILABLE +SWIGINTERNINLINE PyObject* +SWIG_From_unsigned_SS_long_SS_long (unsigned long long value) +{ + return (value > LONG_MAX) ? + PyLong_FromUnsignedLongLong(value) : PyInt_FromLong(static_cast< long >(value)); +} +#endif + + +SWIGINTERNINLINE PyObject * +SWIG_From_size_t (size_t value) +{ +#ifdef SWIG_LONG_LONG_AVAILABLE + if (sizeof(size_t) <= sizeof(unsigned long)) { +#endif + return SWIG_From_unsigned_SS_long (static_cast< unsigned long >(value)); +#ifdef SWIG_LONG_LONG_AVAILABLE + } else { + /* assume sizeof(size_t) <= sizeof(unsigned long long) */ + return SWIG_From_unsigned_SS_long_SS_long (static_cast< unsigned long long >(value)); + } +#endif +} + +SWIGINTERN std::vector< unsigned char,std::allocator< unsigned char > > *std_vector_Sl_uint8_t_Sg____getslice__(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i,std::vector< unsigned char >::difference_type j){ + return swig::getslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____setslice____SWIG_0(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i,std::vector< unsigned char >::difference_type j){ + swig::setslice(self, i, j, 1, std::vector< unsigned char,std::allocator< unsigned char > >()); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____setslice____SWIG_1(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i,std::vector< unsigned char >::difference_type j,std::vector< unsigned char,std::allocator< unsigned char > > const &v){ + swig::setslice(self, i, j, 1, v); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____delslice__(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i,std::vector< unsigned char >::difference_type j){ + swig::delslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____delitem____SWIG_0(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i){ + swig::erase(self, swig::getpos(self, i)); + } +SWIGINTERN std::vector< unsigned char,std::allocator< unsigned char > > *std_vector_Sl_uint8_t_Sg____getitem____SWIG_0(std::vector< uint8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return NULL; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type id = i; + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type jd = j; + return swig::getslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____setitem____SWIG_0(std::vector< uint8_t > *self,SWIGPY_SLICEOBJECT *slice,std::vector< unsigned char,std::allocator< unsigned char > > const &v){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type id = i; + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type jd = j; + swig::setslice(self, id, jd, step, v); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____setitem____SWIG_1(std::vector< uint8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type id = i; + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg____delitem____SWIG_1(std::vector< uint8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type id = i; + std::vector< unsigned char,std::allocator< unsigned char > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN std::vector< unsigned char >::value_type const &std_vector_Sl_uint8_t_Sg____getitem____SWIG_1(std::vector< uint8_t > const *self,std::vector< unsigned char >::difference_type i){ + return *(swig::cgetpos(self, i)); + } + +namespace swig { + static PyObject* container_owner_attribute() { + static PyObject* attr = SWIG_Python_str_FromChar("__swig_container"); + return attr; + } + + template + struct container_owner { + // By default, do not add the back-reference (for value types) + // Specialization below will check the reference for pointer types. + static bool back_reference(PyObject* /*child*/, PyObject* /*owner*/) { + return false; + } + }; + + template <> + struct container_owner { + /* + * Call to add a back-reference to the owning object when returning a + * reference from a container. Will only set the reference if child + * is a SWIG wrapper object that does not own the pointer. + * + * returns whether the reference was set or not + */ + static bool back_reference(PyObject* child, PyObject* owner) { + SwigPyObject* swigThis = SWIG_Python_GetSwigThis(child); + if (swigThis && (swigThis->own & SWIG_POINTER_OWN) != SWIG_POINTER_OWN) { + return PyObject_SetAttr(child, container_owner_attribute(), owner) != -1; + } + return false; + } + }; +} + +SWIGINTERN void std_vector_Sl_uint8_t_Sg____setitem____SWIG_2(std::vector< uint8_t > *self,std::vector< unsigned char >::difference_type i,std::vector< unsigned char >::value_type const &x){ + *(swig::getpos(self,i)) = x; + } +SWIGINTERN std::vector< unsigned char >::value_type std_vector_Sl_uint8_t_Sg__pop(std::vector< uint8_t > *self){ + if (self->size() == 0) + throw std::out_of_range("pop from empty container"); + std::vector< unsigned char,std::allocator< unsigned char > >::value_type x = self->back(); + self->pop_back(); + return x; + } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__append(std::vector< uint8_t > *self,std::vector< unsigned char >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN std::vector< unsigned char >::iterator std_vector_Sl_uint8_t_Sg__erase__SWIG_0(std::vector< uint8_t > *self,std::vector< unsigned char >::iterator pos){ return self->erase(pos); } +SWIGINTERN std::vector< unsigned char >::iterator std_vector_Sl_uint8_t_Sg__erase__SWIG_1(std::vector< uint8_t > *self,std::vector< unsigned char >::iterator first,std::vector< unsigned char >::iterator last){ return self->erase(first, last); } +SWIGINTERN std::vector< unsigned char >::iterator std_vector_Sl_uint8_t_Sg__insert__SWIG_0(std::vector< uint8_t > *self,std::vector< unsigned char >::iterator pos,std::vector< unsigned char >::value_type const &x){ return self->insert(pos, x); } +SWIGINTERN void std_vector_Sl_uint8_t_Sg__insert__SWIG_1(std::vector< uint8_t > *self,std::vector< unsigned char >::iterator pos,std::vector< unsigned char >::size_type n,std::vector< unsigned char >::value_type const &x){ self->insert(pos, n, x); } + +SWIGINTERN int +SWIG_AsVal_signed_SS_char (PyObject * obj, signed char *val) +{ + long v; + int res = SWIG_AsVal_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v < SCHAR_MIN || v > SCHAR_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< signed char >(v); + } + } + return res; +} + + +SWIGINTERNINLINE PyObject * +SWIG_From_signed_SS_char (signed char value) +{ + return SWIG_From_long (value); +} + + +namespace swig { + template <> struct traits< signed char > { + typedef value_category category; + static const char* type_name() { return"signed char"; } + }; + template <> struct traits_asval< signed char > { + typedef signed char value_type; + static int asval(PyObject *obj, value_type *val) { + return SWIG_AsVal_signed_SS_char (obj, val); + } + }; + template <> struct traits_from< signed char > { + typedef signed char value_type; + static PyObject *from(const value_type& val) { + return SWIG_From_signed_SS_char (val); + } + }; +} + + + namespace swig { + template <> struct traits > > { + typedef pointer_category category; + static const char* type_name() { + return "std::vector<" "signed char" "," "std::allocator< signed char >" " >"; + } + }; + } + +SWIGINTERN swig::SwigPyIterator *std_vector_Sl_int8_t_Sg__iterator(std::vector< int8_t > *self,PyObject **PYTHON_SELF){ + return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF); + } +SWIGINTERN bool std_vector_Sl_int8_t_Sg____nonzero__(std::vector< int8_t > const *self){ + return !(self->empty()); + } +SWIGINTERN bool std_vector_Sl_int8_t_Sg____bool__(std::vector< int8_t > const *self){ + return !(self->empty()); + } +SWIGINTERN std::vector< signed char >::size_type std_vector_Sl_int8_t_Sg____len__(std::vector< int8_t > const *self){ + return self->size(); + } +SWIGINTERN std::vector< signed char,std::allocator< signed char > > *std_vector_Sl_int8_t_Sg____getslice__(std::vector< int8_t > *self,std::vector< signed char >::difference_type i,std::vector< signed char >::difference_type j){ + return swig::getslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____setslice____SWIG_0(std::vector< int8_t > *self,std::vector< signed char >::difference_type i,std::vector< signed char >::difference_type j){ + swig::setslice(self, i, j, 1, std::vector< signed char,std::allocator< signed char > >()); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____setslice____SWIG_1(std::vector< int8_t > *self,std::vector< signed char >::difference_type i,std::vector< signed char >::difference_type j,std::vector< signed char,std::allocator< signed char > > const &v){ + swig::setslice(self, i, j, 1, v); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____delslice__(std::vector< int8_t > *self,std::vector< signed char >::difference_type i,std::vector< signed char >::difference_type j){ + swig::delslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____delitem____SWIG_0(std::vector< int8_t > *self,std::vector< signed char >::difference_type i){ + swig::erase(self, swig::getpos(self, i)); + } +SWIGINTERN std::vector< signed char,std::allocator< signed char > > *std_vector_Sl_int8_t_Sg____getitem____SWIG_0(std::vector< int8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return NULL; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< signed char,std::allocator< signed char > >::difference_type id = i; + std::vector< signed char,std::allocator< signed char > >::difference_type jd = j; + return swig::getslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____setitem____SWIG_0(std::vector< int8_t > *self,SWIGPY_SLICEOBJECT *slice,std::vector< signed char,std::allocator< signed char > > const &v){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< signed char,std::allocator< signed char > >::difference_type id = i; + std::vector< signed char,std::allocator< signed char > >::difference_type jd = j; + swig::setslice(self, id, jd, step, v); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____setitem____SWIG_1(std::vector< int8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< signed char,std::allocator< signed char > >::difference_type id = i; + std::vector< signed char,std::allocator< signed char > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____delitem____SWIG_1(std::vector< int8_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< signed char,std::allocator< signed char > >::difference_type id = i; + std::vector< signed char,std::allocator< signed char > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN std::vector< signed char >::value_type const &std_vector_Sl_int8_t_Sg____getitem____SWIG_1(std::vector< int8_t > const *self,std::vector< signed char >::difference_type i){ + return *(swig::cgetpos(self, i)); + } +SWIGINTERN void std_vector_Sl_int8_t_Sg____setitem____SWIG_2(std::vector< int8_t > *self,std::vector< signed char >::difference_type i,std::vector< signed char >::value_type const &x){ + *(swig::getpos(self,i)) = x; + } +SWIGINTERN std::vector< signed char >::value_type std_vector_Sl_int8_t_Sg__pop(std::vector< int8_t > *self){ + if (self->size() == 0) + throw std::out_of_range("pop from empty container"); + std::vector< signed char,std::allocator< signed char > >::value_type x = self->back(); + self->pop_back(); + return x; + } +SWIGINTERN void std_vector_Sl_int8_t_Sg__append(std::vector< int8_t > *self,std::vector< signed char >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN std::vector< signed char >::iterator std_vector_Sl_int8_t_Sg__erase__SWIG_0(std::vector< int8_t > *self,std::vector< signed char >::iterator pos){ return self->erase(pos); } +SWIGINTERN std::vector< signed char >::iterator std_vector_Sl_int8_t_Sg__erase__SWIG_1(std::vector< int8_t > *self,std::vector< signed char >::iterator first,std::vector< signed char >::iterator last){ return self->erase(first, last); } +SWIGINTERN std::vector< signed char >::iterator std_vector_Sl_int8_t_Sg__insert__SWIG_0(std::vector< int8_t > *self,std::vector< signed char >::iterator pos,std::vector< signed char >::value_type const &x){ return self->insert(pos, x); } +SWIGINTERN void std_vector_Sl_int8_t_Sg__insert__SWIG_1(std::vector< int8_t > *self,std::vector< signed char >::iterator pos,std::vector< signed char >::size_type n,std::vector< signed char >::value_type const &x){ self->insert(pos, n, x); } + +SWIGINTERN swig_type_info* +SWIG_pchar_descriptor(void) +{ + static int init = 0; + static swig_type_info* info = 0; + if (!init) { + info = SWIG_TypeQuery("_p_char"); + init = 1; + } + return info; +} + + +/* Return string from Python obj. NOTE: obj must remain in scope in order + to use the returned cptr (but only when alloc is set to SWIG_OLDOBJ) */ +SWIGINTERN int +SWIG_AsCharPtrAndSize(PyObject *obj, char **cptr, size_t *psize, int *alloc) +{ +#if PY_VERSION_HEX>=0x03000000 +#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) + if (PyBytes_Check(obj)) +#else + if (PyUnicode_Check(obj)) +#endif +#else + if (PyString_Check(obj)) +#endif + { + char *cstr; Py_ssize_t len; + PyObject *bytes = NULL; + int ret = SWIG_OK; + if (alloc) + *alloc = SWIG_OLDOBJ; +#if PY_VERSION_HEX>=0x03000000 && defined(SWIG_PYTHON_STRICT_BYTE_CHAR) + if (PyBytes_AsStringAndSize(obj, &cstr, &len) == -1) + return SWIG_TypeError; +#else + cstr = (char *)SWIG_PyUnicode_AsUTF8AndSize(obj, &len, &bytes); + if (!cstr) + return SWIG_TypeError; + /* The returned string is only duplicated if the char * returned is not owned and memory managed by obj */ + if (bytes && cptr) { + if (alloc) { + cstr = reinterpret_cast< char* >(memcpy(new char[len + 1], cstr, sizeof(char)*(len + 1))); + *alloc = SWIG_NEWOBJ; + } else { + /* alloc must be set in order to clean up allocated memory */ + return SWIG_RuntimeError; + } + } +#endif + if (cptr) *cptr = cstr; + if (psize) *psize = len + 1; + Py_XDECREF(bytes); + return ret; + } else { +#if defined(SWIG_PYTHON_2_UNICODE) +#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) +#error "Cannot use both SWIG_PYTHON_2_UNICODE and SWIG_PYTHON_STRICT_BYTE_CHAR at once" +#endif +#if PY_VERSION_HEX<0x03000000 + if (PyUnicode_Check(obj)) { + char *cstr; Py_ssize_t len; + if (!alloc && cptr) { + return SWIG_RuntimeError; + } + obj = PyUnicode_AsUTF8String(obj); + if (!obj) + return SWIG_TypeError; + if (PyString_AsStringAndSize(obj, &cstr, &len) != -1) { + if (cptr) { + if (alloc) *alloc = SWIG_NEWOBJ; + *cptr = reinterpret_cast< char* >(memcpy(new char[len + 1], cstr, sizeof(char)*(len + 1))); + } + if (psize) *psize = len + 1; + + Py_XDECREF(obj); + return SWIG_OK; + } else { + Py_XDECREF(obj); + } + } +#endif +#endif + + swig_type_info* pchar_descriptor = SWIG_pchar_descriptor(); + if (pchar_descriptor) { + void* vptr = 0; + if (SWIG_ConvertPtr(obj, &vptr, pchar_descriptor, 0) == SWIG_OK) { + if (cptr) *cptr = (char *) vptr; + if (psize) *psize = vptr ? (strlen((char *)vptr) + 1) : 0; + if (alloc) *alloc = SWIG_OLDOBJ; + return SWIG_OK; + } + } + } + return SWIG_TypeError; +} + + + + + +SWIGINTERN int +SWIG_AsVal_int (PyObject * obj, int *val) +{ + long v; + int res = SWIG_AsVal_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v < INT_MIN || v > INT_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< int >(v); + } + } + return res; +} + + +SWIGINTERN int +SWIG_AsPtr_std_string (PyObject * obj, std::string **val) +{ + char* buf = 0 ; size_t size = 0; int alloc = SWIG_OLDOBJ; + if (SWIG_IsOK((SWIG_AsCharPtrAndSize(obj, &buf, &size, &alloc)))) { + if (buf) { + if (val) *val = new std::string(buf, size - 1); + if (alloc == SWIG_NEWOBJ) delete[] buf; + return SWIG_NEWOBJ; + } else { + if (val) *val = 0; + return SWIG_OLDOBJ; + } + } else { + static int init = 0; + static swig_type_info* descriptor = 0; + if (!init) { + descriptor = SWIG_TypeQuery("std::string" " *"); + init = 1; + } + if (descriptor) { + std::string *vptr; + int res = SWIG_ConvertPtr(obj, (void**)&vptr, descriptor, 0); + if (SWIG_IsOK(res) && val) *val = vptr; + return res; + } + } + return SWIG_ERROR; +} + + +struct SWIG_null_deleter { + void operator() (void const *) const { + } +}; +#define SWIG_NO_NULL_DELETER_0 , SWIG_null_deleter() +#define SWIG_NO_NULL_DELETER_1 +#define SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW +#define SWIG_NO_NULL_DELETER_SWIG_POINTER_OWN + + +#define SWIG_NO_NULL_DELETER_SWIG_BUILTIN_INIT + + +SWIGINTERNINLINE PyObject * +SWIG_FromCharPtrAndSize(const char* carray, size_t size) +{ + if (carray) { + if (size > INT_MAX) { + swig_type_info* pchar_descriptor = SWIG_pchar_descriptor(); + return pchar_descriptor ? + SWIG_InternalNewPointerObj(const_cast< char * >(carray), pchar_descriptor, 0) : SWIG_Py_Void(); + } else { +#if PY_VERSION_HEX >= 0x03000000 +#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) + return PyBytes_FromStringAndSize(carray, static_cast< Py_ssize_t >(size)); +#else + return PyUnicode_DecodeUTF8(carray, static_cast< Py_ssize_t >(size), "surrogateescape"); +#endif +#else + return PyString_FromStringAndSize(carray, static_cast< Py_ssize_t >(size)); +#endif + } + } else { + return SWIG_Py_Void(); + } +} + + +SWIGINTERNINLINE PyObject * +SWIG_From_std_string (const std::string& s) +{ + return SWIG_FromCharPtrAndSize(s.data(), s.size()); +} + + +SWIGINTERN int +SWIG_AsVal_unsigned_SS_short (PyObject * obj, unsigned short *val) +{ + unsigned long v; + int res = SWIG_AsVal_unsigned_SS_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v > USHRT_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< unsigned short >(v); + } + } + return res; +} + + +SWIGINTERNINLINE PyObject * +SWIG_From_unsigned_SS_short (unsigned short value) +{ + return SWIG_From_unsigned_SS_long (value); +} + + +SWIGINTERN int +SWIG_AsVal_bool (PyObject *obj, bool *val) +{ + int r; + if (!PyBool_Check(obj)) + return SWIG_ERROR; + r = PyObject_IsTrue(obj); + if (r == -1) + return SWIG_ERROR; + if (val) *val = r ? true : false; + return SWIG_OK; +} + + +#include + + +SWIGINTERNINLINE PyObject * +SWIG_From_short (short value) +{ + return SWIG_From_long (value); +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_unsigned_SS_int (unsigned int value) +{ + return PyInt_FromSize_t((size_t) value); +} + + + +/* --------------------------------------------------- + * C++ director class methods + * --------------------------------------------------- */ + +#include "wedpr_python_transportPYTHON_wrap.h" + +SwigDirector_ErrorCallback::SwigDirector_ErrorCallback(PyObject *self): ppc::front::ErrorCallback(), Swig::Director(self) { + SWIG_DIRECTOR_RGTR((ppc::front::ErrorCallback *)this, this); +} + + + + +SwigDirector_ErrorCallback::~SwigDirector_ErrorCallback() { +} + +void SwigDirector_ErrorCallback::onError(bcos::Error::Ptr error) { + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + + swig::SwigVar_PyObject obj0; + + smartarg1 = error ? new std::shared_ptr< bcos::Error >(error) : 0; + obj0 = SWIG_NewPointerObj(SWIG_as_voidptr(smartarg1), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN | 0 ); + + if (!swig_get_self()) { + Swig::DirectorException::raise("'self' uninitialized, maybe you forgot to call ErrorCallback.__init__."); + } +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) + const size_t swig_method_index = 0; + const char *const swig_method_name = "onError"; + PyObject *method = swig_get_method(swig_method_index, swig_method_name); + swig::SwigVar_PyObject result = PyObject_CallFunctionObjArgs(method ,(PyObject *)obj0, NULL); +#else + swig::SwigVar_PyObject swig_method_name = SWIG_Python_str_FromChar("onError"); + swig::SwigVar_PyObject result = PyObject_CallMethodObjArgs(swig_get_self(), (PyObject *) swig_method_name ,(PyObject *)obj0, NULL); +#endif + if (!result) { + PyObject *error = PyErr_Occurred(); + if (error) { + Swig::DirectorMethodException::raise("Error detected when calling 'ErrorCallback.onError'"); + } + } +} + + +SwigDirector_MessageDispatcherHandler::SwigDirector_MessageDispatcherHandler(PyObject *self): ppc::front::MessageDispatcherHandler(), Swig::Director(self) { + SWIG_DIRECTOR_RGTR((ppc::front::MessageDispatcherHandler *)this, this); +} + + + + +SwigDirector_MessageDispatcherHandler::~SwigDirector_MessageDispatcherHandler() { +} + +void SwigDirector_MessageDispatcherHandler::onMessage(ppc::protocol::Message::Ptr msg) { + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + + swig::SwigVar_PyObject obj0; + + smartarg1 = msg ? new std::shared_ptr< ppc::protocol::Message >(msg) : 0; + obj0 = SWIG_NewPointerObj(SWIG_as_voidptr(smartarg1), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN | 0 ); + + if (!swig_get_self()) { + Swig::DirectorException::raise("'self' uninitialized, maybe you forgot to call MessageDispatcherHandler.__init__."); + } +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) + const size_t swig_method_index = 0; + const char *const swig_method_name = "onMessage"; + PyObject *method = swig_get_method(swig_method_index, swig_method_name); + swig::SwigVar_PyObject result = PyObject_CallFunctionObjArgs(method ,(PyObject *)obj0, NULL); +#else + swig::SwigVar_PyObject swig_method_name = SWIG_Python_str_FromChar("onMessage"); + swig::SwigVar_PyObject result = PyObject_CallMethodObjArgs(swig_get_self(), (PyObject *) swig_method_name ,(PyObject *)obj0, NULL); +#endif + if (!result) { + PyObject *error = PyErr_Occurred(); + if (error) { + Swig::DirectorMethodException::raise("Error detected when calling 'MessageDispatcherHandler.onMessage'"); + } + } +} + + +SwigDirector_IMessageHandler::SwigDirector_IMessageHandler(PyObject *self): ppc::front::IMessageHandler(), Swig::Director(self) { + SWIG_DIRECTOR_RGTR((ppc::front::IMessageHandler *)this, this); +} + + + + +SwigDirector_IMessageHandler::~SwigDirector_IMessageHandler() { +} + +void SwigDirector_IMessageHandler::onMessage(bcos::Error::Ptr e,ppc::protocol::Message::Ptr msg,ppc::front::SendResponseHandler sendResponseHandler) { + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + std::shared_ptr< ppc::protocol::Message > *smartarg2 = 0 ; + + swig::SwigVar_PyObject obj0; + + smartarg1 = e ? new std::shared_ptr< bcos::Error >(e) : 0; + obj0 = SWIG_NewPointerObj(SWIG_as_voidptr(smartarg1), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN | 0 ); + + swig::SwigVar_PyObject obj1; + + smartarg2 = msg ? new std::shared_ptr< ppc::protocol::Message >(msg) : 0; + obj1 = SWIG_NewPointerObj(SWIG_as_voidptr(smartarg2), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN | 0 ); + + swig::SwigVar_PyObject obj2; + obj2 = SWIG_NewPointerObj((new ppc::front::SendResponseHandler(SWIG_STD_MOVE(sendResponseHandler))), SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_OWN | 0 ); + if (!swig_get_self()) { + Swig::DirectorException::raise("'self' uninitialized, maybe you forgot to call IMessageHandler.__init__."); + } +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) + const size_t swig_method_index = 0; + const char *const swig_method_name = "onMessage"; + PyObject *method = swig_get_method(swig_method_index, swig_method_name); + swig::SwigVar_PyObject result = PyObject_CallFunctionObjArgs(method ,(PyObject *)obj0,(PyObject *)obj1,(PyObject *)obj2, NULL); +#else + swig::SwigVar_PyObject swig_method_name = SWIG_Python_str_FromChar("onMessage"); + swig::SwigVar_PyObject result = PyObject_CallMethodObjArgs(swig_get_self(), (PyObject *) swig_method_name ,(PyObject *)obj0,(PyObject *)obj1,(PyObject *)obj2, NULL); +#endif + if (!result) { + PyObject *error = PyErr_Occurred(); + if (error) { + Swig::DirectorMethodException::raise("Error detected when calling 'IMessageHandler.onMessage'"); + } + } +} + + +SwigDirector_GetPeersInfoHandler::SwigDirector_GetPeersInfoHandler(PyObject *self): ppc::front::GetPeersInfoHandler(), Swig::Director(self) { + SWIG_DIRECTOR_RGTR((ppc::front::GetPeersInfoHandler *)this, this); +} + + + + +SwigDirector_GetPeersInfoHandler::~SwigDirector_GetPeersInfoHandler() { +} + +void SwigDirector_GetPeersInfoHandler::onPeersInfo(bcos::Error::Ptr e,std::string const &peersInfo) { + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + + swig::SwigVar_PyObject obj0; + + smartarg1 = e ? new std::shared_ptr< bcos::Error >(e) : 0; + obj0 = SWIG_NewPointerObj(SWIG_as_voidptr(smartarg1), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN | 0 ); + + swig::SwigVar_PyObject obj1; + obj1 = SWIG_From_std_string(static_cast< std::string >(peersInfo)); + if (!swig_get_self()) { + Swig::DirectorException::raise("'self' uninitialized, maybe you forgot to call GetPeersInfoHandler.__init__."); + } +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) + const size_t swig_method_index = 0; + const char *const swig_method_name = "onPeersInfo"; + PyObject *method = swig_get_method(swig_method_index, swig_method_name); + swig::SwigVar_PyObject result = PyObject_CallFunctionObjArgs(method ,(PyObject *)obj0,(PyObject *)obj1, NULL); +#else + swig::SwigVar_PyObject swig_method_name = SWIG_Python_str_FromChar("onPeersInfo"); + swig::SwigVar_PyObject result = PyObject_CallMethodObjArgs(swig_get_self(), (PyObject *) swig_method_name ,(PyObject *)obj0,(PyObject *)obj1, NULL); +#endif + if (!result) { + PyObject *error = PyErr_Occurred(); + if (error) { + Swig::DirectorMethodException::raise("Error detected when calling 'GetPeersInfoHandler.onPeersInfo'"); + } + } +} + + +#ifdef __cplusplus +extern "C" { +#endif +SWIGINTERN PyObject *_wrap_delete_SwigPyIterator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SwigPyIterator" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_value(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + PyObject *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_value" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (PyObject *)((swig::SwigPyIterator const *)arg1)->value(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = result; + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_incr__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + size_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_incr" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator_incr" "', argument " "2"" of type '" "size_t""'"); + } + arg2 = static_cast< size_t >(val2); + try { + result = (swig::SwigPyIterator *)(arg1)->incr(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_incr__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_incr" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (swig::SwigPyIterator *)(arg1)->incr(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_incr(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "SwigPyIterator_incr", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_SwigPyIterator_incr__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_SwigPyIterator_incr__SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'SwigPyIterator_incr'.\n" + " Possible C/C++ prototypes are:\n" + " swig::SwigPyIterator::incr(size_t)\n" + " swig::SwigPyIterator::incr()\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_decr__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + size_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_decr" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator_decr" "', argument " "2"" of type '" "size_t""'"); + } + arg2 = static_cast< size_t >(val2); + try { + result = (swig::SwigPyIterator *)(arg1)->decr(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_decr__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_decr" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (swig::SwigPyIterator *)(arg1)->decr(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_decr(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "SwigPyIterator_decr", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_SwigPyIterator_decr__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_SwigPyIterator_decr__SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'SwigPyIterator_decr'.\n" + " Possible C/C++ prototypes are:\n" + " swig::SwigPyIterator::decr(size_t)\n" + " swig::SwigPyIterator::decr()\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_distance(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + swig::SwigPyIterator *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + ptrdiff_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator_distance", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_distance" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_swig__SwigPyIterator, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SwigPyIterator_distance" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SwigPyIterator_distance" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + arg2 = reinterpret_cast< swig::SwigPyIterator * >(argp2); + try { + result = ((swig::SwigPyIterator const *)arg1)->distance((swig::SwigPyIterator const &)*arg2); + } catch(std::invalid_argument &_e) { + SWIG_Python_Raise(SWIG_NewPointerObj((new std::invalid_argument(static_cast< const std::invalid_argument& >(_e))),SWIGTYPE_p_std__invalid_argument,SWIG_POINTER_OWN), "std::invalid_argument", SWIGTYPE_p_std__invalid_argument); SWIG_fail; + } + resultobj = SWIG_From_ptrdiff_t(static_cast< ptrdiff_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_equal(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + swig::SwigPyIterator *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator_equal", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_equal" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_swig__SwigPyIterator, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SwigPyIterator_equal" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SwigPyIterator_equal" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + arg2 = reinterpret_cast< swig::SwigPyIterator * >(argp2); + try { + result = (bool)((swig::SwigPyIterator const *)arg1)->equal((swig::SwigPyIterator const &)*arg2); + } catch(std::invalid_argument &_e) { + SWIG_Python_Raise(SWIG_NewPointerObj((new std::invalid_argument(static_cast< const std::invalid_argument& >(_e))),SWIGTYPE_p_std__invalid_argument,SWIG_POINTER_OWN), "std::invalid_argument", SWIGTYPE_p_std__invalid_argument); SWIG_fail; + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_copy(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_copy" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + result = (swig::SwigPyIterator *)((swig::SwigPyIterator const *)arg1)->copy(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_next(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + PyObject *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_next" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (PyObject *)(arg1)->next(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = result; + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___next__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + PyObject *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___next__" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (PyObject *)(arg1)->__next__(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = result; + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_previous(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + PyObject *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_previous" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + try { + result = (PyObject *)(arg1)->previous(); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = result; + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator_advance(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + ptrdiff_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator_advance", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator_advance" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator_advance" "', argument " "2"" of type '" "ptrdiff_t""'"); + } + arg2 = static_cast< ptrdiff_t >(val2); + try { + result = (swig::SwigPyIterator *)(arg1)->advance(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___eq__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + swig::SwigPyIterator *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator___eq__", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___eq__" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_swig__SwigPyIterator, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SwigPyIterator___eq__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SwigPyIterator___eq__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + arg2 = reinterpret_cast< swig::SwigPyIterator * >(argp2); + result = (bool)((swig::SwigPyIterator const *)arg1)->operator ==((swig::SwigPyIterator const &)*arg2); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_TypeError)) { + return NULL; + } + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___ne__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + swig::SwigPyIterator *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator___ne__", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___ne__" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_swig__SwigPyIterator, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SwigPyIterator___ne__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SwigPyIterator___ne__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + arg2 = reinterpret_cast< swig::SwigPyIterator * >(argp2); + result = (bool)((swig::SwigPyIterator const *)arg1)->operator !=((swig::SwigPyIterator const &)*arg2); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_TypeError)) { + return NULL; + } + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___iadd__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + ptrdiff_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator___iadd__", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___iadd__" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator___iadd__" "', argument " "2"" of type '" "ptrdiff_t""'"); + } + arg2 = static_cast< ptrdiff_t >(val2); + try { + result = (swig::SwigPyIterator *) &(arg1)->operator +=(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___isub__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + ptrdiff_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator___isub__", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___isub__" "', argument " "1"" of type '" "swig::SwigPyIterator *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator___isub__" "', argument " "2"" of type '" "ptrdiff_t""'"); + } + arg2 = static_cast< ptrdiff_t >(val2); + try { + result = (swig::SwigPyIterator *) &(arg1)->operator -=(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___add__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + ptrdiff_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SwigPyIterator___add__", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___add__" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator___add__" "', argument " "2"" of type '" "ptrdiff_t""'"); + } + arg2 = static_cast< ptrdiff_t >(val2); + try { + result = (swig::SwigPyIterator *)((swig::SwigPyIterator const *)arg1)->operator +(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_TypeError)) { + return NULL; + } + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___sub____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + ptrdiff_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + swig::SwigPyIterator *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___sub__" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SwigPyIterator___sub__" "', argument " "2"" of type '" "ptrdiff_t""'"); + } + arg2 = static_cast< ptrdiff_t >(val2); + try { + result = (swig::SwigPyIterator *)((swig::SwigPyIterator const *)arg1)->operator -(arg2); + } catch(swig::stop_iteration &_e) { + { + (void)_e; + SWIG_SetErrorObj(PyExc_StopIteration, SWIG_Py_Void()); + SWIG_fail; + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_TypeError)) { + return NULL; + } + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___sub____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + swig::SwigPyIterator *arg1 = (swig::SwigPyIterator *) 0 ; + swig::SwigPyIterator *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + ptrdiff_t result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_swig__SwigPyIterator, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SwigPyIterator___sub__" "', argument " "1"" of type '" "swig::SwigPyIterator const *""'"); + } + arg1 = reinterpret_cast< swig::SwigPyIterator * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_swig__SwigPyIterator, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SwigPyIterator___sub__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SwigPyIterator___sub__" "', argument " "2"" of type '" "swig::SwigPyIterator const &""'"); + } + arg2 = reinterpret_cast< swig::SwigPyIterator * >(argp2); + result = ((swig::SwigPyIterator const *)arg1)->operator -((swig::SwigPyIterator const &)*arg2); + resultobj = SWIG_From_ptrdiff_t(static_cast< ptrdiff_t >(result)); + return resultobj; +fail: + if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_TypeError)) { + return NULL; + } + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *_wrap_SwigPyIterator___sub__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "SwigPyIterator___sub__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_SwigPyIterator___sub____SWIG_1(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_swig__SwigPyIterator, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_SwigPyIterator___sub____SWIG_0(self, argc, argv); + } + } + } + +fail: + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; +} + + +SWIGINTERN PyObject *SwigPyIterator_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_swig__SwigPyIterator, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_new_SharedBcosError(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< bcos::Error > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedBcosError", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >(); + { + std::shared_ptr< bcos::Error > *smartresult = (result && *result) ? new std::shared_ptr< bcos::Error >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedBcosError(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< bcos::Error > *arg1 = (std::shared_ptr< bcos::Error > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedBcosError" "', argument " "1"" of type '" "std::shared_ptr< bcos::Error > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedBcosError_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedBcosError_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedFrontConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::FrontConfig > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedFrontConfig", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >(); + { + std::shared_ptr< ppc::front::FrontConfig > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::FrontConfig >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedFrontConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::FrontConfig > *arg1 = (std::shared_ptr< ppc::front::FrontConfig > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedFrontConfig" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::FrontConfig > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedFrontConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedFrontConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedGrpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::GrpcConfig > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedGrpcConfig", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >(); + { + std::shared_ptr< ppc::protocol::GrpcConfig > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::GrpcConfig >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedGrpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::GrpcConfig > *arg1 = (std::shared_ptr< ppc::protocol::GrpcConfig > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedGrpcConfig" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::GrpcConfig > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedGrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedGrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedFront(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IFront > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedFront", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >(); + { + std::shared_ptr< ppc::front::IFront > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::IFront >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedFront(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IFront > *arg1 = (std::shared_ptr< ppc::front::IFront > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedFront" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::IFront > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedFront_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedFront_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedFrontClient(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IFrontClient > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedFrontClient", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >(); + { + std::shared_ptr< ppc::front::IFrontClient > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::IFrontClient >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedFrontClient(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IFrontClient > *arg1 = (std::shared_ptr< ppc::front::IFrontClient > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFrontClient > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedFrontClient" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::IFrontClient > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedFrontClient_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedFrontClient_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedErrorCallback(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::ErrorCallback > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedErrorCallback", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >(); + { + std::shared_ptr< ppc::front::ErrorCallback > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::ErrorCallback >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedErrorCallback(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::ErrorCallback > *arg1 = (std::shared_ptr< ppc::front::ErrorCallback > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedErrorCallback" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::ErrorCallback > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedErrorCallback_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedErrorCallback_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessageDispatcherHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageDispatcherHandler", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >(); + { + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::MessageDispatcherHandler >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessageDispatcherHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *arg1 = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessageDispatcherHandler" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::MessageDispatcherHandler > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessageDispatcherHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessageDispatcherHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedIMessageHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IMessageHandler > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedIMessageHandler", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >(); + { + std::shared_ptr< ppc::front::IMessageHandler > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::IMessageHandler >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedIMessageHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::IMessageHandler > *arg1 = (std::shared_ptr< ppc::front::IMessageHandler > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedIMessageHandler" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::IMessageHandler > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedIMessageHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedIMessageHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedGetPeersInfoHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedGetPeersInfoHandler", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >(); + { + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartresult = (result && *result) ? new std::shared_ptr< ppc::front::GetPeersInfoHandler >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedGetPeersInfoHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *arg1 = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedGetPeersInfoHandler" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::GetPeersInfoHandler > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedGetPeersInfoHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedGetPeersInfoHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedGateway(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::gateway::IGateway > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedGateway", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >(); + { + std::shared_ptr< ppc::gateway::IGateway > *smartresult = (result && *result) ? new std::shared_ptr< ppc::gateway::IGateway >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedGateway(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::gateway::IGateway > *arg1 = (std::shared_ptr< ppc::gateway::IGateway > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::gateway::IGateway > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedGateway" "', argument " "1"" of type '" "std::shared_ptr< ppc::gateway::IGateway > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::gateway::IGateway > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::gateway::IGateway > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::gateway::IGateway > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedGateway_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedGateway_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::Message > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessage", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >(); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::Message >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::Message > *arg1 = (std::shared_ptr< ppc::protocol::Message > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessage" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::Message > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessage_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessage_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessageHeader(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageHeader > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageHeader", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >(); + { + std::shared_ptr< ppc::protocol::MessageHeader > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeader >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessageHeader(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageHeader > *arg1 = (std::shared_ptr< ppc::protocol::MessageHeader > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessageHeader" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessageHeader > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessageHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessageHeader_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessagePayload(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessagePayload > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessagePayload", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >(); + { + std::shared_ptr< ppc::protocol::MessagePayload > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayload >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessagePayload(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessagePayload > *arg1 = (std::shared_ptr< ppc::protocol::MessagePayload > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessagePayload" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessagePayload > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessagePayload_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessagePayload_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedRouteInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedRouteInfo", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedRouteInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *arg1 = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedRouteInfo" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessageOptionalHeader > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedRouteInfo_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedRouteInfo_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessageBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageBuilder > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageBuilder", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >(); + { + std::shared_ptr< ppc::protocol::MessageBuilder > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessageBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageBuilder > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessageBuilder" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessageBuilder > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessageBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessageBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessageHeaderBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageHeaderBuilder", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(); + { + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessageHeaderBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessageHeaderBuilder" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessageHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessageHeaderBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedMessagePayloadBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedMessagePayloadBuilder", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(); + { + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedMessagePayloadBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedMessagePayloadBuilder" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedMessagePayloadBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedMessagePayloadBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SharedRouteInfoBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SharedRouteInfoBuilder", 0, 0, 0)) SWIG_fail; + result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN); + if (SWIG_POINTER_NEW) delete result; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SharedRouteInfoBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *arg1 = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *) 0 ; + void *argp1 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedRouteInfoBuilder" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1) : &tempshared1; + } + } + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SharedRouteInfoBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SharedRouteInfoBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_ubytes_iterator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + PyObject **arg2 = (PyObject **) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + swig::SwigPyIterator *result = 0 ; + + arg2 = &swig_obj[0]; + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_iterator" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (swig::SwigPyIterator *)std_vector_Sl_uint8_t_Sg__iterator(arg1,arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___nonzero__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___nonzero__" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (bool)std_vector_Sl_uint8_t_Sg____nonzero__((std::vector< unsigned char > const *)arg1); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___bool__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___bool__" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (bool)std_vector_Sl_uint8_t_Sg____bool__((std::vector< unsigned char > const *)arg1); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___len__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___len__" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = std_vector_Sl_uint8_t_Sg____len__((std::vector< unsigned char > const *)arg1); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___getslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + std::vector< unsigned char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< unsigned char,std::allocator< unsigned char > > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes___getslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___getslice__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___getslice__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes___getslice__" "', argument " "3"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned char >::difference_type >(val3); + try { + result = (std::vector< unsigned char,std::allocator< unsigned char > > *)std_vector_Sl_uint8_t_Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + std::vector< unsigned char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___setslice__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___setslice__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes___setslice__" "', argument " "3"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned char >::difference_type >(val3); + try { + std_vector_Sl_uint8_t_Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + std::vector< unsigned char >::difference_type arg3 ; + std::vector< unsigned char,std::allocator< unsigned char > > *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + int res4 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___setslice__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___setslice__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes___setslice__" "', argument " "3"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned char >::difference_type >(val3); + { + std::vector< unsigned char,std::allocator< unsigned char > > *ptr = (std::vector< unsigned char,std::allocator< unsigned char > > *)0; + res4 = swig::asptr(swig_obj[3], &ptr); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "ubytes___setslice__" "', argument " "4"" of type '" "std::vector< unsigned char,std::allocator< unsigned char > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ubytes___setslice__" "', argument " "4"" of type '" "std::vector< unsigned char,std::allocator< unsigned char > > const &""'"); + } + arg4 = ptr; + } + try { + std_vector_Sl_uint8_t_Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< unsigned char,std::allocator< unsigned char > > const &)*arg4); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res4)) delete arg4; + return resultobj; +fail: + if (SWIG_IsNewObj(res4)) delete arg4; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setslice__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes___setslice__", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes___setslice____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = swig::asptr(argv[3], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_ubytes___setslice____SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes___setslice__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::__setslice__(std::vector< unsigned char >::difference_type,std::vector< unsigned char >::difference_type)\n" + " std::vector< uint8_t >::__setslice__(std::vector< unsigned char >::difference_type,std::vector< unsigned char >::difference_type,std::vector< unsigned char,std::allocator< unsigned char > > const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes___delslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + std::vector< unsigned char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes___delslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___delslice__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___delslice__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes___delslice__" "', argument " "3"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned char >::difference_type >(val3); + try { + std_vector_Sl_uint8_t_Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___delitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___delitem__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + try { + std_vector_Sl_uint8_t_Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned char,std::allocator< unsigned char > > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___getitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + result = (std::vector< unsigned char,std::allocator< unsigned char > > *)std_vector_Sl_uint8_t_Sg____getitem____SWIG_0(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + std::vector< unsigned char,std::allocator< unsigned char > > *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res3 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___setitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + std::vector< unsigned char,std::allocator< unsigned char > > *ptr = (std::vector< unsigned char,std::allocator< unsigned char > > *)0; + res3 = swig::asptr(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "ubytes___setitem__" "', argument " "3"" of type '" "std::vector< unsigned char,std::allocator< unsigned char > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ubytes___setitem__" "', argument " "3"" of type '" "std::vector< unsigned char,std::allocator< unsigned char > > const &""'"); + } + arg3 = ptr; + } + try { + std_vector_Sl_uint8_t_Sg____setitem____SWIG_0(arg1,arg2,(std::vector< unsigned char,std::allocator< unsigned char > > const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res3)) delete arg3; + return resultobj; +fail: + if (SWIG_IsNewObj(res3)) delete arg3; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___setitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + std_vector_Sl_uint8_t_Sg____setitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___delitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + std_vector_Sl_uint8_t_Sg____delitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___delitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes___delitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ubytes___delitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes___delitem____SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes___delitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::__delitem__(std::vector< unsigned char >::difference_type)\n" + " std::vector< uint8_t >::__delitem__(SWIGPY_SLICEOBJECT *)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned char >::value_type *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___getitem__" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___getitem__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + try { + result = (std::vector< unsigned char >::value_type *) &std_vector_Sl_uint8_t_Sg____getitem____SWIG_1((std::vector< unsigned char > const *)arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___getitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes___getitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ubytes___getitem____SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes___getitem____SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes___getitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::__getitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint8_t >::__getitem__(std::vector< unsigned char >::difference_type) const\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::difference_type arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned char >::value_type temp3 ; + unsigned char val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes___setitem__" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes___setitem__" "', argument " "2"" of type '" "std::vector< unsigned char >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::difference_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes___setitem__" "', argument " "3"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned char >::value_type >(val3); + arg3 = &temp3; + try { + std_vector_Sl_uint8_t_Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(unsigned char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes___setitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes___setitem__", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ubytes___setitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + int res = swig::asptr(argv[2], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_ubytes___setitem____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes___setitem____SWIG_2(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes___setitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< unsigned char,std::allocator< unsigned char > > const &)\n" + " std::vector< uint8_t >::__setitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint8_t >::__setitem__(std::vector< unsigned char >::difference_type,std::vector< unsigned char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes_pop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::value_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_pop" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + try { + result = (std::vector< unsigned char >::value_type)std_vector_Sl_uint8_t_Sg__pop(arg1); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_append(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned char >::value_type temp2 ; + unsigned char val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes_append", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_append" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_append" "', argument " "2"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned char >::value_type >(val2); + arg2 = &temp2; + std_vector_Sl_uint8_t_Sg__append(arg1,(unsigned char const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ubytes__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + std::vector< uint8_t > *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + result = (std::vector< uint8_t > *)new std::vector< uint8_t >(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ubytes__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = 0 ; + int res1 = SWIG_OLDOBJ ; + std::vector< uint8_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + std::vector< unsigned char,std::allocator< unsigned char > > *ptr = (std::vector< unsigned char,std::allocator< unsigned char > > *)0; + res1 = swig::asptr(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_ubytes" "', argument " "1"" of type '" "std::vector< uint8_t > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_ubytes" "', argument " "1"" of type '" "std::vector< uint8_t > const &""'"); + } + arg1 = ptr; + } + result = (std::vector< uint8_t > *)new std::vector< uint8_t >((std::vector< uint8_t > const &)*arg1); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NEW | 0 ); + if (SWIG_IsNewObj(res1)) delete arg1; + return resultobj; +fail: + if (SWIG_IsNewObj(res1)) delete arg1; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_empty(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_empty" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (bool)((std::vector< uint8_t > const *)arg1)->empty(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_size" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = ((std::vector< uint8_t > const *)arg1)->size(); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_swap(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< uint8_t > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes_swap", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_swap" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_uint8_t_t, 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "ubytes_swap" "', argument " "2"" of type '" "std::vector< uint8_t > &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ubytes_swap" "', argument " "2"" of type '" "std::vector< uint8_t > &""'"); + } + arg2 = reinterpret_cast< std::vector< uint8_t > * >(argp2); + (arg1)->swap(*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_begin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned char >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_begin" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (arg1)->begin(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_end(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned char >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_end" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (arg1)->end(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_rbegin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned char >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_rbegin" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (arg1)->rbegin(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_rend(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned char >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_rend" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (arg1)->rend(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_clear(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_clear" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + (arg1)->clear(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_get_allocator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::allocator< unsigned char > > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_get_allocator" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = ((std::vector< uint8_t > const *)arg1)->get_allocator(); + resultobj = SWIG_NewPointerObj((new std::vector< unsigned char >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_unsigned_char_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ubytes__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned char >::size_type arg1 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< uint8_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_ubytes" "', argument " "1"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned char >::size_type >(val1); + result = (std::vector< uint8_t > *)new std::vector< uint8_t >(arg1); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_pop_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_pop_back" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + (arg1)->pop_back(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_resize" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_resize" "', argument " "2"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::size_type >(val2); + (arg1)->resize(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_erase" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } + } + result = std_vector_Sl_uint8_t_Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > arg2 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + swig::SwigPyIterator *iter3 = 0 ; + int res3 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_erase" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } + } + res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res3) || !iter3) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "3"" of type '" "std::vector< unsigned char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter3); + if (iter_t) { + arg3 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_erase" "', argument " "3"" of type '" "std::vector< unsigned char >::iterator""'"); + } + } + result = std_vector_Sl_uint8_t_Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_erase(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes_erase", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_ubytes_erase__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_ubytes_erase__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes_erase'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::erase(std::vector< unsigned char >::iterator)\n" + " std::vector< uint8_t >::erase(std::vector< unsigned char >::iterator,std::vector< unsigned char >::iterator)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_new_ubytes__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned char >::size_type arg1 ; + std::vector< unsigned char >::value_type *arg2 = 0 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< unsigned char >::value_type temp2 ; + unsigned char val2 ; + int ecode2 = 0 ; + std::vector< uint8_t > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_ubytes" "', argument " "1"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned char >::size_type >(val1); + ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_ubytes" "', argument " "2"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned char >::value_type >(val2); + arg2 = &temp2; + result = (std::vector< uint8_t > *)new std::vector< uint8_t >(arg1,(std::vector< unsigned char >::value_type const &)*arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ubytes(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_ubytes", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_ubytes__SWIG_0(self, argc, argv); + } + if (argc == 1) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_ubytes__SWIG_2(self, argc, argv); + } + } + if (argc == 1) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_new_ubytes__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_char(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_ubytes__SWIG_3(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_ubytes'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::vector()\n" + " std::vector< uint8_t >::vector(std::vector< uint8_t > const &)\n" + " std::vector< uint8_t >::vector(std::vector< unsigned char >::size_type)\n" + " std::vector< uint8_t >::vector(std::vector< unsigned char >::size_type,std::vector< unsigned char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes_push_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned char >::value_type temp2 ; + unsigned char val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes_push_back", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_push_back" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_push_back" "', argument " "2"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned char >::value_type >(val2); + arg2 = &temp2; + (arg1)->push_back((std::vector< unsigned char >::value_type const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_front(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_front" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (std::vector< unsigned char >::value_type *) &((std::vector< uint8_t > const *)arg1)->front(); + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_back" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = (std::vector< unsigned char >::value_type *) &((std::vector< uint8_t > const *)arg1)->back(); + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_assign(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::size_type arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned char >::value_type temp3 ; + unsigned char val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes_assign", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_assign" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_assign" "', argument " "2"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes_assign" "', argument " "3"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned char >::value_type >(val3); + arg3 = &temp3; + (arg1)->assign(arg2,(std::vector< unsigned char >::value_type const &)*arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::size_type arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned char >::value_type temp3 ; + unsigned char val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_resize" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_resize" "', argument " "2"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes_resize" "', argument " "3"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned char >::value_type >(val3); + arg3 = &temp3; + (arg1)->resize(arg2,(std::vector< unsigned char >::value_type const &)*arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_resize(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes_resize", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes_resize__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes_resize__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes_resize'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::resize(std::vector< unsigned char >::size_type)\n" + " std::vector< uint8_t >::resize(std::vector< unsigned char >::size_type,std::vector< unsigned char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > arg2 ; + std::vector< unsigned char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + std::vector< unsigned char >::value_type temp3 ; + unsigned char val3 ; + int ecode3 = 0 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_insert" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_insert" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_insert" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_unsigned_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes_insert" "', argument " "3"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned char >::value_type >(val3); + arg3 = &temp3; + result = std_vector_Sl_uint8_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(unsigned char const &)*arg3); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + SwigValueWrapper< std::vector< unsigned char >::iterator > arg2 ; + std::vector< unsigned char >::size_type arg3 ; + std::vector< unsigned char >::value_type *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + size_t val3 ; + int ecode3 = 0 ; + std::vector< unsigned char >::value_type temp4 ; + unsigned char val4 ; + int ecode4 = 0 ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_insert" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_insert" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ubytes_insert" "', argument " "2"" of type '" "std::vector< unsigned char >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ubytes_insert" "', argument " "3"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg3 = static_cast< std::vector< unsigned char >::size_type >(val3); + ecode4 = SWIG_AsVal_unsigned_SS_char(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "ubytes_insert" "', argument " "4"" of type '" "std::vector< unsigned char >::value_type""'"); + } + temp4 = static_cast< std::vector< unsigned char >::value_type >(val4); + arg4 = &temp4; + std_vector_Sl_uint8_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(unsigned char const &)*arg4); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_insert(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ubytes_insert", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes_insert__SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned char,std::allocator< unsigned char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_char(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ubytes_insert__SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ubytes_insert'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint8_t >::insert(std::vector< unsigned char >::iterator,std::vector< unsigned char >::value_type const &)\n" + " std::vector< uint8_t >::insert(std::vector< unsigned char >::iterator,std::vector< unsigned char >::size_type,std::vector< unsigned char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ubytes_reserve(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + std::vector< unsigned char >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ubytes_reserve", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_reserve" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ubytes_reserve" "', argument " "2"" of type '" "std::vector< unsigned char >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned char >::size_type >(val2); + (arg1)->reserve(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ubytes_capacity(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ubytes_capacity" "', argument " "1"" of type '" "std::vector< uint8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + result = ((std::vector< uint8_t > const *)arg1)->capacity(); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ubytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint8_t > *arg1 = (std::vector< uint8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ubytes" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ubytes_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ubytes_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_ibytes_iterator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + PyObject **arg2 = (PyObject **) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + swig::SwigPyIterator *result = 0 ; + + arg2 = &swig_obj[0]; + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_iterator" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (swig::SwigPyIterator *)std_vector_Sl_int8_t_Sg__iterator(arg1,arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___nonzero__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___nonzero__" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (bool)std_vector_Sl_int8_t_Sg____nonzero__((std::vector< signed char > const *)arg1); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___bool__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___bool__" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (bool)std_vector_Sl_int8_t_Sg____bool__((std::vector< signed char > const *)arg1); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___len__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___len__" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = std_vector_Sl_int8_t_Sg____len__((std::vector< signed char > const *)arg1); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___getslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + std::vector< signed char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< signed char,std::allocator< signed char > > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes___getslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___getslice__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___getslice__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes___getslice__" "', argument " "3"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg3 = static_cast< std::vector< signed char >::difference_type >(val3); + try { + result = (std::vector< signed char,std::allocator< signed char > > *)std_vector_Sl_int8_t_Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + std::vector< signed char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___setslice__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___setslice__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes___setslice__" "', argument " "3"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg3 = static_cast< std::vector< signed char >::difference_type >(val3); + try { + std_vector_Sl_int8_t_Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + std::vector< signed char >::difference_type arg3 ; + std::vector< signed char,std::allocator< signed char > > *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + int res4 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___setslice__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___setslice__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes___setslice__" "', argument " "3"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg3 = static_cast< std::vector< signed char >::difference_type >(val3); + { + std::vector< signed char,std::allocator< signed char > > *ptr = (std::vector< signed char,std::allocator< signed char > > *)0; + res4 = swig::asptr(swig_obj[3], &ptr); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "ibytes___setslice__" "', argument " "4"" of type '" "std::vector< signed char,std::allocator< signed char > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ibytes___setslice__" "', argument " "4"" of type '" "std::vector< signed char,std::allocator< signed char > > const &""'"); + } + arg4 = ptr; + } + try { + std_vector_Sl_int8_t_Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< signed char,std::allocator< signed char > > const &)*arg4); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res4)) delete arg4; + return resultobj; +fail: + if (SWIG_IsNewObj(res4)) delete arg4; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setslice__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes___setslice__", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes___setslice____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = swig::asptr(argv[3], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_ibytes___setslice____SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes___setslice__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::__setslice__(std::vector< signed char >::difference_type,std::vector< signed char >::difference_type)\n" + " std::vector< int8_t >::__setslice__(std::vector< signed char >::difference_type,std::vector< signed char >::difference_type,std::vector< signed char,std::allocator< signed char > > const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes___delslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + std::vector< signed char >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes___delslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___delslice__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___delslice__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes___delslice__" "', argument " "3"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg3 = static_cast< std::vector< signed char >::difference_type >(val3); + try { + std_vector_Sl_int8_t_Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___delitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___delitem__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + try { + std_vector_Sl_int8_t_Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< signed char,std::allocator< signed char > > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___getitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + result = (std::vector< signed char,std::allocator< signed char > > *)std_vector_Sl_int8_t_Sg____getitem____SWIG_0(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + std::vector< signed char,std::allocator< signed char > > *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res3 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___setitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + std::vector< signed char,std::allocator< signed char > > *ptr = (std::vector< signed char,std::allocator< signed char > > *)0; + res3 = swig::asptr(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "ibytes___setitem__" "', argument " "3"" of type '" "std::vector< signed char,std::allocator< signed char > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ibytes___setitem__" "', argument " "3"" of type '" "std::vector< signed char,std::allocator< signed char > > const &""'"); + } + arg3 = ptr; + } + try { + std_vector_Sl_int8_t_Sg____setitem____SWIG_0(arg1,arg2,(std::vector< signed char,std::allocator< signed char > > const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res3)) delete arg3; + return resultobj; +fail: + if (SWIG_IsNewObj(res3)) delete arg3; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___setitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + std_vector_Sl_int8_t_Sg____setitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___delitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + try { + std_vector_Sl_int8_t_Sg____delitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___delitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes___delitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ibytes___delitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes___delitem____SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes___delitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::__delitem__(std::vector< signed char >::difference_type)\n" + " std::vector< int8_t >::__delitem__(SWIGPY_SLICEOBJECT *)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< signed char >::value_type *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___getitem__" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___getitem__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + try { + result = (std::vector< signed char >::value_type *) &std_vector_Sl_int8_t_Sg____getitem____SWIG_1((std::vector< signed char > const *)arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_From_signed_SS_char(static_cast< signed char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___getitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes___getitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ibytes___getitem____SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes___getitem____SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes___getitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::__getitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< int8_t >::__getitem__(std::vector< signed char >::difference_type) const\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::difference_type arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< signed char >::value_type temp3 ; + signed char val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes___setitem__" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes___setitem__" "', argument " "2"" of type '" "std::vector< signed char >::difference_type""'"); + } + arg2 = static_cast< std::vector< signed char >::difference_type >(val2); + ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes___setitem__" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'"); + } + temp3 = static_cast< std::vector< signed char >::value_type >(val3); + arg3 = &temp3; + try { + std_vector_Sl_int8_t_Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes___setitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes___setitem__", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_ibytes___setitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + int res = swig::asptr(argv[2], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_ibytes___setitem____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_signed_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes___setitem____SWIG_2(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes___setitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< signed char,std::allocator< signed char > > const &)\n" + " std::vector< int8_t >::__setitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< int8_t >::__setitem__(std::vector< signed char >::difference_type,std::vector< signed char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes_pop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::value_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_pop" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + try { + result = (std::vector< signed char >::value_type)std_vector_Sl_int8_t_Sg__pop(arg1); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } + resultobj = SWIG_From_signed_SS_char(static_cast< signed char >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_append(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< signed char >::value_type temp2 ; + signed char val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes_append", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_append" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_signed_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_append" "', argument " "2"" of type '" "std::vector< signed char >::value_type""'"); + } + temp2 = static_cast< std::vector< signed char >::value_type >(val2); + arg2 = &temp2; + std_vector_Sl_int8_t_Sg__append(arg1,(signed char const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ibytes__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + std::vector< int8_t > *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + result = (std::vector< int8_t > *)new std::vector< int8_t >(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ibytes__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = 0 ; + int res1 = SWIG_OLDOBJ ; + std::vector< int8_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + std::vector< signed char,std::allocator< signed char > > *ptr = (std::vector< signed char,std::allocator< signed char > > *)0; + res1 = swig::asptr(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > const &""'"); + } + arg1 = ptr; + } + result = (std::vector< int8_t > *)new std::vector< int8_t >((std::vector< int8_t > const &)*arg1); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_NEW | 0 ); + if (SWIG_IsNewObj(res1)) delete arg1; + return resultobj; +fail: + if (SWIG_IsNewObj(res1)) delete arg1; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_empty(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_empty" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (bool)((std::vector< int8_t > const *)arg1)->empty(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_size" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = ((std::vector< int8_t > const *)arg1)->size(); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_swap(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< int8_t > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes_swap", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_swap" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_int8_t_t, 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "ibytes_swap" "', argument " "2"" of type '" "std::vector< int8_t > &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "ibytes_swap" "', argument " "2"" of type '" "std::vector< int8_t > &""'"); + } + arg2 = reinterpret_cast< std::vector< int8_t > * >(argp2); + (arg1)->swap(*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_begin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< signed char >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_begin" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (arg1)->begin(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_end(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< signed char >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_end" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (arg1)->end(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_rbegin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< signed char >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_rbegin" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (arg1)->rbegin(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_rend(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< signed char >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_rend" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (arg1)->rend(); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_clear(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_clear" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + (arg1)->clear(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_get_allocator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::allocator< signed char > > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_get_allocator" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = ((std::vector< int8_t > const *)arg1)->get_allocator(); + resultobj = SWIG_NewPointerObj((new std::vector< signed char >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_signed_char_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ibytes__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< signed char >::size_type arg1 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< int8_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_ibytes" "', argument " "1"" of type '" "std::vector< signed char >::size_type""'"); + } + arg1 = static_cast< std::vector< signed char >::size_type >(val1); + result = (std::vector< int8_t > *)new std::vector< int8_t >(arg1); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_pop_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_pop_back" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + (arg1)->pop_back(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_resize" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_resize" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'"); + } + arg2 = static_cast< std::vector< signed char >::size_type >(val2); + (arg1)->resize(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SwigValueWrapper< std::vector< signed char >::iterator > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + SwigValueWrapper< std::vector< signed char >::iterator > result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_erase" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } + } + result = std_vector_Sl_int8_t_Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SwigValueWrapper< std::vector< signed char >::iterator > arg2 ; + SwigValueWrapper< std::vector< signed char >::iterator > arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + swig::SwigPyIterator *iter3 = 0 ; + int res3 ; + SwigValueWrapper< std::vector< signed char >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_erase" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } + } + res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res3) || !iter3) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "3"" of type '" "std::vector< signed char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter3); + if (iter_t) { + arg3 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_erase" "', argument " "3"" of type '" "std::vector< signed char >::iterator""'"); + } + } + result = std_vector_Sl_int8_t_Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_erase(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_erase", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_ibytes_erase__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_ibytes_erase__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_erase'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::erase(std::vector< signed char >::iterator)\n" + " std::vector< int8_t >::erase(std::vector< signed char >::iterator,std::vector< signed char >::iterator)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_new_ibytes__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< signed char >::size_type arg1 ; + std::vector< signed char >::value_type *arg2 = 0 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< signed char >::value_type temp2 ; + signed char val2 ; + int ecode2 = 0 ; + std::vector< int8_t > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_ibytes" "', argument " "1"" of type '" "std::vector< signed char >::size_type""'"); + } + arg1 = static_cast< std::vector< signed char >::size_type >(val1); + ecode2 = SWIG_AsVal_signed_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_ibytes" "', argument " "2"" of type '" "std::vector< signed char >::value_type""'"); + } + temp2 = static_cast< std::vector< signed char >::value_type >(val2); + arg2 = &temp2; + result = (std::vector< int8_t > *)new std::vector< int8_t >(arg1,(std::vector< signed char >::value_type const &)*arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_ibytes(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_ibytes", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_ibytes__SWIG_0(self, argc, argv); + } + if (argc == 1) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_ibytes__SWIG_2(self, argc, argv); + } + } + if (argc == 1) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_new_ibytes__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_signed_SS_char(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_ibytes__SWIG_3(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_ibytes'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::vector()\n" + " std::vector< int8_t >::vector(std::vector< int8_t > const &)\n" + " std::vector< int8_t >::vector(std::vector< signed char >::size_type)\n" + " std::vector< int8_t >::vector(std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes_push_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< signed char >::value_type temp2 ; + signed char val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes_push_back", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_push_back" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_signed_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_push_back" "', argument " "2"" of type '" "std::vector< signed char >::value_type""'"); + } + temp2 = static_cast< std::vector< signed char >::value_type >(val2); + arg2 = &temp2; + (arg1)->push_back((std::vector< signed char >::value_type const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_front(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_front" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (std::vector< signed char >::value_type *) &((std::vector< int8_t > const *)arg1)->front(); + resultobj = SWIG_From_signed_SS_char(static_cast< signed char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_back" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = (std::vector< signed char >::value_type *) &((std::vector< int8_t > const *)arg1)->back(); + resultobj = SWIG_From_signed_SS_char(static_cast< signed char >(*result)); + (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_assign(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::size_type arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< signed char >::value_type temp3 ; + signed char val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes_assign", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_assign" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_assign" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'"); + } + arg2 = static_cast< std::vector< signed char >::size_type >(val2); + ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_assign" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'"); + } + temp3 = static_cast< std::vector< signed char >::value_type >(val3); + arg3 = &temp3; + (arg1)->assign(arg2,(std::vector< signed char >::value_type const &)*arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::size_type arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< signed char >::value_type temp3 ; + signed char val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_resize" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_resize" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'"); + } + arg2 = static_cast< std::vector< signed char >::size_type >(val2); + ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_resize" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'"); + } + temp3 = static_cast< std::vector< signed char >::value_type >(val3); + arg3 = &temp3; + (arg1)->resize(arg2,(std::vector< signed char >::value_type const &)*arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_resize(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_resize", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes_resize__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_signed_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes_resize__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_resize'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::resize(std::vector< signed char >::size_type)\n" + " std::vector< int8_t >::resize(std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SwigValueWrapper< std::vector< signed char >::iterator > arg2 ; + std::vector< signed char >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + std::vector< signed char >::value_type temp3 ; + signed char val3 ; + int ecode3 = 0 ; + SwigValueWrapper< std::vector< signed char >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'"); + } + temp3 = static_cast< std::vector< signed char >::value_type >(val3); + arg3 = &temp3; + result = std_vector_Sl_int8_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3); + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + SwigValueWrapper< std::vector< signed char >::iterator > arg2 ; + std::vector< signed char >::size_type arg3 ; + std::vector< signed char >::value_type *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + size_t val3 ; + int ecode3 = 0 ; + std::vector< signed char >::value_type temp4 ; + signed char val4 ; + int ecode4 = 0 ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::size_type""'"); + } + arg3 = static_cast< std::vector< signed char >::size_type >(val3); + ecode4 = SWIG_AsVal_signed_SS_char(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "ibytes_insert" "', argument " "4"" of type '" "std::vector< signed char >::value_type""'"); + } + temp4 = static_cast< std::vector< signed char >::value_type >(val4); + arg4 = &temp4; + std_vector_Sl_int8_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(signed char const &)*arg4); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_insert(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_insert", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_signed_SS_char(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes_insert__SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_signed_SS_char(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_ibytes_insert__SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_insert'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::value_type const &)\n" + " std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_ibytes_reserve(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + std::vector< signed char >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ibytes_reserve", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_reserve" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_reserve" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'"); + } + arg2 = static_cast< std::vector< signed char >::size_type >(val2); + (arg1)->reserve(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ibytes_capacity(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< signed char >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_capacity" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + result = ((std::vector< int8_t > const *)arg1)->capacity(); + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ibytes_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ibytes_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + int res1 ; + char *buf1 = 0 ; + int alloc1 = 0 ; + int res2 ; + char *buf2 = 0 ; + int alloc2 = 0 ; + int val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + int res5 = SWIG_OLDOBJ ; + bcos::Error result; + + (void)self; + if ((nobjs < 5) || (nobjs > 5)) SWIG_fail; + res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'"); + } + arg1 = reinterpret_cast< char * >(buf1); + res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'"); + } + arg2 = reinterpret_cast< char * >(buf2); + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'"); + } + arg4 = static_cast< int32_t >(val4); + { + std::string *ptr = (std::string *)0; + res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr); + if (!SWIG_IsOK(res5)) { + SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + arg5 = ptr; + } + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5); + { + std::shared_ptr< bcos::Error > *smartresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return resultobj; +fail: + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + bcos::Error *arg6 = 0 ; + int res1 ; + char *buf1 = 0 ; + int alloc1 = 0 ; + int res2 ; + char *buf2 = 0 ; + int alloc2 = 0 ; + int val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + int res5 = SWIG_OLDOBJ ; + void *argp6 = 0 ; + int res6 = 0 ; + std::shared_ptr< bcos::Error const > tempshared6 ; + bcos::Error result; + + (void)self; + if ((nobjs < 6) || (nobjs > 6)) SWIG_fail; + res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'"); + } + arg1 = reinterpret_cast< char * >(buf1); + res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'"); + } + arg2 = reinterpret_cast< char * >(buf2); + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'"); + } + arg4 = static_cast< int32_t >(val4); + { + std::string *ptr = (std::string *)0; + res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr); + if (!SWIG_IsOK(res5)) { + SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + arg5 = ptr; + } + { + int newmem = 0; + res6 = SWIG_ConvertPtrAndOwn(swig_obj[5], &argp6, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 , &newmem); + if (!SWIG_IsOK(res6)) { + SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'"); + } + if (!argp6) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared6 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6); + delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6); + arg6 = const_cast< bcos::Error * >(tempshared6.get()); + } else { + arg6 = const_cast< bcos::Error * >(reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6)->get()); + } + } + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6); + { + std::shared_ptr< bcos::Error > *smartresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return resultobj; +fail: + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + char *arg1 = (char *) 0 ; + char *arg2 = (char *) 0 ; + int arg3 ; + int32_t arg4 ; + std::string *arg5 = 0 ; + std::exception *arg6 = 0 ; + int res1 ; + char *buf1 = 0 ; + int alloc1 = 0 ; + int res2 ; + char *buf2 = 0 ; + int alloc2 = 0 ; + int val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + int res5 = SWIG_OLDOBJ ; + void *argp6 = 0 ; + int res6 = 0 ; + bcos::Error result; + + (void)self; + if ((nobjs < 6) || (nobjs > 6)) SWIG_fail; + res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'"); + } + arg1 = reinterpret_cast< char * >(buf1); + res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'"); + } + arg2 = reinterpret_cast< char * >(buf2); + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'"); + } + arg4 = static_cast< int32_t >(val4); + { + std::string *ptr = (std::string *)0; + res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr); + if (!SWIG_IsOK(res5)) { + SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); + } + arg5 = ptr; + } + res6 = SWIG_ConvertPtr(swig_obj[5], &argp6, SWIGTYPE_p_std__exception, 0 | 0); + if (!SWIG_IsOK(res6)) { + SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); + } + if (!argp6) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); + } + arg6 = reinterpret_cast< std::exception * >(argp6); + result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6); + { + std::shared_ptr< bcos::Error > *smartresult = new std::shared_ptr< bcos::Error >(new bcos::Error(result)); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return resultobj; +fail: + if (alloc1 == SWIG_NEWOBJ) delete[] buf1; + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res5)) delete arg5; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_buildError(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[7] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "Error_buildError", 0, 6, argv))) SWIG_fail; + --argc; + if (argc == 5) { + int _v = 0; + int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_int(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_Error_buildError__SWIG_0(self, argc, argv); + } + } + } + } + } + } + if (argc == 6) { + int _v = 0; + int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_int(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_Error_buildError__SWIG_1(self, argc, argv); + } + } + } + } + } + } + } + if (argc == 6) { + int _v = 0; + int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_int(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__exception, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_Error_buildError__SWIG_2(self, argc, argv); + } + } + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'Error_buildError'.\n" + " Possible C/C++ prototypes are:\n" + " bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &)\n" + " bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,bcos::Error const &)\n" + " bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,std::exception const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_new_Error__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + bcos::Error *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + result = (bcos::Error *)new bcos::Error(); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_Error__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + int64_t arg1 ; + std::string arg2 ; + long long val1 ; + int ecode1 = 0 ; + bcos::Error *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + ecode1 = SWIG_AsVal_long_SS_long(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_Error" "', argument " "1"" of type '" "int64_t""'"); + } + arg1 = static_cast< int64_t >(val1); + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "new_Error" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + result = (bcos::Error *)new bcos::Error(arg1,arg2); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_Error(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_Error", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_Error__SWIG_0(self, argc, argv); + } + if (argc == 2) { + int _v = 0; + { + int res = SWIG_AsVal_long_SS_long(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_new_Error__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_Error'.\n" + " Possible C/C++ prototypes are:\n" + " bcos::Error::Error()\n" + " bcos::Error::Error(int64_t,std::string)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_delete_Error(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::Error *arg1 = (bcos::Error *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error > tempshared1 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Error" "', argument " "1"" of type '" "bcos::Error *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_errorCode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::Error *arg1 = (bcos::Error *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error const > tempshared1 ; + std::shared_ptr< bcos::Error const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + int64_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorCode" "', argument " "1"" of type '" "bcos::Error const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (int64_t)((bcos::Error const *)arg1)->errorCode(); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_errorMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::Error *arg1 = (bcos::Error *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error const > tempshared1 ; + std::shared_ptr< bcos::Error const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorMessage" "', argument " "1"" of type '" "bcos::Error const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((bcos::Error const *)arg1)->errorMessage(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_setErrorCode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::Error *arg1 = (bcos::Error *) 0 ; + int64_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error > tempshared1 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + long long val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Error_setErrorCode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorCode" "', argument " "1"" of type '" "bcos::Error *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_long_SS_long(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Error_setErrorCode" "', argument " "2"" of type '" "int64_t""'"); + } + arg2 = static_cast< int64_t >(val2); + (arg1)->setErrorCode(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Error_setErrorMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::Error *arg1 = (bcos::Error *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< bcos::Error > tempshared1 ; + std::shared_ptr< bcos::Error > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Error_setErrorMessage", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorMessage" "', argument " "1"" of type '" "bcos::Error *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + delete reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1); + arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setErrorMessage((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *Error_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *Error_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_InputBuffer_data_set(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *arg1 = (InputBuffer *) 0 ; + unsigned char *arg2 = (unsigned char *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "InputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_set" "', argument " "1"" of type '" "InputBuffer *""'"); + } + arg1 = reinterpret_cast< InputBuffer * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "InputBuffer_data_set" "', argument " "2"" of type '" "unsigned char const *""'"); + } + arg2 = reinterpret_cast< unsigned char * >(argp2); + if (arg1) (arg1)->data = (unsigned char const *)arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_InputBuffer_data_get(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *arg1 = (InputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + unsigned char *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_get" "', argument " "1"" of type '" "InputBuffer *""'"); + } + arg1 = reinterpret_cast< InputBuffer * >(argp1); + result = (unsigned char *) ((arg1)->data); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_InputBuffer_len_set(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *arg1 = (InputBuffer *) 0 ; + uint64_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + unsigned long long val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "InputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_set" "', argument " "1"" of type '" "InputBuffer *""'"); + } + arg1 = reinterpret_cast< InputBuffer * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "InputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'"); + } + arg2 = static_cast< uint64_t >(val2); + if (arg1) (arg1)->len = arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_InputBuffer_len_get(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *arg1 = (InputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint64_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_get" "', argument " "1"" of type '" "InputBuffer *""'"); + } + arg1 = reinterpret_cast< InputBuffer * >(argp1); + result = (uint64_t) ((arg1)->len); + resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_InputBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_InputBuffer", 0, 0, 0)) SWIG_fail; + result = (InputBuffer *)new InputBuffer(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_InputBuffer, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_InputBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + InputBuffer *arg1 = (InputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_InputBuffer" "', argument " "1"" of type '" "InputBuffer *""'"); + } + arg1 = reinterpret_cast< InputBuffer * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *InputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_InputBuffer, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *InputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_OutputBuffer_data_set(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + unsigned char *arg2 = (unsigned char *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_set" "', argument " "1"" of type '" "OutputBuffer *""'"); + } + arg1 = reinterpret_cast< OutputBuffer * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "OutputBuffer_data_set" "', argument " "2"" of type '" "unsigned char *""'"); + } + arg2 = reinterpret_cast< unsigned char * >(argp2); + if (arg1) (arg1)->data = arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_OutputBuffer_data_get(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + unsigned char *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_get" "', argument " "1"" of type '" "OutputBuffer *""'"); + } + arg1 = reinterpret_cast< OutputBuffer * >(argp1); + result = (unsigned char *) ((arg1)->data); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_OutputBuffer_len_set(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + uint64_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + unsigned long long val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_set" "', argument " "1"" of type '" "OutputBuffer *""'"); + } + arg1 = reinterpret_cast< OutputBuffer * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "OutputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'"); + } + arg2 = static_cast< uint64_t >(val2); + if (arg1) (arg1)->len = arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_OutputBuffer_len_get(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint64_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_get" "', argument " "1"" of type '" "OutputBuffer *""'"); + } + arg1 = reinterpret_cast< OutputBuffer * >(argp1); + result = (uint64_t) ((arg1)->len); + resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_OutputBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_OutputBuffer", 0, 0, 0)) SWIG_fail; + result = (OutputBuffer *)new OutputBuffer(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_OutputBuffer, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + OutputBuffer *arg1 = (OutputBuffer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_OutputBuffer" "', argument " "1"" of type '" "OutputBuffer *""'"); + } + arg1 = reinterpret_cast< OutputBuffer * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *OutputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_OutputBuffer, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *OutputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_delete_FrontConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_threadPoolSize(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_threadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize(); + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setThreadPoolSize(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setThreadPoolSize", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + (arg1)->setThreadPoolSize(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_nodeID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_nodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setNodeID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setNodeID", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setNodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setNodeID((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_selfEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_selfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_mutableSelfEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setSelfEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + ppc::protocol::EndPoint *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setSelfEndPoint", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); + } + arg2 = reinterpret_cast< ppc::protocol::EndPoint * >(argp2); + (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setGatewayGrpcTarget(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGatewayGrpcTarget", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setGatewayGrpcTarget((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_gatewayGrpcTarget(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_gatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setGrpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + ppc::protocol::GrpcConfig::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGrpcConfig", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGrpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGrpcConfig" "', argument " "2"" of type '" "ppc::protocol::GrpcConfig::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2); + } + (arg1)->setGrpcConfig(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_grpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::GrpcConfig::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_grpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig(); + { + std::shared_ptr< ppc::protocol::GrpcConfig > *smartresult = *result ? new std::shared_ptr< ppc::protocol::GrpcConfig >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_generateNodeInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::INodeInfo::Ptr result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_generateNodeInfo" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo(); + resultobj = SWIG_NewPointerObj((new ppc::protocol::INodeInfo::Ptr(result)), SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_getComponents(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< std::string,std::allocator< std::string > > *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_getComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::vector< std::string,std::allocator< std::string > > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setComponents", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); + } + arg2 = reinterpret_cast< std::vector< std::string,std::allocator< std::string > > * >(argp2); + (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_addComponent(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "FrontConfig_addComponent", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_addComponent" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->addComponent((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfig_mutableComponents(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::FrontConfig > tempshared1 ; + std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< std::string,std::allocator< std::string > > *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1); + arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *FrontConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_FrontConfigBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfigBuilder" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ppc::front::FrontConfig::Ptr result; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); + } + arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1); + result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(); + { + std::shared_ptr< ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr< ppc::front::FrontConfig >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ; + int arg2 ; + std::string arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + ppc::front::FrontConfig::Ptr result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); + } + arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfigBuilder_build" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "FrontConfigBuilder_build" "', argument " "3"" of type '" "std::string""'"); + } + arg3 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3); + { + std::shared_ptr< ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr< ppc::front::FrontConfig >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "FrontConfigBuilder_build", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_FrontConfigBuilder_build__SWIG_0(self, argc, argv); + } + } + if (argc == 3) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_int(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_FrontConfigBuilder_build__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'FrontConfigBuilder_build'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::front::FrontConfigBuilder::build() const\n" + " ppc::front::FrontConfigBuilder::build(int,std::string) const\n"); + return 0; +} + + +SWIGINTERN PyObject *FrontConfigBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_printFrontDesc(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig::Ptr *arg1 = 0 ; + void *argp1 ; + int res1 = 0 ; + ppc::front::FrontConfig::Ptr tempshared1 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printFrontDesc" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1); + delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1) : &tempshared1; + } + } + result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::string *arg1 = 0 ; + uint16_t arg2 ; + int res1 = SWIG_OLDOBJ ; + unsigned short val2 ; + int ecode2 = 0 ; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + std::string *ptr = (std::string *)0; + res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); + } + arg1 = ptr; + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_EndPoint" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW | 0 ); + if (SWIG_IsNewObj(res1)) delete arg1; + return resultobj; +fail: + if (SWIG_IsNewObj(res1)) delete arg1; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_EndPoint(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_EndPoint", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_EndPoint__SWIG_0(self, argc, argv); + } + if (argc == 2) { + int _v = 0; + int res = SWIG_AsPtr_std_string(argv[0], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_short(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_EndPoint__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_EndPoint'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::EndPoint::EndPoint()\n" + " ppc::protocol::EndPoint::EndPoint(std::string const &,uint16_t)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_delete_EndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_EndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_host(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_host" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_port(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_port" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_setHost(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "EndPoint_setHost", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setHost" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "EndPoint_setHost" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + (arg1)->setHost(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_setPort(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "EndPoint_setPort", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setPort" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "EndPoint_setPort" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setPort(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_setListenIp(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "EndPoint_setListenIp", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setListenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setListenIp((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_entryPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_entryPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint(); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_listenEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint(); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_EndPoint_listenIp(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *EndPoint_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *EndPoint_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::EndPoint arg1 ; + bool arg2 ; + void *argp1 ; + int res1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + ppc::protocol::GrpcServerConfig *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); + } + if (!argp1) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); + } else { + ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp1); + arg1 = *temp; + if (SWIG_IsNewObj(res1)) delete temp; + } + } + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_GrpcServerConfig(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_GrpcServerConfig", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_GrpcServerConfig__SWIG_0(self, argc, argv); + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_bool(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_GrpcServerConfig__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_GrpcServerConfig'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::GrpcServerConfig::GrpcServerConfig()\n" + " ppc::protocol::GrpcServerConfig::GrpcServerConfig(ppc::protocol::EndPoint,bool)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint(); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + ppc::protocol::EndPoint arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEndPoint", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); + } else { + ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + (arg1)->setEndPoint(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + bool arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + (arg1)->setEnableHealthCheck(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::EndPoint *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); + } + arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *GrpcServerConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *GrpcServerConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail; + result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig(); + { + std::shared_ptr< ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr< ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_loadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setLoadBalancePolicy", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setLoadBalancePolicy((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + bool arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + (arg1)->setEnableHealthCheck(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + bool arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableDnslookup", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + (arg1)->setEnableDnslookup(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ; + std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1); + arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeader" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_encode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes > tempshared2 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_encode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2)->get()); + } + } + ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_decode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytesConstRef arg2 ; + uint64_t arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + unsigned long long val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + int64_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_decode", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "MessageOptionalHeader_decode" "', argument " "3"" of type '" "uint64_t""'"); + } + arg3 = static_cast< uint64_t >(val3); + result = (int64_t)(arg1)->decode(arg2,arg3); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_componentType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_componentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType(); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setComponentType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setComponentType", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + (arg1)->setComponentType(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bcos::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNode(); + { + std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + OutputBuffer result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer(); + { + resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes const > tempshared2 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get()); + } + } + (arg1)->setSrcNode((bcos::bytes const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int res; Py_ssize_t size = 0; const void *buf = 0; +#ifndef Py_LIMITED_API + Py_buffer view; + res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO); +#else +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size); +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + if (res < 0) { + PyErr_Clear(); + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "(char* data, uint64_t length)""'"); + } +#ifndef Py_LIMITED_API + size = view.len; + buf = view.buf; + PyBuffer_Release(&view); +#endif + arg2 = (char *) buf; + arg3 = (uint64_t) (size / sizeof(char)); + } + (arg1)->setSrcNode(arg2,arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcNode", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageOptionalHeader_setSrcNode__SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + if (argc <= 2) { + return _wrap_MessageOptionalHeader_setSrcNode__SWIG_1(self, argc, argv); + } + { + int res = SWIG_AsVal_unsigned_SS_long_SS_long(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_MessageOptionalHeader_setSrcNode__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeader_setSrcNode'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessageOptionalHeader::setSrcNode(bcos::bytes const &)\n" + " ppc::protocol::MessageOptionalHeader::setSrcNode(char *,uint64_t)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bcos::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNode(); + { + std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + OutputBuffer result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer(); + { + resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes const > tempshared2 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstNode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setDstNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setDstNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get()); + } + } + (arg1)->setDstNode((bcos::bytes const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNodeBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstNodeBuffer", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int res; Py_ssize_t size = 0; const void *buf = 0; +#ifndef Py_LIMITED_API + Py_buffer view; + res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO); +#else +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size); +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + if (res < 0) { + PyErr_Clear(); + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'"); + } +#ifndef Py_LIMITED_API + size = view.len; + buf = view.buf; + PyBuffer_Release(&view); +#endif + arg2 = (char *) buf; + arg3 = (uint64_t) (size / sizeof(char)); + } + (arg1)->setDstNodeBuffer(arg2,arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstInst(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstInst(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstInst", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setDstInst((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_topic(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_topic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::unique_ptr< std::string > rvrdeleter2 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__string, SWIG_POINTER_RELEASE | 0 ); + if (!SWIG_IsOK(res2)) { + if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::string &&""'"); + } else { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); + } + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); + } + arg2 = reinterpret_cast< std::string * >(argp2); + rvrdeleter2.reset(arg2); + (arg1)->setTopic((std::string &&)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setTopic((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setTopic", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_std__string, SWIG_POINTER_NO_NULL); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageOptionalHeader_setTopic__SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageOptionalHeader_setTopic__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeader_setTopic'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessageOptionalHeader::setTopic(std::string &&)\n" + " ppc::protocol::MessageOptionalHeader::setTopic(std::string const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst(); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcInst(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcInst", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setSrcInst((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *MessageOptionalHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_MessageHeader(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeader" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_encode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes > tempshared2 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_encode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2)->get()); + } + } + ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_decode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + int64_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_decode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (int64_t)(arg1)->decode(arg2); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_version(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint8_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_version" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version(); + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setVersion(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setVersion", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setVersion" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setVersion(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_traceID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_traceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setTraceID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTraceID", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageHeader_setTraceID" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + (arg1)->setTraceID(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_srcGwNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setSrcGwNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setSrcGwNode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setSrcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setSrcGwNode((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_dstGwNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setDstGwNode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setDstGwNode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setDstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setDstGwNode((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_packetType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_packetType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setPacketType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setPacketType", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setPacketType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setPacketType" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setPacketType(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_ttl(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + int16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ttl" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl(); + resultobj = SWIG_From_short(static_cast< short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setTTL(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTTL", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTTL" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setTTL" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setTTL(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_ext(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ext" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setExt(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setExt", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setExt" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setExt" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setExt(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_optionalField(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_optionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField(); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setOptionalField(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setOptionalField", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setOptionalField" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2); + } + (arg1)->setOptionalField(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_length(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_length" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_isRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->setRespPacket(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_srcP2PNodeIDView(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string_view result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView(); + resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_dstP2PNodeIDView(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string_view result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView(); + resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_routeType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_routeType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_setRouteType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + ppc::protocol::RouteType arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setRouteType", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRouteType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); + } else { + ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + (arg1)->setRouteType(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_hasOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *MessageHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Message" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_header(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessageHeader::Ptr result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_header" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::Message const *)arg1)->header(); + { + std::shared_ptr< ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setHeader(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + ppc::protocol::MessageHeader::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setHeader", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setHeader" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setHeader" "', argument " "2"" of type '" "ppc::protocol::MessageHeader::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2); + } + (arg1)->setHeader(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_version(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_version" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::Message const *)arg1)->version(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setVersion(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setVersion", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setVersion" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setVersion" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setVersion(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_packetType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_packetType" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setPacketType(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setPacketType", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPacketType" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setPacketType" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setPacketType(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_seq(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_seq" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setSeq(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::string arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setSeq", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setSeq" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "Message_setSeq" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + (arg1)->setSeq(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_ext(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_ext" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setExt(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setExt", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setExt" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setExt" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setExt(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_isRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->setRespPacket(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint32_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_length" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint32_t)((ppc::protocol::Message const *)arg1)->length(); + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::shared_ptr< bcos::bytes > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payload" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::Message const *)arg1)->payload(); + { + std::shared_ptr< bcos::bytes > *smartresult = result ? new std::shared_ptr< bcos::bytes >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + OutputBuffer result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payloadBuffer" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::Message const *)arg1)->payloadBuffer(); + { + resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + std::shared_ptr< bcos::bytes > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setPayload", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setPayload" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes >""'"); + } + if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + } + (arg1)->setPayload(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + ppc::protocol::MessagePayload::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_setFrontMessage", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setFrontMessage" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setFrontMessage" "', argument " "2"" of type '" "ppc::protocol::MessagePayload::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2); + } + (arg1)->setFrontMessage(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message const > tempshared1 ; + std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessagePayload::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_frontMessage" "', argument " "1"" of type '" "ppc::protocol::Message const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage(); + { + std::shared_ptr< ppc::protocol::MessagePayload > *smartresult = *result ? new std::shared_ptr< ppc::protocol::MessagePayload >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_encode__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes > tempshared2 ; + bool result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_encode" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2)->get()); + } + } + result = (bool)(arg1)->encode(*arg2); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_encode__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::boostssl::EncodedMsg *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + bool result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_encode" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__boostssl__EncodedMsg, 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::boostssl::EncodedMsg &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::boostssl::EncodedMsg &""'"); + } + arg2 = reinterpret_cast< bcos::boostssl::EncodedMsg * >(argp2); + result = (bool)(arg1)->encode(*arg2); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "Message_encode", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_Message_encode__SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_bcos__boostssl__EncodedMsg, SWIG_POINTER_NO_NULL); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_Message_encode__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'Message_encode'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::Message::encode(bcos::bytes &)\n" + " ppc::protocol::Message::encode(bcos::boostssl::EncodedMsg &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::Message > tempshared1 ; + std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + int64_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "Message_decode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_decode" "', argument " "1"" of type '" "ppc::protocol::Message *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1); + arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (int64_t)(arg1)->decode(arg2); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *Message_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_MessageHeaderBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::MessageHeader::Ptr result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (arg1)->build(arg2); + { + std::shared_ptr< ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageHeader::Ptr result; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (arg1)->build(); + { + std::shared_ptr< ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2); + arg2 = &tempshared2; + } else { + arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2; + } + } + result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessageHeaderBuilder_build", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageHeaderBuilder_build__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageHeaderBuilder_build__SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageHeaderBuilder_build__SWIG_2(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageHeaderBuilder_build'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessageHeaderBuilder::build(bcos::bytesConstRef)\n" + " ppc::protocol::MessageHeaderBuilder::build()\n" + " ppc::protocol::MessageHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *MessageHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_MessageBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + ppc::protocol::Message::Ptr result; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (arg1)->build(); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = result ? new std::shared_ptr< ppc::protocol::Message >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::Message::Ptr result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (arg1)->build(arg2); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = result ? new std::shared_ptr< ppc::protocol::Message >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ; + ppc::protocol::RouteType arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + bcos::bytes *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + void *argp3 ; + int res3 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ; + void *argp4 = 0 ; + int res4 = 0 ; + std::unique_ptr< bcos::bytes > rvrdeleter4 ; + ppc::protocol::Message::Ptr result; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); + } else { + ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "MessageBuilder_build" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + arg3 = &tempshared3; + } else { + arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3; + } + } + res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE | 0 ); + if (!SWIG_IsOK(res4)) { + if (res4 == SWIG_ERROR_RELEASE_NOT_OWNED) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', cannot release ownership as memory is not owned for argument " "4"" of type '" "bcos::bytes &&""'"); + } else { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); + } + } + if (!argp4) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); + } + arg4 = reinterpret_cast< bcos::bytes * >(argp4); + rvrdeleter4.reset(arg4); + result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = result ? new std::shared_ptr< ppc::protocol::Message >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageBuilder_build(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessageBuilder_build", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageBuilder_build__SWIG_0(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageBuilder_build__SWIG_1(self, argc, argv); + } + } + } + if (argc == 4) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_ppc__protocol__RouteType, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[3], &vptr, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NO_NULL); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageBuilder_build__SWIG_2(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageBuilder_build'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessageBuilder::build()\n" + " ppc::protocol::MessageBuilder::build(bcos::bytesConstRef)\n" + " ppc::protocol::MessageBuilder::build(ppc::protocol::RouteType,ppc::protocol::MessageOptionalHeader::Ptr const &,bcos::bytes &&)\n"); + return 0; +} + + +SWIGINTERN PyObject *MessageBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeaderBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2); + arg2 = &tempshared2; + } else { + arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2; + } + } + result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr result; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (arg1)->build(); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeaderBuilder_build", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageOptionalHeaderBuilder_build__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessageOptionalHeaderBuilder_build__SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeaderBuilder_build'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessageOptionalHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n" + " ppc::protocol::MessageOptionalHeaderBuilder::build()\n"); + return 0; +} + + +SWIGINTERN PyObject *MessageOptionalHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_printOptionalField(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessageOptionalHeader::Ptr arg1 ; + void *argp1 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'"); + } + if (argp1) arg1 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1); + } + result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1)); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_printMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::Message::Ptr *arg1 = 0 ; + void *argp1 ; + int res1 = 0 ; + ppc::protocol::Message::Ptr tempshared1 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printMessage" "', argument " "1"" of type '" "ppc::protocol::Message::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp1) tempshared1 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1); + delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1); + arg1 = &tempshared1; + } else { + arg1 = (argp1) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1) : &tempshared1; + } + } + result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_printWsMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + bcos::boostssl::MessageFace::Ptr *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr, 0 | 0); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printWsMessage" "', argument " "1"" of type '" "bcos::boostssl::MessageFace::Ptr const &""'"); + } + if (!argp1) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "printWsMessage" "', argument " "1"" of type '" "bcos::boostssl::MessageFace::Ptr const &""'"); + } + arg1 = reinterpret_cast< bcos::boostssl::MessageFace::Ptr * >(argp1); + result = ppc::protocol::printWsMessage((bcos::boostssl::MessageFace::Ptr const &)*arg1); + resultobj = SWIG_From_std_string(static_cast< std::string >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_MessagePayload(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayload" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_encode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes > tempshared2 ; + PyObject *swig_obj[2] ; + int64_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_encode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_encode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2)->get()); + } + } + result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_decode(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + int64_t result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_decode", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_decode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (int64_t)(arg1)->decode(arg2); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_version(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint8_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_version" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version(); + resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint8_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + unsigned char val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setVersion", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setVersion" "', argument " "2"" of type '" "uint8_t""'"); + } + arg2 = static_cast< uint8_t >(val2); + (arg1)->setVersion(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_data(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bcos::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_data" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bcos::bytes *) &((ppc::protocol::MessagePayload const *)arg1)->data(); + { + std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + OutputBuffer result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_dataBuffer" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer(); + { + resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setData__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::unique_ptr< bcos::bytes > rvrdeleter2 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setData" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE | 0 ); + if (!SWIG_IsOK(res2)) { + if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "bcos::bytes &&""'"); + } else { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes &&""'"); + } + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes &&""'"); + } + arg2 = reinterpret_cast< bcos::bytes * >(argp2); + rvrdeleter2.reset(arg2); + (arg1)->setData((bcos::bytes &&)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setData__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + bcos::bytes *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::shared_ptr< bcos::bytes const > tempshared2 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setData" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2); + arg2 = const_cast< bcos::bytes * >(tempshared2.get()); + } else { + arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get()); + } + } + (arg1)->setData((bcos::bytes const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setData(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessagePayload_setData", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NO_NULL); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessagePayload_setData__SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessagePayload_setData__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessagePayload_setData'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessagePayload::setData(bcos::bytes &&)\n" + " ppc::protocol::MessagePayload::setData(bcos::bytes const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_seq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setSeq(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setSeq", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setSeq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setSeq" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setSeq(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_length(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + int64_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_length" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length(); + resultobj = SWIG_From_long_SS_long(static_cast< long long >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_traceID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_traceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID(); + resultobj = SWIG_From_std_string(static_cast< std::string >(*result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setTraceID(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setTraceID", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->setTraceID((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + uint16_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_ext" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext(); + resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setExt(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + uint16_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setExt", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setExt" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setExt" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + (arg1)->setExt(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_setRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->setRespPacket(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayload_isRespPacket(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (bool)(arg1)->isRespPacket(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *MessagePayload_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_MessagePayloadBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayloadBuilder" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + ppc::protocol::MessagePayload::Ptr result; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (arg1)->build(); + { + std::shared_ptr< ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessagePayload >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ; + bcos::bytesConstRef arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ; + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::MessagePayload::Ptr result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1); + arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); + } else { + bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2); + arg2 = *temp; + if (SWIG_IsNewObj(res2)) delete temp; + } + } + result = (arg1)->build(arg2); + { + std::shared_ptr< ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr< ppc::protocol::MessagePayload >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "MessagePayloadBuilder_build", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 1) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessagePayloadBuilder_build__SWIG_0(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_MessagePayloadBuilder_build__SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessagePayloadBuilder_build'.\n" + " Possible C/C++ prototypes are:\n" + " ppc::protocol::MessagePayloadBuilder::build()\n" + " ppc::protocol::MessagePayloadBuilder::build(bcos::bytesConstRef)\n"); + return 0; +} + + +SWIGINTERN PyObject *MessagePayloadBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_IFrontClient(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFrontClient > tempshared1 ; + std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontClient" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFrontClient_onReceiveMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ; + ppc::protocol::Message::Ptr *arg2 = 0 ; + SwigValueWrapper< std::function< void (std::shared_ptr< bcos::Error >) > > arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFrontClient > tempshared1 ; + std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + ppc::protocol::Message::Ptr tempshared2 ; + void *argp3 ; + int res3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFrontClient_onReceiveMessage", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontClient_onReceiveMessage" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1); + arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFrontClient_onReceiveMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2); + delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2); + arg2 = &tempshared2; + } else { + arg2 = (argp2) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2) : &tempshared2; + } + } + { + res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t, 0 | 0); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'"); + } + if (!argp3) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'"); + } else { + ppc::protocol::ReceiveMsgFunc * temp = reinterpret_cast< ppc::protocol::ReceiveMsgFunc * >(argp3); + arg3 = *temp; + if (SWIG_IsNewObj(res3)) delete temp; + } + } + (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *IFrontClient_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_new_ErrorCallback(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + PyObject *arg1 = (PyObject *) 0 ; + PyObject *swig_obj[1] ; + ppc::front::ErrorCallback *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + arg1 = swig_obj[0]; + if ( arg1 != Py_None ) { + /* subclassed */ + result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(arg1); + } else { + SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); + SWIG_fail; + } + + { + std::shared_ptr< ppc::front::ErrorCallback > *smartresult = result ? new std::shared_ptr< ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ErrorCallback(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ; + std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ErrorCallback_onError(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ; + bcos::Error::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ; + std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + Swig::Director *director = 0; + bool upcall = false; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ErrorCallback_onError", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ErrorCallback_onError" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "ErrorCallback_onError" "', argument " "2"" of type '" "bcos::Error::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2); + } + director = SWIG_DIRECTOR_CAST(arg1); + upcall = (director && (director->swig_get_self()==swig_obj[0])); + try { + if (upcall) { + Swig::DirectorPureVirtualException::raise("ppc::front::ErrorCallback::onError"); + } else { + (arg1)->onError(arg2); + } + } catch (Swig::DirectorException&) { + SWIG_fail; + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_disown_ErrorCallback(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ; + std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1); + arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + Swig::Director *director = SWIG_DIRECTOR_CAST(arg1); + if (director) director->swig_disown(); + } + + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ErrorCallback_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ErrorCallback_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_MessageDispatcherHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + PyObject *arg1 = (PyObject *) 0 ; + PyObject *swig_obj[1] ; + ppc::front::MessageDispatcherHandler *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + arg1 = swig_obj[0]; + if ( arg1 != Py_None ) { + /* subclassed */ + result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(arg1); + } else { + SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); + SWIG_fail; + } + + { + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartresult = result ? new std::shared_ptr< ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_MessageDispatcherHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_MessageDispatcherHandler_onMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ; + ppc::protocol::Message::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + Swig::Director *director = 0; + bool upcall = false; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "MessageDispatcherHandler_onMessage", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2); + } + director = SWIG_DIRECTOR_CAST(arg1); + upcall = (director && (director->swig_get_self()==swig_obj[0])); + try { + if (upcall) { + Swig::DirectorPureVirtualException::raise("ppc::front::MessageDispatcherHandler::onMessage"); + } else { + (arg1)->onMessage(arg2); + } + } catch (Swig::DirectorException&) { + SWIG_fail; + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_disown_MessageDispatcherHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ; + std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1); + arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + Swig::Director *director = SWIG_DIRECTOR_CAST(arg1); + if (director) director->swig_disown(); + } + + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *MessageDispatcherHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *MessageDispatcherHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SendResponseHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + SwigValueWrapper< std::function< void (std::shared_ptr< bcos::bytes > &&) > > arg1 ; + void *argp1 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + ppc::front::SendResponseHandler *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t, 0 | 0); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'"); + } + if (!argp1) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'"); + } else { + ppc::protocol::SendResponseFunction * temp = reinterpret_cast< ppc::protocol::SendResponseFunction * >(argp1); + arg1 = *temp; + if (SWIG_IsNewObj(res1)) delete temp; + } + } + result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SendResponseHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SendResponseHandler" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); + } + arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SendResponseHandler_sendResponse(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ; + std::shared_ptr< bcos::bytes > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + std::unique_ptr< std::shared_ptr< bcos::bytes > > rvrdeleter2 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SendResponseHandler_sendResponse", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SendResponseHandler_sendResponse" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); + } + arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_RELEASE | 0 ); + if (!SWIG_IsOK(res2)) { + if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); + } else { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); + } + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); + } + arg2 = reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2); + rvrdeleter2.reset(arg2); + (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SendResponseHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SendResponseHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_IMessageHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + PyObject *arg1 = (PyObject *) 0 ; + PyObject *swig_obj[1] ; + ppc::front::IMessageHandler *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + arg1 = swig_obj[0]; + if ( arg1 != Py_None ) { + /* subclassed */ + result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(arg1); + } else { + SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); + SWIG_fail; + } + + { + std::shared_ptr< ppc::front::IMessageHandler > *smartresult = result ? new std::shared_ptr< ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_IMessageHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ; + std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IMessageHandler_onMessage(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ; + bcos::Error::Ptr arg2 ; + ppc::protocol::Message::Ptr arg3 ; + SwigValueWrapper< ppc::front::SendResponseHandler > arg4 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ; + std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + void *argp3 ; + int res3 = 0 ; + void *argp4 ; + int res4 = 0 ; + PyObject *swig_obj[4] ; + Swig::Director *director = 0; + bool upcall = false; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IMessageHandler_onMessage", 4, 4, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IMessageHandler_onMessage" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IMessageHandler_onMessage" "', argument " "2"" of type '" "bcos::Error::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2); + } + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IMessageHandler_onMessage" "', argument " "3"" of type '" "ppc::protocol::Message::Ptr""'"); + } + if (argp3) arg3 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3); + } + { + res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_ppc__front__SendResponseHandler, 0 | 0); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'"); + } + if (!argp4) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'"); + } else { + ppc::front::SendResponseHandler * temp = reinterpret_cast< ppc::front::SendResponseHandler * >(argp4); + arg4 = *temp; + if (SWIG_IsNewObj(res4)) delete temp; + } + } + director = SWIG_DIRECTOR_CAST(arg1); + upcall = (director && (director->swig_get_self()==swig_obj[0])); + try { + if (upcall) { + Swig::DirectorPureVirtualException::raise("ppc::front::IMessageHandler::onMessage"); + } else { + (arg1)->onMessage(arg2,arg3,arg4); + } + } catch (Swig::DirectorException&) { + SWIG_fail; + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_disown_IMessageHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ; + std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1); + arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + Swig::Director *director = SWIG_DIRECTOR_CAST(arg1); + if (director) director->swig_disown(); + } + + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *IMessageHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *IMessageHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_GetPeersInfoHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + PyObject *arg1 = (PyObject *) 0 ; + PyObject *swig_obj[1] ; + ppc::front::GetPeersInfoHandler *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + arg1 = swig_obj[0]; + if ( arg1 != Py_None ) { + /* subclassed */ + result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(arg1); + } else { + SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); + SWIG_fail; + } + + { + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartresult = result ? new std::shared_ptr< ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_GetPeersInfoHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_GetPeersInfoHandler_onPeersInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ; + bcos::Error::Ptr arg2 ; + std::string *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + int res3 = SWIG_OLDOBJ ; + PyObject *swig_obj[3] ; + Swig::Director *director = 0; + bool upcall = false; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "GetPeersInfoHandler_onPeersInfo", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "2"" of type '" "bcos::Error::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2); + } + { + std::string *ptr = (std::string *)0; + res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); + } + arg3 = ptr; + } + director = SWIG_DIRECTOR_CAST(arg1); + upcall = (director && (director->swig_get_self()==swig_obj[0])); + try { + if (upcall) { + Swig::DirectorPureVirtualException::raise("ppc::front::GetPeersInfoHandler::onPeersInfo"); + } else { + (arg1)->onPeersInfo(arg2,(std::string const &)*arg3); + } + } catch (Swig::DirectorException&) { + SWIG_fail; + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res3)) delete arg3; + return resultobj; +fail: + if (SWIG_IsNewObj(res3)) delete arg3; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_disown_GetPeersInfoHandler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ; + std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1); + arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + Swig::Director *director = SWIG_DIRECTOR_CAST(arg1); + if (director) director->swig_disown(); + } + + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *GetPeersInfoHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *GetPeersInfoHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFront" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_start" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->start(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_stop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_stop" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->stop(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + void *argp3 ; + int res3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_register_topic_handler", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_topic_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_topic_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'"); + } + if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3); + } + (arg1)->register_topic_handler((std::string const &)*arg2,arg3); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + ppc::front::MessageDispatcherHandler::Ptr arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + void *argp3 ; + int res3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_register_msg_handler", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_msg_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_msg_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'"); + } + if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3); + } + (arg1)->register_msg_handler((std::string const &)*arg2,arg3); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_async_send_message(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + char *arg4 = (char *) 0 ; + uint64_t arg5 ; + int arg6 ; + long arg7 ; + ppc::front::ErrorCallback::Ptr arg8 ; + ppc::front::IMessageHandler::Ptr arg9 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + void *argp3 ; + int res3 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ; + int val6 ; + int ecode6 = 0 ; + long val7 ; + int ecode7 = 0 ; + void *argp8 ; + int res8 = 0 ; + void *argp9 ; + int res9 = 0 ; + PyObject *swig_obj[8] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_message", 8, 8, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_message" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_async_send_message" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_async_send_message" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + arg3 = &tempshared3; + } else { + arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3; + } + } + { + int res; Py_ssize_t size = 0; const void *buf = 0; +#ifndef Py_LIMITED_API + Py_buffer view; + res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO); +#else +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size); +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + if (res < 0) { + PyErr_Clear(); + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_message" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'"); + } +#ifndef Py_LIMITED_API + size = view.len; + buf = view.buf; + PyBuffer_Release(&view); +#endif + arg4 = (char *) buf; + arg5 = (uint64_t) (size / sizeof(char)); + } + ecode6 = SWIG_AsVal_int(swig_obj[4], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_async_send_message" "', argument " "6"" of type '" "int""'"); + } + arg6 = static_cast< int >(val6); + ecode7 = SWIG_AsVal_long(swig_obj[5], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_message" "', argument " "7"" of type '" "long""'"); + } + arg7 = static_cast< long >(val7); + { + int newmem = 0; + res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 , &newmem); + if (!SWIG_IsOK(res8)) { + SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_message" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'"); + } + if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8); + } + { + int newmem = 0; + res9 = SWIG_ConvertPtrAndOwn(swig_obj[7], &argp9, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res9)) { + SWIG_exception_fail(SWIG_ArgError(res9), "in method '" "IFront_async_send_message" "', argument " "9"" of type '" "ppc::front::IMessageHandler::Ptr""'"); + } + if (argp9) arg9 = *(reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9); + } + (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + char *arg2 = (char *) 0 ; + uint64_t arg3 ; + std::string *arg4 = 0 ; + char *arg5 = (char *) 0 ; + uint64_t arg6 ; + int arg7 ; + ppc::front::ErrorCallback::Ptr arg8 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 ; + char *buf2 = 0 ; + int alloc2 = 0 ; + unsigned long long val3 ; + int ecode3 = 0 ; + int res4 = SWIG_OLDOBJ ; + int val7 ; + int ecode7 = 0 ; + void *argp8 ; + int res8 = 0 ; + PyObject *swig_obj[7] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_response", 7, 7, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_response" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_async_send_response" "', argument " "2"" of type '" "char *""'"); + } + arg2 = reinterpret_cast< char * >(buf2); + ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_async_send_response" "', argument " "3"" of type '" "uint64_t""'"); + } + arg3 = static_cast< uint64_t >(val3); + { + std::string *ptr = (std::string *)0; + res4 = SWIG_AsPtr_std_string(swig_obj[3], &ptr); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); + } + arg4 = ptr; + } + { + int res; Py_ssize_t size = 0; const void *buf = 0; +#ifndef Py_LIMITED_API + Py_buffer view; + res = PyObject_GetBuffer(swig_obj[4], &view, PyBUF_CONTIG_RO); +#else +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + res = PyObject_AsReadBuffer(swig_obj[4], &buf, &size); +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + if (res < 0) { + PyErr_Clear(); + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_response" "', argument " "5"" of type '" "(char* payload, uint64_t payloadSize)""'"); + } +#ifndef Py_LIMITED_API + size = view.len; + buf = view.buf; + PyBuffer_Release(&view); +#endif + arg5 = (char *) buf; + arg6 = (uint64_t) (size / sizeof(char)); + } + ecode7 = SWIG_AsVal_int(swig_obj[5], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_response" "', argument " "7"" of type '" "int""'"); + } + arg7 = static_cast< int >(val7); + { + int newmem = 0; + res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 , &newmem); + if (!SWIG_IsOK(res8)) { + SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_response" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'"); + } + if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8); + } + (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8); + resultobj = SWIG_Py_Void(); + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res4)) delete arg4; + return resultobj; +fail: + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res4)) delete arg4; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_push(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + bcos::bytes *arg4 = 0 ; + int arg5 ; + long arg6 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + void *argp3 ; + int res3 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ; + void *argp4 = 0 ; + int res4 = 0 ; + std::unique_ptr< bcos::bytes > rvrdeleter4 ; + int val5 ; + int ecode5 = 0 ; + long val6 ; + int ecode6 = 0 ; + PyObject *swig_obj[6] ; + bcos::Error::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_push", 6, 6, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_push" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_push" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_push" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + arg3 = &tempshared3; + } else { + arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3; + } + } + res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE | 0 ); + if (!SWIG_IsOK(res4)) { + if (res4 == SWIG_ERROR_RELEASE_NOT_OWNED) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_push" "', cannot release ownership as memory is not owned for argument " "4"" of type '" "bcos::bytes &&""'"); + } else { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_push" "', argument " "4"" of type '" "bcos::bytes &&""'"); + } + } + if (!argp4) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_push" "', argument " "4"" of type '" "bcos::bytes &&""'"); + } + arg4 = reinterpret_cast< bcos::bytes * >(argp4); + rvrdeleter4.reset(arg4); + ecode5 = SWIG_AsVal_int(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "IFront_push" "', argument " "5"" of type '" "int""'"); + } + arg5 = static_cast< int >(val5); + ecode6 = SWIG_AsVal_long(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_push" "', argument " "6"" of type '" "long""'"); + } + arg6 = static_cast< long >(val6); + result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + uint16_t arg2 ; + ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ; + char *arg4 = (char *) 0 ; + uint64_t arg5 ; + int arg6 ; + long arg7 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + unsigned short val2 ; + int ecode2 = 0 ; + void *argp3 ; + int res3 = 0 ; + ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ; + int val6 ; + int ecode6 = 0 ; + long val7 ; + int ecode7 = 0 ; + PyObject *swig_obj[6] ; + bcos::Error::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_push_msg", 6, 6, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_push_msg" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_push_msg" "', argument " "2"" of type '" "uint16_t""'"); + } + arg2 = static_cast< uint16_t >(val2); + { + int newmem = 0; + res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 , &newmem); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_push_msg" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3); + arg3 = &tempshared3; + } else { + arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3; + } + } + { + int res; Py_ssize_t size = 0; const void *buf = 0; +#ifndef Py_LIMITED_API + Py_buffer view; + res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO); +#else +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size); +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif +#endif + if (res < 0) { + PyErr_Clear(); + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_push_msg" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'"); + } +#ifndef Py_LIMITED_API + size = view.len; + buf = view.buf; + PyBuffer_Release(&view); +#endif + arg4 = (char *) buf; + arg5 = (uint64_t) (size / sizeof(char)); + } + ecode6 = SWIG_AsVal_int(swig_obj[4], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_push_msg" "', argument " "6"" of type '" "int""'"); + } + arg6 = static_cast< int >(val6); + ecode7 = SWIG_AsVal_long(swig_obj[5], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_push_msg" "', argument " "7"" of type '" "long""'"); + } + arg7 = static_cast< long >(val7); + result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_pop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + long arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + long val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + ppc::protocol::Message::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_pop", 3, 3, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_pop" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + ecode3 = SWIG_AsVal_long(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_pop" "', argument " "3"" of type '" "long""'"); + } + arg3 = static_cast< long >(val3); + result = (arg1)->pop((std::string const &)*arg2,arg3); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = result ? new std::shared_ptr< ppc::protocol::Message >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + } + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_peek(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + ppc::protocol::Message::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_peek", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_peek" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + result = (arg1)->peek((std::string const &)*arg2); + { + std::shared_ptr< ppc::protocol::Message > *smartresult = result ? new std::shared_ptr< ppc::protocol::Message >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN); + } + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_asyncGetPeers(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + ppc::front::GetPeersInfoHandler::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_asyncGetPeers", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_asyncGetPeers" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_asyncGetPeers" "', argument " "2"" of type '" "ppc::front::GetPeersInfoHandler::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2); + } + (arg1)->asyncGetPeers(arg2); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_registerNodeInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + ppc::protocol::INodeInfo::Ptr *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + bcos::Error::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_registerNodeInfo", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerNodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr, 0 | 0); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerNodeInfo" "', argument " "2"" of type '" "ppc::protocol::INodeInfo::Ptr const &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerNodeInfo" "', argument " "2"" of type '" "ppc::protocol::INodeInfo::Ptr const &""'"); + } + arg2 = reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp2); + result = (arg1)->registerNodeInfo((ppc::protocol::INodeInfo::Ptr const &)*arg2); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_unRegisterNodeInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + bcos::Error::Ptr result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterNodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (arg1)->unRegisterNodeInfo(); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_nodeInfo(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::INodeInfo::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_nodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_registerTopic(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + bcos::Error::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_registerTopic", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + result = (arg1)->registerTopic((std::string const &)*arg2); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_unRegisterTopic(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + bcos::Error::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterTopic", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + result = (arg1)->unRegisterTopic((std::string const &)*arg2); + { + std::shared_ptr< bcos::Error > *smartresult = result ? new std::shared_ptr< bcos::Error >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN); + } + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_registerComponent(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_registerComponent", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->registerComponent((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFront_unRegisterComponent(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ; + std::string *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::front::IFront > tempshared1 ; + std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ; + int res2 = SWIG_OLDOBJ ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterComponent", 2, 2, swig_obj)) SWIG_fail; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1); + arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0)); + } + } + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; + } + (arg1)->unRegisterComponent((std::string const &)*arg2); + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res2)) delete arg2; + return resultobj; +fail: + if (SWIG_IsNewObj(res2)) delete arg2; + return NULL; +} + + +SWIGINTERN PyObject *IFront_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_delete_IFrontBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontBuilder" "', argument " "1"" of type '" "ppc::front::IFrontBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_IFrontBuilder_buildClient(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ; + std::string arg2 ; + std::function< void () > arg3 ; + bool arg4 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp3 ; + int res3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + PyObject *swig_obj[4] ; + ppc::front::IFrontClient::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "IFrontBuilder_buildClient", 4, 4, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontBuilder_buildClient" "', argument " "1"" of type '" "ppc::front::IFrontBuilder const *""'"); + } + arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1); + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "IFrontBuilder_buildClient" "', argument " "2"" of type '" "std::string""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + { + res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fF_t, 0 | 0); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'"); + } + if (!argp3) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'"); + } else { + std::function< void () > * temp = reinterpret_cast< std::function< void () > * >(argp3); + arg3 = *temp; + if (SWIG_IsNewObj(res3)) delete temp; + } + } + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "IFrontBuilder_buildClient" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4); + { + std::shared_ptr< ppc::front::IFrontClient > *smartresult = result ? new std::shared_ptr< ppc::front::IFrontClient >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *IFrontBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_new_TransportBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::TransportBuilder *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_TransportBuilder", 0, 0, 0)) SWIG_fail; + result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_TransportBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_TransportBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1); + delete arg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_TransportBuilder_buildProTransport(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + ppc::front::FrontConfig::Ptr arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + ppc::sdk::Transport::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildProTransport", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildProTransport" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1); + { + int newmem = 0; + res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "TransportBuilder_buildProTransport" "', argument " "2"" of type '" "ppc::front::FrontConfig::Ptr""'"); + } + if (argp2) arg2 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2); + } + result = (arg1)->buildProTransport(arg2); + { + std::shared_ptr< ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr< ppc::sdk::Transport >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_TransportBuilder_buildConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + int arg2 ; + std::string arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[3] ; + ppc::front::FrontConfig::Ptr result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildConfig", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildConfig" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "TransportBuilder_buildConfig" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + std::string *ptr = (std::string *)0; + int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "TransportBuilder_buildConfig" "', argument " "3"" of type '" "std::string""'"); + } + arg3 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + result = (arg1)->buildConfig(arg2,arg3); + { + std::shared_ptr< ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr< ppc::front::FrontConfig >(result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + ppc::front::FrontConfigBuilder::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_frontConfigBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); + } + arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1); + result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *TransportBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *TransportBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_Transport(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::front::FrontConfig::Ptr arg1 ; + void *argp1 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + ppc::sdk::Transport *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_Transport" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr""'"); + } + if (argp1) arg1 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1)); + if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1); + } + result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1); + { + std::shared_ptr< ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr< ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_Transport(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Transport" "', argument " "1"" of type '" "ppc::sdk::Transport *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (void)arg1; delete smartarg1; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_start(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_start" "', argument " "1"" of type '" "ppc::sdk::Transport *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->start(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_stop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_stop" "', argument " "1"" of type '" "ppc::sdk::Transport *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + (arg1)->stop(); + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_getFront(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::front::IFront::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getFront" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront(); + { + std::shared_ptr< ppc::front::IFront > *smartresult = *result ? new std::shared_ptr< ppc::front::IFront >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_gateway(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::gateway::IGateway::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_gateway" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway(); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__gateway__IGateway__Ptr, 0 | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_msgPayloadBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessagePayloadBuilder::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgPayloadBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder(); + { + std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartresult = *result ? new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_msgBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessageBuilder::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder(); + { + std::shared_ptr< ppc::protocol::MessageBuilder > *smartresult = *result ? new std::shared_ptr< ppc::protocol::MessageBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_msgHeaderBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessageHeaderBuilder::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgHeaderBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder(); + { + std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartresult = *result ? new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_routeInfoBuilder(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::protocol::MessageOptionalHeaderBuilder::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_routeInfoBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder(); + { + std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = *result ? new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_Transport_getConfig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::shared_ptr< ppc::sdk::Transport const > tempshared1 ; + std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ; + PyObject *swig_obj[1] ; + ppc::front::FrontConfig::Ptr *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + int newmem = 0; + res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 | 0 , &newmem); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getConfig" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'"); + } + if (newmem & SWIG_CAST_NEW_MEMORY) { + tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get()); + } else { + smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1); + arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0)); + } + } + result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig(); + { + std::shared_ptr< ppc::front::FrontConfig > *smartresult = *result ? new std::shared_ptr< ppc::front::FrontConfig >(*result) : 0; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *Transport_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *Transport_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +static PyMethodDef SwigMethods[] = { + { "delete_SwigPyIterator", _wrap_delete_SwigPyIterator, METH_O, NULL}, + { "SwigPyIterator_value", _wrap_SwigPyIterator_value, METH_O, NULL}, + { "SwigPyIterator_incr", _wrap_SwigPyIterator_incr, METH_VARARGS, NULL}, + { "SwigPyIterator_decr", _wrap_SwigPyIterator_decr, METH_VARARGS, NULL}, + { "SwigPyIterator_distance", _wrap_SwigPyIterator_distance, METH_VARARGS, NULL}, + { "SwigPyIterator_equal", _wrap_SwigPyIterator_equal, METH_VARARGS, NULL}, + { "SwigPyIterator_copy", _wrap_SwigPyIterator_copy, METH_O, NULL}, + { "SwigPyIterator_next", _wrap_SwigPyIterator_next, METH_O, NULL}, + { "SwigPyIterator___next__", _wrap_SwigPyIterator___next__, METH_O, NULL}, + { "SwigPyIterator_previous", _wrap_SwigPyIterator_previous, METH_O, NULL}, + { "SwigPyIterator_advance", _wrap_SwigPyIterator_advance, METH_VARARGS, NULL}, + { "SwigPyIterator___eq__", _wrap_SwigPyIterator___eq__, METH_VARARGS, NULL}, + { "SwigPyIterator___ne__", _wrap_SwigPyIterator___ne__, METH_VARARGS, NULL}, + { "SwigPyIterator___iadd__", _wrap_SwigPyIterator___iadd__, METH_VARARGS, NULL}, + { "SwigPyIterator___isub__", _wrap_SwigPyIterator___isub__, METH_VARARGS, NULL}, + { "SwigPyIterator___add__", _wrap_SwigPyIterator___add__, METH_VARARGS, NULL}, + { "SwigPyIterator___sub__", _wrap_SwigPyIterator___sub__, METH_VARARGS, NULL}, + { "SwigPyIterator_swigregister", SwigPyIterator_swigregister, METH_O, NULL}, + { "new_SharedBcosError", _wrap_new_SharedBcosError, METH_NOARGS, NULL}, + { "delete_SharedBcosError", _wrap_delete_SharedBcosError, METH_O, NULL}, + { "SharedBcosError_swigregister", SharedBcosError_swigregister, METH_O, NULL}, + { "SharedBcosError_swiginit", SharedBcosError_swiginit, METH_VARARGS, NULL}, + { "new_SharedFrontConfig", _wrap_new_SharedFrontConfig, METH_NOARGS, NULL}, + { "delete_SharedFrontConfig", _wrap_delete_SharedFrontConfig, METH_O, NULL}, + { "SharedFrontConfig_swigregister", SharedFrontConfig_swigregister, METH_O, NULL}, + { "SharedFrontConfig_swiginit", SharedFrontConfig_swiginit, METH_VARARGS, NULL}, + { "new_SharedGrpcConfig", _wrap_new_SharedGrpcConfig, METH_NOARGS, NULL}, + { "delete_SharedGrpcConfig", _wrap_delete_SharedGrpcConfig, METH_O, NULL}, + { "SharedGrpcConfig_swigregister", SharedGrpcConfig_swigregister, METH_O, NULL}, + { "SharedGrpcConfig_swiginit", SharedGrpcConfig_swiginit, METH_VARARGS, NULL}, + { "new_SharedFront", _wrap_new_SharedFront, METH_NOARGS, NULL}, + { "delete_SharedFront", _wrap_delete_SharedFront, METH_O, NULL}, + { "SharedFront_swigregister", SharedFront_swigregister, METH_O, NULL}, + { "SharedFront_swiginit", SharedFront_swiginit, METH_VARARGS, NULL}, + { "new_SharedFrontClient", _wrap_new_SharedFrontClient, METH_NOARGS, NULL}, + { "delete_SharedFrontClient", _wrap_delete_SharedFrontClient, METH_O, NULL}, + { "SharedFrontClient_swigregister", SharedFrontClient_swigregister, METH_O, NULL}, + { "SharedFrontClient_swiginit", SharedFrontClient_swiginit, METH_VARARGS, NULL}, + { "new_SharedErrorCallback", _wrap_new_SharedErrorCallback, METH_NOARGS, NULL}, + { "delete_SharedErrorCallback", _wrap_delete_SharedErrorCallback, METH_O, NULL}, + { "SharedErrorCallback_swigregister", SharedErrorCallback_swigregister, METH_O, NULL}, + { "SharedErrorCallback_swiginit", SharedErrorCallback_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessageDispatcherHandler", _wrap_new_SharedMessageDispatcherHandler, METH_NOARGS, NULL}, + { "delete_SharedMessageDispatcherHandler", _wrap_delete_SharedMessageDispatcherHandler, METH_O, NULL}, + { "SharedMessageDispatcherHandler_swigregister", SharedMessageDispatcherHandler_swigregister, METH_O, NULL}, + { "SharedMessageDispatcherHandler_swiginit", SharedMessageDispatcherHandler_swiginit, METH_VARARGS, NULL}, + { "new_SharedIMessageHandler", _wrap_new_SharedIMessageHandler, METH_NOARGS, NULL}, + { "delete_SharedIMessageHandler", _wrap_delete_SharedIMessageHandler, METH_O, NULL}, + { "SharedIMessageHandler_swigregister", SharedIMessageHandler_swigregister, METH_O, NULL}, + { "SharedIMessageHandler_swiginit", SharedIMessageHandler_swiginit, METH_VARARGS, NULL}, + { "new_SharedGetPeersInfoHandler", _wrap_new_SharedGetPeersInfoHandler, METH_NOARGS, NULL}, + { "delete_SharedGetPeersInfoHandler", _wrap_delete_SharedGetPeersInfoHandler, METH_O, NULL}, + { "SharedGetPeersInfoHandler_swigregister", SharedGetPeersInfoHandler_swigregister, METH_O, NULL}, + { "SharedGetPeersInfoHandler_swiginit", SharedGetPeersInfoHandler_swiginit, METH_VARARGS, NULL}, + { "new_SharedGateway", _wrap_new_SharedGateway, METH_NOARGS, NULL}, + { "delete_SharedGateway", _wrap_delete_SharedGateway, METH_O, NULL}, + { "SharedGateway_swigregister", SharedGateway_swigregister, METH_O, NULL}, + { "SharedGateway_swiginit", SharedGateway_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessage", _wrap_new_SharedMessage, METH_NOARGS, NULL}, + { "delete_SharedMessage", _wrap_delete_SharedMessage, METH_O, NULL}, + { "SharedMessage_swigregister", SharedMessage_swigregister, METH_O, NULL}, + { "SharedMessage_swiginit", SharedMessage_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessageHeader", _wrap_new_SharedMessageHeader, METH_NOARGS, NULL}, + { "delete_SharedMessageHeader", _wrap_delete_SharedMessageHeader, METH_O, NULL}, + { "SharedMessageHeader_swigregister", SharedMessageHeader_swigregister, METH_O, NULL}, + { "SharedMessageHeader_swiginit", SharedMessageHeader_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessagePayload", _wrap_new_SharedMessagePayload, METH_NOARGS, NULL}, + { "delete_SharedMessagePayload", _wrap_delete_SharedMessagePayload, METH_O, NULL}, + { "SharedMessagePayload_swigregister", SharedMessagePayload_swigregister, METH_O, NULL}, + { "SharedMessagePayload_swiginit", SharedMessagePayload_swiginit, METH_VARARGS, NULL}, + { "new_SharedRouteInfo", _wrap_new_SharedRouteInfo, METH_NOARGS, NULL}, + { "delete_SharedRouteInfo", _wrap_delete_SharedRouteInfo, METH_O, NULL}, + { "SharedRouteInfo_swigregister", SharedRouteInfo_swigregister, METH_O, NULL}, + { "SharedRouteInfo_swiginit", SharedRouteInfo_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessageBuilder", _wrap_new_SharedMessageBuilder, METH_NOARGS, NULL}, + { "delete_SharedMessageBuilder", _wrap_delete_SharedMessageBuilder, METH_O, NULL}, + { "SharedMessageBuilder_swigregister", SharedMessageBuilder_swigregister, METH_O, NULL}, + { "SharedMessageBuilder_swiginit", SharedMessageBuilder_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessageHeaderBuilder", _wrap_new_SharedMessageHeaderBuilder, METH_NOARGS, NULL}, + { "delete_SharedMessageHeaderBuilder", _wrap_delete_SharedMessageHeaderBuilder, METH_O, NULL}, + { "SharedMessageHeaderBuilder_swigregister", SharedMessageHeaderBuilder_swigregister, METH_O, NULL}, + { "SharedMessageHeaderBuilder_swiginit", SharedMessageHeaderBuilder_swiginit, METH_VARARGS, NULL}, + { "new_SharedMessagePayloadBuilder", _wrap_new_SharedMessagePayloadBuilder, METH_NOARGS, NULL}, + { "delete_SharedMessagePayloadBuilder", _wrap_delete_SharedMessagePayloadBuilder, METH_O, NULL}, + { "SharedMessagePayloadBuilder_swigregister", SharedMessagePayloadBuilder_swigregister, METH_O, NULL}, + { "SharedMessagePayloadBuilder_swiginit", SharedMessagePayloadBuilder_swiginit, METH_VARARGS, NULL}, + { "new_SharedRouteInfoBuilder", _wrap_new_SharedRouteInfoBuilder, METH_NOARGS, NULL}, + { "delete_SharedRouteInfoBuilder", _wrap_delete_SharedRouteInfoBuilder, METH_O, NULL}, + { "SharedRouteInfoBuilder_swigregister", SharedRouteInfoBuilder_swigregister, METH_O, NULL}, + { "SharedRouteInfoBuilder_swiginit", SharedRouteInfoBuilder_swiginit, METH_VARARGS, NULL}, + { "ubytes_iterator", _wrap_ubytes_iterator, METH_O, NULL}, + { "ubytes___nonzero__", _wrap_ubytes___nonzero__, METH_O, NULL}, + { "ubytes___bool__", _wrap_ubytes___bool__, METH_O, NULL}, + { "ubytes___len__", _wrap_ubytes___len__, METH_O, NULL}, + { "ubytes___getslice__", _wrap_ubytes___getslice__, METH_VARARGS, NULL}, + { "ubytes___setslice__", _wrap_ubytes___setslice__, METH_VARARGS, NULL}, + { "ubytes___delslice__", _wrap_ubytes___delslice__, METH_VARARGS, NULL}, + { "ubytes___delitem__", _wrap_ubytes___delitem__, METH_VARARGS, NULL}, + { "ubytes___getitem__", _wrap_ubytes___getitem__, METH_VARARGS, NULL}, + { "ubytes___setitem__", _wrap_ubytes___setitem__, METH_VARARGS, NULL}, + { "ubytes_pop", _wrap_ubytes_pop, METH_O, NULL}, + { "ubytes_append", _wrap_ubytes_append, METH_VARARGS, NULL}, + { "ubytes_empty", _wrap_ubytes_empty, METH_O, NULL}, + { "ubytes_size", _wrap_ubytes_size, METH_O, NULL}, + { "ubytes_swap", _wrap_ubytes_swap, METH_VARARGS, NULL}, + { "ubytes_begin", _wrap_ubytes_begin, METH_O, NULL}, + { "ubytes_end", _wrap_ubytes_end, METH_O, NULL}, + { "ubytes_rbegin", _wrap_ubytes_rbegin, METH_O, NULL}, + { "ubytes_rend", _wrap_ubytes_rend, METH_O, NULL}, + { "ubytes_clear", _wrap_ubytes_clear, METH_O, NULL}, + { "ubytes_get_allocator", _wrap_ubytes_get_allocator, METH_O, NULL}, + { "ubytes_pop_back", _wrap_ubytes_pop_back, METH_O, NULL}, + { "ubytes_erase", _wrap_ubytes_erase, METH_VARARGS, NULL}, + { "new_ubytes", _wrap_new_ubytes, METH_VARARGS, NULL}, + { "ubytes_push_back", _wrap_ubytes_push_back, METH_VARARGS, NULL}, + { "ubytes_front", _wrap_ubytes_front, METH_O, NULL}, + { "ubytes_back", _wrap_ubytes_back, METH_O, NULL}, + { "ubytes_assign", _wrap_ubytes_assign, METH_VARARGS, NULL}, + { "ubytes_resize", _wrap_ubytes_resize, METH_VARARGS, NULL}, + { "ubytes_insert", _wrap_ubytes_insert, METH_VARARGS, NULL}, + { "ubytes_reserve", _wrap_ubytes_reserve, METH_VARARGS, NULL}, + { "ubytes_capacity", _wrap_ubytes_capacity, METH_O, NULL}, + { "delete_ubytes", _wrap_delete_ubytes, METH_O, NULL}, + { "ubytes_swigregister", ubytes_swigregister, METH_O, NULL}, + { "ubytes_swiginit", ubytes_swiginit, METH_VARARGS, NULL}, + { "ibytes_iterator", _wrap_ibytes_iterator, METH_O, NULL}, + { "ibytes___nonzero__", _wrap_ibytes___nonzero__, METH_O, NULL}, + { "ibytes___bool__", _wrap_ibytes___bool__, METH_O, NULL}, + { "ibytes___len__", _wrap_ibytes___len__, METH_O, NULL}, + { "ibytes___getslice__", _wrap_ibytes___getslice__, METH_VARARGS, NULL}, + { "ibytes___setslice__", _wrap_ibytes___setslice__, METH_VARARGS, NULL}, + { "ibytes___delslice__", _wrap_ibytes___delslice__, METH_VARARGS, NULL}, + { "ibytes___delitem__", _wrap_ibytes___delitem__, METH_VARARGS, NULL}, + { "ibytes___getitem__", _wrap_ibytes___getitem__, METH_VARARGS, NULL}, + { "ibytes___setitem__", _wrap_ibytes___setitem__, METH_VARARGS, NULL}, + { "ibytes_pop", _wrap_ibytes_pop, METH_O, NULL}, + { "ibytes_append", _wrap_ibytes_append, METH_VARARGS, NULL}, + { "ibytes_empty", _wrap_ibytes_empty, METH_O, NULL}, + { "ibytes_size", _wrap_ibytes_size, METH_O, NULL}, + { "ibytes_swap", _wrap_ibytes_swap, METH_VARARGS, NULL}, + { "ibytes_begin", _wrap_ibytes_begin, METH_O, NULL}, + { "ibytes_end", _wrap_ibytes_end, METH_O, NULL}, + { "ibytes_rbegin", _wrap_ibytes_rbegin, METH_O, NULL}, + { "ibytes_rend", _wrap_ibytes_rend, METH_O, NULL}, + { "ibytes_clear", _wrap_ibytes_clear, METH_O, NULL}, + { "ibytes_get_allocator", _wrap_ibytes_get_allocator, METH_O, NULL}, + { "ibytes_pop_back", _wrap_ibytes_pop_back, METH_O, NULL}, + { "ibytes_erase", _wrap_ibytes_erase, METH_VARARGS, NULL}, + { "new_ibytes", _wrap_new_ibytes, METH_VARARGS, NULL}, + { "ibytes_push_back", _wrap_ibytes_push_back, METH_VARARGS, NULL}, + { "ibytes_front", _wrap_ibytes_front, METH_O, NULL}, + { "ibytes_back", _wrap_ibytes_back, METH_O, NULL}, + { "ibytes_assign", _wrap_ibytes_assign, METH_VARARGS, NULL}, + { "ibytes_resize", _wrap_ibytes_resize, METH_VARARGS, NULL}, + { "ibytes_insert", _wrap_ibytes_insert, METH_VARARGS, NULL}, + { "ibytes_reserve", _wrap_ibytes_reserve, METH_VARARGS, NULL}, + { "ibytes_capacity", _wrap_ibytes_capacity, METH_O, NULL}, + { "delete_ibytes", _wrap_delete_ibytes, METH_O, NULL}, + { "ibytes_swigregister", ibytes_swigregister, METH_O, NULL}, + { "ibytes_swiginit", ibytes_swiginit, METH_VARARGS, NULL}, + { "Error_buildError", _wrap_Error_buildError, METH_VARARGS, NULL}, + { "new_Error", _wrap_new_Error, METH_VARARGS, NULL}, + { "delete_Error", _wrap_delete_Error, METH_O, NULL}, + { "Error_errorCode", _wrap_Error_errorCode, METH_O, NULL}, + { "Error_errorMessage", _wrap_Error_errorMessage, METH_O, NULL}, + { "Error_setErrorCode", _wrap_Error_setErrorCode, METH_VARARGS, NULL}, + { "Error_setErrorMessage", _wrap_Error_setErrorMessage, METH_VARARGS, NULL}, + { "Error_swigregister", Error_swigregister, METH_O, NULL}, + { "Error_swiginit", Error_swiginit, METH_VARARGS, NULL}, + { "InputBuffer_data_set", _wrap_InputBuffer_data_set, METH_VARARGS, NULL}, + { "InputBuffer_data_get", _wrap_InputBuffer_data_get, METH_O, NULL}, + { "InputBuffer_len_set", _wrap_InputBuffer_len_set, METH_VARARGS, NULL}, + { "InputBuffer_len_get", _wrap_InputBuffer_len_get, METH_O, NULL}, + { "new_InputBuffer", _wrap_new_InputBuffer, METH_NOARGS, NULL}, + { "delete_InputBuffer", _wrap_delete_InputBuffer, METH_O, NULL}, + { "InputBuffer_swigregister", InputBuffer_swigregister, METH_O, NULL}, + { "InputBuffer_swiginit", InputBuffer_swiginit, METH_VARARGS, NULL}, + { "OutputBuffer_data_set", _wrap_OutputBuffer_data_set, METH_VARARGS, NULL}, + { "OutputBuffer_data_get", _wrap_OutputBuffer_data_get, METH_O, NULL}, + { "OutputBuffer_len_set", _wrap_OutputBuffer_len_set, METH_VARARGS, NULL}, + { "OutputBuffer_len_get", _wrap_OutputBuffer_len_get, METH_O, NULL}, + { "new_OutputBuffer", _wrap_new_OutputBuffer, METH_NOARGS, NULL}, + { "delete_OutputBuffer", _wrap_delete_OutputBuffer, METH_O, NULL}, + { "OutputBuffer_swigregister", OutputBuffer_swigregister, METH_O, NULL}, + { "OutputBuffer_swiginit", OutputBuffer_swiginit, METH_VARARGS, NULL}, + { "delete_FrontConfig", _wrap_delete_FrontConfig, METH_O, NULL}, + { "FrontConfig_threadPoolSize", _wrap_FrontConfig_threadPoolSize, METH_O, NULL}, + { "FrontConfig_setThreadPoolSize", _wrap_FrontConfig_setThreadPoolSize, METH_VARARGS, NULL}, + { "FrontConfig_nodeID", _wrap_FrontConfig_nodeID, METH_O, NULL}, + { "FrontConfig_setNodeID", _wrap_FrontConfig_setNodeID, METH_VARARGS, NULL}, + { "FrontConfig_selfEndPoint", _wrap_FrontConfig_selfEndPoint, METH_O, NULL}, + { "FrontConfig_mutableSelfEndPoint", _wrap_FrontConfig_mutableSelfEndPoint, METH_O, NULL}, + { "FrontConfig_setSelfEndPoint", _wrap_FrontConfig_setSelfEndPoint, METH_VARARGS, NULL}, + { "FrontConfig_setGatewayGrpcTarget", _wrap_FrontConfig_setGatewayGrpcTarget, METH_VARARGS, NULL}, + { "FrontConfig_gatewayGrpcTarget", _wrap_FrontConfig_gatewayGrpcTarget, METH_O, NULL}, + { "FrontConfig_setGrpcConfig", _wrap_FrontConfig_setGrpcConfig, METH_VARARGS, NULL}, + { "FrontConfig_grpcConfig", _wrap_FrontConfig_grpcConfig, METH_O, NULL}, + { "FrontConfig_generateNodeInfo", _wrap_FrontConfig_generateNodeInfo, METH_O, NULL}, + { "FrontConfig_getComponents", _wrap_FrontConfig_getComponents, METH_O, NULL}, + { "FrontConfig_setComponents", _wrap_FrontConfig_setComponents, METH_VARARGS, NULL}, + { "FrontConfig_addComponent", _wrap_FrontConfig_addComponent, METH_VARARGS, NULL}, + { "FrontConfig_mutableComponents", _wrap_FrontConfig_mutableComponents, METH_O, NULL}, + { "FrontConfig_swigregister", FrontConfig_swigregister, METH_O, NULL}, + { "delete_FrontConfigBuilder", _wrap_delete_FrontConfigBuilder, METH_O, NULL}, + { "FrontConfigBuilder_build", _wrap_FrontConfigBuilder_build, METH_VARARGS, NULL}, + { "FrontConfigBuilder_swigregister", FrontConfigBuilder_swigregister, METH_O, NULL}, + { "printFrontDesc", _wrap_printFrontDesc, METH_O, NULL}, + { "new_EndPoint", _wrap_new_EndPoint, METH_VARARGS, NULL}, + { "delete_EndPoint", _wrap_delete_EndPoint, METH_O, NULL}, + { "EndPoint_host", _wrap_EndPoint_host, METH_O, NULL}, + { "EndPoint_port", _wrap_EndPoint_port, METH_O, NULL}, + { "EndPoint_setHost", _wrap_EndPoint_setHost, METH_VARARGS, NULL}, + { "EndPoint_setPort", _wrap_EndPoint_setPort, METH_VARARGS, NULL}, + { "EndPoint_setListenIp", _wrap_EndPoint_setListenIp, METH_VARARGS, NULL}, + { "EndPoint_entryPoint", _wrap_EndPoint_entryPoint, METH_O, NULL}, + { "EndPoint_listenEndPoint", _wrap_EndPoint_listenEndPoint, METH_O, NULL}, + { "EndPoint_listenIp", _wrap_EndPoint_listenIp, METH_O, NULL}, + { "EndPoint_swigregister", EndPoint_swigregister, METH_O, NULL}, + { "EndPoint_swiginit", EndPoint_swiginit, METH_VARARGS, NULL}, + { "new_GrpcServerConfig", _wrap_new_GrpcServerConfig, METH_VARARGS, NULL}, + { "GrpcServerConfig_listenEndPoint", _wrap_GrpcServerConfig_listenEndPoint, METH_O, NULL}, + { "GrpcServerConfig_setEndPoint", _wrap_GrpcServerConfig_setEndPoint, METH_VARARGS, NULL}, + { "GrpcServerConfig_setEnableHealthCheck", _wrap_GrpcServerConfig_setEnableHealthCheck, METH_VARARGS, NULL}, + { "GrpcServerConfig_endPoint", _wrap_GrpcServerConfig_endPoint, METH_O, NULL}, + { "GrpcServerConfig_mutableEndPoint", _wrap_GrpcServerConfig_mutableEndPoint, METH_O, NULL}, + { "GrpcServerConfig_enableHealthCheck", _wrap_GrpcServerConfig_enableHealthCheck, METH_O, NULL}, + { "delete_GrpcServerConfig", _wrap_delete_GrpcServerConfig, METH_O, NULL}, + { "GrpcServerConfig_swigregister", GrpcServerConfig_swigregister, METH_O, NULL}, + { "GrpcServerConfig_swiginit", GrpcServerConfig_swiginit, METH_VARARGS, NULL}, + { "new_GrpcConfig", _wrap_new_GrpcConfig, METH_NOARGS, NULL}, + { "delete_GrpcConfig", _wrap_delete_GrpcConfig, METH_O, NULL}, + { "GrpcConfig_loadBalancePolicy", _wrap_GrpcConfig_loadBalancePolicy, METH_O, NULL}, + { "GrpcConfig_setLoadBalancePolicy", _wrap_GrpcConfig_setLoadBalancePolicy, METH_VARARGS, NULL}, + { "GrpcConfig_enableHealthCheck", _wrap_GrpcConfig_enableHealthCheck, METH_O, NULL}, + { "GrpcConfig_setEnableHealthCheck", _wrap_GrpcConfig_setEnableHealthCheck, METH_VARARGS, NULL}, + { "GrpcConfig_setEnableDnslookup", _wrap_GrpcConfig_setEnableDnslookup, METH_VARARGS, NULL}, + { "GrpcConfig_enableDnslookup", _wrap_GrpcConfig_enableDnslookup, METH_O, NULL}, + { "GrpcConfig_swigregister", GrpcConfig_swigregister, METH_O, NULL}, + { "GrpcConfig_swiginit", GrpcConfig_swiginit, METH_VARARGS, NULL}, + { "delete_MessageOptionalHeader", _wrap_delete_MessageOptionalHeader, METH_O, NULL}, + { "MessageOptionalHeader_encode", _wrap_MessageOptionalHeader_encode, METH_VARARGS, NULL}, + { "MessageOptionalHeader_decode", _wrap_MessageOptionalHeader_decode, METH_VARARGS, NULL}, + { "MessageOptionalHeader_componentType", _wrap_MessageOptionalHeader_componentType, METH_O, NULL}, + { "MessageOptionalHeader_setComponentType", _wrap_MessageOptionalHeader_setComponentType, METH_VARARGS, NULL}, + { "MessageOptionalHeader_srcNode", _wrap_MessageOptionalHeader_srcNode, METH_O, NULL}, + { "MessageOptionalHeader_srcNodeBuffer", _wrap_MessageOptionalHeader_srcNodeBuffer, METH_O, " for swig-wrapper(pass the binary data)"}, + { "MessageOptionalHeader_setSrcNode", _wrap_MessageOptionalHeader_setSrcNode, METH_VARARGS, NULL}, + { "MessageOptionalHeader_dstNode", _wrap_MessageOptionalHeader_dstNode, METH_O, NULL}, + { "MessageOptionalHeader_dstNodeBuffer", _wrap_MessageOptionalHeader_dstNodeBuffer, METH_O, NULL}, + { "MessageOptionalHeader_setDstNode", _wrap_MessageOptionalHeader_setDstNode, METH_VARARGS, NULL}, + { "MessageOptionalHeader_setDstNodeBuffer", _wrap_MessageOptionalHeader_setDstNodeBuffer, METH_VARARGS, NULL}, + { "MessageOptionalHeader_dstInst", _wrap_MessageOptionalHeader_dstInst, METH_O, NULL}, + { "MessageOptionalHeader_setDstInst", _wrap_MessageOptionalHeader_setDstInst, METH_VARARGS, NULL}, + { "MessageOptionalHeader_topic", _wrap_MessageOptionalHeader_topic, METH_O, NULL}, + { "MessageOptionalHeader_setTopic", _wrap_MessageOptionalHeader_setTopic, METH_VARARGS, NULL}, + { "MessageOptionalHeader_srcInst", _wrap_MessageOptionalHeader_srcInst, METH_O, NULL}, + { "MessageOptionalHeader_setSrcInst", _wrap_MessageOptionalHeader_setSrcInst, METH_VARARGS, NULL}, + { "MessageOptionalHeader_swigregister", MessageOptionalHeader_swigregister, METH_O, NULL}, + { "delete_MessageHeader", _wrap_delete_MessageHeader, METH_O, NULL}, + { "MessageHeader_encode", _wrap_MessageHeader_encode, METH_VARARGS, NULL}, + { "MessageHeader_decode", _wrap_MessageHeader_decode, METH_VARARGS, NULL}, + { "MessageHeader_version", _wrap_MessageHeader_version, METH_O, NULL}, + { "MessageHeader_setVersion", _wrap_MessageHeader_setVersion, METH_VARARGS, NULL}, + { "MessageHeader_traceID", _wrap_MessageHeader_traceID, METH_O, NULL}, + { "MessageHeader_setTraceID", _wrap_MessageHeader_setTraceID, METH_VARARGS, NULL}, + { "MessageHeader_srcGwNode", _wrap_MessageHeader_srcGwNode, METH_O, NULL}, + { "MessageHeader_setSrcGwNode", _wrap_MessageHeader_setSrcGwNode, METH_VARARGS, NULL}, + { "MessageHeader_dstGwNode", _wrap_MessageHeader_dstGwNode, METH_O, NULL}, + { "MessageHeader_setDstGwNode", _wrap_MessageHeader_setDstGwNode, METH_VARARGS, NULL}, + { "MessageHeader_packetType", _wrap_MessageHeader_packetType, METH_O, NULL}, + { "MessageHeader_setPacketType", _wrap_MessageHeader_setPacketType, METH_VARARGS, NULL}, + { "MessageHeader_ttl", _wrap_MessageHeader_ttl, METH_O, NULL}, + { "MessageHeader_setTTL", _wrap_MessageHeader_setTTL, METH_VARARGS, NULL}, + { "MessageHeader_ext", _wrap_MessageHeader_ext, METH_O, NULL}, + { "MessageHeader_setExt", _wrap_MessageHeader_setExt, METH_VARARGS, NULL}, + { "MessageHeader_optionalField", _wrap_MessageHeader_optionalField, METH_O, NULL}, + { "MessageHeader_setOptionalField", _wrap_MessageHeader_setOptionalField, METH_VARARGS, NULL}, + { "MessageHeader_length", _wrap_MessageHeader_length, METH_O, NULL}, + { "MessageHeader_isRespPacket", _wrap_MessageHeader_isRespPacket, METH_O, NULL}, + { "MessageHeader_setRespPacket", _wrap_MessageHeader_setRespPacket, METH_O, NULL}, + { "MessageHeader_srcP2PNodeIDView", _wrap_MessageHeader_srcP2PNodeIDView, METH_O, NULL}, + { "MessageHeader_dstP2PNodeIDView", _wrap_MessageHeader_dstP2PNodeIDView, METH_O, NULL}, + { "MessageHeader_routeType", _wrap_MessageHeader_routeType, METH_O, NULL}, + { "MessageHeader_setRouteType", _wrap_MessageHeader_setRouteType, METH_VARARGS, NULL}, + { "MessageHeader_hasOptionalField", _wrap_MessageHeader_hasOptionalField, METH_O, NULL}, + { "MessageHeader_swigregister", MessageHeader_swigregister, METH_O, NULL}, + { "delete_Message", _wrap_delete_Message, METH_O, NULL}, + { "Message_header", _wrap_Message_header, METH_O, NULL}, + { "Message_setHeader", _wrap_Message_setHeader, METH_VARARGS, NULL}, + { "Message_version", _wrap_Message_version, METH_O, " the overloaed implementation ==="}, + { "Message_setVersion", _wrap_Message_setVersion, METH_VARARGS, NULL}, + { "Message_packetType", _wrap_Message_packetType, METH_O, NULL}, + { "Message_setPacketType", _wrap_Message_setPacketType, METH_VARARGS, NULL}, + { "Message_seq", _wrap_Message_seq, METH_O, NULL}, + { "Message_setSeq", _wrap_Message_setSeq, METH_VARARGS, NULL}, + { "Message_ext", _wrap_Message_ext, METH_O, NULL}, + { "Message_setExt", _wrap_Message_setExt, METH_VARARGS, NULL}, + { "Message_isRespPacket", _wrap_Message_isRespPacket, METH_O, NULL}, + { "Message_setRespPacket", _wrap_Message_setRespPacket, METH_O, NULL}, + { "Message_length", _wrap_Message_length, METH_O, NULL}, + { "Message_payload", _wrap_Message_payload, METH_O, NULL}, + { "Message_payloadBuffer", _wrap_Message_payloadBuffer, METH_O, NULL}, + { "Message_setPayload", _wrap_Message_setPayload, METH_VARARGS, NULL}, + { "Message_setFrontMessage", _wrap_Message_setFrontMessage, METH_VARARGS, NULL}, + { "Message_frontMessage", _wrap_Message_frontMessage, METH_O, NULL}, + { "Message_encode", _wrap_Message_encode, METH_VARARGS, NULL}, + { "Message_decode", _wrap_Message_decode, METH_VARARGS, NULL}, + { "Message_swigregister", Message_swigregister, METH_O, NULL}, + { "delete_MessageHeaderBuilder", _wrap_delete_MessageHeaderBuilder, METH_O, NULL}, + { "MessageHeaderBuilder_build", _wrap_MessageHeaderBuilder_build, METH_VARARGS, NULL}, + { "MessageHeaderBuilder_swigregister", MessageHeaderBuilder_swigregister, METH_O, NULL}, + { "delete_MessageBuilder", _wrap_delete_MessageBuilder, METH_O, NULL}, + { "MessageBuilder_build", _wrap_MessageBuilder_build, METH_VARARGS, NULL}, + { "MessageBuilder_swigregister", MessageBuilder_swigregister, METH_O, NULL}, + { "delete_MessageOptionalHeaderBuilder", _wrap_delete_MessageOptionalHeaderBuilder, METH_O, NULL}, + { "MessageOptionalHeaderBuilder_build", _wrap_MessageOptionalHeaderBuilder_build, METH_VARARGS, NULL}, + { "MessageOptionalHeaderBuilder_swigregister", MessageOptionalHeaderBuilder_swigregister, METH_O, NULL}, + { "printOptionalField", _wrap_printOptionalField, METH_O, NULL}, + { "printMessage", _wrap_printMessage, METH_O, NULL}, + { "printWsMessage", _wrap_printWsMessage, METH_O, NULL}, + { "delete_MessagePayload", _wrap_delete_MessagePayload, METH_O, NULL}, + { "MessagePayload_encode", _wrap_MessagePayload_encode, METH_VARARGS, NULL}, + { "MessagePayload_decode", _wrap_MessagePayload_decode, METH_VARARGS, NULL}, + { "MessagePayload_version", _wrap_MessagePayload_version, METH_O, NULL}, + { "MessagePayload_setVersion", _wrap_MessagePayload_setVersion, METH_VARARGS, NULL}, + { "MessagePayload_data", _wrap_MessagePayload_data, METH_O, NULL}, + { "MessagePayload_dataBuffer", _wrap_MessagePayload_dataBuffer, METH_O, NULL}, + { "MessagePayload_setData", _wrap_MessagePayload_setData, METH_VARARGS, NULL}, + { "MessagePayload_seq", _wrap_MessagePayload_seq, METH_O, NULL}, + { "MessagePayload_setSeq", _wrap_MessagePayload_setSeq, METH_VARARGS, NULL}, + { "MessagePayload_length", _wrap_MessagePayload_length, METH_O, NULL}, + { "MessagePayload_traceID", _wrap_MessagePayload_traceID, METH_O, NULL}, + { "MessagePayload_setTraceID", _wrap_MessagePayload_setTraceID, METH_VARARGS, NULL}, + { "MessagePayload_ext", _wrap_MessagePayload_ext, METH_O, NULL}, + { "MessagePayload_setExt", _wrap_MessagePayload_setExt, METH_VARARGS, NULL}, + { "MessagePayload_setRespPacket", _wrap_MessagePayload_setRespPacket, METH_O, NULL}, + { "MessagePayload_isRespPacket", _wrap_MessagePayload_isRespPacket, METH_O, NULL}, + { "MessagePayload_swigregister", MessagePayload_swigregister, METH_O, NULL}, + { "delete_MessagePayloadBuilder", _wrap_delete_MessagePayloadBuilder, METH_O, NULL}, + { "MessagePayloadBuilder_build", _wrap_MessagePayloadBuilder_build, METH_VARARGS, NULL}, + { "MessagePayloadBuilder_swigregister", MessagePayloadBuilder_swigregister, METH_O, NULL}, + { "delete_IFrontClient", _wrap_delete_IFrontClient, METH_O, NULL}, + { "IFrontClient_onReceiveMessage", _wrap_IFrontClient_onReceiveMessage, METH_VARARGS, "\n" + ": receive message from gateway, call by gateway\n" + ":param _message:: received ppc message\n" + ":rtype: void\n" + ":return: void\n" + ""}, + { "IFrontClient_swigregister", IFrontClient_swigregister, METH_O, NULL}, + { "new_ErrorCallback", _wrap_new_ErrorCallback, METH_O, NULL}, + { "delete_ErrorCallback", _wrap_delete_ErrorCallback, METH_O, NULL}, + { "ErrorCallback_onError", _wrap_ErrorCallback_onError, METH_VARARGS, NULL}, + { "disown_ErrorCallback", _wrap_disown_ErrorCallback, METH_O, NULL}, + { "ErrorCallback_swigregister", ErrorCallback_swigregister, METH_O, NULL}, + { "ErrorCallback_swiginit", ErrorCallback_swiginit, METH_VARARGS, NULL}, + { "new_MessageDispatcherHandler", _wrap_new_MessageDispatcherHandler, METH_O, NULL}, + { "delete_MessageDispatcherHandler", _wrap_delete_MessageDispatcherHandler, METH_O, NULL}, + { "MessageDispatcherHandler_onMessage", _wrap_MessageDispatcherHandler_onMessage, METH_VARARGS, NULL}, + { "disown_MessageDispatcherHandler", _wrap_disown_MessageDispatcherHandler, METH_O, NULL}, + { "MessageDispatcherHandler_swigregister", MessageDispatcherHandler_swigregister, METH_O, NULL}, + { "MessageDispatcherHandler_swiginit", MessageDispatcherHandler_swiginit, METH_VARARGS, NULL}, + { "new_SendResponseHandler", _wrap_new_SendResponseHandler, METH_O, NULL}, + { "delete_SendResponseHandler", _wrap_delete_SendResponseHandler, METH_O, NULL}, + { "SendResponseHandler_sendResponse", _wrap_SendResponseHandler_sendResponse, METH_VARARGS, NULL}, + { "SendResponseHandler_swigregister", SendResponseHandler_swigregister, METH_O, NULL}, + { "SendResponseHandler_swiginit", SendResponseHandler_swiginit, METH_VARARGS, NULL}, + { "new_IMessageHandler", _wrap_new_IMessageHandler, METH_O, NULL}, + { "delete_IMessageHandler", _wrap_delete_IMessageHandler, METH_O, NULL}, + { "IMessageHandler_onMessage", _wrap_IMessageHandler_onMessage, METH_VARARGS, NULL}, + { "disown_IMessageHandler", _wrap_disown_IMessageHandler, METH_O, NULL}, + { "IMessageHandler_swigregister", IMessageHandler_swigregister, METH_O, NULL}, + { "IMessageHandler_swiginit", IMessageHandler_swiginit, METH_VARARGS, NULL}, + { "new_GetPeersInfoHandler", _wrap_new_GetPeersInfoHandler, METH_O, NULL}, + { "delete_GetPeersInfoHandler", _wrap_delete_GetPeersInfoHandler, METH_O, NULL}, + { "GetPeersInfoHandler_onPeersInfo", _wrap_GetPeersInfoHandler_onPeersInfo, METH_VARARGS, NULL}, + { "disown_GetPeersInfoHandler", _wrap_disown_GetPeersInfoHandler, METH_O, NULL}, + { "GetPeersInfoHandler_swigregister", GetPeersInfoHandler_swigregister, METH_O, NULL}, + { "GetPeersInfoHandler_swiginit", GetPeersInfoHandler_swiginit, METH_VARARGS, NULL}, + { "delete_IFront", _wrap_delete_IFront, METH_O, NULL}, + { "IFront_start", _wrap_IFront_start, METH_O, "\n" + "start the IFront\n" + "\n" + ":param front: the IFront to start\n" + ""}, + { "IFront_stop", _wrap_IFront_stop, METH_O, "\n" + "stop the IFront\n" + "\n" + ":param front: the IFront to stop\n" + ""}, + { "IFront_register_topic_handler", _wrap_IFront_register_topic_handler, METH_VARARGS, NULL}, + { "IFront_register_msg_handler", _wrap_IFront_register_msg_handler, METH_VARARGS, NULL}, + { "IFront_async_send_message", _wrap_IFront_async_send_message, METH_VARARGS, NULL}, + { "IFront_async_send_response", _wrap_IFront_async_send_response, METH_VARARGS, NULL}, + { "IFront_push", _wrap_IFront_push, METH_VARARGS, NULL}, + { "IFront_push_msg", _wrap_IFront_push_msg, METH_VARARGS, NULL}, + { "IFront_pop", _wrap_IFront_pop, METH_VARARGS, NULL}, + { "IFront_peek", _wrap_IFront_peek, METH_VARARGS, NULL}, + { "IFront_asyncGetPeers", _wrap_IFront_asyncGetPeers, METH_VARARGS, NULL}, + { "IFront_registerNodeInfo", _wrap_IFront_registerNodeInfo, METH_VARARGS, "\n" + "register the nodeInfo to the gateway\n" + ":type nodeInfo: ppc::protocol::INodeInfo::Ptr\n" + ":param nodeInfo: the nodeInfo\n" + ""}, + { "IFront_unRegisterNodeInfo", _wrap_IFront_unRegisterNodeInfo, METH_O, "unRegister the nodeInfo to the gateway"}, + { "IFront_nodeInfo", _wrap_IFront_nodeInfo, METH_O, NULL}, + { "IFront_registerTopic", _wrap_IFront_registerTopic, METH_VARARGS, "\n" + "register the topic\n" + "\n" + ":type topic: string\n" + ":param topic: the topic to register\n" + ""}, + { "IFront_unRegisterTopic", _wrap_IFront_unRegisterTopic, METH_VARARGS, "\n" + "unRegister the topic\n" + "\n" + ":type topic: string\n" + ":param topic: the topic to unregister\n" + ""}, + { "IFront_registerComponent", _wrap_IFront_registerComponent, METH_VARARGS, NULL}, + { "IFront_unRegisterComponent", _wrap_IFront_unRegisterComponent, METH_VARARGS, NULL}, + { "IFront_swigregister", IFront_swigregister, METH_O, NULL}, + { "delete_IFrontBuilder", _wrap_delete_IFrontBuilder, METH_O, NULL}, + { "IFrontBuilder_buildClient", _wrap_IFrontBuilder_buildClient, METH_VARARGS, NULL}, + { "IFrontBuilder_swigregister", IFrontBuilder_swigregister, METH_O, NULL}, + { "new_TransportBuilder", _wrap_new_TransportBuilder, METH_NOARGS, NULL}, + { "delete_TransportBuilder", _wrap_delete_TransportBuilder, METH_O, NULL}, + { "TransportBuilder_buildProTransport", _wrap_TransportBuilder_buildProTransport, METH_VARARGS, NULL}, + { "TransportBuilder_buildConfig", _wrap_TransportBuilder_buildConfig, METH_VARARGS, NULL}, + { "TransportBuilder_frontConfigBuilder", _wrap_TransportBuilder_frontConfigBuilder, METH_O, NULL}, + { "TransportBuilder_swigregister", TransportBuilder_swigregister, METH_O, NULL}, + { "TransportBuilder_swiginit", TransportBuilder_swiginit, METH_VARARGS, NULL}, + { "new_Transport", _wrap_new_Transport, METH_O, NULL}, + { "delete_Transport", _wrap_delete_Transport, METH_O, NULL}, + { "Transport_start", _wrap_Transport_start, METH_O, NULL}, + { "Transport_stop", _wrap_Transport_stop, METH_O, NULL}, + { "Transport_getFront", _wrap_Transport_getFront, METH_O, NULL}, + { "Transport_gateway", _wrap_Transport_gateway, METH_O, NULL}, + { "Transport_msgPayloadBuilder", _wrap_Transport_msgPayloadBuilder, METH_O, NULL}, + { "Transport_msgBuilder", _wrap_Transport_msgBuilder, METH_O, NULL}, + { "Transport_msgHeaderBuilder", _wrap_Transport_msgHeaderBuilder, METH_O, NULL}, + { "Transport_routeInfoBuilder", _wrap_Transport_routeInfoBuilder, METH_O, NULL}, + { "Transport_getConfig", _wrap_Transport_getConfig, METH_O, NULL}, + { "Transport_swigregister", Transport_swigregister, METH_O, NULL}, + { "Transport_swiginit", Transport_swiginit, METH_VARARGS, NULL}, + { NULL, NULL, 0, NULL } +}; + + +/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (BEGIN) -------- */ + +static void *_p_ppc__front__IFrontTo_p_ppc__front__IFrontClient(void *x, int *SWIGUNUSEDPARM(newmemory)) { + return (void *)((ppc::front::IFrontClient *) ((ppc::front::IFront *) x)); +} +static void *_p_std__shared_ptrT_ppc__front__IFront_tTo_p_std__shared_ptrT_ppc__front__IFrontClient_t(void *x, int *newmemory) { + *newmemory = SWIG_CAST_NEW_MEMORY; + return (void *) new std::shared_ptr< ppc::front::IFrontClient >(*(std::shared_ptr< ppc::front::IFront > *)x); +} +static swig_type_info _swigt__p_ConstPtr = {"_p_ConstPtr", "ConstPtr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_InputBuffer = {"_p_InputBuffer", "InputBuffer *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_OutputBuffer = {"_p_OutputBuffer", "OutputBuffer *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_PrevStdError = {"_p_PrevStdError", "PrevStdError *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_Ptr = {"_p_Ptr", "Ptr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_UniqueConstPtr = {"_p_UniqueConstPtr", "UniqueConstPtr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_UniquePtr = {"_p_UniquePtr", "UniquePtr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_allocator_type = {"_p_allocator_type", "allocator_type *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_bcos__Error = {"_p_bcos__Error", "bcos::Error *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_bcos__boostssl__EncodedMsg = {"_p_bcos__boostssl__EncodedMsg", "bcos::boostssl::EncodedMsg *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_bcos__boostssl__MessageFace__Ptr = {"_p_bcos__boostssl__MessageFace__Ptr", "bcos::boostssl::MessageFace::Ptr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_bcos__bytesConstRef = {"_p_bcos__bytesConstRef", "bcos::bytesConstRef *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_char = {"_p_char", "char *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_difference_type = {"_p_difference_type", "difference_type *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_int = {"_p_int", "int32_t *|int_fast16_t *|int_fast32_t *|int_least32_t *|intptr_t *|int *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_long_long = {"_p_long_long", "int64_t *|int_fast64_t *|int_least64_t *|intmax_t *|long long *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_p_PyObject = {"_p_p_PyObject", "PyObject **", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__ErrorCallback = {"_p_ppc__front__ErrorCallback", "ppc::front::ErrorCallback *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__FrontConfig = {"_p_ppc__front__FrontConfig", "ppc::front::FrontConfig *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__FrontConfigBuilder = {"_p_ppc__front__FrontConfigBuilder", "ppc::front::FrontConfigBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__GetPeersInfoHandler = {"_p_ppc__front__GetPeersInfoHandler", "ppc::front::GetPeersInfoHandler *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__IFront = {"_p_ppc__front__IFront", "ppc::front::IFront *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__IFrontBuilder = {"_p_ppc__front__IFrontBuilder", "ppc::front::IFrontBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__IFrontClient = {"_p_ppc__front__IFrontClient", "ppc::front::IFrontClient *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__IMessageHandler = {"_p_ppc__front__IMessageHandler", "ppc::front::IMessageHandler *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__MessageDispatcherHandler = {"_p_ppc__front__MessageDispatcherHandler", "ppc::front::MessageDispatcherHandler *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__front__SendResponseHandler = {"_p_ppc__front__SendResponseHandler", "ppc::front::SendResponseHandler *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__gateway__IGateway__Ptr = {"_p_ppc__gateway__IGateway__Ptr", "ppc::gateway::IGateway::Ptr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__EndPoint = {"_p_ppc__protocol__EndPoint", "ppc::protocol::EndPoint *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__GrpcConfig = {"_p_ppc__protocol__GrpcConfig", "ppc::protocol::GrpcConfig *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__GrpcServerConfig = {"_p_ppc__protocol__GrpcServerConfig", "ppc::protocol::GrpcServerConfig *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__INodeInfo__Ptr = {"_p_ppc__protocol__INodeInfo__Ptr", "ppc::protocol::INodeInfo::Ptr *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__Message = {"_p_ppc__protocol__Message", "ppc::protocol::Message *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessageBuilder = {"_p_ppc__protocol__MessageBuilder", "ppc::protocol::MessageBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessageHeader = {"_p_ppc__protocol__MessageHeader", "ppc::protocol::MessageHeader *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessageHeaderBuilder = {"_p_ppc__protocol__MessageHeaderBuilder", "ppc::protocol::MessageHeaderBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessageOptionalHeader = {"_p_ppc__protocol__MessageOptionalHeader", "ppc::protocol::MessageOptionalHeader *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessageOptionalHeaderBuilder = {"_p_ppc__protocol__MessageOptionalHeaderBuilder", "ppc::protocol::MessageOptionalHeaderBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessagePayload = {"_p_ppc__protocol__MessagePayload", "ppc::protocol::MessagePayload *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__MessagePayloadBuilder = {"_p_ppc__protocol__MessagePayloadBuilder", "ppc::protocol::MessagePayloadBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__protocol__RouteType = {"_p_ppc__protocol__RouteType", "ppc::protocol::RouteType *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__sdk__Transport = {"_p_ppc__sdk__Transport", "ppc::sdk::Transport *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_ppc__sdk__TransportBuilder = {"_p_ppc__sdk__TransportBuilder", "ppc::sdk::TransportBuilder *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_short = {"_p_short", "int16_t *|int_least16_t *|short *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_signed_char = {"_p_signed_char", "int8_t *|int_fast8_t *|int_least8_t *|signed char *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_size_type = {"_p_size_type", "size_type *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__allocatorT_signed_char_t = {"_p_std__allocatorT_signed_char_t", "std::vector< signed char >::allocator_type *|std::allocator< signed char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__allocatorT_unsigned_char_t = {"_p_std__allocatorT_unsigned_char_t", "std::vector< unsigned char >::allocator_type *|std::allocator< unsigned char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__exception = {"_p_std__exception", "std::exception *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__functionT_void_fF_t = {"_p_std__functionT_void_fF_t", "std::function< void () > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t = {"_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t", "ppc::protocol::ReceiveMsgFunc *|std::function< void (std::shared_ptr< bcos::Error >) > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t = {"_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t", "ppc::protocol::MessageCallback *|std::function< void (std::shared_ptr< bcos::Error >,std::shared_ptr< ppc::protocol::Message >,std::function< void (std::shared_ptr< bcos::bytes > &&) >) > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t = {"_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t", "ppc::protocol::SendResponseFunction *|std::function< void (std::shared_ptr< bcos::bytes > &&) > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t = {"_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t", "ppc::protocol::MessageDispatcherCallback *|std::function< void (std::shared_ptr< ppc::protocol::Message >) > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__invalid_argument = {"_p_std__invalid_argument", "std::invalid_argument *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_bcos__Error_t = {"_p_std__shared_ptrT_bcos__Error_t", "std::shared_ptr< bcos::Error > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_bcos__bytes_t = {"_p_std__shared_ptrT_bcos__bytes_t", "std::shared_ptr< bcos::bytes > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__ErrorCallback_t = {"_p_std__shared_ptrT_ppc__front__ErrorCallback_t", "std::shared_ptr< ppc::front::ErrorCallback > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t = {"_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t", "ppc::front::FrontConfigBuilder::Ptr *|std::shared_ptr< ppc::front::FrontConfigBuilder > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__FrontConfig_t = {"_p_std__shared_ptrT_ppc__front__FrontConfig_t", "std::shared_ptr< ppc::front::FrontConfig > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t = {"_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t", "std::shared_ptr< ppc::front::GetPeersInfoHandler > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t = {"_p_std__shared_ptrT_ppc__front__IFrontClient_t", "std::shared_ptr< ppc::front::IFrontClient > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IFront_t = {"_p_std__shared_ptrT_ppc__front__IFront_t", "std::shared_ptr< ppc::front::IFront > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t = {"_p_std__shared_ptrT_ppc__front__IMessageHandler_t", "std::shared_ptr< ppc::front::IMessageHandler > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t = {"_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t", "std::shared_ptr< ppc::front::MessageDispatcherHandler > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t = {"_p_std__shared_ptrT_ppc__gateway__IGateway_t", "std::shared_ptr< ppc::gateway::IGateway > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t = {"_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t", "std::shared_ptr< ppc::protocol::GrpcConfig > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t", "std::shared_ptr< ppc::protocol::MessageBuilder > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t", "std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t = {"_p_std__shared_ptrT_ppc__protocol__MessageHeader_t", "std::shared_ptr< ppc::protocol::MessageHeader > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t", "std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t = {"_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t", "std::shared_ptr< ppc::protocol::MessageOptionalHeader > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t", "std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessagePayload_t = {"_p_std__shared_ptrT_ppc__protocol__MessagePayload_t", "std::shared_ptr< ppc::protocol::MessagePayload > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__Message_t = {"_p_std__shared_ptrT_ppc__protocol__Message_t", "std::shared_ptr< ppc::protocol::Message > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__shared_ptrT_ppc__sdk__Transport_t = {"_p_std__shared_ptrT_ppc__sdk__Transport_t", "std::shared_ptr< ppc::sdk::Transport > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__string = {"_p_std__string", "std::string *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__string_view = {"_p_std__string_view", "std::string_view *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_int8_t_t = {"_p_std__vectorT_int8_t_t", "std::vector< int8_t > *|std::vector< signed char,std::allocator< signed char > > *|std::vector< signed char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t = {"_p_std__vectorT_std__string_std__allocatorT_std__string_t_t", "std::vector< std::string,std::allocator< std::string > > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_uint8_t_t = {"_p_std__vectorT_uint8_t_t", "bcos::bytes *|std::vector< uint8_t > *|std::vector< unsigned char,std::allocator< unsigned char > > *|std::vector< unsigned char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_swig__SwigPyIterator = {"_p_swig__SwigPyIterator", "swig::SwigPyIterator *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_unsigned_char = {"_p_unsigned_char", "bcos::byte *|uint8_t *|uint_fast8_t *|uint_least8_t *|unsigned char *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_unsigned_int = {"_p_unsigned_int", "uint32_t *|uint_fast16_t *|uint_fast32_t *|uint_least32_t *|uintptr_t *|unsigned int *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_unsigned_long_long = {"_p_unsigned_long_long", "uint64_t *|uint_fast64_t *|uint_least64_t *|uintmax_t *|unsigned long long *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_unsigned_short = {"_p_unsigned_short", "uint16_t *|uint_least16_t *|unsigned short *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_value_type = {"_p_value_type", "value_type *", 0, 0, (void*)0, 0}; + +static swig_type_info *swig_type_initial[] = { + &_swigt__p_ConstPtr, + &_swigt__p_InputBuffer, + &_swigt__p_OutputBuffer, + &_swigt__p_PrevStdError, + &_swigt__p_Ptr, + &_swigt__p_UniqueConstPtr, + &_swigt__p_UniquePtr, + &_swigt__p_allocator_type, + &_swigt__p_bcos__Error, + &_swigt__p_bcos__boostssl__EncodedMsg, + &_swigt__p_bcos__boostssl__MessageFace__Ptr, + &_swigt__p_bcos__bytesConstRef, + &_swigt__p_char, + &_swigt__p_difference_type, + &_swigt__p_int, + &_swigt__p_long_long, + &_swigt__p_p_PyObject, + &_swigt__p_ppc__front__ErrorCallback, + &_swigt__p_ppc__front__FrontConfig, + &_swigt__p_ppc__front__FrontConfigBuilder, + &_swigt__p_ppc__front__GetPeersInfoHandler, + &_swigt__p_ppc__front__IFront, + &_swigt__p_ppc__front__IFrontBuilder, + &_swigt__p_ppc__front__IFrontClient, + &_swigt__p_ppc__front__IMessageHandler, + &_swigt__p_ppc__front__MessageDispatcherHandler, + &_swigt__p_ppc__front__SendResponseHandler, + &_swigt__p_ppc__gateway__IGateway__Ptr, + &_swigt__p_ppc__protocol__EndPoint, + &_swigt__p_ppc__protocol__GrpcConfig, + &_swigt__p_ppc__protocol__GrpcServerConfig, + &_swigt__p_ppc__protocol__INodeInfo__Ptr, + &_swigt__p_ppc__protocol__Message, + &_swigt__p_ppc__protocol__MessageBuilder, + &_swigt__p_ppc__protocol__MessageHeader, + &_swigt__p_ppc__protocol__MessageHeaderBuilder, + &_swigt__p_ppc__protocol__MessageOptionalHeader, + &_swigt__p_ppc__protocol__MessageOptionalHeaderBuilder, + &_swigt__p_ppc__protocol__MessagePayload, + &_swigt__p_ppc__protocol__MessagePayloadBuilder, + &_swigt__p_ppc__protocol__RouteType, + &_swigt__p_ppc__sdk__Transport, + &_swigt__p_ppc__sdk__TransportBuilder, + &_swigt__p_short, + &_swigt__p_signed_char, + &_swigt__p_size_type, + &_swigt__p_std__allocatorT_signed_char_t, + &_swigt__p_std__allocatorT_unsigned_char_t, + &_swigt__p_std__exception, + &_swigt__p_std__functionT_void_fF_t, + &_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t, + &_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t, + &_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t, + &_swigt__p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t, + &_swigt__p_std__invalid_argument, + &_swigt__p_std__shared_ptrT_bcos__Error_t, + &_swigt__p_std__shared_ptrT_bcos__bytes_t, + &_swigt__p_std__shared_ptrT_ppc__front__ErrorCallback_t, + &_swigt__p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, + &_swigt__p_std__shared_ptrT_ppc__front__FrontConfig_t, + &_swigt__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, + &_swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t, + &_swigt__p_std__shared_ptrT_ppc__front__IFront_t, + &_swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t, + &_swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, + &_swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__MessagePayload_t, + &_swigt__p_std__shared_ptrT_ppc__protocol__Message_t, + &_swigt__p_std__shared_ptrT_ppc__sdk__Transport_t, + &_swigt__p_std__string, + &_swigt__p_std__string_view, + &_swigt__p_std__vectorT_int8_t_t, + &_swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t, + &_swigt__p_std__vectorT_uint8_t_t, + &_swigt__p_swig__SwigPyIterator, + &_swigt__p_unsigned_char, + &_swigt__p_unsigned_int, + &_swigt__p_unsigned_long_long, + &_swigt__p_unsigned_short, + &_swigt__p_value_type, +}; + +static swig_cast_info _swigc__p_ConstPtr[] = { {&_swigt__p_ConstPtr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_InputBuffer[] = { {&_swigt__p_InputBuffer, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_OutputBuffer[] = { {&_swigt__p_OutputBuffer, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_PrevStdError[] = { {&_swigt__p_PrevStdError, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_Ptr[] = { {&_swigt__p_Ptr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_UniqueConstPtr[] = { {&_swigt__p_UniqueConstPtr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_UniquePtr[] = { {&_swigt__p_UniquePtr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_allocator_type[] = { {&_swigt__p_allocator_type, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_bcos__Error[] = { {&_swigt__p_bcos__Error, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_bcos__boostssl__EncodedMsg[] = { {&_swigt__p_bcos__boostssl__EncodedMsg, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_bcos__boostssl__MessageFace__Ptr[] = { {&_swigt__p_bcos__boostssl__MessageFace__Ptr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_bcos__bytesConstRef[] = { {&_swigt__p_bcos__bytesConstRef, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_char[] = { {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_difference_type[] = { {&_swigt__p_difference_type, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_int[] = { {&_swigt__p_int, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_long_long[] = { {&_swigt__p_long_long, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_p_PyObject[] = { {&_swigt__p_p_PyObject, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__ErrorCallback[] = { {&_swigt__p_ppc__front__ErrorCallback, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__FrontConfig[] = { {&_swigt__p_ppc__front__FrontConfig, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__FrontConfigBuilder[] = { {&_swigt__p_ppc__front__FrontConfigBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__GetPeersInfoHandler[] = { {&_swigt__p_ppc__front__GetPeersInfoHandler, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__IFront[] = { {&_swigt__p_ppc__front__IFront, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__IFrontBuilder[] = { {&_swigt__p_ppc__front__IFrontBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__IFrontClient[] = { {&_swigt__p_ppc__front__IFrontClient, 0, 0, 0}, {&_swigt__p_ppc__front__IFront, _p_ppc__front__IFrontTo_p_ppc__front__IFrontClient, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__IMessageHandler[] = { {&_swigt__p_ppc__front__IMessageHandler, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__MessageDispatcherHandler[] = { {&_swigt__p_ppc__front__MessageDispatcherHandler, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__front__SendResponseHandler[] = { {&_swigt__p_ppc__front__SendResponseHandler, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__gateway__IGateway__Ptr[] = { {&_swigt__p_ppc__gateway__IGateway__Ptr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__EndPoint[] = { {&_swigt__p_ppc__protocol__EndPoint, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__GrpcConfig[] = { {&_swigt__p_ppc__protocol__GrpcConfig, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__GrpcServerConfig[] = { {&_swigt__p_ppc__protocol__GrpcServerConfig, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__INodeInfo__Ptr[] = { {&_swigt__p_ppc__protocol__INodeInfo__Ptr, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__Message[] = { {&_swigt__p_ppc__protocol__Message, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessageBuilder[] = { {&_swigt__p_ppc__protocol__MessageBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessageHeader[] = { {&_swigt__p_ppc__protocol__MessageHeader, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessageHeaderBuilder[] = { {&_swigt__p_ppc__protocol__MessageHeaderBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessageOptionalHeader[] = { {&_swigt__p_ppc__protocol__MessageOptionalHeader, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessageOptionalHeaderBuilder[] = { {&_swigt__p_ppc__protocol__MessageOptionalHeaderBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessagePayload[] = { {&_swigt__p_ppc__protocol__MessagePayload, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__MessagePayloadBuilder[] = { {&_swigt__p_ppc__protocol__MessagePayloadBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__protocol__RouteType[] = { {&_swigt__p_ppc__protocol__RouteType, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__sdk__Transport[] = { {&_swigt__p_ppc__sdk__Transport, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_ppc__sdk__TransportBuilder[] = { {&_swigt__p_ppc__sdk__TransportBuilder, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_short[] = { {&_swigt__p_short, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_signed_char[] = { {&_swigt__p_signed_char, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_size_type[] = { {&_swigt__p_size_type, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__allocatorT_signed_char_t[] = { {&_swigt__p_std__allocatorT_signed_char_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__allocatorT_unsigned_char_t[] = { {&_swigt__p_std__allocatorT_unsigned_char_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__exception[] = { {&_swigt__p_std__exception, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__functionT_void_fF_t[] = { {&_swigt__p_std__functionT_void_fF_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t[] = { {&_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t[] = { {&_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t[] = { {&_swigt__p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t[] = { {&_swigt__p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__invalid_argument[] = { {&_swigt__p_std__invalid_argument, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_bcos__Error_t[] = { {&_swigt__p_std__shared_ptrT_bcos__Error_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_bcos__bytes_t[] = { {&_swigt__p_std__shared_ptrT_bcos__bytes_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__ErrorCallback_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__FrontConfig_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__FrontConfig_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IFrontClient_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t, 0, 0, 0}, {&_swigt__p_std__shared_ptrT_ppc__front__IFront_t, _p_std__shared_ptrT_ppc__front__IFront_tTo_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IFront_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__IFront_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IMessageHandler_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t[] = { {&_swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__gateway__IGateway_t[] = { {&_swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeader_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessagePayload_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__Message_t[] = { {&_swigt__p_std__shared_ptrT_ppc__protocol__Message_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__shared_ptrT_ppc__sdk__Transport_t[] = { {&_swigt__p_std__shared_ptrT_ppc__sdk__Transport_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__string[] = { {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__string_view[] = { {&_swigt__p_std__string_view, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_int8_t_t[] = { {&_swigt__p_std__vectorT_int8_t_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_std__string_std__allocatorT_std__string_t_t[] = { {&_swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_uint8_t_t[] = { {&_swigt__p_std__vectorT_uint8_t_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_swig__SwigPyIterator[] = { {&_swigt__p_swig__SwigPyIterator, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_unsigned_char[] = { {&_swigt__p_unsigned_char, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_unsigned_int[] = { {&_swigt__p_unsigned_int, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_unsigned_long_long[] = { {&_swigt__p_unsigned_long_long, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_unsigned_short[] = { {&_swigt__p_unsigned_short, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_value_type[] = { {&_swigt__p_value_type, 0, 0, 0},{0, 0, 0, 0}}; + +static swig_cast_info *swig_cast_initial[] = { + _swigc__p_ConstPtr, + _swigc__p_InputBuffer, + _swigc__p_OutputBuffer, + _swigc__p_PrevStdError, + _swigc__p_Ptr, + _swigc__p_UniqueConstPtr, + _swigc__p_UniquePtr, + _swigc__p_allocator_type, + _swigc__p_bcos__Error, + _swigc__p_bcos__boostssl__EncodedMsg, + _swigc__p_bcos__boostssl__MessageFace__Ptr, + _swigc__p_bcos__bytesConstRef, + _swigc__p_char, + _swigc__p_difference_type, + _swigc__p_int, + _swigc__p_long_long, + _swigc__p_p_PyObject, + _swigc__p_ppc__front__ErrorCallback, + _swigc__p_ppc__front__FrontConfig, + _swigc__p_ppc__front__FrontConfigBuilder, + _swigc__p_ppc__front__GetPeersInfoHandler, + _swigc__p_ppc__front__IFront, + _swigc__p_ppc__front__IFrontBuilder, + _swigc__p_ppc__front__IFrontClient, + _swigc__p_ppc__front__IMessageHandler, + _swigc__p_ppc__front__MessageDispatcherHandler, + _swigc__p_ppc__front__SendResponseHandler, + _swigc__p_ppc__gateway__IGateway__Ptr, + _swigc__p_ppc__protocol__EndPoint, + _swigc__p_ppc__protocol__GrpcConfig, + _swigc__p_ppc__protocol__GrpcServerConfig, + _swigc__p_ppc__protocol__INodeInfo__Ptr, + _swigc__p_ppc__protocol__Message, + _swigc__p_ppc__protocol__MessageBuilder, + _swigc__p_ppc__protocol__MessageHeader, + _swigc__p_ppc__protocol__MessageHeaderBuilder, + _swigc__p_ppc__protocol__MessageOptionalHeader, + _swigc__p_ppc__protocol__MessageOptionalHeaderBuilder, + _swigc__p_ppc__protocol__MessagePayload, + _swigc__p_ppc__protocol__MessagePayloadBuilder, + _swigc__p_ppc__protocol__RouteType, + _swigc__p_ppc__sdk__Transport, + _swigc__p_ppc__sdk__TransportBuilder, + _swigc__p_short, + _swigc__p_signed_char, + _swigc__p_size_type, + _swigc__p_std__allocatorT_signed_char_t, + _swigc__p_std__allocatorT_unsigned_char_t, + _swigc__p_std__exception, + _swigc__p_std__functionT_void_fF_t, + _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t, + _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t, + _swigc__p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t, + _swigc__p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t, + _swigc__p_std__invalid_argument, + _swigc__p_std__shared_ptrT_bcos__Error_t, + _swigc__p_std__shared_ptrT_bcos__bytes_t, + _swigc__p_std__shared_ptrT_ppc__front__ErrorCallback_t, + _swigc__p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, + _swigc__p_std__shared_ptrT_ppc__front__FrontConfig_t, + _swigc__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, + _swigc__p_std__shared_ptrT_ppc__front__IFrontClient_t, + _swigc__p_std__shared_ptrT_ppc__front__IFront_t, + _swigc__p_std__shared_ptrT_ppc__front__IMessageHandler_t, + _swigc__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, + _swigc__p_std__shared_ptrT_ppc__gateway__IGateway_t, + _swigc__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeader_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, + _swigc__p_std__shared_ptrT_ppc__protocol__MessagePayload_t, + _swigc__p_std__shared_ptrT_ppc__protocol__Message_t, + _swigc__p_std__shared_ptrT_ppc__sdk__Transport_t, + _swigc__p_std__string, + _swigc__p_std__string_view, + _swigc__p_std__vectorT_int8_t_t, + _swigc__p_std__vectorT_std__string_std__allocatorT_std__string_t_t, + _swigc__p_std__vectorT_uint8_t_t, + _swigc__p_swig__SwigPyIterator, + _swigc__p_unsigned_char, + _swigc__p_unsigned_int, + _swigc__p_unsigned_long_long, + _swigc__p_unsigned_short, + _swigc__p_value_type, +}; + + +/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (END) -------- */ + +static swig_const_info swig_const_table[] = { +{0, 0, 0, 0.0, 0, 0}}; + +#ifdef __cplusplus +} +#endif +/* ----------------------------------------------------------------------------- + * Type initialization: + * This problem is tough by the requirement that no dynamic + * memory is used. Also, since swig_type_info structures store pointers to + * swig_cast_info structures and swig_cast_info structures store pointers back + * to swig_type_info structures, we need some lookup code at initialization. + * The idea is that swig generates all the structures that are needed. + * The runtime then collects these partially filled structures. + * The SWIG_InitializeModule function takes these initial arrays out of + * swig_module, and does all the lookup, filling in the swig_module.types + * array with the correct data and linking the correct swig_cast_info + * structures together. + * + * The generated swig_type_info structures are assigned statically to an initial + * array. We just loop through that array, and handle each type individually. + * First we lookup if this type has been already loaded, and if so, use the + * loaded structure instead of the generated one. Then we have to fill in the + * cast linked list. The cast data is initially stored in something like a + * two-dimensional array. Each row corresponds to a type (there are the same + * number of rows as there are in the swig_type_initial array). Each entry in + * a column is one of the swig_cast_info structures for that type. + * The cast_initial array is actually an array of arrays, because each row has + * a variable number of columns. So to actually build the cast linked list, + * we find the array of casts associated with the type, and loop through it + * adding the casts to the list. The one last trick we need to do is making + * sure the type pointer in the swig_cast_info struct is correct. + * + * First off, we lookup the cast->type name to see if it is already loaded. + * There are three cases to handle: + * 1) If the cast->type has already been loaded AND the type we are adding + * casting info to has not been loaded (it is in this module), THEN we + * replace the cast->type pointer with the type pointer that has already + * been loaded. + * 2) If BOTH types (the one we are adding casting info to, and the + * cast->type) are loaded, THEN the cast info has already been loaded by + * the previous module so we just ignore it. + * 3) Finally, if cast->type has not already been loaded, then we add that + * swig_cast_info to the linked list (because the cast->type) pointer will + * be correct. + * ----------------------------------------------------------------------------- */ + +#ifdef __cplusplus +extern "C" { +#if 0 +} /* c-mode */ +#endif +#endif + +#if 0 +#define SWIGRUNTIME_DEBUG +#endif + +#ifndef SWIG_INIT_CLIENT_DATA_TYPE +#define SWIG_INIT_CLIENT_DATA_TYPE void * +#endif + +SWIGRUNTIME void +SWIG_InitializeModule(SWIG_INIT_CLIENT_DATA_TYPE clientdata) { + size_t i; + swig_module_info *module_head, *iter; + int init; + + /* check to see if the circular list has been setup, if not, set it up */ + if (swig_module.next==0) { + /* Initialize the swig_module */ + swig_module.type_initial = swig_type_initial; + swig_module.cast_initial = swig_cast_initial; + swig_module.next = &swig_module; + init = 1; + } else { + init = 0; + } + + /* Try and load any already created modules */ + module_head = SWIG_GetModule(clientdata); + if (!module_head) { + /* This is the first module loaded for this interpreter */ + /* so set the swig module into the interpreter */ + SWIG_SetModule(clientdata, &swig_module); + } else { + /* the interpreter has loaded a SWIG module, but has it loaded this one? */ + iter=module_head; + do { + if (iter==&swig_module) { + /* Our module is already in the list, so there's nothing more to do. */ + return; + } + iter=iter->next; + } while (iter!= module_head); + + /* otherwise we must add our module into the list */ + swig_module.next = module_head->next; + module_head->next = &swig_module; + } + + /* When multiple interpreters are used, a module could have already been initialized in + a different interpreter, but not yet have a pointer in this interpreter. + In this case, we do not want to continue adding types... everything should be + set up already */ + if (init == 0) return; + + /* Now work on filling in swig_module.types */ +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: size %lu\n", (unsigned long)swig_module.size); +#endif + for (i = 0; i < swig_module.size; ++i) { + swig_type_info *type = 0; + swig_type_info *ret; + swig_cast_info *cast; + +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: type %lu %s\n", (unsigned long)i, swig_module.type_initial[i]->name); +#endif + + /* if there is another module already loaded */ + if (swig_module.next != &swig_module) { + type = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, swig_module.type_initial[i]->name); + } + if (type) { + /* Overwrite clientdata field */ +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: found type %s\n", type->name); +#endif + if (swig_module.type_initial[i]->clientdata) { + type->clientdata = swig_module.type_initial[i]->clientdata; +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: found and overwrite type %s \n", type->name); +#endif + } + } else { + type = swig_module.type_initial[i]; + } + + /* Insert casting types */ + cast = swig_module.cast_initial[i]; + while (cast->type) { + /* Don't need to add information already in the list */ + ret = 0; +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: look cast %s\n", cast->type->name); +#endif + if (swig_module.next != &swig_module) { + ret = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, cast->type->name); +#ifdef SWIGRUNTIME_DEBUG + if (ret) printf("SWIG_InitializeModule: found cast %s\n", ret->name); +#endif + } + if (ret) { + if (type == swig_module.type_initial[i]) { +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: skip old type %s\n", ret->name); +#endif + cast->type = ret; + ret = 0; + } else { + /* Check for casting already in the list */ + swig_cast_info *ocast = SWIG_TypeCheck(ret->name, type); +#ifdef SWIGRUNTIME_DEBUG + if (ocast) printf("SWIG_InitializeModule: skip old cast %s\n", ret->name); +#endif + if (!ocast) ret = 0; + } + } + + if (!ret) { +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: adding cast %s\n", cast->type->name); +#endif + if (type->cast) { + type->cast->prev = cast; + cast->next = type->cast; + } + type->cast = cast; + } + cast++; + } + /* Set entry in modules->types array equal to the type */ + swig_module.types[i] = type; + } + swig_module.types[i] = 0; + +#ifdef SWIGRUNTIME_DEBUG + printf("**** SWIG_InitializeModule: Cast List ******\n"); + for (i = 0; i < swig_module.size; ++i) { + int j = 0; + swig_cast_info *cast = swig_module.cast_initial[i]; + printf("SWIG_InitializeModule: type %lu %s\n", (unsigned long)i, swig_module.type_initial[i]->name); + while (cast->type) { + printf("SWIG_InitializeModule: cast type %s\n", cast->type->name); + cast++; + ++j; + } + printf("---- Total casts: %d\n",j); + } + printf("**** SWIG_InitializeModule: Cast List ******\n"); +#endif +} + +/* This function will propagate the clientdata field of type to +* any new swig_type_info structures that have been added into the list +* of equivalent types. It is like calling +* SWIG_TypeClientData(type, clientdata) a second time. +*/ +SWIGRUNTIME void +SWIG_PropagateClientData(void) { + size_t i; + swig_cast_info *equiv; + static int init_run = 0; + + if (init_run) return; + init_run = 1; + + for (i = 0; i < swig_module.size; i++) { + if (swig_module.types[i]->clientdata) { + equiv = swig_module.types[i]->cast; + while (equiv) { + if (!equiv->converter) { + if (equiv->type && !equiv->type->clientdata) + SWIG_TypeClientData(equiv->type, swig_module.types[i]->clientdata); + } + equiv = equiv->next; + } + } + } +} + +#ifdef __cplusplus +#if 0 +{ + /* c-mode */ +#endif +} +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + /* ----------------------------------------------------------------------------- + * constants/methods manipulation + * ----------------------------------------------------------------------------- */ + + /* Install Constants */ + SWIGINTERN void + SWIG_Python_InstallConstants(PyObject *d, swig_const_info constants[]) { + PyObject *obj = 0; + size_t i; + for (i = 0; constants[i].type; ++i) { + switch(constants[i].type) { + case SWIG_PY_POINTER: + obj = SWIG_InternalNewPointerObj(constants[i].pvalue, *(constants[i]).ptype,0); + break; + case SWIG_PY_BINARY: + obj = SWIG_NewPackedObj(constants[i].pvalue, constants[i].lvalue, *(constants[i].ptype)); + break; + default: + obj = 0; + break; + } + if (obj) { + PyDict_SetItemString(d, constants[i].name, obj); + Py_DECREF(obj); + } + } + } + + /* ----------------------------------------------------------------------------- + * Patch %callback methods' docstrings to hold the callback ptrs + * -----------------------------------------------------------------------------*/ + + SWIGINTERN void + SWIG_Python_FixMethods(PyMethodDef *methods, const swig_const_info *const_table, swig_type_info **types, swig_type_info **types_initial) { + size_t i; + for (i = 0; methods[i].ml_name; ++i) { + const char *c = methods[i].ml_doc; + if (!c) continue; + c = strstr(c, "swig_ptr: "); + if (c) { + int j; + const swig_const_info *ci = 0; + const char *name = c + 10; + for (j = 0; const_table[j].type; ++j) { + if (strncmp(const_table[j].name, name, + strlen(const_table[j].name)) == 0) { + ci = &(const_table[j]); + break; + } + } + if (ci) { + void *ptr = (ci->type == SWIG_PY_POINTER) ? ci->pvalue : 0; + if (ptr) { + size_t shift = (ci->ptype) - types; + swig_type_info *ty = types_initial[shift]; + size_t ldoc = (c - methods[i].ml_doc); + size_t lptr = strlen(ty->name)+2*sizeof(void*)+2; + char *ndoc = (char*)malloc(ldoc + lptr + 10); + if (ndoc) { + char *buff = ndoc; + memcpy(buff, methods[i].ml_doc, ldoc); + buff += ldoc; + memcpy(buff, "swig_ptr: ", 10); + buff += 10; + SWIG_PackVoidPtr(buff, ptr, ty->name, lptr); + methods[i].ml_doc = ndoc; + } + } + } + } + } + } + +#ifdef __cplusplus +} +#endif + + + + +/* -----------------------------------------------------------------------------* + * Partial Init method + * -----------------------------------------------------------------------------*/ + +#ifdef __cplusplus +extern "C" +#endif + +SWIGEXPORT +#if PY_VERSION_HEX >= 0x03000000 +PyObject* +#else +void +#endif +SWIG_init(void) { + PyObject *m, *d, *md, *globals; + +#if PY_VERSION_HEX >= 0x03000000 + static struct PyModuleDef SWIG_module = { + PyModuleDef_HEAD_INIT, + SWIG_name, + NULL, + -1, + SwigMethods, + NULL, + NULL, + NULL, + NULL + }; +#endif + +#if defined(SWIGPYTHON_BUILTIN) + static SwigPyClientData SwigPyObject_clientdata = { + 0, 0, 0, 0, 0, 0, 0 + }; + static PyGetSetDef this_getset_def = { + (char *)"this", &SwigPyBuiltin_ThisClosure, NULL, NULL, NULL + }; + static SwigPyGetSet thisown_getset_closure = { + SwigPyObject_own, + SwigPyObject_own + }; + static PyGetSetDef thisown_getset_def = { + (char *)"thisown", SwigPyBuiltin_GetterClosure, SwigPyBuiltin_SetterClosure, NULL, &thisown_getset_closure + }; + PyTypeObject *builtin_pytype; + int builtin_base_count; + swig_type_info *builtin_basetype; + PyObject *tuple; + PyGetSetDescrObject *static_getset; + PyTypeObject *metatype; + PyTypeObject *swigpyobject; + SwigPyClientData *cd; + PyObject *public_interface, *public_symbol; + PyObject *this_descr; + PyObject *thisown_descr; + PyObject *self = 0; + int i; + + (void)builtin_pytype; + (void)builtin_base_count; + (void)builtin_basetype; + (void)tuple; + (void)static_getset; + (void)self; + + /* Metaclass is used to implement static member variables */ + metatype = SwigPyObjectType(); + assert(metatype); +#endif + + (void)globals; + + /* Create singletons now to avoid potential deadlocks with multi-threaded usage after module initialization */ + SWIG_This(); + SWIG_Python_TypeCache(); + SwigPyPacked_type(); +#ifndef SWIGPYTHON_BUILTIN + SwigPyObject_type(); +#endif + + /* Fix SwigMethods to carry the callback ptrs when needed */ + SWIG_Python_FixMethods(SwigMethods, swig_const_table, swig_types, swig_type_initial); + +#if PY_VERSION_HEX >= 0x03000000 + m = PyModule_Create(&SWIG_module); +#else + m = Py_InitModule(SWIG_name, SwigMethods); +#endif + + md = d = PyModule_GetDict(m); + (void)md; + + SWIG_InitializeModule(0); + +#ifdef SWIGPYTHON_BUILTIN + swigpyobject = SwigPyObject_TypeOnce(); + + SwigPyObject_stype = SWIG_MangledTypeQuery("_p_SwigPyObject"); + assert(SwigPyObject_stype); + cd = (SwigPyClientData*) SwigPyObject_stype->clientdata; + if (!cd) { + SwigPyObject_stype->clientdata = &SwigPyObject_clientdata; + SwigPyObject_clientdata.pytype = swigpyobject; + } else if (swigpyobject->tp_basicsize != cd->pytype->tp_basicsize) { + PyErr_SetString(PyExc_RuntimeError, "Import error: attempted to load two incompatible swig-generated modules."); +# if PY_VERSION_HEX >= 0x03000000 + return NULL; +# else + return; +# endif + } + + /* All objects have a 'this' attribute */ + this_descr = PyDescr_NewGetSet(SwigPyObject_type(), &this_getset_def); + (void)this_descr; + + /* All objects have a 'thisown' attribute */ + thisown_descr = PyDescr_NewGetSet(SwigPyObject_type(), &thisown_getset_def); + (void)thisown_descr; + + public_interface = PyList_New(0); + public_symbol = 0; + (void)public_symbol; + + PyDict_SetItemString(md, "__all__", public_interface); + Py_DECREF(public_interface); + for (i = 0; SwigMethods[i].ml_name != NULL; ++i) + SwigPyBuiltin_AddPublicSymbol(public_interface, SwigMethods[i].ml_name); + for (i = 0; swig_const_table[i].name != 0; ++i) + SwigPyBuiltin_AddPublicSymbol(public_interface, swig_const_table[i].name); +#endif + + SWIG_InstallConstants(d,swig_const_table); + + SWIG_Python_SetConstant(d, "SHARED_PTR_DISOWN",SWIG_From_int(static_cast< int >(0))); + + // thread safe initialization + swig::container_owner_attribute(); + + SWIG_Python_SetConstant(d, "FrontMsgExtFlag_Response",SWIG_From_int(static_cast< int >(ppc::protocol::FrontMsgExtFlag::Response))); + SWIG_Python_SetConstant(d, "SDKMode_AIR",SWIG_From_int(static_cast< int >(ppc::sdk::SDKMode::AIR))); + SWIG_Python_SetConstant(d, "SDKMode_PRO",SWIG_From_int(static_cast< int >(ppc::sdk::SDKMode::PRO))); +#if PY_VERSION_HEX >= 0x03000000 + return m; +#else + return; +#endif +} + diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.h b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.h new file mode 100644 index 00000000..36d8b00b --- /dev/null +++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.h @@ -0,0 +1,184 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.2.1 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + +#ifndef SWIG_wedpr_python_transport_WRAP_H_ +#define SWIG_wedpr_python_transport_WRAP_H_ + +#include +#include + + +class SwigDirector_ErrorCallback : public ppc::front::ErrorCallback, public Swig::Director { + +public: + SwigDirector_ErrorCallback(PyObject *self); + virtual ~SwigDirector_ErrorCallback(); + virtual void onError(bcos::Error::Ptr error); + +/* Internal director utilities */ +public: + bool swig_get_inner(const char *swig_protected_method_name) const { + std::map::const_iterator iv = swig_inner.find(swig_protected_method_name); + return (iv != swig_inner.end() ? iv->second : false); + } + void swig_set_inner(const char *swig_protected_method_name, bool swig_val) const { + swig_inner[swig_protected_method_name] = swig_val; + } +private: + mutable std::map swig_inner; + +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) +/* VTable implementation */ + PyObject *swig_get_method(size_t method_index, const char *method_name) const { + PyObject *method = vtable[method_index]; + if (!method) { + swig::SwigVar_PyObject name = SWIG_Python_str_FromChar(method_name); + method = PyObject_GetAttr(swig_get_self(), name); + if (!method) { + std::string msg = "Method in class ErrorCallback doesn't exist, undefined "; + msg += method_name; + Swig::DirectorMethodException::raise(msg.c_str()); + } + vtable[method_index] = method; + } + return method; + } +private: + mutable swig::SwigVar_PyObject vtable[1]; +#endif + +}; + + +class SwigDirector_MessageDispatcherHandler : public ppc::front::MessageDispatcherHandler, public Swig::Director { + +public: + SwigDirector_MessageDispatcherHandler(PyObject *self); + virtual ~SwigDirector_MessageDispatcherHandler(); + virtual void onMessage(ppc::protocol::Message::Ptr msg); + +/* Internal director utilities */ +public: + bool swig_get_inner(const char *swig_protected_method_name) const { + std::map::const_iterator iv = swig_inner.find(swig_protected_method_name); + return (iv != swig_inner.end() ? iv->second : false); + } + void swig_set_inner(const char *swig_protected_method_name, bool swig_val) const { + swig_inner[swig_protected_method_name] = swig_val; + } +private: + mutable std::map swig_inner; + +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) +/* VTable implementation */ + PyObject *swig_get_method(size_t method_index, const char *method_name) const { + PyObject *method = vtable[method_index]; + if (!method) { + swig::SwigVar_PyObject name = SWIG_Python_str_FromChar(method_name); + method = PyObject_GetAttr(swig_get_self(), name); + if (!method) { + std::string msg = "Method in class MessageDispatcherHandler doesn't exist, undefined "; + msg += method_name; + Swig::DirectorMethodException::raise(msg.c_str()); + } + vtable[method_index] = method; + } + return method; + } +private: + mutable swig::SwigVar_PyObject vtable[1]; +#endif + +}; + + +class SwigDirector_IMessageHandler : public ppc::front::IMessageHandler, public Swig::Director { + +public: + SwigDirector_IMessageHandler(PyObject *self); + virtual ~SwigDirector_IMessageHandler(); + virtual void onMessage(bcos::Error::Ptr e,ppc::protocol::Message::Ptr msg,ppc::front::SendResponseHandler sendResponseHandler); + +/* Internal director utilities */ +public: + bool swig_get_inner(const char *swig_protected_method_name) const { + std::map::const_iterator iv = swig_inner.find(swig_protected_method_name); + return (iv != swig_inner.end() ? iv->second : false); + } + void swig_set_inner(const char *swig_protected_method_name, bool swig_val) const { + swig_inner[swig_protected_method_name] = swig_val; + } +private: + mutable std::map swig_inner; + +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) +/* VTable implementation */ + PyObject *swig_get_method(size_t method_index, const char *method_name) const { + PyObject *method = vtable[method_index]; + if (!method) { + swig::SwigVar_PyObject name = SWIG_Python_str_FromChar(method_name); + method = PyObject_GetAttr(swig_get_self(), name); + if (!method) { + std::string msg = "Method in class IMessageHandler doesn't exist, undefined "; + msg += method_name; + Swig::DirectorMethodException::raise(msg.c_str()); + } + vtable[method_index] = method; + } + return method; + } +private: + mutable swig::SwigVar_PyObject vtable[1]; +#endif + +}; + + +class SwigDirector_GetPeersInfoHandler : public ppc::front::GetPeersInfoHandler, public Swig::Director { + +public: + SwigDirector_GetPeersInfoHandler(PyObject *self); + virtual ~SwigDirector_GetPeersInfoHandler(); + virtual void onPeersInfo(bcos::Error::Ptr e,std::string const &peersInfo); + +/* Internal director utilities */ +public: + bool swig_get_inner(const char *swig_protected_method_name) const { + std::map::const_iterator iv = swig_inner.find(swig_protected_method_name); + return (iv != swig_inner.end() ? iv->second : false); + } + void swig_set_inner(const char *swig_protected_method_name, bool swig_val) const { + swig_inner[swig_protected_method_name] = swig_val; + } +private: + mutable std::map swig_inner; + +#if defined(SWIG_PYTHON_DIRECTOR_VTABLE) +/* VTable implementation */ + PyObject *swig_get_method(size_t method_index, const char *method_name) const { + PyObject *method = vtable[method_index]; + if (!method) { + swig::SwigVar_PyObject name = SWIG_Python_str_FromChar(method_name); + method = PyObject_GetAttr(swig_get_self(), name); + if (!method) { + std::string msg = "Method in class GetPeersInfoHandler doesn't exist, undefined "; + msg += method_name; + Swig::DirectorMethodException::raise(msg.c_str()); + } + vtable[method_index] = method; + } + return method; + } +private: + mutable swig::SwigVar_PyObject vtable[1]; +#endif + +}; + + +#endif diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i index 60e4f350..242d681f 100644 --- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i +++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i @@ -1,18 +1,19 @@ %define MODULEIMPORT " -from utils.lib_loader import LibLoader -_wedpr_python_transport = LibLoader.load_lib() +# Import the low-level C/C++ module +from libs import _wedpr_python_transport " %enddef %module(moduleimport=MODULEIMPORT) wedpr_python_transport +%module(directors="1") wedpr_python_transport %include %include %include %include %include - +%include "typemaps.i" // shared_ptr definition %shared_ptr(ppc::front::FrontConfig); @@ -40,6 +41,8 @@ _wedpr_python_transport = LibLoader.load_lib() %{ #define SWIG_FILE_WITH_INIT +#include +#include #include #include #include @@ -130,6 +133,14 @@ namespace bcos{ %template(ubytes) std::vector; %template(ibytes) std::vector; +%include +%pybuffer_binary(char* data, uint64_t length) +%pybuffer_binary(char* payload, uint64_t payloadSize) + +%typemap(out) OutputBuffer { + $result = PyBytes_FromStringAndSize((const char *)$1.data, $1.len); +} + /// callbacks %feature("director") ppc::front::ErrorCallback; %feature("director") ppc::front::MessageDispatcherHandler; From 71269067490fd48742e516fe8bc34475cb36c471 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 11 Oct 2024 21:29:25 +0800 Subject: [PATCH 052/120] send message by component support not specify the dstInst (#49) * check the parameters * send message by component support not specify the dstInst --- .../gateway/router/PeerRouterTable.cpp | 45 ++++++++++++---- .../gateway/router/PeerRouterTable.h | 3 ++ .../sdk-wrapper/java/bindings/build.gradle | 2 + .../wedpr/sdk/jni/demo/TransportDemo.java | 3 +- .../sdk/jni/transport/IMessageBuilder.java | 7 +++ .../sdk/jni/transport/impl/TransportImpl.java | 54 ++++++++++++++++++- 6 files changed, 102 insertions(+), 12 deletions(-) diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp index 9685e82c..3564f832 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp @@ -190,19 +190,47 @@ GatewayNodeInfos PeerRouterTable::selectRouterByAgency(Message::Ptr const& msg) return it->second; } +// Note: selectRouterByComponent support not specified the dstInst GatewayNodeInfos PeerRouterTable::selectRouterByComponent(Message::Ptr const& msg) const { GatewayNodeInfos result; - bcos::ReadGuard l(x_mutex); - auto it = m_agency2GatewayInfos.find(msg->header()->optionalField()->dstInst()); - // no router found - if (it == m_agency2GatewayInfos.end()) + auto dstInst = msg->header()->optionalField()->dstInst(); + std::vector selectedRouterInfos; { - return result; + bcos::ReadGuard l(x_mutex); + if (dstInst.size() > 0) + { + // specified the dstInst + auto it = m_agency2GatewayInfos.find(dstInst); + // no router found + if (it == m_agency2GatewayInfos.end()) + { + return result; + } + selectedRouterInfos.emplace_back(it->second); + } + else + { + // the dstInst not specified, query from all agencies + for (auto const& it : m_agency2GatewayInfos) + { + selectedRouterInfos.emplace_back(it.second); + } + } } - auto const& gatewayInfos = it->second; + for (auto const& it : selectedRouterInfos) + { + selectRouterByComponent(result, msg, it); + } + return result; +} + + +void PeerRouterTable::selectRouterByComponent(GatewayNodeInfos& choosedGateway, + Message::Ptr const& msg, GatewayNodeInfos const& singleAgencyGatewayInfos) const +{ // foreach all gateways to find the component - for (auto const& it : gatewayInfos) + for (auto const& it : singleAgencyGatewayInfos) { auto const& nodeListInfo = it->nodeList(); for (auto const& nodeInfo : nodeListInfo) @@ -210,12 +238,11 @@ GatewayNodeInfos PeerRouterTable::selectRouterByComponent(Message::Ptr const& ms if (nodeInfo.second->components().count( msg->header()->optionalField()->componentType())) { - result.insert(it); + choosedGateway.insert(it); break; } } } - return result; } void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h index fdb7bc6c..e0a82c80 100644 --- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h +++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h @@ -54,6 +54,9 @@ class PeerRouterTable private: virtual GatewayNodeInfos selectRouterByNodeID(ppc::protocol::Message::Ptr const& msg) const; virtual GatewayNodeInfos selectRouterByComponent(ppc::protocol::Message::Ptr const& msg) const; + void selectRouterByComponent(GatewayNodeInfos& choosedGateway, + ppc::protocol::Message::Ptr const& msg, + GatewayNodeInfos const& singleAgencyGatewayInfos) const; virtual GatewayNodeInfos selectRouterByAgency(ppc::protocol::Message::Ptr const& msg) const; void removeP2PNodeIDFromNodeIDInfos(GatewayNodeInfo::Ptr const& gatewayInfo); void insertGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle index c703444d..5010ee85 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle @@ -24,6 +24,7 @@ ext { junitVersion = "4.13.2" log4jVersion = "2.23.0" jmhVersion = "1.36" + apacheCommonLangVersion = "3.14.0" println("*ossrhUsername: " + ossrhUsername) } @@ -81,6 +82,7 @@ dependencies { api ('org.slf4j:slf4j-api:1.7.36') implementation("org.projectlombok:lombok:${lombokVersion}") + implementation("org.apache.commons:commons-lang3:${apacheCommonLangVersion}") jmhImplementation ("org.openjdk.jmh:jmh-core:${jmhVersion}") jmhAnnotationProcessor ("org.openjdk.jmh:jmh-generator-annprocess:${jmhVersion}") diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java index 6c1c2c02..6bd5713d 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java @@ -52,7 +52,8 @@ public void onMessage(IMessage message) { + ", payload: " + new String(message.getPayload()) + "#######"); - + // the case access payload multiple times + String payload = new String(message.getPayload()); String response = "#### hello, response!"; System.out.println( "#### sendResponse: " diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java index d63c99e9..e4c75358 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java @@ -15,10 +15,13 @@ package com.webank.wedpr.sdk.jni.transport; +import com.webank.wedpr.sdk.jni.common.WeDPRSDKException; import com.webank.wedpr.sdk.jni.generated.Message; import com.webank.wedpr.sdk.jni.generated.MessageOptionalHeader; import com.webank.wedpr.sdk.jni.generated.MessageOptionalHeaderBuilder; import com.webank.wedpr.sdk.jni.transport.impl.MessageImpl; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; public class IMessageBuilder { public static IMessage build(Message msg) { @@ -28,8 +31,12 @@ public static IMessage build(Message msg) { return new MessageImpl(msg); } + @SneakyThrows(Exception.class) public static MessageOptionalHeader buildRouteInfo( MessageOptionalHeaderBuilder routeInfoBuilder, String topic) { + if (StringUtils.isBlank(topic)) { + throw new WeDPRSDKException("buildRouteInfo for invalid empty topic!"); + } // return the ownership to cpp, since it is created by cpp MessageOptionalHeader routeInfo = routeInfoBuilder.build(); routeInfo.setTopic(topic); diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java index c1e4eb48..17679b24 100644 --- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java +++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java @@ -29,6 +29,8 @@ import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback; import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback; import java.math.BigInteger; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,6 +149,7 @@ public void registerTopicHandler(String topic, MessageDispatcherCallback message * @param timeout the timeout setting * @param errorCallback the handler called after receive the message related to the topic */ + @SneakyThrows(Exception.class) @Override public void asyncSendMessageByNodeID( String topic, @@ -158,6 +161,10 @@ public void asyncSendMessageByNodeID( MessageCallback msgCallback) { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); + if (dstNode == null) { + throw new WeDPRSDKException( + "asyncSendMessageByNodeID failed for the dstNode information is empty"); + } routeInfo.setDstNodeBuffer(dstNode, BigInteger.valueOf(dstNode.length)); this.transport .getFront() @@ -182,6 +189,7 @@ public void asyncSendMessageByNodeID( * @param timeout the timeout * @param errorCallback the handler called after receive the message related to the topic */ + @SneakyThrows(Exception.class) @Override public void asyncSendMessageByAgency( String topic, @@ -193,6 +201,10 @@ public void asyncSendMessageByAgency( MessageCallback msgCallback) { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); + if (StringUtils.isBlank(agency)) { + throw new WeDPRSDKException( + "asyncSendMessageByAgency failed for the dstInst information is empty"); + } routeInfo.setDstInst(agency); this.transport .getFront() @@ -207,6 +219,7 @@ public void asyncSendMessageByAgency( msgCallback); } + @SneakyThrows(Exception.class) @Override public void asyncSendMessageByComponent( String topic, @@ -220,7 +233,14 @@ public void asyncSendMessageByComponent( // set the routeInfo MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); - routeInfo.setDstInst(dstInst); + // Note: support not specify the dstInst + if (StringUtils.isNotBlank(dstInst)) { + routeInfo.setDstInst(dstInst); + } + if (StringUtils.isBlank(component)) { + throw new WeDPRSDKException( + "asyncSendMessageByComponent failed for the component information is empty"); + } routeInfo.setComponentType(component); this.transport .getFront() @@ -244,6 +264,7 @@ public void asyncSendMessageByComponent( * @param timeout the timeout * @param errorCallback the handler */ + @SneakyThrows(Exception.class) @Override public void asyncSendMessageByTopic( String topic, @@ -256,6 +277,9 @@ public void asyncSendMessageByTopic( // set the routeInfo MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); + if (StringUtils.isBlank(dstInst)) { + throw new WeDPRSDKException("asyncSendMessageByTopic failed for the dstInst is empty"); + } routeInfo.setDstInst(dstInst); this.transport .getFront() @@ -270,6 +294,7 @@ public void asyncSendMessageByTopic( msgCallback); } + @SneakyThrows(Exception.class) @Override public void asyncSendResponse( byte[] dstNode, @@ -277,6 +302,9 @@ public void asyncSendResponse( byte[] payload, int seq, MessageErrorCallback errorCallback) { + if (dstNode == null) { + throw new WeDPRSDKException("asyncSendResponse failed for the dstNode is empty"); + } this.transport .getFront() .async_send_response( @@ -290,8 +318,12 @@ public void asyncSendResponse( } /** @param topic the topic to remove */ + @SneakyThrows(Exception.class) @Override public void removeTopic(String topic) throws WeDPRSDKException { + if (StringUtils.isBlank(topic)) { + throw new WeDPRSDKException("removeTopic failed for the topic is empty"); + } Error result = this.transport.getFront().unRegisterTopic(topic); Common.checkResult("removeTopic", result); } @@ -302,6 +334,9 @@ public void pushByNodeID(String topic, byte[] dstNodeID, int seq, byte[] payload throws WeDPRSDKException { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); + if (dstNodeID == null) { + throw new WeDPRSDKException("pushByNodeID failed for the dstNode is empty"); + } routeInfo.setDstNodeBuffer(dstNodeID, BigInteger.valueOf(dstNodeID.length)); Error result = this.transport @@ -322,7 +357,12 @@ public void pushByComponent( throws WeDPRSDKException { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); - routeInfo.setDstInst(dstInst); + if (StringUtils.isNotBlank(dstInst)) { + routeInfo.setDstInst(dstInst); + } + if (StringUtils.isBlank(component)) { + throw new WeDPRSDKException("pushByComponent failed for the component is empty"); + } routeInfo.setComponentType(component); Error result = this.transport @@ -342,6 +382,9 @@ public void pushByInst(String topic, String dstInst, int seq, byte[] payload, in throws WeDPRSDKException { MessageOptionalHeader routeInfo = IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder(), topic); + if (StringUtils.isBlank(dstInst)) { + throw new WeDPRSDKException("pushByInst failed for the dstInst is empty"); + } routeInfo.setDstInst(dstInst); Error result = this.transport @@ -358,6 +401,9 @@ public void pushByInst(String topic, String dstInst, int seq, byte[] payload, in @Override public IMessage pop(String topic, int timeout) throws WeDPRSDKException { + if (StringUtils.isBlank(topic)) { + throw new WeDPRSDKException("pop failed for the topic is empty"); + } Message msg = this.transport.getFront().pop(topic, timeout); if (msg == null) { throw new WeDPRSDKException( @@ -366,8 +412,12 @@ public IMessage pop(String topic, int timeout) throws WeDPRSDKException { return IMessageBuilder.build(msg); } + @SneakyThrows(Exception.class) @Override public IMessage peek(String topic) { + if (StringUtils.isBlank(topic)) { + throw new WeDPRSDKException("peek failed for the topic is empty"); + } Message msg = this.transport.getFront().peek(topic); if (msg == null) { return null; From f48a01f0f09a43044516809d592658cbea677adc Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Mon, 14 Oct 2024 20:09:59 +0800 Subject: [PATCH 053/120] remove unused pir module (#50) --- java/ppc-pir-services/build.gradle | 104 -------- .../gradle/wrapper/gradle-wrapper.jar | Bin 60756 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 - java/ppc-pir-services/gradlew | 240 ------------------ java/ppc-pir-services/gradlew.bat | 91 ------- java/ppc-pir-services/script/start.sh | 126 --------- java/ppc-pir-services/script/stop.sh | 45 ---- java/ppc-pir-services/sdk/build.gradle | 26 -- .../main/java/com/wedpr/pir/demo/Demo.java | 47 ---- .../java/com/wedpr/pir/sdk/PirClient.java | 90 ------- .../pir/sdk/entity/body/PirDataBody.java | 17 -- .../pir/sdk/entity/body/PirResultBody.java | 14 - .../pir/sdk/entity/body/ServerResultBody.java | 19 -- .../pir/sdk/entity/body/ServerResultList.java | 17 -- .../sdk/entity/body/ServiceConfigBody.java | 14 - .../pir/sdk/entity/body/SimpleEntity.java | 15 -- .../pir/sdk/entity/param/PirJobParam.java | 67 ----- .../sdk/entity/request/ClientAuthRequest.java | 13 - .../entity/request/ClientDecryptRequest.java | 17 -- .../entity/request/ClientDirectRequest.java | 16 -- .../sdk/entity/request/ClientOTRequest.java | 17 -- .../sdk/entity/request/PirBaseRequest.java | 22 -- .../sdk/entity/request/ServerJobRequest.java | 21 -- .../sdk/entity/request/ServerOTRequest.java | 20 -- .../entity/response/ClientAuthResponse.java | 15 -- .../response/ClientDecryptResponse.java | 14 - .../entity/response/ClientJobResponse.java | 26 -- .../sdk/entity/response/ClientOTResponse.java | 20 -- .../entity/response/PirResultResponse.java | 14 - .../sdk/entity/response/ServerOTResponse.java | 18 -- .../com/wedpr/pir/sdk/enums/ParamEnum.java | 43 ---- .../pir/sdk/exception/WedprException.java | 23 -- .../pir/sdk/exception/WedprStatusEnum.java | 73 ------ .../wedpr/pir/sdk/gateway/GatewayClient.java | 84 ------ .../com/wedpr/pir/sdk/helper/AESHelper.java | 75 ------ .../wedpr/pir/sdk/helper/BasicTypeHelper.java | 15 -- .../com/wedpr/pir/sdk/helper/Constant.java | 17 -- .../pir/sdk/helper/ConvertTypeHelper.java | 44 ---- .../pir/sdk/helper/CryptoOperatorHelper.java | 95 ------- .../pir/sdk/service/ClientDecryptService.java | 64 ----- .../pir/sdk/service/ClientOTService.java | 85 ------- .../wedpr/pir/sdk/service/PirJobService.java | 63 ----- java/ppc-pir-services/settings.gradle | 11 - .../webank/wedpr/pir/WedprPirApplication.java | 22 -- .../wedpr/pir/crypto/entity/Dataset.java | 16 -- .../pir/crypto/entity/DatasetStoragePath.java | 13 - .../wedpr/pir/crypto/entity/PirTable.java | 17 -- .../pir/crypto/entity/WedprAuthTable.java | 17 -- .../pir/crypto/service/PirOTService.java | 25 -- .../pir/crypto/service/PirTableService.java | 17 -- .../crypto/service/impl/PirOTServiceImpl.java | 143 ----------- .../service/impl/PirTableServiceImpl.java | 58 ----- .../wedpr/pir/http/common/CSVFileParser.java | 220 ---------------- .../wedpr/pir/http/common/Constant.java | 15 -- .../wedpr/pir/http/config/HadoopConfig.java | 33 --- .../wedpr/pir/http/config/JacksonConfig.java | 59 ----- .../pir/http/controller/PirController.java | 105 -------- .../mapper/WedprServiceAuthTableMapper.java | 47 ---- .../wedpr/pir/http/service/PirAppService.java | 48 ---- .../pir/http/service/PirAuthService.java | 50 ---- .../pir/http/service/PirDataService.java | 90 ------- .../main/resources/application-dev.properties | 14 - .../resources/application-mysql.properties | 6 - .../src/main/resources/application.properties | 7 - .../src/main/resources/log4j2.xml | 111 -------- 65 files changed, 2995 deletions(-) delete mode 100644 java/ppc-pir-services/build.gradle delete mode 100644 java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar delete mode 100644 java/ppc-pir-services/gradle/wrapper/gradle-wrapper.properties delete mode 100644 java/ppc-pir-services/gradlew delete mode 100644 java/ppc-pir-services/gradlew.bat delete mode 100644 java/ppc-pir-services/script/start.sh delete mode 100644 java/ppc-pir-services/script/stop.sh delete mode 100644 java/ppc-pir-services/sdk/build.gradle delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java delete mode 100644 java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java delete mode 100644 java/ppc-pir-services/settings.gradle delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java delete mode 100644 java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java delete mode 100644 java/ppc-pir-services/src/main/resources/application-dev.properties delete mode 100644 java/ppc-pir-services/src/main/resources/application-mysql.properties delete mode 100644 java/ppc-pir-services/src/main/resources/application.properties delete mode 100644 java/ppc-pir-services/src/main/resources/log4j2.xml diff --git a/java/ppc-pir-services/build.gradle b/java/ppc-pir-services/build.gradle deleted file mode 100644 index 095c7cd8..00000000 --- a/java/ppc-pir-services/build.gradle +++ /dev/null @@ -1,104 +0,0 @@ -plugins { - id 'java' - id 'com.github.sherter.google-java-format' version '0.8' -} - -group 'cn.webank.wedpr' -version '3.0.0' - -repositories { - maven { - url = "https://mirrors.huaweicloud.com/repository/maven/" - } -} - -configurations.all { - exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" - exclude group: "org.slf4j", module: "slf4j-log4j12" -} - -dependencies { - implementation 'org.springframework.boot:spring-boot-starter-web:2.7.12' - implementation 'org.springframework.boot:spring-boot-starter-data-jpa:2.7.11' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.0' - implementation 'com.mysql:mysql-connector-j:8.0.33' - implementation 'org.mybatis:mybatis:3.5.13' - implementation 'org.mybatis.spring.boot:mybatis-spring-boot-starter:2.2.2' - implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.19.0' - implementation 'commons-codec:commons-codec:1.15' - implementation("org.apache.hadoop:hadoop-common:2.7.5"){ - exclude group: 'javax.servlet', module: 'servlet-api' - } - implementation("org.apache.hadoop:hadoop-client:2.7.5"){ - exclude group: 'javax.servlet', module: 'servlet-api' - } - implementation 'com.opencsv:opencsv:5.9' - implementation 'com.squareup.okhttp3:okhttp:4.9.1' - implementation project(":ppc-pir-sdk") - implementation 'com.google.code.gson:gson:2.8.8' - - compileOnly 'org.projectlombok:lombok:1.18.24' - annotationProcessor 'org.projectlombok:lombok:1.18.24' -} - -jar { - destinationDir file('dist/app') - archiveName project.name + "-" + project.version + '.jar' - exclude '**/*.xml' - exclude '**/*.toml' - exclude '**/*.properties' - exclude '**/*.yml' - exclude '**/*.crt' - exclude '**/*.key' - exclude '**/*.sql' - exclude '**/*.pem' - - doLast { - copy { - from file('src/main/resources/') - into 'dist/conf' - } - copy { - from configurations.runtimeClasspath - into 'dist/libs' - } - copy { - from file('.').listFiles().findAll { File f -> (f.name.endsWith('.sh') || f.name.endsWith('.env')) } - into 'dist' - } - copy { - from file('script/') - into 'dist' - } - } -} - -sourceSets { - main { - java { - srcDir 'src/main/java' - } - resources { - srcDir 'src/main/resources' - srcDir 'src/main/java' - } - } -} - -test { - useJUnitPlatform() -} - - -googleJavaFormat { - options style: 'AOSP' - source = sourceSets*.allJava - include '**/*.java' -} - -clean { - file("dist/").deleteDir() -} - - - diff --git a/java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar b/java/ppc-pir-services/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 249e5832f090a2944b7473328c07c9755baa3196..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60756 zcmb5WV{~QRw(p$^Dz@00IL3?^hro$gg*4VI_WAaTyVM5Foj~O|-84 z$;06hMwt*rV;^8iB z1~&0XWpYJmG?Ts^K9PC62H*`G}xom%S%yq|xvG~FIfP=9*f zZoDRJBm*Y0aId=qJ?7dyb)6)JGWGwe)MHeNSzhi)Ko6J<-m@v=a%NsP537lHe0R* z`If4$aaBA#S=w!2z&m>{lpTy^Lm^mg*3?M&7HFv}7K6x*cukLIGX;bQG|QWdn{%_6 zHnwBKr84#B7Z+AnBXa16a?or^R?+>$4`}{*a_>IhbjvyTtWkHw)|ay)ahWUd-qq$~ zMbh6roVsj;_qnC-R{G+Cy6bApVOinSU-;(DxUEl!i2)1EeQ9`hrfqj(nKI7?Z>Xur zoJz-a`PxkYit1HEbv|jy%~DO^13J-ut986EEG=66S}D3!L}Efp;Bez~7tNq{QsUMm zh9~(HYg1pA*=37C0}n4g&bFbQ+?-h-W}onYeE{q;cIy%eZK9wZjSwGvT+&Cgv z?~{9p(;bY_1+k|wkt_|N!@J~aoY@|U_RGoWX<;p{Nu*D*&_phw`8jYkMNpRTWx1H* z>J-Mi_!`M468#5Aix$$u1M@rJEIOc?k^QBc?T(#=n&*5eS#u*Y)?L8Ha$9wRWdH^3D4|Ps)Y?m0q~SiKiSfEkJ!=^`lJ(%W3o|CZ zSrZL-Xxc{OrmsQD&s~zPfNJOpSZUl%V8tdG%ei}lQkM+z@-4etFPR>GOH9+Y_F<3=~SXln9Kb-o~f>2a6Xz@AS3cn^;c_>lUwlK(n>z?A>NbC z`Ud8^aQy>wy=$)w;JZzA)_*Y$Z5hU=KAG&htLw1Uh00yE!|Nu{EZkch zY9O6x7Y??>!7pUNME*d!=R#s)ghr|R#41l!c?~=3CS8&zr6*aA7n9*)*PWBV2w+&I zpW1-9fr3j{VTcls1>ua}F*bbju_Xq%^v;-W~paSqlf zolj*dt`BBjHI)H9{zrkBo=B%>8}4jeBO~kWqO!~Thi!I1H(in=n^fS%nuL=X2+s!p}HfTU#NBGiwEBF^^tKU zbhhv+0dE-sbK$>J#t-J!B$TMgN@Wh5wTtK2BG}4BGfsZOoRUS#G8Cxv|6EI*n&Xxq zt{&OxCC+BNqz$9b0WM7_PyBJEVObHFh%%`~!@MNZlo*oXDCwDcFwT~Rls!aApL<)^ zbBftGKKBRhB!{?fX@l2_y~%ygNFfF(XJzHh#?`WlSL{1lKT*gJM zs>bd^H9NCxqxn(IOky5k-wALFowQr(gw%|`0991u#9jXQh?4l|l>pd6a&rx|v=fPJ z1mutj{YzpJ_gsClbWFk(G}bSlFi-6@mwoQh-XeD*j@~huW4(8ub%^I|azA)h2t#yG z7e_V_<4jlM3D(I+qX}yEtqj)cpzN*oCdYHa!nm%0t^wHm)EmFP*|FMw!tb@&`G-u~ zK)=Sf6z+BiTAI}}i{*_Ac$ffr*Wrv$F7_0gJkjx;@)XjYSh`RjAgrCck`x!zP>Ifu z&%he4P|S)H*(9oB4uvH67^0}I-_ye_!w)u3v2+EY>eD3#8QR24<;7?*hj8k~rS)~7 zSXs5ww)T(0eHSp$hEIBnW|Iun<_i`}VE0Nc$|-R}wlSIs5pV{g_Dar(Zz<4X3`W?K z6&CAIl4U(Qk-tTcK{|zYF6QG5ArrEB!;5s?tW7 zrE3hcFY&k)+)e{+YOJ0X2uDE_hd2{|m_dC}kgEKqiE9Q^A-+>2UonB+L@v3$9?AYw zVQv?X*pK;X4Ovc6Ev5Gbg{{Eu*7{N3#0@9oMI~}KnObQE#Y{&3mM4`w%wN+xrKYgD zB-ay0Q}m{QI;iY`s1Z^NqIkjrTlf`B)B#MajZ#9u41oRBC1oM1vq0i|F59> z#StM@bHt|#`2)cpl_rWB($DNJ3Lap}QM-+A$3pe}NyP(@+i1>o^fe-oxX#Bt`mcQc zb?pD4W%#ep|3%CHAYnr*^M6Czg>~L4?l16H1OozM{P*en298b+`i4$|w$|4AHbzqB zHpYUsHZET$Z0ztC;U+0*+amF!@PI%^oUIZy{`L{%O^i{Xk}X0&nl)n~tVEpcAJSJ} zverw15zP1P-O8h9nd!&hj$zuwjg?DoxYIw{jWM zW5_pj+wFy8Tsa9g<7Qa21WaV&;ejoYflRKcz?#fSH_)@*QVlN2l4(QNk| z4aPnv&mrS&0|6NHq05XQw$J^RR9T{3SOcMKCXIR1iSf+xJ0E_Wv?jEc*I#ZPzyJN2 zUG0UOXHl+PikM*&g$U@g+KbG-RY>uaIl&DEtw_Q=FYq?etc!;hEC_}UX{eyh%dw2V zTTSlap&5>PY{6I#(6`j-9`D&I#|YPP8a;(sOzgeKDWsLa!i-$frD>zr-oid!Hf&yS z!i^cr&7tN}OOGmX2)`8k?Tn!!4=tz~3hCTq_9CdiV!NIblUDxHh(FJ$zs)B2(t5@u z-`^RA1ShrLCkg0)OhfoM;4Z{&oZmAec$qV@ zGQ(7(!CBk<5;Ar%DLJ0p0!ResC#U<+3i<|vib1?{5gCebG7$F7URKZXuX-2WgF>YJ^i zMhHDBsh9PDU8dlZ$yJKtc6JA#y!y$57%sE>4Nt+wF1lfNIWyA`=hF=9Gj%sRwi@vd z%2eVV3y&dvAgyuJ=eNJR+*080dbO_t@BFJO<@&#yqTK&+xc|FRR;p;KVk@J3$S{p` zGaMj6isho#%m)?pOG^G0mzOAw0z?!AEMsv=0T>WWcE>??WS=fII$t$(^PDPMU(P>o z_*0s^W#|x)%tx8jIgZY~A2yG;US0m2ZOQt6yJqW@XNY_>_R7(Nxb8Ged6BdYW6{prd!|zuX$@Q2o6Ona8zzYC1u!+2!Y$Jc9a;wy+pXt}o6~Bu1oF1c zp7Y|SBTNi@=I(K%A60PMjM#sfH$y*c{xUgeSpi#HB`?|`!Tb&-qJ3;vxS!TIzuTZs-&%#bAkAyw9m4PJgvey zM5?up*b}eDEY+#@tKec)-c(#QF0P?MRlD1+7%Yk*jW;)`f;0a-ZJ6CQA?E%>i2Dt7T9?s|9ZF|KP4;CNWvaVKZ+Qeut;Jith_y{v*Ny6Co6!8MZx;Wgo z=qAi%&S;8J{iyD&>3CLCQdTX*$+Rx1AwA*D_J^0>suTgBMBb=*hefV+Ars#mmr+YsI3#!F@Xc1t4F-gB@6aoyT+5O(qMz*zG<9Qq*f0w^V!03rpr*-WLH}; zfM{xSPJeu6D(%8HU%0GEa%waFHE$G?FH^kMS-&I3)ycx|iv{T6Wx}9$$D&6{%1N_8 z_CLw)_9+O4&u94##vI9b-HHm_95m)fa??q07`DniVjAy`t7;)4NpeyAY(aAk(+T_O z1om+b5K2g_B&b2DCTK<>SE$Ode1DopAi)xaJjU>**AJK3hZrnhEQ9E`2=|HHe<^tv z63e(bn#fMWuz>4erc47}!J>U58%<&N<6AOAewyzNTqi7hJc|X{782&cM zHZYclNbBwU6673=!ClmxMfkC$(CykGR@10F!zN1Se83LR&a~$Ht&>~43OX22mt7tcZUpa;9@q}KDX3O&Ugp6< zLZLfIMO5;pTee1vNyVC$FGxzK2f>0Z-6hM82zKg44nWo|n}$Zk6&;5ry3`(JFEX$q zK&KivAe${e^5ZGc3a9hOt|!UOE&OocpVryE$Y4sPcs4rJ>>Kbi2_subQ9($2VN(3o zb~tEzMsHaBmBtaHAyES+d3A(qURgiskSSwUc9CfJ@99&MKp2sooSYZu+-0t0+L*!I zYagjOlPgx|lep9tiU%ts&McF6b0VE57%E0Ho%2oi?=Ks+5%aj#au^OBwNwhec zta6QAeQI^V!dF1C)>RHAmB`HnxyqWx?td@4sd15zPd*Fc9hpDXP23kbBenBxGeD$k z;%0VBQEJ-C)&dTAw_yW@k0u?IUk*NrkJ)(XEeI z9Y>6Vel>#s_v@=@0<{4A{pl=9cQ&Iah0iD0H`q)7NeCIRz8zx;! z^OO;1+IqoQNak&pV`qKW+K0^Hqp!~gSohcyS)?^P`JNZXw@gc6{A3OLZ?@1Uc^I2v z+X!^R*HCm3{7JPq{8*Tn>5;B|X7n4QQ0Bs79uTU%nbqOJh`nX(BVj!#f;#J+WZxx4 z_yM&1Y`2XzhfqkIMO7tB3raJKQS+H5F%o83bM+hxbQ zeeJm=Dvix$2j|b4?mDacb67v-1^lTp${z=jc1=j~QD>7c*@+1?py>%Kj%Ejp7Y-!? z8iYRUlGVrQPandAaxFfks53@2EC#0)%mrnmGRn&>=$H$S8q|kE_iWko4`^vCS2aWg z#!`RHUGyOt*k?bBYu3*j3u0gB#v(3tsije zgIuNNWNtrOkx@Pzs;A9un+2LX!zw+p3_NX^Sh09HZAf>m8l@O*rXy_82aWT$Q>iyy zqO7Of)D=wcSn!0+467&!Hl))eff=$aneB?R!YykdKW@k^_uR!+Q1tR)+IJb`-6=jj zymzA>Sv4>Z&g&WWu#|~GcP7qP&m*w-S$)7Xr;(duqCTe7p8H3k5>Y-n8438+%^9~K z3r^LIT_K{i7DgEJjIocw_6d0!<;wKT`X;&vv+&msmhAAnIe!OTdybPctzcEzBy88_ zWO{6i4YT%e4^WQZB)KHCvA(0tS zHu_Bg+6Ko%a9~$EjRB90`P(2~6uI@SFibxct{H#o&y40MdiXblu@VFXbhz>Nko;7R z70Ntmm-FePqhb%9gL+7U8@(ch|JfH5Fm)5${8|`Lef>LttM_iww6LW2X61ldBmG0z zax3y)njFe>j*T{i0s8D4=L>X^j0)({R5lMGVS#7(2C9@AxL&C-lZQx~czI7Iv+{%1 z2hEG>RzX4S8x3v#9sgGAnPzptM)g&LB}@%E>fy0vGSa(&q0ch|=ncKjNrK z`jA~jObJhrJ^ri|-)J^HUyeZXz~XkBp$VhcTEcTdc#a2EUOGVX?@mYx#Vy*!qO$Jv zQ4rgOJ~M*o-_Wptam=~krnmG*p^j!JAqoQ%+YsDFW7Cc9M%YPiBOrVcD^RY>m9Pd< zu}#9M?K{+;UIO!D9qOpq9yxUquQRmQNMo0pT`@$pVt=rMvyX)ph(-CCJLvUJy71DI zBk7oc7)-%ngdj~s@76Yse3L^gV0 z2==qfp&Q~L(+%RHP0n}+xH#k(hPRx(!AdBM$JCfJ5*C=K3ts>P?@@SZ_+{U2qFZb>4kZ{Go37{# zSQc+-dq*a-Vy4?taS&{Ht|MLRiS)Sn14JOONyXqPNnpq&2y~)6wEG0oNy>qvod$FF z`9o&?&6uZjhZ4_*5qWVrEfu(>_n2Xi2{@Gz9MZ8!YmjYvIMasE9yVQL10NBrTCczq zcTY1q^PF2l!Eraguf{+PtHV3=2A?Cu&NN&a8V(y;q(^_mFc6)%Yfn&X&~Pq zU1?qCj^LF(EQB1F`8NxNjyV%fde}dEa(Hx=r7$~ts2dzDwyi6ByBAIx$NllB4%K=O z$AHz1<2bTUb>(MCVPpK(E9wlLElo(aSd(Os)^Raum`d(g9Vd_+Bf&V;l=@mM=cC>) z)9b0enb)u_7V!!E_bl>u5nf&Rl|2r=2F3rHMdb7y9E}}F82^$Rf+P8%dKnOeKh1vs zhH^P*4Ydr^$)$h@4KVzxrHyy#cKmWEa9P5DJ|- zG;!Qi35Tp7XNj60=$!S6U#!(${6hyh7d4q=pF{`0t|N^|L^d8pD{O9@tF~W;#Je*P z&ah%W!KOIN;SyAEhAeTafJ4uEL`(RtnovM+cb(O#>xQnk?dzAjG^~4$dFn^<@-Na3 z395;wBnS{t*H;Jef2eE!2}u5Ns{AHj>WYZDgQJt8v%x?9{MXqJsGP|l%OiZqQ1aB! z%E=*Ig`(!tHh>}4_z5IMpg{49UvD*Pp9!pxt_gdAW%sIf3k6CTycOT1McPl=_#0?8 zVjz8Hj*Vy9c5-krd-{BQ{6Xy|P$6LJvMuX$* zA+@I_66_ET5l2&gk9n4$1M3LN8(yEViRx&mtd#LD}AqEs?RW=xKC(OCWH;~>(X6h!uDxXIPH06xh z*`F4cVlbDP`A)-fzf>MuScYsmq&1LUMGaQ3bRm6i7OsJ|%uhTDT zlvZA1M}nz*SalJWNT|`dBm1$xlaA>CCiQ zK`xD-RuEn>-`Z?M{1%@wewf#8?F|(@1e0+T4>nmlSRrNK5f)BJ2H*$q(H>zGD0>eL zQ!tl_Wk)k*e6v^m*{~A;@6+JGeWU-q9>?+L_#UNT%G?4&BnOgvm9@o7l?ov~XL+et zbGT)|G7)KAeqb=wHSPk+J1bdg7N3$vp(ekjI1D9V$G5Cj!=R2w=3*4!z*J-r-cyeb zd(i2KmX!|Lhey!snRw z?#$Gu%S^SQEKt&kep)up#j&9}e+3=JJBS(s>MH+|=R(`8xK{mmndWo_r`-w1#SeRD&YtAJ#GiVI*TkQZ}&aq<+bU2+coU3!jCI6E+Ad_xFW*ghnZ$q zAoF*i&3n1j#?B8x;kjSJD${1jdRB;)R*)Ao!9bd|C7{;iqDo|T&>KSh6*hCD!rwv= zyK#F@2+cv3=|S1Kef(E6Niv8kyLVLX&e=U;{0x{$tDfShqkjUME>f8d(5nzSkY6@! z^-0>DM)wa&%m#UF1F?zR`8Y3X#tA!*7Q$P3lZJ%*KNlrk_uaPkxw~ zxZ1qlE;Zo;nb@!SMazSjM>;34ROOoygo%SF);LL>rRonWwR>bmSd1XD^~sGSu$Gg# zFZ`|yKU0%!v07dz^v(tY%;So(e`o{ZYTX`hm;@b0%8|H>VW`*cr8R%3n|ehw2`(9B+V72`>SY}9^8oh$En80mZK9T4abVG*to;E z1_S6bgDOW?!Oy1LwYy=w3q~KKdbNtyH#d24PFjX)KYMY93{3-mPP-H>@M-_>N~DDu zENh~reh?JBAK=TFN-SfDfT^=+{w4ea2KNWXq2Y<;?(gf(FgVp8Zp-oEjKzB%2Iqj;48GmY3h=bcdYJ}~&4tS`Q1sb=^emaW$IC$|R+r-8V- zf0$gGE(CS_n4s>oicVk)MfvVg#I>iDvf~Ov8bk}sSxluG!6#^Z_zhB&U^`eIi1@j( z^CK$z^stBHtaDDHxn+R;3u+>Lil^}fj?7eaGB z&5nl^STqcaBxI@v>%zG|j))G(rVa4aY=B@^2{TFkW~YP!8!9TG#(-nOf^^X-%m9{Z zCC?iC`G-^RcBSCuk=Z`(FaUUe?hf3{0C>>$?Vs z`2Uud9M+T&KB6o4o9kvdi^Q=Bw!asPdxbe#W-Oaa#_NP(qpyF@bVxv5D5))srkU#m zj_KA+#7sqDn*Ipf!F5Byco4HOSd!Ui$l94|IbW%Ny(s1>f4|Mv^#NfB31N~kya9!k zWCGL-$0ZQztBate^fd>R!hXY_N9ZjYp3V~4_V z#eB)Kjr8yW=+oG)BuNdZG?jaZlw+l_ma8aET(s+-x+=F-t#Qoiuu1i`^x8Sj>b^U} zs^z<()YMFP7CmjUC@M=&lA5W7t&cxTlzJAts*%PBDAPuqcV5o7HEnqjif_7xGt)F% zGx2b4w{@!tE)$p=l3&?Bf#`+!-RLOleeRk3 z7#pF|w@6_sBmn1nECqdunmG^}pr5(ZJQVvAt$6p3H(16~;vO>?sTE`Y+mq5YP&PBo zvq!7#W$Gewy`;%6o^!Dtjz~x)T}Bdk*BS#=EY=ODD&B=V6TD2z^hj1m5^d6s)D*wk zu$z~D7QuZ2b?5`p)E8e2_L38v3WE{V`bVk;6fl#o2`) z99JsWhh?$oVRn@$S#)uK&8DL8>An0&S<%V8hnGD7Z^;Y(%6;^9!7kDQ5bjR_V+~wp zfx4m3z6CWmmZ<8gDGUyg3>t8wgJ5NkkiEm^(sedCicP^&3D%}6LtIUq>mXCAt{9eF zNXL$kGcoUTf_Lhm`t;hD-SE)m=iBnxRU(NyL}f6~1uH)`K!hmYZjLI%H}AmEF5RZt z06$wn63GHnApHXZZJ}s^s)j9(BM6e*7IBK6Bq(!)d~zR#rbxK9NVIlgquoMq z=eGZ9NR!SEqP6=9UQg#@!rtbbSBUM#ynF);zKX+|!Zm}*{H z+j=d?aZ2!?@EL7C~%B?6ouCKLnO$uWn;Y6Xz zX8dSwj732u(o*U3F$F=7xwxm>E-B+SVZH;O-4XPuPkLSt_?S0)lb7EEg)Mglk0#eS z9@jl(OnH4juMxY+*r03VDfPx_IM!Lmc(5hOI;`?d37f>jPP$?9jQQIQU@i4vuG6MagEoJrQ=RD7xt@8E;c zeGV*+Pt+t$@pt!|McETOE$9k=_C!70uhwRS9X#b%ZK z%q(TIUXSS^F0`4Cx?Rk07C6wI4!UVPeI~-fxY6`YH$kABdOuiRtl73MqG|~AzZ@iL&^s?24iS;RK_pdlWkhcF z@Wv-Om(Aealfg)D^adlXh9Nvf~Uf@y;g3Y)i(YP zEXDnb1V}1pJT5ZWyw=1i+0fni9yINurD=EqH^ciOwLUGi)C%Da)tyt=zq2P7pV5-G zR7!oq28-Fgn5pW|nlu^b!S1Z#r7!Wtr{5J5PQ>pd+2P7RSD?>(U7-|Y z7ZQ5lhYIl_IF<9?T9^IPK<(Hp;l5bl5tF9>X-zG14_7PfsA>6<$~A338iYRT{a@r_ zuXBaT=`T5x3=s&3=RYx6NgG>No4?5KFBVjE(swfcivcIpPQFx5l+O;fiGsOrl5teR z_Cm+;PW}O0Dwe_(4Z@XZ)O0W-v2X><&L*<~*q3dg;bQW3g7)a#3KiQP>+qj|qo*Hk z?57>f2?f@`=Fj^nkDKeRkN2d$Z@2eNKpHo}ksj-$`QKb6n?*$^*%Fb3_Kbf1(*W9K>{L$mud2WHJ=j0^=g30Xhg8$#g^?36`p1fm;;1@0Lrx+8t`?vN0ZorM zSW?rhjCE8$C|@p^sXdx z|NOHHg+fL;HIlqyLp~SSdIF`TnSHehNCU9t89yr@)FY<~hu+X`tjg(aSVae$wDG*C zq$nY(Y494R)hD!i1|IIyP*&PD_c2FPgeY)&mX1qujB1VHPG9`yFQpLFVQ0>EKS@Bp zAfP5`C(sWGLI?AC{XEjLKR4FVNw(4+9b?kba95ukgR1H?w<8F7)G+6&(zUhIE5Ef% z=fFkL3QKA~M@h{nzjRq!Y_t!%U66#L8!(2-GgFxkD1=JRRqk=n%G(yHKn%^&$dW>; zSjAcjETMz1%205se$iH_)ZCpfg_LwvnsZQAUCS#^FExp8O4CrJb6>JquNV@qPq~3A zZ<6dOU#6|8+fcgiA#~MDmcpIEaUO02L5#T$HV0$EMD94HT_eXLZ2Zi&(! z&5E>%&|FZ`)CN10tM%tLSPD*~r#--K(H-CZqIOb99_;m|D5wdgJ<1iOJz@h2Zkq?} z%8_KXb&hf=2Wza(Wgc;3v3TN*;HTU*q2?#z&tLn_U0Nt!y>Oo>+2T)He6%XuP;fgn z-G!#h$Y2`9>Jtf}hbVrm6D70|ERzLAU>3zoWhJmjWfgM^))T+2u$~5>HF9jQDkrXR z=IzX36)V75PrFjkQ%TO+iqKGCQ-DDXbaE;C#}!-CoWQx&v*vHfyI>$HNRbpvm<`O( zlx9NBWD6_e&J%Ous4yp~s6)Ghni!I6)0W;9(9$y1wWu`$gs<$9Mcf$L*piP zPR0Av*2%ul`W;?-1_-5Zy0~}?`e@Y5A&0H!^ApyVTT}BiOm4GeFo$_oPlDEyeGBbh z1h3q&Dx~GmUS|3@4V36&$2uO8!Yp&^pD7J5&TN{?xphf*-js1fP?B|`>p_K>lh{ij zP(?H%e}AIP?_i^f&Li=FDSQ`2_NWxL+BB=nQr=$ zHojMlXNGauvvwPU>ZLq!`bX-5F4jBJ&So{kE5+ms9UEYD{66!|k~3vsP+mE}x!>%P za98bAU0!h0&ka4EoiDvBM#CP#dRNdXJcb*(%=<(g+M@<)DZ!@v1V>;54En?igcHR2 zhubQMq}VSOK)onqHfczM7YA@s=9*ow;k;8)&?J3@0JiGcP! zP#00KZ1t)GyZeRJ=f0^gc+58lc4Qh*S7RqPIC6GugG1gXe$LIQMRCo8cHf^qXgAa2 z`}t>u2Cq1CbSEpLr~E=c7~=Qkc9-vLE%(v9N*&HF`(d~(0`iukl5aQ9u4rUvc8%m) zr2GwZN4!s;{SB87lJB;veebPmqE}tSpT>+`t?<457Q9iV$th%i__Z1kOMAswFldD6 ztbOvO337S5o#ZZgN2G99_AVqPv!?Gmt3pzgD+Hp3QPQ`9qJ(g=kjvD+fUSS3upJn! zqoG7acIKEFRX~S}3|{EWT$kdz#zrDlJU(rPkxjws_iyLKU8+v|*oS_W*-guAb&Pj1 z35Z`3z<&Jb@2Mwz=KXucNYdY#SNO$tcVFr9KdKm|%^e-TXzs6M`PBper%ajkrIyUe zp$vVxVs9*>Vp4_1NC~Zg)WOCPmOxI1V34QlG4!aSFOH{QqSVq1^1)- z0P!Z?tT&E-ll(pwf0?=F=yOzik=@nh1Clxr9}Vij89z)ePDSCYAqw?lVI?v?+&*zH z)p$CScFI8rrwId~`}9YWPFu0cW1Sf@vRELs&cbntRU6QfPK-SO*mqu|u~}8AJ!Q$z znzu}50O=YbjwKCuSVBs6&CZR#0FTu)3{}qJJYX(>QPr4$RqWiwX3NT~;>cLn*_&1H zaKpIW)JVJ>b{uo2oq>oQt3y=zJjb%fU@wLqM{SyaC6x2snMx-}ivfU<1- znu1Lh;i$3Tf$Kh5Uk))G!D1UhE8pvx&nO~w^fG)BC&L!_hQk%^p`Kp@F{cz>80W&T ziOK=Sq3fdRu*V0=S53rcIfWFazI}Twj63CG(jOB;$*b`*#B9uEnBM`hDk*EwSRdwP8?5T?xGUKs=5N83XsR*)a4|ijz|c{4tIU+4j^A5C<#5 z*$c_d=5ml~%pGxw#?*q9N7aRwPux5EyqHVkdJO=5J>84!X6P>DS8PTTz>7C#FO?k#edkntG+fJk8ZMn?pmJSO@`x-QHq;7^h6GEXLXo1TCNhH z8ZDH{*NLAjo3WM`xeb=X{((uv3H(8&r8fJJg_uSs_%hOH%JDD?hu*2NvWGYD+j)&` zz#_1%O1wF^o5ryt?O0n;`lHbzp0wQ?rcbW(F1+h7_EZZ9{>rePvLAPVZ_R|n@;b$;UchU=0j<6k8G9QuQf@76oiE*4 zXOLQ&n3$NR#p4<5NJMVC*S);5x2)eRbaAM%VxWu9ohlT;pGEk7;002enCbQ>2r-us z3#bpXP9g|mE`65VrN`+3mC)M(eMj~~eOf)do<@l+fMiTR)XO}422*1SL{wyY(%oMpBgJagtiDf zz>O6(m;};>Hi=t8o{DVC@YigqS(Qh+ix3Rwa9aliH}a}IlOCW1@?%h_bRbq-W{KHF z%Vo?-j@{Xi@=~Lz5uZP27==UGE15|g^0gzD|3x)SCEXrx`*MP^FDLl%pOi~~Il;dc z^hrwp9sYeT7iZ)-ajKy@{a`kr0-5*_!XfBpXwEcFGJ;%kV$0Nx;apKrur zJN2J~CAv{Zjj%FolyurtW8RaFmpn&zKJWL>(0;;+q(%(Hx!GMW4AcfP0YJ*Vz!F4g z!ZhMyj$BdXL@MlF%KeInmPCt~9&A!;cRw)W!Hi@0DY(GD_f?jeV{=s=cJ6e}JktJw zQORnxxj3mBxfrH=x{`_^Z1ddDh}L#V7i}$njUFRVwOX?qOTKjfPMBO4y(WiU<)epb zvB9L=%jW#*SL|Nd_G?E*_h1^M-$PG6Pc_&QqF0O-FIOpa4)PAEPsyvB)GKasmBoEt z?_Q2~QCYGH+hW31x-B=@5_AN870vY#KB~3a*&{I=f);3Kv7q4Q7s)0)gVYx2#Iz9g(F2;=+Iy4 z6KI^8GJ6D@%tpS^8boU}zpi=+(5GfIR)35PzrbuXeL1Y1N%JK7PG|^2k3qIqHfX;G zQ}~JZ-UWx|60P5?d1e;AHx!_;#PG%d=^X(AR%i`l0jSpYOpXoKFW~7ip7|xvN;2^? zsYC9fanpO7rO=V7+KXqVc;Q5z%Bj})xHVrgoR04sA2 zl~DAwv=!(()DvH*=lyhIlU^hBkA0$e*7&fJpB0|oB7)rqGK#5##2T`@_I^|O2x4GO z;xh6ROcV<9>?e0)MI(y++$-ksV;G;Xe`lh76T#Htuia+(UrIXrf9?

L(tZ$0BqX1>24?V$S+&kLZ`AodQ4_)P#Q3*4xg8}lMV-FLwC*cN$< zt65Rf%7z41u^i=P*qO8>JqXPrinQFapR7qHAtp~&RZ85$>ob|Js;GS^y;S{XnGiBc zGa4IGvDl?x%gY`vNhv8wgZnP#UYI-w*^4YCZnxkF85@ldepk$&$#3EAhrJY0U)lR{F6sM3SONV^+$;Zx8BD&Eku3K zKNLZyBni3)pGzU0;n(X@1fX8wYGKYMpLmCu{N5-}epPDxClPFK#A@02WM3!myN%bkF z|GJ4GZ}3sL{3{qXemy+#Uk{4>Kf8v11;f8I&c76+B&AQ8udd<8gU7+BeWC`akUU~U zgXoxie>MS@rBoyY8O8Tc&8id!w+_ooxcr!1?#rc$-|SBBtH6S?)1e#P#S?jFZ8u-Bs&k`yLqW|{j+%c#A4AQ>+tj$Y z^CZajspu$F%73E68Lw5q7IVREED9r1Ijsg#@DzH>wKseye>hjsk^{n0g?3+gs@7`i zHx+-!sjLx^fS;fY!ERBU+Q zVJ!e0hJH%P)z!y%1^ZyG0>PN@5W~SV%f>}c?$H8r;Sy-ui>aruVTY=bHe}$e zi&Q4&XK!qT7-XjCrDaufT@>ieQ&4G(SShUob0Q>Gznep9fR783jGuUynAqc6$pYX; z7*O@@JW>O6lKIk0G00xsm|=*UVTQBB`u1f=6wGAj%nHK_;Aqmfa!eAykDmi-@u%6~ z;*c!pS1@V8r@IX9j&rW&d*}wpNs96O2Ute>%yt{yv>k!6zfT6pru{F1M3P z2WN1JDYqoTB#(`kE{H676QOoX`cnqHl1Yaru)>8Ky~VU{)r#{&s86Vz5X)v15ULHA zAZDb{99+s~qI6;-dQ5DBjHJP@GYTwn;Dv&9kE<0R!d z8tf1oq$kO`_sV(NHOSbMwr=To4r^X$`sBW4$gWUov|WY?xccQJN}1DOL|GEaD_!@& z15p?Pj+>7d`@LvNIu9*^hPN)pwcv|akvYYq)ks%`G>!+!pW{-iXPZsRp8 z35LR;DhseQKWYSD`%gO&k$Dj6_6q#vjWA}rZcWtQr=Xn*)kJ9kacA=esi*I<)1>w^ zO_+E>QvjP)qiSZg9M|GNeLtO2D7xT6vsj`88sd!94j^AqxFLi}@w9!Y*?nwWARE0P znuI_7A-saQ+%?MFA$gttMV-NAR^#tjl_e{R$N8t2NbOlX373>e7Ox=l=;y#;M7asp zRCz*CLnrm$esvSb5{T<$6CjY zmZ(i{Rs_<#pWW>(HPaaYj`%YqBra=Ey3R21O7vUbzOkJJO?V`4-D*u4$Me0Bx$K(lYo`JO}gnC zx`V}a7m-hLU9Xvb@K2ymioF)vj12<*^oAqRuG_4u%(ah?+go%$kOpfb`T96P+L$4> zQ#S+sA%VbH&mD1k5Ak7^^dZoC>`1L%i>ZXmooA!%GI)b+$D&ziKrb)a=-ds9xk#~& z7)3iem6I|r5+ZrTRe_W861x8JpD`DDIYZNm{$baw+$)X^Jtjnl0xlBgdnNY}x%5za zkQ8E6T<^$sKBPtL4(1zi_Rd(tVth*3Xs!ulflX+70?gb&jRTnI8l+*Aj9{|d%qLZ+ z>~V9Z;)`8-lds*Zgs~z1?Fg?Po7|FDl(Ce<*c^2=lFQ~ahwh6rqSjtM5+$GT>3WZW zj;u~w9xwAhOc<kF}~`CJ68 z?(S5vNJa;kriPlim33{N5`C{9?NWhzsna_~^|K2k4xz1`xcui*LXL-1#Y}Hi9`Oo!zQ>x-kgAX4LrPz63uZ+?uG*84@PKq-KgQlMNRwz=6Yes) zY}>YN+qP}nwr$(CZQFjUOI=-6J$2^XGvC~EZ+vrqWaOXB$k?%Suf5k=4>AveC1aJ! ziaW4IS%F$_Babi)kA8Y&u4F7E%99OPtm=vzw$$ zEz#9rvn`Iot_z-r3MtV>k)YvErZ<^Oa${`2>MYYODSr6?QZu+be-~MBjwPGdMvGd!b!elsdi4% z`37W*8+OGulab8YM?`KjJ8e+jM(tqLKSS@=jimq3)Ea2EB%88L8CaM+aG7;27b?5` z4zuUWBr)f)k2o&xg{iZ$IQkJ+SK>lpq4GEacu~eOW4yNFLU!Kgc{w4&D$4ecm0f}~ zTTzquRW@`f0}|IILl`!1P+;69g^upiPA6F{)U8)muWHzexRenBU$E^9X-uIY2%&1w z_=#5*(nmxJ9zF%styBwivi)?#KMG96-H@hD-H_&EZiRNsfk7mjBq{L%!E;Sqn!mVX*}kXhwH6eh;b42eD!*~upVG@ z#smUqz$ICm!Y8wY53gJeS|Iuard0=;k5i5Z_hSIs6tr)R4n*r*rE`>38Pw&lkv{_r!jNN=;#?WbMj|l>cU(9trCq; z%nN~r^y7!kH^GPOf3R}?dDhO=v^3BeP5hF|%4GNQYBSwz;x({21i4OQY->1G=KFyu z&6d`f2tT9Yl_Z8YACZaJ#v#-(gcyeqXMhYGXb=t>)M@fFa8tHp2x;ODX=Ap@a5I=U z0G80^$N0G4=U(>W%mrrThl0DjyQ-_I>+1Tdd_AuB3qpYAqY54upwa3}owa|x5iQ^1 zEf|iTZxKNGRpI>34EwkIQ2zHDEZ=(J@lRaOH>F|2Z%V_t56Km$PUYu^xA5#5Uj4I4RGqHD56xT%H{+P8Ag>e_3pN$4m8n>i%OyJFPNWaEnJ4McUZPa1QmOh?t8~n& z&RulPCors8wUaqMHECG=IhB(-tU2XvHP6#NrLVyKG%Ee*mQ5Ps%wW?mcnriTVRc4J`2YVM>$ixSF2Xi+Wn(RUZnV?mJ?GRdw%lhZ+t&3s7g!~g{%m&i<6 z5{ib-<==DYG93I(yhyv4jp*y3#*WNuDUf6`vTM%c&hiayf(%=x@4$kJ!W4MtYcE#1 zHM?3xw63;L%x3drtd?jot!8u3qeqctceX3m;tWetK+>~q7Be$h>n6riK(5@ujLgRS zvOym)k+VAtyV^mF)$29Y`nw&ijdg~jYpkx%*^ z8dz`C*g=I?;clyi5|!27e2AuSa$&%UyR(J3W!A=ZgHF9OuKA34I-1U~pyD!KuRkjA zbkN!?MfQOeN>DUPBxoy5IX}@vw`EEB->q!)8fRl_mqUVuRu|C@KD-;yl=yKc=ZT0% zB$fMwcC|HE*0f8+PVlWHi>M`zfsA(NQFET?LrM^pPcw`cK+Mo0%8*x8@65=CS_^$cG{GZQ#xv($7J z??R$P)nPLodI;P!IC3eEYEHh7TV@opr#*)6A-;EU2XuogHvC;;k1aI8asq7ovoP!* z?x%UoPrZjj<&&aWpsbr>J$Er-7!E(BmOyEv!-mbGQGeJm-U2J>74>o5x`1l;)+P&~ z>}f^=Rx(ZQ2bm+YE0u=ZYrAV@apyt=v1wb?R@`i_g64YyAwcOUl=C!i>=Lzb$`tjv zOO-P#A+)t-JbbotGMT}arNhJmmGl-lyUpMn=2UacVZxmiG!s!6H39@~&uVokS zG=5qWhfW-WOI9g4!R$n7!|ViL!|v3G?GN6HR0Pt_L5*>D#FEj5wM1DScz4Jv@Sxnl zB@MPPmdI{(2D?;*wd>3#tjAirmUnQoZrVv`xM3hARuJksF(Q)wd4P$88fGYOT1p6U z`AHSN!`St}}UMBT9o7i|G`r$ zrB=s$qV3d6$W9@?L!pl0lf%)xs%1ko^=QY$ty-57=55PvP(^6E7cc zGJ*>m2=;fOj?F~yBf@K@9qwX0hA803Xw+b0m}+#a(>RyR8}*Y<4b+kpp|OS+!whP( zH`v{%s>jsQI9rd$*vm)EkwOm#W_-rLTHcZRek)>AtF+~<(did)*oR1|&~1|e36d-d zgtm5cv1O0oqgWC%Et@P4Vhm}Ndl(Y#C^MD03g#PH-TFy+7!Osv1z^UWS9@%JhswEq~6kSr2DITo59+; ze=ZC}i2Q?CJ~Iyu?vn|=9iKV>4j8KbxhE4&!@SQ^dVa-gK@YfS9xT(0kpW*EDjYUkoj! zE49{7H&E}k%5(>sM4uGY)Q*&3>{aitqdNnRJkbOmD5Mp5rv-hxzOn80QsG=HJ_atI-EaP69cacR)Uvh{G5dTpYG7d zbtmRMq@Sexey)||UpnZ?;g_KMZq4IDCy5}@u!5&B^-=6yyY{}e4Hh3ee!ZWtL*s?G zxG(A!<9o!CL+q?u_utltPMk+hn?N2@?}xU0KlYg?Jco{Yf@|mSGC<(Zj^yHCvhmyx z?OxOYoxbptDK()tsJ42VzXdINAMWL$0Gcw?G(g8TMB)Khw_|v9`_ql#pRd2i*?CZl z7k1b!jQB=9-V@h%;Cnl7EKi;Y^&NhU0mWEcj8B|3L30Ku#-9389Q+(Yet0r$F=+3p z6AKOMAIi|OHyzlHZtOm73}|ntKtFaXF2Fy|M!gOh^L4^62kGUoWS1i{9gsds_GWBc zLw|TaLP64z3z9?=R2|T6Xh2W4_F*$cq>MtXMOy&=IPIJ`;!Tw?PqvI2b*U1)25^<2 zU_ZPoxg_V0tngA0J+mm?3;OYw{i2Zb4x}NedZug!>EoN3DC{1i)Z{Z4m*(y{ov2%- zk(w>+scOO}MN!exSc`TN)!B=NUX`zThWO~M*ohqq;J2hx9h9}|s#?@eR!=F{QTrq~ zTcY|>azkCe$|Q0XFUdpFT=lTcyW##i;-e{}ORB4D?t@SfqGo_cS z->?^rh$<&n9DL!CF+h?LMZRi)qju!meugvxX*&jfD!^1XB3?E?HnwHP8$;uX{Rvp# zh|)hM>XDv$ZGg=$1{+_bA~u-vXqlw6NH=nkpyWE0u}LQjF-3NhATL@9rRxMnpO%f7 z)EhZf{PF|mKIMFxnC?*78(}{Y)}iztV12}_OXffJ;ta!fcFIVjdchyHxH=t%ci`Xd zX2AUB?%?poD6Zv*&BA!6c5S#|xn~DK01#XvjT!w!;&`lDXSJT4_j$}!qSPrb37vc{ z9^NfC%QvPu@vlxaZ;mIbn-VHA6miwi8qJ~V;pTZkKqqOii<1Cs}0i?uUIss;hM4dKq^1O35y?Yp=l4i zf{M!@QHH~rJ&X~8uATV><23zZUbs-J^3}$IvV_ANLS08>k`Td7aU_S1sLsfi*C-m1 z-e#S%UGs4E!;CeBT@9}aaI)qR-6NU@kvS#0r`g&UWg?fC7|b^_HyCE!8}nyh^~o@< zpm7PDFs9yxp+byMS(JWm$NeL?DNrMCNE!I^ko-*csB+dsf4GAq{=6sfyf4wb>?v1v zmb`F*bN1KUx-`ra1+TJ37bXNP%`-Fd`vVQFTwWpX@;s(%nDQa#oWhgk#mYlY*!d>( zE&!|ySF!mIyfING+#%RDY3IBH_fW$}6~1%!G`suHub1kP@&DoAd5~7J55;5_noPI6eLf{t;@9Kf<{aO0`1WNKd?<)C-|?C?)3s z>wEq@8=I$Wc~Mt$o;g++5qR+(6wt9GI~pyrDJ%c?gPZe)owvy^J2S=+M^ z&WhIE`g;;J^xQLVeCtf7b%Dg#Z2gq9hp_%g)-%_`y*zb; zn9`f`mUPN-Ts&fFo(aNTsXPA|J!TJ{0hZp0^;MYHLOcD=r_~~^ymS8KLCSeU3;^QzJNqS z5{5rEAv#l(X?bvwxpU;2%pQftF`YFgrD1jt2^~Mt^~G>T*}A$yZc@(k9orlCGv&|1 zWWvVgiJsCAtamuAYT~nzs?TQFt<1LSEx!@e0~@yd6$b5!Zm(FpBl;(Cn>2vF?k zOm#TTjFwd2D-CyA!mqR^?#Uwm{NBemP>(pHmM}9;;8`c&+_o3#E5m)JzfwN?(f-a4 zyd%xZc^oQx3XT?vcCqCX&Qrk~nu;fxs@JUoyVoi5fqpi&bUhQ2y!Ok2pzsFR(M(|U zw3E+kH_zmTRQ9dUMZWRE%Zakiwc+lgv7Z%|YO9YxAy`y28`Aw;WU6HXBgU7fl@dnt z-fFBV)}H-gqP!1;V@Je$WcbYre|dRdp{xt!7sL3Eoa%IA`5CAA%;Wq8PktwPdULo! z8!sB}Qt8#jH9Sh}QiUtEPZ6H0b*7qEKGJ%ITZ|vH)5Q^2m<7o3#Z>AKc%z7_u`rXA zqrCy{-{8;9>dfllLu$^M5L z-hXs))h*qz%~ActwkIA(qOVBZl2v4lwbM>9l70Y`+T*elINFqt#>OaVWoja8RMsep z6Or3f=oBnA3vDbn*+HNZP?8LsH2MY)x%c13@(XfuGR}R?Nu<|07{$+Lc3$Uv^I!MQ z>6qWgd-=aG2Y^24g4{Bw9ueOR)(9h`scImD=86dD+MnSN4$6 z^U*o_mE-6Rk~Dp!ANp#5RE9n*LG(Vg`1)g6!(XtDzsov$Dvz|Gv1WU68J$CkshQhS zCrc|cdkW~UK}5NeaWj^F4MSgFM+@fJd{|LLM)}_O<{rj z+?*Lm?owq?IzC%U%9EBga~h-cJbIu=#C}XuWN>OLrc%M@Gu~kFEYUi4EC6l#PR2JS zQUkGKrrS#6H7}2l0F@S11DP`@pih0WRkRJl#F;u{c&ZC{^$Z+_*lB)r)-bPgRFE;* zl)@hK4`tEP=P=il02x7-C7p%l=B`vkYjw?YhdJU9!P!jcmY$OtC^12w?vy3<<=tlY zUwHJ_0lgWN9vf>1%WACBD{UT)1qHQSE2%z|JHvP{#INr13jM}oYv_5#xsnv9`)UAO zuwgyV4YZ;O)eSc3(mka6=aRohi!HH@I#xq7kng?Acdg7S4vDJb6cI5fw?2z%3yR+| zU5v@Hm}vy;${cBp&@D=HQ9j7NcFaOYL zj-wV=eYF{|XTkFNM2uz&T8uH~;)^Zo!=KP)EVyH6s9l1~4m}N%XzPpduPg|h-&lL` zAXspR0YMOKd2yO)eMFFJ4?sQ&!`dF&!|niH*!^*Ml##o0M(0*uK9&yzekFi$+mP9s z>W9d%Jb)PtVi&-Ha!o~Iyh@KRuKpQ@)I~L*d`{O8!kRObjO7=n+Gp36fe!66neh+7 zW*l^0tTKjLLzr`x4`_8&on?mjW-PzheTNox8Hg7Nt@*SbE-%kP2hWYmHu#Fn@Q^J(SsPUz*|EgOoZ6byg3ew88UGdZ>9B2Tq=jF72ZaR=4u%1A6Vm{O#?@dD!(#tmR;eP(Fu z{$0O%=Vmua7=Gjr8nY%>ul?w=FJ76O2js&17W_iq2*tb!i{pt#`qZB#im9Rl>?t?0c zicIC}et_4d+CpVPx)i4~$u6N-QX3H77ez z?ZdvXifFk|*F8~L(W$OWM~r`pSk5}#F?j_5u$Obu9lDWIknO^AGu+Blk7!9Sb;NjS zncZA?qtASdNtzQ>z7N871IsPAk^CC?iIL}+{K|F@BuG2>qQ;_RUYV#>hHO(HUPpk@ z(bn~4|F_jiZi}Sad;_7`#4}EmD<1EiIxa48QjUuR?rC}^HRocq`OQPM@aHVKP9E#q zy%6bmHygCpIddPjE}q_DPC`VH_2m;Eey&ZH)E6xGeStOK7H)#+9y!%-Hm|QF6w#A( zIC0Yw%9j$s-#odxG~C*^MZ?M<+&WJ+@?B_QPUyTg9DJGtQN#NIC&-XddRsf3n^AL6 zT@P|H;PvN;ZpL0iv$bRb7|J{0o!Hq+S>_NrH4@coZtBJu#g8#CbR7|#?6uxi8d+$g z87apN>EciJZ`%Zv2**_uiET9Vk{pny&My;+WfGDw4EVL#B!Wiw&M|A8f1A@ z(yFQS6jfbH{b8Z-S7D2?Ixl`j0{+ZnpT=;KzVMLW{B$`N?Gw^Fl0H6lT61%T2AU**!sX0u?|I(yoy&Xveg7XBL&+>n6jd1##6d>TxE*Vj=8lWiG$4=u{1UbAa5QD>5_ z;Te^42v7K6Mmu4IWT6Rnm>oxrl~b<~^e3vbj-GCdHLIB_>59}Ya+~OF68NiH=?}2o zP(X7EN=quQn&)fK>M&kqF|<_*H`}c zk=+x)GU>{Af#vx&s?`UKUsz})g^Pc&?Ka@t5$n$bqf6{r1>#mWx6Ep>9|A}VmWRnowVo`OyCr^fHsf# zQjQ3Ttp7y#iQY8l`zEUW)(@gGQdt(~rkxlkefskT(t%@i8=|p1Y9Dc5bc+z#n$s13 zGJk|V0+&Ekh(F};PJzQKKo+FG@KV8a<$gmNSD;7rd_nRdc%?9)p!|B-@P~kxQG}~B zi|{0}@}zKC(rlFUYp*dO1RuvPC^DQOkX4<+EwvBAC{IZQdYxoq1Za!MW7%p7gGr=j zzWnAq%)^O2$eItftC#TTSArUyL$U54-O7e|)4_7%Q^2tZ^0-d&3J1}qCzR4dWX!)4 zzIEKjgnYgMus^>6uw4Jm8ga6>GBtMjpNRJ6CP~W=37~||gMo_p@GA@#-3)+cVYnU> zE5=Y4kzl+EbEh%dhQokB{gqNDqx%5*qBusWV%!iprn$S!;oN_6E3?0+umADVs4ako z?P+t?m?};gev9JXQ#Q&KBpzkHPde_CGu-y z<{}RRAx=xlv#mVi+Ibrgx~ujW$h{?zPfhz)Kp7kmYS&_|97b&H&1;J-mzrBWAvY} zh8-I8hl_RK2+nnf&}!W0P+>5?#?7>npshe<1~&l_xqKd0_>dl_^RMRq@-Myz&|TKZBj1=Q()) zF{dBjv5)h=&Z)Aevx}+i|7=R9rG^Di!sa)sZCl&ctX4&LScQ-kMncgO(9o6W6)yd< z@Rk!vkja*X_N3H=BavGoR0@u0<}m-7|2v!0+2h~S2Q&a=lTH91OJsvms2MT~ zY=c@LO5i`mLpBd(vh|)I&^A3TQLtr>w=zoyzTd=^f@TPu&+*2MtqE$Avf>l>}V|3-8Fp2hzo3y<)hr_|NO(&oSD z!vEjTWBxbKTiShVl-U{n*B3#)3a8$`{~Pk}J@elZ=>Pqp|MQ}jrGv7KrNcjW%TN_< zZz8kG{#}XoeWf7qY?D)L)8?Q-b@Na&>i=)(@uNo zr;cH98T3$Iau8Hn*@vXi{A@YehxDE2zX~o+RY`)6-X{8~hMpc#C`|8y> zU8Mnv5A0dNCf{Ims*|l-^ z(MRp{qoGohB34|ggDI*p!Aw|MFyJ|v+<+E3brfrI)|+l3W~CQLPbnF@G0)P~Ly!1TJLp}xh8uW`Q+RB-v`MRYZ9Gam3cM%{ zb4Cb*f)0deR~wtNb*8w-LlIF>kc7DAv>T0D(a3@l`k4TFnrO+g9XH7;nYOHxjc4lq zMmaW6qpgAgy)MckYMhl?>sq;-1E)-1llUneeA!ya9KM$)DaNGu57Z5aE>=VST$#vb zFo=uRHr$0M{-ha>h(D_boS4zId;3B|Tpqo|?B?Z@I?G(?&Iei+-{9L_A9=h=Qfn-U z1wIUnQe9!z%_j$F_{rf&`ZFSott09gY~qrf@g3O=Y>vzAnXCyL!@(BqWa)Zqt!#_k zfZHuwS52|&&)aK;CHq9V-t9qt0au{$#6c*R#e5n3rje0hic7c7m{kW$p(_`wB=Gw7 z4k`1Hi;Mc@yA7dp@r~?@rfw)TkjAW++|pkfOG}0N|2guek}j8Zen(!+@7?qt_7ndX zB=BG6WJ31#F3#Vk3=aQr8T)3`{=p9nBHlKzE0I@v`{vJ}h8pd6vby&VgFhzH|q;=aonunAXL6G2y(X^CtAhWr*jI zGjpY@raZDQkg*aMq}Ni6cRF z{oWv}5`nhSAv>usX}m^GHt`f(t8@zHc?K|y5Zi=4G*UG1Sza{$Dpj%X8 zzEXaKT5N6F5j4J|w#qlZP!zS7BT)9b+!ZSJdToqJts1c!)fwih4d31vfb{}W)EgcA zH2pZ^8_k$9+WD2n`6q5XbOy8>3pcYH9 z07eUB+p}YD@AH!}p!iKv><2QF-Y^&xx^PAc1F13A{nUeCDg&{hnix#FiO!fe(^&%Qcux!h znu*S!s$&nnkeotYsDthh1dq(iQrE|#f_=xVgfiiL&-5eAcC-> z5L0l|DVEM$#ulf{bj+Y~7iD)j<~O8CYM8GW)dQGq)!mck)FqoL^X zwNdZb3->hFrbHFm?hLvut-*uK?zXn3q1z|UX{RZ;-WiLoOjnle!xs+W0-8D)kjU#R z+S|A^HkRg$Ij%N4v~k`jyHffKaC~=wg=9)V5h=|kLQ@;^W!o2^K+xG&2n`XCd>OY5Ydi= zgHH=lgy++erK8&+YeTl7VNyVm9-GfONlSlVb3)V9NW5tT!cJ8d7X)!b-$fb!s76{t z@d=Vg-5K_sqHA@Zx-L_}wVnc@L@GL9_K~Zl(h5@AR#FAiKad8~KeWCo@mgXIQ#~u{ zgYFwNz}2b6Vu@CP0XoqJ+dm8px(5W5-Jpis97F`+KM)TuP*X8H@zwiVKDKGVp59pI zifNHZr|B+PG|7|Y<*tqap0CvG7tbR1R>jn70t1X`XJixiMVcHf%Ez*=xm1(CrTSDt z0cle!+{8*Ja&EOZ4@$qhBuKQ$U95Q%rc7tg$VRhk?3=pE&n+T3upZg^ZJc9~c2es% zh7>+|mrmA-p&v}|OtxqmHIBgUxL~^0+cpfkSK2mhh+4b=^F1Xgd2)}U*Yp+H?ls#z zrLxWg_hm}AfK2XYWr!rzW4g;+^^&bW%LmbtRai9f3PjU${r@n`JThy-cphbcwn)rq9{A$Ht`lmYKxOacy z6v2R(?gHhD5@&kB-Eg?4!hAoD7~(h>(R!s1c1Hx#s9vGPePUR|of32bS`J5U5w{F) z>0<^ktO2UHg<0{oxkdOQ;}coZDQph8p6ruj*_?uqURCMTac;>T#v+l1Tc~%^k-Vd@ zkc5y35jVNc49vZpZx;gG$h{%yslDI%Lqga1&&;mN{Ush1c7p>7e-(zp}6E7f-XmJb4nhk zb8zS+{IVbL$QVF8pf8}~kQ|dHJAEATmmnrb_wLG}-yHe>W|A&Y|;muy-d^t^<&)g5SJfaTH@P1%euONny=mxo+C z4N&w#biWY41r8k~468tvuYVh&XN&d#%QtIf9;iVXfWY)#j=l`&B~lqDT@28+Y!0E+MkfC}}H*#(WKKdJJq=O$vNYCb(ZG@p{fJgu;h z21oHQ(14?LeT>n5)s;uD@5&ohU!@wX8w*lB6i@GEH0pM>YTG+RAIWZD;4#F1&F%Jp zXZUml2sH0!lYJT?&sA!qwez6cXzJEd(1ZC~kT5kZSp7(@=H2$Azb_*W&6aA|9iwCL zdX7Q=42;@dspHDwYE?miGX#L^3xD&%BI&fN9^;`v4OjQXPBaBmOF1;#C)8XA(WFlH zycro;DS2?(G&6wkr6rqC>rqDv3nfGw3hmN_9Al>TgvmGsL8_hXx09};l9Ow@)F5@y z#VH5WigLDwZE4nh^7&@g{1FV^UZ%_LJ-s<{HN*2R$OPg@R~Z`c-ET*2}XB@9xvAjrK&hS=f|R8Gr9 zr|0TGOsI7RD+4+2{ZiwdVD@2zmg~g@^D--YL;6UYGSM8i$NbQr4!c7T9rg!8;TM0E zT#@?&S=t>GQm)*ua|?TLT2ktj#`|R<_*FAkOu2Pz$wEc%-=Y9V*$&dg+wIei3b*O8 z2|m$!jJG!J!ZGbbIa!(Af~oSyZV+~M1qGvelMzPNE_%5?c2>;MeeG2^N?JDKjFYCy z7SbPWH-$cWF9~fX%9~v99L!G(wi!PFp>rB!9xj7=Cv|F+7CsGNwY0Q_J%FID%C^CBZQfJ9K(HK%k31j~e#&?hQ zNuD6gRkVckU)v+53-fc} z7ZCzYN-5RG4H7;>>Hg?LU9&5_aua?A0)0dpew1#MMlu)LHe(M;OHjHIUl7|%%)YPo z0cBk;AOY00%Fe6heoN*$(b<)Cd#^8Iu;-2v@>cE-OB$icUF9EEoaC&q8z9}jMTT2I z8`9;jT%z0;dy4!8U;GW{i`)3!c6&oWY`J3669C!tM<5nQFFrFRglU8f)5Op$GtR-3 zn!+SPCw|04sv?%YZ(a7#L?vsdr7ss@WKAw&A*}-1S|9~cL%uA+E~>N6QklFE>8W|% zyX-qAUGTY1hQ-+um`2|&ji0cY*(qN!zp{YpDO-r>jPk*yuVSay<)cUt`t@&FPF_&$ zcHwu1(SQ`I-l8~vYyUxm@D1UEdFJ$f5Sw^HPH7b!9 zzYT3gKMF((N(v0#4f_jPfVZ=ApN^jQJe-X$`A?X+vWjLn_%31KXE*}5_}d8 zw_B1+a#6T1?>M{ronLbHIlEsMf93muJ7AH5h%;i99<~JX^;EAgEB1uHralD*!aJ@F zV2ruuFe9i2Q1C?^^kmVy921eb=tLDD43@-AgL^rQ3IO9%+vi_&R2^dpr}x{bCVPej z7G0-0o64uyWNtr*loIvslyo0%)KSDDKjfThe0hcqs)(C-MH1>bNGBDRTW~scy_{w} zp^aq8Qb!h9Lwielq%C1b8=?Z=&U)ST&PHbS)8Xzjh2DF?d{iAv)Eh)wsUnf>UtXN( zL7=$%YrZ#|^c{MYmhn!zV#t*(jdmYdCpwqpZ{v&L8KIuKn`@IIZfp!uo}c;7J57N` zAxyZ-uA4=Gzl~Ovycz%MW9ZL7N+nRo&1cfNn9(1H5eM;V_4Z_qVann7F>5f>%{rf= zPBZFaV@_Sobl?Fy&KXyzFDV*FIdhS5`Uc~S^Gjo)aiTHgn#<0C=9o-a-}@}xDor;D zZyZ|fvf;+=3MZd>SR1F^F`RJEZo+|MdyJYQAEauKu%WDol~ayrGU3zzbHKsnHKZ*z zFiwUkL@DZ>!*x05ql&EBq@_Vqv83&?@~q5?lVmffQZ+V-=qL+!u4Xs2Z2zdCQ3U7B&QR9_Iggy} z(om{Y9eU;IPe`+p1ifLx-XWh?wI)xU9ik+m#g&pGdB5Bi<`PR*?92lE0+TkRuXI)z z5LP!N2+tTc%cB6B1F-!fj#}>S!vnpgVU~3!*U1ej^)vjUH4s-bd^%B=ItQqDCGbrEzNQi(dJ`J}-U=2{7-d zK8k^Rlq2N#0G?9&1?HSle2vlkj^KWSBYTwx`2?9TU_DX#J+f+qLiZCqY1TXHFxXZqYMuD@RU$TgcnCC{_(vwZ-*uX)~go#%PK z@}2Km_5aQ~(<3cXeJN6|F8X_1@L%@xTzs}$_*E|a^_URF_qcF;Pfhoe?FTFwvjm1o z8onf@OY@jC2tVcMaZS;|T!Ks(wOgPpRzRnFS-^RZ4E!9dsnj9sFt609a|jJbb1Dt@ z<=Gal2jDEupxUSwWu6zp<<&RnAA;d&4gKVG0iu6g(DsST(4)z6R)zDpfaQ}v{5ARt zyhwvMtF%b-YazR5XLz+oh=mn;y-Mf2a8>7?2v8qX;19y?b>Z5laGHvzH;Nu9S`B8} zI)qN$GbXIQ1VL3lnof^6TS~rvPVg4V?Dl2Bb*K2z4E{5vy<(@@K_cN@U>R!>aUIRnb zL*)=787*cs#zb31zBC49x$`=fkQbMAef)L2$dR{)6BAz!t5U_B#1zZG`^neKSS22oJ#5B=gl%U=WeqL9REF2g zZnfCb0?quf?Ztj$VXvDSWoK`0L=Zxem2q}!XWLoT-kYMOx)!7fcgT35uC~0pySEme z`{wGWTkGr7>+Kb^n;W?BZH6ZP(9tQX%-7zF>vc2}LuWDI(9kh1G#7B99r4x6;_-V+k&c{nPUrR zAXJGRiMe~aup{0qzmLNjS_BC4cB#sXjckx{%_c&^xy{M61xEb>KW_AG5VFXUOjAG4 z^>Qlm9A#1N{4snY=(AmWzatb!ngqiqPbBZ7>Uhb3)dTkSGcL#&SH>iMO-IJBPua`u zo)LWZ>=NZLr758j{%(|uQuZ)pXq_4c!!>s|aDM9#`~1bzK3J1^^D#<2bNCccH7~-X}Ggi!pIIF>uFx%aPARGQsnC8ZQc8lrQ5o~smqOg>Ti^GNme94*w z)JZy{_{#$jxGQ&`M z!OMvZMHR>8*^>eS%o*6hJwn!l8VOOjZQJvh)@tnHVW&*GYPuxqXw}%M!(f-SQf`=L z5;=5w2;%82VMH6Xi&-K3W)o&K^+vJCepWZ-rW%+Dc6X3(){z$@4zjYxQ|}8UIojeC zYZpQ1dU{fy=oTr<4VX?$q)LP}IUmpiez^O&N3E_qPpchGTi5ZM6-2ScWlQq%V&R2Euz zO|Q0Hx>lY1Q1cW5xHv5!0OGU~PVEqSuy#fD72d#O`N!C;o=m+YioGu-wH2k6!t<~K zSr`E=W9)!g==~x9VV~-8{4ZN9{~-A9zJpRe%NGg$+MDuI-dH|b@BD)~>pPCGUNNzY zMDg||0@XGQgw`YCt5C&A{_+J}mvV9Wg{6V%2n#YSRN{AP#PY?1FF1#|vO_%e+#`|2*~wGAJaeRX6=IzFNeWhz6gJc8+(03Ph4y6ELAm=AkN7TOgMUEw*N{= z_)EIDQx5q22oUR+_b*tazu9+pX|n1c*IB-}{DqIj z-?E|ks{o3AGRNb;+iKcHkZvYJvFsW&83RAPs1Oh@IWy%l#5x2oUP6ZCtv+b|q>jsf zZ_9XO;V!>n`UxH1LvH8)L4?8raIvasEhkpQoJ`%!5rBs!0Tu(s_D{`4opB;57)pkX z4$A^8CsD3U5*!|bHIEqsn~{q+Ddj$ME@Gq4JXtgVz&7l{Ok!@?EA{B3P~NAqb9)4? zkQo30A^EbHfQ@87G5&EQTd`frrwL)&Yw?%-W@uy^Gn23%j?Y!Iea2xw<-f;esq zf%w5WN@E1}zyXtYv}}`U^B>W`>XPmdLj%4{P298|SisrE;7HvXX;A}Ffi8B#3Lr;1 zHt6zVb`8{#+e$*k?w8|O{Uh|&AG}|DG1PFo1i?Y*cQm$ZwtGcVgMwtBUDa{~L1KT-{jET4w60>{KZ27vXrHJ;fW{6| z=|Y4!&UX020wU1>1iRgB@Q#m~1^Z^9CG1LqDhYBrnx%IEdIty z!46iOoKlKs)c}newDG)rWUikD%j`)p z_w9Ph&e40=(2eBy;T!}*1p1f1SAUDP9iWy^u^Ubdj21Kn{46;GR+hwLO=4D11@c~V zI8x&(D({K~Df2E)Nx_yQvYfh4;MbMJ@Z}=Dt3_>iim~QZ*hZIlEs0mEb z_54+&*?wMD`2#vsQRN3KvoT>hWofI_Vf(^C1ff-Ike@h@saEf7g}<9T`W;HAne-Nd z>RR+&SP35w)xKn8^U$7))PsM!jKwYZ*RzEcG-OlTrX3}9a{q%#Un5E5W{{hp>w~;` zGky+3(vJvQyGwBo`tCpmo0mo((?nM8vf9aXrrY1Ve}~TuVkB(zeds^jEfI}xGBCM2 zL1|#tycSaWCurP+0MiActG3LCas@_@tao@(R1ANlwB$4K53egNE_;!&(%@Qo$>h`^1S_!hN6 z)vZtG$8fN!|BXBJ=SI>e(LAU(y(i*PHvgQ2llulxS8>qsimv7yL}0q_E5WiAz7)(f zC(ahFvG8&HN9+6^jGyLHM~$)7auppeWh_^zKk&C_MQ~8;N??OlyH~azgz5fe^>~7F zl3HnPN3z-kN)I$4@`CLCMQx3sG~V8hPS^}XDXZrQA>}mQPw%7&!sd(Pp^P=tgp-s^ zjl}1-KRPNWXgV_K^HkP__SR`S-|OF0bR-N5>I%ODj&1JUeAQ3$9i;B~$S6}*^tK?= z**%aCiH7y?xdY?{LgVP}S0HOh%0%LI$wRx;$T|~Y8R)Vdwa}kGWv8?SJVm^>r6+%I z#lj1aR94{@MP;t-scEYQWc#xFA30^}?|BeX*W#9OL;Q9#WqaaM546j5j29((^_8Nu z4uq}ESLr~r*O7E7$D{!k9W>`!SLoyA53i9QwRB{!pHe8um|aDE`Cg0O*{jmor)^t)3`>V>SWN-2VJcFmj^1?~tT=JrP`fVh*t zXHarp=8HEcR#vFe+1a%XXuK+)oFs`GDD}#Z+TJ}Ri`FvKO@ek2ayn}yaOi%(8p%2$ zpEu)v0Jym@f}U|-;}CbR=9{#<^z28PzkkTNvyKvJDZe+^VS2bES3N@Jq!-*}{oQlz z@8bgC_KnDnT4}d#&Cpr!%Yb?E!brx0!eVOw~;lLwUoz#Np%d$o%9scc3&zPm`%G((Le|6o1 zM(VhOw)!f84zG^)tZ1?Egv)d8cdNi+T${=5kV+j;Wf%2{3g@FHp^Gf*qO0q!u$=m9 zCaY`4mRqJ;FTH5`a$affE5dJrk~k`HTP_7nGTY@B9o9vvnbytaID;^b=Tzp7Q#DmD zC(XEN)Ktn39z5|G!wsVNnHi) z%^q94!lL|hF`IijA^9NR0F$@h7k5R^ljOW(;Td9grRN0Mb)l_l7##{2nPQ@?;VjXv zaLZG}yuf$r$<79rVPpXg?6iiieX|r#&`p#Con2i%S8*8F}(E) zI5E6c3tG*<;m~6>!&H!GJ6zEuhH7mkAzovdhLy;)q z{H2*8I^Pb}xC4s^6Y}6bJvMu=8>g&I)7!N!5QG$xseeU#CC?ZM-TbjsHwHgDGrsD= z{%f;@Sod+Ch66Ko2WF~;Ty)v>&x^aovCbCbD7>qF*!?BXmOV3(s|nxsb*Lx_2lpB7 zokUnzrk;P=T-&kUHO}td+Zdj!3n&NR?K~cRU zAXU!DCp?51{J4w^`cV#ye}(`SQhGQkkMu}O3M*BWt4UsC^jCFUy;wTINYmhD$AT;4 z?Xd{HaJjP`raZ39qAm;%beDbrLpbRf(mkKbANan7XsL>_pE2oo^$TgdidjRP!5-`% zv0d!|iKN$c0(T|L0C~XD0aS8t{*&#LnhE;1Kb<9&=c2B+9JeLvJr*AyyRh%@jHej=AetOMSlz^=!kxX>>B{2B1uIrQyfd8KjJ+DBy!h)~*(!|&L4^Q_07SQ~E zcemVP`{9CwFvPFu7pyVGCLhH?LhEVb2{7U+Z_>o25#+3<|8%1T^5dh}*4(kfJGry} zm%r#hU+__Z;;*4fMrX=Bkc@7|v^*B;HAl0((IBPPii%X9+u3DDF6%bI&6?Eu$8&aWVqHIM7mK6?Uvq$1|(-T|)IV<>e?!(rY zqkmO1MRaLeTR=)io(0GVtQT@s6rN%C6;nS3@eu;P#ry4q;^O@1ZKCJyp_Jo)Ty^QW z+vweTx_DLm{P-XSBj~Sl<%_b^$=}odJ!S2wAcxenmzFGX1t&Qp8Vxz2VT`uQsQYtdn&_0xVivIcxZ_hnrRtwq4cZSj1c-SG9 z7vHBCA=fd0O1<4*=lu$6pn~_pVKyL@ztw1swbZi0B?spLo56ZKu5;7ZeUml1Ws1?u zqMf1p{5myAzeX$lAi{jIUqo1g4!zWLMm9cfWcnw`k6*BR^?$2(&yW?>w;G$EmTA@a z6?y#K$C~ZT8+v{87n5Dm&H6Pb_EQ@V0IWmG9cG=O;(;5aMWWrIPzz4Q`mhK;qQp~a z+BbQrEQ+w{SeiuG-~Po5f=^EvlouB@_|4xQXH@A~KgpFHrwu%dwuCR)=B&C(y6J4J zvoGk9;lLs9%iA-IJGU#RgnZZR+@{5lYl8(e1h6&>Vc_mvg0d@);X zji4T|n#lB!>pfL|8tQYkw?U2bD`W{na&;*|znjmalA&f;*U++_aBYerq;&C8Kw7mI z7tsG*?7*5j&dU)Lje;^{D_h`%(dK|pB*A*1(Jj)w^mZ9HB|vGLkF1GEFhu&rH=r=8 zMxO42e{Si6$m+Zj`_mXb&w5Q(i|Yxyg?juUrY}78uo@~3v84|8dfgbPd0iQJRdMj< zncCNGdMEcsxu#o#B5+XD{tsg*;j-eF8`mp~K8O1J!Z0+>0=7O=4M}E?)H)ENE;P*F z$Ox?ril_^p0g7xhDUf(q652l|562VFlC8^r8?lQv;TMvn+*8I}&+hIQYh2 z1}uQQaag&!-+DZ@|C+C$bN6W;S-Z@)d1|en+XGvjbOxCa-qAF*LA=6s(Jg+g;82f$ z(Vb)8I)AH@cdjGFAR5Rqd0wiNCu!xtqWbcTx&5kslzTb^7A78~Xzw1($UV6S^VWiP zFd{Rimd-0CZC_Bu(WxBFW7+k{cOW7DxBBkJdJ;VsJ4Z@lERQr%3eVv&$%)b%<~ zCl^Y4NgO}js@u{|o~KTgH}>!* z_iDNqX2(As7T0xivMH|3SC1ivm8Q}6Ffcd7owUKN5lHAtzMM4<0v+ykUT!QiowO;`@%JGv+K$bBx@*S7C8GJVqQ_K>12}M`f_Ys=S zKFh}HM9#6Izb$Y{wYzItTy+l5U2oL%boCJn?R3?jP@n$zSIwlmyGq30Cw4QBO|14` zW5c);AN*J3&eMFAk$SR~2k|&+&Bc$e>s%c{`?d~85S-UWjA>DS5+;UKZ}5oVa5O(N zqqc@>)nee)+4MUjH?FGv%hm2{IlIF-QX}ym-7ok4Z9{V+ZHVZQl$A*x!(q%<2~iVv znUa+BX35&lCb#9VE-~Y^W_f;Xhl%vgjwdjzMy$FsSIj&ok}L+X`4>J=9BkN&nu^E*gbhj3(+D>C4E z@Fwq_=N)^bKFSHTzZk?-gNU$@l}r}dwGyh_fNi=9b|n}J>&;G!lzilbWF4B}BBq4f zYIOl?b)PSh#XTPp4IS5ZR_2C!E)Z`zH0OW%4;&~z7UAyA-X|sh9@~>cQW^COA9hV4 zXcA6qUo9P{bW1_2`eo6%hgbN%(G-F1xTvq!sc?4wN6Q4`e9Hku zFwvlAcRY?6h^Fj$R8zCNEDq8`=uZB8D-xn)tA<^bFFy}4$vA}Xq0jAsv1&5!h!yRA zU()KLJya5MQ`q&LKdH#fwq&(bNFS{sKlEh_{N%{XCGO+po#(+WCLmKW6&5iOHny>g z3*VFN?mx!16V5{zyuMWDVP8U*|BGT$(%IO|)?EF|OI*sq&RovH!N%=>i_c?K*A>>k zyg1+~++zY4Q)J;VWN0axhoIKx;l&G$gvj(#go^pZskEVj8^}is3Jw26LzYYVos0HX zRPvmK$dVxM8(Tc?pHFe0Z3uq){{#OK3i-ra#@+;*=ui8)y6hsRv z4Fxx1c1+fr!VI{L3DFMwXKrfl#Q8hfP@ajgEau&QMCxd{g#!T^;ATXW)nUg&$-n25 zruy3V!!;{?OTobo|0GAxe`Acn3GV@W=&n;~&9 zQM>NWW~R@OYORkJAo+eq1!4vzmf9K%plR4(tB@TR&FSbDoRgJ8qVcH#;7lQub*nq&?Z>7WM=oeEVjkaG zT#f)=o!M2DO5hLR+op>t0CixJCIeXH*+z{-XS|%jx)y(j&}Wo|3!l7{o)HU3m7LYyhv*xF&tq z%IN7N;D4raue&&hm0xM=`qv`+TK@;_xAcGKuK(2|75~ar2Yw)geNLSmVxV@x89bQu zpViVKKnlkwjS&&c|-X6`~xdnh}Ps)Hs z4VbUL^{XNLf7_|Oi>tA%?SG5zax}esF*FH3d(JH^Gvr7Rp*n=t7frH!U;!y1gJB^i zY_M$KL_}mW&XKaDEi9K-wZR|q*L32&m+2n_8lq$xRznJ7p8}V>w+d@?uB!eS3#u<} zIaqi!b!w}a2;_BfUUhGMy#4dPx>)_>yZ`ai?Rk`}d0>~ce-PfY-b?Csd(28yX22L% zI7XI>OjIHYTk_@Xk;Gu^F52^Gn6E1&+?4MxDS2G_#PQ&yXPXP^<-p|2nLTb@AAQEY zI*UQ9Pmm{Kat}wuazpjSyXCdnrD&|C1c5DIb1TnzF}f4KIV6D)CJ!?&l&{T)e4U%3HTSYqsQ zo@zWB1o}ceQSV)<4G<)jM|@@YpL+XHuWsr5AYh^Q{K=wSV99D~4RRU52FufmMBMmd z_H}L#qe(}|I9ZyPRD6kT>Ivj&2Y?qVZq<4bG_co_DP`sE*_Xw8D;+7QR$Uq(rr+u> z8bHUWbV19i#)@@G4bCco@Xb<8u~wVDz9S`#k@ciJtlu@uP1U0X?yov8v9U3VOig2t zL9?n$P3=1U_Emi$#slR>N5wH-=J&T=EdUHA}_Z zZIl3nvMP*AZS9{cDqFanrA~S5BqxtNm9tlu;^`)3X&V4tMAkJ4gEIPl= zoV!Gyx0N{3DpD@)pv^iS*dl2FwANu;1;%EDl}JQ7MbxLMAp>)UwNwe{=V}O-5C*>F zu?Ny+F64jZn<+fKjF01}8h5H_3pey|;%bI;SFg$w8;IC<8l|3#Lz2;mNNik6sVTG3 z+Su^rIE#40C4a-587$U~%KedEEw1%r6wdvoMwpmlXH$xPnNQN#f%Z7|p)nC>WsuO= z4zyqapLS<8(UJ~Qi9d|dQijb_xhA2)v>la)<1md5s^R1N&PiuA$^k|A<+2C?OiHbj z>Bn$~t)>Y(Zb`8hW7q9xQ=s>Rv81V+UiuZJc<23HplI88isqRCId89fb`Kt|CxVIg znWcwprwXnotO>3s&Oypkte^9yJjlUVVxSe%_xlzmje|mYOVPH^vjA=?6xd0vaj0Oz zwJ4OJNiFdnHJX3rw&inskjryukl`*fRQ#SMod5J|KroJRsVXa5_$q7whSQ{gOi*s0 z1LeCy|JBWRsDPn7jCb4s(p|JZiZ8+*ExC@Vj)MF|*Vp{B(ziccSn`G1Br9bV(v!C2 z6#?eqpJBc9o@lJ#^p-`-=`4i&wFe>2)nlPK1p9yPFzJCzBQbpkcR>={YtamIw)3nt z(QEF;+)4`>8^_LU)_Q3 zC5_7lgi_6y>U%m)m@}Ku4C}=l^J=<<7c;99ec3p{aR+v=diuJR7uZi%aQv$oP?dn?@6Yu_+*^>T0ptf(oobdL;6)N-I!TO`zg^Xbv3#L0I~sn@WGk-^SmPh5>W+LB<+1PU}AKa?FCWF|qMNELOgdxR{ zbqE7@jVe+FklzdcD$!(A$&}}H*HQFTJ+AOrJYnhh}Yvta(B zQ_bW4Rr;R~&6PAKwgLWXS{Bnln(vUI+~g#kl{r+_zbngT`Y3`^Qf=!PxN4IYX#iW4 zucW7@LLJA9Zh3(rj~&SyN_pjO8H&)|(v%!BnMWySBJV=eSkB3YSTCyIeJ{i;(oc%_hk{$_l;v>nWSB)oVeg+blh=HB5JSlG_r7@P z3q;aFoZjD_qS@zygYqCn=;Zxjo!?NK!%J$ z52lOP`8G3feEj+HTp@Tnn9X~nG=;tS+z}u{mQX_J0kxtr)O30YD%oo)L@wy`jpQYM z@M>Me=95k1p*FW~rHiV1CIfVc{K8r|#Kt(ApkXKsDG$_>76UGNhHExFCw#Ky9*B-z zNq2ga*xax!HMf_|Vp-86r{;~YgQKqu7%szk8$hpvi_2I`OVbG1doP(`gn}=W<8%Gn z%81#&WjkH4GV;4u43EtSW>K_Ta3Zj!XF?;SO3V#q=<=>Tc^@?A`i;&`-cYj|;^ zEo#Jl5zSr~_V-4}y8pnufXLa80vZY4z2ko7fj>DR)#z=wWuS1$$W!L?(y}YC+yQ|G z@L&`2upy3f>~*IquAjkVNU>}c10(fq#HdbK$~Q3l6|=@-eBbo>B9(6xV`*)sae58*f zym~RRVx;xoCG3`JV`xo z!lFw)=t2Hy)e!IFs?0~7osWk(d%^wxq&>_XD4+U#y&-VF%4z?XH^i4w`TxpF{`XhZ z%G}iEzf!T(l>g;W9<~K+)$g!{UvhW{E0Lis(S^%I8OF&%kr!gJ&fMOpM=&=Aj@wuL zBX?*6i51Qb$uhkwkFYkaD_UDE+)rh1c;(&Y=B$3)J&iJfQSx!1NGgPtK!$c9OtJuu zX(pV$bfuJpRR|K(dp@^j}i&HeJOh@|7lWo8^$*o~Xqo z5Sb+!EtJ&e@6F+h&+_1ETbg7LfP5GZjvIUIN3ibCOldAv z)>YdO|NH$x7AC8dr=<2ekiY1%fN*r~e5h6Yaw<{XIErujKV~tiyrvV_DV0AzEknC- zR^xKM3i<1UkvqBj3C{wDvytOd+YtDSGu!gEMg+!&|8BQrT*|p)(dwQLEy+ zMtMzij3zo40)CA!BKZF~yWg?#lWhqD3@qR)gh~D{uZaJO;{OWV8XZ_)J@r3=)T|kt zUS1pXr6-`!Z}w2QR7nP%d?ecf90;K_7C3d!UZ`N(TZoWNN^Q~RjVhQG{Y<%E1PpV^4 z-m-K+$A~-+VDABs^Q@U*)YvhY4Znn2^w>732H?NRK(5QSS$V@D7yz2BVX4)f5A04~$WbxGOam22>t&uD)JB8-~yiQW6ik;FGblY_I>SvB_z2?PS z*Qm&qbKI{H1V@YGWzpx`!v)WeLT02};JJo*#f$a*FH?IIad-^(;9XC#YTWN6;Z6+S zm4O1KH=#V@FJw7Pha0!9Vb%ZIM$)a`VRMoiN&C|$YA3~ZC*8ayZRY^fyuP6$n%2IU z$#XceYZeqLTXw(m$_z|33I$B4k~NZO>pP6)H_}R{E$i%USGy{l{-jOE;%CloYPEU+ zRFxOn4;7lIOh!7abb23YKD+_-?O z0FP9otcAh+oSj;=f#$&*ExUHpd&e#bSF%#8*&ItcL2H$Sa)?pt0Xtf+t)z$_u^wZi z44oE}r4kIZGy3!Mc8q$B&6JqtnHZ>Znn!Zh@6rgIu|yU+zG8q`q9%B18|T|oN3zMq z`l&D;U!OL~%>vo&q0>Y==~zLiCZk4v%s_7!9DxQ~id1LLE93gf*gg&2$|hB#j8;?3 z5v4S;oM6rT{Y;I+#FdmNw z){d%tNM<<#GN%n9ox7B=3#;u7unZ~tLB_vRZ52a&2=IM)2VkXm=L+Iqq~uk#Dug|x z>S84e+A7EiOY5lj*!q?6HDkNh~0g;0Jy(al!ZHHDtur9T$y-~)94HelX1NHjXWIM7UAe}$?jiz z9?P4`I0JM=G5K{3_%2jPLC^_Mlw?-kYYgb7`qGa3@dn|^1fRMwiyM@Ch z;CB&o7&&?c5e>h`IM;Wnha0QKnEp=$hA8TJgR-07N~U5(>9vJzeoFsSRBkDq=x(YgEMpb=l4TDD`2 zwVJpWGTA_u7}?ecW7s6%rUs&NXD3+n;jB86`X?8(l3MBo6)PdakI6V6a}22{)8ilT zM~T*mU}__xSy|6XSrJ^%lDAR3Lft%+yxC|ZUvSO_nqMX!_ul3;R#*{~4DA=h$bP)%8Yv9X zyp><|e8=_ttI}ZAwOd#dlnSjck#6%273{E$kJuCGu=I@O)&6ID{nWF5@gLb16sj|&Sb~+du4e4O_%_o`Ix4NRrAsyr1_}MuP94s>de8cH-OUkVPk3+K z&jW)It9QiU-ti~AuJkL`XMca8Oh4$SyJ=`-5WU<{cIh+XVH#e4d&zive_UHC!pN>W z3TB;Mn5i)9Qn)#6@lo4QpI3jFYc0~+jS)4AFz8fVC;lD^+idw^S~Qhq>Tg(!3$yLD zzktzoFrU@6s4wwCMz}edpF5i5Q1IMmEJQHzp(LAt)pgN3&O!&d?3W@6U4)I^2V{;- z6A(?zd93hS*uQmnh4T)nHnE{wVhh(=MMD(h(P4+^p83Om6t<*cUW>l(qJzr%5vp@K zN27ka(L{JX=1~e2^)F^i=TYj&;<7jyUUR2Bek^A8+3Up*&Xwc{)1nRR5CT8vG>ExV zHnF3UqXJOAno_?bnhCX-&kwI~Ti8t4`n0%Up>!U`ZvK^w2+0Cs-b9%w%4`$+To|k= zKtgc&l}P`*8IS>8DOe?EB84^kx4BQp3<7P{Pq}&p%xF_81pg!l2|u=&I{AuUgmF5n zJQCTLv}%}xbFGYtKfbba{CBo)lWW%Z>i(_NvLhoQZ*5-@2l&x>e+I~0Nld3UI9tdL zRzu8}i;X!h8LHVvN?C+|M81e>Jr38%&*9LYQec9Ax>?NN+9(_>XSRv&6hlCYB`>Qm z1&ygi{Y()OU4@D_jd_-7vDILR{>o|7-k)Sjdxkjgvi{@S>6GqiF|o`*Otr;P)kLHN zZkpts;0zw_6;?f(@4S1FN=m!4^mv~W+lJA`&7RH%2$)49z0A+8@0BCHtj|yH--AEL z0tW6G%X-+J+5a{5*WKaM0QDznf;V?L5&uQw+yegDNDP`hA;0XPYc6e0;Xv6|i|^F2WB)Z$LR|HR4 zTQsRAby9(^Z@yATyOgcfQw7cKyr^3Tz7lc7+JEwwzA7)|2x+PtEb>nD(tpxJQm)Kn zW9K_*r!L%~N*vS8<5T=iv|o!zTe9k_2jC_j*7ik^M_ zaf%k{WX{-;0*`t`G!&`eW;gChVXnJ-Rn)To8vW-?>>a%QU1v`ZC=U)f8iA@%JG0mZ zDqH;~mgBnrCP~1II<=V9;EBL)J+xzCoiRBaeH&J6rL!{4zIY8tZka?_FBeQeNO3q6 zyG_alW54Ba&wQf{&F1v-r1R6ID)PTsqjIBc+5MHkcW5Fnvi~{-FjKe)t1bl}Y;z@< z=!%zvpRua>>t_x}^}z0<7MI!H2v6|XAyR9!t50q-A)xk0nflgF4*OQlCGK==4S|wc zRMsSscNhRzHMBU8TdcHN!q^I}x0iXJ%uehac|Zs_B$p@CnF)HeXPpB_Za}F{<@6-4 zl%kml@}kHQ(ypD8FsPJ2=14xXJE|b20RUIgs!2|R3>LUMGF6X*B_I|$`Qg=;zm7C z{mEDy9dTmPbued7mlO@phdmAmJ7p@GR1bjCkMw6*G7#4+`k>fk1czdJUB!e@Q(~6# zwo%@p@V5RL0ABU2LH7Asq^quDUho@H>eTZH9f*no9fY0T zD_-9px3e}A!>>kv5wk91%C9R1J_Nh!*&Kk$J3KNxC}c_@zlgpJZ+5L)Nw|^p=2ue}CJtm;uj*Iqr)K})kA$xtNUEvX;4!Px*^&9T_`IN{D z{6~QY=Nau6EzpvufB^hflc#XIsSq0Y9(nf$d~6ZwK}fal92)fr%T3=q{0mP-EyP_G z)UR5h@IX}3Qll2b0oCAcBF>b*@Etu*aTLPU<%C>KoOrk=x?pN!#f_Og-w+;xbFgjQ zXp`et%lDBBh~OcFnMKMUoox0YwBNy`N0q~bSPh@+enQ=4RUw1) zpovN`QoV>vZ#5LvC;cl|6jPr}O5tu!Ipoyib8iXqy}TeJ;4+_7r<1kV0v5?Kv>fYp zg>9L`;XwXa&W7-jf|9~uP2iyF5`5AJ`Q~p4eBU$MCC00`rcSF>`&0fbd^_eqR+}mK z4n*PMMa&FOcc)vTUR zlDUAn-mh`ahi_`f`=39JYTNVjsTa_Y3b1GOIi)6dY)D}xeshB0T8Eov5%UhWd1)u}kjEQ|LDo{tqKKrYIfVz~@dp!! zMOnah@vp)%_-jDTUG09l+;{CkDCH|Q{NqX*uHa1YxFShy*1+;J`gywKaz|2Q{lG8x zP?KBur`}r`!WLKXY_K;C8$EWG>jY3UIh{+BLv0=2)KH%P}6xE2kg)%(-uA6lC?u8}{K(#P*c zE9C8t*u%j2r_{;Rpe1A{9nNXU;b_N0vNgyK!EZVut~}+R2rcbsHilqsOviYh-pYX= zHw@53nlmwYI5W5KP>&`dBZe0Jn?nAdC^HY1wlR6$u^PbpB#AS&5L6zqrXN&7*N2Q` z+Rae1EwS)H=aVSIkr8Ek^1jy2iS2o7mqm~Mr&g5=jjt7VxwglQ^`h#Mx+x2v|9ZAwE$i_9918MjJxTMr?n!bZ6n$}y11u8I9COTU`Z$Fi z!AeAQLMw^gp_{+0QTEJrhL424pVDp%wpku~XRlD3iv{vQ!lAf!_jyqd_h}+Tr1XG| z`*FT*NbPqvHCUsYAkFnM`@l4u_QH&bszpUK#M~XLJt{%?00GXY?u_{gj3Hvs!=N(I z(=AuWPijyoU!r?aFTsa8pLB&cx}$*%;K$e*XqF{~*rA-qn)h^!(-;e}O#B$|S~c+U zN4vyOK0vmtx$5K!?g*+J@G1NmlEI=pyZXZ69tAv=@`t%ag_Hk{LP~OH9iE)I= zaJ69b4kuCkV0V zo(M0#>phpQ_)@j;h%m{-a*LGi(72TP)ws2w*@4|C-3+;=5DmC4s7Lp95%n%@Ko zfdr3-a7m*dys9iIci$A=4NPJ`HfJ;hujLgU)ZRuJI`n;Pw|yksu!#LQnJ#dJysgNb z@@qwR^wrk(jbq4H?d!lNyy72~Dnn87KxsgQ!)|*m(DRM+eC$wh7KnS-mho3|KE)7h zK3k;qZ;K1Lj6uEXLYUYi)1FN}F@-xJ z@@3Hb84sl|j{4$3J}aTY@cbX@pzB_qM~APljrjju6P0tY{C@ zpUCOz_NFmALMv1*blCcwUD3?U6tYs+N%cmJ98D%3)%)Xu^uvzF zS5O!sc#X6?EwsYkvPo6A%O8&y8sCCQH<%f2togVwW&{M;PR!a(ZT_A+jVAbf{@5kL zB@Z(hb$3U{T_}SKA_CoQVU-;j>2J=L#lZ~aQCFg-d<9rzs$_gO&d5N6eFSc z1ml8)P*FSi+k@!^M9nDWR5e@ATD8oxtDu=36Iv2!;dZzidIS(PCtEuXAtlBb1;H%Z zwnC^Ek*D)EX4#Q>R$$WA2sxC_t(!!6Tr?C#@{3}n{<^o;9id1RA&-Pig1e-2B1XpG zliNjgmd3c&%A}s>qf{_j#!Z`fu0xIwm4L0)OF=u(OEmp;bLCIaZX$&J_^Z%4Sq4GZ zPn6sV_#+6pJmDN_lx@1;Zw6Md_p0w9h6mHtzpuIEwNn>OnuRSC2=>fP^Hqgc)xu^4 z<3!s`cORHJh#?!nKI`Et7{3C27+EuH)Gw1f)aoP|B3y?fuVfvpYYmmukx0ya-)TQX zR{ggy5cNf4X|g)nl#jC9p>7|09_S7>1D2GTRBUTW zAkQ=JMRogZqG#v;^=11O6@rPPwvJkr{bW-Qg8`q8GoD#K`&Y+S#%&B>SGRL>;ZunM@49!}Uy zN|bBCJ%sO;@3wl0>0gbl3L@1^O60ONObz8ZI7nder>(udj-jt`;yj^nTQ$L9`OU9W zX4alF#$|GiR47%x@s&LV>2Sz2R6?;2R~5k6V>)nz!o_*1Y!$p>BC5&?hJg_MiE6UBy>RkVZj`9UWbRkN-Hk!S`=BS3t3uyX6)7SF#)71*}`~Ogz z1rap5H6~dhBJ83;q-Y<5V35C2&F^JI-it(=5D#v!fAi9p#UwV~2tZQI+W(Dv?1t9? zfh*xpxxO{-(VGB>!Q&0%^YW_F!@aZS#ucP|YaD#>wd1Fv&Z*SR&mc;asi}1G) z_H>`!akh-Zxq9#io(7%;a$)w+{QH)Y$?UK1Dt^4)up!Szcxnu}kn$0afcfJL#IL+S z5gF_Y30j;{lNrG6m~$Ay?)*V9fZuU@3=kd40=LhazjFrau>(Y>SJNtOz>8x_X-BlA zIpl{i>OarVGj1v(4?^1`R}aQB&WCRQzS~;7R{tDZG=HhgrW@B`W|#cdyj%YBky)P= zpxuOZkW>S6%q7U{VsB#G(^FMsH5QuGXhb(sY+!-R8Bmv6Sx3WzSW<1MPPN1!&PurYky(@`bP9tz z52}LH9Q?+FF5jR6-;|+GVdRA!qtd;}*-h&iIw3Tq3qF9sDIb1FFxGbo&fbG5n8$3F zyY&PWL{ys^dTO}oZ#@sIX^BKW*bon=;te9j5k+T%wJ zNJtoN1~YVj4~YRrlZl)b&kJqp+Z`DqT!la$x&&IxgOQw#yZd-nBP3!7FijBXD|IsU8Zl^ zc6?MKpJQ+7ka|tZQLfchD$PD|;K(9FiLE|eUZX#EZxhG!S-63C$jWX1Yd!6-Yxi-u zjULIr|0-Q%D9jz}IF~S%>0(jOqZ(Ln<$9PxiySr&2Oic7vb<8q=46)Ln%Z|<*z5&> z3f~Zw@m;vR(bESB<=Jqkxn(=#hQw42l(7)h`vMQQTttz9XW6^|^8EK7qhju4r_c*b zJIi`)MB$w@9epwdIfnEBR+?~);yd6C(LeMC& zn&&N*?-g&BBJcV;8&UoZi4Lmxcj16ojlxR~zMrf=O_^i1wGb9X-0@6_rpjPYemIin zmJb+;lHe;Yp=8G)Q(L1bzH*}I>}uAqhj4;g)PlvD9_e_ScR{Ipq|$8NvAvLD8MYr}xl=bU~)f%B3E>r3Bu9_t|ThF3C5~BdOve zEbk^r&r#PT&?^V1cb{72yEWH}TXEE}w>t!cY~rA+hNOTK8FAtIEoszp!qqptS&;r$ zaYV-NX96-h$6aR@1xz6_E0^N49mU)-v#bwtGJm)ibygzJ8!7|WIrcb`$XH~^!a#s& z{Db-0IOTFq#9!^j!n_F}#Z_nX{YzBK8XLPVmc&X`fT7!@$U-@2KM9soGbmOSAmqV z{nr$L^MBo_u^Joyf0E^=eo{Rt0{{e$IFA(#*kP@SQd6lWT2-#>` zP1)7_@IO!9lk>Zt?#CU?cuhiLF&)+XEM9B)cS(gvQT!X3`wL*{fArTS;Ak`J<84du zALKPz4}3nlG8Fo^MH0L|oK2-4xIY!~Oux~1sw!+It)&D3p;+N8AgqKI`ld6v71wy8I!eP0o~=RVcFQR2Gr(eP_JbSytoQ$Yt}l*4r@A8Me94y z8cTDWhqlq^qoAhbOzGBXv^Wa4vUz$(7B!mX`T=x_ueKRRDfg&Uc-e1+z4x$jyW_Pm zp?U;-R#xt^Z8Ev~`m`iL4*c#65Nn)q#=Y0l1AuD&+{|8-Gsij3LUZXpM0Bx0u7WWm zH|%yE@-#XEph2}-$-thl+S;__ciBxSSzHveP%~v}5I%u!z_l_KoW{KRx2=eB33umE zIYFtu^5=wGU`Jab8#}cnYry@9p5UE#U|VVvx_4l49JQ;jQdp(uw=$^A$EA$LM%vmE zvdEOaIcp5qX8wX{mYf0;#51~imYYPn4=k&#DsKTxo{_Mg*;S495?OBY?#gv=edYC* z^O@-sd-qa+U24xvcbL0@C7_6o!$`)sVr-jSJE4XQUQ$?L7}2(}Eixqv;L8AdJAVqc zq}RPgpnDb@E_;?6K58r3h4-!4rT4Ab#rLHLX?eMOfluJk=3i1@Gt1i#iA=O`M0@x! z(HtJP9BMHXEzuD93m|B&woj0g6T?f#^)>J>|I4C5?Gam>n9!8CT%~aT;=oco5d6U8 zMXl(=W;$ND_8+DD*?|5bJ!;8ebESXMUKBAf7YBwNVJibGaJ*(2G`F%wx)grqVPjudiaq^Kl&g$8A2 zWMxMr@_$c}d+;_B`#kUX-t|4VKH&_f^^EP0&=DPLW)H)UzBG%%Tra*5 z%$kyZe3I&S#gfie^z5)!twG={3Cuh)FdeA!Kj<-9** zvT*5%Tb`|QbE!iW-XcOuy39>D3oe6x{>&<#E$o8Ac|j)wq#kQzz|ATd=Z0K!p2$QE zPu?jL8Lb^y3_CQE{*}sTDe!2!dtlFjq&YLY@2#4>XS`}v#PLrpvc4*@q^O{mmnr5D zmyJq~t?8>FWU5vZdE(%4cuZuao0GNjp3~Dt*SLaxI#g_u>hu@k&9Ho*#CZP~lFJHj z(e!SYlLigyc?&5-YxlE{uuk$9b&l6d`uIlpg_z15dPo*iU&|Khx2*A5Fp;8iK_bdP z?T6|^7@lcx2j0T@x>X7|kuuBSB7<^zeY~R~4McconTxA2flHC0_jFxmSTv-~?zVT| zG_|yDqa9lkF*B6_{j=T>=M8r<0s;@z#h)3BQ4NLl@`Xr__o7;~M&dL3J8fP&zLfDfy z);ckcTev{@OUlZ`bCo(-3? z1u1xD`PKgSg?RqeVVsF<1SLF;XYA@Bsa&cY!I48ZJn1V<3d!?s=St?TLo zC0cNr`qD*M#s6f~X>SCNVkva^9A2ZP>CoJ9bvgXe_c}WdX-)pHM5m7O zrHt#g$F0AO+nGA;7dSJ?)|Mo~cf{z2L)Rz!`fpi73Zv)H=a5K)*$5sf_IZypi($P5 zsPwUc4~P-J1@^3C6-r9{V-u0Z&Sl7vNfmuMY4yy*cL>_)BmQF!8Om9Dej%cHxbIzA zhtV0d{=%cr?;bpBPjt@4w=#<>k5ee=TiWAXM2~tUGfm z$s&!Dm0R^V$}fOR*B^kGaipi~rx~A2cS0;t&khV1a4u38*XRUP~f za!rZMtay8bsLt6yFYl@>-y^31(*P!L^^s@mslZy(SMsv9bVoX`O#yBgEcjCmGpyc* zeH$Dw6vB5P*;jor+JOX@;6K#+xc)Z9B8M=x2a@Wx-{snPGpRmOC$zpsqW*JCh@M2Y z#K+M(>=#d^>Of9C`))h<=Bsy)6zaMJ&x-t%&+UcpLjV`jo4R2025 zXaG8EA!0lQa)|dx-@{O)qP6`$rhCkoQqZ`^SW8g-kOwrwsK8 z3ms*AIcyj}-1x&A&vSq{r=QMyp3CHdWH35!sad#!Sm>^|-|afB+Q;|Iq@LFgqIp#Z zD1%H+3I?6RGnk&IFo|u+E0dCxXz4yI^1i!QTu7uvIEH>i3rR{srcST`LIRwdV1P;W z+%AN1NIf@xxvVLiSX`8ILA8MzNqE&7>%jMzGt9wm78bo9<;h*W84i29^w!>V>{N+S zd`5Zmz^G;f=icvoOZfK5#1ctx*~UwD=ab4DGQXehQ!XYnak*dee%YN$_ZPL%KZuz$ zD;$PpT;HM^$KwtQm@7uvT`i6>Hae1CoRVM2)NL<2-k2PiX=eAx+-6j#JI?M}(tuBW zkF%jjLR)O`gI2fcPBxF^HeI|DWwQWHVR!;;{BXXHskxh8F@BMDn`oEi-NHt;CLymW z=KSv5)3dyzec0T5B*`g-MQ<;gz=nIWKUi9ko<|4I(-E0k$QncH>E4l z**1w&#={&zv4Tvhgz#c29`m|;lU-jmaXFMC11 z*dlXDMEOG>VoLMc>!rApwOu2prKSi*!w%`yzGmS+k(zm*CsLK*wv{S_0WX^8A-rKy zbk^Gf_92^7iB_uUF)EE+ET4d|X|>d&mdN?x@vxKAQk`O+r4Qdu>XGy(a(19g;=jU} zFX{O*_NG>!$@jh!U369Lnc+D~qch3uT+_Amyi}*k#LAAwh}k8IPK5a-WZ81ufD>l> z$4cF}GSz>ce`3FAic}6W4Z7m9KGO?(eWqi@L|5Hq0@L|&2flN1PVl}XgQ2q*_n2s3 zt5KtowNkTYB5b;SVuoXA@i5irXO)A&%7?V`1@HGCB&)Wgk+l|^XXChq;u(nyPB}b3 zY>m5jkxpZgi)zfbgv&ec4Zqdvm+D<?Im*mXweS9H+V>)zF#Zp3)bhl$PbISY{5=_z!8&*Jv~NYtI-g!>fDs zmvL5O^U%!^VaKA9gvKw|5?-jk>~%CVGvctKmP$kpnpfN{D8@X*Aazi$txfa%vd-|E z>kYmV66W!lNekJPom29LdZ%(I+ZLZYTXzTg*to~m?7vp%{V<~>H+2}PQ?PPAq`36R z<%wR8v6UkS>Wt#hzGk#44W<%9S=nBfB);6clKwnxY}T*w21Qc3_?IJ@4gYzC7s;WP zVQNI(M=S=JT#xsZy7G`cR(BP9*je0bfeN8JN5~zY(DDs0t{LpHOIbN);?T-69Pf3R zSNe*&p2%AwXHL>__g+xd4Hlc_vu<25H?(`nafS%)3UPP7_4;gk-9ckt8SJRTv5v0M z_Hww`qPudL?ajIR&X*;$y-`<)6dxx1U~5eGS13CB!lX;3w7n&lDDiArbAhSycd}+b zya_3p@A`$kQy;|NJZ~s44Hqo7Hwt}X86NK=(ey>lgWTtGL6k@Gy;PbO!M%1~Wcn2k zUFP|*5d>t-X*RU8g%>|(wwj*~#l4z^Aatf^DWd1Wj#Q*AY0D^V@sC`M zjJc6qXu0I7Y*2;;gGu!plAFzG=J;1%eIOdn zQA>J&e05UN*7I5@yRhK|lbBSfJ+5Uq;!&HV@xfPZrgD}kE*1DSq^=%{o%|LChhl#0 zlMb<^a6ixzpd{kNZr|3jTGeEzuo}-eLT-)Q$#b{!vKx8Tg}swCni>{#%vDY$Ww$84 zew3c9BBovqb}_&BRo#^!G(1Eg((BScRZ}C)Oz?y`T5wOrv);)b^4XR8 zhJo7+<^7)qB>I;46!GySzdneZ>n_E1oWZY;kf94#)s)kWjuJN1c+wbVoNQcmnv}{> zN0pF+Sl3E}UQ$}slSZeLJrwT>Sr}#V(dVaezCQl2|4LN`7L7v&siYR|r7M(*JYfR$ zst3=YaDw$FSc{g}KHO&QiKxuhEzF{f%RJLKe3p*7=oo`WNP)M(9X1zIQPP0XHhY3c znrP{$4#Ol$A0s|4S7Gx2L23dv*Gv2o;h((XVn+9+$qvm}s%zi6nI-_s6?mG! zj{DV;qesJb&owKeEK?=J>UcAlYckA7Sl+I&IN=yasrZOkejir*kE@SN`fk<8Fgx*$ zy&fE6?}G)d_N`){P~U@1jRVA|2*69)KSe_}!~?+`Yb{Y=O~_+@!j<&oVQQMnhoIRU zA0CyF1OFfkK44n*JD~!2!SCPM;PRSk%1XL=0&rz00wxPs&-_eapJy#$h!eqY%nS0{ z!aGg58JIJPF3_ci%n)QSVpa2H`vIe$RD43;#IRfDV&Ibit z+?>HW4{2wOfC6Fw)}4x}i1maDxcE1qi@BS*qcxD2gE@h3#4cgU*D-&3z7D|tVZWt= z-Cy2+*Cm@P4GN_TPUtaVyVesbVDazF@)j8VJ4>XZv!f%}&eO1SvIgr}4`A*3#vat< z_MoByL(qW6L7SFZ#|Gc1fFN)L2PxY+{B8tJp+pxRyz*87)vXR}*=&ahXjBlQKguuf zX6x<<6fQulE^C*KH8~W%ptpaC0l?b=_{~*U4?5Vt;dgM4t_{&UZ1C2j?b>b+5}{IF_CUyvz-@QZPMlJ)r_tS$9kH%RPv#2_nMb zRLj5;chJ72*U`Z@Dqt4$@_+k$%|8m(HqLG!qT4P^DdfvGf&){gKnGCX#H0!;W=AGP zbA&Z`-__a)VTS}kKFjWGk z%|>yE?t*EJ!qeQ%dPk$;xIQ+P0;()PCBDgjJm6Buj{f^awNoVx+9<|lg3%-$G(*f) zll6oOkN|yamn1uyl2*N-lnqRI1cvs_JxLTeahEK=THV$Sz*gQhKNb*p0fNoda#-&F zB-qJgW^g}!TtM|0bS2QZekW7_tKu%GcJ!4?lObt0z_$mZ4rbQ0o=^curCs3bJK6sq z9fu-aW-l#>z~ca(B;4yv;2RZ?tGYAU)^)Kz{L|4oPj zdOf_?de|#yS)p2v8-N||+XL=O*%3+y)oI(HbM)Ds?q8~HPzIP(vs*G`iddbWq}! z(2!VjP&{Z1w+%eUq^ '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" -APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ - "$@" - -# Stop when "xargs" is not available. -if ! command -v xargs >/dev/null 2>&1 -then - die "xargs is not available" -fi - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/java/ppc-pir-services/gradlew.bat b/java/ppc-pir-services/gradlew.bat deleted file mode 100644 index f127cfd4..00000000 --- a/java/ppc-pir-services/gradlew.bat +++ /dev/null @@ -1,91 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%"=="" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%"=="" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if %ERRORLEVEL% equ 0 goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if %ERRORLEVEL% equ 0 goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -set EXIT_CODE=%ERRORLEVEL% -if %EXIT_CODE% equ 0 set EXIT_CODE=1 -if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% -exit /b %EXIT_CODE% - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/java/ppc-pir-services/script/start.sh b/java/ppc-pir-services/script/start.sh deleted file mode 100644 index fd8114ca..00000000 --- a/java/ppc-pir-services/script/start.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd $dirpath - -JAVA_HOME=$JAVA_HOME -APP_MAIN=cn.webank.wedpr.http.PpcsPirApplication -CLASSPATH='conf/:app/*:libs/*' -CURRENT_DIR=`pwd` -LOG_DIR=${CURRENT_DIR}/logs -CONF_DIR=${CURRENT_DIR}/conf -os_platform=`uname -s` - - -if [ "${JAVA_HOME}" = "" ];then - echo "JAVA_HOME has not been configured" - exit -1 -fi - -log_path="/data/app/ppc/wedpr-pir-node/logs/wedpr-pir.log" -# 检查文件是否存在 -if [ ! -f "$log_path" ]; then - # 文件不存在，创建文件 - touch "$log_path" -fi - -# 初始化全局变量，用于标识系统的PID（0表示未启动） -tradePortalPID=0 -start_timestamp=0 -APP_NAME="wedpr-pir" - -getTradeProtalPID(){ - javaps=`$JAVA_HOME/bin/jps -l | grep $APP_MAIN` - if [ -n "$javaps" ]; then - tradePortalPID=`echo $javaps | awk '{print $1}'` - else - tradePortalPID=0 - fi -} - -#JAVA_OPTS=" -Dfile.encoding=UTF-8 -Dcom.sun.management.jmxremote.port=${JMXREMOTE_PORT} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -JAVA_OPTS=" -Dlog4j.configurationfile=${CONF_DIR}/log4j2.xml -Dindex.log.home=${LOG_DIR} -Dconfig=${CONF_DIR}/" -JAVA_OPTS+=" -Xmx3072m -Xms3072m -XX:NewSize=1024m -XX:MaxNewSize=1024m -XX:PermSize=512m -XX:MaxPermSize=512m" -JAVA_OPTS+=" -XX:CompileThreshold=20000 -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70" -JAVA_OPTS+=" -XX:+DisableExplicitGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${LOG_DIR}/jvm.log" -JAVA_OPTS+=" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOG_DIR}/ -XX:ErrorFile=${LOG_DIR}/heap_error.log" - -function get_start_time() { - start_time=$(date "+%Y-%m-%d %H:%M:%S") - if [[ "${os_platform}" = "Darwin" ]];then - start_timestamp=$(date -j -f "%Y-%m-%d %H:%M:%S" "${start_time}" +%s) - elif [[ "${os_platform}" = "Linux" ]];then - start_timestamp=$(date -d "${start_time}" +%s) - else - echo -e "\033[31m[ERROR] Server start fail!\033[0m" - echo "check platform...[failed]" - echo "===============================================================================================" - kill $tradePortalPID - exit 1 - fi -} - -function waiting_for_start() { - echo "" - i=0 - while [ $i -le 30 ] - do - for j in '\\' '|' '/' '-' - do - printf "%c%c%c%c%c Waiting for server started %c%c%c%c%c\r" \ - "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" "$j" - check_time=$(cat ${dirpath}/logs/${APP_NAME}.log | grep -a "JVM running for" | tail -n1 | awk -F "]" '{print $3}' | awk -F "[" '{print $2}' | awk -F " " '{print $1, $2}') - if [ -n "$check_time" ]; then - if [[ "${os_platform}" = "Darwin" ]];then - timestamp=$(date -j -f "%Y-%m-%d %H:%M:%S" "${check_time}" +%s) - elif [[ "${os_platform}" = "Linux" ]];then - timestamp=$(date -d "${check_time}" +%s) - else - echo -e "\033[31m[ERROR] Server start fail!\033[0m" - echo "check platform...[failed]" - echo "===============================================================================================" - kill $tradePortalPID - exit 1 - fi - if [[ ${timestamp} -gt ${start_timestamp} ]]; then - echo "" - echo -e "\033[32m[INFO] Server start Successful!\033[0m" - echo "(PID=$tradePortalPID)...[Success]" - echo "===============================================================================================" - exit 0 - fi - fi - sleep 1 - done - let i=i+5 - done - echo "" - echo -e "\033[31m[ERROR] Server start fail!\033[0m" - echo "timeout...[failed]" - echo "===============================================================================================" - kill $tradePortalPID - exit 1 -} - -start(){ - getTradeProtalPID - echo "===============================================================================================" - if [ $tradePortalPID -ne 0 ]; then - echo "$APP_MAIN already started(PID=$tradePortalPID)" - echo "===============================================================================================" - else - get_start_time - nohup $JAVA_HOME/bin/java $JAVA_OPTS -cp $CLASSPATH $APP_MAIN >start.out 2>&1 & - sleep 1 - getTradeProtalPID - if [ $tradePortalPID -ne 0 ]; then - waiting_for_start - else - echo -e "\033[31m[ERROR] Server start fail!\033[0m" - echo "[Failed]" - echo "===============================================================================================" - fi - fi -} - -start diff --git a/java/ppc-pir-services/script/stop.sh b/java/ppc-pir-services/script/stop.sh deleted file mode 100644 index ac04bbde..00000000 --- a/java/ppc-pir-services/script/stop.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -rm -rf start.out - -dirpath="$(cd "$(dirname "$0")" && pwd)" -cd $dirpath - -JAVA_HOME=$JAVA_HOME -APP_MAIN=cn.webank.wedpr.http.PpcsPirApplication -CURRENT_DIR=`pwd` -CONF_DIR=${CURRENT_DIR}/conf - -# 初始化全局变量，用于标识系统的PID（0表示未启动） -tradePortalPID=0 - -getTradeProtalPID(){ - javaps=`$JAVA_HOME/bin/jps -l | grep $APP_MAIN` - if [ -n "$javaps" ]; then - tradePortalPID=`echo $javaps | awk '{print $1}'` - else - tradePortalPID=0 - fi -} - -stop(){ - getTradeProtalPID - echo "===============================================================================================" - if [ $tradePortalPID -ne 0 ]; then - echo -n "Stopping $APP_MAIN(PID=$tradePortalPID)..." - kill $tradePortalPID > /dev/null - if [ $? -eq 0 ]; then - echo "[Success]" - echo "===============================================================================================" - else - echo "[Failed]" - echo "===============================================================================================" - fi - getTradeProtalPID - else - echo "$APP_MAIN is not running" - echo "===============================================================================================" - fi -} - -stop diff --git a/java/ppc-pir-services/sdk/build.gradle b/java/ppc-pir-services/sdk/build.gradle deleted file mode 100644 index 355c101d..00000000 --- a/java/ppc-pir-services/sdk/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -plugins { - id 'java' - id 'com.github.sherter.google-java-format' -} - -repositories { - maven { - url = "https://mirrors.huaweicloud.com/repository/maven/" - } -} - -project.version="1.0.0" - -dependencies { - implementation 'com.squareup.okhttp3:okhttp:4.9.1' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.0' - - compileOnly 'org.projectlombok:lombok:1.18.24' - annotationProcessor 'org.projectlombok:lombok:1.18.24' -} - -googleJavaFormat { - options style: 'AOSP' - source = sourceSets*.allJava - include '**/*.java' -} \ No newline at end of file diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java deleted file mode 100644 index fd1ad363..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/demo/Demo.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.wedpr.pir.demo; - -import com.wedpr.pir.sdk.PirClient; -import com.wedpr.pir.sdk.entity.param.PirJobParam; -import com.wedpr.pir.sdk.entity.response.PirResultResponse; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import java.util.Arrays; - -/** - * @author zachma - * @date 2024/8/22 - */ -public class Demo { - public static void main(String[] args) throws WedprException { - testIdObfuscation(); - testIdFilter(); - } - - private static void testIdFilter() throws WedprException { - PirJobParam pirJobParam = new PirJobParam(); - pirJobParam.setServiceId("s-123456789"); - pirJobParam.setAccessKeyId("a-123456789"); - pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); - pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idFilter.getValue()); - pirJobParam.setGatewayUrl("localhost:5831"); - pirJobParam.setSearchIdList(Arrays.asList("1", "2", "3")); - - PirClient pirClient = new PirClient(pirJobParam); - PirResultResponse pirResultResponse = pirClient.executePirJob(); - System.out.println(pirResultResponse); - } - - private static void testIdObfuscation() throws WedprException { - PirJobParam pirJobParam = new PirJobParam(); - pirJobParam.setServiceId("s-123456789"); - pirJobParam.setAccessKeyId("a-123456789"); - pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); - pirJobParam.setJobAlgorithmType(ParamEnum.AlgorithmType.idObfuscation.getValue()); - pirJobParam.setGatewayUrl("localhost:5831"); - pirJobParam.setSearchIdList(Arrays.asList("3", "2", "1")); - - PirClient pirClient = new PirClient(pirJobParam); - PirResultResponse pirResultResponse = pirClient.executePirJob(); - System.out.println(pirResultResponse); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java deleted file mode 100644 index 41da2bd7..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/PirClient.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.wedpr.pir.sdk; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import com.wedpr.pir.sdk.entity.body.SimpleEntity; -import com.wedpr.pir.sdk.entity.param.PirJobParam; -import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; -import com.wedpr.pir.sdk.entity.request.ServerJobRequest; -import com.wedpr.pir.sdk.entity.response.ClientAuthResponse; -import com.wedpr.pir.sdk.entity.response.ClientOTResponse; -import com.wedpr.pir.sdk.entity.response.PirResultResponse; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import com.wedpr.pir.sdk.gateway.GatewayClient; -import com.wedpr.pir.sdk.helper.Constant; -import com.wedpr.pir.sdk.service.PirJobService; -import java.util.Objects; - -/** - * @author zachma - * @date 2024/8/19 - */ -public class PirClient { - private final PirJobService pirJobService; - - public PirClient(PirJobParam pirJobParam) throws WedprException { - pirJobParam.check(); - this.pirJobService = new PirJobService(pirJobParam); - } - - public PirResultResponse executePirJob() throws WedprException { - try { - ObjectMapper objectMapper = new ObjectMapper(); - // 认证 - PirJobParam pirJobParam = pirJobService.getPirJobParam(); - ServiceConfigBody serviceConfigBody = pirJobParam.getServiceConfigBody(); - Integer invokeType = pirJobParam.getPirInvokeType(); - if (invokeType.equals(ParamEnum.JobMode.SDKMode.getValue())) { - ClientAuthRequest clientAuthRequest = new ClientAuthRequest(); - clientAuthRequest.setAccessKeyId(pirJobParam.getAccessKeyId()); - clientAuthRequest.setServiceId(pirJobParam.getServiceId()); - String authUrl = pirJobParam.getGatewayUrl() + Constant.PERMISSION; - String body = objectMapper.writeValueAsString(clientAuthRequest); - ClientAuthResponse authResult = - GatewayClient.sendPostRequestWithRetry( - authUrl, body, ClientAuthResponse.class); - if (!authResult.getCode().equals(WedprStatusEnum.SUCCESS.getCode())) { - throw new WedprException( - WedprStatusEnum.CALL_PSI_ERROR, "认证失败：" + authResult.getMessage()); - } - pirJobParam.setServiceConfigBody(authResult.getData()); - } - - // 执行任务 - ClientOTResponse otParamResponse = pirJobService.requesterOtCipher(); - ServerJobRequest serverJobRequest = - generatePirJobRequestFromJobAndOTParam(pirJobParam, otParamResponse); - - String pirUrl = pirJobParam.getGatewayUrl() + Constant.SERVER; - String body = objectMapper.writeValueAsString(serverJobRequest); - SimpleEntity otResult = - GatewayClient.sendPostRequestWithRetry(pirUrl, body, SimpleEntity.class); - if (Objects.isNull(otResult) - || !otResult.getCode().equals(WedprStatusEnum.SUCCESS.getCode())) { - throw new WedprException(WedprStatusEnum.TASK_EXECUTE_ERROR); - } - return pirJobService.requesterOtRecover(otParamResponse, pirJobParam, otResult); - } catch (Exception e) { - throw new WedprException(WedprStatusEnum.CALL_PSI_ERROR, e.getMessage()); - } - } - - /** 根据 ClientOTResponse 和 PirJobParam 构建PirJobRequest */ - private ServerJobRequest generatePirJobRequestFromJobAndOTParam( - PirJobParam pirJobParam, ClientOTResponse otParamResponse) { - ServerJobRequest serverJobRequest = new ServerJobRequest(); - serverJobRequest.setServiceId(pirJobParam.getServiceId()); - serverJobRequest.setServiceConfigBody(pirJobParam.getServiceConfigBody()); - serverJobRequest.setAccessKeyId(pirJobParam.getAccessKeyId()); - serverJobRequest.setJobAlgorithmType(pirJobParam.getJobAlgorithmType()); - serverJobRequest.setX(otParamResponse.getX()); - serverJobRequest.setY(otParamResponse.getY()); - serverJobRequest.setDataBodyList(otParamResponse.getDataBodyList()); - for (int i = 0; i < serverJobRequest.getDataBodyList().size(); i++) { - serverJobRequest.getDataBodyList().get(i).setIdIndex(0); - } - return serverJobRequest; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java deleted file mode 100644 index 0bf38256..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirDataBody.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import java.math.BigInteger; -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -public class PirDataBody { - BigInteger z0; - String filter; - int idIndex; - List idHashList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java deleted file mode 100644 index c8bf5524..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/PirResultBody.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/21 - */ -@Data -public class PirResultBody { - String searchId; - Boolean searchExist; - String searchValue; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java deleted file mode 100644 index f71b9896..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultBody.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import java.math.BigInteger; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ServerResultBody { - BigInteger e; - BigInteger w; - String c; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java deleted file mode 100644 index 3f448180..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServerResultList.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import java.util.List; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ServerResultList { - List resultBodyList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java deleted file mode 100644 index 5ffbd332..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/ServiceConfigBody.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Data -public class ServiceConfigBody { - private String datasetId; - private String[] exists; - private String[] values; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java deleted file mode 100644 index 7726c772..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/body/SimpleEntity.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.wedpr.pir.sdk.entity.body; - -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/21 - */ -@Data -public class SimpleEntity { - String code; - String message; - ServerOTResponse data; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java deleted file mode 100644 index 4095dafc..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/param/PirJobParam.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.wedpr.pir.sdk.entity.param; - -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import com.wedpr.pir.sdk.entity.request.PirBaseRequest; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import com.wedpr.pir.sdk.helper.BasicTypeHelper; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * @author zachma - * @date 2024/8/19 - */ -@Data -@EqualsAndHashCode(callSuper = true) -public class PirJobParam extends PirBaseRequest { - - private Integer filterLength = 4; - - private String gatewayUrl; - - private List searchIdList; - - private ServiceConfigBody serviceConfigBody; - - public void check() throws WedprException { - if (BasicTypeHelper.isStringEmpty(gatewayUrl)) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "访问gatewayUrl不能为空"); - } - - if (getPirInvokeType().equals(ParamEnum.JobMode.SDKMode.getValue()) - && BasicTypeHelper.isStringEmpty(getServiceId())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "serviceId 不能为空"); - } - - if (getPirInvokeType().equals(ParamEnum.JobMode.SDKMode.getValue()) - && BasicTypeHelper.isStringEmpty(getAccessKeyId())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "accessKeyId 不能为空"); - } - - if (Arrays.stream(ParamEnum.JobType.values()) - .noneMatch(enumValue -> enumValue.getValue().equals(getJobType()))) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobType输入错误"); - } - - if (Arrays.stream(ParamEnum.AlgorithmType.values()) - .noneMatch(enumValue -> enumValue.getValue().equals(getJobAlgorithmType()))) { - throw new WedprException( - WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "jobAlgorithmType输入错误"); - } - - if (Objects.isNull(searchIdList) || searchIdList.size() == 0) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "searchId列表不能为空"); - } - - if (getPirInvokeType().equals(ParamEnum.JobMode.DirectorMode.getValue()) - && Objects.isNull(serviceConfigBody)) { - throw new WedprException( - WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "向导模式下serviceConfigBody不能为空"); - } - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java deleted file mode 100644 index 0841ba6c..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientAuthRequest.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Data -public class ClientAuthRequest { - String serviceId; - String accessKeyId; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java deleted file mode 100644 index 7a3ecfb3..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDecryptRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import java.math.BigInteger; -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/21 - */ -@Data -public class ClientDecryptRequest { - BigInteger b; - List dataBodyList; - ServerOTResponse serverResult; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java deleted file mode 100644 index 9079ceb5..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientDirectRequest.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/5 - */ -@Data -public class ClientDirectRequest { - private String algorithmType; - private List searchIds; - private ServiceConfigBody serviceConfigBody; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java deleted file mode 100644 index b4d54dcb..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ClientOTRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/20 - */ -@Data -public class ClientOTRequest { - int filterLength = 4; - - int obfuscationOrder; - - List dataBodyList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java deleted file mode 100644 index f38200b0..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/PirBaseRequest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import com.wedpr.pir.sdk.enums.ParamEnum; -import lombok.Data; - -@Data -public class PirBaseRequest { - // 任务ID - // 任务类型(0: 查询存在性, 1: 查询对应值) - String serviceId; - - String accessKeyId; - - String jobType; - // 数据方机构id - // 匿踪算法类型(0: hash披露算法, 1: hash混淆算法) - String jobAlgorithmType; - // 查询范围 - Integer obfuscationOrder = 9; - - Integer pirInvokeType = ParamEnum.JobMode.SDKMode.getValue(); -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java deleted file mode 100644 index 296b00c1..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerJobRequest.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import com.wedpr.pir.sdk.entity.body.PirDataBody; -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import java.math.BigInteger; -import java.util.List; -import lombok.Data; -import lombok.EqualsAndHashCode; - -/** - * @author zachma - * @date 2024/8/18 - */ -@EqualsAndHashCode(callSuper = true) -@Data -public class ServerJobRequest extends PirBaseRequest { - BigInteger x; - BigInteger y; - List dataBodyList; - ServiceConfigBody serviceConfigBody; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java deleted file mode 100644 index 99ebce12..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/request/ServerOTRequest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.wedpr.pir.sdk.entity.request; - -import com.wedpr.pir.sdk.entity.body.PirDataBody; -import java.math.BigInteger; -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -public class ServerOTRequest { - String jobType; - String datasetId; - String[] params; - BigInteger x; - BigInteger y; - List dataBodyList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java deleted file mode 100644 index b2743f9a..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientAuthResponse.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Data -public class ClientAuthResponse { - String code; - String message; - ServiceConfigBody data; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java deleted file mode 100644 index cf720cb8..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientDecryptResponse.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import com.wedpr.pir.sdk.entity.body.PirResultBody; -import java.util.List; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/21 - */ -@Data -public class ClientDecryptResponse { - List detail; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java deleted file mode 100644 index d367cf54..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientJobResponse.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -public class ClientJobResponse { - private String code; - private String message; - private Object data; - - public ClientJobResponse(WedprStatusEnum wedprStatusEnum) { - this.code = wedprStatusEnum.getCode(); - this.message = wedprStatusEnum.getMessage(); - } - - public ClientJobResponse(WedprStatusEnum wedprStatusEnum, Object data) { - this.code = wedprStatusEnum.getCode(); - this.message = wedprStatusEnum.getMessage(); - this.data = data; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java deleted file mode 100644 index a26ce00b..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ClientOTResponse.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import com.wedpr.pir.sdk.entity.body.PirDataBody; -import java.math.BigInteger; -import java.util.List; -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/20 - */ -@Data -@AllArgsConstructor -public class ClientOTResponse { - BigInteger b; - BigInteger x; - BigInteger y; - List dataBodyList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java deleted file mode 100644 index 1b250263..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/PirResultResponse.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/21 - */ -@Data -public class PirResultResponse { - String jobType; - // List detail; - ClientDecryptResponse detail; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java deleted file mode 100644 index 45d0a312..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/entity/response/ServerOTResponse.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.wedpr.pir.sdk.entity.response; - -import com.wedpr.pir.sdk.entity.body.ServerResultList; -import java.util.List; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class ServerOTResponse { - List resultBodyList; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java deleted file mode 100644 index 27cdfa5f..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/enums/ParamEnum.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.wedpr.pir.sdk.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author zachma - * @date 2024/8/18 - */ -public class ParamEnum { - @Getter - @AllArgsConstructor - public enum JobType { - SearchValue("0"), - SearchExist("1"); - private String value; - } - - @Getter - @AllArgsConstructor - public enum AlgorithmType { - idFilter("0"), - idObfuscation("1"); - - private String value; - } - - @Getter - @AllArgsConstructor - public enum HttpType { - HttpTimeout(180), - HttpMaxRetries(1); - private Integer value; - } - - @Getter - @AllArgsConstructor - public enum JobMode { - SDKMode(0), - DirectorMode(1); - private Integer value; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java deleted file mode 100644 index 3565318d..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprException.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.wedpr.pir.sdk.exception; - -/** - * @author zachma - * @date 2024/8/18 - */ -public class WedprException extends Exception { - private final WedprStatusEnum status; - - public WedprException(WedprStatusEnum status) { - super(status.getMessage()); - this.status = status; - } - - public WedprException(WedprStatusEnum status, String message) { - super(message); - this.status = status; - } - - public WedprStatusEnum getStatus() { - return status; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java deleted file mode 100644 index 2313402d..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/exception/WedprStatusEnum.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.wedpr.pir.sdk.exception; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @author zachma - * @date 2024/8/18 - */ -@AllArgsConstructor -@Getter -public enum WedprStatusEnum { - SUCCESS("0", "成功"), - FAILURE("1", "系统内部错误"), - INTERNAL_SERVER_ERROR("3", "系统内部错误"), - SYSTEM_CONFIG_ERROR("4", "系统配置参数错误"), - HANDLE_FILE_ERROR("5", "处理文件错误"), - TASK_EXECUTE_ERROR("6", "任务执行失败"), - - // 1000 未定义错误 - SYSTEM_EXCEPTION("1000", "系统抛出异常，请检查日志记录"), - CLIENT_PARAM_EXCEPTION("1001", "发起方参数错误"), - - // 2000 参数解析错误 - INVALID_INPUT_VALUE("2000", "输入参数异常"), - JSON_ERROR("2001", "输入参数JSON解析失败"), - INVALID_SYSTEM_KEY("2002", "系统密钥解析失败"), - INVALID_CUSTOMER_TYPE("2003", "未定义的客户类型"), - INVALID_HASH_VALUE("2004", "输入的哈希值不合法"), - INVALID_INPUT_LENGTH("2005", "输入参数列表长度不合法"), - INVALID_INPUT_PREFIX("2006", "输入参数前缀不匹配"), - INVALID_TIMESTAMP_VALUE("2007", "输入时间戳不合法"), - JSON_GET_ERROR("2008", "JSON解析返回值失败"), - INVALID_TRANS_TYPE("2009", "不合法的交易类型"), - INVALID_FILE_PATH("2010", "不合法交集文件路径"), - INVALID_ROLE_VALUE("2011", "pir.Role设置错误"), - CSV_ERROR("2012", "csv解析是错误"), - - // 3000 db错误 - DB_ERROR("3000", "DB查询失败"), - // DB_TIMEOUT("3000", "DB访问超时"), - DB_INSERT_FAILURE("3001", "DB插入失败"), - DB_DELETE_FAILURE("3002", "DB删除失败"), - UNENCRYPTED_DATA("3003", "查询的数据未被加密"), - DB_MISSED_USER("3004", "无法写入未命中用户"), - - // 4000 被请求服务端异常 - SERVICE_UNAVAILABLE("4000", "对方PIR服务访问失败"), - INVALID_ENCRYPTION_KEY("4001", "服务端加密密钥异常"), - SEARCHED_DB_UNAVAILABLE("4002", "服务端db无法访问"), - HTTP_CALL_ERROR("4003", "HTTP调用异常"), - CALL_PSI_ERROR("4004", "调用PIR服务异常"), - UPLOAD_FPS_ERROR("4005", "上传文件至Fps失败"), - DOWNLOAD_FPS_ERROR("4006", "下载Fps文件失败"), - CLIENT_LOCAL_FILE("4007", "本地初始化./client/local.txt没有设置"), - SERVER_LOCAL_FILE("4008", "本地初始化./server/local.txt没有设置"), - - // 5000 密码学运算错误 - AES_ENCRYPT_ERROR("5000", "AES加密明文信息错误"), - AES_DECRYPT_ERROR("5001", "AES解密密文信息错误"), - MATCH_RESULT_ERROR("5002", "找不到匹配结果查询值"), - ENCODE_BIG_NUMBER_ERROR("5003", "转换输入哈希到BIG NUMBER类型失败"), - DECODE_ECC_POINT_ERROR("5004", "解码密文到椭圆曲线上点失败"), - MATCH_CIPHER_ERROR("5005", "计算匹配结果失败"), - CLIENT_CIPHER_ERROR("5006", "计算客户端密文失败"), - CIPHER_LENGTH_ERROR("5007", "根据前缀索引到的密文太多"), - WRONG_HASH_ERROR("5008", "输入哈希算法错误"), - - CRYPTO_SPE_ERROR("5099", "密码运算通用错误"); - - private String code; - private String message; -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java deleted file mode 100644 index c9cfaceb..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/gateway/GatewayClient.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.wedpr.pir.sdk.gateway; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.io.IOException; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import okhttp3.Call; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; - -/** - * @author zachma - * @date 2024/8/20 - */ -public class GatewayClient { - - private static final MediaType JSON_MEDIA_TYPE = - MediaType.get("application/json; charset=utf-8"); - - private static String sendGetRequest(String url) throws IOException, WedprException { - int timeout = ParamEnum.HttpType.HttpTimeout.getValue(); - OkHttpClient httpClient = - new OkHttpClient() - .newBuilder() - .connectTimeout(timeout, TimeUnit.SECONDS) - .readTimeout(timeout, TimeUnit.SECONDS) - .writeTimeout(timeout, TimeUnit.SECONDS) - .build(); - - Request getRequest = new Request.Builder().url(url).get().build(); - - Call call = httpClient.newCall(getRequest); - Response response = call.execute(); - String responseBody = ""; - if (response.isSuccessful() && Objects.nonNull(response.body())) { - responseBody = response.body().string(); - } else { - throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); - } - return responseBody; - } - - private static Response sendPostRequest(String url, String jsonBody) throws IOException { - if (!url.startsWith("http")) { - url = "http://" + url; - } - OkHttpClient httpClient = new OkHttpClient().newBuilder().build(); - RequestBody body = RequestBody.create(jsonBody, JSON_MEDIA_TYPE); - Request request = new Request.Builder().post(body).url(url).build(); - Call call = httpClient.newCall(request); - return call.execute(); - } - - public static T sendPostRequestWithRetry(String url, String body, Class responseType) - throws WedprException { - int maxRetries = ParamEnum.HttpType.HttpMaxRetries.getValue(); - int retries = 1; - while (retries <= maxRetries) { - try { - Response response = sendPostRequest(url, body); - if (response.isSuccessful() && Objects.nonNull(response.body())) { - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.readValue(response.body().string(), responseType); - } else { - throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); - } - - } catch (Exception e) { - e.printStackTrace(); - retries++; - if (retries > maxRetries) { - throw new WedprException(WedprStatusEnum.HTTP_CALL_ERROR); - } - } - } - return null; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java deleted file mode 100644 index f3260060..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/AESHelper.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.wedpr.pir.sdk.helper; - -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.util.Base64; -import javax.crypto.Cipher; -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.GCMParameterSpec; -import javax.crypto.spec.SecretKeySpec; - -/** - * @author zachma - * @date 2024/8/21 - */ -public class AESHelper { - private static final SecureRandom SECURE_RANDOM = new SecureRandom(); - - private static final String AES_ALGORITHM_TRANSFORMATION = "AES/GCM/NoPadding"; - - public static String generateRandomKey() { - // 随机生成 16 位字符串格式的密钥 - byte[] keyBytes = new byte[16]; - SECURE_RANDOM.nextBytes(keyBytes); - return Base64.getEncoder().encodeToString(keyBytes); - } - - public static String encrypt(String plainText, String keyString) throws WedprException { - try { - byte[] iv = new byte[12]; - SECURE_RANDOM.nextBytes(iv); - byte[] contentBytes = plainText.getBytes(StandardCharsets.UTF_8); - Cipher cipher = Cipher.getInstance(AES_ALGORITHM_TRANSFORMATION); - GCMParameterSpec params = new GCMParameterSpec(128, iv); - cipher.init(Cipher.ENCRYPT_MODE, getSecretKey(keyString), params); - byte[] encryptData = cipher.doFinal(contentBytes); - assert encryptData.length == contentBytes.length + 16; - byte[] message = new byte[12 + contentBytes.length + 16]; - System.arraycopy(iv, 0, message, 0, 12); - System.arraycopy(encryptData, 0, message, 12, encryptData.length); - return Base64.getEncoder().encodeToString(message); - } catch (Exception e) { - throw new WedprException(WedprStatusEnum.AES_ENCRYPT_ERROR); - } - } - - public static String decrypt(String encryptedText, String keyString) throws WedprException { - try { - byte[] content = Base64.getDecoder().decode(encryptedText); - if (content.length < 12 + 16) { - throw new IllegalArgumentException(); - } - GCMParameterSpec params = new GCMParameterSpec(128, content, 0, 12); - Cipher cipher = Cipher.getInstance(AES_ALGORITHM_TRANSFORMATION); - cipher.init(Cipher.DECRYPT_MODE, getSecretKey(keyString), params); - byte[] decryptData = cipher.doFinal(content, 12, content.length - 12); - return new String(decryptData, StandardCharsets.UTF_8); - } catch (Exception e) { - throw new WedprException(WedprStatusEnum.AES_DECRYPT_ERROR); - } - } - - private static SecretKeySpec getSecretKey(String key) throws NoSuchAlgorithmException { - KeyGenerator kg = KeyGenerator.getInstance("AES"); - // 初始化密钥生成器，AES要求密钥长度为128位、192位、256位 - SecureRandom secureRandom = SecureRandom.getInstance("SHA1PRNG"); - secureRandom.setSeed(key.getBytes(StandardCharsets.UTF_8)); - kg.init(128, secureRandom); - SecretKey secretKey = kg.generateKey(); - return new SecretKeySpec(secretKey.getEncoded(), "AES"); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java deleted file mode 100644 index 3e8aebc0..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/BasicTypeHelper.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.wedpr.pir.sdk.helper; - -/** - * @author zachma - * @date 2024/8/21 - */ -public class BasicTypeHelper { - public static boolean isStringEmpty(String str) { - return str == null || str.length() == 0; - } - - public static boolean isStringNotEmpty(String str) { - return !isStringEmpty(str); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java deleted file mode 100644 index 976252fd..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/Constant.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.wedpr.pir.sdk.helper; -/** - * @author zachma - * @date 2024/9/3 - */ -public class Constant { - - public static final String SERVER = "/api/pir/v3/server"; - - public static final String PERMISSION = "/api/pir/v3/auth"; - - public static String PIR_TEMP_TABLE_PREFIX = "wedpr_pir_"; - - public static String datasetId2tableId(String datasetId) { - return PIR_TEMP_TABLE_PREFIX + datasetId.substring(2); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java deleted file mode 100644 index 6e5728fd..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/ConvertTypeHelper.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.wedpr.pir.sdk.helper; - -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * @author zachma - * @date 2024/8/19 - */ -public class ConvertTypeHelper { - public static int bytesToInt(byte[] bytes, ByteOrder byteOrder) { - ByteBuffer buffer = ByteBuffer.wrap(bytes).order(byteOrder); - return buffer.getInt(); - } - - public static byte[] intToBytes(int number, ByteOrder byteOrder) { - ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder); - buffer.putInt(number); - return buffer.array(); - } - - public static BigInteger bytesToBigInteger(byte[] bytes) { - return new BigInteger(1, bytes); - } - - public static byte[] bigIntegerToBytes(BigInteger number) { - byte[] bytes = number.toByteArray(); - if (bytes[0] == 0) { - byte[] tmpBytes = new byte[bytes.length - 1]; - System.arraycopy(bytes, 1, tmpBytes, 0, tmpBytes.length); - return tmpBytes; - } - return bytes; - } - - public static String bytesToHexString(byte[] bytes) { - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) { - sb.append(String.format("%02X ", b)); - } - return sb.toString().trim(); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java deleted file mode 100644 index 2bec5edf..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/helper/CryptoOperatorHelper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.wedpr.pir.sdk.helper; - -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * @author zachma - * @date 2024/8/19 - */ -public class CryptoOperatorHelper { - private static final BigInteger DEFAULT_G = - new BigInteger( - "9020881489161854992071763483314773468341853433975756385639545080944698236944020124874820917267762049756743282301106459062535797137327360192691469027152272"); - private static final BigInteger DEFAULT_N = - new BigInteger( - "102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885212534669604964926915306738569799518792945024759514373214412797317972739022405456550476153212687312211184540248262330559143446510677062823907392904449451177"); - private static final BigInteger DEFAULT_FI = - new BigInteger( - "102724610959913950919762303151320427896415051258714708724768326174083057407299433043362228762657118029566890747043004760241559786931866234640457856691885192126363163670343672910761259882348623401714459980712242233796355982147797162316532450768783823909695360736554767341443201861573989081253763975895939627220"); - - private static final SecureRandom RANDOM = new SecureRandom(); - - /** 生成随机数 * */ - public static BigInteger getRandomInt() { - BigInteger num = new BigInteger(DEFAULT_N.bitLength(), RANDOM); - while (num.compareTo(DEFAULT_N) >= 0) { - num = new BigInteger(DEFAULT_N.bitLength(), RANDOM); - } - return num; - } - - /** b*G mod N */ - public static BigInteger powMod(BigInteger b) { - return DEFAULT_G.modPow(b, DEFAULT_N); - } - - /** a^b mod N */ - public static BigInteger OTPow(BigInteger a, BigInteger b) { - return a.modPow(b, DEFAULT_N); - } - - /** a*b mod FI */ - public static BigInteger mulMod(BigInteger a, BigInteger b) { - return a.multiply(b).mod(DEFAULT_FI); - } - - /** a*b mod N */ - public static BigInteger OTMul(BigInteger a, BigInteger b) { - return a.multiply(b).mod(DEFAULT_N); - } - - /** 批量获取searchID的Hash值 */ - public static List getIdHashVec(int obfuscationOrder, Integer idIndex, String searchId) - throws WedprException { - - List clientDataBodyList = new ArrayList<>(); - String searchIdTemp; - for (int i = 0; i < obfuscationOrder + 1; i++) { - if (idIndex == i) { - searchIdTemp = searchId; - } else { - String uuid = UUID.randomUUID().toString(); - searchIdTemp = makeHash(uuid.getBytes()); - } - clientDataBodyList.add(searchIdTemp); - } - return clientDataBodyList; - } - - private static String makeHash(byte[] data) throws WedprException { - try { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] hashBytes = digest.digest(data); - - StringBuilder hexString = new StringBuilder(); - for (byte b : hashBytes) { - String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) { - hexString.append('0'); - } - hexString.append(hex); - } - return hexString.toString(); - } catch (NoSuchAlgorithmException e) { - throw new WedprException(WedprStatusEnum.WRONG_HASH_ERROR); - } - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java deleted file mode 100644 index 2321dec4..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientDecryptService.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.wedpr.pir.sdk.service; - -import com.wedpr.pir.sdk.entity.body.PirResultBody; -import com.wedpr.pir.sdk.entity.body.ServerResultBody; -import com.wedpr.pir.sdk.entity.request.ClientDecryptRequest; -import com.wedpr.pir.sdk.entity.response.ClientDecryptResponse; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.helper.AESHelper; -import com.wedpr.pir.sdk.helper.BasicTypeHelper; -import com.wedpr.pir.sdk.helper.ConvertTypeHelper; -import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -/** - * @author zachma - * @date 2024/8/21 - */ -public class ClientDecryptService { - private void decryptServerResultList( - PirResultBody pirResultBody, List serverResultList, BigInteger b) { - for (ServerResultBody body : serverResultList) { - BigInteger e = body.getE(); - BigInteger w = body.getW(); - String cipherStr = body.getC(); - BigInteger w1 = CryptoOperatorHelper.OTPow(w, b); - try { - // 对整数AES密钥OT解密，报错后不处理 - BigInteger messageRecover = w1.xor(e); - byte[] convertedBytes = ConvertTypeHelper.bigIntegerToBytes(messageRecover); - String convertedStr = new String(convertedBytes, StandardCharsets.UTF_8); - String decryptedText = AESHelper.decrypt(cipherStr, convertedStr); - pirResultBody.setSearchValue(decryptedText); - } catch (WedprException ignored) { - - } - } - } - - protected ClientDecryptResponse runDecryptOTparam(ClientDecryptRequest clientDecryptRequest) { - List pirResultBodyArrayList = new ArrayList<>(); - for (int i = 0; - i < clientDecryptRequest.getServerResult().getResultBodyList().size(); - i++) { - PirResultBody pirResultBody = new PirResultBody(); - pirResultBody.setSearchId(clientDecryptRequest.getDataBodyList().get(i)); - List serverResultlist = - clientDecryptRequest - .getServerResult() - .getResultBodyList() - .get(i) - .getResultBodyList(); - decryptServerResultList(pirResultBody, serverResultlist, clientDecryptRequest.getB()); - pirResultBody.setSearchExist( - BasicTypeHelper.isStringNotEmpty(pirResultBody.getSearchValue())); - pirResultBodyArrayList.add(pirResultBody); - } - ClientDecryptResponse clientDecryptResponse = new ClientDecryptResponse(); - clientDecryptResponse.setDetail(pirResultBodyArrayList); - return clientDecryptResponse; - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java deleted file mode 100644 index 7d867479..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/ClientOTService.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.wedpr.pir.sdk.service; - -import com.wedpr.pir.sdk.entity.body.PirDataBody; -import com.wedpr.pir.sdk.entity.request.ClientOTRequest; -import com.wedpr.pir.sdk.entity.response.ClientOTResponse; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.helper.ConvertTypeHelper; -import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -/** - * @author zachma - * @date 2024/8/20 - */ -public class ClientOTService { - private final Random rand = new Random(); - - /* hash披露, 请求方选择id，生成随机数a、b */ - protected ClientOTResponse runClientOTParam(ClientOTRequest clientOTRequest) { - int filterLength = clientOTRequest.getFilterLength(); - BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); - BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); - - BigInteger x = CryptoOperatorHelper.powMod(blindingA); - BigInteger y = CryptoOperatorHelper.powMod(blindingB); - BigInteger blindingC = CryptoOperatorHelper.mulMod(blindingA, blindingB); - - List pirDataArrayList = new ArrayList<>(); - for (String searchId : clientOTRequest.getDataBodyList()) { - String filter = - searchId.length() < filterLength - ? searchId - : searchId.substring(0, filterLength); - BigInteger z0 = calculateZ0(searchId, blindingC); - PirDataBody pirDataBody = new PirDataBody(); - pirDataBody.setFilter(filter); - pirDataBody.setZ0(z0); - pirDataArrayList.add(pirDataBody); - } - return new ClientOTResponse(blindingB, x, y, pirDataArrayList); - } - - /* hash筛选, 请求方选择顺序\delta\in \{0,1,..,m-1\}，生成随机数a、b */ - protected ClientOTResponse runClientOTCipher(ClientOTRequest clientOTRequest) - throws WedprException { - int obfuscationOrder = clientOTRequest.getObfuscationOrder(); - BigInteger blindingA = CryptoOperatorHelper.getRandomInt(); - BigInteger blindingB = CryptoOperatorHelper.getRandomInt(); - - BigInteger x = CryptoOperatorHelper.powMod(blindingA); - BigInteger y = CryptoOperatorHelper.powMod(blindingB); - BigInteger blindingC = CryptoOperatorHelper.mulMod(blindingA, blindingB); - - List pirDataArrayList = new ArrayList<>(); - for (String searchId : clientOTRequest.getDataBodyList()) { - int idIndex = rand.nextInt(obfuscationOrder + 1); - BigInteger z0 = calculateIndexZ0(idIndex, blindingC); - List idHashVecList = - CryptoOperatorHelper.getIdHashVec(obfuscationOrder, idIndex, searchId); - - PirDataBody pirDataBody = new PirDataBody(); - pirDataBody.setZ0(z0); - pirDataBody.setIdIndex(idIndex); - pirDataBody.setIdHashList(idHashVecList); - pirDataArrayList.add(pirDataBody); - } - return new ClientOTResponse(blindingB, x, y, pirDataArrayList); - } - - private BigInteger calculateZ0(String searchId, BigInteger blindingC) { - byte[] idBytes = searchId.getBytes(StandardCharsets.UTF_8); - BigInteger idNumber = ConvertTypeHelper.bytesToBigInteger(idBytes); - return CryptoOperatorHelper.powMod(blindingC.subtract(idNumber)); - } - - private BigInteger calculateIndexZ0(Integer idIndex, BigInteger blindingC) { - // 将整数转长整数 - BigInteger idNumber = BigInteger.valueOf(idIndex); - return CryptoOperatorHelper.powMod(blindingC.subtract(idNumber)); - } -} diff --git a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java b/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java deleted file mode 100644 index bdd97265..00000000 --- a/java/ppc-pir-services/sdk/src/main/java/com/wedpr/pir/sdk/service/PirJobService.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.wedpr.pir.sdk.service; - -import com.wedpr.pir.sdk.entity.body.SimpleEntity; -import com.wedpr.pir.sdk.entity.param.PirJobParam; -import com.wedpr.pir.sdk.entity.request.ClientDecryptRequest; -import com.wedpr.pir.sdk.entity.request.ClientOTRequest; -import com.wedpr.pir.sdk.entity.response.ClientDecryptResponse; -import com.wedpr.pir.sdk.entity.response.ClientOTResponse; -import com.wedpr.pir.sdk.entity.response.PirResultResponse; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import lombok.Getter; - -/** - * @author zachma - * @date 2024/8/20 - */ -public class PirJobService { - - @Getter private final PirJobParam pirJobParam; - private final ClientOTService clientOTService; - private final ClientDecryptService clientDecryptService; - - public PirJobService(PirJobParam pirJobParam) { - this.pirJobParam = pirJobParam; - this.clientOTService = new ClientOTService(); - this.clientDecryptService = new ClientDecryptService(); - } - - public ClientOTResponse requesterOtCipher() throws WedprException { - ClientOTRequest clientOTRequest = new ClientOTRequest(); - ClientOTResponse otParamResponse; - clientOTRequest.setDataBodyList(pirJobParam.getSearchIdList()); - if (pirJobParam.getJobAlgorithmType().equals(ParamEnum.AlgorithmType.idFilter.getValue())) { - clientOTRequest.setFilterLength(pirJobParam.getFilterLength()); - otParamResponse = clientOTService.runClientOTParam(clientOTRequest); - } else if (pirJobParam - .getJobAlgorithmType() - .equals(ParamEnum.AlgorithmType.idObfuscation.getValue())) { - otParamResponse = clientOTService.runClientOTCipher(clientOTRequest); - } else { - throw new WedprException(WedprStatusEnum.INVALID_INPUT_VALUE); - } - return otParamResponse; - } - - public PirResultResponse requesterOtRecover( - ClientOTResponse otParamResponse, PirJobParam clientJobRequest, SimpleEntity otResult) { - ClientDecryptRequest clientDecryptRequest = new ClientDecryptRequest(); - clientDecryptRequest.setB(otParamResponse.getB()); - clientDecryptRequest.setDataBodyList(clientJobRequest.getSearchIdList()); - clientDecryptRequest.setServerResult(otResult.getData()); - - ClientDecryptResponse clientDecryptResponse = - clientDecryptService.runDecryptOTparam(clientDecryptRequest); - - PirResultResponse pirResultResponse = new PirResultResponse(); - pirResultResponse.setJobType(clientJobRequest.getJobType()); - pirResultResponse.setDetail(clientDecryptResponse); - return pirResultResponse; - } -} diff --git a/java/ppc-pir-services/settings.gradle b/java/ppc-pir-services/settings.gradle deleted file mode 100644 index 301af393..00000000 --- a/java/ppc-pir-services/settings.gradle +++ /dev/null @@ -1,11 +0,0 @@ -pluginManagement { - repositories { - gradlePluginPortal() - } -} - -rootProject.name = "ppc-pir-services" - -include ':ppc-pir-sdk' -project(":ppc-pir-sdk").projectDir = file("sdk") - diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java deleted file mode 100644 index f2c23ed7..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/WedprPirApplication.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.webank.wedpr.pir; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.domain.EntityScan; - -/** - * @author zachma - * @date 2024/8/18 - */ -@SpringBootApplication -@EnableAutoConfiguration -@MapperScan("com.webank.wedpr.pir.http.mapper") -@EntityScan(basePackages = {"cn.webank.wedpr.http", "cn.webank.wedpr.pir"}) -public class WedprPirApplication { - public static void main(String[] args) { - SpringApplication.run(WedprPirApplication.class, args); - System.out.println("Start WedprPirApplication Server successfully!"); - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java deleted file mode 100644 index 4cb4f463..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/Dataset.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.webank.wedpr.pir.crypto.entity; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/4 - */ -@Data -public class Dataset { - private String datasetId; - - private String datasetFields; - - private String datasetStoragePath; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java deleted file mode 100644 index b6173658..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/DatasetStoragePath.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.webank.wedpr.pir.crypto.entity; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/4 - */ -@Data -public class DatasetStoragePath { - private String storageType; - private String filePath; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java deleted file mode 100644 index d153c4a8..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/PirTable.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedpr.pir.crypto.entity; - -import lombok.Data; - -/** - * @author zachma - * @date 2024/8/19 - */ -@Data -public class PirTable { - - private Integer id; - - private String pirKey; - - private String pirValue; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java deleted file mode 100644 index 85a347ec..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/entity/WedprAuthTable.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedpr.pir.crypto.entity; - -import java.time.LocalDateTime; -import javax.persistence.Entity; -import lombok.Data; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Entity -@Data -public class WedprAuthTable { - private String serviceId; - private String accessKeyId; - private LocalDateTime expireTime; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java deleted file mode 100644 index 68be68e0..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirOTService.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.webank.wedpr.pir.crypto.service; - -import com.wedpr.pir.sdk.entity.request.ServerOTRequest; -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import com.wedpr.pir.sdk.exception.WedprException; - -/** - * @author zachma - * @date 2024/8/19 - */ -public interface PirOTService { - /** - * @param serverOTRequest - * @return PirOTResponse - * @throws WedprException - */ - ServerOTResponse runServerOTParam(ServerOTRequest serverOTRequest) throws WedprException; - - /** - * @param serverOTRequest - * @return PirOTResponse - * @throws WedprException - */ - ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) throws WedprException; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java deleted file mode 100644 index a337c02d..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/PirTableService.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedpr.pir.crypto.service; - -import com.webank.wedpr.pir.crypto.entity.PirTable; -import java.util.List; - -/** - * @author zachma - * @date 2024/8/19 - */ -public interface PirTableService { - /** 根据id查询 */ - List idFilterTable(String datasetId, String filter, String[] param); - - List idObfuscationTable(String datasetId, String searchId, String[] params); - - List objectsToPirTableList(List params); -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java deleted file mode 100644 index 56a54e82..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirOTServiceImpl.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.webank.wedpr.pir.crypto.service.impl; - -import com.webank.wedpr.pir.crypto.entity.PirTable; -import com.webank.wedpr.pir.crypto.service.PirOTService; -import com.webank.wedpr.pir.crypto.service.PirTableService; -import com.wedpr.pir.sdk.entity.body.ServerResultBody; -import com.wedpr.pir.sdk.entity.body.ServerResultList; -import com.wedpr.pir.sdk.entity.request.ServerOTRequest; -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.helper.AESHelper; -import com.wedpr.pir.sdk.helper.Constant; -import com.wedpr.pir.sdk.helper.ConvertTypeHelper; -import com.wedpr.pir.sdk.helper.CryptoOperatorHelper; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Service -public class PirOTServiceImpl implements PirOTService { - private static final Logger logger = LoggerFactory.getLogger(PirOTServiceImpl.class); - @Autowired PirTableService pirTableService; - - public ServerOTResponse runServerOTParam(ServerOTRequest serverOTRequest) - throws WedprException { - logger.info("Server start runServerOTParam."); - String datasetId = serverOTRequest.getDatasetId(); - String tableId = Constant.datasetId2tableId(datasetId); - BigInteger x = serverOTRequest.getX(); - BigInteger y = serverOTRequest.getY(); - String[] params = serverOTRequest.getParams(); - List serverResultlist = new ArrayList<>(); - for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { - BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); - String filter = serverOTRequest.getDataBodyList().get(i).getFilter(); - List objects = pirTableService.idFilterTable(tableId, filter, params); - List pirTables = pirTableService.objectsToPirTableList(objects); - List serverResultBodyTempList = new ArrayList<>(); - for (PirTable pirTable : pirTables) { - ServerResultBody serverResultBody = - cipherPirTableContentIdSearch(pirTable, x, y, z0); - serverResultBodyTempList.add(serverResultBody); - } - serverResultlist.add(new ServerResultList(serverResultBodyTempList)); - } - logger.info("Server runServerOTParam success"); - return new ServerOTResponse(serverResultlist); - } - - public ServerOTResponse runServerOTCipher(ServerOTRequest serverOTRequest) - throws WedprException { - logger.info("Server start runServerOTCipher."); - String datasetId = serverOTRequest.getDatasetId(); - BigInteger x = serverOTRequest.getX(); - BigInteger y = serverOTRequest.getY(); - String tableId = Constant.datasetId2tableId(datasetId); - List serverResultlist = new ArrayList<>(); - String[] params = serverOTRequest.getParams(); - for (int i = 0; i < serverOTRequest.getDataBodyList().size(); i++) { - BigInteger z0 = serverOTRequest.getDataBodyList().get(i).getZ0(); - List idHashList = serverOTRequest.getDataBodyList().get(i).getIdHashList(); - List serverResultBodyTempList = new ArrayList<>(); - - for (int j = 0; j < idHashList.size(); j++) { - List objects = - pirTableService.idObfuscationTable(tableId, idHashList.get(j), params); - List pirTables = pirTableService.objectsToPirTableList(objects); - for (PirTable pirTable : pirTables) { - ServerResultBody serverResultBody = - cipherPirTableContentIdObfuscation(pirTable, x, y, z0, j); - serverResultBodyTempList.add(serverResultBody); - } - } - serverResultlist.add(new ServerResultList(serverResultBodyTempList)); - } - logger.info("Server runServerOTCipher success."); - return new ServerOTResponse(serverResultlist); - } - - private ServerResultBody cipherPirTableContentIdSearch( - PirTable pirTable, BigInteger x, BigInteger y, BigInteger z0) throws WedprException { - String uid = pirTable.getPirKey(); - String message = pirTable.getPirValue(); - byte[] uidBytes = uid.getBytes(StandardCharsets.UTF_8); - BigInteger uidNum = ConvertTypeHelper.bytesToBigInteger(uidBytes); - - BigInteger blindingR = CryptoOperatorHelper.getRandomInt(); - BigInteger blindingS = CryptoOperatorHelper.getRandomInt(); - BigInteger w = - CryptoOperatorHelper.OTMul( - CryptoOperatorHelper.OTPow(x, blindingS), - CryptoOperatorHelper.powMod(blindingR)); - BigInteger z1 = CryptoOperatorHelper.OTMul(z0, CryptoOperatorHelper.powMod(uidNum)); - BigInteger key = - CryptoOperatorHelper.OTMul( - CryptoOperatorHelper.OTPow(z1, blindingS), - CryptoOperatorHelper.OTPow(y, blindingR)); - - String aesKey = AESHelper.generateRandomKey(); - BigInteger aesNum = - ConvertTypeHelper.bytesToBigInteger(aesKey.getBytes(StandardCharsets.UTF_8)); - BigInteger messageCipherNum = key.xor(aesNum); - String c = AESHelper.encrypt(message, aesKey); - - return new ServerResultBody(messageCipherNum, w, c); - } - - private ServerResultBody cipherPirTableContentIdObfuscation( - PirTable pirTable, BigInteger x, BigInteger y, BigInteger z0, int index) - throws WedprException { - String message = pirTable.getPirValue(); - BigInteger blindingR = CryptoOperatorHelper.getRandomInt(); - BigInteger blindingS = CryptoOperatorHelper.getRandomInt(); - BigInteger w = - CryptoOperatorHelper.OTMul( - CryptoOperatorHelper.OTPow(x, blindingS), - CryptoOperatorHelper.powMod(blindingR)); - BigInteger z1 = - CryptoOperatorHelper.OTMul( - z0, CryptoOperatorHelper.powMod(BigInteger.valueOf(index))); - BigInteger key = - CryptoOperatorHelper.OTMul( - CryptoOperatorHelper.OTPow(z1, blindingS), - CryptoOperatorHelper.OTPow(y, blindingR)); - - String aesKey = AESHelper.generateRandomKey(); - BigInteger aesNum = - ConvertTypeHelper.bytesToBigInteger(aesKey.getBytes(StandardCharsets.UTF_8)); - BigInteger messageCipherNum = key.xor(aesNum); - String c = AESHelper.encrypt(message, aesKey); - - return new ServerResultBody(messageCipherNum, w, c); - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java deleted file mode 100644 index ef38830a..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/crypto/service/impl/PirTableServiceImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.webank.wedpr.pir.crypto.service.impl; - -import com.webank.wedpr.pir.crypto.entity.PirTable; -import com.webank.wedpr.pir.crypto.service.PirTableService; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; -import org.springframework.stereotype.Service; - -/** - * @author zachma - * @date 2024/8/19 - */ -@Service -public class PirTableServiceImpl implements PirTableService { - - @PersistenceContext private EntityManager entityManager; - - @Override - public List idFilterTable(String datasetId, String filter, String[] params) { - String param = processParams(params); - String sqlFormat = "SELECT t.id as t_id, %s FROM %s t WHERE t.id LIKE CONCAT(%s, '%%')"; - String sql = String.format(sqlFormat, param, datasetId, filter); - Query query = entityManager.createNativeQuery(sql); - return query.getResultList(); - } - - @Override - public List idObfuscationTable(String datasetId, String searchId, String[] params) { - String param = processParams(params); - String sqlFormat = "SELECT t.id as t_id, %s FROM %s t WHERE t.id = %s"; - String sql = String.format(sqlFormat, param, datasetId, searchId); - Query query = entityManager.createNativeQuery(sql); - return query.getResultList(); - } - - public List objectsToPirTableList(List values) { - List result = new LinkedList<>(); - for (int i = 0; i < values.size(); i++) { - Object[] temp = (Object[]) values.get(i); - PirTable pirTable = new PirTable(); - pirTable.setId(i + 1); - pirTable.setPirKey(String.valueOf(temp[0])); - Object[] objects = Arrays.copyOfRange(temp, 1, temp.length); - pirTable.setPirValue(Arrays.toString(objects)); - result.add(pirTable); - } - return result; - } - - private String processParams(String[] params) { - String[] temp = Arrays.stream(params).map(s -> "t." + s).toArray(String[]::new); - return String.join(",", temp); - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java deleted file mode 100644 index 4fd4c8e4..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/CSVFileParser.java +++ /dev/null @@ -1,220 +0,0 @@ -package com.webank.wedpr.pir.http.common; - -import com.opencsv.CSVReaderHeaderAware; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CSVFileParser { - private static final Logger logger = LoggerFactory.getLogger(CSVFileParser.class); - - private interface ParseHandler { - Object call(CSVReaderHeaderAware reader) throws Exception; - } - - private static Object loadCSVFile(String filePath, int chunkSize, ParseHandler handler) - throws Exception { - try (Reader fileReader = new BufferedReader(new FileReader(filePath), chunkSize); - CSVReaderHeaderAware reader = new CSVReaderHeaderAware(fileReader)) { - if (handler != null) { - return handler.call(reader); - } - return null; - } catch (Exception e) { - logger.warn("CSVFileParser exception, filePath: {}, error: ", filePath, e); - throw new WedprException( - WedprStatusEnum.CSV_ERROR, "loadCSVFile exception for " + e.getMessage()); - } - } - - public static List> processCsv2SqlMap(String[] tableFields, String csvFilePath) - throws Exception { - return (List>) - loadCSVFile( - csvFilePath, - Constant.DEFAULT_READ_TRUNK_SIZE, - reader -> { - List> resultValue = new ArrayList<>(); - Map row; - while ((row = reader.readMap()) != null) { - List rowValue = new ArrayList<>(); - for (String field : tableFields) { - if (!row.keySet().contains(field.trim())) { - String errorMsg = - "extractFields failed for the field " - + field - + " not existed in the file " - + tableFields.toString(); - logger.warn(errorMsg); - throw new WedprException( - WedprStatusEnum.CSV_ERROR, errorMsg); - } - rowValue.add(row.get(field)); - } - resultValue.add(rowValue); - } - return resultValue; - }); - } - - public static Set getFields(String filePath) throws Exception { - return (Set) - (loadCSVFile( - filePath, - Constant.DEFAULT_READ_TRUNK_SIZE, - new ParseHandler() { - @Override - public Object call(CSVReaderHeaderAware reader) throws Exception { - return reader.readMap().keySet(); - } - })); - } - - public static class ExtractConfig { - private String originalFilePath; - private List extractFields; - private String extractFilePath; - private String fieldSplitter = ","; - private Integer writeChunkSize = Constant.DEFAULT_WRITE_TRUNK_SIZE; - private Integer readChunkSize = Constant.DEFAULT_READ_TRUNK_SIZE; - - public ExtractConfig() {} - - public ExtractConfig( - String originalFilePath, List extractFields, String extractFilePath) { - this.originalFilePath = originalFilePath; - this.extractFields = extractFields; - this.extractFilePath = extractFilePath; - } - - public String getOriginalFilePath() { - return originalFilePath; - } - - public void setOriginalFilePath(String originalFilePath) { - this.originalFilePath = originalFilePath; - } - - public List getExtractFields() { - return extractFields; - } - - public void setExtractFields(List extractFields) { - this.extractFields = extractFields; - } - - public String getExtractFilePath() { - return extractFilePath; - } - - public void setExtractFilePath(String extractFilePath) { - this.extractFilePath = extractFilePath; - } - - public String getFieldSplitter() { - return fieldSplitter; - } - - public void setFieldSplitter(String fieldSplitter) { - this.fieldSplitter = fieldSplitter; - } - - public Integer getWriteChunkSize() { - return writeChunkSize; - } - - public void setWriteChunkSize(Integer writeChunkSize) { - this.writeChunkSize = writeChunkSize; - } - - public Integer getReadChunkSize() { - return readChunkSize; - } - - public void setReadChunkSize(Integer readChunkSize) { - this.readChunkSize = readChunkSize; - } - - @Override - public String toString() { - return "ExtractConfig{" - + "originalFilePath='" - + originalFilePath - + '\'' - + ", extractFields=" - + extractFields - + ", extractFilePath='" - + extractFilePath - + '\'' - + ", fieldSplitter='" - + fieldSplitter - + '\'' - + ", writeChunkSize=" - + writeChunkSize - + ", readChunkSize=" - + readChunkSize - + '}'; - } - } - - public static void extractFields(ExtractConfig extractConfig) throws Exception { - loadCSVFile( - extractConfig.getOriginalFilePath(), - extractConfig.getReadChunkSize(), - new ParseHandler() { - @Override - public Object call(CSVReaderHeaderAware reader) throws Exception { - // check the fields - Map headerInfo = reader.readMap(); - Set fields = headerInfo.keySet(); - for (String field : extractConfig.getExtractFields()) { - if (!fields.contains(field.trim())) { - String errorMsg = - "extractFields failed for the field " - + field - + " not existed in the file " - + extractConfig.getOriginalFilePath(); - logger.warn(errorMsg); - throw new WedprException(WedprStatusEnum.CSV_ERROR, errorMsg); - } - } - Map row; - try (Writer writer = - new BufferedWriter( - new FileWriter(extractConfig.getExtractFilePath()), - extractConfig.getWriteChunkSize())) { - // write the data(Note: here no need to write the header) - while ((row = reader.readMap()) != null) { - int column = 0; - for (String field : extractConfig.getExtractFields()) { - writer.write(row.get(field)); - if (column < extractConfig.getExtractFields().size() - 1) { - writer.write(extractConfig.getFieldSplitter()); - } - column++; - } - writer.write(Constant.DEFAULT_LINE_SPLITTER); - } - } catch (Exception e) { - logger.warn( - "extractFields exception, config: {}, error", - extractConfig.toString(), - e); - throw new WedprException( - WedprStatusEnum.CSV_ERROR, - "extractFields exception, detail: " - + extractConfig.toString() - + ", error: " - + e.getMessage()); - } - return null; - } - }); - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java deleted file mode 100644 index f967d323..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/common/Constant.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.webank.wedpr.pir.http.common; - -/** - * @author zachma - * @date 2024/8/18 - */ -public class Constant { - - public static final String PIR_API_PREFIX = "/api/pir/v3"; - - public static final Integer DEFAULT_READ_TRUNK_SIZE = 1024 * 1024; - public static final Integer DEFAULT_WRITE_TRUNK_SIZE = 1024 * 1024; - - public static final String DEFAULT_LINE_SPLITTER = "\n"; -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java deleted file mode 100644 index def233a0..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/HadoopConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.webank.wedpr.pir.http.config; - -import java.io.IOException; -import java.net.URI; -import org.apache.hadoop.fs.FileSystem; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @author zachma - * @date 2024/9/4 - */ -@Configuration -public class HadoopConfig { - @Value("${hdfs.uri}") - private String hdfsUri; - - @Value("${hdfs.user}") - private String hdfsUser; - - @Value("${hdfs.local}") - private String hdfsLocal; - - @Bean - public FileSystem fileSystem() throws IOException, InterruptedException { - org.apache.hadoop.conf.Configuration configuration = - new org.apache.hadoop.conf.Configuration(); - configuration.set("fs.defaultFS", hdfsUri); - FileSystem fileSystem = FileSystem.get(URI.create(hdfsUri), configuration, hdfsUser); - return fileSystem; - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java deleted file mode 100644 index ef6adad2..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/config/JacksonConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -/** Copyright (C) @2014-2022 Webank */ -package com.webank.wedpr.pir.http.config; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer; -import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; -import com.fasterxml.jackson.datatype.jsr310.deser.LocalTimeDeserializer; -import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer; -import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; -import com.fasterxml.jackson.datatype.jsr310.ser.LocalTimeSerializer; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JacksonConfig { - @Value("${spring.jackson.local-date-time-format:yyyy-MM-dd HH:mm:ss}") - String localDateTimeFormat; - - @Value("${spring.jackson.local-date-format:yyyy-MM-dd}") - String localDateFormat; - - @Value("${spring.jackson.local-time-format:HH:mm:ss}") - String localTimeFormat; - - @Bean - public ObjectMapper objectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - JavaTimeModule javaTimeModule = new JavaTimeModule(); - javaTimeModule.addSerializer( - LocalDateTime.class, - new LocalDateTimeSerializer(DateTimeFormatter.ofPattern(localDateTimeFormat))); - javaTimeModule.addSerializer( - LocalDate.class, - new LocalDateSerializer(DateTimeFormatter.ofPattern(localDateFormat))); - javaTimeModule.addSerializer( - LocalTime.class, - new LocalTimeSerializer(DateTimeFormatter.ofPattern(localTimeFormat))); - - javaTimeModule.addDeserializer( - LocalDateTime.class, - new LocalDateTimeDeserializer(DateTimeFormatter.ofPattern(localDateTimeFormat))); - javaTimeModule.addDeserializer( - LocalDate.class, - new LocalDateDeserializer(DateTimeFormatter.ofPattern(localDateFormat))); - javaTimeModule.addDeserializer( - LocalTime.class, - new LocalTimeDeserializer(DateTimeFormatter.ofPattern(localTimeFormat))); - objectMapper.registerModule(javaTimeModule); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - return objectMapper; - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java deleted file mode 100644 index e876b9d2..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/controller/PirController.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.webank.wedpr.pir.http.controller; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.webank.wedpr.pir.http.common.Constant; -import com.webank.wedpr.pir.http.service.PirAppService; -import com.webank.wedpr.pir.http.service.PirAuthService; -import com.webank.wedpr.pir.http.service.PirDataService; -import com.wedpr.pir.sdk.PirClient; -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import com.wedpr.pir.sdk.entity.param.PirJobParam; -import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; -import com.wedpr.pir.sdk.entity.request.ClientDirectRequest; -import com.wedpr.pir.sdk.entity.request.ServerJobRequest; -import com.wedpr.pir.sdk.entity.response.ClientJobResponse; -import com.wedpr.pir.sdk.entity.response.PirResultResponse; -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -/** - * @author zachma - * @date 2024/8/18 - */ -@RestController -@Slf4j -public class PirController { - private static final Logger logger = LoggerFactory.getLogger(PirController.class); - - @Autowired private ObjectMapper objectMapper; - @Autowired private PirAppService pirAppService; - @Autowired private PirAuthService pirAuthService; - @Autowired private PirDataService pirDataService; - - @PostMapping(Constant.PIR_API_PREFIX + "/auth") - public ClientJobResponse pirAuthController(@RequestBody ClientAuthRequest clientAuthRequest) { - try { - logger.info( - "ClientAuthRequest: clientAuthRequest: {}.", - objectMapper.writeValueAsString(clientAuthRequest)); - // 1. 发起认证auth - ServiceConfigBody serviceConfigBody = - pirAuthService.checkServiceAuth(clientAuthRequest); - return new ClientJobResponse(WedprStatusEnum.SUCCESS, serviceConfigBody); - } catch (Exception e) { - logger.warn( - "匿踪任务认证失败, serviceId: {}, response: {}", - clientAuthRequest.getServiceId(), - WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); - return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, null); - } - } - - @PostMapping(Constant.PIR_API_PREFIX + "/client") - public PirResultResponse pirClientController( - @RequestBody ClientDirectRequest clientDirectRequest) { - try { - logger.info( - "ClientDirectRequest: clientDirectRequest: {}.", - objectMapper.writeValueAsString(clientDirectRequest)); - PirJobParam pirJobParam = new PirJobParam(); - pirJobParam.setPirInvokeType(ParamEnum.JobMode.DirectorMode.getValue()); - pirJobParam.setJobType(ParamEnum.JobType.SearchValue.getValue()); - pirJobParam.setJobAlgorithmType(clientDirectRequest.getAlgorithmType()); - - // todo:对接网关 - pirJobParam.setGatewayUrl("localhost:5831"); - - pirJobParam.setSearchIdList(clientDirectRequest.getSearchIds()); - pirJobParam.setServiceConfigBody(clientDirectRequest.getServiceConfigBody()); - - PirClient pirClient = new PirClient(pirJobParam); - return pirClient.executePirJob(); - } catch (Exception e) { - logger.warn("匿踪任务失败, response: {}", WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); - return null; - } - } - - @PostMapping(Constant.PIR_API_PREFIX + "/server") - public ClientJobResponse pirServerController(@RequestBody ServerJobRequest serverJobRequest) { - try { - pirDataService.processPirDataset(serverJobRequest.getServiceConfigBody()); - logger.info( - "WedprServerjob: serverJobRequest: {}.", - objectMapper.writeValueAsString(serverJobRequest)); - // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 - ServerOTResponse serverOTResponse = pirAppService.providerOtCipher(serverJobRequest); - return new ClientJobResponse(WedprStatusEnum.SUCCESS, serverOTResponse); - } catch (Exception e) { - e.printStackTrace(); - logger.warn( - "匿踪任务失败, serviceId: {}, response: {}", - serverJobRequest.getServiceId(), - WedprStatusEnum.SYSTEM_EXCEPTION.getMessage()); - return new ClientJobResponse(WedprStatusEnum.SYSTEM_EXCEPTION, e.getMessage()); - } - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java deleted file mode 100644 index 061fbe60..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/mapper/WedprServiceAuthTableMapper.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.webank.wedpr.pir.http.mapper; - -import com.webank.wedpr.pir.crypto.entity.Dataset; -import com.webank.wedpr.pir.crypto.entity.WedprAuthTable; -import java.util.List; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Result; -import org.apache.ibatis.annotations.Results; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Mapper -public interface WedprServiceAuthTableMapper { - - @Select( - "SELECT service_id, access_key_id, expire_time FROM wedpr_service_auth_table WHERE service_id = #{serviceId} AND access_key_id = #{accessKeyId}") - @Results({ - @Result(property = "serviceId", column = "service_id"), - @Result(property = "accessKeyId", column = "access_key_id"), - @Result(property = "expireTime", column = "expire_time") - }) - WedprAuthTable searchWedprAuthTable( - @Param("serviceId") String serviceId, @Param("accessKeyId") String accessKeyId); - - @Select("SELECT service_config FROM wedpr_published_service WHERE service_id = #{serviceId}") - String searchWedprServicePublishTable(@Param("serviceId") String serviceId); - - @Select("SHOW TABLES") - List showAllTables(); - - @Select( - "SELECT dataset_id, dataset_fields, dataset_storage_path FROM wedpr_dataset WHERE dataset_id = #{datasetId}") - @Results({ - @Result(property = "datasetId", column = "dataset_id"), - @Result(property = "datasetFields", column = "dataset_fields"), - @Result(property = "datasetStoragePath", column = "dataset_storage_path") - }) - Dataset searchWedprDatasetId(@Param("datasetId") String datasetId); - - @Update(value = "${nativeSql}") - void executeTableByNativeSql(@Param("nativeSql") String nativeSql); -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java deleted file mode 100644 index 1852c523..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAppService.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.webank.wedpr.pir.http.service; - -import com.webank.wedpr.pir.crypto.service.PirOTService; -import com.wedpr.pir.sdk.entity.request.ServerJobRequest; -import com.wedpr.pir.sdk.entity.request.ServerOTRequest; -import com.wedpr.pir.sdk.entity.response.ServerOTResponse; -import com.wedpr.pir.sdk.enums.ParamEnum; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * @author zachma - * @date 2024/8/18 - */ -@Service -public class PirAppService { - - @Autowired private PirOTService pirOTService; - - public ServerOTResponse providerOtCipher(ServerJobRequest serverJobRequest) - throws WedprException { - // 1. 根据请求，筛选数据，加密密钥，返回筛选结果及AES消息密文 - ServerOTRequest serverOTRequest = new ServerOTRequest(); - serverOTRequest.setJobType(serverJobRequest.getJobType()); - serverOTRequest.setDatasetId(serverJobRequest.getServiceConfigBody().getDatasetId()); - serverOTRequest.setX(serverJobRequest.getX()); - serverOTRequest.setY(serverJobRequest.getY()); - serverOTRequest.setDataBodyList(serverJobRequest.getDataBodyList()); - - ServerOTResponse otResultResponse; - if (serverJobRequest - .getJobAlgorithmType() - .equals(ParamEnum.AlgorithmType.idFilter.getValue())) { - serverOTRequest.setParams(serverJobRequest.getServiceConfigBody().getValues()); - otResultResponse = pirOTService.runServerOTParam(serverOTRequest); - } else if (serverJobRequest - .getJobAlgorithmType() - .equals(ParamEnum.AlgorithmType.idObfuscation.getValue())) { - serverOTRequest.setParams(serverJobRequest.getServiceConfigBody().getExists()); - otResultResponse = pirOTService.runServerOTCipher(serverOTRequest); - } else { - throw new WedprException(WedprStatusEnum.INVALID_INPUT_VALUE); - } - return otResultResponse; - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java deleted file mode 100644 index ca0d979e..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirAuthService.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.webank.wedpr.pir.http.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.webank.wedpr.pir.crypto.entity.WedprAuthTable; -import com.webank.wedpr.pir.http.mapper.WedprServiceAuthTableMapper; -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import com.wedpr.pir.sdk.entity.request.ClientAuthRequest; -import com.wedpr.pir.sdk.exception.WedprException; -import com.wedpr.pir.sdk.exception.WedprStatusEnum; -import java.time.LocalDateTime; -import java.util.Objects; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * @author zachma - * @date 2024/9/3 - */ -@Service -public class PirAuthService { - - @Autowired private WedprServiceAuthTableMapper wedprServiceAuthTableMapper; - - public ServiceConfigBody checkServiceAuth(ClientAuthRequest clientAuthRequest) - throws WedprException { - try { - WedprAuthTable wedprAuthTable = - wedprServiceAuthTableMapper.searchWedprAuthTable( - clientAuthRequest.getServiceId(), clientAuthRequest.getAccessKeyId()); - if (Objects.isNull(wedprAuthTable)) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "用户未申请认证"); - } - - if (LocalDateTime.now().isAfter(wedprAuthTable.getExpireTime())) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, "认证已经过期"); - } - - String serviceConfigStr = - wedprServiceAuthTableMapper.searchWedprServicePublishTable( - clientAuthRequest.getServiceId()); - ObjectMapper objectMapper = new ObjectMapper(); - ServiceConfigBody serviceConfigBody = - objectMapper.readValue(serviceConfigStr, ServiceConfigBody.class); - - return serviceConfigBody; - } catch (Exception e) { - throw new WedprException(WedprStatusEnum.CLIENT_PARAM_EXCEPTION, e.getMessage()); - } - } -} diff --git a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java b/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java deleted file mode 100644 index 25946eb8..00000000 --- a/java/ppc-pir-services/src/main/java/com/webank/wedpr/pir/http/service/PirDataService.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.webank.wedpr.pir.http.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.webank.wedpr.pir.crypto.entity.Dataset; -import com.webank.wedpr.pir.crypto.entity.DatasetStoragePath; -import com.webank.wedpr.pir.http.common.CSVFileParser; -import com.webank.wedpr.pir.http.mapper.WedprServiceAuthTableMapper; -import com.wedpr.pir.sdk.entity.body.ServiceConfigBody; -import com.wedpr.pir.sdk.helper.Constant; -import java.util.Arrays; -import java.util.List; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * @author zachma - * @date 2024/9/4 - */ -@Service -public class PirDataService { - - @Autowired private WedprServiceAuthTableMapper wedprServiceAuthTableMapper; - - @Autowired private FileSystem fileSystem; - - @PersistenceContext private EntityManager entityManager; - - public void processPirDataset(ServiceConfigBody serviceConfigBody) throws Exception { - // 数据预处理 - List allTables = wedprServiceAuthTableMapper.showAllTables(); - String tableId = Constant.datasetId2tableId(serviceConfigBody.getDatasetId()); - if (allTables.contains(tableId)) { - return; - } - Dataset dataset = - wedprServiceAuthTableMapper.searchWedprDatasetId(serviceConfigBody.getDatasetId()); - ObjectMapper objectMapper = new ObjectMapper(); - DatasetStoragePath datasetStoragePath = - objectMapper.readValue(dataset.getDatasetStoragePath(), DatasetStoragePath.class); - - String csvFilePath = "./" + serviceConfigBody.getDatasetId(); - fileSystem.copyToLocalFile( - new Path(datasetStoragePath.getFilePath()), new Path(csvFilePath)); - - String[] datasetFields = - Arrays.stream(dataset.getDatasetFields().trim().split(",")) - .map(String::trim) - .toArray(String[]::new); - processCsv2Db(serviceConfigBody.getDatasetId(), datasetFields, csvFilePath); - } - - private void processCsv2Db(String datasetId, String[] datasetFields, String csvFilePath) - throws Exception { - List> sqlValues = CSVFileParser.processCsv2SqlMap(datasetFields, csvFilePath); - if (sqlValues.size() == 0) { - throw new Exception("数据集为空,请退回"); - } - - String tableId = Constant.datasetId2tableId(datasetId); - String createSqlFormat = "CREATE TABLE %s ( %s , PRIMARY KEY (`id`) USING BTREE )"; - - String[] fieldsWithType = new String[datasetFields.length]; - for (int i = 0; i < datasetFields.length; i++) { - if ("id".equalsIgnoreCase(datasetFields[i])) { - fieldsWithType[i] = datasetFields[i] + " VARCHAR(64)"; - } else { - fieldsWithType[i] = datasetFields[i] + " TEXT"; - } - } - String sql = String.format(createSqlFormat, tableId, String.join(",", fieldsWithType)); - wedprServiceAuthTableMapper.executeTableByNativeSql(sql); - - StringBuilder sb = new StringBuilder(); - for (List values : sqlValues) { - sb.append("(").append(String.join(",", values)).append("), "); - } - String insertValues = sb.toString(); - insertValues = insertValues.substring(0, insertValues.length() - 2); - - String insertSqlFormat = "INSERT INTO %s (%s) VALUES %s "; - sql = - String.format( - insertSqlFormat, tableId, String.join(",", datasetFields), insertValues); - wedprServiceAuthTableMapper.executeTableByNativeSql(sql); - } -} diff --git a/java/ppc-pir-services/src/main/resources/application-dev.properties b/java/ppc-pir-services/src/main/resources/application-dev.properties deleted file mode 100644 index ac03ea13..00000000 --- a/java/ppc-pir-services/src/main/resources/application-dev.properties +++ /dev/null @@ -1,14 +0,0 @@ -### Spring boot configuration -server.address=0.0.0.0 -server.port=5831 -server.session.timeout=60 -banner.charset=UTF-8 -spring.jackson.local-date-time-format=yyyy-MM-dd HH:mm:ss -spring.jackson.local-date-format=yyyy-MM-dd -spring.jackson.local-time-format=HH:mm:ss -spring.jackson.time-zone=GMT+8 -spring.jackson.open-in-view=false - -hdfs.uri=hdfs://139.159.202.235:9900 -hdfs.user=ppc -hdfs.local=./temp \ No newline at end of file diff --git a/java/ppc-pir-services/src/main/resources/application-mysql.properties b/java/ppc-pir-services/src/main/resources/application-mysql.properties deleted file mode 100644 index 3f2227a1..00000000 --- a/java/ppc-pir-services/src/main/resources/application-mysql.properties +++ /dev/null @@ -1,6 +0,0 @@ -spring.datasource.url=jdbc:mysql://xxxxxxx:3306/xxxxx?serverTimezone=Asia/Shanghai&useUnicode=true&autoReconnect=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&useFractionalSeconds=true&useUnbufferedInput=false&useSSL=false -spring.datasource.username=xxxx -spring.datasource.password=xxxx -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver -spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5InnoDBDialect -spring.jpa.database=mysql diff --git a/java/ppc-pir-services/src/main/resources/application.properties b/java/ppc-pir-services/src/main/resources/application.properties deleted file mode 100644 index 80c5d0e0..00000000 --- a/java/ppc-pir-services/src/main/resources/application.properties +++ /dev/null @@ -1,7 +0,0 @@ - -spring.profiles.active=dev -# mysql or dm or gauss or kingbase or shentong or postgresql -spring.profiles.include=mysql - -server.shutdown=graceful -spring.lifecycle.timeout-per-shutdown-phase=30s diff --git a/java/ppc-pir-services/src/main/resources/log4j2.xml b/java/ppc-pir-services/src/main/resources/log4j2.xml deleted file mode 100644 index b3c6d126..00000000 --- a/java/ppc-pir-services/src/main/resources/log4j2.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - info - log location - ./logs - - - wedpr-pir - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 20700048babf82c432599f3b76a11844fbac30d0 Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Tue, 15 Oct 2024 14:23:00 +0800 Subject: [PATCH 054/120] Feature milestone2 (#44) * update secure lr * update model and predict * update ppc_dev --- python/aes_key.bin | 2 + python/ppc_dev/__init__.py | 0 python/ppc_dev/common/__init__.py | 0 python/ppc_dev/common/base_context.py | 13 ++ python/ppc_dev/common/base_result.py | 8 ++ python/ppc_dev/job_exceuter/__init__.py | 0 python/ppc_dev/job_exceuter/hdfs_client.py | 53 +++++++ python/ppc_dev/job_exceuter/pws_client.py | 66 +++++++++ python/ppc_dev/result/__init__.py | 0 python/ppc_dev/result/fe_result.py | 27 ++++ python/ppc_dev/result/model_result.py | 56 ++++++++ python/ppc_dev/result/psi_result.py | 27 ++++ python/ppc_dev/test/__init__.py | 0 python/ppc_dev/test/test_dev.py | 70 +++++++++ python/ppc_dev/utils/__init__.py | 0 python/ppc_dev/utils/agency.py | 5 + python/ppc_dev/utils/utils.py | 12 ++ python/ppc_dev/wedpr_data/__init__.py | 0 python/ppc_dev/wedpr_data/data_context.py | 35 +++++ python/ppc_dev/wedpr_data/wedpr_data.py | 64 +++++++++ python/ppc_dev/wedpr_session/__init__.py | 0 python/ppc_dev/wedpr_session/wedpr_session.py | 135 ++++++++++++++++++ python/ppc_model/common/base_context.py | 26 ++++ python/ppc_model/model_crypto/__init__.py | 0 python/ppc_model/model_crypto/crypto_aes.py | 78 ++++++++++ python/ppc_model/model_crypto/test_aes.py | 26 ++++ .../secure_lgbm/secure_lgbm_context.py | 1 + .../ppc_model/secure_lgbm/vertical/booster.py | 95 +++++++++++- .../ppc_model/secure_lr/secure_lr_context.py | 1 + .../secure_lr/secure_lr_prediction_engine.py | 38 +++++ .../secure_lr/secure_lr_training_engine.py | 1 - .../secure_lr/test/test_secure_lr_training.py | 56 ++++---- .../ppc_model/secure_lr/vertical/booster.py | 73 +++++++++- 33 files changed, 933 insertions(+), 35 deletions(-) create mode 100644 python/aes_key.bin create mode 100644 python/ppc_dev/__init__.py create mode 100644 python/ppc_dev/common/__init__.py create mode 100644 python/ppc_dev/common/base_context.py create mode 100644 python/ppc_dev/common/base_result.py create mode 100644 python/ppc_dev/job_exceuter/__init__.py create mode 100644 python/ppc_dev/job_exceuter/hdfs_client.py create mode 100644 python/ppc_dev/job_exceuter/pws_client.py create mode 100644 python/ppc_dev/result/__init__.py create mode 100644 python/ppc_dev/result/fe_result.py create mode 100644 python/ppc_dev/result/model_result.py create mode 100644 python/ppc_dev/result/psi_result.py create mode 100644 python/ppc_dev/test/__init__.py create mode 100644 python/ppc_dev/test/test_dev.py create mode 100644 python/ppc_dev/utils/__init__.py create mode 100644 python/ppc_dev/utils/agency.py create mode 100644 python/ppc_dev/utils/utils.py create mode 100644 python/ppc_dev/wedpr_data/__init__.py create mode 100644 python/ppc_dev/wedpr_data/data_context.py create mode 100644 python/ppc_dev/wedpr_data/wedpr_data.py create mode 100644 python/ppc_dev/wedpr_session/__init__.py create mode 100644 python/ppc_dev/wedpr_session/wedpr_session.py create mode 100644 python/ppc_model/model_crypto/__init__.py create mode 100644 python/ppc_model/model_crypto/crypto_aes.py create mode 100644 python/ppc_model/model_crypto/test_aes.py create mode 100644 python/ppc_model/secure_lr/secure_lr_prediction_engine.py diff --git a/python/aes_key.bin b/python/aes_key.bin new file mode 100644 index 00000000..0d516172 --- /dev/null +++ b/python/aes_key.bin @@ -0,0 +1,2 @@ +m}9H +褊c?Ӈ!<> \ No newline at end of file diff --git a/python/ppc_dev/__init__.py b/python/ppc_dev/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/common/__init__.py b/python/ppc_dev/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/common/base_context.py b/python/ppc_dev/common/base_context.py new file mode 100644 index 00000000..1496f381 --- /dev/null +++ b/python/ppc_dev/common/base_context.py @@ -0,0 +1,13 @@ +import os + + +class BaseContext: + + def __init__(self, project_id, user_name, pws_endpoint=None, hdfs_endpoint=None, token=None): + + self.project_id = project_id + self.user_name = user_name + self.pws_endpoint = pws_endpoint + self.hdfs_endpoint = hdfs_endpoint + self.token = token + self.workspace = os.path.join(self.project_id, self.user_name) diff --git a/python/ppc_dev/common/base_result.py b/python/ppc_dev/common/base_result.py new file mode 100644 index 00000000..ace5f8e3 --- /dev/null +++ b/python/ppc_dev/common/base_result.py @@ -0,0 +1,8 @@ +from ppc_dev.common.base_context import BaseContext + + +class BaseResult: + + def __init__(self, ctx: BaseContext): + + self.ctx = ctx diff --git a/python/ppc_dev/job_exceuter/__init__.py b/python/ppc_dev/job_exceuter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/job_exceuter/hdfs_client.py b/python/ppc_dev/job_exceuter/hdfs_client.py new file mode 100644 index 00000000..ca77360a --- /dev/null +++ b/python/ppc_dev/job_exceuter/hdfs_client.py @@ -0,0 +1,53 @@ +import requests +import pandas as pd +import io + + +class HDFSApi: + def __init__(self, base_url): + self.base_url = base_url + + def upload(self, dataframe, hdfs_path): + """ + 上传Pandas DataFrame到HDFS + :param dataframe: 要上传的Pandas DataFrame + :param hdfs_path: HDFS目标路径 + :return: 响应信息 + """ + # 将DataFrame转换为CSV格式 + csv_buffer = io.StringIO() + dataframe.to_csv(csv_buffer, index=False) + + # 发送PUT请求上传CSV数据 + response = requests.put( + f"{self.base_url}/upload?path={hdfs_path}", + data=csv_buffer.getvalue(), + headers={'Content-Type': 'text/csv'} + ) + return response.json() + + def download(self, hdfs_path): + """ + 从HDFS下载数据并返回为Pandas DataFrame + :param hdfs_path: HDFS文件路径 + :return: Pandas DataFrame + """ + response = requests.get(f"{self.base_url}/download?path={hdfs_path}") + if response.status_code == 200: + # 读取CSV数据并转换为DataFrame + dataframe = pd.read_csv(io.StringIO(response.text)) + return dataframe + else: + raise Exception(f"下载失败: {response.json()}") + + def download_data(self, hdfs_path): + """ + 从HDFS下载数据并返回为Pandas DataFrame + :param hdfs_path: HDFS文件路径 + :return: text + """ + response = requests.get(f"{self.base_url}/download?path={hdfs_path}") + if response.status_code == 200: + return response.text + else: + raise Exception(f"下载失败: {response.json()}") diff --git a/python/ppc_dev/job_exceuter/pws_client.py b/python/ppc_dev/job_exceuter/pws_client.py new file mode 100644 index 00000000..8404620a --- /dev/null +++ b/python/ppc_dev/job_exceuter/pws_client.py @@ -0,0 +1,66 @@ +import random +import time + +from ppc_common.ppc_utils import http_utils +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode + + +class PWSApi: + def __init__(self, endpoint, token, + polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): + self.endpoint = endpoint + self.token = token + self.polling_interval_s = polling_interval_s + self.max_retries = max_retries + self.retry_delay_s = retry_delay_s + self._async_run_task_method = 'asyncRunTask' + self._get_task_status_method = 'getTaskStatus' + self._completed_status = 'COMPLETED' + self._failed_status = 'FAILED' + + def run(self, datasets, params): + params = { + 'jsonrpc': '1', + 'method': self._async_run_task_method, + 'token': self.token, + 'id': random.randint(1, 65535), + 'dataset': datasets, + 'params': params + } + response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + if response.status_code != 200: + raise Exception(f"创建任务失败: {response.json()}") + return self._poll_task_status(response.job_id, self.token) + + def _poll_task_status(self, job_id, token): + while True: + params = { + 'jsonrpc': '1', + 'method': self._get_task_status_method, + 'token': token, + 'id': random.randint(1, 65535), + 'params': { + 'taskID': job_id, + } + } + response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + if response.status_code != 200: + raise Exception(f"轮询任务失败: {response.json()}") + if response['result']['status'] == self._completed_status: + return response['result'] + elif response['result']['status'] == self._failed_status: + raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) + time.sleep(self.polling_interval_s) + + def _send_request_with_retry(self, request_func, *args, **kwargs): + attempt = 0 + while attempt < self.max_retries: + try: + response = request_func(*args, **kwargs) + return response + except Exception as e: + attempt += 1 + if attempt < self.max_retries: + time.sleep(self.retry_delay_s) + else: + raise e diff --git a/python/ppc_dev/result/__init__.py b/python/ppc_dev/result/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/result/fe_result.py b/python/ppc_dev/result/fe_result.py new file mode 100644 index 00000000..65c4dfc5 --- /dev/null +++ b/python/ppc_dev/result/fe_result.py @@ -0,0 +1,27 @@ +import os + +from ppc_dev.wedpr_data.data_context import DataContext +from ppc_dev.common.base_result import BaseResult + + +class FeResult(BaseResult): + + FE_RESULT_FILE = "fe_result.csv" + + def __init__(self, dataset: DataContext, job_id: str): + + super().__init__(dataset.ctx) + self.job_id = job_id + + participant_id_list = [] + for dataset in self.dataset.datasets: + participant_id_list.append(dataset.agency.agency_id) + self.participant_id_list = participant_id_list + + result_list = [] + for dataset in self.dataset.datasets: + dataset.update_path(os.path.join(self.job_id, self.FE_RESULT_FILE)) + result_list.append(dataset) + + fe_result = DataContext(*result_list) + return fe_result diff --git a/python/ppc_dev/result/model_result.py b/python/ppc_dev/result/model_result.py new file mode 100644 index 00000000..f1cc4e07 --- /dev/null +++ b/python/ppc_dev/result/model_result.py @@ -0,0 +1,56 @@ +import os +import numpy as np + +from ppc_common.ppc_utils import utils + +from ppc_dev.wedpr_data.data_context import DataContext +from ppc_dev.common.base_result import BaseResult +from ppc_dev.job_exceuter.hdfs_client import HDFSApi + + +class ModelResult(BaseResult): + + FEATURE_BIN_FILE = "feature_bin.json" + MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' + TEST_MODEL_OUTPUT_FILE = "xgb_output.csv" + TRAIN_MODEL_OUTPUT_FILE = "xgb_train_output.csv" + + def __init__(self, dataset: DataContext, job_id: str, job_type: str): + + super().__init__(dataset.ctx) + self.job_id = job_id + + participant_id_list = [] + for dataset in self.dataset.datasets: + participant_id_list.append(dataset.agency.agency_id) + self.participant_id_list = participant_id_list + + if job_type == 'xgb_training': + self._xgb_train_result() + + def _xgb_train_result(self): + + # train_praba, test_praba, train_y, test_y, feature_importance, split_xbin, trees, params + # 从hdfs读取结果文件信息，构造为属性 + train_praba_path = os.path.join(self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) + test_praba_path = os.path.join(self.job_id, self.TEST_MODEL_OUTPUT_FILE) + train_output = HDFSApi.download(train_praba_path) + test_output = HDFSApi.download(test_praba_path) + self.train_praba = train_output['class_pred'].values + self.test_praba = test_output['class_pred'].values + if 'class_label' in train_output.columns: + self.train_y = train_output['class_label'].values + self.test_y = test_output['class_label'].values + else: + self.train_y = None + self.test_y = None + + feature_bin_path = os.path.join(self.job_id, self.FEATURE_BIN_FILE) + model_path = os.path.join(self.job_id, self.MODEL_DATA_FILE) + feature_bin_data = HDFSApi.download_data(feature_bin_path) + model_data = HDFSApi.download_data(model_path) + + self.feature_importance = ... + self.split_xbin = feature_bin_data + self.trees = model_data + self.params = ... diff --git a/python/ppc_dev/result/psi_result.py b/python/ppc_dev/result/psi_result.py new file mode 100644 index 00000000..dae03f58 --- /dev/null +++ b/python/ppc_dev/result/psi_result.py @@ -0,0 +1,27 @@ +import os + +from ppc_dev.wedpr_data.data_context import DataContext +from ppc_dev.common.base_result import BaseResult + + +class PSIResult(BaseResult): + + PSI_RESULT_FILE = "psi_result.csv" + + def __init__(self, dataset: DataContext, job_id: str): + + super().__init__(dataset.ctx) + self.job_id = job_id + + participant_id_list = [] + for dataset in self.dataset.datasets: + participant_id_list.append(dataset.agency.agency_id) + self.participant_id_list = participant_id_list + + result_list = [] + for dataset in self.dataset.datasets: + dataset.update_path(os.path.join(self.job_id, self.PSI_RESULT_FILE)) + result_list.append(dataset) + + psi_result = DataContext(*result_list) + return psi_result diff --git a/python/ppc_dev/test/__init__.py b/python/ppc_dev/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/test/test_dev.py b/python/ppc_dev/test/test_dev.py new file mode 100644 index 00000000..03bad07a --- /dev/null +++ b/python/ppc_dev/test/test_dev.py @@ -0,0 +1,70 @@ +import unittest +import numpy as np +import pandas as pd +from sklearn import metrics + +from ppc_dev.common.base_context import BaseContext +from ppc_dev.utils.agency import Agency +from ppc_dev.wedpr_data.wedpr_data import WedprData +from ppc_dev.wedpr_data.data_context import DataContext +from ppc_dev.wedpr_session.wedpr_session import WedprSession + + +# 从jupyter环境中获取project_id等信息 +# create workspace +# 相同项目/刷新专家模式project_id固定 +project_id = 'p-123' +user = 'admin' +my_agency='WeBank' +pws_endpoint = '0.0.0.0:0000' +hdfs_endpoint = '0.0.0.0:0001' +token = 'abc...' + + +# 自定义合作方机构 +partner_agency1='SG' +partner_agency2='TX' + +# 初始化project ctx 信息 +ctx = BaseContext(project_id, user, pws_endpoint, hdfs_endpoint, token) + +# 注册 agency +agency1 = Agency(agency_id=my_agency) +agency2 = Agency(agency_id=partner_agency1) + +# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path +# pd.Dataframe +df = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 +}) +dataset1 = WedprData(ctx, values=df, agency=agency1) +dataset1.storage_client = None +dataset1.save_values(path='./project_id/user/data/d-101') +# hdfs_path +dataset2 = WedprData(ctx, dataset_path='./data_path/d-123', agency=agency2, is_label_holder=True) +dataset2.storage_client = None +dataset2.load_values() + +# 支持更新dataset的values数据 +df2 = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + 'y': np.random.randint(0, 2, size=100), + **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 +}) +dataset2.update_values(values=df2) + +# 构建 dataset context +dataset = DataContext(dataset1, dataset2) + +# 初始化 wedpr task session（含数据） +task = WedprSession(dataset, my_agency=my_agency) +print(task.participant_id_list, task.result_receiver_id_list) +# 执行psi任务 +psi_result = task.psi() + +# 初始化 wedpr task session（不含数据） （推荐：使用更灵活） +task = WedprSession(my_agency=my_agency) +# 执行psi任务 +fe_result = task.proprecessing(dataset) +print(task.participant_id_list, task.result_receiver_id_list) diff --git a/python/ppc_dev/utils/__init__.py b/python/ppc_dev/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/utils/agency.py b/python/ppc_dev/utils/agency.py new file mode 100644 index 00000000..461bce03 --- /dev/null +++ b/python/ppc_dev/utils/agency.py @@ -0,0 +1,5 @@ +class Agency: + + def __init__(self, agency_id): + + self.agency_id = agency_id diff --git a/python/ppc_dev/utils/utils.py b/python/ppc_dev/utils/utils.py new file mode 100644 index 00000000..25e0ec72 --- /dev/null +++ b/python/ppc_dev/utils/utils.py @@ -0,0 +1,12 @@ +import uuid +from enum import Enum + + +class IdPrefixEnum(Enum): + DATASET = "d-" + ALGORITHM = "a-" + JOB = "j-" + + +def make_id(prefix): + return prefix + str(uuid.uuid4()).replace("-", "") diff --git a/python/ppc_dev/wedpr_data/__init__.py b/python/ppc_dev/wedpr_data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/wedpr_data/data_context.py b/python/ppc_dev/wedpr_data/data_context.py new file mode 100644 index 00000000..177a828f --- /dev/null +++ b/python/ppc_dev/wedpr_data/data_context.py @@ -0,0 +1,35 @@ +import os + +from ppc_dev.utils import utils + + +class DataContext: + + def __init__(self, *datasets): + self.datasets = list(datasets) + self.ctx = self.datasets[0].ctx + + self._check_datasets() + + def _save_dataset(self, dataset): + if dataset.dataset_path is None: + dataset.dataset_id = utils.make_id(utils.IdPrefixEnum.DATASET.value) + dataset.dataset_path = os.path.join(dataset.ctx.workspace, dataset.dataset_id) + if self.storage_client is not None: + self.storage_client.upload(self.values, self.dataset_path) + + def _check_datasets(self): + for dataset in self.datasets: + self._save_dataset(dataset) + + def to_psi_format(self): + dataset_psi = [] + for dataset in self.datasets: + dataset_psi.append(dataset.dataset_path) + return dataset_psi + + def to_model_formort(self): + dataset_model = [] + for dataset in self.datasets: + dataset_model.append(dataset.dataset_path) + return dataset_model diff --git a/python/ppc_dev/wedpr_data/wedpr_data.py b/python/ppc_dev/wedpr_data/wedpr_data.py new file mode 100644 index 00000000..2e4c9575 --- /dev/null +++ b/python/ppc_dev/wedpr_data/wedpr_data.py @@ -0,0 +1,64 @@ +import pandas as pd + +from ppc_dev.common.base_context import BaseContext +from ppc_dev.job_exceuter.hdfs_client import HDFSApi + + +class WedprData: + + def __init__(self, + ctx: BaseContext, + dataset_id=None, + dataset_path=None, + agency=None, + values=None, + is_label_holder=False): + + # super().__init__(project_id) + self.ctx = ctx + + self.dataset_id = dataset_id + self.dataset_path = dataset_path + self.agency = agency + self.values = values + self.is_label_holder = is_label_holder + self.columns = None + self.shape = None + + self.storage_client = HDFSApi(self.ctx.hdfs_endpoint) + + if self.values is not None: + self.columns = self.values.columns + self.shape = self.values.shape + + def load_values(self): + # 加载hdfs的数据集 + if self.storage_client is not None: + self.values = self.storage_client.download(self.dataset_path) + self.columns = self.values.columns + self.shape = self.values.shape + + def save_values(self, path=None): + # 保存数据到hdfs目录 + if path is not None: + self.dataset_path = path + if self.storage_client is not None: + self.storage_client.upload(self.values, self.dataset_path) + + def update_values(self, values: pd.DataFrame = None, path: str = None): + # 将数据集存入hdfs相同路径，替换旧数据集 + if values is not None: + self.values = values + self.columns = self.values.columns + self.shape = self.values.shape + if path is not None: + self.dataset_path = path + if self.storage_client is not None: + self.storage_client.upload(self.values, self.dataset_path) + + def update_path(self, path: str = None): + # 将数据集存入hdfs相同路径，替换旧数据集 + if path is not None: + self.dataset_path = path + if self.values is not None: + self.values = None diff --git a/python/ppc_dev/wedpr_session/__init__.py b/python/ppc_dev/wedpr_session/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_dev/wedpr_session/wedpr_session.py b/python/ppc_dev/wedpr_session/wedpr_session.py new file mode 100644 index 00000000..e3c34e01 --- /dev/null +++ b/python/ppc_dev/wedpr_session/wedpr_session.py @@ -0,0 +1,135 @@ +from ppc_dev.wedpr_data.data_context import DataContext +from ppc_dev.job_exceuter.pws_client import PWSApi +from ppc_dev.result.psi_result import PSIResult +from ppc_dev.result.fe_result import FeResult +from ppc_dev.result.model_result import ModelResult + + +class WedprSession: + + def __init__(self, dataset: DataContext = None, my_agency = None): + + self.dataset = dataset + self.create_agency = my_agency + + if self.dataset is not None: + self.participant_id_list = self.get_agencies() + self.label_holder_agency = self.get_label_holder_agency() + self.result_receiver_id_list = [my_agency] # 仅限jupyter所在机构 + + self.excute = PWSApi(self.dataset.ctx.pws_endpoint, self.dataset.ctx.token) + + def task(self, datasets: list, params: dict = {}): + + self.check_agencies() + job_response = self.excute.run(datasets, params) + + return job_response.job_id + + def psi(self, dataset: DataContext = None, merge_filed: str = 'id'): + + if dataset is not None: + self.update_dataset(dataset) + + # 构造参数 + params = {merge_filed: merge_filed} + + # 执行任务 + job_id = self.task(self.dataset.to_psi_format(), params) + + # 结果处理 + psi_result = PSIResult(dataset, job_id) + + return psi_result + + def proprecessing(self, dataset: DataContext = None, psi_result = None, params: dict = None): + + if dataset is not None: + self.update_dataset(dataset) + + job_id = self.task(self.dataset.to_model_formort()) + + # 结果处理 + datasets_pre = FeResult(dataset, job_id) + + return datasets_pre + + def feature_engineering(self, dataset: DataContext = None, psi_result = None, params: dict = None): + + if dataset is not None: + self.update_dataset(dataset) + + job_id = self.task(self.dataset.to_model_formort()) + + # 结果处理 + datasets_fe = FeResult(dataset, job_id) + + return datasets_fe + + def xgb_training(self, dataset: DataContext = None, psi_result = None, params: dict = None): + + if dataset is not None: + self.update_dataset(dataset) + self.check_datasets() + + job_id = self.task(self.dataset.to_model_formort()) + + # 结果处理 + model_result = ModelResult(dataset, job_id, job_type='xgb_training') + + return model_result + + def xgb_predict(self, dataset: DataContext = None, psi_result = None, model = None): + + if dataset is not None: + self.update_dataset(dataset) + self.check_datasets() + + # 上传模型到hdfs + job_id = self.task(self.dataset.to_model_formort()) + + # 结果处理 + model_result = ModelResult(dataset, job_id, job_type='xgb_predict') + + # 结果处理 + return model_result + + def update_dataset(self, dataset: DataContext): + self.dataset = dataset + self.participant_id_list = self.get_agencies() + self.label_holder_agency = self.get_label_holder_agency() + + def get_agencies(self): + participant_id_list = [] + for dataset in self.dataset.datasets: + participant_id_list.append(dataset.agency.agency_id) + return participant_id_list + + def get_label_holder_agency(self): + label_holder_agency = None + for dataset in self.dataset.datasets: + if dataset.is_label_holder: + label_holder_agency = dataset.agency.agency_id + return label_holder_agency + + def check_agencies(self): + """ + 校验机构数和任务是否匹配 + """ + if len(self.participant_id_list) < 2: + raise ValueError("至少需要传入两个机构") + + def check_datasets(self): + """ + 校验是否包含标签提供方 + """ + if not self.label_holder_agency or self.label_holder_agency not in self.participant_id_list: + raise ValueError("数据集中标签提供方配置错误") + + # def get_agencies(self): + # """ + # 返回所有机构名称的列表。 + + # :return: 机构名称的列表 + # """ + # return self.agencies diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py index 43f1e873..9b16680b 100644 --- a/python/ppc_model/common/base_context.py +++ b/python/ppc_model/common/base_context.py @@ -22,6 +22,9 @@ class BaseContext: # TRAIN_MODEL_OUTPUT_FILE = "train_model_output.csv" TRAIN_MODEL_OUTPUT_FILE = "xgb_train_output.csv" + MODEL_FILE = "model.kpl" + MODEL_ENC_FILE = "model_enc.kpl" + def __init__(self, job_id: str, job_temp_dir: str): self.job_id = job_id self.workspace = os.path.join(job_temp_dir, self.job_id) @@ -85,6 +88,11 @@ def __init__(self, job_id: str, job_temp_dir: str): self.metrics_iteration_file = os.path.join( self.workspace, utils.METRICS_OVER_ITERATION_FILE) + self.model_file = os.path.join( + self.workspace, self.MODEL_FILE) + self.model_enc_file = os.path.join( + self.workspace, self.MODEL_ENC_FILE) + self.remote_summary_evaluation_file = os.path.join( self.job_id, utils.MPC_XGB_EVALUATION_TABLE) self.remote_feature_importance_file = os.path.join( @@ -123,6 +131,24 @@ def __init__(self, job_id: str, job_temp_dir: str): self.remote_metrics_iteration_file = os.path.join( self.job_id, utils.METRICS_OVER_ITERATION_FILE) + self.remote_model_file = os.path.join( + self.job_id, self.MODEL_FILE) + self.remote_model_enc_file = os.path.join( + self.job_id, self.MODEL_ENC_FILE) + + # self.get_key_pair() + self.load_key('aes_key.bin') + @staticmethod def feature_engineering_input_path(job_id: str, job_temp_dir: str): return os.path.join(job_temp_dir, job_id, BaseContext.MODEL_PREPARE_FILE) + + def get_key_pair(self): + with open('public_key.pem', 'rb') as f: + self.public_pem = f.read() + with open('private_key.pem', 'rb') as f: + self.private_pem = f.read() + + def load_key(self, filename): + with open(filename, 'rb') as file: + self.key = file.read() diff --git a/python/ppc_model/model_crypto/__init__.py b/python/ppc_model/model_crypto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/model_crypto/crypto_aes.py b/python/ppc_model/model_crypto/crypto_aes.py new file mode 100644 index 00000000..e63ee2ea --- /dev/null +++ b/python/ppc_model/model_crypto/crypto_aes.py @@ -0,0 +1,78 @@ +import os +import base64 + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding + + +# 生成256位（32字节）的AES密钥 +def generate_aes_key(): + return os.urandom(32) # 32 bytes == 256 bits + + +# 将密钥保存到文件 +def save_key_to_file(key, filename): + with open(filename, 'wb') as file: + file.write(key) + + +# 从文件中加载密钥 +def load_key_from_file(filename): + with open(filename, 'rb') as file: + key = file.read() + return key + + +# key = load_key_from_file('aes_key.bin') + + +# AES加密函数 +def encrypt_data(key, plaintext): + # 使用随机生成的初始向量 (IV) + iv = os.urandom(16) # AES块大小为128位（16字节） + + # 创建AES加密器 + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + encryptor = cipher.encryptor() + + # 对数据进行填充（AES要求输入的块大小为128位） + padder = padding.PKCS7(128).padder() + padded_data = padder.update(plaintext) + padder.finalize() + + # 加密数据 + ciphertext = encryptor.update(padded_data) + encryptor.finalize() + + # 返回IV和密文 + return iv + ciphertext + + +# AES解密函数 +def decrypt_data(key, ciphertext): + # 提取IV和密文 + iv = ciphertext[:16] # 前16字节是IV + actual_ciphertext = ciphertext[16:] + + # 创建AES解密器 + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + decryptor = cipher.decryptor() + + # 解密数据 + decrypted_padded_data = decryptor.update(actual_ciphertext) + decryptor.finalize() + + # 去除填充 + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(decrypted_padded_data) + unpadder.finalize() + + return plaintext + + +def cipher_to_base64(ciphertext): + # 将bytes类型转换为Base64字符串 + encoded_ciphertext = base64.b64encode(ciphertext).decode('utf-8') + return encoded_ciphertext + + +def base64_to_cipher(data): + decoded_ciphertext = base64.b64decode(data) + return decoded_ciphertext diff --git a/python/ppc_model/model_crypto/test_aes.py b/python/ppc_model/model_crypto/test_aes.py new file mode 100644 index 00000000..802ee54e --- /dev/null +++ b/python/ppc_model/model_crypto/test_aes.py @@ -0,0 +1,26 @@ +from ppc_model.model_crypto.crypto_aes import generate_aes_key, save_key_to_file +from ppc_model.model_crypto.crypto_aes import encrypt_data, decrypt_data +from ppc_model.model_crypto.crypto_aes import base64_to_cipher, cipher_to_base64 + + +key = generate_aes_key() +save_key_to_file(key, 'aes_key.bin') +print("AES密钥已生成并保存到aes_key.bin文件中。") + +plaintext = "需要加密的内容".encode('utf-8') +ciphertext = encrypt_data(key, plaintext) +print(f"加密后的内容: {ciphertext}") + +decrypted_text = decrypt_data(key, ciphertext) +print(f"解密后的内容: {decrypted_text.decode('utf-8')}") + +# 保存密文到文件 +# ciphertext = encrypt_data(key, plaintext) +encoded_ciphertext = cipher_to_base64(ciphertext) +print(f"encoded_ciphertext: {encoded_ciphertext}") + +# 使用AES密钥解密 +decoded_ciphertext = base64_to_cipher(encoded_ciphertext) +print(f"encoded_ciphertext: {decoded_ciphertext}") +decrypted_text = decrypt_data(key, decoded_ciphertext) +print(f"解密后的内容字符串: {decrypted_text.decode('utf-8')}") diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py index ea536a14..c366c8e8 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -271,3 +271,4 @@ class LGBMMessage(Enum): VALID_LEAF_MASK = "PREDICT_VALID_LEAF_MASK" STOP_ITERATION = "STOP_ITERATION" PREDICT_PRABA = "PREDICT_PRABA" + MODEL_DATA = "MODEL_DATA" diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index 9bfab7b6..aedff0c4 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -6,6 +6,7 @@ from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.model_crypto.crypto_aes import encrypt_data, decrypt_data, cipher_to_base64, base64_to_cipher from ppc_model.interface.model_base import VerticalModel from ppc_model.datasets.dataset import SecureDataset from ppc_model.common.protocol import PheMessage @@ -241,6 +242,89 @@ def save_model(self, file_path=None): log.info( f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.") + self.merge_model_file() + + def merge_model_file(self): + + # 加密文件 + lgbm_model = {} + with open(self.ctx.feature_bin_file, 'rb') as f: + feature_bin_data = f.read() + with open(self.ctx.model_data_file, 'rb') as f: + model_data = f.read() + feature_bin_enc = encrypt_data(self.ctx.key, feature_bin_data) + model_data_enc = encrypt_data(self.ctx.key, model_data) + + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + lgbm_model[my_agency_id] = [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + + # 发送&接受文件 + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + self._send_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', + feature_bin_enc, partner_index) + self._send_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', + model_data_enc, partner_index) + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + feature_bin_enc = self._receive_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', partner_index) + model_data_enc = self._receive_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', partner_index) + lgbm_model[self.ctx.participant_id_list[partner_index]] = \ + [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + + # 上传密文模型 + with open(self.ctx.model_enc_file, 'w') as f: + json.dump(lgbm_model, f) + ResultFileHandling._upload_file(self.ctx.components.storage_client, + self.ctx.model_enc_file, self.ctx.remote_model_enc_file) + self.ctx.components.logger().info( + f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") + + def split_model_file(self): + # 下载密文模型 + try: + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.remote_model_enc_file, self.ctx.model_enc_file) + except: + pass + + # 发送/接受文件 + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + if os.path.exists(self.ctx.model_enc_file): + + with open(self.ctx.model_enc_file, 'r') as f: + lgbm_model = json.load(f) + + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + feature_bin_enc, model_data_enc = \ + [base64_to_cipher(i) for i in lgbm_model[self.ctx.participant_id_list[partner_index]]] + self._send_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', + feature_bin_enc, partner_index) + self._send_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', + model_data_enc, partner_index) + feature_bin_enc, model_data_enc = [base64_to_cipher(i) for i in lgbm_model[my_agency_id]] + + else: + feature_bin_enc = self._receive_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', 0) + model_data_enc = self._receive_byte_data( + self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', 0) + + # 解密文件 + feature_bin_data = decrypt_data(self.ctx.key, feature_bin_enc) + model_data = decrypt_data(self.ctx.key, model_data_enc) + with open(self.ctx.feature_bin_file, 'wb') as f: + f.write(feature_bin_data) + with open(self.ctx.model_data_file, 'wb') as f: + f.write(model_data) + def load_model(self, file_path=None): log = self.ctx.components.logger() if file_path is not None: @@ -254,10 +338,13 @@ def load_model(self, file_path=None): self.ctx.remote_model_data_file = os.path.join( self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) - ResultFileHandling._download_file(self.ctx.components.storage_client, - self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) - ResultFileHandling._download_file(self.ctx.components.storage_client, - self.ctx.model_data_file, self.ctx.remote_model_data_file) + try: + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + except: + self.split_model_file() with open(self.ctx.feature_bin_file, 'r') as f: X_split_dict = json.load(f) diff --git a/python/ppc_model/secure_lr/secure_lr_context.py b/python/ppc_model/secure_lr/secure_lr_context.py index 4ecebb9f..f80ad340 100644 --- a/python/ppc_model/secure_lr/secure_lr_context.py +++ b/python/ppc_model/secure_lr/secure_lr_context.py @@ -218,3 +218,4 @@ class LRMessage(Enum): TEST_LEAF_MASK = "PREDICT_TEST_LEAF_MASK" VALID_LEAF_MASK = "PREDICT_VALID_LEAF_MASK" PREDICT_PRABA = "PREDICT_PRABA" + MODEL_DATA = "MODEL_DATA" diff --git a/python/ppc_model/secure_lr/secure_lr_prediction_engine.py b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py new file mode 100644 index 00000000..218264b6 --- /dev/null +++ b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py @@ -0,0 +1,38 @@ +from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +from ppc_model.common.protocol import TaskRole, ModelTask +from ppc_model.common.global_context import components +from ppc_model.interface.task_engine import TaskEngine +from ppc_model.datasets.dataset import SecureDataset +from ppc_model.metrics.evaluation import Evaluation +from ppc_model.common.model_result import ResultFileHandling +from ppc_model.secure_lr.secure_lr_context import SecureLRContext +from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty + + +class SecureLGBMPredictionEngine(TaskEngine): + task_type = ModelTask.LR_PREDICTING + + @staticmethod + def run(args): + + task_info = SecureLRContext(args, components) + secure_dataset = SecureDataset(task_info) + + if task_info.role == TaskRole.ACTIVE_PARTY: + booster = VerticalLRActiveParty(task_info, secure_dataset) + elif task_info.role == TaskRole.PASSIVE_PARTY: + booster = VerticalLRPassiveParty(task_info, secure_dataset) + else: + raise PpcException(PpcErrorCode.ROLE_TYPE_ERROR.get_code(), + PpcErrorCode.ROLE_TYPE_ERROR.get_message()) + + booster.load_model() + booster.predict() + + # 获取测试集的预测概率值 + test_praba = booster.get_test_praba() + + # 获取测试集的预测值评估指标 + Evaluation(task_info, secure_dataset, test_praba=test_praba) + + ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lr/secure_lr_training_engine.py b/python/ppc_model/secure_lr/secure_lr_training_engine.py index c848fd76..66497f23 100644 --- a/python/ppc_model/secure_lr/secure_lr_training_engine.py +++ b/python/ppc_model/secure_lr/secure_lr_training_engine.py @@ -36,5 +36,4 @@ def run(args): # 获取训练集和验证集的预测值评估指标 Evaluation(task_info, secure_dataset, train_praba, test_praba) - ModelPlot(booster) ResultFileHandling(task_info) diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_training.py index 14a9bce5..d90e66a7 100644 --- a/python/ppc_model/secure_lr/test/test_secure_lr_training.py +++ b/python/ppc_model/secure_lr/test/test_secure_lr_training.py @@ -137,40 +137,40 @@ def test_fit(self): def active_worker(): try: booster_a.fit() - # booster_a.save_model() - # train_praba = booster_a.get_train_praba() - # test_praba = booster_a.get_test_praba() - # Evaluation(task_info_a, secure_dataset_a, - # train_praba, test_praba) - # ResultFileHandling(task_info_a) - # booster_a.load_model() - # booster_a.predict() - # test_praba = booster_a.get_test_praba() - # task_info_a.algorithm_type = 'Predict' - # task_info_a.sync_file_list = {} - # Evaluation(task_info_a, secure_dataset_a, - # test_praba=test_praba) - # ResultFileHandling(task_info_a) + booster_a.save_model() + train_praba = booster_a.get_train_praba() + test_praba = booster_a.get_test_praba() + Evaluation(task_info_a, secure_dataset_a, + train_praba, test_praba) + ResultFileHandling(task_info_a) + booster_a.load_model() + booster_a.predict() + test_praba = booster_a.get_test_praba() + task_info_a.algorithm_type = 'Predict' + task_info_a.sync_file_list = {} + Evaluation(task_info_a, secure_dataset_a, + test_praba=test_praba) + ResultFileHandling(task_info_a) except Exception as e: task_info_a.components.logger().info(traceback.format_exc()) def passive_worker(): try: booster_b.fit() - # booster_b.save_model() - # train_praba = booster_b.get_train_praba() - # test_praba = booster_b.get_test_praba() - # Evaluation(task_info_b, secure_dataset_b, - # train_praba, test_praba) - # ResultFileHandling(task_info_b) - # booster_b.load_model() - # booster_b.predict() - # test_praba = booster_b.get_test_praba() - # task_info_b.algorithm_type = 'Predict' - # task_info_b.sync_file_list = {} - # Evaluation(task_info_b, secure_dataset_b, - # test_praba=test_praba) - # ResultFileHandling(task_info_b) + booster_b.save_model() + train_praba = booster_b.get_train_praba() + test_praba = booster_b.get_test_praba() + Evaluation(task_info_b, secure_dataset_b, + train_praba, test_praba) + ResultFileHandling(task_info_b) + booster_b.load_model() + booster_b.predict() + test_praba = booster_b.get_test_praba() + task_info_b.algorithm_type = 'Predict' + task_info_b.sync_file_list = {} + Evaluation(task_info_b, secure_dataset_b, + test_praba=test_praba) + ResultFileHandling(task_info_b) except Exception as e: task_info_b.components.logger().info(traceback.format_exc()) diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py index 68a68d5f..24f233cc 100644 --- a/python/ppc_model/secure_lr/vertical/booster.py +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -7,6 +7,7 @@ from ppc_common.ppc_protos.generated.ppc_model_pb2 import BestSplitInfo from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.model_crypto.crypto_aes import encrypt_data, decrypt_data, cipher_to_base64, base64_to_cipher from ppc_model.interface.model_base import VerticalModel from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection from ppc_model.datasets.dataset import SecureDataset @@ -258,6 +259,71 @@ def save_model(self, file_path=None): log.info( f"task {self.ctx.task_id}: Saved serial_weight to {self.ctx.model_data_file} finished.") + self.merge_model_file() + + def merge_model_file(self): + + # 加密文件 + lr_model = {} + with open(self.ctx.model_data_file, 'rb') as f: + model_data = f.read() + model_data_enc = encrypt_data(self.ctx.key, model_data) + + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + lr_model[my_agency_id] = cipher_to_base64(model_data_enc) + + # 发送&接受文件 + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + self._send_byte_data( + self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', + model_data_enc, partner_index) + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + model_data_enc = self._receive_byte_data( + self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', partner_index) + lr_model[self.ctx.participant_id_list[partner_index]] = cipher_to_base64(model_data_enc) + + # 上传密文模型 + with open(self.ctx.model_enc_file, 'w') as f: + json.dump(lr_model, f) + ResultFileHandling._upload_file(self.ctx.components.storage_client, + self.ctx.model_enc_file, self.ctx.remote_model_enc_file) + self.ctx.components.logger().info( + f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") + + def split_model_file(self): + # 下载密文模型 + try: + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.remote_model_enc_file, self.ctx.model_enc_file) + except: + pass + + # 发送/接受文件 + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] + if os.path.exists(self.ctx.model_enc_file): + + with open(self.ctx.model_enc_file, 'r') as f: + lr_model = json.load(f) + + for partner_index in range(0, len(self.ctx.participant_id_list)): + if self.ctx.participant_id_list[partner_index] != my_agency_id: + model_data_enc = base64_to_cipher(lr_model[self.ctx.participant_id_list[partner_index]]) + self._send_byte_data( + self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', + model_data_enc, partner_index) + model_data_enc = base64_to_cipher(lr_model[my_agency_id]) + + else: + model_data_enc = self._receive_byte_data( + self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', 0) + + # 解密文件 + model_data = decrypt_data(self.ctx.key, model_data_enc) + with open(self.ctx.model_data_file, 'wb') as f: + f.write(model_data) + def load_model(self, file_path=None): log = self.ctx.components.logger() if file_path is not None: @@ -267,8 +333,11 @@ def load_model(self, file_path=None): self.ctx.remote_model_data_file = os.path.join( self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) - ResultFileHandling._download_file(self.ctx.components.storage_client, - self.ctx.model_data_file, self.ctx.remote_model_data_file) + try: + ResultFileHandling._download_file(self.ctx.components.storage_client, + self.ctx.model_data_file, self.ctx.remote_model_data_file) + except: + self.split_model_file() with open(self.ctx.model_data_file, 'r') as f: serial_weight = json.load(f) From d2f2e1ab7b50619df12ad0a5e561c0810fe0ed78 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 15 Oct 2024 17:54:48 +0800 Subject: [PATCH 055/120] refactor modelSetting (#51) * refactor modelSetting * use T_MODEL insteadOf t_proprecessing * fix task_result_handler --- python/ppc_dev/result/model_result.py | 12 +-- python/ppc_model/common/base_context.py | 4 +- python/ppc_model/common/model_setting.py | 82 ++++++++++++++----- python/ppc_model/conf/application-sample.yml | 4 +- python/ppc_model/conf/logging.conf | 4 +- .../model_result/task_result_handler.py | 73 ++++++++++------- python/ppc_model/ppc_model_app.py | 5 +- .../secure_lgbm/secure_lgbm_context.py | 3 + .../ppc_model/secure_lr/secure_lr_context.py | 36 +++++--- python/ppc_model/task/task_manager.py | 2 +- python/ppc_model/tools/start.sh | 5 +- python/ppc_model_gateway/conf/logging.conf | 4 +- .../ppc_model_gateway_app.py | 20 +++-- python/ppc_model_gateway/tools/start.sh | 3 - .../demo/default_flow_config_sample.py | 38 ++++----- python/ppc_scheduler/job/job_manager.py | 30 ++++--- python/ppc_scheduler/node/node_manager.py | 12 ++- .../workflow/common/worker_type.py | 6 +- .../workflow/worker/engine/model_engine.py | 22 ++--- .../workflow/worker/worker_factory.py | 11 +-- 20 files changed, 221 insertions(+), 155 deletions(-) diff --git a/python/ppc_dev/result/model_result.py b/python/ppc_dev/result/model_result.py index f1cc4e07..5930e45c 100644 --- a/python/ppc_dev/result/model_result.py +++ b/python/ppc_dev/result/model_result.py @@ -12,14 +12,14 @@ class ModelResult(BaseResult): FEATURE_BIN_FILE = "feature_bin.json" MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' - TEST_MODEL_OUTPUT_FILE = "xgb_output.csv" - TRAIN_MODEL_OUTPUT_FILE = "xgb_train_output.csv" + TEST_MODEL_OUTPUT_FILE = "test_output.csv" + TRAIN_MODEL_OUTPUT_FILE = "train_output.csv" def __init__(self, dataset: DataContext, job_id: str, job_type: str): super().__init__(dataset.ctx) self.job_id = job_id - + participant_id_list = [] for dataset in self.dataset.datasets: participant_id_list.append(dataset.agency.agency_id) @@ -32,8 +32,10 @@ def _xgb_train_result(self): # train_praba, test_praba, train_y, test_y, feature_importance, split_xbin, trees, params # 从hdfs读取结果文件信息，构造为属性 - train_praba_path = os.path.join(self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) - test_praba_path = os.path.join(self.job_id, self.TEST_MODEL_OUTPUT_FILE) + train_praba_path = os.path.join( + self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) + test_praba_path = os.path.join( + self.job_id, self.TEST_MODEL_OUTPUT_FILE) train_output = HDFSApi.download(train_praba_path) test_output = HDFSApi.download(test_praba_path) self.train_praba = train_output['class_pred'].values diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py index 9b16680b..bd19d2c7 100644 --- a/python/ppc_model/common/base_context.py +++ b/python/ppc_model/common/base_context.py @@ -17,10 +17,10 @@ class BaseContext: MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' TEST_MODEL_RESULT_FILE = "model_result.csv" # TEST_MODEL_OUTPUT_FILE = "model_output.csv" - TEST_MODEL_OUTPUT_FILE = "xgb_output.csv" + TEST_MODEL_OUTPUT_FILE = "test_model_output.csv" TRAIN_MODEL_RESULT_FILE = "train_model_result.csv" # TRAIN_MODEL_OUTPUT_FILE = "train_model_output.csv" - TRAIN_MODEL_OUTPUT_FILE = "xgb_train_output.csv" + TRAIN_MODEL_OUTPUT_FILE = "train_model_output.csv" MODEL_FILE = "model.kpl" MODEL_ENC_FILE = "model_enc.kpl" diff --git a/python/ppc_model/common/model_setting.py b/python/ppc_model/common/model_setting.py index 4d1aa98e..d2340878 100644 --- a/python/ppc_model/common/model_setting.py +++ b/python/ppc_model/common/model_setting.py @@ -2,7 +2,7 @@ from ppc_common.ppc_utils import common_func -class ModelSetting: +class PreprocessingSetting: def __init__(self, model_dict): self.use_psi = common_func.get_config_value( "use_psi", False, model_dict, False) @@ -28,26 +28,57 @@ def __init__(self, model_dict): "psi_select_bins", 4, model_dict, False)) self.corr_select = float(common_func.get_config_value( "corr_select", 0, model_dict, False)) + self.use_goss = common_func.get_config_value( + "use_goss", False, model_dict, False) + + +class FeatureEngineeringEngineSetting: + def __init__(self, model_dict): self.use_iv = common_func.get_config_value( "use_iv", False, model_dict, False) self.group_num = int(common_func.get_config_value( "group_num", 4, model_dict, False)) self.iv_thresh = float(common_func.get_config_value( "iv_thresh", 0.1, model_dict, False)) - self.use_goss = common_func.get_config_value( - "use_goss", False, model_dict, False) - self.test_size = float(common_func.get_config_value( - "test_dataset_percentage", 0.3, model_dict, False)) + + +class CommmonModelSetting: + def __init__(self, model_dict): self.learning_rate = float(common_func.get_config_value( "learning_rate", 0.1, model_dict, False)) + + self.eval_set_column = common_func.get_config_value( + "eval_set_column", "", model_dict, False) + self.train_set_value = common_func.get_config_value( + "train_set_value", "", model_dict, False) + self.eval_set_value = common_func.get_config_value( + "eval_set_value", "", model_dict, False) + self.verbose_eval = int(common_func.get_config_value( + "verbose_eval", 1, model_dict, False)) + self.silent = common_func.get_config_value( + "silent", False, model_dict, False) + self.train_features = common_func.get_config_value( + "train_features", "", model_dict, False) + random_state_str = common_func.get_config_value( + "random_state", "", model_dict, False) + if len(random_state_str) > 0: + self.random_state = int(random_state_str) + self.n_jobs = int(common_func.get_config_value( + "n_jobs", 0, model_dict, False)) + + +class SecureLGBMSetting(CommmonModelSetting): + def __init__(self, model_dict): + super().__init__(model_dict) + self.test_size = float(common_func.get_config_value( + "test_dataset_percentage", 0.3, model_dict, False)) self.num_trees = int(common_func.get_config_value( "num_trees", 6, model_dict, False)) self.max_depth = int(common_func.get_config_value( "max_depth", 3, model_dict, False)) self.max_bin = int(common_func.get_config_value( "max_bin", 4, model_dict, False)) - self.silent = common_func.get_config_value( - "silent", False, model_dict, False) + self.subsample = float(common_func.get_config_value( "subsample", 1, model_dict, False)) self.colsample_bytree = float(common_func.get_config_value( @@ -70,21 +101,30 @@ def __init__(self, model_dict): "early_stopping_rounds", 5, model_dict, False)) self.eval_metric = common_func.get_config_value( "eval_metric", "auc", model_dict, False) - self.verbose_eval = int(common_func.get_config_value( - "verbose_eval", 1, model_dict, False)) - self.eval_set_column = common_func.get_config_value( - "eval_set_column", "", model_dict, False) - self.train_set_value = common_func.get_config_value( - "train_set_value", "", model_dict, False) - self.eval_set_value = common_func.get_config_value( - "eval_set_value", "", model_dict, False) - self.train_features = common_func.get_config_value( - "train_features", "", model_dict, False) - self.epochs = int(common_func.get_config_value( - "epochs", 3, model_dict, False)) - self.batch_size = int(common_func.get_config_value( - "batch_size", 16, model_dict, False)) self.threads = int(common_func.get_config_value( "threads", 8, model_dict, False)) self.one_hot = common_func.get_config_value( "one_hot", 0, model_dict, False) + + +class SecureLRSetting(CommmonModelSetting): + def __init__(self, model_dict): + super().__init__(model_dict) + self.feature_rate = float(common_func.get_config_value( + "feature_rate", 1.0, model_dict, False)) + self.batch_size = int(common_func.get_config_value( + "batch_size", 16, model_dict, False)) + self.epochs = int(common_func.get_config_value( + "epochs", 3, model_dict, False)) + + +class ModelSetting(PreprocessingSetting, FeatureEngineeringEngineSetting, SecureLGBMSetting, SecureLRSetting): + def __init__(self, model_dict): + # init PreprocessingSetting + super().__init__(model_dict) + # init FeatureEngineeringEngineSetting + super(FeatureEngineeringEngineSetting, self).__init__(model_dict) + # init SecureLGBMSetting + super(SecureLGBMSetting, self).__init__(model_dict) + # init SecureLRSetting + super(SecureLRSetting, self).__init__(model_dict) diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml index b7458d70..330f1de7 100644 --- a/python/ppc_model/conf/application-sample.yml +++ b/python/ppc_model/conf/application-sample.yml @@ -16,8 +16,8 @@ SSL_CRT: "./ssl.crt" SSL_KEY: "./ssl.key" -PEM_PATH: "/data/app/ppcs-model4ef/wedpr-model-node/ppc_model_service/server.pem" -SHARE_PATH: "/data/app/ppcs-model4ef/wedpr-model-node/ppc_model_service/dataset_share/" +PEM_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/server.pem" +SHARE_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/dataset_share/" DB_TYPE: "mysql" SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4" diff --git a/python/ppc_model/conf/logging.conf b/python/ppc_model/conf/logging.conf index 31f30acb..b9b3bdb8 100644 --- a/python/ppc_model/conf/logging.conf +++ b/python/ppc_model/conf/logging.conf @@ -16,13 +16,13 @@ keys=fileHandler,consoleHandler,accessHandler [handler_accessHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/logs/ppcs-model4ef/appmonitor.log', 'D', 1, 30, 'utf-8') +args=('logs/appmonitor.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter [handler_fileHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/logs/ppcs-model4ef/ppcs-model4ef-node.log', 'D', 1, 30, 'utf-8') +args=('logs/wedpr-model.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py index 935446d1..8c5d87de 100644 --- a/python/ppc_model/model_result/task_result_handler.py +++ b/python/ppc_model/model_result/task_result_handler.py @@ -2,6 +2,7 @@ from ppc_common.ppc_utils.utils import PpcException, PpcErrorCode from ppc_common.ppc_utils import utils from ppc_model.common.protocol import ModelTask +from ppc_model.common.base_context import BaseContext from ppc_common.ppc_ml.model.algorithm_info import ClassificationType from ppc_model.common.model_result import ResultFileHandling from ppc_common.ppc_ml.model.algorithm_info import EvaluationType @@ -194,7 +195,7 @@ def to_dict(self): return self.result -class XGBJobResult: +class ModelJobResult: DEFAULT_PROPERTY_NAME = "outputModelResult" MODEL_RESULT = "ModelResult" MODEL_RESULT_PATH = "modelResultPath" @@ -202,8 +203,9 @@ class XGBJobResult: TEST_RESULT_PATH = "testResultPath" WOE_RESULT_PATH = "woeIVResultPath" - def __init__(self, job_id, components, property_name=DEFAULT_PROPERTY_NAME): + def __init__(self, xgb_job, job_id, components, property_name=DEFAULT_PROPERTY_NAME): self.job_id = job_id + self.xgb_job = xgb_job self.components = components self.logger = components.logger() self.property_name = property_name @@ -212,12 +214,14 @@ def __init__(self, job_id, components, property_name=DEFAULT_PROPERTY_NAME): self.model_result_path = None self.train_result_path = None self.woe_iv_result_path = None - self.xgb_result_path = None + self.model_result_path_dict = None self.evaluation_table = None self.feature_importance_table = None self.iteration_metrics = None - def fetch_model_result(self): + def fetch_xgb_model_result(self): + if not self.xgb_job: + return self.model_result_list = [] i = 0 # while True: @@ -240,25 +244,25 @@ def load_result(self, result_path, result_property): self.job_result = job_result_object.to_dict() def load_model_result_path(self, predict: bool): - self.xgb_result_path = dict() + self.model_result_path_dict = dict() self.model_result_path = ResultFileHandling.get_remote_path( self.components, self.job_id, BaseContext.MODEL_DATA_FILE) - self.xgb_result_path.update( - {XGBJobResult.MODEL_RESULT_PATH: self.model_result_path}) + self.model_result_path_dict.update( + {ModelJobResult.MODEL_RESULT_PATH: self.model_result_path}) self.train_result_path = ResultFileHandling.get_remote_path( self.components, self.job_id, BaseContext.TRAIN_MODEL_OUTPUT_FILE) - self.xgb_result_path.update( - {XGBJobResult.TRAIN_RESULT_PATH: self.train_result_path}) + self.model_result_path_dict.update( + {ModelJobResult.TRAIN_RESULT_PATH: self.train_result_path}) - self.xgb_result_path.update( - {XGBJobResult.TEST_RESULT_PATH: ResultFileHandling.get_remote_path( + self.model_result_path_dict.update( + {ModelJobResult.TEST_RESULT_PATH: ResultFileHandling.get_remote_path( self.components, self.job_id, BaseContext.TEST_MODEL_OUTPUT_FILE)}) self.woe_iv_result_path = ResultFileHandling.get_remote_path( self.components, self.job_id, BaseContext.WOE_IV_FILE) - self.xgb_result_path.update( - {XGBJobResult.WOE_RESULT_PATH: self.woe_iv_result_path}) + self.model_result_path_dict.update( + {ModelJobResult.WOE_RESULT_PATH: self.woe_iv_result_path}) def load_evaluation_table(self, evaluation_path, property): evaluation_table_object = TableResult(self.components, @@ -267,12 +271,16 @@ def load_evaluation_table(self, evaluation_path, property): type=DataType.TABLE).to_dict()} def load_feature_importance_table(self, feature_importance_path, property): + if not self.xgb_job: + return feature_importance_table = TableResult(self.components, self.job_id, ResultFileMeta(feature_importance_path)) self.feature_importance_table = {property: DataItem(name=property, data=feature_importance_table.to_dict(), type=DataType.TABLE).to_dict()} def load_iteration_metrics(self, iteration_path, property): + if not self.xgb_job: + return try: iteration_metrics_data = DataItem(data=ResultFileHandling.make_graph_data(self.components, self.job_id, utils.METRICS_OVER_ITERATION_FILE), name='iteration_metrics', name_property="ModelPlotName", data_property="ModelPlotData", @@ -295,9 +303,9 @@ def to_dict(self): result.update(self.feature_importance_table) if self.iteration_metrics is not None: result.update({self.iteration_property: self.iteration_metrics}) - if self.xgb_result_path is not None: + if self.model_result_path_dict is not None: result.update( - {XGBJobResult.MODEL_RESULT: self.xgb_result_path}) + {ModelJobResult.MODEL_RESULT: self.model_result_path_dict}) return result @@ -308,8 +316,11 @@ def __init__(self, task_result_request: TaskResultRequest, components): self.logger = components.logger() self.result_list = [] self.predict = False - if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name: + self.xgb_job = False + if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.LR_PREDICTING.name: self.predict = True + if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.XGB_TRAINING.name: + self.xgb_job = True self.logger.info( f"Init jobResultHandler for: {self.task_result_request.job_id}") self._get_evaluation_result() @@ -323,7 +334,7 @@ def get_response(self): return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response) def _get_evaluation_result(self): - if self.task_result_request.task_type == ModelTask.XGB_TRAINING.name: + if not self.predict: # the train evaluation result self.train_evaluation_result = JobEvaluationResult( property_name="outputMetricsGraphs", @@ -347,18 +358,18 @@ def _get_evaluation_result(self): "mpc_metric_ks.csv", "KSTable") self.result_list.append(self.validation_evaluation_result) - self.xgb_model = XGBJobResult( - self.task_result_request.job_id, self.components, XGBJobResult.DEFAULT_PROPERTY_NAME) - self.xgb_model.fetch_model_result() + self.model = ModelJobResult(self.xgb_job, + self.task_result_request.job_id, self.components, ModelJobResult.DEFAULT_PROPERTY_NAME) + self.model.fetch_xgb_model_result() # the ks-auc table - self.xgb_model.load_evaluation_table( + self.model.load_evaluation_table( utils.MPC_XGB_EVALUATION_TABLE, "EvaluationTable") # the feature-importance table - self.xgb_model.load_feature_importance_table( + self.model.load_feature_importance_table( utils.XGB_FEATURE_IMPORTANCE_TABLE, "FeatureImportance") - self.result_list.append(self.xgb_model) + self.result_list.append(self.model) # the metrics iteration graph - self.xgb_model.load_iteration_metrics( + self.model.load_iteration_metrics( utils.METRICS_OVER_ITERATION_FILE, "IterationGraph") if self.predict: @@ -374,13 +385,13 @@ def _get_evaluation_result(self): "mpc_eval_metric_ks.csv", "KSTable") self.result_list.append(self.predict_evaluation_result) - # load xgb_result - self.xgb_result = XGBJobResult( - self.task_result_request.job_id, self.components, XGBJobResult.DEFAULT_PROPERTY_NAME) - self.xgb_result.load_result( - "xgb_train_output.csv", "outputTrainPreview") - self.xgb_result.load_model_result_path(self.predict) - self.result_list.append(self.xgb_result) + # load model_result + self.model_result = ModelJobResult(self.xgb_job, + self.task_result_request.job_id, self.components, ModelJobResult.DEFAULT_PROPERTY_NAME) + self.model_result.load_result( + BaseContext.TRAIN_MODEL_OUTPUT_FILE, "outputTrainPreview") + self.model_result.load_model_result_path(self.predict) + self.result_list.append(self.model_result) def _get_feature_processing_result(self): self.feature_processing_result = FeatureProcessingResult( diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py index dc5865ea..63d77156 100644 --- a/python/ppc_model/ppc_model_app.py +++ b/python/ppc_model/ppc_model_app.py @@ -1,4 +1,7 @@ # Note: here can't be refactored by autopep +import sys +sys.path.append("../") + from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine @@ -21,8 +24,6 @@ from concurrent import futures import os import multiprocessing -import sys -sys.path.append("../") app = Flask(__name__) diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py index c366c8e8..4871c3a6 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -227,6 +227,9 @@ def __init__(self, if model_setting.train_features is not None and len(model_setting.train_features) > 0: self.model_params.train_feature = model_setting.train_features.split( ',') + if model_setting.categorical is not None and len(model_setting.categorical) > 0: + self.model_params.categorical_feature = model_setting.categorical.split( + ',') self.model_params.n_estimators = model_setting.num_trees self.model_params.feature_rate = model_setting.colsample_bytree self.model_params.min_split_gain = model_setting.gamma diff --git a/python/ppc_model/secure_lr/secure_lr_context.py b/python/ppc_model/secure_lr/secure_lr_context.py index f80ad340..7ecbe338 100644 --- a/python/ppc_model/secure_lr/secure_lr_context.py +++ b/python/ppc_model/secure_lr/secure_lr_context.py @@ -182,6 +182,9 @@ def __init__(self, if model_setting.train_features is not None and len(model_setting.train_features) > 0: self.model_params.train_feature = model_setting.train_features.split( ',') + if model_setting.categorical is not None and len(model_setting.categorical) > 0: + self.model_params.categorical_feature = model_setting.categorical.split( + ',') self.model_params.random_state = model_setting.seed self.sync_file_list = {} if self.algorithm_type == AlgorithmType.Train.name: @@ -196,17 +199,28 @@ def get_model_params(self): return self.model_params def set_sync_file(self): - self.sync_file_list['summary_evaluation'] = [self.summary_evaluation_file, self.remote_summary_evaluation_file] - self.sync_file_list['train_ks_table'] = [self.train_metric_ks_table, self.remote_train_metric_ks_table] - self.sync_file_list['train_metric_roc'] = [self.train_metric_roc_file, self.remote_train_metric_roc_file] - self.sync_file_list['train_metric_ks'] = [self.train_metric_ks_file, self.remote_train_metric_ks_file] - self.sync_file_list['train_metric_pr'] = [self.train_metric_pr_file, self.remote_train_metric_pr_file] - self.sync_file_list['train_metric_acc'] = [self.train_metric_acc_file, self.remote_train_metric_acc_file] - self.sync_file_list['test_ks_table'] = [self.test_metric_ks_table, self.remote_test_metric_ks_table] - self.sync_file_list['test_metric_roc'] = [self.test_metric_roc_file, self.remote_test_metric_roc_file] - self.sync_file_list['test_metric_ks'] = [self.test_metric_ks_file, self.remote_test_metric_ks_file] - self.sync_file_list['test_metric_pr'] = [self.test_metric_pr_file, self.remote_test_metric_pr_file] - self.sync_file_list['test_metric_acc'] = [self.test_metric_acc_file, self.remote_test_metric_acc_file] + self.sync_file_list['summary_evaluation'] = [ + self.summary_evaluation_file, self.remote_summary_evaluation_file] + self.sync_file_list['train_ks_table'] = [ + self.train_metric_ks_table, self.remote_train_metric_ks_table] + self.sync_file_list['train_metric_roc'] = [ + self.train_metric_roc_file, self.remote_train_metric_roc_file] + self.sync_file_list['train_metric_ks'] = [ + self.train_metric_ks_file, self.remote_train_metric_ks_file] + self.sync_file_list['train_metric_pr'] = [ + self.train_metric_pr_file, self.remote_train_metric_pr_file] + self.sync_file_list['train_metric_acc'] = [ + self.train_metric_acc_file, self.remote_train_metric_acc_file] + self.sync_file_list['test_ks_table'] = [ + self.test_metric_ks_table, self.remote_test_metric_ks_table] + self.sync_file_list['test_metric_roc'] = [ + self.test_metric_roc_file, self.remote_test_metric_roc_file] + self.sync_file_list['test_metric_ks'] = [ + self.test_metric_ks_file, self.remote_test_metric_ks_file] + self.sync_file_list['test_metric_pr'] = [ + self.test_metric_pr_file, self.remote_test_metric_pr_file] + self.sync_file_list['test_metric_acc'] = [ + self.test_metric_acc_file, self.remote_test_metric_acc_file] class LRMessage(Enum): diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py index d3ea8e33..49654e82 100644 --- a/python/ppc_model/task/task_manager.py +++ b/python/ppc_model/task/task_manager.py @@ -162,7 +162,7 @@ def record_model_job_log(self, job_id): log_file = self._get_log_file_path() if log_file is None or log_file == "": current_working_dir = os.getcwd() - relative_log_path = "logs/ppcs-model4ef-node.log" + relative_log_path = "logs/wedpr-model.log" log_file = os.path.join(current_working_dir, relative_log_path) start_keyword = LOG_START_FLAG_FORMATTER.format(job_id=job_id) diff --git a/python/ppc_model/tools/start.sh b/python/ppc_model/tools/start.sh index 3c1c818b..477ac215 100644 --- a/python/ppc_model/tools/start.sh +++ b/python/ppc_model/tools/start.sh @@ -2,7 +2,6 @@ dirpath="$(cd "$(dirname "$0")" && pwd)" cd $dirpath -LOG_DIR=/data/app/logs/ppcs-model4ef/ # kill crypto process crypto_pro_num=`ps -ef | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | wc -l` @@ -34,6 +33,4 @@ check_service() { } sleep 5 -check_service ppc_model_app.py -rm -rf logs -ln -s ${LOG_DIR} logs \ No newline at end of file +check_service ppc_model_app.py \ No newline at end of file diff --git a/python/ppc_model_gateway/conf/logging.conf b/python/ppc_model_gateway/conf/logging.conf index bb34233c..4a7b0aa9 100644 --- a/python/ppc_model_gateway/conf/logging.conf +++ b/python/ppc_model_gateway/conf/logging.conf @@ -16,13 +16,13 @@ keys=fileHandler,consoleHandler,accessHandler [handler_accessHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/logs/ppcs-modelgateway/appmonitor.log', 'D', 1, 30, 'utf-8') +args=('logs/appmonitor.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter [handler_fileHandler] class=handlers.TimedRotatingFileHandler -args=('/data/app/logs/ppcs-modelgateway/ppcs-modelgateway-gateway.log', 'D', 1, 30, 'utf-8') +args=('logs/ppcs-modelgateway-gateway.log', 'D', 1, 30, 'utf-8') level=INFO formatter=simpleFormatter diff --git a/python/ppc_model_gateway/ppc_model_gateway_app.py b/python/ppc_model_gateway/ppc_model_gateway_app.py index ef55c4e6..6d53286e 100644 --- a/python/ppc_model_gateway/ppc_model_gateway_app.py +++ b/python/ppc_model_gateway/ppc_model_gateway_app.py @@ -1,16 +1,18 @@ -from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService -from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService -from ppc_model_gateway import config -from ppc_common.ppc_utils import utils -from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc -import grpc -from threading import Thread -from concurrent import futures -import os # Note: here can't be refactored by autopep import sys sys.path.append("../") +import os +from concurrent import futures +from threading import Thread +import grpc +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_utils import utils +from ppc_model_gateway import config +from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService +from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService + + log = config.get_logger() diff --git a/python/ppc_model_gateway/tools/start.sh b/python/ppc_model_gateway/tools/start.sh index 2a3fb1d9..a25aa05e 100644 --- a/python/ppc_model_gateway/tools/start.sh +++ b/python/ppc_model_gateway/tools/start.sh @@ -2,7 +2,6 @@ dirpath="$(cd "$(dirname "$0")" && pwd)" cd $dirpath -LOG_DIR=/data/app/logs/ppcs-modelgateway/ export PYTHONPATH=$dirpath/../ source /data/app/ppcs-modelgateway/gateway_env/bin/deactivate @@ -31,5 +30,3 @@ check_service() { sleep 5 check_service ppc_model_gateway_app.py -rm -rf logs -ln -s ${LOG_DIR} logs diff --git a/python/ppc_scheduler/demo/default_flow_config_sample.py b/python/ppc_scheduler/demo/default_flow_config_sample.py index 4cf0b5de..123df9c2 100644 --- a/python/ppc_scheduler/demo/default_flow_config_sample.py +++ b/python/ppc_scheduler/demo/default_flow_config_sample.py @@ -1,5 +1,5 @@ flow_dict = { - + "SHELL": [ { "index": 1, @@ -24,7 +24,7 @@ ] } ], - + "PSI": [ { "index": 1, @@ -61,18 +61,18 @@ "PREPROCESSING": [ { "index": 1, - "type": "T_PREPROCESSING" + "type": "T_MODEL" } ], "FEATURE_ENGINEERING": [ { "index": 1, - "type": "T_PREPROCESSING" + "type": "T_MODEL" }, { "index": 2, - "type": "T_FEATURE_ENGINEERING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -84,11 +84,11 @@ "TRAINING": [ { "index": 1, - "type": "T_PREPROCESSING" + "type": "T_MODEL" }, { "index": 2, - "type": "T_TRAINING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -100,11 +100,11 @@ "PREDICTION": [ { "index": 1, - "type": "T_PREPROCESSING" + "type": "T_MODEL" }, { "index": 2, - "type": "T_PREDICTION", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -116,11 +116,11 @@ "FEATURE_ENGINEERING_TRAINING": [ { "index": 1, - "type": "T_PREPROCESSING" + "type": "T_MODEL" }, { "index": 2, - "type": "T_FEATURE_ENGINEERING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -129,7 +129,7 @@ }, { "index": 3, - "type": "T_TRAINING", + "type": "T_MODEL", "upstreams": [ { "index": 2 @@ -145,7 +145,7 @@ }, { "index": 2, - "type": "T_PREPROCESSING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -154,7 +154,7 @@ }, { "index": 3, - "type": "T_FEATURE_ENGINEERING", + "type": "T_MODEL", "upstreams": [ { "index": 2 @@ -170,7 +170,7 @@ }, { "index": 2, - "type": "T_PREPROCESSING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -179,7 +179,7 @@ }, { "index": 3, - "type": "T_TRAINING", + "type": "T_MODEL", "upstreams": [ { "index": 2 @@ -195,7 +195,7 @@ }, { "index": 2, - "type": "T_PREPROCESSING", + "type": "T_MODEL", "upstreams": [ { "index": 1 @@ -204,7 +204,7 @@ }, { "index": 3, - "type": "T_FEATURE_ENGINEERING", + "type": "T_MODEL", "upstreams": [ { "index": 2 @@ -213,7 +213,7 @@ }, { "index": 4, - "type": "T_TRAINING", + "type": "T_MODEL", "upstreams": [ { "index": 3 diff --git a/python/ppc_scheduler/job/job_manager.py b/python/ppc_scheduler/job/job_manager.py index 29cc5b38..05e85a00 100644 --- a/python/ppc_scheduler/job/job_manager.py +++ b/python/ppc_scheduler/job/job_manager.py @@ -44,25 +44,30 @@ def run_task(self, job_id, request_body): # TODO: The database persists job information with self._rw_lock.gen_wlock(): if job_id in self._jobs: - self.logger.info(f"Job already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}") + self.logger.info( + f"Job already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}") return - self._jobs[job_id] = [JobStatus.RUNNING, datetime.datetime.now(), 0] + self._jobs[job_id] = [ + JobStatus.RUNNING, datetime.datetime.now(), 0] self.logger.info(log_utils.job_start_log_info(job_id)) - + # Create job context - job_context = JobContext.create_job_context(request_body, self._workspace) + job_context = JobContext.create_job_context( + request_body, self._workspace) # Build job workflow - flow_context = self._flow_builder.build_flow_context(job_id=job_context.job_id, workflow_configs=job_context.workflow_configs) + flow_context = self._flow_builder.build_flow_context( + job_id=job_context.job_id, workflow_configs=job_context.workflow_configs) # Persistent workflow self._flow_builder.save_flow_context(job_context.job_id, flow_context) # Run workflow - self._async_executor.execute(job_id, self._run_job_flow, self._on_task_finish, (job_context, flow_context)) + self._async_executor.execute( + job_id, self._run_job_flow, self._on_task_finish, (job_context, flow_context)) def _run_job_flow(self, job_context, flow_context): """ run job flow """ - + # the scheduler module starts scheduling tasks self._scheduler.run(job_context, flow_context) @@ -98,17 +103,20 @@ def _on_task_finish(self, job_id: str, is_succeeded: bool, e: Exception = None): self._jobs[job_id][2] = time_costs if is_succeeded: self._jobs[job_id][0] = JobStatus.SUCCESS - self.logger.info(f"Job {job_id} completed, time_costs: {time_costs}s") + self.logger.info( + f"Job {job_id} completed, time_costs: {time_costs}s") else: self._jobs[job_id][0] = JobStatus.FAILURE - self.logger.warn(f"Job {job_id} failed, time_costs: {time_costs}s, error: {e}") + self.logger.warn( + f"Job {job_id} failed, time_costs: {time_costs}s, error: {e}") self.logger.info(log_utils.job_end_log_info(job_id)) def _loop_action(self): while True: time.sleep(20) self._terminate_timeout_jobs() - self._cleanup_finished_jobs() + # TODO: store into the database + # self._cleanup_finished_jobs() self._report_jobs() def _terminate_timeout_jobs(self): @@ -139,7 +147,7 @@ def _cleanup_finished_jobs(self): del self._jobs[job_id] self._thread_event_manager.remove_event(job_id) self.logger.info(f"Cleanup job cache, job_id: {job_id}") - + def _report_jobs(self): with self._rw_lock.gen_rlock(): job_count = len(self._jobs) diff --git a/python/ppc_scheduler/node/node_manager.py b/python/ppc_scheduler/node/node_manager.py index cadf3e29..c180714b 100644 --- a/python/ppc_scheduler/node/node_manager.py +++ b/python/ppc_scheduler/node/node_manager.py @@ -5,12 +5,8 @@ class ComputingNodeManager: type_map = { WorkerType.T_PSI: 'PSI', - WorkerType.T_ML_PSI: 'PSI', WorkerType.T_MPC: 'MPC', - WorkerType.T_PREPROCESSING: 'MODEL', - WorkerType.T_FEATURE_ENGINEERING: 'MODEL', - WorkerType.T_TRAINING: 'MODEL', - WorkerType.T_PREDICTION: 'MODEL', + WorkerType.T_MODEL: 'MODEL' } def __init__(self, components): @@ -18,11 +14,13 @@ def __init__(self, components): def add_node(self, node_id: str, url: str, worker_type: str): with self.components.create_sql_session() as session: - computing_node_mapper.insert_computing_node(session, node_id, url, self.type_map[worker_type], 0) + computing_node_mapper.insert_computing_node( + session, node_id, url, self.type_map[worker_type], 0) def remove_node(self, url: str, worker_type: str): with self.components.create_sql_session() as session: - computing_node_mapper.delete_computing_node(session, url, self.type_map[worker_type]) + computing_node_mapper.delete_computing_node( + session, url, self.type_map[worker_type]) def get_node(self, worker_type: str): with self.components.create_sql_session() as session: diff --git a/python/ppc_scheduler/workflow/common/worker_type.py b/python/ppc_scheduler/workflow/common/worker_type.py index 6f430978..1a7314f2 100644 --- a/python/ppc_scheduler/workflow/common/worker_type.py +++ b/python/ppc_scheduler/workflow/common/worker_type.py @@ -6,12 +6,8 @@ class WorkerType: # specific job worker T_PSI = 'PSI' - T_ML_PSI = 'ML_PSI' T_MPC = 'MPC' - T_PREPROCESSING = 'PREPROCESSING' - T_FEATURE_ENGINEERING = 'FEATURE_ENGINEERING' - T_TRAINING = 'XGB_TRAINING' - T_PREDICTION = 'XGB_PREDICTING' + T_MODEL = "MODEL" # finish job T_ON_SUCCESS = 'T_ON_SUCCESS' diff --git a/python/ppc_scheduler/workflow/worker/engine/model_engine.py b/python/ppc_scheduler/workflow/worker/engine/model_engine.py index 8f2b24ba..cd104310 100644 --- a/python/ppc_scheduler/workflow/worker/engine/model_engine.py +++ b/python/ppc_scheduler/workflow/worker/engine/model_engine.py @@ -4,6 +4,7 @@ from ppc_scheduler.workflow.common.worker_type import WorkerType from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine + class ModelWorkerEngine(WorkerEngine): def __init__(self, model_client, worker_type, worker_id, components, job_context: JobContext): self.model_client = model_client @@ -14,25 +15,24 @@ def __init__(self, model_client, worker_type, worker_id, components, job_context self.logger = self.components.logger() def run(self, *args) -> list: - if self.worker_type == WorkerType.T_PREPROCESSING\ - or self.worker_type == WorkerType.T_FEATURE_ENGINEERING\ - or self.worker_type == WorkerType.T_TRAINING\ - or self.worker_type == WorkerType.T_PREDICTION: + if self.worker_type == WorkerType.T_MODEL: pass else: raise ValueError(f"Unsupported worker type: {self.worker_type}") - + job_id = self.job_context.job_id start_time = time.time() - self.logger.info(f"## model engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}") - + self.logger.info( + f"## model engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}") + # send job request to model node and wait for the job to finish self.model_client.run(*args) - + time_costs = time.time() - start_time - self.logger.info(f"## model engine run finished, job_id={job_id}, timecost: {time_costs}s") - + self.logger.info( + f"## model engine run finished, job_id={job_id}, timecost: {time_costs}s") + # args = { # 'job_id': job_id, # 'task_id': task_id, @@ -48,5 +48,5 @@ def run(self, *args) -> list: # self.log.info( # f"call compute_xgb_job service success, job: {job_id}, " # f"task_id: {task_id}, timecost: {time.time() - start}") - + return [] diff --git a/python/ppc_scheduler/workflow/worker/worker_factory.py b/python/ppc_scheduler/workflow/worker/worker_factory.py index 9b15db5d..c8573185 100644 --- a/python/ppc_scheduler/workflow/worker/worker_factory.py +++ b/python/ppc_scheduler/workflow/worker/worker_factory.py @@ -20,18 +20,15 @@ def build_worker(job_context, worker_id, worker_type, worker_args, *args, **kwar return PythonWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs) elif worker_type == WorkerType.T_SHELL: return ShellWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) - elif worker_type == WorkerType.T_PSI or \ - worker_type == WorkerType.T_ML_PSI: + elif worker_type == WorkerType.T_PSI: return PsiWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_MPC: return MpcWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) - elif worker_type == WorkerType.T_PREPROCESSING or \ - worker_type == WorkerType.T_FEATURE_ENGINEERING or \ - worker_type == WorkerType.T_TRAINING or \ - worker_type == WorkerType.T_PREDICTION: + elif worker_type == WorkerType.T_MODEL: return ModelWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) elif worker_type == WorkerType.T_ON_SUCCESS or \ worker_type == WorkerType.T_ON_FAILURE: return DefaultWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs) else: - raise PpcException(PpcErrorCode.UNSUPPORTED_WORK_TYPE, f"Unsupported worker type: {worker_type}") + raise PpcException(PpcErrorCode.UNSUPPORTED_WORK_TYPE, + f"Unsupported worker type: {worker_type}") From 904b457a4db3a93e1fa1642c5d73aa40af41a924 Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:58:56 +0800 Subject: [PATCH 056/120] Feature milestone2 (#52) * update secure lr * update model and predict * update ppc_dev * update model setting * Update booster.py --- .../secure_lgbm/vertical/active_party.py | 4 ++ .../ppc_model/secure_lgbm/vertical/booster.py | 49 +++++++------------ .../secure_lgbm/vertical/passive_party.py | 5 ++ .../secure_lr/vertical/active_party.py | 4 ++ .../ppc_model/secure_lr/vertical/booster.py | 43 +++++++--------- .../secure_lr/vertical/passive_party.py | 5 ++ 6 files changed, 52 insertions(+), 58 deletions(-) diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py index ddec192a..fe4c2ff7 100644 --- a/python/ppc_model/secure_lgbm/vertical/active_party.py +++ b/python/ppc_model/secure_lgbm/vertical/active_party.py @@ -1,5 +1,6 @@ import itertools import time +import json import numpy as np from pandas import DataFrame @@ -143,6 +144,9 @@ def _init_active_data(self): [s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) self._all_feature_num += len([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + for i in range(1, len(self.ctx.participant_id_list)): + self._send_byte_data(self.ctx, LGBMMessage.FEATURE_NAME.value, + json.dumps(self._all_feature_name).encode('utf-8'), i) self.log.info(f'task {self.ctx.task_id}: total feature number:{self._all_feature_num}, ' f'total feature name: {self._all_feature_name}.') diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index aedff0c4..a8868dba 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -248,6 +248,17 @@ def merge_model_file(self): # 加密文件 lgbm_model = {} + lgbm_model['model_type'] = 'xgb_model' + lgbm_model['label_provider'] = self.ctx.participant_id_list[0] + lgbm_model['label_column'] = 'y' + lgbm_model['participant_agency_list'] = [] + for partner_index in range(0, len(self.ctx.participant_id_list)): + agency_info = {'agency': self.ctx.participant_id_list[partner_index]} + agency_info['fields'] = self.ctx._all_feature_name[partner_index] + lgbm_model['participant_agency_list'].append(agency_info) + + lgbm_model['model_dict'] = self.ctx.model_params + model_text = {} with open(self.ctx.feature_bin_file, 'rb') as f: feature_bin_data = f.read() with open(self.ctx.model_data_file, 'rb') as f: @@ -256,7 +267,7 @@ def merge_model_file(self): model_data_enc = encrypt_data(self.ctx.key, model_data) my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - lgbm_model[my_agency_id] = [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + model_text[my_agency_id] = [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] # 发送&接受文件 for partner_index in range(0, len(self.ctx.participant_id_list)): @@ -273,8 +284,9 @@ def merge_model_file(self): self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', partner_index) model_data_enc = self._receive_byte_data( self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', partner_index) - lgbm_model[self.ctx.participant_id_list[partner_index]] = \ + model_text[self.ctx.participant_id_list[partner_index]] = \ [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + lgbm_model['model_text'] = model_text # 上传密文模型 with open(self.ctx.model_enc_file, 'w') as f: @@ -285,38 +297,11 @@ def merge_model_file(self): f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") def split_model_file(self): - # 下载密文模型 - try: - ResultFileHandling._download_file(self.ctx.components.storage_client, - self.ctx.remote_model_enc_file, self.ctx.model_enc_file) - except: - pass - - # 发送/接受文件 + # 传入模型 my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - if os.path.exists(self.ctx.model_enc_file): - - with open(self.ctx.model_enc_file, 'r') as f: - lgbm_model = json.load(f) + model_text = self.ctx.model_predict_algorithm['model_text'] + feature_bin_enc, model_data_enc = [base64_to_cipher(i) for i in model_text[my_agency_id]] - for partner_index in range(0, len(self.ctx.participant_id_list)): - if self.ctx.participant_id_list[partner_index] != my_agency_id: - feature_bin_enc, model_data_enc = \ - [base64_to_cipher(i) for i in lgbm_model[self.ctx.participant_id_list[partner_index]]] - self._send_byte_data( - self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', - feature_bin_enc, partner_index) - self._send_byte_data( - self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', - model_data_enc, partner_index) - feature_bin_enc, model_data_enc = [base64_to_cipher(i) for i in lgbm_model[my_agency_id]] - - else: - feature_bin_enc = self._receive_byte_data( - self.ctx, f'{LGBMMessage.MODEL_DATA.value}_feature_bin', 0) - model_data_enc = self._receive_byte_data( - self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', 0) - # 解密文件 feature_bin_data = decrypt_data(self.ctx.key, feature_bin_enc) model_data = decrypt_data(self.ctx.key, model_data_enc) diff --git a/python/ppc_model/secure_lgbm/vertical/passive_party.py b/python/ppc_model/secure_lgbm/vertical/passive_party.py index 31e47826..325f241b 100644 --- a/python/ppc_model/secure_lgbm/vertical/passive_party.py +++ b/python/ppc_model/secure_lgbm/vertical/passive_party.py @@ -1,5 +1,6 @@ import multiprocessing import time +import json import numpy as np from pandas import DataFrame @@ -16,6 +17,7 @@ class VerticalLGBMPassiveParty(VerticalBooster): def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: super().__init__(ctx, dataset) self.params = ctx.model_params + self._all_feature_name = [] self.log = ctx.components.logger() self.log.info( f'task {self.ctx.task_id}: print all params: {self.params.get_all_params()}') @@ -94,6 +96,9 @@ def _init_passive_data(self): b''.join(s.encode('utf-8') + b' ' for s in self.dataset.feature_name), 0) self.params.my_categorical_idx = self._get_categorical_idx( self.dataset.feature_name, self.params.categorical_feature) + feature_name_bytes = self._receive_byte_data( + self.ctx, LGBMMessage.FEATURE_NAME.value, 0) + self._all_feature_name = json.loads(feature_name_bytes.decode('utf-8')) # 初始化分桶数据集 feat_bin = FeatureBinning(self.ctx) diff --git a/python/ppc_model/secure_lr/vertical/active_party.py b/python/ppc_model/secure_lr/vertical/active_party.py index 1d9e3a2f..cd7c731f 100644 --- a/python/ppc_model/secure_lr/vertical/active_party.py +++ b/python/ppc_model/secure_lr/vertical/active_party.py @@ -1,5 +1,6 @@ import itertools import time +import json import numpy as np from pandas import DataFrame @@ -115,6 +116,9 @@ def _init_active_data(self): [s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) self._all_feature_num += len([s.decode('utf-8') for s in feature_name_bytes.split(b' ') if s]) + for i in range(1, len(self.ctx.participant_id_list)): + self._send_byte_data(self.ctx, LRMessage.FEATURE_NAME.value, + json.dumps(self._all_feature_name).encode('utf-8'), i) self.log.info(f'task {self.ctx.task_id}: total feature number:{self._all_feature_num}, ' f'total feature name: {self._all_feature_name}.') diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py index 24f233cc..c80f1a5d 100644 --- a/python/ppc_model/secure_lr/vertical/booster.py +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -265,12 +265,23 @@ def merge_model_file(self): # 加密文件 lr_model = {} + lr_model['model_type'] = 'lr_model' + lr_model['label_provider'] = self.ctx.participant_id_list[0] + lr_model['label_column'] = 'y' + lr_model['participant_agency_list'] = [] + for partner_index in range(0, len(self.ctx.participant_id_list)): + agency_info = {'agency': self.ctx.participant_id_list[partner_index]} + agency_info['fields'] = self.ctx._all_feature_name[partner_index] + lr_model['participant_agency_list'].append(agency_info) + + lr_model['model_dict'] = self.ctx.model_params + model_text = {} with open(self.ctx.model_data_file, 'rb') as f: model_data = f.read() model_data_enc = encrypt_data(self.ctx.key, model_data) my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - lr_model[my_agency_id] = cipher_to_base64(model_data_enc) + model_text[my_agency_id] = cipher_to_base64(model_data_enc) # 发送&接受文件 for partner_index in range(0, len(self.ctx.participant_id_list)): @@ -282,7 +293,8 @@ def merge_model_file(self): if self.ctx.participant_id_list[partner_index] != my_agency_id: model_data_enc = self._receive_byte_data( self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', partner_index) - lr_model[self.ctx.participant_id_list[partner_index]] = cipher_to_base64(model_data_enc) + model_text[self.ctx.participant_id_list[partner_index]] = cipher_to_base64(model_data_enc) + lr_model['model_text'] = model_text # 上传密文模型 with open(self.ctx.model_enc_file, 'w') as f: @@ -293,32 +305,11 @@ def merge_model_file(self): f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") def split_model_file(self): - # 下载密文模型 - try: - ResultFileHandling._download_file(self.ctx.components.storage_client, - self.ctx.remote_model_enc_file, self.ctx.model_enc_file) - except: - pass - - # 发送/接受文件 + # 传入模型 my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - if os.path.exists(self.ctx.model_enc_file): - - with open(self.ctx.model_enc_file, 'r') as f: - lr_model = json.load(f) - - for partner_index in range(0, len(self.ctx.participant_id_list)): - if self.ctx.participant_id_list[partner_index] != my_agency_id: - model_data_enc = base64_to_cipher(lr_model[self.ctx.participant_id_list[partner_index]]) - self._send_byte_data( - self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', - model_data_enc, partner_index) - model_data_enc = base64_to_cipher(lr_model[my_agency_id]) + model_text = self.ctx.model_predict_algorithm['model_text'] + model_data_enc = base64_to_cipher(model_text[my_agency_id]) - else: - model_data_enc = self._receive_byte_data( - self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', 0) - # 解密文件 model_data = decrypt_data(self.ctx.key, model_data_enc) with open(self.ctx.model_data_file, 'wb') as f: diff --git a/python/ppc_model/secure_lr/vertical/passive_party.py b/python/ppc_model/secure_lr/vertical/passive_party.py index 46101559..372c3d6a 100644 --- a/python/ppc_model/secure_lr/vertical/passive_party.py +++ b/python/ppc_model/secure_lr/vertical/passive_party.py @@ -1,6 +1,7 @@ import itertools import multiprocessing import time +import json import numpy as np from pandas import DataFrame @@ -19,6 +20,7 @@ class VerticalLRPassiveParty(VerticalBooster): def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None: super().__init__(ctx, dataset) self.params = ctx.model_params + self._all_feature_name = [] self._loss_func = BinaryLoss() self.log = ctx.components.logger() self.log.info( @@ -86,6 +88,9 @@ def _init_passive_data(self): b''.join(s.encode('utf-8') + b' ' for s in self.dataset.feature_name), 0) self.params.my_categorical_idx = self._get_categorical_idx( self.dataset.feature_name, self.params.categorical_feature) + feature_name_bytes = self._receive_byte_data( + self.ctx, LRMessage.FEATURE_NAME.value, 0) + self._all_feature_name = json.loads(feature_name_bytes.decode('utf-8')) def _build_iter(self, feature_select, idx): From abf582f2f06b7e2b403de29f4d1b2776291253e0 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 16 Oct 2024 10:24:04 +0800 Subject: [PATCH 057/120] fix saveModel (#53) * add modelData response * fix saveModel --- python/ppc_model/interface/model_base.py | 4 ++++ .../model_result/task_result_handler.py | 16 +++++++++++++- .../ppc_model/secure_lgbm/vertical/booster.py | 21 +++++++++++-------- .../ppc_model/secure_lr/vertical/booster.py | 4 ++-- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/python/ppc_model/interface/model_base.py b/python/ppc_model/interface/model_base.py index 050980ae..0e96565f 100644 --- a/python/ppc_model/interface/model_base.py +++ b/python/ppc_model/interface/model_base.py @@ -31,3 +31,7 @@ def load_model(self, file_path): class VerticalModel(ModelBase): mode = "VERTICAL" + + def __init__(self, ctx): + super().__init__(ctx) + self._all_feature_name = [] diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py index 8c5d87de..02697ee2 100644 --- a/python/ppc_model/model_result/task_result_handler.py +++ b/python/ppc_model/model_result/task_result_handler.py @@ -206,6 +206,7 @@ class ModelJobResult: def __init__(self, xgb_job, job_id, components, property_name=DEFAULT_PROPERTY_NAME): self.job_id = job_id self.xgb_job = xgb_job + self.base_context = BaseContext(job_id, ".tmp") self.components = components self.logger = components.logger() self.property_name = property_name @@ -278,6 +279,12 @@ def load_feature_importance_table(self, feature_importance_path, property): self.feature_importance_table = {property: DataItem(name=property, data=feature_importance_table.to_dict(), type=DataType.TABLE).to_dict()} + def load_encrypted_model_data(self): + try: + return self.components.storage_client.get_data(self.base_context.remote_model_enc_file).decode("utf-8") + except: + pass + def load_iteration_metrics(self, iteration_path, property): if not self.xgb_job: return @@ -317,6 +324,7 @@ def __init__(self, task_result_request: TaskResultRequest, components): self.result_list = [] self.predict = False self.xgb_job = False + self.model_data = None if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.LR_PREDICTING.name: self.predict = True if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.XGB_TRAINING.name: @@ -330,7 +338,12 @@ def get_response(self): merged_result = dict() for result in self.result_list: merged_result.update(result.to_dict()) - response = {"jobPlanetResult": merged_result} + + if self.model_data is None: + response = {"jobPlanetResult": merged_result} + else: + response = {"jobPlanetResult": merged_result, + "modelData": self.model_data} return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response) def _get_evaluation_result(self): @@ -371,6 +384,7 @@ def _get_evaluation_result(self): # the metrics iteration graph self.model.load_iteration_metrics( utils.METRICS_OVER_ITERATION_FILE, "IterationGraph") + self.model_data = self.model.load_encrypted_model_data() if self.predict: # the train evaluation result diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index a8868dba..076f855c 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -33,7 +33,6 @@ def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: self._train_praba = None self._test_weights = None self._test_praba = None - random.seed(ctx.model_params.random_state) np.random.seed(ctx.model_params.random_state) @@ -253,11 +252,12 @@ def merge_model_file(self): lgbm_model['label_column'] = 'y' lgbm_model['participant_agency_list'] = [] for partner_index in range(0, len(self.ctx.participant_id_list)): - agency_info = {'agency': self.ctx.participant_id_list[partner_index]} - agency_info['fields'] = self.ctx._all_feature_name[partner_index] + agency_info = { + 'agency': self.ctx.participant_id_list[partner_index]} + agency_info['fields'] = self._all_feature_name[partner_index] lgbm_model['participant_agency_list'].append(agency_info) - - lgbm_model['model_dict'] = self.ctx.model_params + + lgbm_model['model_dict'] = self.ctx.model_params.get_all_params() model_text = {} with open(self.ctx.feature_bin_file, 'rb') as f: feature_bin_data = f.read() @@ -265,9 +265,10 @@ def merge_model_file(self): model_data = f.read() feature_bin_enc = encrypt_data(self.ctx.key, feature_bin_data) model_data_enc = encrypt_data(self.ctx.key, model_data) - + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - model_text[my_agency_id] = [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + model_text[my_agency_id] = [cipher_to_base64( + feature_bin_enc), cipher_to_base64(model_data_enc)] # 发送&接受文件 for partner_index in range(0, len(self.ctx.participant_id_list)): @@ -285,7 +286,8 @@ def merge_model_file(self): model_data_enc = self._receive_byte_data( self.ctx, f'{LGBMMessage.MODEL_DATA.value}_model_data', partner_index) model_text[self.ctx.participant_id_list[partner_index]] = \ - [cipher_to_base64(feature_bin_enc), cipher_to_base64(model_data_enc)] + [cipher_to_base64(feature_bin_enc), + cipher_to_base64(model_data_enc)] lgbm_model['model_text'] = model_text # 上传密文模型 @@ -300,7 +302,8 @@ def split_model_file(self): # 传入模型 my_agency_id = self.ctx.components.config_data['AGENCY_ID'] model_text = self.ctx.model_predict_algorithm['model_text'] - feature_bin_enc, model_data_enc = [base64_to_cipher(i) for i in model_text[my_agency_id]] + feature_bin_enc, model_data_enc = [ + base64_to_cipher(i) for i in model_text[my_agency_id]] # 解密文件 feature_bin_data = decrypt_data(self.ctx.key, feature_bin_enc) diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py index c80f1a5d..23623787 100644 --- a/python/ppc_model/secure_lr/vertical/booster.py +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -271,10 +271,10 @@ def merge_model_file(self): lr_model['participant_agency_list'] = [] for partner_index in range(0, len(self.ctx.participant_id_list)): agency_info = {'agency': self.ctx.participant_id_list[partner_index]} - agency_info['fields'] = self.ctx._all_feature_name[partner_index] + agency_info['fields'] = self._all_feature_name[partner_index] lr_model['participant_agency_list'].append(agency_info) - lr_model['model_dict'] = self.ctx.model_params + lr_model['model_dict'] = self.ctx.model_params.get_all_params() model_text = {} with open(self.ctx.model_data_file, 'rb') as f: model_data = f.read() From 456e83cd7c831dddb139bd89ac7a8b7a42bc990e Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:59:19 +0800 Subject: [PATCH 058/120] rename ppc_dev to wedpr-ml-toolkit (#55) * update secure lr * update model and predict * update ppc_dev * update model setting * Update booster.py * update wedpr_ml_toolkit --- python/ppc_dev/job_exceuter/hdfs_client.py | 53 ------------------- python/ppc_dev/wedpr_data/data_context.py | 35 ------------ .../{ppc_dev => wedpr_ml_toolkit}/__init__.py | 0 .../common/__init__.py | 0 .../common/base_context.py | 2 +- .../common/base_result.py | 2 +- .../job_exceuter/__init__.py | 0 .../job_exceuter/hdfs_client.py | 47 ++++++++++++++++ .../job_exceuter/pws_client.py | 37 ++++++++----- .../result/__init__.py | 0 .../result/fe_result.py | 4 +- .../result/model_result.py | 6 +-- .../result/psi_result.py | 4 +- .../test/__init__.py | 0 .../test/test_dev.py | 49 +++++++++-------- .../utils/__init__.py | 0 .../utils/agency.py | 0 .../utils/utils.py | 0 .../wedpr_data/__init__.py | 0 .../wedpr_data/data_context.py | 46 ++++++++++++++++ .../wedpr_data/wedpr_data.py | 10 ++-- .../wedpr_session/__init__.py | 0 .../wedpr_session/wedpr_session.py | 51 +++++++++++++----- 23 files changed, 197 insertions(+), 149 deletions(-) delete mode 100644 python/ppc_dev/job_exceuter/hdfs_client.py delete mode 100644 python/ppc_dev/wedpr_data/data_context.py rename python/{ppc_dev => wedpr_ml_toolkit}/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/common/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/common/base_context.py (81%) rename python/{ppc_dev => wedpr_ml_toolkit}/common/base_result.py (58%) rename python/{ppc_dev => wedpr_ml_toolkit}/job_exceuter/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py rename python/{ppc_dev => wedpr_ml_toolkit}/job_exceuter/pws_client.py (64%) rename python/{ppc_dev => wedpr_ml_toolkit}/result/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/result/fe_result.py (84%) rename python/{ppc_dev => wedpr_ml_toolkit}/result/model_result.py (91%) rename python/{ppc_dev => wedpr_ml_toolkit}/result/psi_result.py (84%) rename python/{ppc_dev => wedpr_ml_toolkit}/test/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/test/test_dev.py (52%) rename python/{ppc_dev => wedpr_ml_toolkit}/utils/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/utils/agency.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/utils/utils.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/wedpr_data/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/wedpr_data/data_context.py rename python/{ppc_dev => wedpr_ml_toolkit}/wedpr_data/wedpr_data.py (80%) rename python/{ppc_dev => wedpr_ml_toolkit}/wedpr_session/__init__.py (100%) rename python/{ppc_dev => wedpr_ml_toolkit}/wedpr_session/wedpr_session.py (68%) diff --git a/python/ppc_dev/job_exceuter/hdfs_client.py b/python/ppc_dev/job_exceuter/hdfs_client.py deleted file mode 100644 index ca77360a..00000000 --- a/python/ppc_dev/job_exceuter/hdfs_client.py +++ /dev/null @@ -1,53 +0,0 @@ -import requests -import pandas as pd -import io - - -class HDFSApi: - def __init__(self, base_url): - self.base_url = base_url - - def upload(self, dataframe, hdfs_path): - """ - 上传Pandas DataFrame到HDFS - :param dataframe: 要上传的Pandas DataFrame - :param hdfs_path: HDFS目标路径 - :return: 响应信息 - """ - # 将DataFrame转换为CSV格式 - csv_buffer = io.StringIO() - dataframe.to_csv(csv_buffer, index=False) - - # 发送PUT请求上传CSV数据 - response = requests.put( - f"{self.base_url}/upload?path={hdfs_path}", - data=csv_buffer.getvalue(), - headers={'Content-Type': 'text/csv'} - ) - return response.json() - - def download(self, hdfs_path): - """ - 从HDFS下载数据并返回为Pandas DataFrame - :param hdfs_path: HDFS文件路径 - :return: Pandas DataFrame - """ - response = requests.get(f"{self.base_url}/download?path={hdfs_path}") - if response.status_code == 200: - # 读取CSV数据并转换为DataFrame - dataframe = pd.read_csv(io.StringIO(response.text)) - return dataframe - else: - raise Exception(f"下载失败: {response.json()}") - - def download_data(self, hdfs_path): - """ - 从HDFS下载数据并返回为Pandas DataFrame - :param hdfs_path: HDFS文件路径 - :return: text - """ - response = requests.get(f"{self.base_url}/download?path={hdfs_path}") - if response.status_code == 200: - return response.text - else: - raise Exception(f"下载失败: {response.json()}") diff --git a/python/ppc_dev/wedpr_data/data_context.py b/python/ppc_dev/wedpr_data/data_context.py deleted file mode 100644 index 177a828f..00000000 --- a/python/ppc_dev/wedpr_data/data_context.py +++ /dev/null @@ -1,35 +0,0 @@ -import os - -from ppc_dev.utils import utils - - -class DataContext: - - def __init__(self, *datasets): - self.datasets = list(datasets) - self.ctx = self.datasets[0].ctx - - self._check_datasets() - - def _save_dataset(self, dataset): - if dataset.dataset_path is None: - dataset.dataset_id = utils.make_id(utils.IdPrefixEnum.DATASET.value) - dataset.dataset_path = os.path.join(dataset.ctx.workspace, dataset.dataset_id) - if self.storage_client is not None: - self.storage_client.upload(self.values, self.dataset_path) - - def _check_datasets(self): - for dataset in self.datasets: - self._save_dataset(dataset) - - def to_psi_format(self): - dataset_psi = [] - for dataset in self.datasets: - dataset_psi.append(dataset.dataset_path) - return dataset_psi - - def to_model_formort(self): - dataset_model = [] - for dataset in self.datasets: - dataset_model.append(dataset.dataset_path) - return dataset_model diff --git a/python/ppc_dev/__init__.py b/python/wedpr_ml_toolkit/__init__.py similarity index 100% rename from python/ppc_dev/__init__.py rename to python/wedpr_ml_toolkit/__init__.py diff --git a/python/ppc_dev/common/__init__.py b/python/wedpr_ml_toolkit/common/__init__.py similarity index 100% rename from python/ppc_dev/common/__init__.py rename to python/wedpr_ml_toolkit/common/__init__.py diff --git a/python/ppc_dev/common/base_context.py b/python/wedpr_ml_toolkit/common/base_context.py similarity index 81% rename from python/ppc_dev/common/base_context.py rename to python/wedpr_ml_toolkit/common/base_context.py index 1496f381..cd2cdb66 100644 --- a/python/ppc_dev/common/base_context.py +++ b/python/wedpr_ml_toolkit/common/base_context.py @@ -10,4 +10,4 @@ def __init__(self, project_id, user_name, pws_endpoint=None, hdfs_endpoint=None, self.pws_endpoint = pws_endpoint self.hdfs_endpoint = hdfs_endpoint self.token = token - self.workspace = os.path.join(self.project_id, self.user_name) + self.workspace = './milestone2' diff --git a/python/ppc_dev/common/base_result.py b/python/wedpr_ml_toolkit/common/base_result.py similarity index 58% rename from python/ppc_dev/common/base_result.py rename to python/wedpr_ml_toolkit/common/base_result.py index ace5f8e3..88bb3f8f 100644 --- a/python/ppc_dev/common/base_result.py +++ b/python/wedpr_ml_toolkit/common/base_result.py @@ -1,4 +1,4 @@ -from ppc_dev.common.base_context import BaseContext +from wedpr_ml_toolkit.common.base_context import BaseContext class BaseResult: diff --git a/python/ppc_dev/job_exceuter/__init__.py b/python/wedpr_ml_toolkit/job_exceuter/__init__.py similarity index 100% rename from python/ppc_dev/job_exceuter/__init__.py rename to python/wedpr_ml_toolkit/job_exceuter/__init__.py diff --git a/python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py b/python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py new file mode 100644 index 00000000..d8c7be68 --- /dev/null +++ b/python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py @@ -0,0 +1,47 @@ +import pandas as pd +import io + +from ppc_common.deps_services import storage_loader + + +class HDFSApi: + def __init__(self, hdfs_endpoint): + self.hdfs_endpoint = hdfs_endpoint + + config_data = {} + config_data['STORAGE_TYPE'] = 'HDFS' + config_data['HDFS_URL'] = self.hdfs_endpoint + config_data['HDFS_ENDPOINT'] = self.hdfs_endpoint + self.storage_client = storage_loader.load(config_data, logger=None) + + def upload(self, dataframe, hdfs_path): + """ + 上传Pandas DataFrame到HDFS + :param dataframe: 要上传的Pandas DataFrame + :param hdfs_path: HDFS目标路径 + :return: 响应信息 + """ + # 将DataFrame转换为CSV格式 + csv_buffer = io.StringIO() + dataframe.to_csv(csv_buffer, index=False) + self.storage_client.save_data(csv_buffer.getvalue(), hdfs_path) + return + + def download(self, hdfs_path): + """ + 从HDFS下载数据并返回为Pandas DataFrame + :param hdfs_path: HDFS文件路径 + :return: Pandas DataFrame + """ + content = self.storage_client.get_data(hdfs_path) + dataframe = pd.read_csv(io.BytesIO(content)) + return dataframe + + def download_byte(self, hdfs_path): + """ + 从HDFS下载数据 + :param hdfs_path: HDFS文件路径 + :return: text + """ + content = self.storage_client.get_data(hdfs_path) + return content diff --git a/python/ppc_dev/job_exceuter/pws_client.py b/python/wedpr_ml_toolkit/job_exceuter/pws_client.py similarity index 64% rename from python/ppc_dev/job_exceuter/pws_client.py rename to python/wedpr_ml_toolkit/job_exceuter/pws_client.py index 8404620a..40df0238 100644 --- a/python/ppc_dev/job_exceuter/pws_client.py +++ b/python/wedpr_ml_toolkit/job_exceuter/pws_client.py @@ -1,36 +1,47 @@ import random import time +import requests from ppc_common.ppc_utils import http_utils from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode +PWS_URL = '/api/wedpr/v3/project/submitJob' + + class PWSApi: def __init__(self, endpoint, token, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): - self.endpoint = endpoint + self.pws_url = endpoint + PWS_URL self.token = token self.polling_interval_s = polling_interval_s self.max_retries = max_retries self.retry_delay_s = retry_delay_s - self._async_run_task_method = 'asyncRunTask' - self._get_task_status_method = 'getTaskStatus' self._completed_status = 'COMPLETED' self._failed_status = 'FAILED' - def run(self, datasets, params): - params = { - 'jsonrpc': '1', - 'method': self._async_run_task_method, - 'token': self.token, - 'id': random.randint(1, 65535), - 'dataset': datasets, - 'params': params + def run(self, params): + + headers = { + "Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MzEzMTUwMTksInVzZXIiOiJ7XCJ1c2VybmFtZVwiOlwiZmx5aHVhbmcxXCIsXCJncm91cEluZm9zXCI6W3tcImdyb3VwSWRcIjpcIjEwMDAwMDAwMDAwMDAwMDBcIixcImdyb3VwTmFtZVwiOlwi5Yid5aeL55So5oi357uEXCIsXCJncm91cEFkbWluTmFtZVwiOlwiYWRtaW5cIn1dLFwicm9sZU5hbWVcIjpcIm9yaWdpbmFsX3VzZXJcIixcInBlcm1pc3Npb25zXCI6bnVsbCxcImFjY2Vzc0tleUlEXCI6bnVsbCxcImFkbWluXCI6ZmFsc2V9In0.1jZFOVbiISzCvvE9SOsTx0IWb0-OQc3o3rJgCu9GM9A", + "content-type": "application/json" } - response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) + + payload = { + "job": { + "jobType": params['jobType'], + "projectName": params['projectName'], + "param": params['param'] + }, + "taskParties": params['taskParties'], + "datasetList": params['datasetList'] + } + + response = requests.request("POST", self.pws_url, json=payload, headers=headers) if response.status_code != 200: raise Exception(f"创建任务失败: {response.json()}") - return self._poll_task_status(response.job_id, self.token) + return + # return self._poll_task_status(response.data, self.token) def _poll_task_status(self, job_id, token): while True: diff --git a/python/ppc_dev/result/__init__.py b/python/wedpr_ml_toolkit/result/__init__.py similarity index 100% rename from python/ppc_dev/result/__init__.py rename to python/wedpr_ml_toolkit/result/__init__.py diff --git a/python/ppc_dev/result/fe_result.py b/python/wedpr_ml_toolkit/result/fe_result.py similarity index 84% rename from python/ppc_dev/result/fe_result.py rename to python/wedpr_ml_toolkit/result/fe_result.py index 65c4dfc5..3caa0fea 100644 --- a/python/ppc_dev/result/fe_result.py +++ b/python/wedpr_ml_toolkit/result/fe_result.py @@ -1,7 +1,7 @@ import os -from ppc_dev.wedpr_data.data_context import DataContext -from ppc_dev.common.base_result import BaseResult +from wedpr_ml_toolkit.wedpr_data.data_context import DataContext +from wedpr_ml_toolkit.common.base_result import BaseResult class FeResult(BaseResult): diff --git a/python/ppc_dev/result/model_result.py b/python/wedpr_ml_toolkit/result/model_result.py similarity index 91% rename from python/ppc_dev/result/model_result.py rename to python/wedpr_ml_toolkit/result/model_result.py index 5930e45c..7609003f 100644 --- a/python/ppc_dev/result/model_result.py +++ b/python/wedpr_ml_toolkit/result/model_result.py @@ -3,9 +3,9 @@ from ppc_common.ppc_utils import utils -from ppc_dev.wedpr_data.data_context import DataContext -from ppc_dev.common.base_result import BaseResult -from ppc_dev.job_exceuter.hdfs_client import HDFSApi +from wedpr_ml_toolkit.wedpr_data.data_context import DataContext +from wedpr_ml_toolkit.common.base_result import BaseResult +from wedpr_ml_toolkit.job_exceuter.hdfs_client import HDFSApi class ModelResult(BaseResult): diff --git a/python/ppc_dev/result/psi_result.py b/python/wedpr_ml_toolkit/result/psi_result.py similarity index 84% rename from python/ppc_dev/result/psi_result.py rename to python/wedpr_ml_toolkit/result/psi_result.py index dae03f58..3b7da74c 100644 --- a/python/ppc_dev/result/psi_result.py +++ b/python/wedpr_ml_toolkit/result/psi_result.py @@ -1,7 +1,7 @@ import os -from ppc_dev.wedpr_data.data_context import DataContext -from ppc_dev.common.base_result import BaseResult +from wedpr_ml_toolkit.wedpr_data.data_context import DataContext +from wedpr_ml_toolkit.common.base_result import BaseResult class PSIResult(BaseResult): diff --git a/python/ppc_dev/test/__init__.py b/python/wedpr_ml_toolkit/test/__init__.py similarity index 100% rename from python/ppc_dev/test/__init__.py rename to python/wedpr_ml_toolkit/test/__init__.py diff --git a/python/ppc_dev/test/test_dev.py b/python/wedpr_ml_toolkit/test/test_dev.py similarity index 52% rename from python/ppc_dev/test/test_dev.py rename to python/wedpr_ml_toolkit/test/test_dev.py index 03bad07a..9ccc864f 100644 --- a/python/ppc_dev/test/test_dev.py +++ b/python/wedpr_ml_toolkit/test/test_dev.py @@ -3,26 +3,26 @@ import pandas as pd from sklearn import metrics -from ppc_dev.common.base_context import BaseContext -from ppc_dev.utils.agency import Agency -from ppc_dev.wedpr_data.wedpr_data import WedprData -from ppc_dev.wedpr_data.data_context import DataContext -from ppc_dev.wedpr_session.wedpr_session import WedprSession +from wedpr_ml_toolkit.common.base_context import BaseContext +from wedpr_ml_toolkit.utils.agency import Agency +from wedpr_ml_toolkit.wedpr_data.wedpr_data import WedprData +from wedpr_ml_toolkit.wedpr_data.data_context import DataContext +from wedpr_ml_toolkit.wedpr_session.wedpr_session import WedprSession # 从jupyter环境中获取project_id等信息 # create workspace # 相同项目/刷新专家模式project_id固定 project_id = 'p-123' -user = 'admin' -my_agency='WeBank' -pws_endpoint = '0.0.0.0:0000' -hdfs_endpoint = '0.0.0.0:0001' +user = 'flyhuang1' +my_agency='sgd' +pws_endpoint = 'http://139.159.202.235:8005' # http +hdfs_endpoint = 'http://192.168.0.18:50070' # client token = 'abc...' # 自定义合作方机构 -partner_agency1='SG' +partner_agency1='webank' partner_agency2='TX' # 初始化project ctx 信息 @@ -36,23 +36,28 @@ # pd.Dataframe df = pd.DataFrame({ 'id': np.arange(0, 100), # id列，顺序整数 + 'y': np.random.randint(0, 2, size=100), **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 }) -dataset1 = WedprData(ctx, values=df, agency=agency1) + +dataset1 = WedprData(ctx, values=df, agency=agency1, is_label_holder=True) dataset1.storage_client = None -dataset1.save_values(path='./project_id/user/data/d-101') +dataset1.save_values(path='d-101') # './milestone2\\sgd\\flyhuang1\\share\\d-101' + # hdfs_path -dataset2 = WedprData(ctx, dataset_path='./data_path/d-123', agency=agency2, is_label_holder=True) +dataset2 = WedprData(ctx, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) dataset2.storage_client = None -dataset2.load_values() - -# 支持更新dataset的values数据 -df2 = pd.DataFrame({ - 'id': np.arange(0, 100), # id列，顺序整数 - 'y': np.random.randint(0, 2, size=100), - **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 -}) -dataset2.update_values(values=df2) +# dataset2.load_values() +if dataset2.storage_client is None: + # 支持更新dataset的values数据 + df2 = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + **{f'z{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 + }) + dataset2.update_values(values=df2) +if dataset1.storage_client is not None: + dataset1.update_values(path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') + dataset1.load_values() # 构建 dataset context dataset = DataContext(dataset1, dataset2) diff --git a/python/ppc_dev/utils/__init__.py b/python/wedpr_ml_toolkit/utils/__init__.py similarity index 100% rename from python/ppc_dev/utils/__init__.py rename to python/wedpr_ml_toolkit/utils/__init__.py diff --git a/python/ppc_dev/utils/agency.py b/python/wedpr_ml_toolkit/utils/agency.py similarity index 100% rename from python/ppc_dev/utils/agency.py rename to python/wedpr_ml_toolkit/utils/agency.py diff --git a/python/ppc_dev/utils/utils.py b/python/wedpr_ml_toolkit/utils/utils.py similarity index 100% rename from python/ppc_dev/utils/utils.py rename to python/wedpr_ml_toolkit/utils/utils.py diff --git a/python/ppc_dev/wedpr_data/__init__.py b/python/wedpr_ml_toolkit/wedpr_data/__init__.py similarity index 100% rename from python/ppc_dev/wedpr_data/__init__.py rename to python/wedpr_ml_toolkit/wedpr_data/__init__.py diff --git a/python/wedpr_ml_toolkit/wedpr_data/data_context.py b/python/wedpr_ml_toolkit/wedpr_data/data_context.py new file mode 100644 index 00000000..cf9e7645 --- /dev/null +++ b/python/wedpr_ml_toolkit/wedpr_data/data_context.py @@ -0,0 +1,46 @@ +import os + +from wedpr_ml_toolkit.utils import utils + + +class DataContext: + + def __init__(self, *datasets): + self.datasets = list(datasets) + self.ctx = self.datasets[0].ctx + + self._check_datasets() + + def _save_dataset(self, dataset): + if dataset.dataset_path is None: + dataset.dataset_id = utils.make_id(utils.IdPrefixEnum.DATASET.value) + dataset.dataset_path = os.path.join(dataset.storage_workspace, dataset.dataset_id) + if dataset.storage_client is not None: + dataset.storage_client.upload(dataset.values, dataset.dataset_path) + + def _check_datasets(self): + for dataset in self.datasets: + self._save_dataset(dataset) + + def to_psi_format(self, merge_filed, result_receiver_id_list): + dataset_psi = [] + for dataset in self.datasets: + if dataset.agency.agency_id in result_receiver_id_list: + result_receiver = "true" + else: + result_receiver = "false" + dataset_psi_info = {"idFields": [merge_filed], + "dataset": {"owner": dataset.ctx.user_name, + "ownerAgency": dataset.agency.agency_id, + "path": dataset.dataset_path, + "storageTypeStr": "HDFS", + "datasetID": dataset.dataset_id}, + "receiveResult": result_receiver} + dataset_psi.append(dataset_psi_info) + return dataset_psi + + def to_model_formort(self): + dataset_model = [] + for dataset in self.datasets: + dataset_model.append(dataset.dataset_path) + return dataset_model diff --git a/python/ppc_dev/wedpr_data/wedpr_data.py b/python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py similarity index 80% rename from python/ppc_dev/wedpr_data/wedpr_data.py rename to python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py index 2e4c9575..eb1f487d 100644 --- a/python/ppc_dev/wedpr_data/wedpr_data.py +++ b/python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py @@ -1,7 +1,8 @@ +import os import pandas as pd -from ppc_dev.common.base_context import BaseContext -from ppc_dev.job_exceuter.hdfs_client import HDFSApi +from wedpr_ml_toolkit.common.base_context import BaseContext +from wedpr_ml_toolkit.job_exceuter.hdfs_client import HDFSApi class WedprData: @@ -26,6 +27,7 @@ def __init__(self, self.shape = None self.storage_client = HDFSApi(self.ctx.hdfs_endpoint) + self.storage_workspace = os.path.join(self.ctx.workspace, self.agency.agency_id, self.ctx.user_name, 'share') if self.values is not None: self.columns = self.values.columns @@ -42,6 +44,8 @@ def save_values(self, path=None): # 保存数据到hdfs目录 if path is not None: self.dataset_path = path + if not self.dataset_path.startswith(self.ctx.workspace): + self.dataset_path = os.path.join(self.storage_workspace, self.dataset_path) if self.storage_client is not None: self.storage_client.upload(self.values, self.dataset_path) @@ -53,7 +57,7 @@ def update_values(self, values: pd.DataFrame = None, path: str = None): self.shape = self.values.shape if path is not None: self.dataset_path = path - if self.storage_client is not None: + if values is not None and self.storage_client is not None: self.storage_client.upload(self.values, self.dataset_path) def update_path(self, path: str = None): diff --git a/python/ppc_dev/wedpr_session/__init__.py b/python/wedpr_ml_toolkit/wedpr_session/__init__.py similarity index 100% rename from python/ppc_dev/wedpr_session/__init__.py rename to python/wedpr_ml_toolkit/wedpr_session/__init__.py diff --git a/python/ppc_dev/wedpr_session/wedpr_session.py b/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py similarity index 68% rename from python/ppc_dev/wedpr_session/wedpr_session.py rename to python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py index e3c34e01..85b81d52 100644 --- a/python/ppc_dev/wedpr_session/wedpr_session.py +++ b/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py @@ -1,8 +1,10 @@ -from ppc_dev.wedpr_data.data_context import DataContext -from ppc_dev.job_exceuter.pws_client import PWSApi -from ppc_dev.result.psi_result import PSIResult -from ppc_dev.result.fe_result import FeResult -from ppc_dev.result.model_result import ModelResult +import json + +from wedpr_ml_toolkit.wedpr_data.data_context import DataContext +from wedpr_ml_toolkit.job_exceuter.pws_client import PWSApi +from wedpr_ml_toolkit.result.psi_result import PSIResult +from wedpr_ml_toolkit.result.fe_result import FeResult +from wedpr_ml_toolkit.result.model_result import ModelResult class WedprSession: @@ -11,18 +13,24 @@ def __init__(self, dataset: DataContext = None, my_agency = None): self.dataset = dataset self.create_agency = my_agency + self.participant_id_list = [] + self.task_parties = [] + self.dataset_id_list = [] + self.dataset_list = [] + self.label_holder_agency = None + self.label_columns = None if self.dataset is not None: - self.participant_id_list = self.get_agencies() - self.label_holder_agency = self.get_label_holder_agency() + self.get_agencies() + self.get_label_holder_agency() self.result_receiver_id_list = [my_agency] # 仅限jupyter所在机构 self.excute = PWSApi(self.dataset.ctx.pws_endpoint, self.dataset.ctx.token) - def task(self, datasets: list, params: dict = {}): + def task(self, params: dict = {}): self.check_agencies() - job_response = self.excute.run(datasets, params) + job_response = self.excute.run(params) return job_response.job_id @@ -31,11 +39,18 @@ def psi(self, dataset: DataContext = None, merge_filed: str = 'id'): if dataset is not None: self.update_dataset(dataset) - # 构造参数 - params = {merge_filed: merge_filed} + self.dataset_list = self.dataset.to_psi_format(merge_filed, self.result_receiver_id_list) + # 构造参数 + # params = {merge_filed: merge_filed} + params = {'jobType': 'PSI', + 'projectName': 'jupyter', + 'param': json.dumps({'dataSetList': self.dataset_list}).replace('"', '\\"'), + 'taskParties': self.task_parties, + 'datasetList': [None, None]} + # 执行任务 - job_id = self.task(self.dataset.to_psi_format(), params) + job_id = self.task(params) # 结果处理 psi_result = PSIResult(dataset, job_id) @@ -101,16 +116,24 @@ def update_dataset(self, dataset: DataContext): def get_agencies(self): participant_id_list = [] + dataset_id_list = [] for dataset in self.dataset.datasets: participant_id_list.append(dataset.agency.agency_id) - return participant_id_list + dataset_id_list.append(dataset.dataset_id) + self.task_parties.append({'userName': dataset.ctx.user_name, + 'agency': dataset.agency.agency_id}) + self.participant_id_list = participant_id_list + self.dataset_id_list = dataset_id_list def get_label_holder_agency(self): label_holder_agency = None + label_columns = None for dataset in self.dataset.datasets: if dataset.is_label_holder: label_holder_agency = dataset.agency.agency_id - return label_holder_agency + label_columns = 'y' + self.label_holder_agency = label_holder_agency + self.label_columns = label_columns def check_agencies(self): """ From f33a82a500c62c6f97bc96417dbb81a124e400b9 Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:35:09 +0800 Subject: [PATCH 059/120] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E9=A2=84=E6=B5=8B?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=9A=84=E7=89=B9=E5=BE=81=E7=AD=9B=E9=80=89?= =?UTF-8?q?=E6=B5=81=E7=A8=8B=20(#56)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update secure lr * update model and predict * update ppc_dev * update model setting * Update booster.py * update wedpr_ml_toolkit * update predict feature selection --- python/ppc_model/datasets/dataset.py | 9 ++++ .../local_processing_party.py | 3 -- .../local_processing/preprocessing.py | 47 ++++--------------- .../preprocessing/processing_context.py | 2 - .../ppc_model/secure_lgbm/vertical/booster.py | 5 -- .../ppc_model/secure_lr/vertical/booster.py | 3 -- 6 files changed, 17 insertions(+), 52 deletions(-) diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py index 1e3a33ff..ab3329a5 100644 --- a/python/ppc_model/datasets/dataset.py +++ b/python/ppc_model/datasets/dataset.py @@ -189,6 +189,15 @@ def _dataset_fe_selected(self, file_path, feature_name): self.model_data = self.model_data.drop(columns=drop_columns) def _construct_dataset(self): + if self.algorithm_type == AlgorithmType.Predict.name: + my_fields = [ + item["fields"] for item in self.ctx.model_predict_algorithm['participant_agency_list'] + if item["agency"] == self.ctx.components.config_data['AGENCY_ID']] + if 'y' in self.model_data.columns and 'y' not in my_fields: + my_fields = ['y'] + my_fields + if 'id' in self.model_data.columns and 'id' not in my_fields: + my_fields = ['id'] + my_fields + self.model_data = self.model_data[my_fields] if os.path.exists(self.iv_selected_file): self._dataset_fe_selected(self.iv_selected_file, 'feature') diff --git a/python/ppc_model/preprocessing/local_processing/local_processing_party.py b/python/ppc_model/preprocessing/local_processing/local_processing_party.py index 6cc13675..8a7aea0b 100644 --- a/python/ppc_model/preprocessing/local_processing/local_processing_party.py +++ b/python/ppc_model/preprocessing/local_processing/local_processing_party.py @@ -25,9 +25,6 @@ def processing(self): dataset_file_path = self.ctx.dataset_file_path storage_client = self.ctx.components.storage_client job_algorithm_type = self.ctx.job_algorithm_type - if job_algorithm_type == utils.AlgorithmType.Predict.name: - storage_client.download_file(os.path.join(self.ctx.training_job_id, self.ctx.PREPROCESSING_RESULT_FILE), - self.ctx.preprocessing_result_file) psi_result_path = self.ctx.psi_result_path model_prepare_file = self.ctx.model_prepare_file storage_client.download_file(dataset_path, dataset_file_path) diff --git a/python/ppc_model/preprocessing/local_processing/preprocessing.py b/python/ppc_model/preprocessing/local_processing/preprocessing.py index f0d39faf..d46f41de 100644 --- a/python/ppc_model/preprocessing/local_processing/preprocessing.py +++ b/python/ppc_model/preprocessing/local_processing/preprocessing.py @@ -79,24 +79,7 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb column_info = {} - if ppc_job_type == utils.AlgorithmType.Predict.name: - column_info_fm = pd.read_csv( - ctx.preprocessing_result_file, index_col=0) - column_info_train_str = json.dumps( - column_info_fm.to_dict(orient='index')) - if column_info_train_str is None: - raise PpcException(-1, "column_info_train is None") - try: - # 对应orient='records' - # column_info_train = json.loads(column_info_train_str, orient='records') - column_info_train = json.loads(column_info_train_str) - except Exception as e: - log.error( - f"jobid: {job_id} column_info_train json.loads error, e:{e}") - raise PpcException(-1, "column_info_train json.loads error") - dataset_df = process_train_dataframe(dataset_df, column_info_train) - column_info = column_info_train - elif ppc_job_type == utils.AlgorithmType.Train.name: + if ppc_job_type != utils.AlgorithmType.Predict.name: # 如果是训练任务 先默认所有数据都存在 column_info = {col: {'isExisted': True} for col in dataset_df.columns} @@ -141,7 +124,7 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb log.info(f"jobid: {job_id} move id column finish.") # 2.1 缺失值筛选 - if ppc_job_type == utils.AlgorithmType.Train.name: + if ppc_job_type != utils.AlgorithmType.Predict.name: if 0 <= model_setting.na_select <= 1: log.info(f"jobid: {job_id} run fillna start") df_filled, column_info = process_na_dataframe( @@ -152,13 +135,9 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb f"jobid: {job_id} xgb_model_dict['na_select'] is range not 0 to 1, xgb_model_dict:{model_setting}") raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), "xgb_model_dict['na_select'] range not 0 to 1") - elif ppc_job_type == utils.AlgorithmType.Predict.name: - log.info(f"jobid: {job_id} don't need run fillna for predict job.") else: - log.error( - f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") - raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), - "ppc_job_type is not Train or Predict") + log.info(f"jobid: {job_id} don't need run fillna for predict job.") + # 2.2 缺失值处理 if model_setting.fillna == 1: # 填充 @@ -184,7 +163,7 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb ), "xgb_model_dict['fillna'] is not 0 or 1") # 6.1 特征选择 进行 psi稳定性指标筛选 计算特征相关性 降维可以减少模型的复杂度，提高模型的泛化能力 - if ppc_job_type == utils.AlgorithmType.Train.name: + if ppc_job_type != utils.AlgorithmType.Predict.name: if model_setting.psi_select_col in df_filled.columns.tolist() and model_setting.psi_select_col != 0: log.info(f"jobid: {job_id} run psi_select_col start") psi_select_base = model_setting.psi_select_base @@ -215,17 +194,12 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb f"jobid: {job_id} xgb_model_dict['psi_select_col'] is not 0 or in col, model_setting:{model_setting}") raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), "xgb_model_dict['psi_select_col'] is not 0 or in col") - elif ppc_job_type == utils.AlgorithmType.Predict.name: + else: log.info( f"jobid: {job_id} don't need run psi_select_col for predict job.") - else: - log.error( - f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") - raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), - "ppc_job_type is not Train or Predict") # 6.2 特征选择 进行 corr_select 计算特征相关性 - if ppc_job_type == utils.AlgorithmType.Train.name: + if ppc_job_type != utils.AlgorithmType.Predict.name: if model_setting.corr_select > 0: log.info(f"jobid: {job_id} run corr_select start") corr_select = model_setting.corr_select @@ -247,14 +221,9 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb f"jobid: {job_id} xgb_model_dict['corr_select'] is not >= 0, model_setting:{model_setting}") raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), "xgb_model_dict['corr_select'] is not >= 0") - elif ppc_job_type == utils.AlgorithmType.Predict.name: + else: log.info( f"jobid: {job_id} don't need run corr_select for predict job.") - else: - log.error( - f"jobid: {job_id} ppc_job_type is not Train or Predict, ppc_job_type:{ppc_job_type}") - raise PpcException(PpcErrorCode.XGB_PREPROCESSING_ERROR.get_code(), - "ppc_job_type is not Train or Predict") # 3. 离群值处理 3-sigma 法 if model_setting.filloutlier == 1: diff --git a/python/ppc_model/preprocessing/processing_context.py b/python/ppc_model/preprocessing/processing_context.py index f46fd11c..5e597ffe 100644 --- a/python/ppc_model/preprocessing/processing_context.py +++ b/python/ppc_model/preprocessing/processing_context.py @@ -20,8 +20,6 @@ def __init__(self, self.job_algorithm_type = args['algorithm_type'] self.need_run_psi = args['need_run_psi'] self.model_dict = args['model_dict'] - self.training_job_id = common_func.get_config_value( - "training_job_id", None, args, False) if "psi_result_path" in args: self.remote_psi_result_path = args["psi_result_path"] self.model_setting = ModelSetting(self.model_dict) diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index 076f855c..4d0c184b 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -320,11 +320,6 @@ def load_model(self, file_path=None): file_path, self.ctx.FEATURE_BIN_FILE) self.ctx.model_data_file = os.path.join( file_path, self.ctx.MODEL_DATA_FILE) - if self.ctx.algorithm_type == AlgorithmType.Predict.name: - self.ctx.remote_feature_bin_file = os.path.join( - self.ctx.model_params.training_job_id, self.ctx.FEATURE_BIN_FILE) - self.ctx.remote_model_data_file = os.path.join( - self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) try: ResultFileHandling._download_file(self.ctx.components.storage_client, diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py index 23623787..9ab285b2 100644 --- a/python/ppc_model/secure_lr/vertical/booster.py +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -320,9 +320,6 @@ def load_model(self, file_path=None): if file_path is not None: self.ctx.model_data_file = os.path.join( file_path, self.ctx.MODEL_DATA_FILE) - if self.ctx.algorithm_type == AlgorithmType.Predict.name: - self.ctx.remote_model_data_file = os.path.join( - self.ctx.model_params.training_job_id, self.ctx.MODEL_DATA_FILE) try: ResultFileHandling._download_file(self.ctx.components.storage_client, From 00b0a6800159b97e278ca764c2cd5e638ccdfdf0 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Wed, 16 Oct 2024 20:29:58 +0800 Subject: [PATCH 060/120] refactor common part of lr-context and xgb-context into SecureModelContext (#57) --- python/ppc_common/ppc_utils/utils.py | 6 +- python/ppc_model/common/context.py | 1 - python/ppc_model/datasets/dataset.py | 7 +- .../model_result/task_result_handler.py | 14 +- python/ppc_model/ppc_model_app.py | 45 ++--- .../secure_lgbm/secure_lgbm_context.py | 169 ++++-------------- .../ppc_model/secure_lgbm/vertical/booster.py | 53 ++---- .../ppc_model/secure_lr/secure_lr_context.py | 127 +------------ .../secure_lr/secure_lr_prediction_engine.py | 2 +- .../ppc_model/secure_lr/vertical/booster.py | 100 +++++------ .../ppc_model/secure_model_base/__init__.py | 0 .../secure_model_base/secure_model_booster.py | 40 +++++ .../secure_model_base/secure_model_context.py | 159 ++++++++++++++++ 13 files changed, 344 insertions(+), 379 deletions(-) create mode 100644 python/ppc_model/secure_model_base/__init__.py create mode 100644 python/ppc_model/secure_model_base/secure_model_booster.py create mode 100644 python/ppc_model/secure_model_base/secure_model_context.py diff --git a/python/ppc_common/ppc_utils/utils.py b/python/ppc_common/ppc_utils/utils.py index 51605969..6450a4a4 100644 --- a/python/ppc_common/ppc_utils/utils.py +++ b/python/ppc_common/ppc_utils/utils.py @@ -67,11 +67,6 @@ MPC_TRAIN_SET_METRIC_PR_FILE = "mpc_train_metric_pr.svg" MPC_TRAIN_SET_METRIC_ACCURACY_FILE = "mpc_train_metric_accuracy.svg" MPC_TRAIN_SET_METRIC_KS_TABLE = "mpc_train_metric_ks.csv" -MPC_EVAL_METRIC_ROC_FILE = "mpc_eval_metric_roc.svg" -MPC_EVAL_METRIC_KS_FILE = "mpc_eval_metric_ks.svg" -MPC_EVAL_METRIC_PR_FILE = "mpc_eval_metric_pr.svg" -MPC_EVAL_METRIC_ACCURACY_FILE = "mpc_eval_metric_accuracy.svg" -MPC_EVAL_METRIC_KS_TABLE = "mpc_eval_metric_ks.csv" MPC_TRAIN_METRIC_CONFUSION_MATRIX_FILE = "mpc_metric_confusion_matrix.svg" METRICS_OVER_ITERATION_FILE = "metrics_over_iterations.svg" @@ -111,6 +106,7 @@ class CryptoType(Enum): ECDSA = 1 GM = 2 + @unique class HashType(Enum): BYTES = 1 diff --git a/python/ppc_model/common/context.py b/python/ppc_model/common/context.py index 02614e47..551b9f83 100644 --- a/python/ppc_model/common/context.py +++ b/python/ppc_model/common/context.py @@ -1,4 +1,3 @@ -import unittest from ppc_model.common.base_context import BaseContext from ppc_model.common.initializer import Initializer diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py index ab3329a5..451ee44a 100644 --- a/python/ppc_model/datasets/dataset.py +++ b/python/ppc_model/datasets/dataset.py @@ -190,9 +190,10 @@ def _dataset_fe_selected(self, file_path, feature_name): def _construct_dataset(self): if self.algorithm_type == AlgorithmType.Predict.name: - my_fields = [ - item["fields"] for item in self.ctx.model_predict_algorithm['participant_agency_list'] - if item["agency"] == self.ctx.components.config_data['AGENCY_ID']] + my_fields = [] + for item in self.ctx.model_predict_algorithm['participant_agency_list']: + if item["agency"] == self.ctx.components.config_data['AGENCY_ID']: + my_fields = item["fields"] if 'y' in self.model_data.columns and 'y' not in my_fields: my_fields = ['y'] + my_fields if 'id' in self.model_data.columns and 'id' not in my_fields: diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py index 02697ee2..d5ef6ca6 100644 --- a/python/ppc_model/model_result/task_result_handler.py +++ b/python/ppc_model/model_result/task_result_handler.py @@ -65,10 +65,10 @@ class JobEvaluationResult: EvaluationType.ACCURACY: utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE} DEFAULT_EVAL_EVALUATION_FILES = { - EvaluationType.ROC: utils.MPC_EVAL_METRIC_ROC_FILE, - EvaluationType.PR: utils.MPC_EVAL_METRIC_PR_FILE, - EvaluationType.KS: utils.MPC_EVAL_METRIC_KS_FILE, - EvaluationType.ACCURACY: utils.MPC_EVAL_METRIC_ACCURACY_FILE + EvaluationType.ROC: utils.MPC_TRAIN_METRIC_ROC_FILE, + EvaluationType.PR: utils.MPC_TRAIN_METRIC_PR_FILE, + EvaluationType.KS: utils.MPC_TRAIN_METRIC_KS_FILE, + EvaluationType.ACCURACY: utils.MPC_TRAIN_METRIC_ACCURACY_FILE } def __init__(self, property_name, classification_type, @@ -357,7 +357,7 @@ def _get_evaluation_result(self): components=self.components) # load the ks table self.train_evaluation_result.load_ks_table( - "mpc_train_metric_ks.csv", "TrainKSTable") + utils.MPC_TRAIN_METRIC_KS_TABLE, "TrainKSTable") self.result_list.append(self.train_evaluation_result) self.validation_evaluation_result = JobEvaluationResult( @@ -368,7 +368,7 @@ def _get_evaluation_result(self): components=self.components) # load the ks_table self.validation_evaluation_result.load_ks_table( - "mpc_metric_ks.csv", "KSTable") + utils.MPC_TRAIN_METRIC_KS_TABLE, "KSTable") self.result_list.append(self.validation_evaluation_result) self.model = ModelJobResult(self.xgb_job, @@ -396,7 +396,7 @@ def _get_evaluation_result(self): components=self.components) # load ks_table self.predict_evaluation_result.load_ks_table( - "mpc_eval_metric_ks.csv", "KSTable") + utils.MPC_TRAIN_METRIC_KS_TABLE, "KSTable") self.result_list.append(self.predict_evaluation_result) # load model_result diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py index 63d77156..6c40bbc3 100644 --- a/python/ppc_model/ppc_model_app.py +++ b/python/ppc_model/ppc_model_app.py @@ -2,28 +2,29 @@ import sys sys.path.append("../") -from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine -from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine -from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine -from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine -from ppc_model.network.http.restx import api -from ppc_model.network.http.model_controller import ns2 as log_namespace -from ppc_model.network.http.model_controller import ns as task_namespace -from ppc_model.network.grpc.grpc_server import ModelService -from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine -from ppc_model.common.protocol import ModelTask -from ppc_model.common.global_context import components -from ppc_common.ppc_utils import utils -from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc -from paste.translogger import TransLogger -from flask import Flask, Blueprint -from cheroot.wsgi import Server as WSGIServer -from cheroot.ssl.builtin import BuiltinSSLAdapter -import grpc -from threading import Thread -from concurrent import futures -import os import multiprocessing +import os +from concurrent import futures +from threading import Thread +import grpc +from cheroot.ssl.builtin import BuiltinSSLAdapter +from cheroot.wsgi import Server as WSGIServer +from flask import Flask, Blueprint +from paste.translogger import TransLogger +from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc +from ppc_common.ppc_utils import utils +from ppc_model.common.global_context import components +from ppc_model.common.protocol import ModelTask +from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine +from ppc_model.network.grpc.grpc_server import ModelService +from ppc_model.network.http.model_controller import ns as task_namespace +from ppc_model.network.http.model_controller import ns2 as log_namespace +from ppc_model.network.http.restx import api +from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine +from ppc_model.secure_lr.secure_lr_prediction_engine import SecureLRPredictionEngine +from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine +from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine +from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine app = Flask(__name__) @@ -53,6 +54,8 @@ def register_task_handler(): ModelTask.XGB_PREDICTING, SecureLGBMPredictionEngine.run) task_manager.register_task_handler( ModelTask.LR_TRAINING, SecureLRTrainingEngine.run) + task_manager.register_task_handler( + ModelTask.LR_PREDICTING, SecureLRPredictionEngine.run) def model_serve(): diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py index 4871c3a6..804c5b97 100644 --- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py +++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py @@ -1,18 +1,19 @@ + import os from enum import Enum -from typing import Any, Dict from sklearn.base import BaseEstimator from ppc_common.ppc_utils.utils import AlgorithmType from ppc_common.ppc_crypto.phe_factory import PheCipherFactory -from ppc_model.common.context import Context from ppc_model.common.initializer import Initializer from ppc_model.common.protocol import TaskRole from ppc_common.ppc_utils import common_func from ppc_model.common.model_setting import ModelSetting +from ppc_model.secure_model_base.secure_model_context import SecureModel +from ppc_model.secure_model_base.secure_model_context import SecureModelContext -class LGBMModel(BaseEstimator): +class LGBMModel(SecureModel): def __init__( self, @@ -36,7 +37,6 @@ def __init__( importance_type: str = 'split', **kwargs ): - self.boosting_type = boosting_type self.objective = objective self.num_leaves = num_leaves @@ -55,60 +55,7 @@ def __init__( self.random_state = random_state self.n_jobs = n_jobs self.importance_type = importance_type - self._other_params: Dict[str, Any] = {} - self.set_params(**kwargs) - - def get_params(self, deep: bool = True) -> Dict[str, Any]: - """Get parameters for this estimator. - - Parameters - ---------- - deep : bool, optional (default=True) - If True, will return the parameters for this estimator and - contained subobjects that are estimators. - - Returns - ------- - params : dict - Parameter names mapped to their values. - """ - params = super().get_params(deep=deep) - params.update(self._other_params) - return params - - def set_model_setting(self, model_setting: ModelSetting) -> "LGBMModel": - # 获取对象的所有属性名 - attrs = dir(model_setting) - # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） - attrs = [attr for attr in attrs if not attr.startswith('_')] - - params = {} - for attr in attrs: - try: - setattr(self, attr, getattr(model_setting, attr)) - except Exception as e: - pass - return self - - def set_params(self, **params: Any) -> "LGBMModel": - """Set the parameters of this estimator. - - Parameters - ---------- - **params - Parameter names with their new values. - - Returns - ------- - self : object - Returns self. - """ - for key, value in params.items(): - setattr(self, key, value) - if hasattr(self, f"_{key}"): - setattr(self, f"_{key}", value) - self._other_params[key] = value - return self + super().__init__(**kwargs) class ModelTaskParams(LGBMModel): @@ -169,98 +116,54 @@ def _get_params(self): """返回LGBMClassifier所有参数""" return LGBMModel().get_params() - def get_all_params(self): - """返回SecureLGBMParams所有参数""" - # 获取对象的所有属性名 - attrs = dir(self) - # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） - attrs = [attr for attr in attrs if not attr.startswith('_')] - - params = {} - for attr in attrs: - try: - # 使用getattr来获取属性的值 - value = getattr(self, attr) - # 检查value是否可调用（例如，方法或函数），如果是，则不打印其值 - if not callable(value): - params[attr] = value - except Exception as e: - pass - return params - -class SecureLGBMContext(Context): +class SecureLGBMContext(SecureModelContext): def __init__(self, args, components: Initializer ): - - if args['is_label_holder']: - role = TaskRole.ACTIVE_PARTY - else: - role = TaskRole.PASSIVE_PARTY - - super().__init__(args['job_id'], - args['task_id'], - components, - role) + super().__init__(args, components) self.phe = PheCipherFactory.build_phe( components.homo_algorithm, components.public_key_length) self.codec = PheCipherFactory.build_codec(components.homo_algorithm) - self.is_label_holder = args['is_label_holder'] - self.result_receiver_id_list = args['result_receiver_id_list'] - self.participant_id_list = args['participant_id_list'] - self.model_predict_algorithm = common_func.get_config_value( - "model_predict_algorithm", None, args, False) - self.algorithm_type = args['algorithm_type'] - if 'dataset_id' in args and args['dataset_id'] is not None: - self.dataset_file_path = os.path.join( - self.workspace, args['dataset_id']) - else: - self.dataset_file_path = None - - self.model_params = SecureLGBMParams() - model_setting = ModelSetting(args['model_dict']) - self.set_model_params(model_setting) - if model_setting.train_features is not None and len(model_setting.train_features) > 0: - self.model_params.train_feature = model_setting.train_features.split( - ',') - if model_setting.categorical is not None and len(model_setting.categorical) > 0: - self.model_params.categorical_feature = model_setting.categorical.split( - ',') - self.model_params.n_estimators = model_setting.num_trees - self.model_params.feature_rate = model_setting.colsample_bytree - self.model_params.min_split_gain = model_setting.gamma - self.model_params.random_state = model_setting.seed - self.sync_file_list = {} - if self.algorithm_type == AlgorithmType.Train.name: - self.set_sync_file() - - def set_model_params(self, model_setting: ModelSetting): - """设置lgbm参数""" - self.model_params.set_model_setting(model_setting) + def create_model_param(self): + return SecureLGBMParams() def get_model_params(self): """获取lgbm参数""" return self.model_params def set_sync_file(self): - self.sync_file_list['metrics_iteration'] = [self.metrics_iteration_file, self.remote_metrics_iteration_file] - self.sync_file_list['feature_importance'] = [self.feature_importance_file, self.remote_feature_importance_file] - self.sync_file_list['summary_evaluation'] = [self.summary_evaluation_file, self.remote_summary_evaluation_file] - self.sync_file_list['train_ks_table'] = [self.train_metric_ks_table, self.remote_train_metric_ks_table] - self.sync_file_list['train_metric_roc'] = [self.train_metric_roc_file, self.remote_train_metric_roc_file] - self.sync_file_list['train_metric_ks'] = [self.train_metric_ks_file, self.remote_train_metric_ks_file] - self.sync_file_list['train_metric_pr'] = [self.train_metric_pr_file, self.remote_train_metric_pr_file] - self.sync_file_list['train_metric_acc'] = [self.train_metric_acc_file, self.remote_train_metric_acc_file] - self.sync_file_list['test_ks_table'] = [self.test_metric_ks_table, self.remote_test_metric_ks_table] - self.sync_file_list['test_metric_roc'] = [self.test_metric_roc_file, self.remote_test_metric_roc_file] - self.sync_file_list['test_metric_ks'] = [self.test_metric_ks_file, self.remote_test_metric_ks_file] - self.sync_file_list['test_metric_pr'] = [self.test_metric_pr_file, self.remote_test_metric_pr_file] - self.sync_file_list['test_metric_acc'] = [self.test_metric_acc_file, self.remote_test_metric_acc_file] + self.sync_file_list['metrics_iteration'] = [ + self.metrics_iteration_file, self.remote_metrics_iteration_file] + self.sync_file_list['feature_importance'] = [ + self.feature_importance_file, self.remote_feature_importance_file] + self.sync_file_list['summary_evaluation'] = [ + self.summary_evaluation_file, self.remote_summary_evaluation_file] + self.sync_file_list['train_ks_table'] = [ + self.train_metric_ks_table, self.remote_train_metric_ks_table] + self.sync_file_list['train_metric_roc'] = [ + self.train_metric_roc_file, self.remote_train_metric_roc_file] + self.sync_file_list['train_metric_ks'] = [ + self.train_metric_ks_file, self.remote_train_metric_ks_file] + self.sync_file_list['train_metric_pr'] = [ + self.train_metric_pr_file, self.remote_train_metric_pr_file] + self.sync_file_list['train_metric_acc'] = [ + self.train_metric_acc_file, self.remote_train_metric_acc_file] + self.sync_file_list['test_ks_table'] = [ + self.test_metric_ks_table, self.remote_test_metric_ks_table] + self.sync_file_list['test_metric_roc'] = [ + self.test_metric_roc_file, self.remote_test_metric_roc_file] + self.sync_file_list['test_metric_ks'] = [ + self.test_metric_ks_file, self.remote_test_metric_ks_file] + self.sync_file_list['test_metric_pr'] = [ + self.test_metric_pr_file, self.remote_test_metric_pr_file] + self.sync_file_list['test_metric_acc'] = [ + self.test_metric_acc_file, self.remote_test_metric_acc_file] + class LGBMMessage(Enum): FEATURE_NAME = "FEATURE_NAME" diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py index 4d0c184b..888c9618 100644 --- a/python/ppc_model/secure_lgbm/vertical/booster.py +++ b/python/ppc_model/secure_lgbm/vertical/booster.py @@ -13,11 +13,12 @@ from ppc_model.network.stub import PushRequest, PullRequest from ppc_model.common.model_result import ResultFileHandling from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning +from ppc_model.secure_model_base.secure_model_booster import SecureModelBooster from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext, LGBMMessage # 抽离sgb的公共部分 -class VerticalBooster(VerticalModel): +class VerticalBooster(SecureModelBooster): def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None: super().__init__(ctx) self.dataset = dataset @@ -129,7 +130,6 @@ def _get_leaf_mask(self, split_info, instance): return left_mask, right_mask def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=False): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -150,12 +150,11 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal ctx.codec, ctx.phe.public_key, enc_data) )) - log.info( + self.logger.info( f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " f"data_length: {len(enc_data)}, time_costs: {time.time() - start_time}s") def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -172,13 +171,12 @@ def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): public_key, enc_data = PheMessage.unpacking_data( ctx.codec, byte_data) - log.info( + self.logger.info( f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") return public_key, enc_data def _send_byte_data(self, ctx, key_type, byte_data, partner_index): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -189,12 +187,11 @@ def _send_byte_data(self, ctx, key_type, byte_data, partner_index): data=byte_data )) - log.info( + self.logger.info( f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") def _receive_byte_data(self, ctx, key_type, partner_index): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -204,7 +201,7 @@ def _receive_byte_data(self, ctx, key_type, partner_index): key=key_type )) - log.info( + self.logger.info( f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") return byte_data @@ -215,13 +212,13 @@ def _split_test_data(ctx, test_X, X_split): return feat_bin.data_binning(test_X, X_split)[0] def save_model(self, file_path=None): - log = self.ctx.components.logger() + super().save_model(file_path, "lgbm_model") + + def save_model_hook(self, file_path): + # save the feature_bin if file_path is not None: self.ctx.feature_bin_file = os.path.join( file_path, self.ctx.FEATURE_BIN_FILE) - self.ctx.model_data_file = os.path.join( - file_path, self.ctx.MODEL_DATA_FILE) - if self._X_split is not None and not os.path.exists(self.ctx.feature_bin_file): X_split_dict = {k: v for k, v in zip( self.dataset.feature_name, self._X_split)} @@ -229,35 +226,19 @@ def save_model(self, file_path=None): json.dump(X_split_dict, f) ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished.") - if not os.path.exists(self.ctx.model_data_file): serial_trees = [self._serial_tree(tree) for tree in self._trees] with open(self.ctx.model_data_file, 'w') as f: json.dump(serial_trees, f) ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.model_data_file, self.ctx.remote_model_data_file) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.") - self.merge_model_file() - - def merge_model_file(self): - + def merge_model_file(self, lgbm_model): # 加密文件 - lgbm_model = {} - lgbm_model['model_type'] = 'xgb_model' - lgbm_model['label_provider'] = self.ctx.participant_id_list[0] - lgbm_model['label_column'] = 'y' - lgbm_model['participant_agency_list'] = [] - for partner_index in range(0, len(self.ctx.participant_id_list)): - agency_info = { - 'agency': self.ctx.participant_id_list[partner_index]} - agency_info['fields'] = self._all_feature_name[partner_index] - lgbm_model['participant_agency_list'].append(agency_info) - - lgbm_model['model_dict'] = self.ctx.model_params.get_all_params() model_text = {} with open(self.ctx.feature_bin_file, 'rb') as f: feature_bin_data = f.read() @@ -295,10 +276,11 @@ def merge_model_file(self): json.dump(lgbm_model, f) ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.model_enc_file, self.ctx.remote_model_enc_file) - self.ctx.components.logger().info( + self.logger.info( f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") def split_model_file(self): + # 传入模型 my_agency_id = self.ctx.components.config_data['AGENCY_ID'] model_text = self.ctx.model_predict_algorithm['model_text'] @@ -314,7 +296,6 @@ def split_model_file(self): f.write(model_data) def load_model(self, file_path=None): - log = self.ctx.components.logger() if file_path is not None: self.ctx.feature_bin_file = os.path.join( file_path, self.ctx.FEATURE_BIN_FILE) @@ -333,13 +314,13 @@ def load_model(self, file_path=None): X_split_dict = json.load(f) feature_name = list(X_split_dict.keys()) x_split = list(X_split_dict.values()) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Load x_split from {self.ctx.feature_bin_file} finished.") assert len(feature_name) == len(self.dataset.feature_name) with open(self.ctx.model_data_file, 'r') as f: serial_trees = json.load(f) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Load serial_trees from {self.ctx.model_data_file} finished.") trees = [self._deserial_tree(tree) for tree in serial_trees] diff --git a/python/ppc_model/secure_lr/secure_lr_context.py b/python/ppc_model/secure_lr/secure_lr_context.py index 7ecbe338..d4e74a01 100644 --- a/python/ppc_model/secure_lr/secure_lr_context.py +++ b/python/ppc_model/secure_lr/secure_lr_context.py @@ -1,18 +1,16 @@ import os from enum import Enum -from typing import Any, Dict -from sklearn.base import BaseEstimator - from ppc_common.ppc_utils.utils import AlgorithmType from ppc_common.ppc_crypto.phe_factory import PheCipherFactory -from ppc_model.common.context import Context from ppc_model.common.initializer import Initializer from ppc_model.common.protocol import TaskRole from ppc_common.ppc_utils import common_func from ppc_model.common.model_setting import ModelSetting +from ppc_model.secure_model_base.secure_model_context import SecureModel +from ppc_model.secure_model_base.secure_model_context import SecureModelContext -class LRModel(BaseEstimator): +class LRModel(SecureModel): def __init__( self, @@ -29,60 +27,7 @@ def __init__( self.learning_rate = learning_rate self.random_state = random_state self.n_jobs = n_jobs - self._other_params: Dict[str, Any] = {} - self.set_params(**kwargs) - - def get_params(self, deep: bool = True) -> Dict[str, Any]: - """Get parameters for this estimator. - - Parameters - ---------- - deep : bool, optional (default=True) - If True, will return the parameters for this estimator and - contained subobjects that are estimators. - - Returns - ------- - params : dict - Parameter names mapped to their values. - """ - params = super().get_params(deep=deep) - params.update(self._other_params) - return params - - def set_model_setting(self, model_setting: ModelSetting) -> "LRModel": - # 获取对象的所有属性名 - attrs = dir(model_setting) - # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） - attrs = [attr for attr in attrs if not attr.startswith('_')] - - params = {} - for attr in attrs: - try: - setattr(self, attr, getattr(model_setting, attr)) - except Exception as e: - pass - return self - - def set_params(self, **params: Any) -> "LRModel": - """Set the parameters of this estimator. - - Parameters - ---------- - **params - Parameter names with their new values. - - Returns - ------- - self : object - Returns self. - """ - for key, value in params.items(): - setattr(self, key, value) - if hasattr(self, f"_{key}"): - setattr(self, f"_{key}", value) - self._other_params[key] = value - return self + super().__init__(**kwargs) class ModelTaskParams(LRModel): @@ -124,75 +69,21 @@ def _get_params(self): """返回LRClassifier所有参数""" return LRModel().get_params() - def get_all_params(self): - """返回SecureLRParams所有参数""" - # 获取对象的所有属性名 - attrs = dir(self) - # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） - attrs = [attr for attr in attrs if not attr.startswith('_')] - - params = {} - for attr in attrs: - try: - # 使用getattr来获取属性的值 - value = getattr(self, attr) - # 检查value是否可调用（例如，方法或函数），如果是，则不打印其值 - if not callable(value): - params[attr] = value - except Exception as e: - pass - return params - -class SecureLRContext(Context): +class SecureLRContext(SecureModelContext): def __init__(self, args, components: Initializer ): - - if args['is_label_holder']: - role = TaskRole.ACTIVE_PARTY - else: - role = TaskRole.PASSIVE_PARTY - - super().__init__(args['job_id'], - args['task_id'], - components, - role) + super().__init__(args, components) self.phe = PheCipherFactory.build_phe( components.homo_algorithm, components.public_key_length) self.codec = PheCipherFactory.build_codec(components.homo_algorithm) - self.is_label_holder = args['is_label_holder'] - self.result_receiver_id_list = args['result_receiver_id_list'] - self.participant_id_list = args['participant_id_list'] - self.model_predict_algorithm = common_func.get_config_value( - "model_predict_algorithm", None, args, False) - self.algorithm_type = args['algorithm_type'] - if 'dataset_id' in args and args['dataset_id'] is not None: - self.dataset_file_path = os.path.join( - self.workspace, args['dataset_id']) - else: - self.dataset_file_path = None - - self.model_params = SecureLRParams() - model_setting = ModelSetting(args['model_dict']) - self.set_model_params(model_setting) - if model_setting.train_features is not None and len(model_setting.train_features) > 0: - self.model_params.train_feature = model_setting.train_features.split( - ',') - if model_setting.categorical is not None and len(model_setting.categorical) > 0: - self.model_params.categorical_feature = model_setting.categorical.split( - ',') - self.model_params.random_state = model_setting.seed - self.sync_file_list = {} - if self.algorithm_type == AlgorithmType.Train.name: - self.set_sync_file() - - def set_model_params(self, model_setting: ModelSetting): - """设置lr参数""" - self.model_params.set_model_setting(model_setting) + + def create_model_param(self): + return SecureLRParams() def get_model_params(self): """获取lr参数""" diff --git a/python/ppc_model/secure_lr/secure_lr_prediction_engine.py b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py index 218264b6..92776269 100644 --- a/python/ppc_model/secure_lr/secure_lr_prediction_engine.py +++ b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py @@ -9,7 +9,7 @@ from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty -class SecureLGBMPredictionEngine(TaskEngine): +class SecureLRPredictionEngine(TaskEngine): task_type = ModelTask.LR_PREDICTING @staticmethod diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py index 9ab285b2..537e3425 100644 --- a/python/ppc_model/secure_lr/vertical/booster.py +++ b/python/ppc_model/secure_lr/vertical/booster.py @@ -16,10 +16,12 @@ from ppc_model.common.model_result import ResultFileHandling from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning from ppc_model.secure_lr.secure_lr_context import SecureLRContext, LRMessage - +from ppc_model.secure_model_base.secure_model_booster import SecureModelBooster # 抽离sgb的公共部分 -class VerticalBooster(VerticalModel): + + +class VerticalBooster(SecureModelBooster): def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None: super().__init__(ctx) self.dataset = dataset @@ -77,8 +79,8 @@ def _get_categorical_idx(feature_name, categorical_feature=[]): def _init_each_iter(self): - idx = self._get_sample_idx(self._iter_id-1, self.dataset.train_X.shape[0], - size = self.params.batch_size) + idx = self._get_sample_idx(self._iter_id-1, self.dataset.train_X.shape[0], + size=self.params.batch_size) feature_select = FeatureSelection.feature_selecting( list(self.dataset.feature_name), self.params.train_feature, self.params.feature_rate) @@ -91,11 +93,12 @@ def _send_d_instance_list(self, d): my_agency_id = self.ctx.components.config_data['AGENCY_ID'] start_time = time.time() - self.log.info(f'task {self.ctx.task_id}: Starting iter-{self._iter_id} ' - f'encrypt d in {my_agency_id} party.') - enc_dlist = self.ctx.phe.encrypt_batch_parallel((d_list).astype('object')) - self.log.info(f'task {self.ctx.task_id}: Finished iter-{self._iter_id} ' - f'encrypt d time_costs: {time.time() - start_time}.') + self.logger.info(f'task {self.ctx.task_id}: Starting iter-{self._iter_id} ' + f'encrypt d in {my_agency_id} party.') + enc_dlist = self.ctx.phe.encrypt_batch_parallel( + (d_list).astype('object')) + self.logger.info(f'task {self.ctx.task_id}: Finished iter-{self._iter_id} ' + f'encrypt d time_costs: {time.time() - start_time}.') for partner_index in range(len(self.ctx.participant_id_list)): if self.ctx.participant_id_list[partner_index] != my_agency_id: @@ -105,7 +108,7 @@ def _send_d_instance_list(self, d): def _receive_d_instance_list(self): my_agency_id = self.ctx.components.config_data['AGENCY_ID'] - + public_key_list = [] d_other_list = [] partner_index_list = [] @@ -127,24 +130,29 @@ def _calculate_deriv(self, x_, d, partner_index_list, d_other_list): # 计算明文*密文 matmul # deriv_other_i = np.matmul(x.T, d_other_list[i]) deriv_other_i = self.enc_matmul(x.T, d_other_list[i]) - + # 发送密文，接受密文并解密 self._send_enc_data(self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', deriv_other_i, partner_index) _, enc_deriv_i = self._receive_enc_data( self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', partner_index) - deriv_i_rec = np.array(self.ctx.phe.decrypt_batch(enc_deriv_i), dtype='object') - deriv_i = self.recover_d(self.ctx, deriv_i_rec, is_square=True) / x_.shape[0] - + deriv_i_rec = np.array( + self.ctx.phe.decrypt_batch(enc_deriv_i), dtype='object') + deriv_i = self.recover_d( + self.ctx, deriv_i_rec, is_square=True) / x_.shape[0] + # 发送明文，接受明文并计算 self._send_byte_data(self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', deriv_i.astype('float').tobytes(), partner_index) deriv_x_i = np.frombuffer(self._receive_byte_data( - self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', partner_index), dtype=np.float) - self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') - self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv_x_i: {deriv_x_i}.') + self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', partner_index), dtype=np.float) + self.logger.info( + f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') + self.logger.info( + f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv_x_i: {deriv_x_i}.') deriv += deriv_x_i - self.log.info(f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') + self.logger.info( + f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.') return deriv def _calculate_deriv1(self, x_, d, partner_index_list, d_other_list): @@ -159,12 +167,13 @@ def _calculate_deriv1(self, x_, d, partner_index_list, d_other_list): deriv_other_i, partner_index) _, enc_deriv_i = self._receive_enc_data( self.ctx, f'{LRMessage.ENC_D_HIST.value}_{self._iter_id}', partner_index) - deriv_i = np.array(self.ctx.phe.decrypt_batch(enc_deriv_i), dtype='object') - deriv += (self.recover_d(self.ctx, deriv_i, is_square=True) / x_.shape[0]) + deriv_i = np.array(self.ctx.phe.decrypt_batch( + enc_deriv_i), dtype='object') + deriv += (self.recover_d(self.ctx, deriv_i, + is_square=True) / x_.shape[0]) return deriv def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=False): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -185,12 +194,11 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal ctx.codec, ctx.phe.public_key, enc_data) )) - log.info( + self.logger.info( f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " f"data_length: {len(enc_data)}, time_costs: {time.time() - start_time}s") def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -207,13 +215,12 @@ def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False): public_key, enc_data = PheMessage.unpacking_data( ctx.codec, byte_data) - log.info( + self.logger.info( f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") return public_key, enc_data def _send_byte_data(self, ctx, key_type, byte_data, partner_index): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -224,12 +231,11 @@ def _send_byte_data(self, ctx, key_type, byte_data, partner_index): data=byte_data )) - log.info( + self.logger.info( f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") def _receive_byte_data(self, ctx, key_type, partner_index): - log = ctx.components.logger() start_time = time.time() partner_id = ctx.participant_id_list[partner_index] @@ -239,47 +245,32 @@ def _receive_byte_data(self, ctx, key_type, partner_index): key=key_type )) - log.info( + self.logger.info( f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, " f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s") return byte_data def save_model(self, file_path=None): - log = self.ctx.components.logger() - if file_path is not None: - self.ctx.model_data_file = os.path.join( - file_path, self.ctx.MODEL_DATA_FILE) + super().save_model(file_path, "lr_model") + def save_model_hook(self, model_file_path): if not os.path.exists(self.ctx.model_data_file): serial_weight = list(self._train_weights) with open(self.ctx.model_data_file, 'w') as f: json.dump(serial_weight, f) ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.model_data_file, self.ctx.remote_model_data_file) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Saved serial_weight to {self.ctx.model_data_file} finished.") - self.merge_model_file() - - def merge_model_file(self): + def merge_model_file(self, lr_model): # 加密文件 - lr_model = {} - lr_model['model_type'] = 'lr_model' - lr_model['label_provider'] = self.ctx.participant_id_list[0] - lr_model['label_column'] = 'y' - lr_model['participant_agency_list'] = [] - for partner_index in range(0, len(self.ctx.participant_id_list)): - agency_info = {'agency': self.ctx.participant_id_list[partner_index]} - agency_info['fields'] = self._all_feature_name[partner_index] - lr_model['participant_agency_list'].append(agency_info) - - lr_model['model_dict'] = self.ctx.model_params.get_all_params() model_text = {} with open(self.ctx.model_data_file, 'rb') as f: model_data = f.read() model_data_enc = encrypt_data(self.ctx.key, model_data) - + my_agency_id = self.ctx.components.config_data['AGENCY_ID'] model_text[my_agency_id] = cipher_to_base64(model_data_enc) @@ -293,7 +284,8 @@ def merge_model_file(self): if self.ctx.participant_id_list[partner_index] != my_agency_id: model_data_enc = self._receive_byte_data( self.ctx, f'{LRMessage.MODEL_DATA.value}_model_data', partner_index) - model_text[self.ctx.participant_id_list[partner_index]] = cipher_to_base64(model_data_enc) + model_text[self.ctx.participant_id_list[partner_index] + ] = cipher_to_base64(model_data_enc) lr_model['model_text'] = model_text # 上传密文模型 @@ -301,7 +293,7 @@ def merge_model_file(self): json.dump(lr_model, f) ResultFileHandling._upload_file(self.ctx.components.storage_client, self.ctx.model_enc_file, self.ctx.remote_model_enc_file) - self.ctx.components.logger().info( + self.logger.info( f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.") def split_model_file(self): @@ -316,7 +308,6 @@ def split_model_file(self): f.write(model_data) def load_model(self, file_path=None): - log = self.ctx.components.logger() if file_path is not None: self.ctx.model_data_file = os.path.join( file_path, self.ctx.MODEL_DATA_FILE) @@ -330,7 +321,7 @@ def load_model(self, file_path=None): with open(self.ctx.model_data_file, 'r') as f: serial_weight = json.load(f) self._train_weights = np.array(serial_weight) - log.info( + self.logger.info( f"task {self.ctx.task_id}: Load serial_weight from {self.ctx.model_data_file} finished.") def get_weights(self): @@ -356,8 +347,9 @@ def rounding_d(d_list: np.ndarray, expand=1000): @staticmethod def recover_d(ctx, d_sum_list: np.ndarray, is_square=False, expand=1000): - - d_sum_list[d_sum_list > 2**(ctx.phe.key_length-1)] -= 2**(ctx.phe.key_length) + + d_sum_list[d_sum_list > 2 ** + (ctx.phe.key_length-1)] -= 2**(ctx.phe.key_length) if is_square: return (d_sum_list / expand / expand).astype('float') diff --git a/python/ppc_model/secure_model_base/__init__.py b/python/ppc_model/secure_model_base/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/ppc_model/secure_model_base/secure_model_booster.py b/python/ppc_model/secure_model_base/secure_model_booster.py new file mode 100644 index 00000000..ac4f4f86 --- /dev/null +++ b/python/ppc_model/secure_model_base/secure_model_booster.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +import os +import json +from ppc_model.interface.model_base import VerticalModel +from ppc_model.common.model_result import ResultFileHandling +from abc import abstractmethod + + +class SecureModelBooster(VerticalModel): + def __init__(self, ctx) -> None: + super().__init__(ctx) + self.logger = self.ctx.components.logger() + + def save_model(self, file_path=None, model_type=None): + if file_path is not None: + self.ctx.model_data_file = os.path.join( + file_path, self.ctx.MODEL_DATA_FILE) + + self.save_model_hook(file_path) + model = {} + model['model_type'] = model_type + model['label_provider'] = self.ctx.participant_id_list[0] + model['label_column'] = 'y' + model['participant_agency_list'] = [] + for partner_index in range(0, len(self.ctx.participant_id_list)): + agency_info = { + 'agency': self.ctx.participant_id_list[partner_index]} + agency_info['fields'] = self._all_feature_name[partner_index] + model['participant_agency_list'].append(agency_info) + + model['model_dict'] = self.ctx.model_params.get_all_params() + self.merge_model_file(model) + + @abstractmethod + def merge_model_file(self, lr_model): + pass + + @abstractmethod + def save_model_hook(self, model_file_path): + pass diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py new file mode 100644 index 00000000..8292d0cb --- /dev/null +++ b/python/ppc_model/secure_model_base/secure_model_context.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +from abc import abstractmethod +from typing import Any, Dict +import json +import os +from ppc_model.common.context import Context +from ppc_model.common.initializer import Initializer +from ppc_model.common.protocol import TaskRole +from ppc_common.ppc_utils import common_func +from ppc_common.ppc_utils.utils import AlgorithmType +from ppc_model.common.model_setting import ModelSetting + +from sklearn.base import BaseEstimator + + +class SecureModel(BaseEstimator): + + def __init__( + self, + **kwargs): + self.train_feature = [] + self.categorical_feature = None + self.random_state = None + self._other_params: Dict[str, Any] = {} + self.set_params(**kwargs) + + def get_params(self, deep: bool = True) -> Dict[str, Any]: + """Get parameters for this estimator. + + Parameters + ---------- + deep : bool, optional (default=True) + If True, will return the parameters for this estimator and + contained subobjects that are estimators. + + Returns + ------- + params : dict + Parameter names mapped to their values. + """ + params = super().get_params(deep=deep) + params.update(self._other_params) + return params + + def set_model_setting(self, model_setting: ModelSetting): + # 获取对象的所有属性名 + attrs = dir(model_setting) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + setattr(self, attr, getattr(model_setting, attr)) + except Exception as e: + pass + return self + + def set_params(self, **params: Any): + """Set the parameters of this estimator. + + Parameters + ---------- + **params + Parameter names with their new values. + + Returns + ------- + self : object + Returns self. + """ + for key, value in params.items(): + setattr(self, key, value) + if hasattr(self, f"_{key}"): + setattr(self, f"_{key}", value) + self._other_params[key] = value + return self + + def get_all_params(self): + """返回SecureLRParams所有参数""" + # 获取对象的所有属性名 + attrs = dir(self) + # 过滤掉以_或者__开头的属性（这些通常是特殊方法或内部属性） + attrs = [attr for attr in attrs if not attr.startswith('_')] + + params = {} + for attr in attrs: + try: + # 使用getattr来获取属性的值 + value = getattr(self, attr) + # 检查value是否可调用（例如，方法或函数），如果是，则不打印其值 + if not callable(value): + params[attr] = value + except Exception as e: + pass + return params + + +class SecureModelContext(Context): + def __init__(self, + args, + components: Initializer): + + if args['is_label_holder']: + role = TaskRole.ACTIVE_PARTY + else: + role = TaskRole.PASSIVE_PARTY + + super().__init__(args['job_id'], + args['task_id'], + components, + role) + self.is_label_holder = args['is_label_holder'] + self.result_receiver_id_list = args['result_receiver_id_list'] + self.participant_id_list = args['participant_id_list'] + + model_predict_algorithm_str = common_func.get_config_value( + "model_predict_algorithm", None, args, False) + if model_predict_algorithm_str is not None: + self.model_predict_algorithm = json.loads( + model_predict_algorithm_str) + self.algorithm_type = args['algorithm_type'] + self.predict = False + if self.algorithm_type == AlgorithmType.Predict.name: + self.predict = True + # check for the predict task + if self.predict and self.model_predict_algorithm is None: + raise f"Not set model_predict_algorithm for the job: {self.task_id}" + + if 'dataset_id' in args and args['dataset_id'] is not None: + self.dataset_file_path = os.path.join( + self.workspace, args['dataset_id']) + else: + self.dataset_file_path = None + self.model_params = self.create_model_param() + self.reset_model_params(ModelSetting(args['model_dict'])) + self.sync_file_list = {} + if self.algorithm_type == AlgorithmType.Train.name: + self.set_sync_file() + + @abstractmethod + def set_sync_file(self): + pass + + @abstractmethod + def create_model_param(self): + pass + + def reset_model_params(self, model_setting: ModelSetting): + """设置lr参数""" + self.model_params.set_model_setting(model_setting) + if model_setting.train_features is not None and len(model_setting.train_features) > 0: + self.model_params.train_feature = model_setting.train_features.split( + ',') + if model_setting.categorical is not None and len(model_setting.categorical) > 0: + self.model_params.categorical_feature = model_setting.categorical.split( + ',') + if model_setting.seed is not None: + self.model_params.random_state = model_setting.seed From 25e376dc3915701975131010bc5917f9886252fb Mon Sep 17 00:00:00 2001 From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com> Date: Thu, 17 Oct 2024 10:43:16 +0800 Subject: [PATCH 061/120] =?UTF-8?q?=E6=9B=B4=E6=96=B0jupyter=20task=20?= =?UTF-8?q?=E5=8F=82=E6=95=B0=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update jupyter task --- python/wedpr_ml_toolkit/job_exceuter/pws_client.py | 6 ++++-- python/wedpr_ml_toolkit/test/test_dev.py | 9 +++++---- python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/python/wedpr_ml_toolkit/job_exceuter/pws_client.py b/python/wedpr_ml_toolkit/job_exceuter/pws_client.py index 40df0238..08cb2e5d 100644 --- a/python/wedpr_ml_toolkit/job_exceuter/pws_client.py +++ b/python/wedpr_ml_toolkit/job_exceuter/pws_client.py @@ -1,3 +1,4 @@ +import json import random import time import requests @@ -40,8 +41,9 @@ def run(self, params): response = requests.request("POST", self.pws_url, json=payload, headers=headers) if response.status_code != 200: raise Exception(f"创建任务失败: {response.json()}") - return - # return self._poll_task_status(response.data, self.token) + print(response.text) + # self._poll_task_status(response.data, self.token) + return json.loads(response.text) def _poll_task_status(self, job_id, token): while True: diff --git a/python/wedpr_ml_toolkit/test/test_dev.py b/python/wedpr_ml_toolkit/test/test_dev.py index 9ccc864f..bfaf7a8d 100644 --- a/python/wedpr_ml_toolkit/test/test_dev.py +++ b/python/wedpr_ml_toolkit/test/test_dev.py @@ -13,16 +13,16 @@ # 从jupyter环境中获取project_id等信息 # create workspace # 相同项目/刷新专家模式project_id固定 -project_id = 'p-123' +project_id = '测试-xinyi' user = 'flyhuang1' -my_agency='sgd' +my_agency='SGD' pws_endpoint = 'http://139.159.202.235:8005' # http hdfs_endpoint = 'http://192.168.0.18:50070' # client token = 'abc...' # 自定义合作方机构 -partner_agency1='webank' +partner_agency1='WeBank' partner_agency2='TX' # 初始化project ctx 信息 @@ -45,7 +45,8 @@ dataset1.save_values(path='d-101') # './milestone2\\sgd\\flyhuang1\\share\\d-101' # hdfs_path -dataset2 = WedprData(ctx, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) +ctx2 = BaseContext(project_id, 'flyhuang', pws_endpoint, hdfs_endpoint, token) +dataset2 = WedprData(ctx2, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) dataset2.storage_client = None # dataset2.load_values() if dataset2.storage_client is None: diff --git a/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py b/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py index 85b81d52..2cdae065 100644 --- a/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py +++ b/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py @@ -32,7 +32,7 @@ def task(self, params: dict = {}): self.check_agencies() job_response = self.excute.run(params) - return job_response.job_id + return job_response['data'] def psi(self, dataset: DataContext = None, merge_filed: str = 'id'): @@ -44,16 +44,16 @@ def psi(self, dataset: DataContext = None, merge_filed: str = 'id'): # 构造参数 # params = {merge_filed: merge_filed} params = {'jobType': 'PSI', - 'projectName': 'jupyter', + 'projectName': self.dataset.ctx.project_id, 'param': json.dumps({'dataSetList': self.dataset_list}).replace('"', '\\"'), 'taskParties': self.task_parties, - 'datasetList': [None, None]} + 'datasetList': self.dataset_id_list} # 执行任务 job_id = self.task(params) # 结果处理 - psi_result = PSIResult(dataset, job_id) + psi_result = PSIResult(dataset, 'psi-' + job_id) return psi_result From 55f7765ab7d3fccb214647589901a1e0c26e116d Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 17 Oct 2024 15:22:51 +0800 Subject: [PATCH 062/120] refactor wedpr_ml_toolkit --- .../wedpr_ml_toolkit/common/base_context.py | 13 -- python/wedpr_ml_toolkit/common/base_result.py | 8 - .../utils}/__init__.py | 0 .../wedpr_ml_toolkit/common/utils/constant.py | 14 ++ python/wedpr_ml_toolkit/common/utils/utils.py | 26 +++ .../{result => context}/__init__.py | 0 .../{wedpr_data => context}/data_context.py | 11 +- .../wedpr_ml_toolkit/context/job_context.py | 139 +++++++++++++++ .../{utils => context/result}/__init__.py | 0 .../context/result/fe_result_context.py | 22 +++ .../context/result/model_result_context.py | 51 ++++++ .../context/result/psi_result_context.py | 24 +++ .../context/result/result_context.py | 14 ++ .../job_exceuter/pws_client.py | 79 --------- python/wedpr_ml_toolkit/result/fe_result.py | 27 --- .../wedpr_ml_toolkit/result/model_result.py | 58 ------- python/wedpr_ml_toolkit/result/psi_result.py | 27 --- python/wedpr_ml_toolkit/test/test_dev.py | 28 ++-- .../{wedpr_data => transport}/__init__.py | 0 .../transport/credential_generator.py | 55 ++++++ .../storage_entrypoint.py} | 13 +- .../transport/wedpr_entrypoint.py | 41 +++++ .../transport/wedpr_remote_job_client.py | 91 ++++++++++ python/wedpr_ml_toolkit/utils/agency.py | 5 - python/wedpr_ml_toolkit/utils/utils.py | 12 -- .../wedpr_ml_toolkit/wedpr_data/wedpr_data.py | 68 -------- python/wedpr_ml_toolkit/wedpr_ml_toolkit.py | 26 +++ .../wedpr_session/__init__.py | 0 .../wedpr_session/wedpr_session.py | 158 ------------------ 29 files changed, 532 insertions(+), 478 deletions(-) delete mode 100644 python/wedpr_ml_toolkit/common/base_context.py delete mode 100644 python/wedpr_ml_toolkit/common/base_result.py rename python/wedpr_ml_toolkit/{job_exceuter => common/utils}/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/common/utils/constant.py create mode 100644 python/wedpr_ml_toolkit/common/utils/utils.py rename python/wedpr_ml_toolkit/{result => context}/__init__.py (100%) rename python/wedpr_ml_toolkit/{wedpr_data => context}/data_context.py (82%) create mode 100644 python/wedpr_ml_toolkit/context/job_context.py rename python/wedpr_ml_toolkit/{utils => context/result}/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/context/result/fe_result_context.py create mode 100644 python/wedpr_ml_toolkit/context/result/model_result_context.py create mode 100644 python/wedpr_ml_toolkit/context/result/psi_result_context.py create mode 100644 python/wedpr_ml_toolkit/context/result/result_context.py delete mode 100644 python/wedpr_ml_toolkit/job_exceuter/pws_client.py delete mode 100644 python/wedpr_ml_toolkit/result/fe_result.py delete mode 100644 python/wedpr_ml_toolkit/result/model_result.py delete mode 100644 python/wedpr_ml_toolkit/result/psi_result.py rename python/wedpr_ml_toolkit/{wedpr_data => transport}/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/transport/credential_generator.py rename python/wedpr_ml_toolkit/{job_exceuter/hdfs_client.py => transport/storage_entrypoint.py} (76%) create mode 100644 python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py create mode 100644 python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py delete mode 100644 python/wedpr_ml_toolkit/utils/agency.py delete mode 100644 python/wedpr_ml_toolkit/utils/utils.py delete mode 100644 python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit.py delete mode 100644 python/wedpr_ml_toolkit/wedpr_session/__init__.py delete mode 100644 python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py diff --git a/python/wedpr_ml_toolkit/common/base_context.py b/python/wedpr_ml_toolkit/common/base_context.py deleted file mode 100644 index cd2cdb66..00000000 --- a/python/wedpr_ml_toolkit/common/base_context.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - - -class BaseContext: - - def __init__(self, project_id, user_name, pws_endpoint=None, hdfs_endpoint=None, token=None): - - self.project_id = project_id - self.user_name = user_name - self.pws_endpoint = pws_endpoint - self.hdfs_endpoint = hdfs_endpoint - self.token = token - self.workspace = './milestone2' diff --git a/python/wedpr_ml_toolkit/common/base_result.py b/python/wedpr_ml_toolkit/common/base_result.py deleted file mode 100644 index 88bb3f8f..00000000 --- a/python/wedpr_ml_toolkit/common/base_result.py +++ /dev/null @@ -1,8 +0,0 @@ -from wedpr_ml_toolkit.common.base_context import BaseContext - - -class BaseResult: - - def __init__(self, ctx: BaseContext): - - self.ctx = ctx diff --git a/python/wedpr_ml_toolkit/job_exceuter/__init__.py b/python/wedpr_ml_toolkit/common/utils/__init__.py similarity index 100% rename from python/wedpr_ml_toolkit/job_exceuter/__init__.py rename to python/wedpr_ml_toolkit/common/utils/__init__.py diff --git a/python/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/common/utils/constant.py new file mode 100644 index 00000000..460f36d8 --- /dev/null +++ b/python/wedpr_ml_toolkit/common/utils/constant.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +class Constant: + NUMERIC_ARRAY = [i for i in range(10)] + HTTP_STATUS_OK = 200 + DEFAULT_SUBMIT_JOB_URI = '/api/wedpr/v3/project/submitJob' + DEFAULT_QUERY_JOB_STATUS_URL = '/api/wedpr/v3/project/queryJobByCondition' + PSI_RESULT_FILE = "psi_result.csv" + + FEATURE_BIN_FILE = "feature_bin.json" + TEST_MODEL_OUTPUT_FILE = "test_output.csv" + TRAIN_MODEL_OUTPUT_FILE = "train_output.csv" + + FE_RESULT_FILE = "fe_result.csv" \ No newline at end of file diff --git a/python/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/common/utils/utils.py new file mode 100644 index 00000000..f517eaba --- /dev/null +++ b/python/wedpr_ml_toolkit/common/utils/utils.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +import uuid +from enum import Enum +import random +from common.utils.constant import Constant +from urllib.parse import urlencode, urlparse, parse_qs, quote + +class IdPrefixEnum(Enum): + DATASET = "d-" + ALGORITHM = "a-" + JOB = "j-" + + +def make_id(prefix): + return prefix + str(uuid.uuid4()).replace("-", "") + +def generate_nonce(nonce_len): + return ''.join(random.choice(Constant.NUMERIC_ARRAY) for _ in range(nonce_len)) + +def add_params_to_url(url, params): + parsed_url = urlparse(url) + query_params = parse_qs(parsed_url.query) + for key, value in params.items(): + query_params[key] = value + new_query = urlencode(query_params, doseq=True) + return parsed_url._replace(query=new_query).geturl() diff --git a/python/wedpr_ml_toolkit/result/__init__.py b/python/wedpr_ml_toolkit/context/__init__.py similarity index 100% rename from python/wedpr_ml_toolkit/result/__init__.py rename to python/wedpr_ml_toolkit/context/__init__.py diff --git a/python/wedpr_ml_toolkit/wedpr_data/data_context.py b/python/wedpr_ml_toolkit/context/data_context.py similarity index 82% rename from python/wedpr_ml_toolkit/wedpr_data/data_context.py rename to python/wedpr_ml_toolkit/context/data_context.py index cf9e7645..2dc611f8 100644 --- a/python/wedpr_ml_toolkit/wedpr_data/data_context.py +++ b/python/wedpr_ml_toolkit/context/data_context.py @@ -8,15 +8,18 @@ class DataContext: def __init__(self, *datasets): self.datasets = list(datasets) self.ctx = self.datasets[0].ctx - + self._check_datasets() def _save_dataset(self, dataset): if dataset.dataset_path is None: - dataset.dataset_id = utils.make_id(utils.IdPrefixEnum.DATASET.value) - dataset.dataset_path = os.path.join(dataset.storage_workspace, dataset.dataset_id) + dataset.dataset_id = utils.make_id( + utils.IdPrefixEnum.DATASET.value) + dataset.dataset_path = os.path.join( + dataset.storage_workspace, dataset.dataset_id) if dataset.storage_client is not None: - dataset.storage_client.upload(dataset.values, dataset.dataset_path) + dataset.storage_client.upload( + dataset.values, dataset.dataset_path) def _check_datasets(self): for dataset in self.datasets: diff --git a/python/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/context/job_context.py new file mode 100644 index 00000000..620bc1fb --- /dev/null +++ b/python/wedpr_ml_toolkit/context/job_context.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +import json + +from wedpr_ml_toolkit.context.data_context import DataContext +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobParam +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo +from abc import abstractmethod +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient + + +class JobContext: + + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, dataset: DataContext = None, my_agency=None): + if dataset is None: + raise Exception("Must define the job related datasets!") + self.remote_job_client = remote_job_client + self.project_name = project_name + self.dataset = dataset + self.create_agency = my_agency + self.participant_id_list = [] + self.task_parties = [] + self.dataset_id_list = [] + self.dataset_list = [] + self.label_holder_agency = None + self.label_columns = None + self.__init_participant__() + self.__init_label_information__() + self.result_receiver_id_list = [my_agency] # 仅限jupyter所在机构 + self.__check__() + + def __check__(self): + """ + 校验机构数和任务是否匹配 + """ + if len(self.participant_id_list) < 2: + raise Exception("至少需要传入两个机构") + if not self.label_holder_agency or self.label_holder_agency not in self.participant_id_list: + raise Exception("数据集中标签提供方配置错误") + + def __init_participant__(self): + participant_id_list = [] + dataset_id_list = [] + for dataset in self.dataset.datasets: + participant_id_list.append(dataset.agency.agency_id) + dataset_id_list.append(dataset.dataset_id) + self.task_parties.append({'userName': dataset.ctx.user_name, + 'agency': dataset.agency.agency_id}) + self.participant_id_list = participant_id_list + self.dataset_id_list = dataset_id_list + + def __init_label_information__(self): + label_holder_agency = None + label_columns = None + for dataset in self.dataset.datasets: + if dataset.is_label_holder: + label_holder_agency = dataset.agency.agency_id + label_columns = 'y' + self.label_holder_agency = label_holder_agency + self.label_columns = label_columns + + @abstractmethod + def build(self) -> JobParam: + pass + + @abstractmethod + def get_job_type(self) -> str: + pass + + def submit(self, project_name): + return self.submit(self.build(project_name)) + + +class PSIJobContext(JobContext): + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'): + super().__init__(remote_job_client, project_name, dataset, my_agency) + self.merge_field = merge_field + + def get_job_type(self) -> str: + return "PSI" + + def build(self) -> JobParam: + self.dataset_list = self.dataset.to_psi_format( + self.merge_field, self.result_receiver_id_list) + job_info = JobInfo(self.get_job_type(), self.project_name, json.dumps( + {'dataSetList': self.dataset_list}).replace('"', '\\"')) + job_param = JobParam(job_info, self.task_parties, self.dataset_id_list) + return job_param + + +class PreprocessingJobContext(JobContext): + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None): + super().__init__(remote_job_client, project_name, dataset, my_agency) + self.model_setting = model_setting + + def get_job_type(self) -> str: + return "PREPROCESSING" + + # TODO: build the request + def build(self) -> JobParam: + return None + + +class FeatureEngineeringJobContext(JobContext): + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None): + super().__init__(remote_job_client, project_name, dataset, my_agency) + self.model_setting = model_setting + + def get_job_type(self) -> str: + return "FEATURE_ENGINEERING" + + # TODO: build the jobParam + def build(self) -> JobParam: + return None + + +class SecureLGBMTrainingJobContext(JobContext): + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None): + super().__init__(remote_job_client, project_name, dataset, my_agency) + self.model_setting = model_setting + + def get_job_type(self) -> str: + return "XGB_TRAINING" + + # TODO: build the jobParam + def build(self) -> JobParam: + return None + + +class SecureLGBMPredictJobContext(JobContext): + def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None): + super().__init__(remote_job_client, project_name, dataset, my_agency) + self.model_setting = model_setting + + def get_job_type(self) -> str: + return "XGB_PREDICTING" + + # TODO: build the jobParam + def build(self) -> JobParam: + return None diff --git a/python/wedpr_ml_toolkit/utils/__init__.py b/python/wedpr_ml_toolkit/context/result/__init__.py similarity index 100% rename from python/wedpr_ml_toolkit/utils/__init__.py rename to python/wedpr_ml_toolkit/context/result/__init__.py diff --git a/python/wedpr_ml_toolkit/context/result/fe_result_context.py b/python/wedpr_ml_toolkit/context/result/fe_result_context.py new file mode 100644 index 00000000..81512241 --- /dev/null +++ b/python/wedpr_ml_toolkit/context/result/fe_result_context.py @@ -0,0 +1,22 @@ +import os + +from wedpr_ml_toolkit.context.data_context import DataContext +from wedpr_ml_toolkit.common.utils.constant import Constant +from wedpr_ml_toolkit.context.result.result_context import ResultContext +from wedpr_ml_toolkit.context.job_context import JobContext + + +class FeResultContext(ResultContext): + + def __init__(self, job_context: JobContext, job_id: str): + super().__init__(job_context, job_id) + + def parse_result(self): + result_list = [] + for dataset in self.job_context.dataset.datasets: + dataset.update_path(os.path.join( + self.job_id, Constant.FE_RESULT_FILE)) + result_list.append(dataset) + + fe_result = DataContext(*result_list) + return fe_result diff --git a/python/wedpr_ml_toolkit/context/result/model_result_context.py b/python/wedpr_ml_toolkit/context/result/model_result_context.py new file mode 100644 index 00000000..c7b3bd48 --- /dev/null +++ b/python/wedpr_ml_toolkit/context/result/model_result_context.py @@ -0,0 +1,51 @@ +import os +import numpy as np + +from ppc_common.ppc_utils import utils +from wedpr_ml_toolkit.context.result.result_context import ResultContext +from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint +from wedpr_ml_toolkit.common.utils.constant import Constant +from wedpr_ml_toolkit.context.job_context import JobContext + + +class ModelResultContext(ResultContext): + def __init__(self, job_context: JobContext, job_id: str, storage_entrypoint: StorageEntryPoint): + super().__init__(job_context, job_id) + self.storage_entrypoint = storage_entrypoint + + +class SecureLGBMResultContext(ModelResultContext): + MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' + + def __init__(self, job_context: JobContext, job_id: str, storage_entrypoint: StorageEntryPoint): + super().__init__(job_context, job_id, storage_entrypoint) + + def parse_result(self): + + # train_praba, test_praba, train_y, test_y, feature_importance, split_xbin, trees, params + # 从hdfs读取结果文件信息，构造为属性 + train_praba_path = os.path.join( + self.job_id, Constant.TRAIN_MODEL_OUTPUT_FILE) + test_praba_path = os.path.join( + self.job_id, Constant.TEST_MODEL_OUTPUT_FILE) + train_output = self.storage_entrypoint.download(train_praba_path) + test_output = self.storage_entrypoint.download(test_praba_path) + self.train_praba = train_output['class_pred'].values + self.test_praba = test_output['class_pred'].values + if 'class_label' in train_output.columns: + self.train_y = train_output['class_label'].values + self.test_y = test_output['class_label'].values + else: + self.train_y = None + self.test_y = None + + feature_bin_path = os.path.join(self.job_id, Constant.FEATURE_BIN_FILE) + model_path = os.path.join(self.job_id, self.MODEL_DATA_FILE) + feature_bin_data = self.storage_entrypoint.download_data( + feature_bin_path) + model_data = self.storage_entrypoint.download_data(model_path) + + self.feature_importance = ... + self.split_xbin = feature_bin_data + self.trees = model_data + self.params = ... diff --git a/python/wedpr_ml_toolkit/context/result/psi_result_context.py b/python/wedpr_ml_toolkit/context/result/psi_result_context.py new file mode 100644 index 00000000..f5a94113 --- /dev/null +++ b/python/wedpr_ml_toolkit/context/result/psi_result_context.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +import os + +from wedpr_ml_toolkit.context.job_context import JobContext +from wedpr_ml_toolkit.context.data_context import DataContext +from wedpr_ml_toolkit.common.utils.constant import Constant +from wedpr_ml_toolkit.context.result.result_context import ResultContext + + +class PSIResultContext(ResultContext): + + PSI_RESULT_FILE = "psi_result.csv" + + def __init__(self, job_context: JobContext, job_id: str): + super().__init__(job_context, job_id) + + def parse_result(self): + result_list = [] + for dataset in self.job_context.dataset.datasets: + dataset.update_path(os.path.join( + self.job_id, Constant.PSI_RESULT_FILE)) + result_list.append(dataset) + + self.psi_result = DataContext(*result_list) diff --git a/python/wedpr_ml_toolkit/context/result/result_context.py b/python/wedpr_ml_toolkit/context/result/result_context.py new file mode 100644 index 00000000..dcb0670a --- /dev/null +++ b/python/wedpr_ml_toolkit/context/result/result_context.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +from wedpr_ml_toolkit.context.job_context import JobContext +from abc import abstractmethod + + +class ResultContext: + def __init__(self, job_context: JobContext, job_id: str): + self.job_id = job_id + self.job_context = job_context + self.parse_result() + + @abstractmethod + def parse_result(self): + pass diff --git a/python/wedpr_ml_toolkit/job_exceuter/pws_client.py b/python/wedpr_ml_toolkit/job_exceuter/pws_client.py deleted file mode 100644 index 08cb2e5d..00000000 --- a/python/wedpr_ml_toolkit/job_exceuter/pws_client.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -import random -import time -import requests - -from ppc_common.ppc_utils import http_utils -from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode - - -PWS_URL = '/api/wedpr/v3/project/submitJob' - - -class PWSApi: - def __init__(self, endpoint, token, - polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5): - self.pws_url = endpoint + PWS_URL - self.token = token - self.polling_interval_s = polling_interval_s - self.max_retries = max_retries - self.retry_delay_s = retry_delay_s - self._completed_status = 'COMPLETED' - self._failed_status = 'FAILED' - - def run(self, params): - - headers = { - "Authorization": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MzEzMTUwMTksInVzZXIiOiJ7XCJ1c2VybmFtZVwiOlwiZmx5aHVhbmcxXCIsXCJncm91cEluZm9zXCI6W3tcImdyb3VwSWRcIjpcIjEwMDAwMDAwMDAwMDAwMDBcIixcImdyb3VwTmFtZVwiOlwi5Yid5aeL55So5oi357uEXCIsXCJncm91cEFkbWluTmFtZVwiOlwiYWRtaW5cIn1dLFwicm9sZU5hbWVcIjpcIm9yaWdpbmFsX3VzZXJcIixcInBlcm1pc3Npb25zXCI6bnVsbCxcImFjY2Vzc0tleUlEXCI6bnVsbCxcImFkbWluXCI6ZmFsc2V9In0.1jZFOVbiISzCvvE9SOsTx0IWb0-OQc3o3rJgCu9GM9A", - "content-type": "application/json" - } - - payload = { - "job": { - "jobType": params['jobType'], - "projectName": params['projectName'], - "param": params['param'] - }, - "taskParties": params['taskParties'], - "datasetList": params['datasetList'] - } - - response = requests.request("POST", self.pws_url, json=payload, headers=headers) - if response.status_code != 200: - raise Exception(f"创建任务失败: {response.json()}") - print(response.text) - # self._poll_task_status(response.data, self.token) - return json.loads(response.text) - - def _poll_task_status(self, job_id, token): - while True: - params = { - 'jsonrpc': '1', - 'method': self._get_task_status_method, - 'token': token, - 'id': random.randint(1, 65535), - 'params': { - 'taskID': job_id, - } - } - response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params) - if response.status_code != 200: - raise Exception(f"轮询任务失败: {response.json()}") - if response['result']['status'] == self._completed_status: - return response['result'] - elif response['result']['status'] == self._failed_status: - raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data']) - time.sleep(self.polling_interval_s) - - def _send_request_with_retry(self, request_func, *args, **kwargs): - attempt = 0 - while attempt < self.max_retries: - try: - response = request_func(*args, **kwargs) - return response - except Exception as e: - attempt += 1 - if attempt < self.max_retries: - time.sleep(self.retry_delay_s) - else: - raise e diff --git a/python/wedpr_ml_toolkit/result/fe_result.py b/python/wedpr_ml_toolkit/result/fe_result.py deleted file mode 100644 index 3caa0fea..00000000 --- a/python/wedpr_ml_toolkit/result/fe_result.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from wedpr_ml_toolkit.wedpr_data.data_context import DataContext -from wedpr_ml_toolkit.common.base_result import BaseResult - - -class FeResult(BaseResult): - - FE_RESULT_FILE = "fe_result.csv" - - def __init__(self, dataset: DataContext, job_id: str): - - super().__init__(dataset.ctx) - self.job_id = job_id - - participant_id_list = [] - for dataset in self.dataset.datasets: - participant_id_list.append(dataset.agency.agency_id) - self.participant_id_list = participant_id_list - - result_list = [] - for dataset in self.dataset.datasets: - dataset.update_path(os.path.join(self.job_id, self.FE_RESULT_FILE)) - result_list.append(dataset) - - fe_result = DataContext(*result_list) - return fe_result diff --git a/python/wedpr_ml_toolkit/result/model_result.py b/python/wedpr_ml_toolkit/result/model_result.py deleted file mode 100644 index 7609003f..00000000 --- a/python/wedpr_ml_toolkit/result/model_result.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import numpy as np - -from ppc_common.ppc_utils import utils - -from wedpr_ml_toolkit.wedpr_data.data_context import DataContext -from wedpr_ml_toolkit.common.base_result import BaseResult -from wedpr_ml_toolkit.job_exceuter.hdfs_client import HDFSApi - - -class ModelResult(BaseResult): - - FEATURE_BIN_FILE = "feature_bin.json" - MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json' - TEST_MODEL_OUTPUT_FILE = "test_output.csv" - TRAIN_MODEL_OUTPUT_FILE = "train_output.csv" - - def __init__(self, dataset: DataContext, job_id: str, job_type: str): - - super().__init__(dataset.ctx) - self.job_id = job_id - - participant_id_list = [] - for dataset in self.dataset.datasets: - participant_id_list.append(dataset.agency.agency_id) - self.participant_id_list = participant_id_list - - if job_type == 'xgb_training': - self._xgb_train_result() - - def _xgb_train_result(self): - - # train_praba, test_praba, train_y, test_y, feature_importance, split_xbin, trees, params - # 从hdfs读取结果文件信息，构造为属性 - train_praba_path = os.path.join( - self.job_id, self.TRAIN_MODEL_OUTPUT_FILE) - test_praba_path = os.path.join( - self.job_id, self.TEST_MODEL_OUTPUT_FILE) - train_output = HDFSApi.download(train_praba_path) - test_output = HDFSApi.download(test_praba_path) - self.train_praba = train_output['class_pred'].values - self.test_praba = test_output['class_pred'].values - if 'class_label' in train_output.columns: - self.train_y = train_output['class_label'].values - self.test_y = test_output['class_label'].values - else: - self.train_y = None - self.test_y = None - - feature_bin_path = os.path.join(self.job_id, self.FEATURE_BIN_FILE) - model_path = os.path.join(self.job_id, self.MODEL_DATA_FILE) - feature_bin_data = HDFSApi.download_data(feature_bin_path) - model_data = HDFSApi.download_data(model_path) - - self.feature_importance = ... - self.split_xbin = feature_bin_data - self.trees = model_data - self.params = ... diff --git a/python/wedpr_ml_toolkit/result/psi_result.py b/python/wedpr_ml_toolkit/result/psi_result.py deleted file mode 100644 index 3b7da74c..00000000 --- a/python/wedpr_ml_toolkit/result/psi_result.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from wedpr_ml_toolkit.wedpr_data.data_context import DataContext -from wedpr_ml_toolkit.common.base_result import BaseResult - - -class PSIResult(BaseResult): - - PSI_RESULT_FILE = "psi_result.csv" - - def __init__(self, dataset: DataContext, job_id: str): - - super().__init__(dataset.ctx) - self.job_id = job_id - - participant_id_list = [] - for dataset in self.dataset.datasets: - participant_id_list.append(dataset.agency.agency_id) - self.participant_id_list = participant_id_list - - result_list = [] - for dataset in self.dataset.datasets: - dataset.update_path(os.path.join(self.job_id, self.PSI_RESULT_FILE)) - result_list.append(dataset) - - psi_result = DataContext(*result_list) - return psi_result diff --git a/python/wedpr_ml_toolkit/test/test_dev.py b/python/wedpr_ml_toolkit/test/test_dev.py index bfaf7a8d..db5efc9a 100644 --- a/python/wedpr_ml_toolkit/test/test_dev.py +++ b/python/wedpr_ml_toolkit/test/test_dev.py @@ -5,9 +5,9 @@ from wedpr_ml_toolkit.common.base_context import BaseContext from wedpr_ml_toolkit.utils.agency import Agency -from wedpr_ml_toolkit.wedpr_data.wedpr_data import WedprData -from wedpr_ml_toolkit.wedpr_data.data_context import DataContext -from wedpr_ml_toolkit.wedpr_session.wedpr_session import WedprSession +from wedpr_ml_toolkit.toolkit import DatasetToolkit +from wedpr_ml_toolkit.context.data_context import DataContext +from wedpr_ml_toolkit.context.job_context import JobContext # 从jupyter环境中获取project_id等信息 @@ -15,15 +15,15 @@ # 相同项目/刷新专家模式project_id固定 project_id = '测试-xinyi' user = 'flyhuang1' -my_agency='SGD' +my_agency = 'sgd' pws_endpoint = 'http://139.159.202.235:8005' # http hdfs_endpoint = 'http://192.168.0.18:50070' # client token = 'abc...' # 自定义合作方机构 -partner_agency1='WeBank' -partner_agency2='TX' +partner_agency1 = 'webank' +partner_agency2 = 'TX' # 初始化project ctx 信息 ctx = BaseContext(project_id, user, pws_endpoint, hdfs_endpoint, token) @@ -40,13 +40,14 @@ **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 }) -dataset1 = WedprData(ctx, values=df, agency=agency1, is_label_holder=True) +dataset1 = DatasetToolkit(ctx, values=df, agency=agency1, is_label_holder=True) dataset1.storage_client = None -dataset1.save_values(path='d-101') # './milestone2\\sgd\\flyhuang1\\share\\d-101' +# './milestone2\\sgd\\flyhuang1\\share\\d-101' +dataset1.save_values(path='d-101') # hdfs_path -ctx2 = BaseContext(project_id, 'flyhuang', pws_endpoint, hdfs_endpoint, token) -dataset2 = WedprData(ctx2, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) +dataset2 = DatasetToolkit( + ctx, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) dataset2.storage_client = None # dataset2.load_values() if dataset2.storage_client is None: @@ -57,20 +58,21 @@ }) dataset2.update_values(values=df2) if dataset1.storage_client is not None: - dataset1.update_values(path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') + dataset1.update_values( + path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') dataset1.load_values() # 构建 dataset context dataset = DataContext(dataset1, dataset2) # 初始化 wedpr task session（含数据） -task = WedprSession(dataset, my_agency=my_agency) +task = JobContext(dataset, my_agency=my_agency) print(task.participant_id_list, task.result_receiver_id_list) # 执行psi任务 psi_result = task.psi() # 初始化 wedpr task session（不含数据） （推荐：使用更灵活） -task = WedprSession(my_agency=my_agency) +task = JobContext(my_agency=my_agency) # 执行psi任务 fe_result = task.proprecessing(dataset) print(task.participant_id_list, task.result_receiver_id_list) diff --git a/python/wedpr_ml_toolkit/wedpr_data/__init__.py b/python/wedpr_ml_toolkit/transport/__init__.py similarity index 100% rename from python/wedpr_ml_toolkit/wedpr_data/__init__.py rename to python/wedpr_ml_toolkit/transport/__init__.py diff --git a/python/wedpr_ml_toolkit/transport/credential_generator.py b/python/wedpr_ml_toolkit/transport/credential_generator.py new file mode 100644 index 00000000..c8a23a7d --- /dev/null +++ b/python/wedpr_ml_toolkit/transport/credential_generator.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +import hashlib +from common import utils +import time + + +class CredentialInfo: + ACCESS_ID_KEY = "accessKeyID" + NONCE_KEY = "nonce" + TIMESTAMP_KEY = "timestamp" + SIGNATURE_KEY = "signature" + + def __init__(self, access_key_id: str, nonce: str, timestamp: str, signature: str): + self.access_key_id = access_key_id + self.nonce = nonce + self.timestamp = timestamp + self.signature = signature + + def to_dict(self): + result = {} + result.update(CredentialInfo.ACCESS_ID_KEY, self.access_key_id) + result.update(CredentialInfo.NONCE_KEY, self.nonce) + result.update(CredentialInfo.TIMESTAMP_KEY, self.timestamp) + result.update(CredentialInfo.SIGNATURE_KEY, self.signature) + + def update_url_with_auth_info(self, url): + auth_params = self.to_dict() + return utils.add_params_to_url(auth_params) + + +class CredentialGenerator: + def __init__(self, access_key_id: str, access_key_secret: str, nonce_len=5): + self.access_key_id = access_key_id + self.access_key_secret = access_key_secret + self.nonce_len = nonce_len + + def generate_credential(self) -> CredentialInfo: + nonce = utils.generate_nonce(self.nonce_len) + timestamp = int(time.time()) + # generate the signature + signature = CredentialGenerator.generate_signature( + self.access_key_id, self.access_key_secret, nonce, timestamp) + return CredentialInfo(self.access_key_id, nonce, timestamp, signature) + + @staticmethod + def generate_signature(access_key_id, access_key_secret, nonce, timestamp) -> str: + anti_replay_info_hash = hashlib.sha3_256() + # hash(access_key_id + nonce + timestamp) + anti_replay_info = f"{access_key_id}{nonce}{timestamp}" + anti_replay_info_hash.update(anti_replay_info) + # hash(anti_replay_info + access_key_secret) + signature_hash = hashlib.sha3_256() + signature_hash.update(anti_replay_info_hash.hexdigest()) + signature_hash.update(access_key_secret) + return signature_hash.hexdigest() diff --git a/python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py b/python/wedpr_ml_toolkit/transport/storage_entrypoint.py similarity index 76% rename from python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py rename to python/wedpr_ml_toolkit/transport/storage_entrypoint.py index d8c7be68..06770d05 100644 --- a/python/wedpr_ml_toolkit/job_exceuter/hdfs_client.py +++ b/python/wedpr_ml_toolkit/transport/storage_entrypoint.py @@ -2,16 +2,17 @@ import io from ppc_common.deps_services import storage_loader +from wedpr_ml_toolkit.config.wedpr_ml_config import StorageConfig -class HDFSApi: - def __init__(self, hdfs_endpoint): - self.hdfs_endpoint = hdfs_endpoint +class StorageEntryPoint: + def __init__(self, storage_config: StorageConfig): + self.storage_config = storage_config config_data = {} config_data['STORAGE_TYPE'] = 'HDFS' - config_data['HDFS_URL'] = self.hdfs_endpoint - config_data['HDFS_ENDPOINT'] = self.hdfs_endpoint + config_data['HDFS_URL'] = self.storage_config.endpoint + config_data['HDFS_ENDPOINT'] = self.storage_config.endpoint self.storage_client = storage_loader.load(config_data, logger=None) def upload(self, dataframe, hdfs_path): @@ -34,7 +35,7 @@ def download(self, hdfs_path): :return: Pandas DataFrame """ content = self.storage_client.get_data(hdfs_path) - dataframe = pd.read_csv(io.BytesIO(content)) + dataframe = pd.read_csv(io.BytesIO(content)) return dataframe def download_byte(self, hdfs_path): diff --git a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py new file mode 100644 index 00000000..0803bc9a --- /dev/null +++ b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +import requests +from transport.credential_generator import CredentialGenerator +from common.utils.constant import Constant +import json + + +class LoadBanlancer: + def __init__(self, remote_entrypoints): + if remote_entrypoints == None or len(remote_entrypoints) == 0: + raise Exception(f"Must define the wedpr entrypoints") + self.remote_entrypoints = remote_entrypoints + self.round_robin_idx = 0 + + # choose with round-robin policy + def select(self, uri_path: str): + selected_idx = self.round_robin_idx + self.round_robin_idx += 1 + selected_entrypoint = self.remote_entrypoints[selected_idx % len( + self.remote_entrypoints)] + return f"{selected_entrypoint}/${uri_path}" + + +class WeDPREntryPoint: + def __init__(self, access_key_id: str, access_key_secret: str, remote_entrypoints, nonce_len: int = 5): + self.credential_generator = CredentialGenerator( + access_key_id, access_key_secret, nonce_len) + self.loadbalancer = LoadBanlancer(remote_entrypoints) + + def send_post_request(self, uri, params, headers, data): + credential_info = self.credential_generator.generate_credential() + url = credential_info.update_url_with_auth_info( + self.loadbalancer.select(uri)) + if not headers: + headers = {'content-type': 'application/json'} + response = requests.post(url, data=data, json=params, headers=headers) + if response.status_code != Constant.HTTP_STATUS_OK: + raise Exception( + f"send post request to {url} failed, response: {response.text}") + # parse the result + return json.loads(response.text) diff --git a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py new file mode 100644 index 00000000..45246c63 --- /dev/null +++ b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py @@ -0,0 +1,91 @@ +from transport.wedpr_entrypoint import WeDPREntryPoint +from common.utils.constant import Constant +from config.wedpr_ml_config import AuthConfig +from config.wedpr_ml_config import JobConfig +import random +import json +import time + + +class JobInfo: + def __init__(self, job_type: str, project_name: str, param: str): + self.jobType = job_type + self.projectName = project_name + self.param = param + + +class JobParam: + def __init__(self, job_info: JobInfo, task_parities, dataset_list): + self.job = job_info + self.taskParties = task_parities + self.datasetList = dataset_list + + +class WeDPRResponse: + def __init__(self, code, msg, data): + self.code = code + self.msg = msg + self.data = data + + def success(self): + return self.code == 0 + + +class QueryJobRequest: + def __init__(self, job_id): + # the job condition + self.job = {} + self.job.update("id", job_id) + + +class WeDPRRemoteJobClient(WeDPREntryPoint): + def __init__(self, auth_config: AuthConfig, job_config: JobConfig): + if auth_config is None: + raise Exception("Must define the auth config!") + if job_config is None: + raise Exception("Must define the job config") + super().__init__(auth_config.access_key_id, auth_config.access_key_secret, + auth_config.remote_entrypoints, auth_config.nonce_len) + self.auth_config = auth_config + self.job_config = job_config + + def get_auth_config(self): + return self.auth_config + + def get_job_config(self): + return self.job_config + + def submit_job(self, job_params: JobParam) -> WeDPRResponse: + wedpr_response = self.send_post_request( + self.job_config._submit_job_uri, None, None, json.dumps(job_params)) + submit_result = WeDPRResponse(**wedpr_response) + # return the job_id + if submit_result.success(): + return submit_result.data + raise Exception( + f"submit_job failed, code: {submit_result.code}, msg: {submit_result.msg}") + + def _poll_task_status(self, job_id, token): + while True: + wedpr_response = WeDPRResponse(self._send_request_with_retry( + self.send_post_request, self.job_config.query_job_status_uri, None, None, json.dumps(QueryJobRequest(job_id)))) + # TODO: check with status + if wedpr_response.success(): + return wedpr_response + else: + raise Exception( + f"_poll_task_status for job {job_id} failed, code: {wedpr_response.code}, msg: {wedpr_response.msg}") + time.sleep(self.job_config.polling_interval_s) + + def _send_request_with_retry(self, request_func, *args, **kwargs): + attempt = 0 + while attempt < self.max_retries: + try: + response = request_func(*args, **kwargs) + return response + except Exception as e: + attempt += 1 + if attempt < self.max_retries: + time.sleep(self.retry_delay_s) + else: + raise e diff --git a/python/wedpr_ml_toolkit/utils/agency.py b/python/wedpr_ml_toolkit/utils/agency.py deleted file mode 100644 index 461bce03..00000000 --- a/python/wedpr_ml_toolkit/utils/agency.py +++ /dev/null @@ -1,5 +0,0 @@ -class Agency: - - def __init__(self, agency_id): - - self.agency_id = agency_id diff --git a/python/wedpr_ml_toolkit/utils/utils.py b/python/wedpr_ml_toolkit/utils/utils.py deleted file mode 100644 index 25e0ec72..00000000 --- a/python/wedpr_ml_toolkit/utils/utils.py +++ /dev/null @@ -1,12 +0,0 @@ -import uuid -from enum import Enum - - -class IdPrefixEnum(Enum): - DATASET = "d-" - ALGORITHM = "a-" - JOB = "j-" - - -def make_id(prefix): - return prefix + str(uuid.uuid4()).replace("-", "") diff --git a/python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py b/python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py deleted file mode 100644 index eb1f487d..00000000 --- a/python/wedpr_ml_toolkit/wedpr_data/wedpr_data.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import pandas as pd - -from wedpr_ml_toolkit.common.base_context import BaseContext -from wedpr_ml_toolkit.job_exceuter.hdfs_client import HDFSApi - - -class WedprData: - - def __init__(self, - ctx: BaseContext, - dataset_id=None, - dataset_path=None, - agency=None, - values=None, - is_label_holder=False): - - # super().__init__(project_id) - self.ctx = ctx - - self.dataset_id = dataset_id - self.dataset_path = dataset_path - self.agency = agency - self.values = values - self.is_label_holder = is_label_holder - self.columns = None - self.shape = None - - self.storage_client = HDFSApi(self.ctx.hdfs_endpoint) - self.storage_workspace = os.path.join(self.ctx.workspace, self.agency.agency_id, self.ctx.user_name, 'share') - - if self.values is not None: - self.columns = self.values.columns - self.shape = self.values.shape - - def load_values(self): - # 加载hdfs的数据集 - if self.storage_client is not None: - self.values = self.storage_client.download(self.dataset_path) - self.columns = self.values.columns - self.shape = self.values.shape - - def save_values(self, path=None): - # 保存数据到hdfs目录 - if path is not None: - self.dataset_path = path - if not self.dataset_path.startswith(self.ctx.workspace): - self.dataset_path = os.path.join(self.storage_workspace, self.dataset_path) - if self.storage_client is not None: - self.storage_client.upload(self.values, self.dataset_path) - - def update_values(self, values: pd.DataFrame = None, path: str = None): - # 将数据集存入hdfs相同路径，替换旧数据集 - if values is not None: - self.values = values - self.columns = self.values.columns - self.shape = self.values.shape - if path is not None: - self.dataset_path = path - if values is not None and self.storage_client is not None: - self.storage_client.upload(self.values, self.dataset_path) - - def update_path(self, path: str = None): - # 将数据集存入hdfs相同路径，替换旧数据集 - if path is not None: - self.dataset_path = path - if self.values is not None: - self.values = None diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py new file mode 100644 index 00000000..02fa8d47 --- /dev/null +++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +from config.wedpr_ml_config import WeDPRMlConfig +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient +from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint + + +class WeDPRMlToolkit: + def __init__(self, config: WeDPRMlConfig): + self.config = config + self.remote_job_client = WeDPRRemoteJobClient( + self.config.auth_config, self.config.job_config) + self.storage_entry_point = StorageEntryPoint( + self.config.storage_config) + + def get_config(self) -> WeDPRMlConfig: + return self.config + + def get_remote_job_client(self) -> WeDPRRemoteJobClient: + return self.remote_job_client + + def get_storage_entry_point(self) -> StorageEntryPoint: + return self.storage_entry_point + + def submit(self, job_param): + return self.remote_job_client.submit_job(job_param) diff --git a/python/wedpr_ml_toolkit/wedpr_session/__init__.py b/python/wedpr_ml_toolkit/wedpr_session/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py b/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py deleted file mode 100644 index 2cdae065..00000000 --- a/python/wedpr_ml_toolkit/wedpr_session/wedpr_session.py +++ /dev/null @@ -1,158 +0,0 @@ -import json - -from wedpr_ml_toolkit.wedpr_data.data_context import DataContext -from wedpr_ml_toolkit.job_exceuter.pws_client import PWSApi -from wedpr_ml_toolkit.result.psi_result import PSIResult -from wedpr_ml_toolkit.result.fe_result import FeResult -from wedpr_ml_toolkit.result.model_result import ModelResult - - -class WedprSession: - - def __init__(self, dataset: DataContext = None, my_agency = None): - - self.dataset = dataset - self.create_agency = my_agency - self.participant_id_list = [] - self.task_parties = [] - self.dataset_id_list = [] - self.dataset_list = [] - self.label_holder_agency = None - self.label_columns = None - - if self.dataset is not None: - self.get_agencies() - self.get_label_holder_agency() - self.result_receiver_id_list = [my_agency] # 仅限jupyter所在机构 - - self.excute = PWSApi(self.dataset.ctx.pws_endpoint, self.dataset.ctx.token) - - def task(self, params: dict = {}): - - self.check_agencies() - job_response = self.excute.run(params) - - return job_response['data'] - - def psi(self, dataset: DataContext = None, merge_filed: str = 'id'): - - if dataset is not None: - self.update_dataset(dataset) - - self.dataset_list = self.dataset.to_psi_format(merge_filed, self.result_receiver_id_list) - - # 构造参数 - # params = {merge_filed: merge_filed} - params = {'jobType': 'PSI', - 'projectName': self.dataset.ctx.project_id, - 'param': json.dumps({'dataSetList': self.dataset_list}).replace('"', '\\"'), - 'taskParties': self.task_parties, - 'datasetList': self.dataset_id_list} - - # 执行任务 - job_id = self.task(params) - - # 结果处理 - psi_result = PSIResult(dataset, 'psi-' + job_id) - - return psi_result - - def proprecessing(self, dataset: DataContext = None, psi_result = None, params: dict = None): - - if dataset is not None: - self.update_dataset(dataset) - - job_id = self.task(self.dataset.to_model_formort()) - - # 结果处理 - datasets_pre = FeResult(dataset, job_id) - - return datasets_pre - - def feature_engineering(self, dataset: DataContext = None, psi_result = None, params: dict = None): - - if dataset is not None: - self.update_dataset(dataset) - - job_id = self.task(self.dataset.to_model_formort()) - - # 结果处理 - datasets_fe = FeResult(dataset, job_id) - - return datasets_fe - - def xgb_training(self, dataset: DataContext = None, psi_result = None, params: dict = None): - - if dataset is not None: - self.update_dataset(dataset) - self.check_datasets() - - job_id = self.task(self.dataset.to_model_formort()) - - # 结果处理 - model_result = ModelResult(dataset, job_id, job_type='xgb_training') - - return model_result - - def xgb_predict(self, dataset: DataContext = None, psi_result = None, model = None): - - if dataset is not None: - self.update_dataset(dataset) - self.check_datasets() - - # 上传模型到hdfs - job_id = self.task(self.dataset.to_model_formort()) - - # 结果处理 - model_result = ModelResult(dataset, job_id, job_type='xgb_predict') - - # 结果处理 - return model_result - - def update_dataset(self, dataset: DataContext): - self.dataset = dataset - self.participant_id_list = self.get_agencies() - self.label_holder_agency = self.get_label_holder_agency() - - def get_agencies(self): - participant_id_list = [] - dataset_id_list = [] - for dataset in self.dataset.datasets: - participant_id_list.append(dataset.agency.agency_id) - dataset_id_list.append(dataset.dataset_id) - self.task_parties.append({'userName': dataset.ctx.user_name, - 'agency': dataset.agency.agency_id}) - self.participant_id_list = participant_id_list - self.dataset_id_list = dataset_id_list - - def get_label_holder_agency(self): - label_holder_agency = None - label_columns = None - for dataset in self.dataset.datasets: - if dataset.is_label_holder: - label_holder_agency = dataset.agency.agency_id - label_columns = 'y' - self.label_holder_agency = label_holder_agency - self.label_columns = label_columns - - def check_agencies(self): - """ - 校验机构数和任务是否匹配 - """ - if len(self.participant_id_list) < 2: - raise ValueError("至少需要传入两个机构") - - def check_datasets(self): - """ - 校验是否包含标签提供方 - """ - if not self.label_holder_agency or self.label_holder_agency not in self.participant_id_list: - raise ValueError("数据集中标签提供方配置错误") - - # def get_agencies(self): - # """ - # 返回所有机构名称的列表。 - - # :return: 机构名称的列表 - # """ - # return self.agencies From e8d19cf971cc5860512d65f1da0734ccc70b9b26 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Thu, 17 Oct 2024 20:03:40 +0800 Subject: [PATCH 063/120] fix wedpr_ml_toolkit (#60) --- .../common/utils/properies_parser.py | 19 +++++ python/wedpr_ml_toolkit/common/utils/utils.py | 5 +- python/wedpr_ml_toolkit/config/__init__.py | 0 .../config/wedpr_ml_config.py | 74 +++++++++++++++++ .../config/wedpr_model_setting.py | 79 +++++++++++++++++++ .../wedpr_ml_toolkit/context/data_context.py | 2 +- .../wedpr_ml_toolkit/context/job_context.py | 44 ++++++++--- .../wedpr_ml_toolkit/test/config.properties | 9 +++ python/wedpr_ml_toolkit/test/test_dev.py | 78 ------------------ .../wedpr_ml_toolkit/test/test_ml_toolkit.py | 72 +++++++++++++++++ .../toolkit/dataset_toolkit.py | 64 +++++++++++++++ .../transport/credential_generator.py | 2 +- .../transport/storage_entrypoint.py | 13 ++- .../transport/wedpr_entrypoint.py | 4 +- .../transport/wedpr_remote_job_client.py | 9 +-- python/wedpr_ml_toolkit/wedpr_ml_toolkit.py | 35 +++++++- 16 files changed, 398 insertions(+), 111 deletions(-) create mode 100644 python/wedpr_ml_toolkit/common/utils/properies_parser.py create mode 100644 python/wedpr_ml_toolkit/config/__init__.py create mode 100644 python/wedpr_ml_toolkit/config/wedpr_ml_config.py create mode 100644 python/wedpr_ml_toolkit/config/wedpr_model_setting.py create mode 100644 python/wedpr_ml_toolkit/test/config.properties delete mode 100644 python/wedpr_ml_toolkit/test/test_dev.py create mode 100644 python/wedpr_ml_toolkit/test/test_ml_toolkit.py create mode 100644 python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py diff --git a/python/wedpr_ml_toolkit/common/utils/properies_parser.py b/python/wedpr_ml_toolkit/common/utils/properies_parser.py new file mode 100644 index 00000000..d42f83fc --- /dev/null +++ b/python/wedpr_ml_toolkit/common/utils/properies_parser.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +class Properties: + def __init__(self, file_path): + self.file_path = file_path + + def getProperties(self): + try: + pro_file = open(self.file_path, 'r', encoding='utf-8') + properties = {} + for line in pro_file: + if line.find('=') > 0: + strs = line.replace('\n', '').split('=') + properties[strs[0].strip()] = strs[1].strip() + except Exception as e: + raise e + else: + pro_file.close() + return properties diff --git a/python/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/common/utils/utils.py index f517eaba..a90af704 100644 --- a/python/wedpr_ml_toolkit/common/utils/utils.py +++ b/python/wedpr_ml_toolkit/common/utils/utils.py @@ -2,9 +2,10 @@ import uuid from enum import Enum import random -from common.utils.constant import Constant +from wedpr_ml_toolkit.common.utils.constant import Constant from urllib.parse import urlencode, urlparse, parse_qs, quote + class IdPrefixEnum(Enum): DATASET = "d-" ALGORITHM = "a-" @@ -14,9 +15,11 @@ class IdPrefixEnum(Enum): def make_id(prefix): return prefix + str(uuid.uuid4()).replace("-", "") + def generate_nonce(nonce_len): return ''.join(random.choice(Constant.NUMERIC_ARRAY) for _ in range(nonce_len)) + def add_params_to_url(url, params): parsed_url = urlparse(url) query_params = parse_qs(parsed_url.query) diff --git a/python/wedpr_ml_toolkit/config/__init__.py b/python/wedpr_ml_toolkit/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/config/wedpr_ml_config.py new file mode 100644 index 00000000..4783b07e --- /dev/null +++ b/python/wedpr_ml_toolkit/config/wedpr_ml_config.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import os +from typing import Any, Dict +from wedpr_ml_toolkit.common.utils.constant import Constant +from wedpr_ml_toolkit.common.utils.properies_parser import Properties + + +class BaseConfig: + def set_params(self, **params: Any): + for key, value in params.items(): + setattr(self, key, value) + if hasattr(self, f"{key}"): + setattr(self, f"{key}", value) + return self + + +class AuthConfig(BaseConfig): + def __init__(self, access_key_id: str = None, access_key_secret: str = None, remote_entrypoints: str = None, nonce_len: int = 5): + self.access_key_id = access_key_id + self.access_key_secret = access_key_secret + self.remote_entrypoints = remote_entrypoints + self.nonce_len = nonce_len + + +class JobConfig(BaseConfig): + def __init__(self, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5, + submit_job_uri: str = Constant.DEFAULT_SUBMIT_JOB_URI, + query_job_status_uri: str = Constant.DEFAULT_QUERY_JOB_STATUS_URL): + self.polling_interval_s = polling_interval_s + self.max_retries = max_retries + self.retry_delay_s = retry_delay_s + self.submit_job_uri = submit_job_uri + self.query_job_status_uri = query_job_status_uri + + +class StorageConfig(BaseConfig): + def __init__(self, storage_endpoint: str = None): + self.storage_endpoint = storage_endpoint + + +class UserConfig(BaseConfig): + def __init__(self, agency_name: str = None, workspace_path: str = None, user_name: str = None): + self.agency_name = agency_name + self.workspace_path = workspace_path + self.user = user_name + + def get_workspace_path(self): + return os.path.join(self.workspace_path, self.user) + + +class WeDPRMlConfig: + def __init__(self, config_dict): + self.auth_config = AuthConfig() + self.auth_config.set_params(**config_dict) + self.job_config = JobConfig() + self.job_config.set_params(**config_dict) + self.storage_config = StorageConfig() + self.storage_config.set_params(**config_dict) + self.user_config = UserConfig() + self.user_config.set_params(**config_dict) + + +class WeDPRMlConfigBuilder: + @staticmethod + def build(config_dict) -> WeDPRMlConfig: + return WeDPRMlConfig(config_dict) + + @staticmethod + def build_from_properties_file(config_file_path): + if not os.path.exists(config_file_path): + raise Exception( + f"build WeDPRMlConfig failed for the config file {config_file_path} not exits!") + properties = Properties(config_file_path) + return WeDPRMlConfigBuilder.build(properties.getProperties()) diff --git a/python/wedpr_ml_toolkit/config/wedpr_model_setting.py b/python/wedpr_ml_toolkit/config/wedpr_model_setting.py new file mode 100644 index 00000000..aa108ac5 --- /dev/null +++ b/python/wedpr_ml_toolkit/config/wedpr_model_setting.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +class PreprocessingModelSetting: + def __init__(self): + self.use_psi = False + self.fillna = False + self.na_select = 1.0 + self.filloutlier = False + self.normalized = False + self.standardized = False + self.categorical = '' + self.psi_select_col = '' + self.psi_select_base = '' + self.psi_select_base = 0.3 + self.psi_select_bins = 4 + self.corr_select = 0 + self.use_goss = False + + +class FeatureEngineeringEngineModelSetting: + def __init__(self): + self.use_iv = False + self.group_num = 4 + self.iv_thresh = 0.1 + + +class CommmonSecureModelSetting: + def __init__(self): + self.learning_rate = 0.1 + self.eval_set_column = '' + self.train_set_value = '' + self.eval_set_value = '' + self.verbose_eval = 1 + self.silent = False + self.train_features = '' + self.random_state = None + self.n_jobs = 0 + + +class SecureLGBMModelSetting(CommmonSecureModelSetting): + def __init__(self): + super().__init__() + self.test_size = 0.3 + self.num_trees = 6 + self.max_depth = 3 + self.max_bin = 4 + self.subsample = 1.0 + self.colsample_bytree = 1 + self.colsample_bylevel = 1 + self.reg_alpha = 0 + self.reg_lambda = 1.0 + self.gamma = 0.0 + self.min_child_weight = 0.0 + self.min_child_samples = 10 + self.seed = 2024 + self.early_stopping_rounds = 5 + self.eval_metric = "auc" + self.threads = 8 + self.one_hot = 0 + + +class SecureLRModelSetting(CommmonSecureModelSetting): + def __init__(self): + super().__init__() + self.feature_rate = 1.0 + self.batch_size = 16 + self.epochs = 3 + + +class ModelSetting(PreprocessingModelSetting, FeatureEngineeringEngineModelSetting, SecureLGBMModelSetting, SecureLRModelSetting): + def __init__(self): + # init PreprocessingSetting + super().__init__() + # init FeatureEngineeringEngineSetting + super(FeatureEngineeringEngineModelSetting, self).__init__(model_dict) + # init SecureLGBMSetting + super(SecureLGBMModelSetting, self).__init__(model_dict) + # init SecureLRSetting + super(SecureLRModelSetting, self).__init__(model_dict) diff --git a/python/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/context/data_context.py index 2dc611f8..ade5b8bb 100644 --- a/python/wedpr_ml_toolkit/context/data_context.py +++ b/python/wedpr_ml_toolkit/context/data_context.py @@ -1,6 +1,6 @@ import os -from wedpr_ml_toolkit.utils import utils +from wedpr_ml_toolkit.common import utils class DataContext: diff --git a/python/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/context/job_context.py index 620bc1fb..51809697 100644 --- a/python/wedpr_ml_toolkit/context/job_context.py +++ b/python/wedpr_ml_toolkit/context/job_context.py @@ -6,6 +6,15 @@ from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo from abc import abstractmethod from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient +from enum import Enum + + +class JobType(Enum): + PSI = "PSI", + PREPROCESSING = "PREPROCESSING", + FEATURE_ENGINEERING = "FEATURE_ENGINEERING", + XGB_TRAINING = "XGB_TRAINING", + XGB_PREDICTING = "XGB_PREDICTING" class JobContext: @@ -63,11 +72,20 @@ def build(self) -> JobParam: pass @abstractmethod - def get_job_type(self) -> str: + def get_job_type(self) -> JobType: + pass + + def submit(self): + return self.remote_job_client.submit_job(self.build()) + + @abstractmethod + def parse_result(self, result_detail): pass - def submit(self, project_name): - return self.submit(self.build(project_name)) + def fetch_job_result(self, job_id, block_until_success): + job_result = self.query_job_status(job_id, block_until_success) + # TODO: determine success or not here + return self.parse_result(self.remote_job_client.query_job_detail(job_id)) class PSIJobContext(JobContext): @@ -75,8 +93,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, d super().__init__(remote_job_client, project_name, dataset, my_agency) self.merge_field = merge_field - def get_job_type(self) -> str: - return "PSI" + def get_job_type(self) -> JobType: + return JobType.PSI def build(self) -> JobParam: self.dataset_list = self.dataset.to_psi_format( @@ -92,8 +110,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, m super().__init__(remote_job_client, project_name, dataset, my_agency) self.model_setting = model_setting - def get_job_type(self) -> str: - return "PREPROCESSING" + def get_job_type(self) -> JobType: + return JobType.PREPROCESSING # TODO: build the request def build(self) -> JobParam: @@ -105,8 +123,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, m super().__init__(remote_job_client, project_name, dataset, my_agency) self.model_setting = model_setting - def get_job_type(self) -> str: - return "FEATURE_ENGINEERING" + def get_job_type(self) -> JobType: + return JobType.FEATURE_ENGINEERING # TODO: build the jobParam def build(self) -> JobParam: @@ -118,8 +136,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, m super().__init__(remote_job_client, project_name, dataset, my_agency) self.model_setting = model_setting - def get_job_type(self) -> str: - return "XGB_TRAINING" + def get_job_type(self) -> JobType: + return JobType.XGB_TRAINING # TODO: build the jobParam def build(self) -> JobParam: @@ -131,8 +149,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, m super().__init__(remote_job_client, project_name, dataset, my_agency) self.model_setting = model_setting - def get_job_type(self) -> str: - return "XGB_PREDICTING" + def get_job_type(self) -> JobType: + return JobType.XGB_PREDICTING # TODO: build the jobParam def build(self) -> JobParam: diff --git a/python/wedpr_ml_toolkit/test/config.properties b/python/wedpr_ml_toolkit/test/config.properties new file mode 100644 index 00000000..c466500f --- /dev/null +++ b/python/wedpr_ml_toolkit/test/config.properties @@ -0,0 +1,9 @@ +access_key_id="" +access_key_secret="" +remote_entrypoints="http://127.0.0.1:16000,http://127.0.0.1:16001" + +agency_name="SGD" +workspace_path="/user/wedpr/milestone2/sgd/" +user="test_user" +storage_endpoint="http://127.0.0.1:50070" + diff --git a/python/wedpr_ml_toolkit/test/test_dev.py b/python/wedpr_ml_toolkit/test/test_dev.py deleted file mode 100644 index db5efc9a..00000000 --- a/python/wedpr_ml_toolkit/test/test_dev.py +++ /dev/null @@ -1,78 +0,0 @@ -import unittest -import numpy as np -import pandas as pd -from sklearn import metrics - -from wedpr_ml_toolkit.common.base_context import BaseContext -from wedpr_ml_toolkit.utils.agency import Agency -from wedpr_ml_toolkit.toolkit import DatasetToolkit -from wedpr_ml_toolkit.context.data_context import DataContext -from wedpr_ml_toolkit.context.job_context import JobContext - - -# 从jupyter环境中获取project_id等信息 -# create workspace -# 相同项目/刷新专家模式project_id固定 -project_id = '测试-xinyi' -user = 'flyhuang1' -my_agency = 'sgd' -pws_endpoint = 'http://139.159.202.235:8005' # http -hdfs_endpoint = 'http://192.168.0.18:50070' # client -token = 'abc...' - - -# 自定义合作方机构 -partner_agency1 = 'webank' -partner_agency2 = 'TX' - -# 初始化project ctx 信息 -ctx = BaseContext(project_id, user, pws_endpoint, hdfs_endpoint, token) - -# 注册 agency -agency1 = Agency(agency_id=my_agency) -agency2 = Agency(agency_id=partner_agency1) - -# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path -# pd.Dataframe -df = pd.DataFrame({ - 'id': np.arange(0, 100), # id列，顺序整数 - 'y': np.random.randint(0, 2, size=100), - **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 -}) - -dataset1 = DatasetToolkit(ctx, values=df, agency=agency1, is_label_holder=True) -dataset1.storage_client = None -# './milestone2\\sgd\\flyhuang1\\share\\d-101' -dataset1.save_values(path='d-101') - -# hdfs_path -dataset2 = DatasetToolkit( - ctx, dataset_path='/user/ppc/milestone2/webank/flyhuang/d-9606695119693829', agency=agency2) -dataset2.storage_client = None -# dataset2.load_values() -if dataset2.storage_client is None: - # 支持更新dataset的values数据 - df2 = pd.DataFrame({ - 'id': np.arange(0, 100), # id列，顺序整数 - **{f'z{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 - }) - dataset2.update_values(values=df2) -if dataset1.storage_client is not None: - dataset1.update_values( - path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') - dataset1.load_values() - -# 构建 dataset context -dataset = DataContext(dataset1, dataset2) - -# 初始化 wedpr task session（含数据） -task = JobContext(dataset, my_agency=my_agency) -print(task.participant_id_list, task.result_receiver_id_list) -# 执行psi任务 -psi_result = task.psi() - -# 初始化 wedpr task session（不含数据） （推荐：使用更灵活） -task = JobContext(my_agency=my_agency) -# 执行psi任务 -fe_result = task.proprecessing(dataset) -print(task.participant_id_list, task.result_receiver_id_list) diff --git a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py new file mode 100644 index 00000000..98ab3ee8 --- /dev/null +++ b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +import unittest +import numpy as np +import pandas as pd +from sklearn import metrics +from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder +from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit +from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit +from wedpr_ml_toolkit.context.data_context import DataContext +from wedpr_ml_toolkit.context.job_context import JobType +from wedpr_ml_toolkit.config.wedpr_model_setting import PreprocessingModelSetting + +wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file( + "config.properties") + +wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config) + +# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path +df = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + 'y': np.random.randint(0, 2, size=100), + **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 +}) + +dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), + storage_workspace=wedpr_config.user_config.get_workspace_path(), + agency=wedpr_config.user_config.agency_name, + values=df, + is_label_holder=True) +dataset1.save_values(path='d-101') + +# hdfs_path +dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), + dataset_path="d-9606695119693829", agency="WeBank") + +dataset2.storage_client = None +# dataset2.load_values() +if dataset2.storage_client is None: + # 支持更新dataset的values数据 + df2 = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + **{f'z{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 + }) + dataset2.update_values(values=df2) +if dataset1.storage_client is not None: + dataset1.update_values( + path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') + dataset1.load_values() + +# 构建 dataset context +dataset = DataContext(dataset1, dataset2) + +# init the job context +project_name = "1" + +psi_job_context = wedpr_ml_toolkit.build_job_context( + JobType.PSI, project_name, dataset, None, "id") +print(psi_job_context.participant_id_list, + psi_job_context.result_receiver_id_list) +# 执行psi任务 +psi_job_id = psi_job_context.submit() +psi_result = psi_job_context.fetch_job_result(psi_job_id, True) + +# 初始化 +preprocessing_data = DataContext(dataset1) +preprocessing_job_context = wedpr_ml_toolkit.build_job_context( + JobType.PREPROCESSING, project_name, preprocessing_data, PreprocessingModelSetting()) +# 执行预处理任务 +fe_job_id = preprocessing_job_context.submit(dataset) +fe_result = preprocessing_job_context.fetch_job_result(fe_job_id, True) +print(preprocessing_job_context.participant_id_list, + preprocessing_job_context.result_receiver_id_list) diff --git a/python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py b/python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py new file mode 100644 index 00000000..2e58a3c7 --- /dev/null +++ b/python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py @@ -0,0 +1,64 @@ +import os +import pandas as pd +from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint + + +class DatasetToolkit: + + def __init__(self, + storage_entrypoint: StorageEntryPoint, + storage_workspace, + dataset_id=None, + dataset_path=None, + agency=None, + values=None, + is_label_holder=False): + self.dataset_id = dataset_id + self.dataset_path = dataset_path + self.agency = agency + self.values = values + self.is_label_holder = is_label_holder + self.columns = None + self.shape = None + + self.storage_client = storage_entrypoint + self.storage_workspace = storage_workspace + + if self.values is not None: + self.columns = self.values.columns + self.shape = self.values.shape + + def load_values(self): + # 加载hdfs的数据集 + if self.storage_client is not None: + self.values = self.storage_client.download(self.dataset_path) + self.columns = self.values.columns + self.shape = self.values.shape + + def save_values(self, path=None): + # 保存数据到hdfs目录 + if path is not None: + self.dataset_path = path + if not self.dataset_path.startswith(self.storage_workspace): + self.dataset_path = os.path.join( + self.storage_workspace, self.dataset_path) + if self.storage_client is not None: + self.storage_client.upload(self.values, self.dataset_path) + + def update_values(self, values: pd.DataFrame = None, path: str = None): + # 将数据集存入hdfs相同路径，替换旧数据集 + if values is not None: + self.values = values + self.columns = self.values.columns + self.shape = self.values.shape + if path is not None: + self.dataset_path = path + if values is not None and self.storage_client is not None: + self.storage_client.upload(self.values, self.dataset_path) + + def update_path(self, path: str = None): + # 将数据集存入hdfs相同路径，替换旧数据集 + if path is not None: + self.dataset_path = path + if self.values is not None: + self.values = None diff --git a/python/wedpr_ml_toolkit/transport/credential_generator.py b/python/wedpr_ml_toolkit/transport/credential_generator.py index c8a23a7d..7800e74d 100644 --- a/python/wedpr_ml_toolkit/transport/credential_generator.py +++ b/python/wedpr_ml_toolkit/transport/credential_generator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import hashlib -from common import utils +from wedpr_ml_toolkit.common import utils import time diff --git a/python/wedpr_ml_toolkit/transport/storage_entrypoint.py b/python/wedpr_ml_toolkit/transport/storage_entrypoint.py index 06770d05..33cfd5c8 100644 --- a/python/wedpr_ml_toolkit/transport/storage_entrypoint.py +++ b/python/wedpr_ml_toolkit/transport/storage_entrypoint.py @@ -2,18 +2,17 @@ import io from ppc_common.deps_services import storage_loader +from ppc_common.deps_services.hdfs_storage import HdfsStorage from wedpr_ml_toolkit.config.wedpr_ml_config import StorageConfig +from wedpr_ml_toolkit.config.wedpr_ml_config import UserConfig class StorageEntryPoint: - def __init__(self, storage_config: StorageConfig): + def __init__(self, user_config: UserConfig, storage_config: StorageConfig): self.storage_config = storage_config - - config_data = {} - config_data['STORAGE_TYPE'] = 'HDFS' - config_data['HDFS_URL'] = self.storage_config.endpoint - config_data['HDFS_ENDPOINT'] = self.storage_config.endpoint - self.storage_client = storage_loader.load(config_data, logger=None) + self.user_config = user_config + self.storage_client = HdfsStorage( + self.storage_config.storage_endpoint, self.user_config.user, self.user_config.get_workspace_path()) def upload(self, dataframe, hdfs_path): """ diff --git a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py index 0803bc9a..9ffe3671 100644 --- a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py +++ b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import requests -from transport.credential_generator import CredentialGenerator -from common.utils.constant import Constant +from wedpr_ml_toolkit.transport.credential_generator import CredentialGenerator +from wedpr_ml_toolkit.common.utils.constant import Constant import json diff --git a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py index 45246c63..1fb1aeb0 100644 --- a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py +++ b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py @@ -1,8 +1,7 @@ -from transport.wedpr_entrypoint import WeDPREntryPoint -from common.utils.constant import Constant -from config.wedpr_ml_config import AuthConfig -from config.wedpr_ml_config import JobConfig -import random +from wedpr_ml_toolkit.transport.wedpr_entrypoint import WeDPREntryPoint +from wedpr_ml_toolkit.common.utils.constant import Constant +from wedpr_ml_toolkit.config.wedpr_ml_config import AuthConfig +from wedpr_ml_toolkit.config.wedpr_ml_config import JobConfig import json import time diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py index 02fa8d47..a4db7de2 100644 --- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py +++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py @@ -1,8 +1,15 @@ # -*- coding: utf-8 -*- -from config.wedpr_ml_config import WeDPRMlConfig +from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfig from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint +from wedpr_ml_toolkit.context.job_context import JobType +from wedpr_ml_toolkit.context.job_context import PSIJobContext +from wedpr_ml_toolkit.context.job_context import PreprocessingJobContext +from wedpr_ml_toolkit.context.job_context import FeatureEngineeringJobContext +from wedpr_ml_toolkit.context.job_context import SecureLGBMPredictJobContext +from wedpr_ml_toolkit.context.job_context import SecureLGBMTrainingJobContext +from wedpr_ml_toolkit.context.data_context import DataContext class WeDPRMlToolkit: @@ -10,8 +17,8 @@ def __init__(self, config: WeDPRMlConfig): self.config = config self.remote_job_client = WeDPRRemoteJobClient( self.config.auth_config, self.config.job_config) - self.storage_entry_point = StorageEntryPoint( - self.config.storage_config) + self.storage_entry_point = StorageEntryPoint(self.config.user_config, + self.config.storage_config) def get_config(self) -> WeDPRMlConfig: return self.config @@ -24,3 +31,25 @@ def get_storage_entry_point(self) -> StorageEntryPoint: def submit(self, job_param): return self.remote_job_client.submit_job(job_param) + + def query_job_status(self, job_id, block_until_success=False): + if not block_until_success: + return self.remote_job_client.query_job_result(job_id) + return self.remote_job_client.wait_job_finished() + + def query_job_detail(self, job_id, block_until_success): + job_result = self.query_job_status(job_id, block_until_success) + # TODO: determine success or not here + return self.remote_job_client.query_job_detail(job_id) + + def build_job_context(self, job_type: JobType, project_name: str, dataset: DataContext, model_setting=None, id_fields='id'): + if job_type == JobType.PSI: + return PSIJobContext(self.remote_job_client, project_name, dataset, self.config.agency_config.agency_name, id_fields) + if job_type == JobType.PREPROCESSING: + return PreprocessingJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + if job_type == JobType.FEATURE_ENGINEERING: + return FeatureEngineeringJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + if job_type == JobType.XGB_TRAINING: + return SecureLGBMTrainingJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + if job_type == JobType.XGB_PREDICTING: + return SecureLGBMPredictJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) From 9a066dc256702376fdcb9dc78dfde2b8e28f31c3 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 18 Oct 2024 14:18:53 +0800 Subject: [PATCH 064/120] complement queryJob and queryJobDetails (#61) --- .../common/utils/base_object.py | 28 +++ .../wedpr_ml_toolkit/common/utils/constant.py | 8 +- .../common/utils/properies_parser.py | 2 +- python/wedpr_ml_toolkit/common/utils/utils.py | 2 +- .../config/wedpr_ml_config.py | 37 +-- .../wedpr_ml_toolkit/context/job_context.py | 12 +- .../wedpr_ml_toolkit/test/config.properties | 14 +- .../wedpr_ml_toolkit/test/test_ml_toolkit.py | 133 ++++++----- .../transport/credential_generator.py | 35 +-- .../transport/wedpr_entrypoint.py | 24 +- .../transport/wedpr_remote_job_client.py | 215 ++++++++++++++---- python/wedpr_ml_toolkit/wedpr_ml_toolkit.py | 34 +-- 12 files changed, 373 insertions(+), 171 deletions(-) create mode 100644 python/wedpr_ml_toolkit/common/utils/base_object.py diff --git a/python/wedpr_ml_toolkit/common/utils/base_object.py b/python/wedpr_ml_toolkit/common/utils/base_object.py new file mode 100644 index 00000000..03b9ef9b --- /dev/null +++ b/python/wedpr_ml_toolkit/common/utils/base_object.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from typing import Any +import time + + +class BaseObject: + def set_params(self, **params: Any): + for key, value in params.items(): + setattr(self, key, value) + if hasattr(self, f"{key}"): + setattr(self, f"{key}", value) + return self + + def as_dict(obj): + return {attr: getattr(obj, attr) for attr in dir(obj) if not callable(getattr(obj, attr)) and not attr.startswith("__")} + + def execute_with_retry(self, request_func, retry_times, retry_wait_seconds, *args, **kwargs): + attempt = 0 + while attempt < retry_times: + try: + response = request_func(*args, **kwargs) + return response + except Exception as e: + attempt += 1 + if attempt < retry_times: + time.sleep(retry_wait_seconds) + else: + raise e diff --git a/python/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/common/utils/constant.py index 460f36d8..b323d962 100644 --- a/python/wedpr_ml_toolkit/common/utils/constant.py +++ b/python/wedpr_ml_toolkit/common/utils/constant.py @@ -3,12 +3,14 @@ class Constant: NUMERIC_ARRAY = [i for i in range(10)] HTTP_STATUS_OK = 200 - DEFAULT_SUBMIT_JOB_URI = '/api/wedpr/v3/project/submitJob' - DEFAULT_QUERY_JOB_STATUS_URL = '/api/wedpr/v3/project/queryJobByCondition' + WEDPR_API_PREFIX = '/api/wedpr/v3/' + DEFAULT_SUBMIT_JOB_URI = f'{WEDPR_API_PREFIX}project/submitJob' + DEFAULT_QUERY_JOB_STATUS_URL = f'{WEDPR_API_PREFIX}project/queryJobByCondition' + DEFAULT_QUERY_JOB_DETAIL_URL = f'{WEDPR_API_PREFIX}scheduler/queryJobDetail' PSI_RESULT_FILE = "psi_result.csv" FEATURE_BIN_FILE = "feature_bin.json" TEST_MODEL_OUTPUT_FILE = "test_output.csv" TRAIN_MODEL_OUTPUT_FILE = "train_output.csv" - FE_RESULT_FILE = "fe_result.csv" \ No newline at end of file + FE_RESULT_FILE = "fe_result.csv" diff --git a/python/wedpr_ml_toolkit/common/utils/properies_parser.py b/python/wedpr_ml_toolkit/common/utils/properies_parser.py index d42f83fc..dd05a475 100644 --- a/python/wedpr_ml_toolkit/common/utils/properies_parser.py +++ b/python/wedpr_ml_toolkit/common/utils/properies_parser.py @@ -10,7 +10,7 @@ def getProperties(self): properties = {} for line in pro_file: if line.find('=') > 0: - strs = line.replace('\n', '').split('=') + strs = line.strip("\"").replace('\n', '').split('=') properties[strs[0].strip()] = strs[1].strip() except Exception as e: raise e diff --git a/python/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/common/utils/utils.py index a90af704..27795c8a 100644 --- a/python/wedpr_ml_toolkit/common/utils/utils.py +++ b/python/wedpr_ml_toolkit/common/utils/utils.py @@ -17,7 +17,7 @@ def make_id(prefix): def generate_nonce(nonce_len): - return ''.join(random.choice(Constant.NUMERIC_ARRAY) for _ in range(nonce_len)) + return ''.join(str(random.choice(Constant.NUMERIC_ARRAY)) for _ in range(nonce_len)) def add_params_to_url(url, params): diff --git a/python/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/config/wedpr_ml_config.py index 4783b07e..3e8e8830 100644 --- a/python/wedpr_ml_toolkit/config/wedpr_ml_config.py +++ b/python/wedpr_ml_toolkit/config/wedpr_ml_config.py @@ -1,44 +1,42 @@ # -*- coding: utf-8 -*- import os -from typing import Any, Dict +from wedpr_ml_toolkit.common.utils.base_object import BaseObject from wedpr_ml_toolkit.common.utils.constant import Constant from wedpr_ml_toolkit.common.utils.properies_parser import Properties -class BaseConfig: - def set_params(self, **params: Any): - for key, value in params.items(): - setattr(self, key, value) - if hasattr(self, f"{key}"): - setattr(self, f"{key}", value) - return self - - -class AuthConfig(BaseConfig): +class AuthConfig(BaseObject): def __init__(self, access_key_id: str = None, access_key_secret: str = None, remote_entrypoints: str = None, nonce_len: int = 5): self.access_key_id = access_key_id self.access_key_secret = access_key_secret self.remote_entrypoints = remote_entrypoints self.nonce_len = nonce_len + def get_remote_entrypoints_list(self) -> []: + if self.remote_entrypoints is None: + return None + return self.remote_entrypoints.split(',') + -class JobConfig(BaseConfig): - def __init__(self, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5, +class JobConfig(BaseObject): + def __init__(self, polling_interval_s: int = 5, max_retries: int = 2, retry_delay_s: int = 5, submit_job_uri: str = Constant.DEFAULT_SUBMIT_JOB_URI, - query_job_status_uri: str = Constant.DEFAULT_QUERY_JOB_STATUS_URL): + query_job_status_uri: str = Constant.DEFAULT_QUERY_JOB_STATUS_URL, + query_job_detail_uri: str = Constant.DEFAULT_QUERY_JOB_DETAIL_URL): self.polling_interval_s = polling_interval_s self.max_retries = max_retries self.retry_delay_s = retry_delay_s self.submit_job_uri = submit_job_uri self.query_job_status_uri = query_job_status_uri + self.query_job_detail_uri = query_job_detail_uri -class StorageConfig(BaseConfig): +class StorageConfig(BaseObject): def __init__(self, storage_endpoint: str = None): self.storage_endpoint = storage_endpoint -class UserConfig(BaseConfig): +class UserConfig(BaseObject): def __init__(self, agency_name: str = None, workspace_path: str = None, user_name: str = None): self.agency_name = agency_name self.workspace_path = workspace_path @@ -48,6 +46,11 @@ def get_workspace_path(self): return os.path.join(self.workspace_path, self.user) +class HttpConfig(BaseObject): + def __init__(self, timeout_seconds=3): + self.timeout_seconds = timeout_seconds + + class WeDPRMlConfig: def __init__(self, config_dict): self.auth_config = AuthConfig() @@ -58,6 +61,8 @@ def __init__(self, config_dict): self.storage_config.set_params(**config_dict) self.user_config = UserConfig() self.user_config.set_params(**config_dict) + self.http_config = HttpConfig() + self.http_config.set_params(**config_dict) class WeDPRMlConfigBuilder: diff --git a/python/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/context/job_context.py index 51809697..90355003 100644 --- a/python/wedpr_ml_toolkit/context/job_context.py +++ b/python/wedpr_ml_toolkit/context/job_context.py @@ -6,15 +6,7 @@ from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo from abc import abstractmethod from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient -from enum import Enum - - -class JobType(Enum): - PSI = "PSI", - PREPROCESSING = "PREPROCESSING", - FEATURE_ENGINEERING = "FEATURE_ENGINEERING", - XGB_TRAINING = "XGB_TRAINING", - XGB_PREDICTING = "XGB_PREDICTING" +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobType class JobContext: @@ -99,7 +91,7 @@ def get_job_type(self) -> JobType: def build(self) -> JobParam: self.dataset_list = self.dataset.to_psi_format( self.merge_field, self.result_receiver_id_list) - job_info = JobInfo(self.get_job_type(), self.project_name, json.dumps( + job_info = JobInfo(job_type=self.get_job_type(), project_name=self.project_name, param=json.dumps( {'dataSetList': self.dataset_list}).replace('"', '\\"')) job_param = JobParam(job_info, self.task_parties, self.dataset_id_list) return job_param diff --git a/python/wedpr_ml_toolkit/test/config.properties b/python/wedpr_ml_toolkit/test/config.properties index c466500f..358d170d 100644 --- a/python/wedpr_ml_toolkit/test/config.properties +++ b/python/wedpr_ml_toolkit/test/config.properties @@ -1,9 +1,9 @@ -access_key_id="" -access_key_secret="" -remote_entrypoints="http://127.0.0.1:16000,http://127.0.0.1:16001" +access_key_id= +access_key_secret= +remote_entrypoints=http://127.0.0.1:16000,http://127.0.0.1:16001 -agency_name="SGD" -workspace_path="/user/wedpr/milestone2/sgd/" -user="test_user" -storage_endpoint="http://127.0.0.1:50070" +agency_name=SGD +workspace_path=/user/wedpr/milestone2/sgd/ +user=test_user +storage_endpoint=http://127.0.0.1:50070 diff --git a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py index 98ab3ee8..3d076a80 100644 --- a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py +++ b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py @@ -10,63 +10,94 @@ from wedpr_ml_toolkit.context.job_context import JobType from wedpr_ml_toolkit.config.wedpr_model_setting import PreprocessingModelSetting -wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file( - "config.properties") -wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config) +class WeDPRMlToolkitTestWrapper: + def __init__(self, config_file_path): + self.wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file( + config_file_path) + self.wedpr_ml_toolkit = WeDPRMlToolkit(self.wedpr_config) -# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path -df = pd.DataFrame({ - 'id': np.arange(0, 100), # id列，顺序整数 - 'y': np.random.randint(0, 2, size=100), - **{f'x{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 -}) + def test_submit_job(self): + # 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path + df = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + 'y': np.random.randint(0, 2, size=100), + # x1到x10列，随机数 + **{f'x{i}': np.random.rand(100) for i in range(1, 11)} + }) -dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), - storage_workspace=wedpr_config.user_config.get_workspace_path(), - agency=wedpr_config.user_config.agency_name, - values=df, - is_label_holder=True) -dataset1.save_values(path='d-101') + dataset1 = DatasetToolkit(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(), + storage_workspace=self.wedpr_config.user_config.get_workspace_path(), + agency=self.wedpr_config.user_config.agency_name, + values=df, + is_label_holder=True) + dataset1.save_values(path='d-101') -# hdfs_path -dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), - dataset_path="d-9606695119693829", agency="WeBank") + # hdfs_path + dataset2 = DatasetToolkit(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(), + dataset_path="d-9606695119693829", agency="WeBank") -dataset2.storage_client = None -# dataset2.load_values() -if dataset2.storage_client is None: - # 支持更新dataset的values数据 - df2 = pd.DataFrame({ - 'id': np.arange(0, 100), # id列，顺序整数 - **{f'z{i}': np.random.rand(100) for i in range(1, 11)} # x1到x10列，随机数 - }) - dataset2.update_values(values=df2) -if dataset1.storage_client is not None: - dataset1.update_values( - path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') - dataset1.load_values() + dataset2.storage_client = None + # dataset2.load_values() + if dataset2.storage_client is None: + # 支持更新dataset的values数据 + df2 = pd.DataFrame({ + 'id': np.arange(0, 100), # id列，顺序整数 + # x1到x10列，随机数 + **{f'z{i}': np.random.rand(100) for i in range(1, 11)} + }) + dataset2.update_values(values=df2) + if dataset1.storage_client is not None: + dataset1.update_values( + path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485') + dataset1.load_values() -# 构建 dataset context -dataset = DataContext(dataset1, dataset2) + # 构建 dataset context + dataset = DataContext(dataset1, dataset2) -# init the job context -project_name = "1" + # init the job context + project_name = "1" -psi_job_context = wedpr_ml_toolkit.build_job_context( - JobType.PSI, project_name, dataset, None, "id") -print(psi_job_context.participant_id_list, - psi_job_context.result_receiver_id_list) -# 执行psi任务 -psi_job_id = psi_job_context.submit() -psi_result = psi_job_context.fetch_job_result(psi_job_id, True) + psi_job_context = self.wedpr_ml_toolkit.build_job_context( + JobType.PSI, project_name, dataset, None, "id") + print(psi_job_context.participant_id_list, + psi_job_context.result_receiver_id_list) + # 执行psi任务 + psi_job_id = psi_job_context.submit() + psi_result = psi_job_context.fetch_job_result(psi_job_id, True) -# 初始化 -preprocessing_data = DataContext(dataset1) -preprocessing_job_context = wedpr_ml_toolkit.build_job_context( - JobType.PREPROCESSING, project_name, preprocessing_data, PreprocessingModelSetting()) -# 执行预处理任务 -fe_job_id = preprocessing_job_context.submit(dataset) -fe_result = preprocessing_job_context.fetch_job_result(fe_job_id, True) -print(preprocessing_job_context.participant_id_list, - preprocessing_job_context.result_receiver_id_list) + # 初始化 + preprocessing_data = DataContext(dataset1) + preprocessing_job_context = self.wedpr_ml_toolkit.build_job_context( + JobType.PREPROCESSING, project_name, preprocessing_data, PreprocessingModelSetting()) + # 执行预处理任务 + fe_job_id = preprocessing_job_context.submit(dataset) + fe_result = preprocessing_job_context.fetch_job_result(fe_job_id, True) + print(preprocessing_job_context.participant_id_list, + preprocessing_job_context.result_receiver_id_list) + + def test_query_job(self, job_id: str, block_until_finish): + job_result = self.wedpr_ml_toolkit.query_job_status( + job_id, block_until_finish) + print(f"#### query_job_status job_result: {job_result}") + job_detail_result = self.wedpr_ml_toolkit.query_job_detail( + job_id, block_until_finish) + print(f"#### query_job_detail job_detail_result: {job_detail_result}") + return (job_result, job_detail_result) + + +class TestMlToolkit(unittest.TestCase): + def test_query_jobs(self): + wrapper = WeDPRMlToolkitTestWrapper("config.properties") + # the success job case + success_job_id = "9630202187032582" + wrapper.test_query_job(success_job_id, False) + # wrapper.test_query_job(success_job_id, True) + # the fail job case + # failed_job_id = "9630156365047814" + # wrapper.test_query_job(success_job_id, False) + # wrapper.test_query_job(success_job_id, True) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/wedpr_ml_toolkit/transport/credential_generator.py b/python/wedpr_ml_toolkit/transport/credential_generator.py index 7800e74d..de028bcd 100644 --- a/python/wedpr_ml_toolkit/transport/credential_generator.py +++ b/python/wedpr_ml_toolkit/transport/credential_generator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import hashlib -from wedpr_ml_toolkit.common import utils -import time +from wedpr_ml_toolkit.common.utils import utils +from datetime import datetime class CredentialInfo: @@ -9,23 +9,24 @@ class CredentialInfo: NONCE_KEY = "nonce" TIMESTAMP_KEY = "timestamp" SIGNATURE_KEY = "signature" + HASH_ALGORITHM_KEY = "hashAlgorithm" - def __init__(self, access_key_id: str, nonce: str, timestamp: str, signature: str): + def __init__(self, access_key_id: str, nonce: str, timestamp: int, signature: str): self.access_key_id = access_key_id self.nonce = nonce self.timestamp = timestamp self.signature = signature + # use SHA3-256 algorithm + self.hash_algorithm = "SHA3-256" def to_dict(self): result = {} - result.update(CredentialInfo.ACCESS_ID_KEY, self.access_key_id) - result.update(CredentialInfo.NONCE_KEY, self.nonce) - result.update(CredentialInfo.TIMESTAMP_KEY, self.timestamp) - result.update(CredentialInfo.SIGNATURE_KEY, self.signature) - - def update_url_with_auth_info(self, url): - auth_params = self.to_dict() - return utils.add_params_to_url(auth_params) + result.update({CredentialInfo.ACCESS_ID_KEY: self.access_key_id}) + result.update({CredentialInfo.NONCE_KEY: self.nonce}) + result.update({CredentialInfo.TIMESTAMP_KEY: self.timestamp}) + result.update({CredentialInfo.SIGNATURE_KEY: self.signature}) + result.update({CredentialInfo.HASH_ALGORITHM_KEY: self.hash_algorithm}) + return result class CredentialGenerator: @@ -36,7 +37,8 @@ def __init__(self, access_key_id: str, access_key_secret: str, nonce_len=5): def generate_credential(self) -> CredentialInfo: nonce = utils.generate_nonce(self.nonce_len) - timestamp = int(time.time()) + # convert to the million-seconds timestamp + timestamp = int(datetime.now().timestamp() * 1000) # generate the signature signature = CredentialGenerator.generate_signature( self.access_key_id, self.access_key_secret, nonce, timestamp) @@ -46,10 +48,11 @@ def generate_credential(self) -> CredentialInfo: def generate_signature(access_key_id, access_key_secret, nonce, timestamp) -> str: anti_replay_info_hash = hashlib.sha3_256() # hash(access_key_id + nonce + timestamp) - anti_replay_info = f"{access_key_id}{nonce}{timestamp}" - anti_replay_info_hash.update(anti_replay_info) + anti_replay_info_hash.update( + bytes(access_key_id + nonce + str(timestamp), encoding='utf-8')) # hash(anti_replay_info + access_key_secret) signature_hash = hashlib.sha3_256() - signature_hash.update(anti_replay_info_hash.hexdigest()) - signature_hash.update(access_key_secret) + signature_hash.update( + bytes(anti_replay_info_hash.hexdigest(), encoding='utf-8')) + signature_hash.update(bytes(access_key_secret, encoding='utf-8')) return signature_hash.hexdigest() diff --git a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py index 9ffe3671..d711528b 100644 --- a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py +++ b/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import requests from wedpr_ml_toolkit.transport.credential_generator import CredentialGenerator +from wedpr_ml_toolkit.config.wedpr_ml_config import HttpConfig from wedpr_ml_toolkit.common.utils.constant import Constant import json class LoadBanlancer: - def __init__(self, remote_entrypoints): + def __init__(self, remote_entrypoints: []): if remote_entrypoints == None or len(remote_entrypoints) == 0: raise Exception(f"Must define the wedpr entrypoints") self.remote_entrypoints = remote_entrypoints @@ -18,22 +19,31 @@ def select(self, uri_path: str): self.round_robin_idx += 1 selected_entrypoint = self.remote_entrypoints[selected_idx % len( self.remote_entrypoints)] - return f"{selected_entrypoint}/${uri_path}" + return f"{selected_entrypoint}/{uri_path}" class WeDPREntryPoint: - def __init__(self, access_key_id: str, access_key_secret: str, remote_entrypoints, nonce_len: int = 5): + def __init__(self, access_key_id: str, access_key_secret: str, remote_entrypoints: [], http_config: HttpConfig, nonce_len: int = 5): self.credential_generator = CredentialGenerator( access_key_id, access_key_secret, nonce_len) + self.http_config = http_config self.loadbalancer = LoadBanlancer(remote_entrypoints) - def send_post_request(self, uri, params, headers, data): + def send_request(self, is_post: bool, uri, params, headers, data): credential_info = self.credential_generator.generate_credential() - url = credential_info.update_url_with_auth_info( - self.loadbalancer.select(uri)) + if params is None: + params = {} + params.update(credential_info.to_dict()) + url = self.loadbalancer.select(uri) if not headers: headers = {'content-type': 'application/json'} - response = requests.post(url, data=data, json=params, headers=headers) + response = None + if is_post: + response = requests.post(url, data=data, params=params, headers=headers, + timeout=self.http_config.timeout_seconds) + else: + response = requests.get(url, params=params, headers=headers, + timeout=self.http_config.timeout_seconds) if response.status_code != Constant.HTTP_STATUS_OK: raise Exception( f"send post request to {url} failed, response: {response.text}") diff --git a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py index 1fb1aeb0..8b3795a3 100644 --- a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py +++ b/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py @@ -2,15 +2,78 @@ from wedpr_ml_toolkit.common.utils.constant import Constant from wedpr_ml_toolkit.config.wedpr_ml_config import AuthConfig from wedpr_ml_toolkit.config.wedpr_ml_config import JobConfig +from wedpr_ml_toolkit.config.wedpr_ml_config import HttpConfig +from wedpr_ml_toolkit.common.utils.base_object import BaseObject import json import time - - -class JobInfo: - def __init__(self, job_type: str, project_name: str, param: str): - self.jobType = job_type +from enum import Enum +from typing import Any + + +class JobType(Enum): + PSI = "PSI", + PREPROCESSING = "PREPROCESSING", + FEATURE_ENGINEERING = "FEATURE_ENGINEERING", + XGB_TRAINING = "XGB_TRAINING", + XGB_PREDICTING = "XGB_PREDICTING" + + +class JobStatus(Enum): + Submitted = "Submitted", + Running = "Running", + RunFailed = "RunFailed", + RunSuccess = "RunSuccess", + WaitToRetry = "WaitToRetry", + WaitToKill = "WaitToKill", + Killing = "Killing", + Killed = "Killed", + ChainInProgress = "ChainInProgress" + + def run_success(self) -> bool: + if self.name == JobStatus.RunSuccess.name: + return True + return False + + def run_failed(self) -> bool: + if self.name == JobStatus.RunFailed.name: + return True + return False + + def scheduling(self) -> bool: + return (not self.run_success()) and (not self.run_failed()) + + @staticmethod + def get_job_status(job_status_tr: str): + try: + if job_status_tr is None or len(job_status_tr) == 0: + return None + return JobStatus[job_status_tr] + except: + return None + + +class JobInfo(BaseObject): + def __init__(self, job_id: str = None, job_type: JobType = None, project_name: str = None, param: str = None, **params: Any): + self.id = job_id + self.name = None + self.owner = None + self.ownerAgency = None + if job_type is not None: + self.jobType = job_type.name + else: + self.jobType = None + self.parties = None self.projectName = project_name self.param = param + self.status = None + self.result = None + self.createTime = None + self.lastUpdateTime = None + self.set_params(**params) + self.job_status = JobStatus.get_job_status(self.status) + + def __repr__(self): + return f"job_id: {self.id}, owner: {self.owner}, ownerAgency: {self.ownerAgency}, jobType: {self.jobType}, status: {self.status}" class JobParam: @@ -20,31 +83,71 @@ def __init__(self, job_info: JobInfo, task_parities, dataset_list): self.datasetList = dataset_list -class WeDPRResponse: - def __init__(self, code, msg, data): - self.code = code - self.msg = msg - self.data = data +class WeDPRResponse(BaseObject): + def __init__(self, **params: Any): + self.code = None + self.msg = None + self.data = None + self.set_params(**params) def success(self): - return self.code == 0 - + return self.code is not None and self.code == 0 -class QueryJobRequest: - def __init__(self, job_id): - # the job condition - self.job = {} - self.job.update("id", job_id) +class QueryJobRequest(BaseObject): + def __init__(self, job_info): + self.job = job_info -class WeDPRRemoteJobClient(WeDPREntryPoint): - def __init__(self, auth_config: AuthConfig, job_config: JobConfig): + def as_dict(self): + result = {} + if self.job is None: + return result + result.update({"job": self.job.as_dict()}) + return result + + def __repr__(self): + return f"job: {self.job}" + + +class JobListResponse(BaseObject): + def __init__(self, **params: Any): + self.jobs = [] + self.job_object_list = [] + self.total = None + self.set_params(**params) + for job_item in self.jobs: + self.job_object_list.append(JobInfo(**job_item)) + + def get_queried_job(self): + if len(self.job_object_list) == 0: + return None + return self.job_object_list[0] + + +class JobDetailResponse(BaseObject): + def __init__(self, job: JobInfo = None, **params: Any): + self.job = job + self.job_object = None + self.modelResultDetail = None + self.resultFileInfo = None + self.model = None + self.set_params(**params) + # deserialize the job_object + self.job_object = JobInfo(**self.job) + # TODO: deserialize the result + + def __repr__(self): + return f"job: {self.job_object}, modelResultDetail: {self.modelResultDetail}, resultFileInfo: {self.resultFileInfo}, model: {self.model}" + + +class WeDPRRemoteJobClient(WeDPREntryPoint, BaseObject): + def __init__(self, http_config: HttpConfig, auth_config: AuthConfig, job_config: JobConfig): if auth_config is None: raise Exception("Must define the auth config!") if job_config is None: raise Exception("Must define the job config") super().__init__(auth_config.access_key_id, auth_config.access_key_secret, - auth_config.remote_entrypoints, auth_config.nonce_len) + auth_config.get_remote_entrypoints_list(), http_config, auth_config.nonce_len) self.auth_config = auth_config self.job_config = job_config @@ -55,8 +158,8 @@ def get_job_config(self): return self.job_config def submit_job(self, job_params: JobParam) -> WeDPRResponse: - wedpr_response = self.send_post_request( - self.job_config._submit_job_uri, None, None, json.dumps(job_params)) + wedpr_response = self.send_request(True, + self.job_config._submit_job_uri, None, None, json.dumps(job_params)) submit_result = WeDPRResponse(**wedpr_response) # return the job_id if submit_result.success(): @@ -64,27 +167,51 @@ def submit_job(self, job_params: JobParam) -> WeDPRResponse: raise Exception( f"submit_job failed, code: {submit_result.code}, msg: {submit_result.msg}") - def _poll_task_status(self, job_id, token): + def query_job_detail(self, job_id, block_until_finish) -> JobDetailResponse: + job_result = self.poll_job_result(job_id, block_until_finish) + # failed case + if job_result == None or job_result.job_status == None or (not job_result.job_status.run_success()): + return JobDetailResponse(job=job_result, params=None) + # success case + params = {} + params["jobID"] = job_id + response_dict = self.execute_with_retry(self.send_request, + self.job_config.max_retries, + self.job_config.retry_delay_s, + False, + self.job_config.query_job_detail_uri, + params, + None, None) + wedpr_response = WeDPRResponse(**response_dict) + if not wedpr_response.success(): + raise Exception( + f"query_job_detail exception, job: {job_id}, code: {wedpr_response.code}, msg: {wedpr_response.msg}") + return JobDetailResponse(**(wedpr_response.data)) + + def poll_job_result(self, job_id, block_until_finish) -> JobInfo: while True: - wedpr_response = WeDPRResponse(self._send_request_with_retry( - self.send_post_request, self.job_config.query_job_status_uri, None, None, json.dumps(QueryJobRequest(job_id)))) - # TODO: check with status - if wedpr_response.success(): - return wedpr_response - else: + query_condition = JobInfo(job_id=job_id) + response_dict = self.execute_with_retry(self.send_request, + self.job_config.max_retries, + self.job_config.retry_delay_s, + True, + self.job_config.query_job_status_uri, + None, None, json.dumps(QueryJobRequest(job_info=query_condition).as_dict())) + wedpr_response = WeDPRResponse(**response_dict) + if not wedpr_response.success(): raise Exception( - f"_poll_task_status for job {job_id} failed, code: {wedpr_response.code}, msg: {wedpr_response.msg}") - time.sleep(self.job_config.polling_interval_s) - - def _send_request_with_retry(self, request_func, *args, **kwargs): - attempt = 0 - while attempt < self.max_retries: - try: - response = request_func(*args, **kwargs) - return response - except Exception as e: - attempt += 1 - if attempt < self.max_retries: - time.sleep(self.retry_delay_s) - else: - raise e + f"poll_job_result failed, job_id: {job_id}, code: {wedpr_response.code}, msg: {wedpr_response.msg}") + # check the result + result = JobListResponse(**(wedpr_response.data)) + result_job = result.get_queried_job() + if result_job is None: + raise Exception( + f"poll_job_result for the queried job {job_id} not exists!") + # run finished + if result_job.job_status.run_success() or result_job.job_status.run_failed(): + return result_job + # wait to finish + if block_until_finish: + time.sleep(self.job_config.polling_interval_s) + else: + return None diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py index a4db7de2..6d8b008e 100644 --- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py +++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py @@ -2,6 +2,8 @@ from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfig from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo +from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint from wedpr_ml_toolkit.context.job_context import JobType from wedpr_ml_toolkit.context.job_context import PSIJobContext @@ -16,7 +18,7 @@ class WeDPRMlToolkit: def __init__(self, config: WeDPRMlConfig): self.config = config self.remote_job_client = WeDPRRemoteJobClient( - self.config.auth_config, self.config.job_config) + self.config.http_config, self.config.auth_config, self.config.job_config) self.storage_entry_point = StorageEntryPoint(self.config.user_config, self.config.storage_config) @@ -32,24 +34,26 @@ def get_storage_entry_point(self) -> StorageEntryPoint: def submit(self, job_param): return self.remote_job_client.submit_job(job_param) - def query_job_status(self, job_id, block_until_success=False): - if not block_until_success: - return self.remote_job_client.query_job_result(job_id) - return self.remote_job_client.wait_job_finished() + def query_job_status(self, job_id, block_until_finish=False) -> JobInfo: + return self.remote_job_client.poll_job_result(job_id, block_until_finish) - def query_job_detail(self, job_id, block_until_success): - job_result = self.query_job_status(job_id, block_until_success) - # TODO: determine success or not here - return self.remote_job_client.query_job_detail(job_id) + def query_job_detail(self, job_id, block_until_finish=False) -> JobDetailResponse: + return self.remote_job_client.query_job_detail(job_id, block_until_finish) - def build_job_context(self, job_type: JobType, project_name: str, dataset: DataContext, model_setting=None, id_fields='id'): + def build_job_context(self, job_type: JobType, project_name: str, dataset: DataContext, model_setting=None, + id_fields='id'): if job_type == JobType.PSI: - return PSIJobContext(self.remote_job_client, project_name, dataset, self.config.agency_config.agency_name, id_fields) + return PSIJobContext(self.remote_job_client, project_name, dataset, self.config.agency_config.agency_name, + id_fields) if job_type == JobType.PREPROCESSING: - return PreprocessingJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + return PreprocessingJobContext(self.remote_job_client, project_name, model_setting, dataset, + self.config.agency_config.agency_name) if job_type == JobType.FEATURE_ENGINEERING: - return FeatureEngineeringJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + return FeatureEngineeringJobContext(self.remote_job_client, project_name, model_setting, dataset, + self.config.agency_config.agency_name) if job_type == JobType.XGB_TRAINING: - return SecureLGBMTrainingJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + return SecureLGBMTrainingJobContext(self.remote_job_client, project_name, model_setting, dataset, + self.config.agency_config.agency_name) if job_type == JobType.XGB_PREDICTING: - return SecureLGBMPredictJobContext(self.remote_job_client, project_name, model_setting, dataset, self.config.agency_config.agency_name) + return SecureLGBMPredictJobContext(self.remote_job_client, project_name, model_setting, dataset, + self.config.agency_config.agency_name) From 9e3172c77fb5968172a406e22b4790736fdf6df4 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Fri, 18 Oct 2024 15:44:56 +0800 Subject: [PATCH 065/120] add setup.py (#62) --- .../ppc_common/deps_services/hdfs_storage.py | 3 +- .../{__init__.py => CHANGELOG.md} | 0 python/wedpr_ml_toolkit/MANIFEST.in | 2 + python/wedpr_ml_toolkit/pyproject.toml | 87 ++++++++++++++++++ python/wedpr_ml_toolkit/requirements.txt | 4 + python/wedpr_ml_toolkit/setup.py | 62 +++++++++++++ .../{common => wedpr_ml_toolkit}/__init__.py | 0 .../wedpr_ml_toolkit/_version.py | 8 ++ .../common}/__init__.py | 0 .../common/utils}/__init__.py | 0 .../common/utils/base_object.py | 0 .../common/utils/constant.py | 0 .../common/utils/hdfs_storage_impl.py | 89 +++++++++++++++++++ .../common/utils/properies_parser.py | 0 .../common/utils/utils.py | 17 ++++ .../config}/__init__.py | 0 .../config/wedpr_ml_config.py | 0 .../config/wedpr_model_setting.py | 0 .../context}/__init__.py | 0 .../context/data_context.py | 0 .../context/job_context.py | 0 .../context/result}/__init__.py | 0 .../context/result/fe_result_context.py | 0 .../context/result/model_result_context.py | 0 .../context/result/psi_result_context.py | 0 .../context/result/result_context.py | 0 .../wedpr_ml_toolkit/toolkit/__init__.py | 0 .../toolkit/dataset_toolkit.py | 0 .../wedpr_ml_toolkit/transport/__init__.py | 0 .../transport/credential_generator.py | 0 .../transport/storage_entrypoint.py | 5 +- .../transport/wedpr_entrypoint.py | 0 .../transport/wedpr_remote_job_client.py | 0 .../wedpr_ml_toolkit.py | 0 34 files changed, 272 insertions(+), 5 deletions(-) rename python/wedpr_ml_toolkit/{__init__.py => CHANGELOG.md} (100%) create mode 100644 python/wedpr_ml_toolkit/MANIFEST.in create mode 100644 python/wedpr_ml_toolkit/pyproject.toml create mode 100644 python/wedpr_ml_toolkit/requirements.txt create mode 100644 python/wedpr_ml_toolkit/setup.py rename python/wedpr_ml_toolkit/{common => wedpr_ml_toolkit}/__init__.py (100%) create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py rename python/wedpr_ml_toolkit/{common/utils => wedpr_ml_toolkit/common}/__init__.py (100%) rename python/wedpr_ml_toolkit/{config => wedpr_ml_toolkit/common/utils}/__init__.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/common/utils/base_object.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/common/utils/constant.py (100%) create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/common/utils/properies_parser.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/common/utils/utils.py (68%) rename python/wedpr_ml_toolkit/{context => wedpr_ml_toolkit/config}/__init__.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/config/wedpr_ml_config.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/config/wedpr_model_setting.py (100%) rename python/wedpr_ml_toolkit/{context/result => wedpr_ml_toolkit/context}/__init__.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/data_context.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/job_context.py (100%) rename python/wedpr_ml_toolkit/{transport => wedpr_ml_toolkit/context/result}/__init__.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/result/fe_result_context.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/result/model_result_context.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/result/psi_result_context.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/context/result/result_context.py (100%) create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/toolkit/dataset_toolkit.py (100%) create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/__init__.py rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/transport/credential_generator.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/transport/storage_entrypoint.py (90%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/transport/wedpr_entrypoint.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/transport/wedpr_remote_job_client.py (100%) rename python/wedpr_ml_toolkit/{ => wedpr_ml_toolkit}/wedpr_ml_toolkit.py (100%) diff --git a/python/ppc_common/deps_services/hdfs_storage.py b/python/ppc_common/deps_services/hdfs_storage.py index 370ed2d3..d48fbc77 100644 --- a/python/ppc_common/deps_services/hdfs_storage.py +++ b/python/ppc_common/deps_services/hdfs_storage.py @@ -13,9 +13,8 @@ class HdfsStorage(StorageApi): DEFAULT_HDFS_USER = "ppc" DEFAULT_HDFS_USER_PATH = "/user/" + # endpoint: http://127.0.0.1:50070 def __init__(self, endpoint, hdfs_user, hdfs_home=None): - - # self.client = Client('http://127.0.0.1:9870') self.endpoint = endpoint self._user = common_func.get_config_value( "HDFS_USER", HdfsStorage.DEFAULT_HDFS_USER, hdfs_user, False) diff --git a/python/wedpr_ml_toolkit/__init__.py b/python/wedpr_ml_toolkit/CHANGELOG.md similarity index 100% rename from python/wedpr_ml_toolkit/__init__.py rename to python/wedpr_ml_toolkit/CHANGELOG.md diff --git a/python/wedpr_ml_toolkit/MANIFEST.in b/python/wedpr_ml_toolkit/MANIFEST.in new file mode 100644 index 00000000..3b258244 --- /dev/null +++ b/python/wedpr_ml_toolkit/MANIFEST.in @@ -0,0 +1,2 @@ +include CHANGELOG.md +include *requirements.txt diff --git a/python/wedpr_ml_toolkit/pyproject.toml b/python/wedpr_ml_toolkit/pyproject.toml new file mode 100644 index 00000000..dacbc3ad --- /dev/null +++ b/python/wedpr_ml_toolkit/pyproject.toml @@ -0,0 +1,87 @@ +# autoflake is used for autoformatting Python code +# +# ref: https://github.com/PyCQA/autoflake#readme +# +[tool.autoflake] +ignore-init-module-imports = true +remove-all-unused-imports = true +remove-duplicate-keys = true +#remove-unused-variables = true + + +# isort is used for autoformatting Python code +# +# ref: https://pycqa.github.io/isort/ +# +[tool.isort] +profile = "black" + + +# black is used for autoformatting Python code +# +# ref: https://black.readthedocs.io/en/stable/ +# +[tool.black] +skip-string-normalization = true +# target-version should be all supported versions, see +# https://github.com/psf/black/issues/751#issuecomment-473066811 +target_version = [ + "py38", + "py39", + "py310", + "py311", +] + + +# pytest is used for running Python based tests +# +# ref: https://docs.pytest.org/en/stable/ +# +[tool.pytest.ini_options] +addopts = "--verbose --color=yes --durations=10 --cov=wedpr_ml_toolkit" +asyncio_mode = "auto" +testpaths = [ + "test" +] + + +# pytest-cov / coverage is used to measure code coverage of tests +# +# ref: https://coverage.readthedocs.io/en/stable/config.html +# +[tool.coverage.run] +omit = [ + "test/**", +] + + +# tbump is used to simplify and standardize the release process when updating +# the version, making a git commit and tag, and pushing changes. +# +# ref: https://github.com/your-tools/tbump#readme +# +[tool.tbump] +github_url = "https://github.com/WeBankBlockchain/WeDPR-Component" + +[tool.tbump.version] +current = "1.0.0.dev" +regex = ''' + (?P\d+) + \. + (?P\d+) + \. + (?P\d+) + (?P

((a|b|rc)\d+)|)
+    \.?
+    (?P(?<=\.)dev\d*|)
+'''
+
+[tool.tbump.git]
+message_template = "Bump to {new_version}"
+tag_template = "{new_version}"
+
+[[tool.tbump.file]]
+src = "setup.py"
+
+[[tool.tbump.file]]
+src = "wedpr_ml_toolkit/_version.py"
diff --git a/python/wedpr_ml_toolkit/requirements.txt b/python/wedpr_ml_toolkit/requirements.txt
new file mode 100644
index 00000000..b6bc7e3b
--- /dev/null
+++ b/python/wedpr_ml_toolkit/requirements.txt
@@ -0,0 +1,4 @@
+hdfs
+requests~=2.31.0
+requests_toolbelt==0.9.1
+
diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
new file mode 100644
index 00000000..0ec8449b
--- /dev/null
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -0,0 +1,62 @@
+import sys
+
+from setuptools import find_packages, setup
+from setuptools.command.bdist_egg import bdist_egg
+
+
+class bdist_egg_disabled(bdist_egg):
+    """Disabled version of bdist_egg
+
+    Prevents setup.py install from performing setuptools' default easy_install,
+    which it should never ever do.
+    """
+
+    def run(self):
+        sys.exit(
+            "Aborting implicit building of eggs. Use `pip install .` to install from source."
+        )
+
+
+setup_args = dict(
+    name='wedpr_ml_toolkit',
+    packages=find_packages(),
+    version="1.0.0.dev",
+    description="wedpr-ml-toolkit: The ML toolkit for WeDPR",
+    long_description_content_type="text/markdown",
+    author="WeDPR Development Team",
+    author_email="wedpr@webank.com",
+    url="https://github.com/WeBankBlockchain/WeDPR-Component",
+    license="Apache-2.0",
+    platforms="Linux, Mac OS X",
+    keywords=['Interactive', 'Interpreter', 'Shell', 'Web'],
+    python_requires=">=3.8",
+    include_package_data=True,
+    classifiers=[
+        'Intended Audience :: Developers',
+        'Intended Audience :: System Administrators',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: Apache License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+    ],
+)
+
+setup_args['cmdclass'] = {
+    'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled,
+}
+
+setup_args['install_requires'] = install_requires = []
+with open('requirements.txt') as f:
+    for line in f.readlines():
+        req = line.strip()
+        if not req or req.startswith(('-e', '#')):
+            continue
+        install_requires.append(req)
+
+
+def main():
+    setup(**setup_args)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/python/wedpr_ml_toolkit/common/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/common/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/__init__.py
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py
new file mode 100644
index 00000000..ec0d7829
--- /dev/null
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py
@@ -0,0 +1,8 @@
+# __version__ should be updated using tbump, based on configuration in
+# pyproject.toml, according to instructions in RELEASE.md.
+#
+__version__ = "1.0.0.dev"
+
+# version_info looks like (1, 2, 3, "dev") if __version__ is 1.2.3.dev
+version_info = tuple(int(p) if p.isdigit()
+                     else p for p in __version__.split("."))
diff --git a/python/wedpr_ml_toolkit/common/utils/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/common/utils/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/__init__.py
diff --git a/python/wedpr_ml_toolkit/config/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/config/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/__init__.py
diff --git a/python/wedpr_ml_toolkit/common/utils/base_object.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py
similarity index 100%
rename from python/wedpr_ml_toolkit/common/utils/base_object.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py
diff --git a/python/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
similarity index 100%
rename from python/wedpr_ml_toolkit/common/utils/constant.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py
new file mode 100644
index 00000000..2e525d94
--- /dev/null
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py
@@ -0,0 +1,89 @@
+import os
+from typing import AnyStr
+
+from hdfs.client import InsecureClient
+from wedpr_ml_toolkit.common.utils import utils
+
+
+class HdfsStorageImpl:
+
+    DEFAULT_HDFS_USER = "ppc"
+    DEFAULT_HDFS_USER_PATH = "/user/"
+
+    # endpoint: http://127.0.0.1:50070
+    def __init__(self, endpoint, hdfs_user, hdfs_home=None):
+        self.endpoint = endpoint
+        self._user = hdfs_user
+        self._hdfs_storage_path = hdfs_home
+        if hdfs_home is None:
+            self._hdfs_storage_path = os.path.join(
+                HdfsStorage.DEFAULT_HDFS_USER_PATH, self._user)
+
+        self.client = InsecureClient(endpoint, user=self._user)
+        # print(self.client.list('/'))
+        # print(self.client.list('/user/root/'))
+
+    def get_home_path(self):
+        return self._hdfs_storage_path
+
+    def download_file(self, hdfs_path, local_file_path, enable_cache=False):
+        # hit the cache
+        if enable_cache is True and utils.file_exists(local_file_path):
+            return
+        if utils.file_exists(local_file_path):
+            utils.delete_file(local_file_path)
+        local_path = os.path.dirname(local_file_path)
+        if len(local_path) > 0 and not os.path.exists(local_path):
+            os.makedirs(local_path)
+        self.client.download(os.path.join(self._hdfs_storage_path,
+                             hdfs_path), local_file_path)
+        return
+
+    def upload_file(self, local_file_path, hdfs_path):
+        self.make_file_path(hdfs_path)
+        self.client.upload(os.path.join(self._hdfs_storage_path, hdfs_path),
+                           local_file_path, overwrite=True)
+        return
+
+    def make_file_path(self, hdfs_path):
+        hdfs_dir = os.path.dirname(hdfs_path)
+        if self.client.status(os.path.join(self._hdfs_storage_path, hdfs_dir), strict=False) is None:
+            self.client.makedirs(os.path.join(
+                self._hdfs_storage_path, hdfs_dir))
+        return
+
+    def delete_file(self, hdfs_path):
+        self.client.delete(os.path.join(
+            self._hdfs_storage_path, hdfs_path), recursive=True)
+        return
+
+    def save_data(self, data: AnyStr, hdfs_path):
+        self.make_file_path(hdfs_path)
+        self.client.write(os.path.join(self._hdfs_storage_path,
+                          hdfs_path), data, overwrite=True)
+        return
+
+    def get_data(self, hdfs_path) -> AnyStr:
+        with self.client.read(os.path.join(self._hdfs_storage_path, hdfs_path)) as reader:
+            content = reader.read()
+        return content
+
+    def mkdir(self, hdfs_dir):
+        self.client.makedirs(hdfs_dir)
+
+    def file_existed(self, hdfs_path):
+        if self.client.status(os.path.join(self._hdfs_storage_path, hdfs_path), strict=False) is None:
+            return False
+        return True
+
+    def file_rename(self, old_hdfs_path, hdfs_path):
+        old_path = os.path.join(self._hdfs_storage_path, old_hdfs_path)
+        new_path = os.path.join(self._hdfs_storage_path, hdfs_path)
+        # return for the file not exists
+        if not self.file_existed(old_path):
+            return
+        parent_path = os.path.dirname(new_path)
+        if len(parent_path) > 0 and not self.file_existed(parent_path):
+            self.mkdir(parent_path)
+        self.client.rename(old_path, new_path)
+        return
diff --git a/python/wedpr_ml_toolkit/common/utils/properies_parser.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/properies_parser.py
similarity index 100%
rename from python/wedpr_ml_toolkit/common/utils/properies_parser.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/properies_parser.py
diff --git a/python/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
similarity index 68%
rename from python/wedpr_ml_toolkit/common/utils/utils.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
index 27795c8a..303c8dae 100644
--- a/python/wedpr_ml_toolkit/common/utils/utils.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 import uuid
 from enum import Enum
+import shutil
+import os
 import random
 from wedpr_ml_toolkit.common.utils.constant import Constant
 from urllib.parse import urlencode, urlparse, parse_qs, quote
@@ -27,3 +29,18 @@ def add_params_to_url(url, params):
         query_params[key] = value
     new_query = urlencode(query_params, doseq=True)
     return parsed_url._replace(query=new_query).geturl()
+
+
+def file_exists(_file):
+    if os.path.exists(_file) and os.path.isfile(_file):
+        return True
+    return False
+
+
+def delete_file(path):
+    if os.path.isfile(path):
+        os.remove(path)
+    elif os.path.isdir(path):
+        shutil.rmtree(path)
+    else:
+        raise (Exception(' path not exisited ! path => %s', path))
diff --git a/python/wedpr_ml_toolkit/context/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/__init__.py
diff --git a/python/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
similarity index 100%
rename from python/wedpr_ml_toolkit/config/wedpr_ml_config.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
diff --git a/python/wedpr_ml_toolkit/config/wedpr_model_setting.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py
similarity index 100%
rename from python/wedpr_ml_toolkit/config/wedpr_model_setting.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py
diff --git a/python/wedpr_ml_toolkit/context/result/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/result/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/__init__.py
diff --git a/python/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/data_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
diff --git a/python/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/job_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
diff --git a/python/wedpr_ml_toolkit/transport/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/__init__.py
similarity index 100%
rename from python/wedpr_ml_toolkit/transport/__init__.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/__init__.py
diff --git a/python/wedpr_ml_toolkit/context/result/fe_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/result/fe_result_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
diff --git a/python/wedpr_ml_toolkit/context/result/model_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/result/model_result_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
diff --git a/python/wedpr_ml_toolkit/context/result/psi_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/result/psi_result_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
diff --git a/python/wedpr_ml_toolkit/context/result/result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
similarity index 100%
rename from python/wedpr_ml_toolkit/context/result/result_context.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
similarity index 100%
rename from python/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/wedpr_ml_toolkit/transport/credential_generator.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/credential_generator.py
similarity index 100%
rename from python/wedpr_ml_toolkit/transport/credential_generator.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/credential_generator.py
diff --git a/python/wedpr_ml_toolkit/transport/storage_entrypoint.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
similarity index 90%
rename from python/wedpr_ml_toolkit/transport/storage_entrypoint.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
index 33cfd5c8..8ba01aa9 100644
--- a/python/wedpr_ml_toolkit/transport/storage_entrypoint.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
@@ -1,8 +1,7 @@
 import pandas as pd
 import io
 
-from ppc_common.deps_services import storage_loader
-from ppc_common.deps_services.hdfs_storage import HdfsStorage
+from wedpr_ml_toolkit.common.utils.hdfs_storage_impl import HdfsStorageImpl
 from wedpr_ml_toolkit.config.wedpr_ml_config import StorageConfig
 from wedpr_ml_toolkit.config.wedpr_ml_config import UserConfig
 
@@ -11,7 +10,7 @@ class StorageEntryPoint:
     def __init__(self, user_config: UserConfig, storage_config: StorageConfig):
         self.storage_config = storage_config
         self.user_config = user_config
-        self.storage_client = HdfsStorage(
+        self.storage_client = HdfsStorageImpl(
             self.storage_config.storage_endpoint, self.user_config.user, self.user_config.get_workspace_path())
 
     def upload(self, dataframe, hdfs_path):
diff --git a/python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_entrypoint.py
similarity index 100%
rename from python/wedpr_ml_toolkit/transport/wedpr_entrypoint.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_entrypoint.py
diff --git a/python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
similarity index 100%
rename from python/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
similarity index 100%
rename from python/wedpr_ml_toolkit/wedpr_ml_toolkit.py
rename to python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py

From 1562aa78026d70c25c5c2e7637f551e83b2a1fa9 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 18 Oct 2024 17:41:32 +0800
Subject: [PATCH 066/120] wedpr_python_gateway_sdk support install (#63)

---
 cpp/cmake/TargetSettings.cmake                   |  2 +-
 .../bindings/{libs/__init__.py => CHANGELOG.md}  |  0
 .../sdk-wrapper/python/bindings/MANIFEST.in      |  3 +++
 .../bindings/{transport => demo}/__init__.py     |  0
 .../{transport => }/demo/transport_demo.py       | 14 +++++++-------
 .../api/__init__.py => requirements.txt}         |  0
 .../sdk-wrapper/python/bindings/setup.py         |  6 +++++-
 .../bindings/transport/impl/message_factory.py   | 11 -----------
 .../{ => wedpr_python_gateway_sdk}/__init__.py   |  0
 .../libs}/__init__.py                            |  0
 .../libs/_wedpr_python_transport.py              |  2 +-
 .../transport}/__init__.py                       |  0
 .../transport/api}/__init__.py                   |  0
 .../transport/api/message_api.py                 |  0
 .../transport/api/transport_api.py               |  2 +-
 .../transport/generated}/__init__.py             |  0
 .../generated/wedpr_python_transport.py          |  2 +-
 .../transport/impl/__init__.py                   |  0
 .../transport/impl/message_factory.py            | 11 +++++++++++
 .../transport/impl/message_impl.py               |  8 ++++----
 .../transport/impl/route_info_builder.py         |  4 ++--
 .../transport/impl/transport.py                  | 16 ++++++++--------
 .../transport/impl/transport_config.py           |  8 ++++----
 .../transport/impl/transport_loader.py           |  6 +++---
 .../wedpr_python_gateway_sdk/utils/__init__.py   |  0
 .../utils/lib_loader.py                          |  0
 .../utils/tests/test_lib_loader.py               |  1 +
 .../python/swig/wedpr_python_transport.i         |  2 +-
 28 files changed, 53 insertions(+), 45 deletions(-)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{libs/__init__.py => CHANGELOG.md} (100%)
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/MANIFEST.in
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport => demo}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport => }/demo/transport_demo.py (88%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport/api/__init__.py => requirements.txt} (100%)
 delete mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport/demo => wedpr_python_gateway_sdk/libs}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/libs/_wedpr_python_transport.py (85%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport/generated => wedpr_python_gateway_sdk/transport}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{transport/impl => wedpr_python_gateway_sdk/transport/api}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/api/message_api.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/api/transport_api.py (93%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{utils => wedpr_python_gateway_sdk/transport/generated}/__init__.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/generated/wedpr_python_transport.py (99%)
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/__init__.py
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_factory.py
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/impl/message_impl.py (88%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/impl/route_info_builder.py (75%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/impl/transport.py (83%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/impl/transport_config.py (80%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/transport/impl/transport_loader.py (57%)
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/__init__.py
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/utils/lib_loader.py (100%)
 rename cpp/wedpr-transport/sdk-wrapper/python/bindings/{ => wedpr_python_gateway_sdk}/utils/tests/test_lib_loader.py (75%)

diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake
index 9a2ebfab..1eabf08e 100644
--- a/cpp/cmake/TargetSettings.cmake
+++ b/cpp/cmake/TargetSettings.cmake
@@ -139,7 +139,7 @@ set(WEDPR_TRANSPORT_SDK_TARGET wedpr-transport-sdk)
 # ==== the swig wrapper ===== 
 set(WEDPR_PYTHON_TRANSPORT "wedpr_python_transport")
 set(WEDPR_PYTHON_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python/generated/${WEDPR_PYTHON_TRANSPORT})
-set(WEDPR_PYTHON_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/python/bindings/libs)
+set(WEDPR_PYTHON_TRANSPORT_LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs)
 
 set(WEDPR_JAVA_TRANSPORT "wedpr_java_transport_jni")
 set(WEDPR_JAVA_TRANSPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/java/generated/${WEDPR_JAVA_TRANSPORT})
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/CHANGELOG.md
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/CHANGELOG.md
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/MANIFEST.in b/cpp/wedpr-transport/sdk-wrapper/python/bindings/MANIFEST.in
new file mode 100644
index 00000000..98b18675
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/MANIFEST.in
@@ -0,0 +1,3 @@
+include CHANGELOG.md
+include *requirements.txt
+include ./wedpr_python_gateway_sdk/libs/libwedpr_python_transport*
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
similarity index 88%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
index 55571d59..bce7efff 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/transport_demo.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
@@ -1,16 +1,16 @@
 # -*- coding: utf-8 -*-
 # Note: here can't be refactored by autopep
-import os
 import sys
+import os
 root_path = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(root_path, "../.."))
-
+sys.path.append(os.path.join(root_path, "../"))
 
-import argparse
-from transport.impl.transport import Transport
-from transport.impl.transport_config import TransportConfig
-from transport.impl.transport_loader import TransportLoader
 import time
+from wedpr_python_gateway_sdk.transport.impl.transport_loader import TransportLoader
+from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
+from wedpr_python_gateway_sdk.transport.impl.transport import Transport
+import argparse
+import sys
 
 
 def parse_args():
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/requirements.txt
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/requirements.txt
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
index e60894f8..23423c02 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr-python-gateway-sdk',
     packages=find_packages(),
-    version="1.0.0-rc1",
+    version="1.0.0-rc1-20241018",
     description="wedpr-python-gateway-sdk: The gateway sdk for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",
@@ -31,6 +31,10 @@ def run(self):
     keywords=['Interactive', 'Interpreter', 'Shell', 'Web'],
     python_requires=">=3.8",
     include_package_data=True,
+    package_data={
+        # the library setting
+        '': ['wedpr_python_gateway_sdk/libs/libwedpr_python_transport.*'],
+    },
     classifiers=[
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py
deleted file mode 100644
index 59f5ef68..00000000
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_factory.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from transport.impl.message_impl import MessageImpl
-from transport.api.message_api import MessageAPI
-from transport.generated.wedpr_python_transport import Message
-
-
-class MessageFactory:
-    @staticmethod
-    def build(message: Message) -> MessageAPI:
-        if message is None:
-            return None
-        return MessageImpl(message)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/demo/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/_wedpr_python_transport.py
similarity index 85%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/_wedpr_python_transport.py
index 1605caa8..fb07b92a 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/libs/_wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/_wedpr_python_transport.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import shutil
 import pkg_resources
-from utils.lib_loader import LibLoader
+from wedpr_python_gateway_sdk.utils.lib_loader import LibLoader
 
 
 def __bootstrap__():
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/message_api.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/message_api.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/message_api.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
similarity index 93%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
index 8167f87c..a61510a6 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/api/transport_api.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from abc import ABC, abstractmethod
-from transport.api.message_api import MessageAPI
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
 
 
 class TransportAPI(ABC):
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/__init__.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/__init__.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/__init__.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
similarity index 99%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 16d734a2..83de8c13 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -8,7 +8,7 @@
 from sys import version_info as _swig_python_version_info
 
 # Import the low-level C/C++ module
-from libs import _wedpr_python_transport
+from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 
 
 try:
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_factory.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_factory.py
new file mode 100644
index 00000000..1ee38f3c
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_factory.py
@@ -0,0 +1,11 @@
+from wedpr_python_gateway_sdk.transport.impl.message_impl import MessageImpl
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Message
+
+
+class MessageFactory:
+    @staticmethod
+    def build(message: Message) -> MessageAPI:
+        if message is None:
+            return None
+        return MessageImpl(message)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
similarity index 88%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
index 218462ba..51a8ef65 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/message_impl.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
-from transport.api.message_api import MessageHeaderAPI
-from transport.api.message_api import MessageAPI
-from transport.generated.wedpr_python_transport import Message
-from transport.generated.wedpr_python_transport import MessageHeader
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageHeaderAPI
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Message
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import MessageHeader
 
 
 class MessageHeaderImpl(MessageHeaderAPI):
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/route_info_builder.py
similarity index 75%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/route_info_builder.py
index d7fe8037..498e78b7 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/route_info_builder.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/route_info_builder.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from transport.generated.wedpr_python_transport import MessageOptionalHeaderBuilder
-from transport.generated.wedpr_python_transport import MessageOptionalHeader
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import MessageOptionalHeaderBuilder
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import MessageOptionalHeader
 
 
 class RouteInfoBuilder:
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
similarity index 83%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
index b6239b6c..86b6695d 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 
-from transport.generated.wedpr_python_transport import TransportBuilder
-from transport.generated.wedpr_python_transport import Transport
-from transport.generated.wedpr_python_transport import Error
-from transport.api.message_api import MessageAPI
-from transport.impl.route_info_builder import RouteInfoBuilder
-from transport.impl.message_factory import MessageFactory
-from transport.generated.wedpr_python_transport import MessageOptionalHeader
-from transport.api.transport_api import TransportAPI
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Transport
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Error
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
+from wedpr_python_gateway_sdk.transport.impl.route_info_builder import RouteInfoBuilder
+from wedpr_python_gateway_sdk.transport.impl.message_factory import MessageFactory
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import MessageOptionalHeader
+from wedpr_python_gateway_sdk.transport.api.transport_api import TransportAPI
 
 from enum import Enum
 import signal
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
similarity index 80%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
index 9b7d7487..0c15b0b8 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_config.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 
-from transport.generated.wedpr_python_transport import FrontConfig
-from transport.generated.wedpr_python_transport import TransportBuilder
-from transport.generated.wedpr_python_transport import EndPoint
-from transport.generated.wedpr_python_transport import GrpcConfig
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import FrontConfig
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import EndPoint
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import GrpcConfig
 
 
 class TransportConfig:
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
similarity index 57%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
index 02b0c547..b6e6492e 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/transport/impl/transport_loader.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-from transport.impl.transport_config import TransportConfig
-from transport.generated.wedpr_python_transport import TransportBuilder
-from transport.impl.transport import Transport
+from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
+from wedpr_python_gateway_sdk.transport.impl.transport import Transport
 import signal
 
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/__init__.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/lib_loader.py
similarity index 100%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/lib_loader.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/lib_loader.py
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/tests/test_lib_loader.py
similarity index 75%
rename from cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py
rename to cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/tests/test_lib_loader.py
index 8de1e1cd..025c6f32 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/utils/tests/test_lib_loader.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/tests/test_lib_loader.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
+from wedpr_python_gateway_sdk.utils.lib_loader import LibLoader
 
 
 class TestLibLoader(unittest.TestCase):
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index 242d681f..7ed79823 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -1,7 +1,7 @@
 %define MODULEIMPORT
 "
 # Import the low-level C/C++ module
-from libs import _wedpr_python_transport
+from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 "
 %enddef
  

From d5549deeee05cdc2892b3c3a400b007237f67bae Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 22 Oct 2024 01:20:16 +0800
Subject: [PATCH 067/120] use wedpr_gateway_sdk replace grpc (#64)

* optimize transport builder

* use wedpr_gateway_sdk replace grpc

* add selectRouter to gateway

* ppc_model adapt to gateway

* update script
---
 cpp/CMakeLists.txt                            |   15 +-
 cpp/ppc-framework/Helper.h                    |   14 +-
 cpp/ppc-framework/front/IFront.h              |    4 +
 cpp/ppc-framework/gateway/IGateway.h          |    4 +
 cpp/ppc-framework/protocol/INodeInfo.h        |    3 +-
 .../ppc-mpc/src/CMakeLists.txt                |    2 +-
 .../grpc/client/GatewayClient.cpp             |   24 +
 .../grpc/client/GatewayClient.h               |    3 +
 .../grpc/client/RemoteFrontBuilder.cpp        |    2 +-
 .../grpc/server/GatewayServer.cpp             |   30 +-
 .../grpc/server/GatewayServer.h               |    5 +
 cpp/wedpr-protocol/proto/pb/Service.proto     |   10 +
 .../protobuf/src/NodeInfoImpl.h               |    5 +
 .../protobuf/src/RequestConverter.h           |   30 +-
 .../ppc-front/ppc-front/FrontImpl.h           |    7 +
 .../ppc-gateway/CMakeLists.txt                |    5 +-
 .../ppc-gateway/gateway/GatewayImpl.cpp       |    4 +-
 .../ppc-gateway/gateway/GatewayImpl.h         |    6 +
 .../gateway/router/GatewayNodeInfo.h          |    5 +
 .../gateway/router/GatewayNodeInfoImpl.cpp    |    2 +-
 .../ppc-gateway/gateway/router/LocalRouter.h  |    1 -
 .../gateway/router/PeerRouterTable.cpp        |   67 +-
 .../gateway/router/PeerRouterTable.h          |   19 +-
 .../wedpr/sdk/jni/demo/TransportDemo.java     |   12 +-
 .../wedpr/sdk/jni/generated/FrontConfig.java  |   12 +-
 .../wedpr/sdk/jni/generated/IFront.java       |   11 +
 ...SWIGTYPE_p_std__vectorT_std__string_t.java |   30 -
 .../wedpr/sdk/jni/generated/StringVec.java    |  163 +
 .../generated/wedpr_java_transportJNI.java    |   39 +-
 .../sdk/jni/transport/IMessageBuilder.java    |    9 +
 .../sdk/jni/transport/WeDPRTransport.java     |    5 +
 .../sdk/jni/transport/impl/TransportImpl.java |   30 +
 .../src/wedpr_java_transportJAVA_wrap.cxx     |  391 ++-
 .../java/swig/wedpr_java_transport.i          |    2 +
 .../sdk-wrapper/python/CMakeLists.txt         |    4 +-
 .../python/bindings/demo/transport_demo.py    |   29 +-
 .../generated/wedpr_python_transport.py       |  114 +
 .../transport/impl/message_impl.py            |    6 +
 .../transport/impl/transport.py               |   29 +-
 .../transport/impl/transport_loader.py        |   65 +-
 .../utils/base_object.py                      |   15 +
 .../wedpr_python_gateway_sdk/utils/utils.py   |    5 +
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 2611 +++++++++++++++--
 .../python/swig/wedpr_python_transport.i      |    2 +
 python/ppc_common/ppc_protos/ppc_model.proto  |   27 -
 python/ppc_model/common/initializer.py        |  121 +-
 .../common/mock/mock_model_transport.py       |   45 +
 .../ppc_model/common/mock/rpc_client_mock.py  |   31 -
 python/ppc_model/common/model_result.py       |   34 +-
 python/ppc_model/common/model_setting.py      |   12 +-
 python/ppc_model/conf/application-sample.yml  |   14 +-
 python/ppc_model/conf/logging.conf            |    2 +-
 python/ppc_model/datasets/dataset.py          |    8 +-
 .../test/feature_engineering_unittest.py      |   31 +-
 .../vertical/active_party.py                  |   40 +-
 .../vertical/passive_party.py                 |   36 +-
 python/ppc_model/interface/model_base.py      |    4 +
 python/ppc_model/network/grpc/__init__.py     |    0
 python/ppc_model/network/grpc/grpc_client.py  |   82 -
 python/ppc_model/network/grpc/grpc_server.py  |   17 -
 python/ppc_model/network/stub.py              |  219 --
 .../ppc_model/network/test/stub_unittest.py   |   83 -
 .../network/wedpr_model_transport.py          |  103 +
 .../network/wedpr_model_transport_api.py      |   23 +
 python/ppc_model/ppc_model_app.py             |   90 +-
 .../test_secure_lgbm_performance_training.py  |   31 +-
 .../test/test_secure_lgbm_training.py         |   37 +-
 .../ppc_model/secure_lgbm/vertical/booster.py |   53 +-
 python/ppc_model/secure_lr/test/test_lr.py    |   21 +-
 .../test_secure_lr_performance_training.py    |   33 +-
 .../secure_lr/test/test_secure_lr_training.py |   42 +-
 .../ppc_model/secure_lr/vertical/booster.py   |   49 +-
 python/ppc_model/task/task_manager.py         |   14 +-
 .../task/test/task_manager_unittest.py        |   38 +-
 python/ppc_model/tools/start.sh               |   63 +-
 python/ppc_model/tools/stop.sh                |   53 +-
 python/ppc_model_gateway/__init__.py          |    0
 python/ppc_model_gateway/clients/__init__.py  |    0
 .../clients/client_manager.py                 |   57 -
 .../conf/application-sample.yml               |   19 -
 python/ppc_model_gateway/conf/logging.conf    |   40 -
 python/ppc_model_gateway/config.py            |   58 -
 .../ppc_model_gateway/endpoints/__init__.py   |    0
 .../endpoints/node_to_partner.py              |   36 -
 .../endpoints/partner_to_node.py              |   34 -
 .../endpoints/response_builder.py             |    8 -
 .../ppc_model_gateway_app.py                  |   93 -
 python/ppc_model_gateway/test/__init__.py     |    0
 python/ppc_model_gateway/test/client.py       |   41 -
 python/ppc_model_gateway/test/server.py       |   37 -
 python/ppc_model_gateway/tools/gen_cert.sh    |   70 -
 python/ppc_model_gateway/tools/start.sh       |   32 -
 python/ppc_model_gateway/tools/stop.sh        |   11 -
 93 files changed, 3998 insertions(+), 1845 deletions(-)
 delete mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/StringVec.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/base_object.py
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/utils.py
 create mode 100644 python/ppc_model/common/mock/mock_model_transport.py
 delete mode 100644 python/ppc_model/common/mock/rpc_client_mock.py
 delete mode 100644 python/ppc_model/network/grpc/__init__.py
 delete mode 100644 python/ppc_model/network/grpc/grpc_client.py
 delete mode 100644 python/ppc_model/network/grpc/grpc_server.py
 delete mode 100644 python/ppc_model/network/stub.py
 delete mode 100644 python/ppc_model/network/test/stub_unittest.py
 create mode 100644 python/ppc_model/network/wedpr_model_transport.py
 create mode 100644 python/ppc_model/network/wedpr_model_transport_api.py
 delete mode 100644 python/ppc_model_gateway/__init__.py
 delete mode 100644 python/ppc_model_gateway/clients/__init__.py
 delete mode 100644 python/ppc_model_gateway/clients/client_manager.py
 delete mode 100644 python/ppc_model_gateway/conf/application-sample.yml
 delete mode 100644 python/ppc_model_gateway/conf/logging.conf
 delete mode 100644 python/ppc_model_gateway/config.py
 delete mode 100644 python/ppc_model_gateway/endpoints/__init__.py
 delete mode 100644 python/ppc_model_gateway/endpoints/node_to_partner.py
 delete mode 100644 python/ppc_model_gateway/endpoints/partner_to_node.py
 delete mode 100644 python/ppc_model_gateway/endpoints/response_builder.py
 delete mode 100644 python/ppc_model_gateway/ppc_model_gateway_app.py
 delete mode 100644 python/ppc_model_gateway/test/__init__.py
 delete mode 100644 python/ppc_model_gateway/test/client.py
 delete mode 100644 python/ppc_model_gateway/test/server.py
 delete mode 100644 python/ppc_model_gateway/tools/gen_cert.sh
 delete mode 100644 python/ppc_model_gateway/tools/start.sh
 delete mode 100644 python/ppc_model_gateway/tools/stop.sh

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 67650d29..74642c67 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -76,16 +76,21 @@ endif()
 set(TRANSPORT_SDK_SOURCE_LIST 
     wedpr-protocol 
     wedpr-transport/ppc-front 
-    wedpr-transport/sdk wedpr-transport/sdk-wrapper)
+    wedpr-transport/sdk)
+
+set(TRANSPORT_SDK_TOOLKIT_SOURCE_LIST
+    ${TRANSPORT_SDK_SOURCE_LIST} 
+    wedpr-transport/sdk-wrapper)
 
 set(ALL_SOURCE_LIST 
         ${SDK_SOURCE_LIST} 
+        ${TRANSPORT_SDK_SOURCE_LIST}
         wedpr-crypto/ppc-crypto 
-        wedpr-protocol
         wedpr-helper/libhelper wedpr-helper/ppc-tools
         wedpr-storage/ppc-io wedpr-storage/ppc-storage
-        wedpr-transport/ppc-gateway wedpr-transport/ppc-front 
-        wedpr-transport/ppc-http wedpr-transport/ppc-rpc wedpr-transport/sdk
+        wedpr-transport/ppc-gateway 
+        wedpr-transport/ppc-rpc
+        wedpr-transport/ppc-http
         wedpr-computing/ppc-psi wedpr-computing/ppc-mpc wedpr-computing/ppc-pir ${CEM_SOURCE}
         wedpr-initializer wedpr-main)
 
@@ -96,7 +101,7 @@ elseif(BUILD_UDF)
 elseif(BUILD_SDK)
     add_sources("${SDK_SOURCE_LIST}")
 elseif(BUILD_WEDPR_TOOLKIT)
-    add_sources("${TRANSPORT_SDK_SOURCE_LIST}")
+    add_sources("${TRANSPORT_SDK_TOOLKIT_SOURCE_LIST}")
 endif()
 ########### set the sources end ########### 
 
diff --git a/cpp/ppc-framework/Helper.h b/cpp/ppc-framework/Helper.h
index d084cff7..6684fa04 100644
--- a/cpp/ppc-framework/Helper.h
+++ b/cpp/ppc-framework/Helper.h
@@ -42,7 +42,17 @@ inline std::string_view printP2PIDElegantly(std::string_view p2pId) noexcept
 template 
 inline std::string_view printNodeID(T const& nodeID)
 {
-    size_t offset = nodeID.size() >= 15 ? 15 : nodeID.size();
-    return std::string_view((const char*)nodeID.data(), offset);
+    return std::string_view((const char*)nodeID.data(), nodeID.size());
+}
+
+template 
+inline std::string printCollection(T const& collection)
+{
+    std::ostringstream oss;
+    for (auto const& it : collection)
+    {
+        oss << it << ",";
+    }
+    return oss.str();
 }
 }  // namespace ppc
\ No newline at end of file
diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h
index 5e0a249a..b0fa8876 100644
--- a/cpp/ppc-framework/front/IFront.h
+++ b/cpp/ppc-framework/front/IFront.h
@@ -260,6 +260,10 @@ class IFront : virtual public IFrontClient
     virtual void registerComponent(std::string const& component) = 0;
     virtual void unRegisterComponent(std::string const& component) = 0;
 
+    // get the target nodeList according to the routeInfo
+    virtual std::vector selectNodesByRoutePolicy(
+        int16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) = 0;
+
 private:
     ppc::protocol::ReceiveMsgFunc populateErrorCallback(ErrorCallback::Ptr errorCallback)
     {
diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h
index 2b5431b0..6b43d101 100644
--- a/cpp/ppc-framework/gateway/IGateway.h
+++ b/cpp/ppc-framework/gateway/IGateway.h
@@ -75,6 +75,10 @@ class IGateway
         bcos::bytesConstRef nodeID, std::string const& topic) = 0;
     virtual bcos::Error::Ptr unRegisterTopic(
         bcos::bytesConstRef nodeID, std::string const& topic) = 0;
+
+    // get the target nodeList according to the routeInfo
+    virtual std::vector selectNodesByRoutePolicy(ppc::protocol::RouteType routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) = 0;
 };
 
 }  // namespace ppc::gateway
diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h
index 37f8b772..6dbf5281 100644
--- a/cpp/ppc-framework/protocol/INodeInfo.h
+++ b/cpp/ppc-framework/protocol/INodeInfo.h
@@ -49,6 +49,7 @@ class INodeInfo
     virtual void setComponents(std::set const& components) = 0;
     virtual bool addComponent(std::string const& component) = 0;
     virtual bool eraseComponent(std::string const& component) = 0;
+    virtual bool componentExist(std::string const& component) const = 0;
     virtual std::set const& components() const = 0;
     virtual std::vector copiedComponents() const = 0;
 
@@ -88,7 +89,7 @@ inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo)
     stringstream << LOG_KV("endPoint", nodeInfo->endPoint())
                  << LOG_KV("nodeID", printNodeID(nodeInfo->nodeID()));
     std::string components = "";
-    for (auto const& it : nodeInfo->components())
+    for (auto const& it : nodeInfo->copiedComponents())
     {
         components = components + it + ",";
     }
diff --git a/cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt
index 1f218fe1..a880eea0 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-mpc/src/CMakeLists.txt
@@ -1,4 +1,4 @@
 file(GLOB_RECURSE SOURCES "*.cpp" "*.h")
 
 add_library(${MPC_TARGET} ${SOURCES})
-target_link_libraries(${MPC_TARGET} PUBLIC ${IO_TARGET})
+target_link_libraries(${MPC_TARGET} PUBLIC ${IO_TARGET} ${TOOLS_TARGET} jsoncpp_static)
diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
index b8108dbe..850d70f9 100644
--- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
@@ -53,6 +53,30 @@ void GatewayClient::asyncSendMessage(RouteType routeType,
             Status status) { callback(toError(status, *response)); });
 }
 
+std::vector GatewayClient::selectNodesByRoutePolicy(
+    RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo)
+{
+    std::unique_ptr request(
+        generateSelectRouteRequest(routeType, routeInfo));
+    auto context = std::make_shared();
+    auto response = std::make_shared();
+    // lambda keeps the lifecycle for clientContext
+    auto status = m_stub->selectNodesByRoutePolicy(context.get(), *request, response.get());
+    if (!status.ok())
+    {
+        throw std::runtime_error(
+            "selectNodesByRoutePolicy failed, code: " + std::to_string(status.error_code()) +
+            ", msg: " + status.error_message());
+    }
+    if (response->error().errorcode() != 0)
+    {
+        throw std::runtime_error("selectNodesByRoutePolicy failed, code: " +
+                                 std::to_string(response->error().errorcode()) +
+                                 ", msg: " + response->error().errormessage());
+    }
+    return std::vector(response->nodelist().begin(), response->nodelist().end());
+}
+
 void GatewayClient::asyncGetPeers(std::function callback)
 {
     auto response = std::make_shared();
diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h
index 2594890e..05fe0d2b 100644
--- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h
+++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h
@@ -66,6 +66,9 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient
     bcos::Error::Ptr registerTopic(bcos::bytesConstRef nodeID, std::string const& topic) override;
     bcos::Error::Ptr unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic) override;
 
+    std::vector selectNodesByRoutePolicy(ppc::protocol::RouteType routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) override;
+
 private:
     std::unique_ptr m_stub;
     std::map> m_broadcastStubs;
diff --git a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp
index a2d06fa9..05129869 100644
--- a/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp
+++ b/cpp/wedpr-protocol/grpc/client/RemoteFrontBuilder.cpp
@@ -32,7 +32,7 @@ IFrontClient::Ptr RemoteFrontBuilder::buildClient(std::string endPoint,
     auto frontClient = std::make_shared(m_grpcConfig, endPoint);
     if (m_healthChecker)
     {
-        auto healthCheckHandler = std::make_shared("front" + endPoint);
+        auto healthCheckHandler = std::make_shared("front_" + endPoint);
         healthCheckHandler->checkHealthHandler = [frontClient]() {
             return frontClient->checkHealth();
         };
diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
index f8e701ed..4222d8af 100644
--- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
@@ -45,7 +45,35 @@ ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* conte
                                     << LOG_KV("error", boost::diagnostic_information(e));
         toSerializedError(reply,
             std::make_shared(-1,
-                "handle message failed for : " + std::string(boost::diagnostic_information(e))));
+                "asyncSendMessage failed for : " + std::string(boost::diagnostic_information(e))));
+        reactor->Finish(Status::OK);
+    }
+    return reactor;
+}
+
+grpc::ServerUnaryReactor* GatewayServer::selectNodesByRoutePolicy(
+    grpc::CallbackServerContext* context, const ppc::proto::SelectRouteRequest* selectRouteRequest,
+    ppc::proto::NodeList* reply)
+{
+    ServerUnaryReactor* reactor(context->DefaultReactor());
+    try
+    {
+        auto routeInfo = generateRouteInfo(m_routeInfoBuilder, selectRouteRequest->routeinfo());
+        auto selectedNodes = m_gateway->selectNodesByRoutePolicy(
+            (ppc::protocol::RouteType)selectRouteRequest->routetype(), routeInfo);
+        for (auto const& it : selectedNodes)
+        {
+            reply->add_nodelist(it);
+        }
+        reactor->Finish(Status::OK);
+    }
+    catch (std::exception const& e)
+    {
+        GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("selectNodesByRoutePolicy exception")
+                                    << LOG_KV("error", boost::diagnostic_information(e));
+        toSerializedError(reply->mutable_error(),
+            std::make_shared(-1, "selectNodesByRoutePolicy failed for : " +
+                                                  std::string(boost::diagnostic_information(e))));
         reactor->Finish(Status::OK);
     }
     return reactor;
diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.h b/cpp/wedpr-protocol/grpc/server/GatewayServer.h
index 7de12ed7..c4ab5828 100644
--- a/cpp/wedpr-protocol/grpc/server/GatewayServer.h
+++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.h
@@ -39,6 +39,10 @@ class GatewayServer : public ppc::proto::Gateway::CallbackService
     grpc::ServerUnaryReactor* asyncSendMessage(grpc::CallbackServerContext* context,
         const ppc::proto::SendedMessageRequest* sendedMsg, ppc::proto::Error* reply) override;
 
+    grpc::ServerUnaryReactor* selectNodesByRoutePolicy(grpc::CallbackServerContext* context,
+        const ppc::proto::SelectRouteRequest* selectRouteRequest,
+        ppc::proto::NodeList* reply) override;
+
     grpc::ServerUnaryReactor* asyncGetPeers(grpc::CallbackServerContext* context,
         const ppc::proto::Empty* request, ppc::proto::PeersInfo* reply) override;
     grpc::ServerUnaryReactor* asyncGetAgencies(grpc::CallbackServerContext* context,
@@ -57,6 +61,7 @@ class GatewayServer : public ppc::proto::Gateway::CallbackService
     grpc::ServerUnaryReactor* unRegisterTopic(grpc::CallbackServerContext* context,
         const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override;
 
+
 private:
     ppc::gateway::IGateway::Ptr m_gateway;
     MessageOptionalHeaderBuilder::Ptr m_routeInfoBuilder;
diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto
index 2f633dba..9b7834cb 100644
--- a/cpp/wedpr-protocol/proto/pb/Service.proto
+++ b/cpp/wedpr-protocol/proto/pb/Service.proto
@@ -31,6 +31,11 @@ message SendedMessageRequest{
     string traceID = 5;
 };
 
+message SelectRouteRequest{
+    int32 routeType = 1;
+    RouteInfo routeInfo = 2;
+}
+
 message AgenciesInfo{
     Error error = 1;
     repeated string agencies = 2;
@@ -47,12 +52,17 @@ message Condition{
     repeated string components = 1;
 };
 
+message NodeList{
+    repeated string nodeList = 1;
+    Error error = 2;
+};
 service Front {
     rpc onReceiveMessage (ReceivedMessage) returns (Error) {}
 }
 service Gateway{
     rpc asyncSendMessage(SendedMessageRequest) returns(Error){}
     rpc asyncGetPeers(Empty)returns(PeersInfo){}
+    rpc selectNodesByRoutePolicy(SelectRouteRequest)returns(NodeList){}
     rpc asyncGetAgencies(Condition)returns(AgenciesInfo){}
     rpc registerNodeInfo(NodeInfo) returns(Error){}
     rpc unRegisterNodeInfo(NodeInfo)returns(Error){}
diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
index 2871074a..04f0496c 100644
--- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
+++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
@@ -65,6 +65,11 @@ class NodeInfoImpl : public INodeInfo
         bcos::ReadGuard l(x_components);
         return m_components;
     }
+    bool componentExist(std::string const& component) const override
+    {
+        bcos::ReadGuard l(x_components);
+        return m_components.count(component);
+    }
 
     std::vector copiedComponents() const override
     {
diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
index 7641103f..753b2aab 100644
--- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
+++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
@@ -43,6 +43,17 @@ inline MessageOptionalHeader::Ptr generateRouteInfo(
     return routeInfo;
 }
 
+inline void setRouteInfo(
+    ppc::proto::RouteInfo* route_info, MessageOptionalHeader::Ptr const& routeInfo)
+{
+    // set the route information
+    route_info->set_topic(routeInfo->topic());
+    route_info->set_componenttype(routeInfo->componentType());
+    route_info->set_srcnode(routeInfo->srcNode().data(), routeInfo->srcNode().size());
+    route_info->set_dstnode(routeInfo->dstNode().data(), routeInfo->dstNode().size());
+    route_info->set_dstinst(routeInfo->dstInst().data(), routeInfo->dstInst().size());
+}
+
 inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& traceID,
     RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload,
     long timeout)
@@ -51,20 +62,23 @@ inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& trac
     request->set_traceid(traceID);
     request->set_routetype(uint16_t(routeType));
     // set the route information
-    request->mutable_routeinfo()->set_topic(routeInfo->topic());
-    request->mutable_routeinfo()->set_componenttype(routeInfo->componentType());
-    request->mutable_routeinfo()->set_srcnode(
-        routeInfo->srcNode().data(), routeInfo->srcNode().size());
-    request->mutable_routeinfo()->set_dstnode(
-        routeInfo->dstNode().data(), routeInfo->dstNode().size());
-    request->mutable_routeinfo()->set_dstinst(
-        routeInfo->dstInst().data(), routeInfo->dstInst().size());
+    setRouteInfo(request->mutable_routeinfo(), routeInfo);
     // set the payload(TODO: optimize here)
     request->set_payload(payload.data(), payload.size());
     request->set_timeout(timeout);
     return request;
 }
 
+inline ppc::proto::SelectRouteRequest* generateSelectRouteRequest(
+    RouteType routeType, MessageOptionalHeader::Ptr const& routeInfo)
+{
+    auto request = new ppc::proto::SelectRouteRequest();
+    request->set_routetype(uint16_t(routeType));
+    // set the route information
+    setRouteInfo(request->mutable_routeinfo(), routeInfo);
+    return request;
+}
+
 inline ppc::proto::NodeInfo* toNodeInfoRequest(
     bcos::bytesConstRef const& nodeID, std::string const& topic)
 {
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
index 13fb00d9..3243535e 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
@@ -177,6 +177,13 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
     void registerComponent(std::string const& component) override;
     void unRegisterComponent(std::string const& component) override;
 
+    std::vector selectNodesByRoutePolicy(
+        int16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) override
+    {
+        return m_gatewayClient->selectNodesByRoutePolicy(
+            (ppc::protocol::RouteType)routeType, routeInfo);
+    }
+
 private:
     void asyncSendMessageToGateway(bool responsePacket,
         ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType,
diff --git a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
index 2d4b7364..dc81e28d 100644
--- a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
+++ b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
@@ -9,8 +9,9 @@ file(GLOB_RECURSE SRCS ppc-gateway/*.cpp)
 
 
 add_library(${GATEWAY_TARGET} ${SRCS})
-target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} 
-    ${HTTP_TARGET} ${PROTOCOL_TARGET} 
+target_link_libraries(${GATEWAY_TARGET} PUBLIC ${TOOLS_TARGET} jsoncpp_static Boost::filesystem 
+    ${BCOS_BOOSTSSL_TARGET} ${BCOS_UTILITIES_TARGET} 
+    ${PROTOCOL_TARGET} 
     ${TARS_PROTOCOL_TARGET} ${PB_PROTOCOL_TARGET}  TBB::tbb)
 
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
index d8bc8fea..273bb7fe 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
@@ -166,7 +166,8 @@ void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType,
         return;
     }
     // try to find the dstP2PNode
-    auto selectedP2PNodes = m_peerRouter->selectRouter(routeType, p2pMessage);
+    auto selectedP2PNodes =
+        m_peerRouter->selectRouter(routeType, p2pMessage->header()->optionalField());
     if (selectedP2PNodes.empty())
     {
         GATEWAY_LOG(INFO) << LOG_DESC("can't find the gateway to send the message")
@@ -252,6 +253,7 @@ bcos::Error::Ptr GatewayImpl::registerNodeInfo(ppc::protocol::INodeInfo::Ptr con
                 return;
             }
             gateway->m_localRouter->unRegisterNode(nodeInfo->nodeID().toBytes());
+            gateway->m_localRouter->increaseSeq();
         },
         true);
     return nullptr;
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
index fd83cb25..aad2fb4f 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
@@ -71,6 +71,12 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this const& components,
         std::function)> callback) override;
 
+    std::vector selectNodesByRoutePolicy(ppc::protocol::RouteType routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) override
+    {
+        return m_peerRouter->selectTargetNodes(routeType, routeInfo);
+    }
+
 protected:
     virtual void onReceiveP2PMessage(
         bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session);
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h
index 6014a6e2..e12fe0d6 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfo.h
@@ -90,6 +90,11 @@ inline std::string printNodeStatus(GatewayNodeInfo::Ptr const& status)
     stringstream << LOG_KV("p2pNodeID", printP2PIDElegantly(status->p2pNodeID()))
                  << LOG_KV("agency", status->agency()) << LOG_KV("statusSeq", status->statusSeq())
                  << LOG_KV("nodeSize", status->nodeSize());
+    auto nodeInfoList = status->nodeList();
+    for (auto const& it : nodeInfoList)
+    {
+        stringstream << printNodeInfo(it.second);
+    }
     return stringstream.str();
 }
 }  // namespace ppc::gateway
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
index 1c52fa8a..00a1e503 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
@@ -64,7 +64,7 @@ bool GatewayNodeInfoImpl::existComponent(std::string const& component) const
     bcos::ReadGuard l(x_nodeList);
     for (auto const& it : m_nodeList)
     {
-        if (it.second->components().count(component))
+        if (it.second->componentExist(component))
         {
             return true;
         }
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
index d1fc9040..7c2e015e 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
@@ -67,7 +67,6 @@ class LocalRouter
 
     GatewayNodeInfo::Ptr const& routerInfo() const { return m_routerInfo; }
 
-private:
     uint32_t increaseSeq()
     {
         uint32_t statusSeq = ++m_statusSeq;
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
index 3564f832..6c7c1f4a 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
@@ -34,6 +34,7 @@ void PeerRouterTable::updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo)
     auto nodeList = gatewayInfo->nodeList();
 
     removeP2PNodeIDFromNodeIDInfos(gatewayInfo);
+    removeP2PNodeIDFromAgencyInfos(gatewayInfo->p2pNodeID());
     insertGatewayInfo(gatewayInfo);
 }
 
@@ -145,17 +146,17 @@ std::set PeerRouterTable::agencies(std::vector const&
 }
 
 GatewayNodeInfos PeerRouterTable::selectRouter(
-    RouteType const& routeType, Message::Ptr const& msg) const
+    RouteType const& routeType, MessageOptionalHeader::Ptr const& routeInfo) const
 {
     switch (routeType)
     {
     case RouteType::ROUTE_THROUGH_NODEID:
-        return selectRouterByNodeID(msg);
+        return selectRouterByNodeID(routeInfo);
     case RouteType::ROUTE_THROUGH_COMPONENT:
-        return selectRouterByComponent(msg);
+        return selectRouterByComponent(routeInfo);
     case RouteType::ROUTE_THROUGH_AGENCY:
     case RouteType::ROUTE_THROUGH_TOPIC:
-        return selectRouterByAgency(msg);
+        return selectRouterByAgency(routeInfo);
     default:
         BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment(
                                   "selectRouter failed for encounter unsupported routeType: " +
@@ -163,11 +164,46 @@ GatewayNodeInfos PeerRouterTable::selectRouter(
     }
 }
 
-GatewayNodeInfos PeerRouterTable::selectRouterByNodeID(Message::Ptr const& msg) const
+std::vector PeerRouterTable::selectTargetNodes(
+    RouteType const& routeType, MessageOptionalHeader::Ptr const& routeInfo) const
+{
+    std::set targetNodeList;
+    auto selectedP2PNodes = selectRouter(routeType, routeInfo);
+    if (selectedP2PNodes.empty())
+    {
+        PEER_ROUTER_LOG(INFO) << LOG_DESC("selectTargetNodes with empty result")
+                              << LOG_KV("routeType", routeType)
+                              << LOG_KV("routeInfo", printOptionalField(routeInfo));
+        return std::vector();
+    }
+    for (auto const& it : selectedP2PNodes)
+    {
+        auto nodeList = it->nodeList();
+        for (auto const& it : nodeList)
+        {
+            if (routeType == RouteType::ROUTE_THROUGH_COMPONENT)
+            {
+                if (it.second->componentExist(routeInfo->componentType()))
+                {
+                    targetNodeList.insert(std::string(it.first.begin(), it.first.end()));
+                }
+                continue;
+            }
+            targetNodeList.insert(std::string(it.first.begin(), it.first.end()));
+        }
+    }
+    PEER_ROUTER_LOG(INFO) << LOG_DESC("selectTargetNodes, result: ")
+                          << printCollection(targetNodeList) << LOG_KV("routeType", routeType)
+                          << LOG_KV("routeInfo", printOptionalField(routeInfo));
+    return std::vector(targetNodeList.begin(), targetNodeList.end());
+}
+
+GatewayNodeInfos PeerRouterTable::selectRouterByNodeID(
+    MessageOptionalHeader::Ptr const& routeInfo) const
 {
     GatewayNodeInfos result;
     bcos::ReadGuard l(x_mutex);
-    auto it = m_nodeID2GatewayInfos.find(msg->header()->optionalField()->dstNode());
+    auto it = m_nodeID2GatewayInfos.find(routeInfo->dstNode());
     // no router found
     if (it == m_nodeID2GatewayInfos.end())
     {
@@ -177,11 +213,12 @@ GatewayNodeInfos PeerRouterTable::selectRouterByNodeID(Message::Ptr const& msg)
 }
 
 
-GatewayNodeInfos PeerRouterTable::selectRouterByAgency(Message::Ptr const& msg) const
+GatewayNodeInfos PeerRouterTable::selectRouterByAgency(
+    MessageOptionalHeader::Ptr const& routeInfo) const
 {
     GatewayNodeInfos result;
     bcos::ReadGuard l(x_mutex);
-    auto it = m_agency2GatewayInfos.find(msg->header()->optionalField()->dstInst());
+    auto it = m_agency2GatewayInfos.find(routeInfo->dstInst());
     // no router found
     if (it == m_agency2GatewayInfos.end())
     {
@@ -191,10 +228,11 @@ GatewayNodeInfos PeerRouterTable::selectRouterByAgency(Message::Ptr const& msg)
 }
 
 // Note: selectRouterByComponent support not specified the dstInst
-GatewayNodeInfos PeerRouterTable::selectRouterByComponent(Message::Ptr const& msg) const
+GatewayNodeInfos PeerRouterTable::selectRouterByComponent(
+    MessageOptionalHeader::Ptr const& routeInfo) const
 {
     GatewayNodeInfos result;
-    auto dstInst = msg->header()->optionalField()->dstInst();
+    auto dstInst = routeInfo->dstInst();
     std::vector selectedRouterInfos;
     {
         bcos::ReadGuard l(x_mutex);
@@ -220,14 +258,15 @@ GatewayNodeInfos PeerRouterTable::selectRouterByComponent(Message::Ptr const& ms
     }
     for (auto const& it : selectedRouterInfos)
     {
-        selectRouterByComponent(result, msg, it);
+        selectRouterByComponent(result, routeInfo, it);
     }
     return result;
 }
 
 
 void PeerRouterTable::selectRouterByComponent(GatewayNodeInfos& choosedGateway,
-    Message::Ptr const& msg, GatewayNodeInfos const& singleAgencyGatewayInfos) const
+    MessageOptionalHeader::Ptr const& routeInfo,
+    GatewayNodeInfos const& singleAgencyGatewayInfos) const
 {
     // foreach all gateways to find the component
     for (auto const& it : singleAgencyGatewayInfos)
@@ -235,8 +274,7 @@ void PeerRouterTable::selectRouterByComponent(GatewayNodeInfos& choosedGateway,
         auto const& nodeListInfo = it->nodeList();
         for (auto const& nodeInfo : nodeListInfo)
         {
-            if (nodeInfo.second->components().count(
-                    msg->header()->optionalField()->componentType()))
+            if (nodeInfo.second->componentExist(routeInfo->componentType()))
             {
                 choosedGateway.insert(it);
                 break;
@@ -245,6 +283,7 @@ void PeerRouterTable::selectRouterByComponent(GatewayNodeInfos& choosedGateway,
     }
 }
 
+
 void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const
 {
     bcos::ReadGuard l(x_mutex);
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
index e0a82c80..330737f0 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
@@ -38,8 +38,11 @@ class PeerRouterTable
 
     virtual void updateGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo);
     virtual void removeP2PID(std::string const& p2pNode);
-    virtual GatewayNodeInfos selectRouter(
-        ppc::protocol::RouteType const& routeType, ppc::protocol::Message::Ptr const& msg) const;
+    virtual GatewayNodeInfos selectRouter(ppc::protocol::RouteType const& routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
+
+    virtual std::vector selectTargetNodes(ppc::protocol::RouteType const& routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
 
     virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const;
 
@@ -51,13 +54,17 @@ class PeerRouterTable
         return m_agency2GatewayInfos;
     }
 
+
 private:
-    virtual GatewayNodeInfos selectRouterByNodeID(ppc::protocol::Message::Ptr const& msg) const;
-    virtual GatewayNodeInfos selectRouterByComponent(ppc::protocol::Message::Ptr const& msg) const;
+    virtual GatewayNodeInfos selectRouterByNodeID(
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
+    virtual GatewayNodeInfos selectRouterByComponent(
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
     void selectRouterByComponent(GatewayNodeInfos& choosedGateway,
-        ppc::protocol::Message::Ptr const& msg,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo,
         GatewayNodeInfos const& singleAgencyGatewayInfos) const;
-    virtual GatewayNodeInfos selectRouterByAgency(ppc::protocol::Message::Ptr const& msg) const;
+    virtual GatewayNodeInfos selectRouterByAgency(
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
     void removeP2PNodeIDFromNodeIDInfos(GatewayNodeInfo::Ptr const& gatewayInfo);
     void insertGatewayInfo(GatewayNodeInfo::Ptr const& gatewayInfo);
     void removeP2PNodeIDFromAgencyInfos(std::string const& p2pNode);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
index 6bd5713d..87482370 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
@@ -23,9 +23,12 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
+import com.webank.wedpr.sdk.jni.transport.impl.RouteType;
 import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl;
 import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint;
+import java.util.List;
 import lombok.SneakyThrows;
+import org.apache.commons.lang3.StringUtils;
 
 public class TransportDemo {
     public static class MessageDispatcherCallbackImpl extends MessageDispatcherCallback {
@@ -140,8 +143,9 @@ public static void main(String[] args) throws Exception {
         WeDPRTransport transport = TransportImpl.build(transportConfig);
 
         transport.start();
+        String component = "WEDPR_COMPONENT_TEST";
+        transport.registerComponent(component);
         System.out.println("####### start the transport success");
-
         // send Message to the gateway
         String topic = "testTopic";
         MessageDispatcherCallback messageDispatcherCallback =
@@ -179,6 +183,12 @@ public static void main(String[] args) throws Exception {
                                 + ", payload: "
                                 + new String(msg.getPayload())
                                 + "####");
+                // selectNodeListByPolicy
+                List nodeList =
+                        transport.selectNodeListByPolicy(
+                                RouteType.ROUTE_THROUGH_COMPONENT, null, component, null);
+                System.out.println(
+                        "###### selectNodeListByPolicy result: " + StringUtils.join(nodeList, ","));
                 i++;
             } catch (Exception e) {
                 System.out.println("#### exception: " + e.getMessage());
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
index 32fb72fc..af3b8bcb 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
@@ -94,22 +94,22 @@ public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr generateNodeInfo() {
                 wedpr_java_transportJNI.FrontConfig_generateNodeInfo(swigCPtr, this), true);
     }
 
-    public SWIGTYPE_p_std__vectorT_std__string_t getComponents() {
-        return new SWIGTYPE_p_std__vectorT_std__string_t(
+    public StringVec getComponents() {
+        return new StringVec(
                 wedpr_java_transportJNI.FrontConfig_getComponents(swigCPtr, this), false);
     }
 
-    public void setComponents(SWIGTYPE_p_std__vectorT_std__string_t components) {
+    public void setComponents(StringVec components) {
         wedpr_java_transportJNI.FrontConfig_setComponents(
-                swigCPtr, this, SWIGTYPE_p_std__vectorT_std__string_t.getCPtr(components));
+                swigCPtr, this, StringVec.getCPtr(components), components);
     }
 
     public void addComponent(String component) {
         wedpr_java_transportJNI.FrontConfig_addComponent(swigCPtr, this, component);
     }
 
-    public SWIGTYPE_p_std__vectorT_std__string_t mutableComponents() {
-        return new SWIGTYPE_p_std__vectorT_std__string_t(
+    public StringVec mutableComponents() {
+        return new StringVec(
                 wedpr_java_transportJNI.FrontConfig_mutableComponents(swigCPtr, this), false);
     }
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
index 935dd715..35bad809 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
@@ -223,4 +223,15 @@ public void registerComponent(String component) {
     public void unRegisterComponent(String component) {
         wedpr_java_transportJNI.IFront_unRegisterComponent(swigCPtr, this, component);
     }
+
+    public StringVec selectNodesByRoutePolicy(short routeType, MessageOptionalHeader routeInfo) {
+        return new StringVec(
+                wedpr_java_transportJNI.IFront_selectNodesByRoutePolicy(
+                        swigCPtr,
+                        this,
+                        routeType,
+                        MessageOptionalHeader.getCPtr(routeInfo),
+                        routeInfo),
+                true);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java
deleted file mode 100644
index 2ab7be1f..00000000
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SWIGTYPE_p_std__vectorT_std__string_t.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* ----------------------------------------------------------------------------
- * This file was automatically generated by SWIG (https://www.swig.org).
- * Version 4.2.1
- *
- * Do not make changes to this file unless you know what you are doing - modify
- * the SWIG interface file instead.
- * ----------------------------------------------------------------------------- */
-
-package com.webank.wedpr.sdk.jni.generated;
-
-public class SWIGTYPE_p_std__vectorT_std__string_t {
-    private transient long swigCPtr;
-
-    protected SWIGTYPE_p_std__vectorT_std__string_t(
-            long cPtr, @SuppressWarnings("unused") boolean futureUse) {
-        swigCPtr = cPtr;
-    }
-
-    protected SWIGTYPE_p_std__vectorT_std__string_t() {
-        swigCPtr = 0;
-    }
-
-    protected static long getCPtr(SWIGTYPE_p_std__vectorT_std__string_t obj) {
-        return (obj == null) ? 0 : obj.swigCPtr;
-    }
-
-    protected static long swigRelease(SWIGTYPE_p_std__vectorT_std__string_t obj) {
-        return (obj == null) ? 0 : obj.swigCPtr;
-    }
-}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/StringVec.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/StringVec.java
new file mode 100644
index 00000000..1138ae64
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/StringVec.java
@@ -0,0 +1,163 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class StringVec extends java.util.AbstractList implements java.util.RandomAccess {
+    private transient long swigCPtr;
+    protected transient boolean swigCMemOwn;
+
+    protected StringVec(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(StringVec obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected static long swigRelease(StringVec obj) {
+        long ptr = 0;
+        if (obj != null) {
+            if (!obj.swigCMemOwn)
+                throw new RuntimeException("Cannot release ownership as memory is not owned");
+            ptr = obj.swigCPtr;
+            obj.swigCMemOwn = false;
+            obj.delete();
+        }
+        return ptr;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_StringVec(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public StringVec(String[] initialElements) {
+        this();
+        reserve(initialElements.length);
+
+        for (String element : initialElements) {
+            add(element);
+        }
+    }
+
+    public StringVec(Iterable initialElements) {
+        this();
+        for (String element : initialElements) {
+            add(element);
+        }
+    }
+
+    public String get(int index) {
+        return doGet(index);
+    }
+
+    public String set(int index, String e) {
+        return doSet(index, e);
+    }
+
+    public boolean add(String e) {
+        modCount++;
+        doAdd(e);
+        return true;
+    }
+
+    public void add(int index, String e) {
+        modCount++;
+        doAdd(index, e);
+    }
+
+    public String remove(int index) {
+        modCount++;
+        return doRemove(index);
+    }
+
+    protected void removeRange(int fromIndex, int toIndex) {
+        modCount++;
+        doRemoveRange(fromIndex, toIndex);
+    }
+
+    public int size() {
+        return doSize();
+    }
+
+    public int capacity() {
+        return doCapacity();
+    }
+
+    public void reserve(int n) {
+        doReserve(n);
+    }
+
+    public StringVec() {
+        this(wedpr_java_transportJNI.new_StringVec__SWIG_0(), true);
+    }
+
+    public StringVec(StringVec other) {
+        this(wedpr_java_transportJNI.new_StringVec__SWIG_1(StringVec.getCPtr(other), other), true);
+    }
+
+    public boolean isEmpty() {
+        return wedpr_java_transportJNI.StringVec_isEmpty(swigCPtr, this);
+    }
+
+    public void clear() {
+        wedpr_java_transportJNI.StringVec_clear(swigCPtr, this);
+    }
+
+    public StringVec(int count, String value) {
+        this(wedpr_java_transportJNI.new_StringVec__SWIG_2(count, value), true);
+    }
+
+    private int doCapacity() {
+        return wedpr_java_transportJNI.StringVec_doCapacity(swigCPtr, this);
+    }
+
+    private void doReserve(int n) {
+        wedpr_java_transportJNI.StringVec_doReserve(swigCPtr, this, n);
+    }
+
+    private int doSize() {
+        return wedpr_java_transportJNI.StringVec_doSize(swigCPtr, this);
+    }
+
+    private void doAdd(String x) {
+        wedpr_java_transportJNI.StringVec_doAdd__SWIG_0(swigCPtr, this, x);
+    }
+
+    private void doAdd(int index, String x) {
+        wedpr_java_transportJNI.StringVec_doAdd__SWIG_1(swigCPtr, this, index, x);
+    }
+
+    private String doRemove(int index) {
+        return wedpr_java_transportJNI.StringVec_doRemove(swigCPtr, this, index);
+    }
+
+    private String doGet(int index) {
+        return wedpr_java_transportJNI.StringVec_doGet(swigCPtr, this, index);
+    }
+
+    private String doSet(int index, String val) {
+        return wedpr_java_transportJNI.StringVec_doSet(swigCPtr, this, index, val);
+    }
+
+    private void doRemoveRange(int fromIndex, int toIndex) {
+        wedpr_java_transportJNI.StringVec_doRemoveRange(swigCPtr, this, fromIndex, toIndex);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index 81c85a3c..82e57708 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -146,6 +146,40 @@ public static final native void ibytes_doRemoveRange(
 
     public static final native void delete_ibytes(long jarg1);
 
+    public static final native long new_StringVec__SWIG_0();
+
+    public static final native long new_StringVec__SWIG_1(long jarg1, StringVec jarg1_);
+
+    public static final native boolean StringVec_isEmpty(long jarg1, StringVec jarg1_);
+
+    public static final native void StringVec_clear(long jarg1, StringVec jarg1_);
+
+    public static final native long new_StringVec__SWIG_2(int jarg1, String jarg2);
+
+    public static final native int StringVec_doCapacity(long jarg1, StringVec jarg1_);
+
+    public static final native void StringVec_doReserve(long jarg1, StringVec jarg1_, int jarg2);
+
+    public static final native int StringVec_doSize(long jarg1, StringVec jarg1_);
+
+    public static final native void StringVec_doAdd__SWIG_0(
+            long jarg1, StringVec jarg1_, String jarg2);
+
+    public static final native void StringVec_doAdd__SWIG_1(
+            long jarg1, StringVec jarg1_, int jarg2, String jarg3);
+
+    public static final native String StringVec_doRemove(long jarg1, StringVec jarg1_, int jarg2);
+
+    public static final native String StringVec_doGet(long jarg1, StringVec jarg1_, int jarg2);
+
+    public static final native String StringVec_doSet(
+            long jarg1, StringVec jarg1_, int jarg2, String jarg3);
+
+    public static final native void StringVec_doRemoveRange(
+            long jarg1, StringVec jarg1_, int jarg2, int jarg3);
+
+    public static final native void delete_StringVec(long jarg1);
+
     public static final native long Error_buildError__SWIG_0(
             byte[] jarg1, byte[] jarg2, int jarg3, int jarg4, String jarg5);
 
@@ -239,7 +273,7 @@ public static final native void FrontConfig_setGrpcConfig(
     public static final native long FrontConfig_getComponents(long jarg1, FrontConfig jarg1_);
 
     public static final native void FrontConfig_setComponents(
-            long jarg1, FrontConfig jarg1_, long jarg2);
+            long jarg1, FrontConfig jarg1_, long jarg2, StringVec jarg2_);
 
     public static final native void FrontConfig_addComponent(
             long jarg1, FrontConfig jarg1_, String jarg2);
@@ -748,6 +782,9 @@ public static final native void IFront_registerComponent(
     public static final native void IFront_unRegisterComponent(
             long jarg1, IFront jarg1_, String jarg2);
 
+    public static final native long IFront_selectNodesByRoutePolicy(
+            long jarg1, IFront jarg1_, short jarg2, long jarg3, MessageOptionalHeader jarg3_);
+
     public static final native void delete_IFrontBuilder(long jarg1);
 
     public static final native long IFrontBuilder_buildClient(
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java
index e4c75358..1618c84c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/IMessageBuilder.java
@@ -43,4 +43,13 @@ public static MessageOptionalHeader buildRouteInfo(
         routeInfo.disOwnMemory();
         return routeInfo;
     }
+
+    @SneakyThrows(Exception.class)
+    public static MessageOptionalHeader buildRouteInfo(
+            MessageOptionalHeaderBuilder routeInfoBuilder) {
+        // return the ownership to cpp, since it is created by cpp
+        MessageOptionalHeader routeInfo = routeInfoBuilder.build();
+        routeInfo.disOwnMemory();
+        return routeInfo;
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
index 5e0545e4..b1e03756 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
@@ -20,6 +20,8 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
+import com.webank.wedpr.sdk.jni.transport.impl.RouteType;
+import java.util.List;
 
 public interface WeDPRTransport {
     // start the transport
@@ -182,4 +184,7 @@ void pushByInst(String topic, String dstInst, int seq, byte[] payload, int timeo
     IMessage pop(String topic, int timeout) throws WeDPRSDKException;
 
     IMessage peek(String topic);
+
+    List selectNodeListByPolicy(
+            RouteType routeType, String dstInst, String dstComponent, String dstNode);
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index 17679b24..7f3ceb28 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -29,6 +29,8 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
 import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
 import lombok.SneakyThrows;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -424,4 +426,32 @@ public IMessage peek(String topic) {
         }
         return IMessageBuilder.build(msg);
     }
+
+    @Override
+    public List selectNodeListByPolicy(
+            RouteType routeType, String dstInst, String dstComponent, String dstNode) {
+        MessageOptionalHeader routeInfo =
+                IMessageBuilder.buildRouteInfo(this.transport.routeInfoBuilder());
+        if (StringUtils.isNotBlank(dstInst)) {
+            routeInfo.setDstInst(dstInst);
+        }
+        if (StringUtils.isNotBlank(dstComponent)) {
+            routeInfo.setComponentType(dstComponent);
+        }
+        if (StringUtils.isNotBlank(dstNode)) {
+            routeInfo.setDstNodeBuffer(dstNode.getBytes(), BigInteger.valueOf(dstNode.length()));
+        }
+        StringVec result =
+                this.transport
+                        .getFront()
+                        .selectNodesByRoutePolicy((short) routeType.ordinal(), routeInfo);
+        if (result == null) {
+            return null;
+        }
+        List nodeList = new ArrayList<>();
+        for (int i = 0; i < result.size(); i++) {
+            nodeList.add(result.get(i));
+        }
+        return nodeList;
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 36134a4e..699684c9 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -1056,6 +1056,68 @@ SWIGINTERN void std_vector_Sl_int8_t_Sg__doRemoveRange(std::vector< int8_t > *se
           throw std::out_of_range("vector index out of range");
         }
       }
+SWIGINTERN std::vector< std::string > *new_std_vector_Sl_std_string_Sg___SWIG_2(jint count,std::string const &value){
+        if (count < 0)
+          throw std::out_of_range("vector count must be positive");
+        return new std::vector< std::string >(static_cast::size_type>(count), value);
+      }
+SWIGINTERN jint std_vector_Sl_std_string_Sg__doCapacity(std::vector< std::string > *self){
+        return SWIG_VectorSize(self->capacity());
+      }
+SWIGINTERN void std_vector_Sl_std_string_Sg__doReserve(std::vector< std::string > *self,jint n){
+        if (n < 0)
+          throw std::out_of_range("vector reserve size must be positive");
+        self->reserve(n);
+      }
+SWIGINTERN jint std_vector_Sl_std_string_Sg__doSize(std::vector< std::string > const *self){
+        return SWIG_VectorSize(self->size());
+      }
+SWIGINTERN void std_vector_Sl_std_string_Sg__doAdd__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::value_type const &x){
+        self->push_back(x);
+      }
+SWIGINTERN void std_vector_Sl_std_string_Sg__doAdd__SWIG_1(std::vector< std::string > *self,jint index,std::vector< std::string >::value_type const &x){
+        jint size = static_cast(self->size());
+        if (0 <= index && index <= size) {
+          self->insert(self->begin() + index, x);
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
+SWIGINTERN std::vector< std::string >::value_type std_vector_Sl_std_string_Sg__doRemove(std::vector< std::string > *self,jint index){
+        jint size = static_cast(self->size());
+        if (0 <= index && index < size) {
+          std::string const old_value = (*self)[index];
+          self->erase(self->begin() + index);
+          return old_value;
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
+SWIGINTERN std::vector< std::string >::value_type const &std_vector_Sl_std_string_Sg__doGet(std::vector< std::string > *self,jint index){
+        jint size = static_cast(self->size());
+        if (index >= 0 && index < size)
+          return (*self)[index];
+        else
+          throw std::out_of_range("vector index out of range");
+      }
+SWIGINTERN std::vector< std::string >::value_type std_vector_Sl_std_string_Sg__doSet(std::vector< std::string > *self,jint index,std::vector< std::string >::value_type const &val){
+        jint size = static_cast(self->size());
+        if (index >= 0 && index < size) {
+          std::string const old_value = (*self)[index];
+          (*self)[index] = val;
+          return old_value;
+        }
+        else
+          throw std::out_of_range("vector index out of range");
+      }
+SWIGINTERN void std_vector_Sl_std_string_Sg__doRemoveRange(std::vector< std::string > *self,jint fromIndex,jint toIndex){
+        jint size = static_cast(self->size());
+        if (0 <= fromIndex && fromIndex <= toIndex && toIndex <= size) {
+          self->erase(self->begin() + fromIndex, self->begin() + toIndex);
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
 
 struct SWIG_null_deleter {
   void operator() (void const *) const {
@@ -2309,6 +2371,308 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1StringVec_1_1SWIG_10(JNIEnv *jenv, jclass jcls) {
+  jlong jresult = 0 ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  result = (std::vector< std::string > *)new std::vector< std::string >();
+  *(std::vector< std::string > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1StringVec_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  std::vector< std::string > *arg1 = 0 ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1;
+  if (!arg1) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< std::string > const & is null");
+    return 0;
+  } 
+  result = (std::vector< std::string > *)new std::vector< std::string >((std::vector< std::string > const &)*arg1);
+  *(std::vector< std::string > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1isEmpty(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jboolean jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  bool result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  result = (bool)((std::vector< std::string > const *)arg1)->empty();
+  jresult = (jboolean)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1clear(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  (arg1)->clear();
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1StringVec_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jint jarg1, jstring jarg2) {
+  jlong jresult = 0 ;
+  jint arg1 ;
+  std::string *arg2 = 0 ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  arg1 = jarg1; 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return 0;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return 0;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  try {
+    result = (std::vector< std::string > *)new_std_vector_Sl_std_string_Sg___SWIG_2(SWIG_STD_MOVE(arg1),(std::string const &)*arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  *(std::vector< std::string > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doCapacity(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  try {
+    result = std_vector_Sl_std_string_Sg__doCapacity(arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doReserve(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    std_vector_Sl_std_string_Sg__doReserve(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::length_error &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  try {
+    result = std_vector_Sl_std_string_Sg__doSize((std::vector< std::string > const *)arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doAdd_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::value_type *arg2 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::vector< std::string >::value_type arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  std_vector_Sl_std_string_Sg__doAdd__SWIG_0(arg1,(std::string const &)*arg2);
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doAdd_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jstring jarg3) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  if(!jarg3) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); 
+  if (!arg3_pstr) return ;
+  std::vector< std::string >::value_type arg3_str(arg3_pstr);
+  arg3 = &arg3_str;
+  jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); 
+  try {
+    std_vector_Sl_std_string_Sg__doAdd__SWIG_1(arg1,SWIG_STD_MOVE(arg2),(std::string const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doRemove(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  jstring jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  std::vector< std::string >::value_type result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    result = std_vector_Sl_std_string_Sg__doRemove(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doGet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  jstring jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  std::vector< std::string >::value_type *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    result = (std::vector< std::string >::value_type *) &std_vector_Sl_std_string_Sg__doGet(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = jenv->NewStringUTF(result->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doSet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jstring jarg3) {
+  jstring jresult = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
+  std::vector< std::string >::value_type result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  if(!jarg3) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return 0;
+  }
+  const char *arg3_pstr = (const char *)jenv->GetStringUTFChars(jarg3, 0); 
+  if (!arg3_pstr) return 0;
+  std::vector< std::string >::value_type arg3_str(arg3_pstr);
+  arg3 = &arg3_str;
+  jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); 
+  try {
+    result = std_vector_Sl_std_string_Sg__doSet(arg1,SWIG_STD_MOVE(arg2),(std::string const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_StringVec_1doRemoveRange(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jint jarg3) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  jint arg2 ;
+  jint arg3 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  arg2 = jarg2; 
+  arg3 = jarg3; 
+  try {
+    std_vector_Sl_std_string_Sg__doRemoveRange(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1StringVec(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  arg1 = *(std::vector< std::string > **)&jarg1; 
+  delete arg1;
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1buildError_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jbyteArray jarg1, jbyteArray jarg2, jint jarg3, jint jarg4, jstring jarg5) {
   jlong jresult = 0 ;
   char *arg1 = (char *) 0 ;
@@ -3078,7 +3442,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
   ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   std::vector< std::string > *arg2 = 0 ;
   std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
@@ -3086,6 +3450,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
+  (void)jarg2_;
   
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
@@ -6734,6 +7099,30 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1selectNodesByRoutePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jshort jarg2, jlong jarg3, jobject jarg3_) {
+  jlong jresult = 0 ;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  int16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ;
+  std::vector< std::string > result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  (void)jarg3_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
+  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
+  arg2 = (int16_t)jarg2; 
+  arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; 
+  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+  *(std::vector< std::string > **)&jresult = new std::vector< std::string >(result); 
+  return jresult;
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
   
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index fd0a6d1c..91b8570c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -115,6 +115,7 @@ namespace ppc::front{
 
 namespace std{
     class vector;
+    class string;
 }
 
 namespace bcos{
@@ -151,6 +152,7 @@ namespace bcos{
 
 %template(ubytes) std::vector;
 %template(ibytes) std::vector;
+%template(StringVec) std::vector;
 
 /// callbacks
 %feature("director") ppc::front::ErrorCallback;
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
index e695da02..fbaa6b04 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
@@ -40,9 +40,9 @@ endif()
 # i.e. we can't use: $<$:${PYTHON_LIBRARIES}>
 # see: https://cmake.org/cmake/help/git-stage/command/target_link_libraries.html#command:target_link_libraries
 if(MSVC)
-  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${Python3_LIBRARIES})
+  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${Python3_LIBRARIES})
 else()
-  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${Python_LIBRARIES})
+  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET})
 endif()
 SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_LIB_DIR})
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
index bce7efff..898f3f76 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
@@ -4,19 +4,23 @@
 import os
 root_path = os.path.abspath(os.path.dirname(__file__))
 sys.path.append(os.path.join(root_path, "../"))
+# Note: here can't be refactored by autopep
 
 import time
 from wedpr_python_gateway_sdk.transport.impl.transport_loader import TransportLoader
 from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
 from wedpr_python_gateway_sdk.transport.impl.transport import Transport
+from wedpr_python_gateway_sdk.transport.impl.transport import RouteType
+import traceback
 import argparse
-import sys
 
 
 def parse_args():
     parser = argparse.ArgumentParser(prog=sys.argv[0])
     parser.add_argument("-t", '--threadpool_size',
                         help='the threadpool size', default=4, required=True)
+    parser.add_argument('--dst_inst',
+                        help='the dist inst', default=None, required=True)
     parser.add_argument("-n", '--node_id',
                         help='the nodeID', required=False)
     parser.add_argument("-g", '--gateway_targets',
@@ -39,6 +43,8 @@ def message_event_loop(args):
     transport = TransportLoader.load(transport_config)
     print(f"Create transport success, config: {transport_config.desc()}")
     transport.start()
+    component = "WEDPR_COMPONENT_TEST"
+    transport.register_component(component)
     print(f"Start transport success")
     test_topic = "sync_message_event_loop_test"
     while Transport.should_exit is False:
@@ -52,7 +58,28 @@ def message_event_loop(args):
                 continue
             print(
                 f"Receive message: {msg.detail()}, buffer: {str(msg.get_payload())}")
+
+            node_list = transport.select_node_list_by_route_policy(route_type=RouteType.ROUTE_THROUGH_COMPONENT,
+                                                                   dst_inst=args.dst_inst,
+                                                                   dst_component=component)
+            if node_list is None:
+                print(
+                    f"####select_node_list_by_route_policy: not find component: {component}")
+                continue
+            for node in node_list:
+                print(
+                    f"##### select_node_list_by_route_policy result: {node}, component: {component}")
+
+            selected_node = transport.select_node_by_route_policy(route_type=RouteType.ROUTE_THROUGH_COMPONENT,
+                                                                  dst_inst=args.dst_inst,
+                                                                  dst_component=component)
+            if selected_node is None:
+                print(
+                    f"####select_node_by_route_policy: not find component: {component}")
+            print(
+                f"##### select_node_by_route_policy, selected_node: {selected_node}, component: {component}")
         except Exception as e:
+            traceback.print_exc()
             print(f"exception: {e}")
         time.sleep(2)
     print(f"stop the transport")
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 83de8c13..0b97ed83 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -624,6 +624,117 @@ def capacity(self):
 _wedpr_python_transport.ibytes_swigregister(ibytes)
 
 
+class StringVec(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def iterator(self):
+        return _wedpr_python_transport.StringVec_iterator(self)
+
+    def __iter__(self):
+        return self.iterator()
+
+    def __nonzero__(self):
+        return _wedpr_python_transport.StringVec___nonzero__(self)
+
+    def __bool__(self):
+        return _wedpr_python_transport.StringVec___bool__(self)
+
+    def __len__(self):
+        return _wedpr_python_transport.StringVec___len__(self)
+
+    def __getslice__(self, i, j):
+        return _wedpr_python_transport.StringVec___getslice__(self, i, j)
+
+    def __setslice__(self, *args):
+        return _wedpr_python_transport.StringVec___setslice__(self, *args)
+
+    def __delslice__(self, i, j):
+        return _wedpr_python_transport.StringVec___delslice__(self, i, j)
+
+    def __delitem__(self, *args):
+        return _wedpr_python_transport.StringVec___delitem__(self, *args)
+
+    def __getitem__(self, *args):
+        return _wedpr_python_transport.StringVec___getitem__(self, *args)
+
+    def __setitem__(self, *args):
+        return _wedpr_python_transport.StringVec___setitem__(self, *args)
+
+    def pop(self):
+        return _wedpr_python_transport.StringVec_pop(self)
+
+    def append(self, x):
+        return _wedpr_python_transport.StringVec_append(self, x)
+
+    def empty(self):
+        return _wedpr_python_transport.StringVec_empty(self)
+
+    def size(self):
+        return _wedpr_python_transport.StringVec_size(self)
+
+    def swap(self, v):
+        return _wedpr_python_transport.StringVec_swap(self, v)
+
+    def begin(self):
+        return _wedpr_python_transport.StringVec_begin(self)
+
+    def end(self):
+        return _wedpr_python_transport.StringVec_end(self)
+
+    def rbegin(self):
+        return _wedpr_python_transport.StringVec_rbegin(self)
+
+    def rend(self):
+        return _wedpr_python_transport.StringVec_rend(self)
+
+    def clear(self):
+        return _wedpr_python_transport.StringVec_clear(self)
+
+    def get_allocator(self):
+        return _wedpr_python_transport.StringVec_get_allocator(self)
+
+    def pop_back(self):
+        return _wedpr_python_transport.StringVec_pop_back(self)
+
+    def erase(self, *args):
+        return _wedpr_python_transport.StringVec_erase(self, *args)
+
+    def __init__(self, *args):
+        _wedpr_python_transport.StringVec_swiginit(
+            self, _wedpr_python_transport.new_StringVec(*args))
+
+    def push_back(self, x):
+        return _wedpr_python_transport.StringVec_push_back(self, x)
+
+    def front(self):
+        return _wedpr_python_transport.StringVec_front(self)
+
+    def back(self):
+        return _wedpr_python_transport.StringVec_back(self)
+
+    def assign(self, n, x):
+        return _wedpr_python_transport.StringVec_assign(self, n, x)
+
+    def resize(self, *args):
+        return _wedpr_python_transport.StringVec_resize(self, *args)
+
+    def insert(self, *args):
+        return _wedpr_python_transport.StringVec_insert(self, *args)
+
+    def reserve(self, n):
+        return _wedpr_python_transport.StringVec_reserve(self, n)
+
+    def capacity(self):
+        return _wedpr_python_transport.StringVec_capacity(self)
+    __swig_destroy__ = _wedpr_python_transport.delete_StringVec
+
+
+# Register StringVec in _wedpr_python_transport:
+_wedpr_python_transport.StringVec_swigregister(StringVec)
+
+
 class Error(object):
     thisown = property(lambda x: x.this.own(), lambda x,
                        v: x.this.own(v), doc="The membership flag")
@@ -1505,6 +1616,9 @@ def registerComponent(self, component):
     def unRegisterComponent(self, component):
         return _wedpr_python_transport.IFront_unRegisterComponent(self, component)
 
+    def selectNodesByRoutePolicy(self, routeType, routeInfo):
+        return _wedpr_python_transport.IFront_selectNodesByRoutePolicy(self, routeType, routeInfo)
+
 
 # Register IFront in _wedpr_python_transport:
 _wedpr_python_transport.IFront_swigregister(IFront)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
index 51a8ef65..aec2cfc5 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/message_impl.py
@@ -62,6 +62,9 @@ def get_topic(self) -> str:
     def detail(self) -> str:
         return f"version: {self.get_version()}, topic: {self.get_topic()}, src_inst: {self.get_src_inst()},src_node: {str(self.get_src_node())}, dst_inst: {self.get_dst_inst()}, dst_node: {str(self.get_dst_node())}"
 
+    def __repr__(self):
+        return self.detail()
+
 
 class MessageImpl(MessageAPI):
     def __init__(self, message: Message):
@@ -85,3 +88,6 @@ def get_length(self) -> int:
 
     def detail(self) -> str:
         return f"seq: {self.get_seq()}, header: {self.get_header().detail()}, length: {self.get_length()}"
+
+    def __repr__(self):
+        return self.detail()
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
index 86b6695d..2de8ba0d 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
@@ -2,12 +2,15 @@
 
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Transport
+from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import StringVec
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import Error
 from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
 from wedpr_python_gateway_sdk.transport.impl.route_info_builder import RouteInfoBuilder
 from wedpr_python_gateway_sdk.transport.impl.message_factory import MessageFactory
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import MessageOptionalHeader
 from wedpr_python_gateway_sdk.transport.api.transport_api import TransportAPI
+from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
+import random
 
 from enum import Enum
 import signal
@@ -23,11 +26,15 @@ class RouteType(Enum):
 class Transport(TransportAPI):
     should_exit = False
 
-    def __init__(self, transport: Transport):
+    def __init__(self, transport: Transport, transport_config: TransportConfig):
         self.__transport = transport
+        self.__transport_config = transport_config
         self.__route_info_builder = RouteInfoBuilder(
             self.__transport.routeInfoBuilder())
 
+    def get_config(self) -> TransportConfig:
+        return self.__transport_config
+
     def start(self):
         self.__transport.start()
 
@@ -40,6 +47,26 @@ def _push_msg(self, route_type: int, route_info: MessageOptionalHeader, payload:
         except Exception as e:
             raise e
 
+    def select_node_list_by_route_policy(self, route_type: RouteType, dst_inst: str, dst_component: str, dst_node: str = None) -> tuple:
+        dst_node_bytes = None
+        if dst_node is not None:
+            dst_node_bytes = bytes(dst_node, encodings="utf-8")
+        route_info = self.__route_info_builder.build(topic=None, dst_inst=dst_inst,
+                                                     component=dst_component, dst_node=dst_node_bytes)
+        try:
+            return self.__transport.getFront().selectNodesByRoutePolicy(
+                route_type.value, route_info)
+        except Exception as e:
+            raise e
+
+    def select_node_by_route_policy(self, route_type: RouteType, dst_inst: str, dst_component: str, dst_node: str = None) -> str:
+        node_list = self.select_node_list_by_route_policy(
+            route_type, dst_inst, dst_component, dst_node)
+        if node_list is None or len(node_list) == 0:
+            return None
+        selected_node_idx = random.randint(0, len(node_list) - 1)
+        return node_list[selected_node_idx]
+
     def push_by_nodeid(self, topic: str, dstNode: bytes, seq: int, payload: bytes, timeout: int):
         route_info = self.__route_info_builder.build(
             topic=topic, dst_node=dstNode, dst_inst=None, component=None)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
index b6e6492e..7ee50de4 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
@@ -1,10 +1,50 @@
 # -*- coding: utf-8 -*-
+from wedpr_python_gateway_sdk.utils.base_object import BaseObject
 from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
 from wedpr_python_gateway_sdk.transport.impl.transport import Transport
+from wedpr_python_gateway_sdk.utils import utils
 import signal
 
 
+class TransportConfigObject(BaseObject):
+    DEFAULT_THREADPOOL_SIZE = 4
+    DEFAULT_LISTEN_IP = "0.0.0.0"
+
+    def __init__(self, transport_threadpool_size: int = DEFAULT_THREADPOOL_SIZE,
+                 transport_node_id: str = None,
+                 transport_gateway_targets: str = None,
+                 transport_host_ip: str = None,
+                 transport_listen_port: int = None,
+                 transport_listen_ip: str = DEFAULT_LISTEN_IP,
+                 **params):
+        self.transport_threadpool_size = transport_threadpool_size
+        self.transport_node_id = transport_node_id
+        self.transport_gateway_targets = transport_gateway_targets
+        self.transport_host_ip = transport_host_ip
+        self.transport_listen_port = transport_listen_port
+        self.transport_listen_ip = transport_listen_ip
+        self.set_params(**params)
+        self.__check()
+
+    def __check(self):
+        if self.transport_threadpool_size <= 0:
+            self.transport_threadpool_size = TransportConfigObject.DEFAULT_THREADPOOL_SIZE
+        utils.require_not_empty("transport_node_id", self.transport_node_id)
+        utils.require_not_empty(
+            "transport_gateway_targets", self.transport_gateway_targets)
+        utils.require_not_empty("transport_host_ip", self.transport_host_ip)
+        if self.transport_listen_port <= 0 or self.transport_listen_port > 65535:
+            raise Exception(
+                f"Invalid transport listen port: {self.transport_listen_port}, must between 0-65535!")
+
+    def __repr__(self):
+        return f"transport_threadpool_size: {self.transport_threadpool_size}, " \
+               f"transport_node_id: {self.transport_node_id}, " \
+               f"transport_gateway_targets: {self.transport_gateway_targets}, " \
+               f"transport_host_ip: {self.transport_host_ip}, transport_listen_port: {self.transport_listen_port}"
+
+
 class TransportLoader:
     transport_builder = TransportBuilder()
 
@@ -12,4 +52,27 @@ class TransportLoader:
     def load(transport_config: TransportConfig) -> Transport:
         transport = TransportLoader.transport_builder.buildProTransport(
             transport_config.get_front_config())
-        return Transport(transport)
+        return Transport(transport, transport_config)
+
+    @staticmethod
+    def build(transport_threadpool_size: int = 4,
+              transport_node_id: str = None,
+              transport_gateway_targets: str = None,
+              transport_host_ip: str = None,
+              transport_listen_port: int = None,
+              **params):
+        config = TransportConfigObject(transport_threadpool_size,
+                                       transport_node_id,
+                                       transport_gateway_targets,
+                                       transport_host_ip,
+                                       transport_listen_port, **params)
+        # build the transport config
+        transport_config = TransportConfig(config.transport_threadpool_size,
+                                           config.transport_node_id,
+                                           config.transport_gateway_targets)
+        # set the self-endpoint
+        transport_config.set_self_endpoint(config.transport_host_ip,
+                                           config.transport_listen_port,
+                                           config.transport_listen_ip)
+
+        return TransportLoader.load(transport_config)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/base_object.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/base_object.py
new file mode 100644
index 00000000..0dd026b8
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/base_object.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+import time
+
+
+class BaseObject:
+    def set_params(self, **params: Any):
+        for key, value in params.items():
+            setattr(self, key, value)
+            if hasattr(self, f"{key}"):
+                setattr(self, f"{key}", value)
+        return self
+
+    def as_dict(obj):
+        return {attr: getattr(obj, attr) for attr in dir(obj) if not callable(getattr(obj, attr)) and not attr.startswith("__")}
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/utils.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/utils.py
new file mode 100644
index 00000000..68254eaa
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/utils/utils.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+def require_not_empty(field, value):
+    if value is None or len(value) == 0:
+        raise Exception(f"The field {field} must not empty!")
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index 12d60c03..fef8030e 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -3692,48 +3692,49 @@ namespace Swig {
 #define SWIGTYPE_p_signed_char swig_types[44]
 #define SWIGTYPE_p_size_type swig_types[45]
 #define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[46]
-#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[47]
-#define SWIGTYPE_p_std__exception swig_types[48]
-#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[49]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[50]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[51]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[52]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[53]
-#define SWIGTYPE_p_std__invalid_argument swig_types[54]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[55]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[56]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[57]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[58]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[59]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[60]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[61]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[62]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[63]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[64]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[65]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[66]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[67]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[68]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[69]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[70]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[71]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[72]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[73]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[74]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[75]
-#define SWIGTYPE_p_std__string swig_types[76]
-#define SWIGTYPE_p_std__string_view swig_types[77]
-#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[78]
-#define SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t swig_types[79]
-#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[80]
-#define SWIGTYPE_p_swig__SwigPyIterator swig_types[81]
-#define SWIGTYPE_p_unsigned_char swig_types[82]
-#define SWIGTYPE_p_unsigned_int swig_types[83]
-#define SWIGTYPE_p_unsigned_long_long swig_types[84]
-#define SWIGTYPE_p_unsigned_short swig_types[85]
-#define SWIGTYPE_p_value_type swig_types[86]
-static swig_type_info *swig_types[88];
-static swig_module_info swig_module = {swig_types, 87, 0, 0, 0, 0};
+#define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[47]
+#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[48]
+#define SWIGTYPE_p_std__exception swig_types[49]
+#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[50]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[51]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[52]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[53]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[54]
+#define SWIGTYPE_p_std__invalid_argument swig_types[55]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[56]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[57]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[58]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[59]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[60]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[61]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[62]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[63]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[64]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[65]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[66]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[67]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[68]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[69]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[70]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[71]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[72]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[73]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[74]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[75]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[76]
+#define SWIGTYPE_p_std__string swig_types[77]
+#define SWIGTYPE_p_std__string_view swig_types[78]
+#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[79]
+#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[80]
+#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[81]
+#define SWIGTYPE_p_swig__SwigPyIterator swig_types[82]
+#define SWIGTYPE_p_unsigned_char swig_types[83]
+#define SWIGTYPE_p_unsigned_int swig_types[84]
+#define SWIGTYPE_p_unsigned_long_long swig_types[85]
+#define SWIGTYPE_p_unsigned_short swig_types[86]
+#define SWIGTYPE_p_value_type swig_types[87]
+static swig_type_info *swig_types[89];
+static swig_module_info swig_module = {swig_types, 88, 0, 0, 0, 0};
 #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name)
 #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name)
 
@@ -5869,25 +5870,6 @@ SWIG_AsCharPtrAndSize(PyObject *obj, char **cptr, size_t *psize, int *alloc)
 }
 
 
-
-
-
-SWIGINTERN int
-SWIG_AsVal_int (PyObject * obj, int *val)
-{
-  long v;
-  int res = SWIG_AsVal_long (obj, &v);
-  if (SWIG_IsOK(res)) {
-    if ((v < INT_MIN || v > INT_MAX)) {
-      return SWIG_OverflowError;
-    } else {
-      if (val) *val = static_cast< int >(v);
-    }
-  }  
-  return res;
-}
-
-
 SWIGINTERN int
 SWIG_AsPtr_std_string (PyObject * obj, std::string **val) 
 {
@@ -5919,17 +5901,22 @@ SWIG_AsPtr_std_string (PyObject * obj, std::string **val)
 }
 
 
-struct SWIG_null_deleter {
-  void operator() (void const *) const {
+SWIGINTERN int
+SWIG_AsVal_std_string (PyObject * obj, std::string *val)
+{
+  std::string* v = (std::string *) 0;
+  int res = SWIG_AsPtr_std_string (obj, &v);
+  if (!SWIG_IsOK(res)) return res;
+  if (v) {
+    if (val) *val = *v;
+    if (SWIG_IsNewObj(res)) {
+      delete v;
+      res = SWIG_DelNewMask(res);
+    }
+    return res;
   }
-};
-#define SWIG_NO_NULL_DELETER_0 , SWIG_null_deleter()
-#define SWIG_NO_NULL_DELETER_1
-#define SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW
-#define SWIG_NO_NULL_DELETER_SWIG_POINTER_OWN
-
-
-#define SWIG_NO_NULL_DELETER_SWIG_BUILTIN_INIT
+  return SWIG_ERROR;
+}
 
 
 SWIGINTERNINLINE PyObject *
@@ -5964,6 +5951,159 @@ SWIG_From_std_string  (const std::string& s)
 }
 
 
+namespace swig {
+  template <> struct traits< std::string > {
+    typedef value_category category;
+    static const char* type_name() { return"std::string"; }
+  };
+  template <>  struct traits_asval< std::string > {
+    typedef std::string value_type;
+    static int asval(PyObject *obj, value_type *val) {
+      return SWIG_AsVal_std_string (obj, val);
+    }
+  };
+  template <>  struct traits_from< std::string > {
+    typedef std::string value_type;
+    static PyObject *from(const value_type& val) {
+      return SWIG_From_std_string  (val);
+    }
+  };
+}
+
+
+      namespace swig {
+	template <>  struct traits > > {
+	  typedef pointer_category category;
+	  static const char* type_name() {
+	    return "std::vector<" "std::string" "," "std::allocator< std::string >" " >";
+	  }
+	};
+      }
+    
+SWIGINTERN swig::SwigPyIterator *std_vector_Sl_std_string_Sg__iterator(std::vector< std::string > *self,PyObject **PYTHON_SELF){
+      return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF);
+    }
+SWIGINTERN bool std_vector_Sl_std_string_Sg____nonzero__(std::vector< std::string > const *self){
+      return !(self->empty());
+    }
+SWIGINTERN bool std_vector_Sl_std_string_Sg____bool__(std::vector< std::string > const *self){
+      return !(self->empty());
+    }
+SWIGINTERN std::vector< std::string >::size_type std_vector_Sl_std_string_Sg____len__(std::vector< std::string > const *self){
+      return self->size();
+    }
+SWIGINTERN std::vector< std::string,std::allocator< std::string > > *std_vector_Sl_std_string_Sg____getslice__(std::vector< std::string > *self,std::vector< std::string >::difference_type i,std::vector< std::string >::difference_type j){
+      return swig::getslice(self, i, j, 1);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____setslice____SWIG_0(std::vector< std::string > *self,std::vector< std::string >::difference_type i,std::vector< std::string >::difference_type j){
+      swig::setslice(self, i, j, 1, std::vector< std::string,std::allocator< std::string > >());
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____setslice____SWIG_1(std::vector< std::string > *self,std::vector< std::string >::difference_type i,std::vector< std::string >::difference_type j,std::vector< std::string,std::allocator< std::string > > const &v){
+      swig::setslice(self, i, j, 1, v);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____delslice__(std::vector< std::string > *self,std::vector< std::string >::difference_type i,std::vector< std::string >::difference_type j){
+      swig::delslice(self, i, j, 1);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____delitem____SWIG_0(std::vector< std::string > *self,std::vector< std::string >::difference_type i){
+      swig::erase(self, swig::getpos(self, i));
+    }
+SWIGINTERN std::vector< std::string,std::allocator< std::string > > *std_vector_Sl_std_string_Sg____getitem____SWIG_0(std::vector< std::string > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return NULL;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::string,std::allocator< std::string > >::difference_type id = i;
+      std::vector< std::string,std::allocator< std::string > >::difference_type jd = j;
+      return swig::getslice(self, id, jd, step);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____setitem____SWIG_0(std::vector< std::string > *self,SWIGPY_SLICEOBJECT *slice,std::vector< std::string,std::allocator< std::string > > const &v){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::string,std::allocator< std::string > >::difference_type id = i;
+      std::vector< std::string,std::allocator< std::string > >::difference_type jd = j;
+      swig::setslice(self, id, jd, step, v);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____setitem____SWIG_1(std::vector< std::string > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::string,std::allocator< std::string > >::difference_type id = i;
+      std::vector< std::string,std::allocator< std::string > >::difference_type jd = j;
+      swig::delslice(self, id, jd, step);
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____delitem____SWIG_1(std::vector< std::string > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::string,std::allocator< std::string > >::difference_type id = i;
+      std::vector< std::string,std::allocator< std::string > >::difference_type jd = j;
+      swig::delslice(self, id, jd, step);
+    }
+SWIGINTERN std::vector< std::string >::value_type const &std_vector_Sl_std_string_Sg____getitem____SWIG_1(std::vector< std::string > const *self,std::vector< std::string >::difference_type i){
+      return *(swig::cgetpos(self, i));
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg____setitem____SWIG_2(std::vector< std::string > *self,std::vector< std::string >::difference_type i,std::vector< std::string >::value_type const &x){
+      *(swig::getpos(self,i)) = x;
+    }
+SWIGINTERN std::vector< std::string >::value_type std_vector_Sl_std_string_Sg__pop(std::vector< std::string > *self){
+      if (self->size() == 0)
+	throw std::out_of_range("pop from empty container");
+      std::vector< std::string,std::allocator< std::string > >::value_type x = self->back();
+      self->pop_back();
+      return x;
+    }
+SWIGINTERN void std_vector_Sl_std_string_Sg__append(std::vector< std::string > *self,std::vector< std::string >::value_type const &x){
+      self->push_back(x);
+    }
+SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__erase__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::iterator pos){ return self->erase(pos); }
+SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__erase__SWIG_1(std::vector< std::string > *self,std::vector< std::string >::iterator first,std::vector< std::string >::iterator last){ return self->erase(first, last); }
+SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__insert__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::value_type const &x){ return self->insert(pos, x); }
+SWIGINTERN void std_vector_Sl_std_string_Sg__insert__SWIG_1(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::size_type n,std::vector< std::string >::value_type const &x){ self->insert(pos, n, x); }
+
+
+
+
+SWIGINTERN int
+SWIG_AsVal_int (PyObject * obj, int *val)
+{
+  long v;
+  int res = SWIG_AsVal_long (obj, &v);
+  if (SWIG_IsOK(res)) {
+    if ((v < INT_MIN || v > INT_MAX)) {
+      return SWIG_OverflowError;
+    } else {
+      if (val) *val = static_cast< int >(v);
+    }
+  }  
+  return res;
+}
+
+
+struct SWIG_null_deleter {
+  void operator() (void const *) const {
+  }
+};
+#define SWIG_NO_NULL_DELETER_0 , SWIG_null_deleter()
+#define SWIG_NO_NULL_DELETER_1
+#define SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW
+#define SWIG_NO_NULL_DELETER_SWIG_POINTER_OWN
+
+
+#define SWIG_NO_NULL_DELETER_SWIG_BUILTIN_INIT
+
+
 SWIGINTERN int
 SWIG_AsVal_unsigned_SS_short (PyObject * obj, unsigned short *val)
 {
@@ -6018,6 +6158,22 @@ SWIGINTERNINLINE PyObject*
 }
 
 
+SWIGINTERN int
+SWIG_AsVal_short (PyObject * obj, short *val)
+{
+  long v;
+  int res = SWIG_AsVal_long (obj, &v);
+  if (SWIG_IsOK(res)) {
+    if ((v < SHRT_MIN || v > SHRT_MAX)) {
+      return SWIG_OverflowError;
+    } else {
+      if (val) *val = static_cast< short >(v);
+    }
+  }  
+  return res;
+}
+
+
 
 /* ---------------------------------------------------
  * C++ director class methods
@@ -11506,96 +11662,1990 @@ SWIGINTERN PyObject *_wrap_ibytes_back(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_assign(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_ibytes_assign(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  std::vector< signed char >::size_type arg2 ;
+  std::vector< signed char >::value_type *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  std::vector< signed char >::value_type temp3 ;
+  signed char val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "ibytes_assign", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_assign" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_assign" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
+  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_assign" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
+  } 
+  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
+  arg3 = &temp3;
+  (arg1)->assign(arg2,(std::vector< signed char >::value_type const &)*arg3);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  std::vector< signed char >::size_type arg2 ;
+  std::vector< signed char >::value_type *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  std::vector< signed char >::value_type temp3 ;
+  signed char val3 ;
+  int ecode3 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_resize" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_resize" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
+  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_resize" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
+  } 
+  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
+  arg3 = &temp3;
+  (arg1)->resize(arg2,(std::vector< signed char >::value_type const &)*arg3);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_resize(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_resize", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_size_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_ibytes_resize__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_size_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        {
+          int res = SWIG_AsVal_signed_SS_char(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          return _wrap_ibytes_resize__SWIG_1(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_resize'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< int8_t >::resize(std::vector< signed char >::size_type)\n"
+    "    std::vector< int8_t >::resize(std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  SwigValueWrapper< std::vector< signed char >::iterator > arg2 ;
+  std::vector< signed char >::value_type *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  std::vector< signed char >::value_type temp3 ;
+  signed char val3 ;
+  int ecode3 = 0 ;
+  SwigValueWrapper< std::vector< signed char >::iterator > result;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+    }
+  }
+  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
+  } 
+  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
+  arg3 = &temp3;
+  result = std_vector_Sl_int8_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3);
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  SwigValueWrapper< std::vector< signed char >::iterator > arg2 ;
+  std::vector< signed char >::size_type arg3 ;
+  std::vector< signed char >::value_type *arg4 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  size_t val3 ;
+  int ecode3 = 0 ;
+  std::vector< signed char >::value_type temp4 ;
+  signed char val4 ;
+  int ecode4 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+    }
+  }
+  ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::size_type""'");
+  } 
+  arg3 = static_cast< std::vector< signed char >::size_type >(val3);
+  ecode4 = SWIG_AsVal_signed_SS_char(swig_obj[3], &val4);
+  if (!SWIG_IsOK(ecode4)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "ibytes_insert" "', argument " "4"" of type '" "std::vector< signed char >::value_type""'");
+  } 
+  temp4 = static_cast< std::vector< signed char >::value_type >(val4);
+  arg4 = &temp4;
+  std_vector_Sl_int8_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(signed char const &)*arg4);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_insert(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[5] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_insert", 0, 4, argv))) SWIG_fail;
+  --argc;
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      if (_v) {
+        {
+          int res = SWIG_AsVal_signed_SS_char(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          return _wrap_ibytes_insert__SWIG_0(self, argc, argv);
+        }
+      }
+    }
+  }
+  if (argc == 4) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      if (_v) {
+        {
+          int res = SWIG_AsVal_size_t(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          {
+            int res = SWIG_AsVal_signed_SS_char(argv[3], NULL);
+            _v = SWIG_CheckState(res);
+          }
+          if (_v) {
+            return _wrap_ibytes_insert__SWIG_1(self, argc, argv);
+          }
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_insert'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::value_type const &)\n"
+    "    std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_reserve(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  std::vector< signed char >::size_type arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "ibytes_reserve", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_reserve" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_reserve" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
+  (arg1)->reserve(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ibytes_capacity(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< signed char >::size_type result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_capacity" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  result = ((std::vector< int8_t > const *)arg1)->capacity();
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *ibytes_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *ibytes_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_StringVec_iterator(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  PyObject **arg2 = (PyObject **) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  swig::SwigPyIterator *result = 0 ;
+  
+  arg2 = &swig_obj[0];
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_iterator" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (swig::SwigPyIterator *)std_vector_Sl_std_string_Sg__iterator(arg1,arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___nonzero__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___nonzero__" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (bool)std_vector_Sl_std_string_Sg____nonzero__((std::vector< std::string > const *)arg1);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___bool__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___bool__" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (bool)std_vector_Sl_std_string_Sg____bool__((std::vector< std::string > const *)arg1);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___len__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::size_type result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___len__" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = std_vector_Sl_std_string_Sg____len__((std::vector< std::string > const *)arg1);
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___getslice__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  std::vector< std::string >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
+  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "StringVec___getslice__", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___getslice__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___getslice__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "StringVec___getslice__" "', argument " "3"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::string >::difference_type >(val3);
+  try {
+    result = (std::vector< std::string,std::allocator< std::string > > *)std_vector_Sl_std_string_Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  std::vector< std::string >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___setslice__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___setslice__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "StringVec___setslice__" "', argument " "3"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::string >::difference_type >(val3);
+  try {
+    std_vector_Sl_std_string_Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  std::vector< std::string >::difference_type arg3 ;
+  std::vector< std::string,std::allocator< std::string > > *arg4 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  int res4 = SWIG_OLDOBJ ;
+  
+  (void)self;
+  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___setslice__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___setslice__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "StringVec___setslice__" "', argument " "3"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::string >::difference_type >(val3);
+  {
+    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
+    res4 = swig::asptr(swig_obj[3], &ptr);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "StringVec___setslice__" "', argument " "4"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec___setslice__" "', argument " "4"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    arg4 = ptr;
+  }
+  try {
+    std_vector_Sl_std_string_Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< std::string,std::allocator< std::string > > const &)*arg4);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res4)) delete arg4;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res4)) delete arg4;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setslice__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[5] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec___setslice__", 0, 4, argv))) SWIG_fail;
+  --argc;
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        {
+          int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          return _wrap_StringVec___setslice____SWIG_0(self, argc, argv);
+        }
+      }
+    }
+  }
+  if (argc == 4) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        {
+          int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          int res = swig::asptr(argv[3], (std::vector< std::string,std::allocator< std::string > >**)(0));
+          _v = SWIG_CheckState(res);
+          if (_v) {
+            return _wrap_StringVec___setslice____SWIG_1(self, argc, argv);
+          }
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec___setslice__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::__setslice__(std::vector< std::string >::difference_type,std::vector< std::string >::difference_type)\n"
+    "    std::vector< std::string >::__setslice__(std::vector< std::string >::difference_type,std::vector< std::string >::difference_type,std::vector< std::string,std::allocator< std::string > > const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___delslice__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  std::vector< std::string >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "StringVec___delslice__", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___delslice__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___delslice__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "StringVec___delslice__" "', argument " "3"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::string >::difference_type >(val3);
+  try {
+    std_vector_Sl_std_string_Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___delitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___delitem__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  try {
+    std_vector_Sl_std_string_Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___getitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
+    }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
+  }
+  try {
+    result = (std::vector< std::string,std::allocator< std::string > > *)std_vector_Sl_std_string_Sg____getitem____SWIG_0(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  std::vector< std::string,std::allocator< std::string > > *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res3 = SWIG_OLDOBJ ;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___setitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
+    }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
+  }
+  {
+    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
+    res3 = swig::asptr(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "StringVec___setitem__" "', argument " "3"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec___setitem__" "', argument " "3"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  try {
+    std_vector_Sl_std_string_Sg____setitem____SWIG_0(arg1,arg2,(std::vector< std::string,std::allocator< std::string > > const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___setitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
+    }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
+  }
+  try {
+    std_vector_Sl_std_string_Sg____setitem____SWIG_1(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___delitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
+    }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
+  }
+  try {
+    std_vector_Sl_std_string_Sg____delitem____SWIG_1(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___delitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec___delitem__", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_StringVec___delitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_StringVec___delitem____SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec___delitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::__delitem__(std::vector< std::string >::difference_type)\n"
+    "    std::vector< std::string >::__delitem__(SWIGPY_SLICEOBJECT *)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  std::vector< std::string >::value_type *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___getitem__" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___getitem__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  try {
+    result = (std::vector< std::string >::value_type *) &std_vector_Sl_std_string_Sg____getitem____SWIG_1((std::vector< std::string > const *)arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  }
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___getitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec___getitem__", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_StringVec___getitem____SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_StringVec___getitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec___getitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::__getitem__(SWIGPY_SLICEOBJECT *)\n"
+    "    std::vector< std::string >::__getitem__(std::vector< std::string >::difference_type) const\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::difference_type arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  int res3 = SWIG_OLDOBJ ;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec___setitem__" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec___setitem__" "', argument " "2"" of type '" "std::vector< std::string >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::difference_type >(val2);
+  {
+    std::string *ptr = (std::string *)0;
+    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "StringVec___setitem__" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec___setitem__" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  try {
+    std_vector_Sl_std_string_Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(std::string const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec___setitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec___setitem__", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_StringVec___setitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        int res = swig::asptr(argv[2], (std::vector< std::string,std::allocator< std::string > >**)(0));
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_StringVec___setitem____SWIG_0(self, argc, argv);
+        }
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0));
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_StringVec___setitem____SWIG_2(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec___setitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< std::string,std::allocator< std::string > > const &)\n"
+    "    std::vector< std::string >::__setitem__(SWIGPY_SLICEOBJECT *)\n"
+    "    std::vector< std::string >::__setitem__(std::vector< std::string >::difference_type,std::vector< std::string >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_pop(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::value_type result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_pop" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  try {
+    result = std_vector_Sl_std_string_Sg__pop(arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  }
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_append(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::value_type *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "StringVec_append", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_append" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "StringVec_append" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_append" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  std_vector_Sl_std_string_Sg__append(arg1,(std::string const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_StringVec__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  result = (std::vector< std::string > *)new std::vector< std::string >();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_StringVec__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = 0 ;
+  int res1 = SWIG_OLDOBJ ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  {
+    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
+    res1 = swig::asptr(swig_obj[0], &ptr);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_StringVec" "', argument " "1"" of type '" "std::vector< std::string > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_StringVec" "', argument " "1"" of type '" "std::vector< std::string > const &""'"); 
+    }
+    arg1 = ptr;
+  }
+  result = (std::vector< std::string > *)new std::vector< std::string >((std::vector< std::string > const &)*arg1);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_NEW |  0 );
+  if (SWIG_IsNewObj(res1)) delete arg1;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res1)) delete arg1;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_empty(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_empty" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (bool)((std::vector< std::string > const *)arg1)->empty();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_size(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::size_type result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_size" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = ((std::vector< std::string > const *)arg1)->size();
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_swap(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string > *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "StringVec_swap", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_swap" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_std__string_t,  0 );
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "StringVec_swap" "', argument " "2"" of type '" "std::vector< std::string > &""'"); 
+  }
+  if (!argp2) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_swap" "', argument " "2"" of type '" "std::vector< std::string > &""'"); 
+  }
+  arg2 = reinterpret_cast< std::vector< std::string > * >(argp2);
+  (arg1)->swap(*arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_begin(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::iterator result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_begin" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (arg1)->begin();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_end(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::iterator result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_end" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (arg1)->end();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_rbegin(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::reverse_iterator result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_rbegin" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (arg1)->rbegin();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::reverse_iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_rend(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::reverse_iterator result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_rend" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (arg1)->rend();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::reverse_iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_clear(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_clear" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  (arg1)->clear();
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_get_allocator(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  SwigValueWrapper< std::allocator< std::string > > result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_get_allocator" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = ((std::vector< std::string > const *)arg1)->get_allocator();
+  resultobj = SWIG_NewPointerObj((new std::vector< std::string >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_std__string_t, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_StringVec__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string >::size_type arg1 ;
+  size_t val1 ;
+  int ecode1 = 0 ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1);
+  if (!SWIG_IsOK(ecode1)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_StringVec" "', argument " "1"" of type '" "std::vector< std::string >::size_type""'");
+  } 
+  arg1 = static_cast< std::vector< std::string >::size_type >(val1);
+  result = (std::vector< std::string > *)new std::vector< std::string >(arg1);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_pop_back(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_pop_back" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  (arg1)->pop_back();
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::size_type arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_resize" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec_resize" "', argument " "2"" of type '" "std::vector< std::string >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::string >::size_type >(val2);
+  (arg1)->resize(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::iterator arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  std::vector< std::string >::iterator result;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_erase" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
+    }
+  }
+  result = std_vector_Sl_std_string_Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2));
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::iterator arg2 ;
+  std::vector< std::string >::iterator arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  swig::SwigPyIterator *iter3 = 0 ;
+  int res3 ;
+  std::vector< std::string >::iterator result;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_erase" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
+    }
+  }
+  res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res3) || !iter3) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "3"" of type '" "std::vector< std::string >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter3);
+    if (iter_t) {
+      arg3 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_erase" "', argument " "3"" of type '" "std::vector< std::string >::iterator""'");
+    }
+  }
+  result = std_vector_Sl_std_string_Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_erase(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec_erase", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      if (_v) {
+        return _wrap_StringVec_erase__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      if (_v) {
+        swig::SwigPyIterator *iter = 0;
+        int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+        _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+        if (_v) {
+          return _wrap_StringVec_erase__SWIG_1(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec_erase'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::erase(std::vector< std::string >::iterator)\n"
+    "    std::vector< std::string >::erase(std::vector< std::string >::iterator,std::vector< std::string >::iterator)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_StringVec__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::string >::size_type arg1 ;
+  std::vector< std::string >::value_type *arg2 = 0 ;
+  size_t val1 ;
+  int ecode1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1);
+  if (!SWIG_IsOK(ecode1)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_StringVec" "', argument " "1"" of type '" "std::vector< std::string >::size_type""'");
+  } 
+  arg1 = static_cast< std::vector< std::string >::size_type >(val1);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_StringVec" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_StringVec" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  result = (std::vector< std::string > *)new std::vector< std::string >(arg1,(std::vector< std::string >::value_type const &)*arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_NEW |  0 );
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_StringVec(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_StringVec", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 0) {
+    return _wrap_new_StringVec__SWIG_0(self, argc, argv);
+  }
+  if (argc == 1) {
+    int _v = 0;
+    {
+      int res = SWIG_AsVal_size_t(argv[0], NULL);
+      _v = SWIG_CheckState(res);
+    }
+    if (_v) {
+      return _wrap_new_StringVec__SWIG_2(self, argc, argv);
+    }
+  }
+  if (argc == 1) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_new_StringVec__SWIG_1(self, argc, argv);
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    {
+      int res = SWIG_AsVal_size_t(argv[0], NULL);
+      _v = SWIG_CheckState(res);
+    }
+    if (_v) {
+      int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0));
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_new_StringVec__SWIG_3(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_StringVec'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::string >::vector()\n"
+    "    std::vector< std::string >::vector(std::vector< std::string > const &)\n"
+    "    std::vector< std::string >::vector(std::vector< std::string >::size_type)\n"
+    "    std::vector< std::string >::vector(std::vector< std::string >::size_type,std::vector< std::string >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_push_back(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::value_type *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "StringVec_push_back", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_push_back" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "StringVec_push_back" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_push_back" "', argument " "2"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->push_back((std::vector< std::string >::value_type const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_front(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::value_type *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_front" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (std::vector< std::string >::value_type *) &((std::vector< std::string > const *)arg1)->front();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_back(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string >::value_type *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_back" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = (std::vector< std::string >::value_type *) &((std::vector< std::string > const *)arg1)->back();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  (void)swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]);
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_StringVec_assign(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
-  std::vector< signed char >::size_type arg2 ;
-  std::vector< signed char >::value_type *arg3 = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::size_type arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   size_t val2 ;
   int ecode2 = 0 ;
-  std::vector< signed char >::value_type temp3 ;
-  signed char val3 ;
-  int ecode3 = 0 ;
+  int res3 = SWIG_OLDOBJ ;
   PyObject *swig_obj[3] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "ibytes_assign", 3, 3, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "StringVec_assign", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_assign" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_assign" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_assign" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
-  } 
-  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
-  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_assign" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec_assign" "', argument " "2"" of type '" "std::vector< std::string >::size_type""'");
   } 
-  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
-  arg3 = &temp3;
-  (arg1)->assign(arg2,(std::vector< signed char >::value_type const &)*arg3);
+  arg2 = static_cast< std::vector< std::string >::size_type >(val2);
+  {
+    std::string *ptr = (std::string *)0;
+    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "StringVec_assign" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_assign" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  (arg1)->assign(arg2,(std::vector< std::string >::value_type const &)*arg3);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_StringVec_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
-  std::vector< signed char >::size_type arg2 ;
-  std::vector< signed char >::value_type *arg3 = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::size_type arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   size_t val2 ;
   int ecode2 = 0 ;
-  std::vector< signed char >::value_type temp3 ;
-  signed char val3 ;
-  int ecode3 = 0 ;
+  int res3 = SWIG_OLDOBJ ;
   
   (void)self;
   if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_resize" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_resize" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_resize" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
-  } 
-  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
-  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_resize" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec_resize" "', argument " "2"" of type '" "std::vector< std::string >::size_type""'");
   } 
-  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
-  arg3 = &temp3;
-  (arg1)->resize(arg2,(std::vector< signed char >::value_type const &)*arg3);
+  arg2 = static_cast< std::vector< std::string >::size_type >(val2);
+  {
+    std::string *ptr = (std::string *)0;
+    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "StringVec_resize" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_resize" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  (arg1)->resize(arg2,(std::vector< std::string >::value_type const &)*arg3);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_resize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_StringVec_resize(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
   PyObject *argv[4] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_resize", 0, 3, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec_resize", 0, 3, argv))) SWIG_fail;
   --argc;
   if (argc == 2) {
     int _v = 0;
-    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
       {
@@ -11603,13 +13653,13 @@ SWIGINTERN PyObject *_wrap_ibytes_resize(PyObject *self, PyObject *args) {
         _v = SWIG_CheckState(res);
       }
       if (_v) {
-        return _wrap_ibytes_resize__SWIG_0(self, argc, argv);
+        return _wrap_StringVec_resize__SWIG_0(self, argc, argv);
       }
     }
   }
   if (argc == 3) {
     int _v = 0;
-    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
       {
@@ -11617,173 +13667,177 @@ SWIGINTERN PyObject *_wrap_ibytes_resize(PyObject *self, PyObject *args) {
         _v = SWIG_CheckState(res);
       }
       if (_v) {
-        {
-          int res = SWIG_AsVal_signed_SS_char(argv[2], NULL);
-          _v = SWIG_CheckState(res);
-        }
+        int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0));
+        _v = SWIG_CheckState(res);
         if (_v) {
-          return _wrap_ibytes_resize__SWIG_1(self, argc, argv);
+          return _wrap_StringVec_resize__SWIG_1(self, argc, argv);
         }
       }
     }
   }
   
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_resize'.\n"
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec_resize'.\n"
     "  Possible C/C++ prototypes are:\n"
-    "    std::vector< int8_t >::resize(std::vector< signed char >::size_type)\n"
-    "    std::vector< int8_t >::resize(std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n");
+    "    std::vector< std::string >::resize(std::vector< std::string >::size_type)\n"
+    "    std::vector< std::string >::resize(std::vector< std::string >::size_type,std::vector< std::string >::value_type const &)\n");
   return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_StringVec_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
-  SwigValueWrapper< std::vector< signed char >::iterator > arg2 ;
-  std::vector< signed char >::value_type *arg3 = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::iterator arg2 ;
+  std::vector< std::string >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   swig::SwigPyIterator *iter2 = 0 ;
   int res2 ;
-  std::vector< signed char >::value_type temp3 ;
-  signed char val3 ;
-  int ecode3 = 0 ;
-  SwigValueWrapper< std::vector< signed char >::iterator > result;
+  int res3 = SWIG_OLDOBJ ;
+  std::vector< std::string >::iterator result;
   
   (void)self;
   if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_insert" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
   if (!SWIG_IsOK(res2) || !iter2) {
-    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_insert" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
   } else {
-    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
     if (iter_t) {
       arg2 = iter_t->get_current();
     } else {
-      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_insert" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
     }
   }
-  ecode3 = SWIG_AsVal_signed_SS_char(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::value_type""'");
-  } 
-  temp3 = static_cast< std::vector< signed char >::value_type >(val3);
-  arg3 = &temp3;
-  result = std_vector_Sl_int8_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(signed char const &)*arg3);
-  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< signed char >::iterator & >(result)),
+  {
+    std::string *ptr = (std::string *)0;
+    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "StringVec_insert" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_insert" "', argument " "3"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  result = std_vector_Sl_std_string_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(std::string const &)*arg3);
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::string >::iterator & >(result)),
     swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_StringVec_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
-  SwigValueWrapper< std::vector< signed char >::iterator > arg2 ;
-  std::vector< signed char >::size_type arg3 ;
-  std::vector< signed char >::value_type *arg4 = 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::iterator arg2 ;
+  std::vector< std::string >::size_type arg3 ;
+  std::vector< std::string >::value_type *arg4 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   swig::SwigPyIterator *iter2 = 0 ;
   int res2 ;
   size_t val3 ;
   int ecode3 = 0 ;
-  std::vector< signed char >::value_type temp4 ;
-  signed char val4 ;
-  int ecode4 = 0 ;
+  int res4 = SWIG_OLDOBJ ;
   
   (void)self;
   if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_insert" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_insert" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
   if (!SWIG_IsOK(res2) || !iter2) {
-    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_insert" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
   } else {
-    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
+    swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2);
     if (iter_t) {
       arg2 = iter_t->get_current();
     } else {
-      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "ibytes_insert" "', argument " "2"" of type '" "std::vector< signed char >::iterator""'");
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "StringVec_insert" "', argument " "2"" of type '" "std::vector< std::string >::iterator""'");
     }
   }
   ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3);
   if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "ibytes_insert" "', argument " "3"" of type '" "std::vector< signed char >::size_type""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "StringVec_insert" "', argument " "3"" of type '" "std::vector< std::string >::size_type""'");
   } 
-  arg3 = static_cast< std::vector< signed char >::size_type >(val3);
-  ecode4 = SWIG_AsVal_signed_SS_char(swig_obj[3], &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "ibytes_insert" "', argument " "4"" of type '" "std::vector< signed char >::value_type""'");
-  } 
-  temp4 = static_cast< std::vector< signed char >::value_type >(val4);
-  arg4 = &temp4;
-  std_vector_Sl_int8_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(signed char const &)*arg4);
+  arg3 = static_cast< std::vector< std::string >::size_type >(val3);
+  {
+    std::string *ptr = (std::string *)0;
+    res4 = SWIG_AsPtr_std_string(swig_obj[3], &ptr);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "StringVec_insert" "', argument " "4"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "StringVec_insert" "', argument " "4"" of type '" "std::vector< std::string >::value_type const &""'"); 
+    }
+    arg4 = ptr;
+  }
+  std_vector_Sl_std_string_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::string const &)*arg4);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res4)) delete arg4;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res4)) delete arg4;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_insert(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_StringVec_insert(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
   PyObject *argv[5] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "ibytes_insert", 0, 4, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "StringVec_insert", 0, 4, argv))) SWIG_fail;
   --argc;
   if (argc == 3) {
     int _v = 0;
-    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
       swig::SwigPyIterator *iter = 0;
       int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
-      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
       if (_v) {
-        {
-          int res = SWIG_AsVal_signed_SS_char(argv[2], NULL);
-          _v = SWIG_CheckState(res);
-        }
+        int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0));
+        _v = SWIG_CheckState(res);
         if (_v) {
-          return _wrap_ibytes_insert__SWIG_0(self, argc, argv);
+          return _wrap_StringVec_insert__SWIG_0(self, argc, argv);
         }
       }
     }
   }
   if (argc == 4) {
     int _v = 0;
-    int res = swig::asptr(argv[0], (std::vector< signed char,std::allocator< signed char > >**)(0));
+    int res = swig::asptr(argv[0], (std::vector< std::string,std::allocator< std::string > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
       swig::SwigPyIterator *iter = 0;
       int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
-      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0));
       if (_v) {
         {
           int res = SWIG_AsVal_size_t(argv[2], NULL);
           _v = SWIG_CheckState(res);
         }
         if (_v) {
-          {
-            int res = SWIG_AsVal_signed_SS_char(argv[3], NULL);
-            _v = SWIG_CheckState(res);
-          }
+          int res = SWIG_AsPtr_std_string(argv[3], (std::string**)(0));
+          _v = SWIG_CheckState(res);
           if (_v) {
-            return _wrap_ibytes_insert__SWIG_1(self, argc, argv);
+            return _wrap_StringVec_insert__SWIG_1(self, argc, argv);
           }
         }
       }
@@ -11791,18 +13845,18 @@ SWIGINTERN PyObject *_wrap_ibytes_insert(PyObject *self, PyObject *args) {
   }
   
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'ibytes_insert'.\n"
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'StringVec_insert'.\n"
     "  Possible C/C++ prototypes are:\n"
-    "    std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::value_type const &)\n"
-    "    std::vector< int8_t >::insert(std::vector< signed char >::iterator,std::vector< signed char >::size_type,std::vector< signed char >::value_type const &)\n");
+    "    std::vector< std::string >::insert(std::vector< std::string >::iterator,std::vector< std::string >::value_type const &)\n"
+    "    std::vector< std::string >::insert(std::vector< std::string >::iterator,std::vector< std::string >::size_type,std::vector< std::string >::value_type const &)\n");
   return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_reserve(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_StringVec_reserve(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
-  std::vector< signed char >::size_type arg2 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
+  std::vector< std::string >::size_type arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   size_t val2 ;
@@ -11810,17 +13864,17 @@ SWIGINTERN PyObject *_wrap_ibytes_reserve(PyObject *self, PyObject *args) {
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "ibytes_reserve", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "StringVec_reserve", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_reserve" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_reserve" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ibytes_reserve" "', argument " "2"" of type '" "std::vector< signed char >::size_type""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "StringVec_reserve" "', argument " "2"" of type '" "std::vector< std::string >::size_type""'");
   } 
-  arg2 = static_cast< std::vector< signed char >::size_type >(val2);
+  arg2 = static_cast< std::vector< std::string >::size_type >(val2);
   (arg1)->reserve(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
@@ -11829,23 +13883,23 @@ SWIGINTERN PyObject *_wrap_ibytes_reserve(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_ibytes_capacity(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_StringVec_capacity(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::vector< signed char >::size_type result;
+  std::vector< std::string >::size_type result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ibytes_capacity" "', argument " "1"" of type '" "std::vector< int8_t > const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "StringVec_capacity" "', argument " "1"" of type '" "std::vector< std::string > const *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
-  result = ((std::vector< int8_t > const *)arg1)->capacity();
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
+  result = ((std::vector< std::string > const *)arg1)->capacity();
   resultobj = SWIG_From_size_t(static_cast< size_t >(result));
   return resultobj;
 fail:
@@ -11853,9 +13907,9 @@ SWIGINTERN PyObject *_wrap_ibytes_capacity(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_StringVec(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::vector< int8_t > *arg1 = (std::vector< int8_t > *) 0 ;
+  std::vector< std::string > *arg1 = (std::vector< std::string > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
@@ -11863,11 +13917,11 @@ SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) {
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_POINTER_DISOWN |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__string_t, SWIG_POINTER_DISOWN |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_StringVec" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
-  arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
+  arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
   delete arg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
@@ -11876,14 +13930,14 @@ SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *ibytes_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *StringVec_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_int8_t_t, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_std__string_t, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *ibytes_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *StringVec_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   return SWIG_Python_InitShadowInstance(args);
 }
 
@@ -13398,7 +15452,7 @@ SWIGINTERN PyObject *_wrap_FrontConfig_getComponents(PyObject *self, PyObject *a
     }
   }
   result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0 |  0 );
+  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(*result));
   return resultobj;
 fail:
   return NULL;
@@ -13413,8 +15467,7 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *a
   int res1 = 0 ;
   std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
   std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
@@ -13434,18 +15487,23 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *a
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t,  0  | 0);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
-  }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+  {
+    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
+    res2 = swig::asptr(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    arg2 = ptr;
   }
-  arg2 = reinterpret_cast< std::vector< std::string,std::allocator< std::string > > * >(argp2);
   (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
@@ -13528,7 +15586,7 @@ SWIGINTERN PyObject *_wrap_FrontConfig_mutableComponents(PyObject *self, PyObjec
     }
   }
   result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0 |  0 );
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
@@ -21609,6 +23667,67 @@ SWIGINTERN PyObject *_wrap_IFront_unRegisterComponent(PyObject *self, PyObject *
 }
 
 
+SWIGINTERN PyObject *_wrap_IFront_selectNodesByRoutePolicy(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  int16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  short val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
+  PyObject *swig_obj[3] ;
+  std::vector< std::string,std::allocator< std::string > > result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_selectNodesByRoutePolicy", 3, 3, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "2"" of type '" "int16_t""'");
+  } 
+  arg2 = static_cast< int16_t >(val2);
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      arg3 = &tempshared3;
+    } else {
+      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
+    }
+  }
+  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *IFront_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@@ -22438,6 +24557,41 @@ static PyMethodDef SwigMethods[] = {
 	 { "delete_ibytes", _wrap_delete_ibytes, METH_O, NULL},
 	 { "ibytes_swigregister", ibytes_swigregister, METH_O, NULL},
 	 { "ibytes_swiginit", ibytes_swiginit, METH_VARARGS, NULL},
+	 { "StringVec_iterator", _wrap_StringVec_iterator, METH_O, NULL},
+	 { "StringVec___nonzero__", _wrap_StringVec___nonzero__, METH_O, NULL},
+	 { "StringVec___bool__", _wrap_StringVec___bool__, METH_O, NULL},
+	 { "StringVec___len__", _wrap_StringVec___len__, METH_O, NULL},
+	 { "StringVec___getslice__", _wrap_StringVec___getslice__, METH_VARARGS, NULL},
+	 { "StringVec___setslice__", _wrap_StringVec___setslice__, METH_VARARGS, NULL},
+	 { "StringVec___delslice__", _wrap_StringVec___delslice__, METH_VARARGS, NULL},
+	 { "StringVec___delitem__", _wrap_StringVec___delitem__, METH_VARARGS, NULL},
+	 { "StringVec___getitem__", _wrap_StringVec___getitem__, METH_VARARGS, NULL},
+	 { "StringVec___setitem__", _wrap_StringVec___setitem__, METH_VARARGS, NULL},
+	 { "StringVec_pop", _wrap_StringVec_pop, METH_O, NULL},
+	 { "StringVec_append", _wrap_StringVec_append, METH_VARARGS, NULL},
+	 { "StringVec_empty", _wrap_StringVec_empty, METH_O, NULL},
+	 { "StringVec_size", _wrap_StringVec_size, METH_O, NULL},
+	 { "StringVec_swap", _wrap_StringVec_swap, METH_VARARGS, NULL},
+	 { "StringVec_begin", _wrap_StringVec_begin, METH_O, NULL},
+	 { "StringVec_end", _wrap_StringVec_end, METH_O, NULL},
+	 { "StringVec_rbegin", _wrap_StringVec_rbegin, METH_O, NULL},
+	 { "StringVec_rend", _wrap_StringVec_rend, METH_O, NULL},
+	 { "StringVec_clear", _wrap_StringVec_clear, METH_O, NULL},
+	 { "StringVec_get_allocator", _wrap_StringVec_get_allocator, METH_O, NULL},
+	 { "StringVec_pop_back", _wrap_StringVec_pop_back, METH_O, NULL},
+	 { "StringVec_erase", _wrap_StringVec_erase, METH_VARARGS, NULL},
+	 { "new_StringVec", _wrap_new_StringVec, METH_VARARGS, NULL},
+	 { "StringVec_push_back", _wrap_StringVec_push_back, METH_VARARGS, NULL},
+	 { "StringVec_front", _wrap_StringVec_front, METH_O, NULL},
+	 { "StringVec_back", _wrap_StringVec_back, METH_O, NULL},
+	 { "StringVec_assign", _wrap_StringVec_assign, METH_VARARGS, NULL},
+	 { "StringVec_resize", _wrap_StringVec_resize, METH_VARARGS, NULL},
+	 { "StringVec_insert", _wrap_StringVec_insert, METH_VARARGS, NULL},
+	 { "StringVec_reserve", _wrap_StringVec_reserve, METH_VARARGS, NULL},
+	 { "StringVec_capacity", _wrap_StringVec_capacity, METH_O, NULL},
+	 { "delete_StringVec", _wrap_delete_StringVec, METH_O, NULL},
+	 { "StringVec_swigregister", StringVec_swigregister, METH_O, NULL},
+	 { "StringVec_swiginit", StringVec_swiginit, METH_VARARGS, NULL},
 	 { "Error_buildError", _wrap_Error_buildError, METH_VARARGS, NULL},
 	 { "new_Error", _wrap_new_Error, METH_VARARGS, NULL},
 	 { "delete_Error", _wrap_delete_Error, METH_O, NULL},
@@ -22697,6 +24851,7 @@ static PyMethodDef SwigMethods[] = {
 		""},
 	 { "IFront_registerComponent", _wrap_IFront_registerComponent, METH_VARARGS, NULL},
 	 { "IFront_unRegisterComponent", _wrap_IFront_unRegisterComponent, METH_VARARGS, NULL},
+	 { "IFront_selectNodesByRoutePolicy", _wrap_IFront_selectNodesByRoutePolicy, METH_VARARGS, NULL},
 	 { "IFront_swigregister", IFront_swigregister, METH_O, NULL},
 	 { "delete_IFrontBuilder", _wrap_delete_IFrontBuilder, METH_O, NULL},
 	 { "IFrontBuilder_buildClient", _wrap_IFrontBuilder_buildClient, METH_VARARGS, NULL},
@@ -22781,6 +24936,7 @@ static swig_type_info _swigt__p_short = {"_p_short", "int16_t *|int_least16_t *|
 static swig_type_info _swigt__p_signed_char = {"_p_signed_char", "int8_t *|int_fast8_t *|int_least8_t *|signed char *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_size_type = {"_p_size_type", "size_type *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__allocatorT_signed_char_t = {"_p_std__allocatorT_signed_char_t", "std::vector< signed char >::allocator_type *|std::allocator< signed char > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__allocatorT_std__string_t = {"_p_std__allocatorT_std__string_t", "std::vector< std::string >::allocator_type *|std::allocator< std::string > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__allocatorT_unsigned_char_t = {"_p_std__allocatorT_unsigned_char_t", "std::vector< unsigned char >::allocator_type *|std::allocator< unsigned char > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__exception = {"_p_std__exception", "std::exception *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__functionT_void_fF_t = {"_p_std__functionT_void_fF_t", "std::function< void () > *", 0, 0, (void*)0, 0};
@@ -22813,7 +24969,7 @@ static swig_type_info _swigt__p_std__shared_ptrT_ppc__sdk__Transport_t = {"_p_st
 static swig_type_info _swigt__p_std__string = {"_p_std__string", "std::string *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__string_view = {"_p_std__string_view", "std::string_view *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__vectorT_int8_t_t = {"_p_std__vectorT_int8_t_t", "std::vector< int8_t > *|std::vector< signed char,std::allocator< signed char > > *|std::vector< signed char > *", 0, 0, (void*)0, 0};
-static swig_type_info _swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t = {"_p_std__vectorT_std__string_std__allocatorT_std__string_t_t", "std::vector< std::string,std::allocator< std::string > > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string,std::allocator< std::string > > *|std::vector< std::string > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__vectorT_uint8_t_t = {"_p_std__vectorT_uint8_t_t", "bcos::bytes *|std::vector< uint8_t > *|std::vector< unsigned char,std::allocator< unsigned char > > *|std::vector< unsigned char > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_swig__SwigPyIterator = {"_p_swig__SwigPyIterator", "swig::SwigPyIterator *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_unsigned_char = {"_p_unsigned_char", "bcos::byte *|uint8_t *|uint_fast8_t *|uint_least8_t *|unsigned char *", 0, 0, (void*)0, 0};
@@ -22870,6 +25026,7 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_signed_char,
   &_swigt__p_size_type,
   &_swigt__p_std__allocatorT_signed_char_t,
+  &_swigt__p_std__allocatorT_std__string_t,
   &_swigt__p_std__allocatorT_unsigned_char_t,
   &_swigt__p_std__exception,
   &_swigt__p_std__functionT_void_fF_t,
@@ -22902,7 +25059,7 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_std__string,
   &_swigt__p_std__string_view,
   &_swigt__p_std__vectorT_int8_t_t,
-  &_swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t,
+  &_swigt__p_std__vectorT_std__string_t,
   &_swigt__p_std__vectorT_uint8_t_t,
   &_swigt__p_swig__SwigPyIterator,
   &_swigt__p_unsigned_char,
@@ -22959,6 +25116,7 @@ static swig_cast_info _swigc__p_short[] = {  {&_swigt__p_short, 0, 0, 0},{0, 0,
 static swig_cast_info _swigc__p_signed_char[] = {  {&_swigt__p_signed_char, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_size_type[] = {  {&_swigt__p_size_type, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__allocatorT_signed_char_t[] = {  {&_swigt__p_std__allocatorT_signed_char_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__allocatorT_std__string_t[] = {  {&_swigt__p_std__allocatorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__allocatorT_unsigned_char_t[] = {  {&_swigt__p_std__allocatorT_unsigned_char_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__exception[] = {  {&_swigt__p_std__exception, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__functionT_void_fF_t[] = {  {&_swigt__p_std__functionT_void_fF_t, 0, 0, 0},{0, 0, 0, 0}};
@@ -22991,7 +25149,7 @@ static swig_cast_info _swigc__p_std__shared_ptrT_ppc__sdk__Transport_t[] = {  {&
 static swig_cast_info _swigc__p_std__string[] = {  {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__string_view[] = {  {&_swigt__p_std__string_view, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__vectorT_int8_t_t[] = {  {&_swigt__p_std__vectorT_int8_t_t, 0, 0, 0},{0, 0, 0, 0}};
-static swig_cast_info _swigc__p_std__vectorT_std__string_std__allocatorT_std__string_t_t[] = {  {&_swigt__p_std__vectorT_std__string_std__allocatorT_std__string_t_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = {  {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__vectorT_uint8_t_t[] = {  {&_swigt__p_std__vectorT_uint8_t_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_swig__SwigPyIterator[] = {  {&_swigt__p_swig__SwigPyIterator, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_unsigned_char[] = {  {&_swigt__p_unsigned_char, 0, 0, 0},{0, 0, 0, 0}};
@@ -23048,6 +25206,7 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_signed_char,
   _swigc__p_size_type,
   _swigc__p_std__allocatorT_signed_char_t,
+  _swigc__p_std__allocatorT_std__string_t,
   _swigc__p_std__allocatorT_unsigned_char_t,
   _swigc__p_std__exception,
   _swigc__p_std__functionT_void_fF_t,
@@ -23080,7 +25239,7 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_std__string,
   _swigc__p_std__string_view,
   _swigc__p_std__vectorT_int8_t_t,
-  _swigc__p_std__vectorT_std__string_std__allocatorT_std__string_t_t,
+  _swigc__p_std__vectorT_std__string_t,
   _swigc__p_std__vectorT_uint8_t_t,
   _swigc__p_swig__SwigPyIterator,
   _swigc__p_unsigned_char,
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index 7ed79823..588e2461 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -96,6 +96,7 @@ namespace ppc::front{
 
 namespace std{
     class vector;
+    class string;
 }
 
 namespace bcos{
@@ -132,6 +133,7 @@ namespace bcos{
 
 %template(ubytes) std::vector;
 %template(ibytes) std::vector;
+%template(StringVec) std::vector;
 
 %include 
 %pybuffer_binary(char* data, uint64_t length)
diff --git a/python/ppc_common/ppc_protos/ppc_model.proto b/python/ppc_common/ppc_protos/ppc_model.proto
index ac1d72f1..228a37c2 100644
--- a/python/ppc_common/ppc_protos/ppc_model.proto
+++ b/python/ppc_common/ppc_protos/ppc_model.proto
@@ -1,36 +1,9 @@
 syntax = "proto3";
 
 option java_multiple_files = true;
-//option java_package = "unknown";
-//option java_outer_classname = "unknown";
 option objc_class_prefix = "PPC";
 
 package ppc.model;
-
-service ModelService {
-  rpc MessageInteraction (ModelRequest) returns (ModelResponse) {}
-}
-
-message ModelRequest {
-  string sender = 1;
-  string receiver = 2;
-  string task_id = 3;
-  string key = 4;
-  int64 seq = 5;
-  int64 slice_num = 6;
-  bytes data = 7;
-}
-
-message BaseResponse {
-  int64 error_code = 1;
-  string message = 2;
-}
-
-message ModelResponse {
-  BaseResponse base_response = 1;
-  bytes data = 2;
-}
-
 message PlainBoolList{
   repeated bool plain_list = 1;
 }
diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py
index 17b79082..8a89404e 100644
--- a/python/ppc_model/common/initializer.py
+++ b/python/ppc_model/common/initializer.py
@@ -6,47 +6,45 @@
 import yaml
 
 from ppc_common.deps_services import storage_loader
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from ppc_common.ppc_utils import common_func
-from ppc_model.network.grpc.grpc_client import GrpcClient
-from ppc_model.network.stub import ModelStub
+from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
+from wedpr_python_gateway_sdk.transport.impl.transport_loader import TransportLoader
+from ppc_model.network.wedpr_model_transport import ModelTransport
 from ppc_model.task.task_manager import TaskManager
 
 
 class Initializer:
     def __init__(self, log_config_path, config_path, plot_lock=None):
         self.log_config_path = log_config_path
+        logging.config.fileConfig(self.log_config_path)
         self.config_path = config_path
         self.config_data = None
-        self.grpc_options = None
-        self.stub = None
-        self.task_manager = None
-        self.thread_event_manager = None
-        self.storage_client = None
         # 只用于测试
         self.mock_logger = None
         self.public_key_length = 2048
         self.homo_algorithm = 0
+        self.init_config()
+        self.job_cache_dir = common_func.get_config_value(
+            "JOB_TEMP_DIR", "/tmp", self.config_data, False)
+        self.thread_event_manager = ThreadEventManager()
+        self.task_manager = TaskManager(
+            logger=self.logger(),
+            thread_event_manager=self.thread_event_manager,
+            task_timeout_h=self.config_data['TASK_TIMEOUT_H']
+        )
+        self.storage_client = storage_loader.load(
+            self.config_data, self.logger())
+        # default send msg timeout
+        self.MODEL_COMPONENT = "WEDPR_MODEL"
+        self.send_msg_timeout_ms = 5000
+        self.pop_msg_timeout_ms = 60000
+        # for UT
+        self.transport = None
         # matplotlib 线程不安全，并行任务绘图增加全局锁
         self.plot_lock = plot_lock
         if plot_lock is None:
             self.plot_lock = threading.Lock()
 
-    def init_all(self):
-        self.init_log()
-        self.init_config()
-        self.init_stub()
-        self.init_task_manager()
-        self.init_storage_client()
-        self.init_cache()
-
-    def init_log(self):
-        logging.config.fileConfig(self.log_config_path)
-
-    def init_cache(self):
-        self.job_cache_dir = common_func.get_config_value(
-            "JOB_TEMP_DIR", "/tmp", self.config_data, False)
-
     def init_config(self):
         with open(self.config_path, 'rb') as f:
             self.config_data = yaml.safe_load(f.read())
@@ -56,54 +54,37 @@ def init_config(self):
             if 'HOMO_ALGORITHM' in self.config_data:
                 self.homo_algorithm = self.config_data['HOMO_ALGORITHM']
 
-    def init_stub(self):
-        self.thread_event_manager = ThreadEventManager()
-        self.grpc_options = [
-            ('grpc.ssl_target_name_override', 'PPCS MODEL GATEWAY'),
-            ('grpc.max_send_message_length',
-             self.config_data['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024),
-            ('grpc.max_receive_message_length',
-             self.config_data['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024),
-            ('grpc.keepalive_time_ms', 15000),  # 每 15 秒发送一次心跳
-            ('grpc.keepalive_timeout_ms', 5000),  # 等待心跳回应的超时时间为 5 秒
-            ('grpc.keepalive_permit_without_calls', True),  # 即使没有调用也允许发送心跳
-            ('grpc.http2.min_time_between_pings_ms', 15000),  # 心跳之间最小时间间隔为 15 秒
-            ('grpc.http2.max_pings_without_data', 0),  # 在发送数据前不限制心跳次数
-            # 在没有数据传输的情况下，确保心跳包之间至少有 20 秒的间隔
-            ('grpc.http2.min_ping_interval_without_data_ms', 20000),
-            ("grpc.so_reuseport", 1),
-            ("grpc.use_local_subchannel_pool", 1),
-            ('grpc.enable_retries', 1),
-            ('grpc.service_config',
-             '{ "retryPolicy":{ "maxAttempts": 4, "initialBackoff": "0.1s", "maxBackoff": "1s", "backoffMutiplier": '
-             '2, "retryableStatusCodes": [ "UNAVAILABLE" ] } }')
-        ]
-        rpc_client = GrpcClient(
-            logger=self.logger(),
-            endpoint=self.config_data['GATEWAY_ENDPOINT'],
-            grpc_options=self.grpc_options,
-            ssl_switch=self.config_data['SSL_SWITCH'],
-            ca_path=self.config_data['CA_CRT'],
-            ssl_key_path=self.config_data['SSL_KEY'],
-            ssl_crt_path=self.config_data['SSL_CRT']
-        )
-        self.stub = ModelStub(
-            agency_id=self.config_data['AGENCY_ID'],
-            thread_event_manager=self.thread_event_manager,
-            rpc_client=rpc_client
-        )
-
-    def init_task_manager(self):
-        self.task_manager = TaskManager(
-            logger=self.logger(),
-            thread_event_manager=self.thread_event_manager,
-            stub=self.stub,
-            task_timeout_h=self.config_data['TASK_TIMEOUT_H']
-        )
+    def init_all(self):
+        agency_id = common_func.get_config_value(
+            "AGENCY_ID", None, self.config_data, True)
+        self.init_transport(task_manager=self.task_manager,
+                            self_agency_id=agency_id,
+                            component_type=self.MODEL_COMPONENT,
+                            send_msg_timeout_ms=self.send_msg_timeout_ms,
+                            pop_msg_timeout_ms=self.pop_msg_timeout_ms)
 
-    def init_storage_client(self):
-        self.storage_client = storage_loader.load(
-            self.config_data, self.logger())
+    def init_transport(self, task_manager: TaskManager,
+                       self_agency_id: str,
+                       component_type: str,
+                       send_msg_timeout_ms: int,
+                       pop_msg_timeout_ms: int):
+        # create the transport
+        transport = TransportLoader.build(**self.config_data)
+        self.logger(
+            f"Create transport success, config: {transport.get_config().desc()}")
+        # start the transport
+        transport.start()
+        self.logger().info(
+            f"Start transport success, config: {transport.get_config().desc()}")
+        transport.register_component(component_type)
+        self.logger().info(
+            f"Register the component {component_type} success")
+        self.transport = ModelTransport(transport=transport,
+                                        self_agency_id=self_agency_id,
+                                        task_manager=task_manager,
+                                        component_type=component_type,
+                                        send_msg_timeout_ms=send_msg_timeout_ms,
+                                        pop_msg_timeout_ms=pop_msg_timeout_ms)
 
     def logger(self, name=None):
         if self.mock_logger is None:
diff --git a/python/ppc_model/common/mock/mock_model_transport.py b/python/ppc_model/common/mock/mock_model_transport.py
new file mode 100644
index 00000000..e4f97521
--- /dev/null
+++ b/python/ppc_model/common/mock/mock_model_transport.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+from ppc_model.network.wedpr_model_transport_api import ModelTransportApi
+from ppc_model.network.wedpr_model_transport_api import ModelRouterApi
+
+
+class MockModelTransportApi(ModelTransportApi):
+    def __init__(self, agency_name):
+        self.agency_name = agency_name
+        self.msg_queue = {}
+
+    @staticmethod
+    def get_topic(task_id: str, task_type: str, dst_agency: str):
+        return f"{dst_agency}_{task_id}{task_type}"
+
+    def push_by_nodeid(self, task_id: str, task_type: str, dst_node: str, payload: bytes, seq: int = 0):
+        self.msg_queue.update({MockModelTransportApi.get_topic(
+            task_id, task_type, self.agency_name): payload})
+
+    def pop(self, task_id: str, task_type: str, dst_inst: str):
+        topic = MockModelTransportApi.get_topic(task_id, task_type, dst_inst)
+        if topic in self.msg_queue.keys():
+            payload = self.msg_queue.get(topic)
+            self.msg_queue.pop(topic)
+            return payload
+        raise Exception(f"Not receive the message of topic:"
+                        f" {self.get_topic(task_id, task_type, dst_inst)} "
+                        f"even after the task has been killed!")
+
+
+class MockModelRouterApi(ModelRouterApi):
+    def __init__(self, participant_id_list):
+        self.participant_id_list = participant_id_list
+        self.transports = {}
+        for participant in self.participant_id_list:
+            self.transports.update(
+                {participant, MockModelTransportApi(participant)})
+
+    def push(self, task_id: str, task_type: str, dst_agency: str, payload: bytes, seq: int = 0):
+        transport = self.transports.get(dst_agency)
+        dst_nodeid = f"{dst_agency}_node"
+        transport.push_by_nodeid(task_id, task_type, dst_nodeid, payload, seq)
+
+    def pop(self, task_id: str, task_type: str, from_inst: str) -> bytes:
+        transport = self.transports.get(from_inst)
+        return transport.pop(task_id, task_type, from_inst)
diff --git a/python/ppc_model/common/mock/rpc_client_mock.py b/python/ppc_model/common/mock/rpc_client_mock.py
deleted file mode 100644
index c3094c4d..00000000
--- a/python/ppc_model/common/mock/rpc_client_mock.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import threading
-
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse
-from ppc_model.interface.rpc_client import RpcClient
-
-
-class RpcClientMock(RpcClient):
-    def __init__(self, need_failed=False):
-        self._need_failed = need_failed
-        self._bad_guy = 0
-        self._lock = threading.Lock()
-        self._on_message_received = None
-
-    def set_message_handler(self, on_message_received):
-        self._on_message_received = on_message_received
-
-    def send(self, request: ModelRequest):
-        # print(
-        #     f"send data to {request.receiver}, task_id: {request.task_id}, "
-        #     f"key: {request.key}, seq: {request.seq}")
-        self._on_message_received(request)
-        response = ModelResponse()
-        if self._need_failed:
-            # 模拟网络断连
-            with self._lock:
-                self._bad_guy += 1
-                response.base_response.error_code = self._bad_guy % 2
-        else:
-            response.base_response.error_code = 0
-            response.base_response.message = "success"
-        return response
diff --git a/python/ppc_model/common/model_result.py b/python/ppc_model/common/model_result.py
index d42f4106..0cf9e36b 100644
--- a/python/ppc_model/common/model_result.py
+++ b/python/ppc_model/common/model_result.py
@@ -8,7 +8,7 @@
 from ppc_common.ppc_utils.utils import AlgorithmType
 from ppc_model.common.context import Context
 from ppc_model.common.protocol import TaskRole
-from ppc_model.network.stub import PushRequest, PullRequest
+from ppc_model.network.wedpr_model_transport import ModelRouter
 
 
 class ResultFileHandling:
@@ -135,20 +135,21 @@ def _remove_workspace(self):
 
     def _sync_result_files(self):
         for key, value in self.ctx.sync_file_list.items():
-            self.sync_result_file(self.ctx, value[0], value[1], key)
+            self.sync_result_file(
+                self.ctx, self.ctx.model_router, value[0], value[1], key)
 
     @staticmethod
-    def sync_result_file(ctx, local_file, remote_file, key_file):
+    def sync_result_file(ctx, model_router: ModelRouter, local_file, remote_file, key_file):
         if ctx.role == TaskRole.ACTIVE_PARTY:
             with open(local_file, 'rb') as f:
                 byte_data = f.read()
             for partner_index in range(1, len(ctx.participant_id_list)):
                 if ctx.participant_id_list[partner_index] in ctx.result_receiver_id_list:
-                    SendMessage._send_byte_data(ctx.components.stub, ctx, f'{CommonMessage.SYNC_FILE.value}_{key_file}',
+                    SendMessage._send_byte_data(model_router, ctx, f'{CommonMessage.SYNC_FILE.value}_{key_file}',
                                                 byte_data, partner_index)
         else:
             if ctx.components.config_data['AGENCY_ID'] in ctx.result_receiver_id_list:
-                byte_data = SendMessage._receive_byte_data(ctx.components.stub, ctx,
+                byte_data = SendMessage._receive_byte_data(model_router, ctx,
                                                            f'{CommonMessage.SYNC_FILE.value}_{key_file}', 0)
                 with open(local_file, 'wb') as f:
                     f.write(byte_data)
@@ -164,34 +165,23 @@ class CommonMessage(Enum):
 class SendMessage:
 
     @staticmethod
-    def _send_byte_data(stub, ctx, key_type, byte_data, partner_index):
+    def _send_byte_data(model_router: ModelRouter, ctx, key_type, byte_data, partner_index):
         log = ctx.components.logger()
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
-
-        stub.push(PushRequest(
-            receiver=partner_id,
-            task_id=ctx.task_id,
-            key=key_type,
-            data=byte_data
-        ))
-
+        model_router.push(task_id=ctx.task_id, task_type=key_type,
+                          dst_agency=partner_id, payload=byte_data)
         log.info(
             f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, "
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
 
     @staticmethod
-    def _receive_byte_data(stub, ctx, key_type, partner_index):
+    def _receive_byte_data(model_router: ModelRouter, ctx, key_type, partner_index):
         log = ctx.components.logger()
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
-
-        byte_data = stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=ctx.task_id,
-            key=key_type
-        ))
-
+        byte_data = model_router.pop(
+            task_id=ctx.task_id, task_type=key_type, from_inst=partner_id)
         log.info(
             f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, "
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
diff --git a/python/ppc_model/common/model_setting.py b/python/ppc_model/common/model_setting.py
index d2340878..d325b73a 100644
--- a/python/ppc_model/common/model_setting.py
+++ b/python/ppc_model/common/model_setting.py
@@ -42,7 +42,7 @@ def __init__(self, model_dict):
             "iv_thresh", 0.1, model_dict, False))
 
 
-class CommmonModelSetting:
+class CommonModelSetting:
     def __init__(self, model_dict):
         self.learning_rate = float(common_func.get_config_value(
             "learning_rate", 0.1, model_dict, False))
@@ -67,7 +67,7 @@ def __init__(self, model_dict):
             "n_jobs", 0, model_dict, False))
 
 
-class SecureLGBMSetting(CommmonModelSetting):
+class SecureLGBMSetting(CommonModelSetting):
     def __init__(self, model_dict):
         super().__init__(model_dict)
         self.test_size = float(common_func.get_config_value(
@@ -107,7 +107,7 @@ def __init__(self, model_dict):
             "one_hot", 0, model_dict, False)
 
 
-class SecureLRSetting(CommmonModelSetting):
+class SecureLRSetting(CommonModelSetting):
     def __init__(self, model_dict):
         super().__init__(model_dict)
         self.feature_rate = float(common_func.get_config_value(
@@ -123,8 +123,8 @@ def __init__(self, model_dict):
         # init PreprocessingSetting
         super().__init__(model_dict)
         # init FeatureEngineeringEngineSetting
-        super(FeatureEngineeringEngineSetting, self).__init__(model_dict)
+        FeatureEngineeringEngineSetting.__init__(self, model_dict)
         # init SecureLGBMSetting
-        super(SecureLGBMSetting, self).__init__(model_dict)
+        SecureLGBMSetting.__init__(self, model_dict)
         # init SecureLRSetting
-        super(SecureLRSetting, self).__init__(model_dict)
+        SecureLRSetting.__init__(self, model_dict)
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index 330f1de7..cf3cbd11 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -1,20 +1,13 @@
 HOST: "0.0.0.0"
 HTTP_PORT: 43471
-RPC_PORT: 43472
 
 AGENCY_ID: 'WeBank'
-GATEWAY_ENDPOINT: "127.0.0.1:43454"
 
 PUBLIC_KEY_LENGTH: 2048
 
 MAX_MESSAGE_LENGTH_MB: 100
 TASK_TIMEOUT_H: 1800
 
-SSL_SWITCH: 0
-CA_CRT: "./ca.crt"
-SSL_CRT: "./ssl.crt"
-SSL_KEY: "./ssl.key"
-
 
 PEM_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/server.pem"
 SHARE_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/dataset_share/"
@@ -41,3 +34,10 @@ UPLOAD_FOLDER: "./upload_data_folder"
 JOB_TEMP_DIR: ".cache/job"
 
 FE_TIMEOUT_S: 5400
+
+# the transport config
+transport_threadpool_size: 4
+transport_node_id: "MODEL_WeBank_NODE"
+transport_gateway_targets: "ipv4:127.0.0.1:40600,127.0.0.1:40601"
+transport_host_ip: "127.0.0.1"
+transport_listen_port: 6500
\ No newline at end of file
diff --git a/python/ppc_model/conf/logging.conf b/python/ppc_model/conf/logging.conf
index b9b3bdb8..f78ab644 100644
--- a/python/ppc_model/conf/logging.conf
+++ b/python/ppc_model/conf/logging.conf
@@ -29,7 +29,7 @@ formatter=simpleFormatter
 [handler_consoleHandler]
 class=StreamHandler
 args=(sys.stdout,)
-level=ERROR
+level=INFO
 formatter=simpleFormatter
 
 [formatters]
diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py
index 451ee44a..994cef06 100644
--- a/python/ppc_model/datasets/dataset.py
+++ b/python/ppc_model/datasets/dataset.py
@@ -112,7 +112,7 @@ def _random_split_dataset(self):
     def _customized_split_dataset(self):
         if self.ctx.role == TaskRole.ACTIVE_PARTY:
             for partner_index in range(1, len(self.ctx.participant_id_list)):
-                byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx,
+                byte_data = SendMessage._receive_byte_data(self.ctx.model_router, self.ctx,
                                                            f'{CommonMessage.EVAL_SET_FILE.value}', partner_index)
                 if not os.path.exists(self.eval_column_file) and byte_data != bytes():
                     with open(self.eval_column_file, 'wb') as f:
@@ -120,7 +120,7 @@ def _customized_split_dataset(self):
             with open(self.eval_column_file, 'rb') as f:
                 byte_data = f.read()
             for partner_index in range(1, len(self.ctx.participant_id_list)):
-                SendMessage._send_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}',
+                SendMessage._send_byte_data(self.ctx.model_router, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}',
                                             byte_data, partner_index)
         else:
             if not os.path.exists(self.eval_column_file):
@@ -128,9 +128,9 @@ def _customized_split_dataset(self):
             else:
                 with open(self.eval_column_file, 'rb') as f:
                     byte_data = f.read()
-            SendMessage._send_byte_data(self.ctx.components.stub, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}',
+            SendMessage._send_byte_data(self.ctx.model_router, self.ctx, f'{CommonMessage.EVAL_SET_FILE.value}',
                                         byte_data, 0)
-            byte_data = SendMessage._receive_byte_data(self.ctx.components.stub, self.ctx,
+            byte_data = SendMessage._receive_byte_data(self.ctx.model_router, self.ctx,
                                                        f'{CommonMessage.EVAL_SET_FILE.value}', 0)
             if not os.path.exists(self.eval_column_file):
                 with open(self.eval_column_file, 'wb') as f:
diff --git a/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
index 67251ad4..4dd89171 100644
--- a/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
+++ b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
@@ -8,12 +8,11 @@
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from ppc_common.ppc_mock.mock_objects import MockLogger, MockStorageClient
 from ppc_model.common.initializer import Initializer
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
+from ppc_model.common.mock.mock_model_transport import MockModelRouterApi
 from ppc_model.common.protocol import TaskRole
 from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext
 from ppc_model.feature_engineering.vertical.active_party import VerticalFeatureEngineeringActiveParty
 from ppc_model.feature_engineering.vertical.passive_party import VerticalFeatureEngineeringPassiveParty
-from ppc_model.network.stub import ModelStub
 
 ACTIVE_PARTY = 'ACTIVE_PARTY'
 
@@ -65,27 +64,9 @@ def mock_args(num_features, iv_thresh):
 class TestFeatureEngineering(unittest.TestCase):
 
     def setUp(self):
-        self._active_rpc_client = RpcClientMock()
-        self._passive_rpc_client = RpcClientMock()
-        self._thread_event_manager = ThreadEventManager()
-        self._active_stub = ModelStub(
-            agency_id=ACTIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._active_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._passive_stub = ModelStub(
-            agency_id=PASSIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._passive_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._active_rpc_client.set_message_handler(
-            self._passive_stub.on_message_received)
-        self._passive_rpc_client.set_message_handler(
-            self._active_stub.on_message_received)
+        participants = [PASSIVE_PARTY, ACTIVE_PARTY]
+        self._active_transport = MockModelRouterApi(participants)
+        self._passive_transport = MockModelRouterApi(participants)
 
     def test_fit(self):
         num_samples = 100000
@@ -96,7 +77,7 @@ def test_fit(self):
 
         active_components = Initializer(log_config_path='', config_path='')
         active_components.homo_algorithm = 0
-        active_components.stub = self._active_stub
+        active_components.transport = self._active_transport
         active_components.config_data = {'JOB_TEMP_DIR': '/tmp'}
         active_components.mock_logger = MockLogger()
         active_components.storage_client = MockStorageClient()
@@ -112,7 +93,7 @@ def test_fit(self):
 
         passive_components = Initializer(log_config_path='', config_path='')
         passive_components.homo_algorithm = 0
-        passive_components.stub = self._passive_stub
+        passive_components.transport = self._passive_transport
         passive_components.config_data = {'JOB_TEMP_DIR': '/tmp'}
         passive_components.mock_logger = MockLogger()
         passive_components.storage_client = MockStorageClient()
diff --git a/python/ppc_model/feature_engineering/vertical/active_party.py b/python/ppc_model/feature_engineering/vertical/active_party.py
index 82523420..cb40d404 100644
--- a/python/ppc_model/feature_engineering/vertical/active_party.py
+++ b/python/ppc_model/feature_engineering/vertical/active_party.py
@@ -11,7 +11,6 @@
 from ppc_model.feature_engineering.vertical.utils import is_continuous_feature, calculate_woe_iv, \
     calculate_woe_iv_with_pos_event
 from ppc_model.interface.model_base import VerticalModel
-from ppc_model.network.stub import PushRequest, PullRequest
 
 
 class VerticalFeatureEngineeringActiveParty(VerticalModel):
@@ -131,12 +130,8 @@ def _process_one_feature(phe, field, count_list, enc_aggr_labels):
     def _get_all_enc_aggr_labels(self, partner_id):
         log = self.ctx.components.logger()
         start_time = time.time()
-        data = self.ctx.components.stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=self.ctx.task_id,
-            key=FeMessage.AGGR_LABELS.value
-        ))
-
+        data = self.ctx.model_router.pop(task_id=self.ctx.task_id,
+                                         task_type=FeMessage.AGGR_LABELS.value, from_inst=partner_id)
         enc_aggr_labels_list_pb = EncAggrLabelsList()
         utils.bytes_to_pb(enc_aggr_labels_list_pb, data)
         public_key = self.ctx.codec.decode_enc_key(
@@ -163,12 +158,10 @@ def _send_enc_labels(self, enc_labels, receiver):
 
         data = PheMessage.packing_data(
             self.ctx.codec, self.ctx.phe.public_key, enc_labels)
-        self.ctx.components.stub.push(PushRequest(
-            receiver=receiver,
-            task_id=self.ctx.task_id,
-            key=FeMessage.ENC_LABELS.value,
-            data=data
-        ))
+        self.ctx.model_router.push(task_id=self.ctx.task_id,
+                                   task_type=FeMessage.ENC_LABELS.value,
+                                   dst_agency=receiver,
+                                   payload=data)
         log.info(
             f"Sending enc labels to {receiver} finished, task_id: {self.ctx.task_id}, label_num: {len(enc_labels)}, "
             f"size: {len(data) / 1024}KB, time_costs: {time.time() - start_time}s")
@@ -183,7 +176,6 @@ def _save_and_sync_fe_results(self):
         self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file,
                                                        self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE)
         log.info(f"Saving fe results finished, task_id: {task_id}")
-
         with open(self.ctx.woe_iv_file, 'rb') as f:
             woe_iv = f.read()
         with open(self.ctx.iv_selected_file, 'rb') as f:
@@ -191,16 +183,12 @@ def _save_and_sync_fe_results(self):
         for i in range(1, len(self.ctx.participant_id_list)):
             partner_id = self.ctx.participant_id_list[i]
             if partner_id in self.ctx.result_receiver_id_list:
-                self.ctx.components.stub.push(PushRequest(
-                    receiver=partner_id,
-                    task_id=self.ctx.task_id,
-                    key=FeMessage.WOE_FILE.value,
-                    data=woe_iv
-                ))
-            self.ctx.components.stub.push(PushRequest(
-                receiver=partner_id,
-                task_id=self.ctx.task_id,
-                key=FeMessage.IV_SELECTED_FILE.value,
-                data=iv_selected
-            ))
+                self.ctx.model_router.push(task_id=self.ctx.task_id,
+                                           task_type=FeMessage.WOE_FILE.value,
+                                           dst_agency=partner_id,
+                                           payload=woe_iv)
+            self.ctx.model_router.push(task_id=self.ctx.task_id,
+                                       task_type=FeMessage.IV_SELECTED_FILE.value,
+                                       dst_agency=partner_id,
+                                       payload=iv_selected)
             log.info(f"Sending fe results finished, task_id: {task_id}")
diff --git a/python/ppc_model/feature_engineering/vertical/passive_party.py b/python/ppc_model/feature_engineering/vertical/passive_party.py
index 4b3ffd47..f3d1aaaa 100644
--- a/python/ppc_model/feature_engineering/vertical/passive_party.py
+++ b/python/ppc_model/feature_engineering/vertical/passive_party.py
@@ -9,7 +9,6 @@
 from ppc_model.feature_engineering.feature_engineering_context import FeatureEngineeringContext, FeMessage
 from ppc_model.feature_engineering.vertical.utils import is_continuous_feature
 from ppc_model.interface.model_base import VerticalModel
-from ppc_model.network.stub import PullRequest, PushRequest
 
 
 class VerticalFeatureEngineeringPassiveParty(VerticalModel):
@@ -44,12 +43,8 @@ def _get_enc_labels(self):
         log = self.ctx.components.logger()
         start_time = time.time()
         active_party = self.ctx.participant_id_list[0]
-        data = self.ctx.components.stub.pull(PullRequest(
-            sender=active_party,
-            task_id=self.ctx.task_id,
-            key=FeMessage.ENC_LABELS.value
-        ))
-
+        data = self.ctx.model_router.pop(
+            task_id=self.ctx.task_id, task_type=FeMessage.ENC_LABELS.value, from_inst=active_party)
         public_key, enc_labels = PheMessage.unpacking_data(
             self.ctx.codec, data)
         log.info(f"All enc labels received, task_id: {self.ctx.task_id}, label_num: {len(enc_labels)}, "
@@ -145,14 +140,10 @@ def _send_all_enc_aggr_labels(self, public_key, aggr_labels_bytes_list):
         self.ctx.components.logger().info(
             f"Encoding all enc aggr labels finished, task_id: {self.ctx.task_id}, "
             f"size: {len(data) / 1024}KB, timecost: {time.time() - start_time}s")
-
-        self.ctx.components.stub.push(PushRequest(
-            receiver=self.ctx.participant_id_list[0],
-            task_id=self.ctx.task_id,
-            key=FeMessage.AGGR_LABELS.value,
-            data=data
-        ))
-
+        self.ctx.model_router.push(task_id=self.ctx.task_id,
+                                   task_type=FeMessage.AGGR_LABELS.value,
+                                   dst_agency=self.ctx.participant_id_list[0],
+                                   payload=data)
         self.ctx.components.logger().info(
             f"Sending all enc aggr labels finished, task_id: {self.ctx.task_id}, "
             f"feature_num: {len(aggr_labels_bytes_list)}, "
@@ -160,13 +151,10 @@ def _send_all_enc_aggr_labels(self, public_key, aggr_labels_bytes_list):
 
     def _get_and_save_result(self):
         active_party = self.ctx.participant_id_list[0]
-        if self.ctx.components.stub.agency_id in self.ctx.result_receiver_id_list:
+        if self.ctx.components.transport.self_agency_id in self.ctx.result_receiver_id_list:
             # 保存来自标签方的woe/iv结果
-            data = self.ctx.components.stub.pull(PullRequest(
-                sender=active_party,
-                task_id=self.ctx.task_id,
-                key=FeMessage.WOE_FILE.value
-            ))
+            data = self.ctx.model_router.pop(
+                task_id=self.ctx.task_id, task_type=FeMessage.WOE_FILE.value, from_inst=active_party)
             self.ctx.components.logger().info(
                 f"Result of woe/iv received, task_id: {self.ctx.task_id}, size: {len(data) / 1024}KB")
             with open(self.ctx.woe_iv_file, 'wb') as f:
@@ -175,10 +163,8 @@ def _get_and_save_result(self):
                                                            self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE)
 
         # 保存来自标签方的iv筛选结果
-        data = self.ctx.components.stub.pull(PullRequest(
-            sender=active_party,
-            task_id=self.ctx.task_id,
-            key=FeMessage.IV_SELECTED_FILE.value))
+        data = self.ctx.model_router.pop(
+            task_id=self.ctx.task_id, task_type=FeMessage.IV_SELECTED_FILE.value, from_inst=active_party)
         self.ctx.components.logger().info(
             f"Result of iv_select received, task_id: {self.ctx.task_id}, size: {len(data) / 1024}KB")
         with open(self.ctx.iv_selected_file, 'wb') as f:
diff --git a/python/ppc_model/interface/model_base.py b/python/ppc_model/interface/model_base.py
index 0e96565f..e755b0b3 100644
--- a/python/ppc_model/interface/model_base.py
+++ b/python/ppc_model/interface/model_base.py
@@ -1,6 +1,7 @@
 from abc import ABC
 
 from pandas import DataFrame
+from ppc_model.network.wedpr_model_transport import ModelRouter
 
 
 class ModelBase(ABC):
@@ -8,6 +9,9 @@ class ModelBase(ABC):
 
     def __init__(self, ctx):
         self.ctx = ctx
+        self.ctx.model_router = ModelRouter(logger=self.ctx.components.logger(),
+                                            transport=self.ctx.components.transport,
+                                            participant_id_list=self.ctx.participant_id_list)
 
     def fit(
         self,
diff --git a/python/ppc_model/network/grpc/__init__.py b/python/ppc_model/network/grpc/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_model/network/grpc/grpc_client.py b/python/ppc_model/network/grpc/grpc_client.py
deleted file mode 100644
index 8169f522..00000000
--- a/python/ppc_model/network/grpc/grpc_client.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import os
-import time
-import traceback
-
-import grpc
-
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse
-from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceStub
-from ppc_common.ppc_utils import utils
-from ppc_model.common.protocol import RpcType
-from ppc_model.interface.rpc_client import RpcClient
-
-
-class GrpcClient(RpcClient):
-    rpc_type = RpcType.GRPC
-
-    def __init__(self, logger, endpoint: str, grpc_options, ssl_switch: int = 0,
-                 ca_path=None, ssl_key_path=None, ssl_crt_path=None):
-        self._logger = logger
-        self._endpoint = endpoint
-        self._ssl_switch = ssl_switch
-        self._grpc_options = grpc_options
-        self._ca_path = ca_path
-        self._ssl_key_path = ssl_key_path
-        self._ssl_crt_path = ssl_crt_path
-        if self._ssl_switch == 0:
-            insecure_channel = grpc.insecure_channel(
-                self._endpoint, options=grpc_options)
-            self._client = ModelServiceStub(insecure_channel)
-        else:
-            channel = self._create_secure_channel(self._endpoint)
-            self._client = ModelServiceStub(channel)
-
-    def _create_secure_channel(self, target):
-        grpc_root_crt = utils.load_credential_from_file(
-            os.path.abspath(self._ca_path))
-        grpc_ssl_key = utils.load_credential_from_file(
-            os.path.abspath(self._ssl_key_path))
-        grpc_ssl_crt = utils.load_credential_from_file(
-            os.path.abspath(self._ssl_crt_path))
-        credentials = grpc.ssl_channel_credentials(
-            root_certificates=grpc_root_crt,
-            private_key=grpc_ssl_key,
-            certificate_chain=grpc_ssl_crt
-        )
-        return grpc.secure_channel(target, credentials, options=self._grpc_options)
-
-    @staticmethod
-    def _build_error_model_response(message: str):
-        model_response = ModelResponse()
-        model_response.base_response.error_code = -1
-        model_response.base_response.message = message
-        return model_response
-
-    def send(self, request: ModelRequest):
-        start_time = time.time()
-        try:
-            self._logger.debug(
-                f"start sending data to {request.receiver}, task_id: {request.task_id}, "
-                f"key: {request.key}, seq: {request.seq}")
-            response = self._client.MessageInteraction(request)
-            end_time = time.time()
-            if response.base_response.error_code != 0:
-                self._logger.warn(
-                    f"[OnWarn]send data to {request.receiver} failed, task_id: {request.task_id}, "
-                    f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, "
-                    f"ret_code: {response.base_response.error_code}, message: {response.base_response.message}, "
-                    f"time_costs: {str(end_time - start_time)}s")
-            else:
-                self._logger.info(
-                    f"finish sending data to {request.receiver}, task_id: {request.task_id}, "
-                    f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, "
-                    f"ret_code: {response.base_response.error_code}, message: {response.base_response.message}, "
-                    f"time_costs: {str(end_time - start_time)}s")
-        except Exception:
-            end_time = time.time()
-            message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \
-                      f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \
-                      f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s"
-            self._logger.warn(message)
-            response = self._build_error_model_response(message)
-        return response
diff --git a/python/ppc_model/network/grpc/grpc_server.py b/python/ppc_model/network/grpc/grpc_server.py
deleted file mode 100644
index 389ba602..00000000
--- a/python/ppc_model/network/grpc/grpc_server.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest, ModelResponse
-from ppc_model.common.global_context import components
-
-
-class ModelService(ppc_model_pb2_grpc.ModelServiceServicer):
-
-    def MessageInteraction(self, model_request: ModelRequest, context):
-        components.logger().info(
-            f"receive a package, sender: {model_request.sender}, task_id: {model_request.task_id}, "
-            f"key: {model_request.key}, seq: {model_request.seq}, slice_num: {model_request.slice_num}")
-
-        components.stub.on_message_received(model_request)
-        model_response = ModelResponse()
-        model_response.base_response.error_code = 0
-        model_response.base_response.message = 'success'
-        return model_response
diff --git a/python/ppc_model/network/stub.py b/python/ppc_model/network/stub.py
deleted file mode 100644
index d472847b..00000000
--- a/python/ppc_model/network/stub.py
+++ /dev/null
@@ -1,219 +0,0 @@
-import os
-import time
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass
-from typing import Dict, Union
-
-from readerwriterlock import rwlock
-
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-
-
-@dataclass
-class PushRequest:
-    receiver: str  # 数据接收方的机构ID
-    task_id: str
-    key: str  # 数据键
-    data: bytes  # 二进制数据
-    slice_size_MB: int = 2  # 切片大小，默认为2MB
-
-    def slice_data(self):
-        """将 data 按 slice_size 进行切片"""
-        if not self.data:
-            return [b'']
-        slice_size = self.slice_size_MB * 1024 * 1024
-        return [self.data[i:i + slice_size] for i in
-                range(0, len(self.data), slice_size)]
-
-
-@dataclass
-class PullRequest:
-    sender: str  # 数据发送方的机构ID
-    task_id: str
-    key: str  # 数据键
-
-
-class ModelStub:
-    def __init__(
-            self,
-            agency_id: str,
-            thread_event_manager: ThreadEventManager,
-            rpc_client,
-            send_retry_times: int = 3,
-            retry_interval_s: Union[int, float] = 5
-    ) -> None:
-        self.agency_id = agency_id
-        self._thread_event_manager = thread_event_manager
-        self._rpc_client = rpc_client
-        self._executor = ThreadPoolExecutor(
-            max_workers=max(1, os.cpu_count() - 1))
-        self._send_retry_times = send_retry_times
-        self._retry_interval_s = retry_interval_s
-        # 缓存收到的消息 [task_id:[sender:[key:[seq: data]]]]
-        # 缓存清理由TaskManager完成
-        self._received_data: Dict[str, Dict[str,
-                                            Dict[str, Dict[int, tuple[int, bytes]]]]] = {}
-        self._received_uuid: Dict[str, set[str]] = {}
-        self._traffic_volume: Dict[str, int] = {}
-        self._data_rw_lock = rwlock.RWLockWrite()
-
-    def push(self, request: PushRequest) -> bytes:
-        """
-        发送消息
-        param request: 消息请求
-        """
-        slices = request.slice_data()
-        futures = []
-        for seq, data in enumerate(slices):
-            model_request = ModelRequest(
-                sender=self.agency_id,
-                receiver=request.receiver,
-                task_id=request.task_id,
-                key=request.key,
-                seq=seq,
-                slice_num=len(slices),
-                data=data
-            )
-            future = self._executor.submit(
-                self._send_with_retry, model_request)
-            futures.append(future)
-
-        ret = bytearray()
-        for future in futures:
-            ret.extend(future.result())
-
-        self._accumulate_traffic_volume(request.task_id, len(request.data))
-
-        return bytes(ret)
-
-    def pull(self, pull_request: PullRequest) -> bytes:
-        """
-        接收消息
-        param request: 待收消息元信息
-        return 消息
-        """
-        task_id = pull_request.task_id
-        sender = pull_request.sender
-        key = pull_request.key
-        while not self._thread_event_manager.event_status(task_id):
-            if self._is_all_data_ready(pull_request):
-                ret = bytearray()
-                with self._data_rw_lock.gen_rlock():
-                    slice_num = len(self._received_data[task_id][sender][key])
-                    for seq in range(slice_num):
-                        ret.extend(
-                            self._received_data[task_id][sender][key][seq][1])
-                # 缓存中删除已获取到的数据
-                with self._data_rw_lock.gen_wlock():
-                    del self._received_data[task_id][sender][key]
-                self._accumulate_traffic_volume(task_id, len(ret))
-                return bytes(ret)
-            # 任务还在执行, 休眠后继续尝试获取数据
-            time.sleep(0.04)
-
-        # 接收到杀任务的信号
-        raise PpcException(PpcErrorCode.TASK_IS_KILLED.get_code(),
-                           PpcErrorCode.TASK_IS_KILLED.get_msg())
-
-    def traffic_volume(self, task_id) -> float:
-        with self._data_rw_lock.gen_rlock():
-            if task_id not in self._traffic_volume:
-                return 0
-            return self._traffic_volume[task_id] / 1024 / 1024
-
-    def on_message_received(self, model_request: ModelRequest):
-        """
-        注册给服务端的回调，服务端收到消息后调用
-        param model_request: 收到的消息
-        """
-        # 消息幂等
-        if not self._is_new_data(model_request):
-            return
-        # 缓存数据
-        self._handle_received_data(model_request)
-
-    def cleanup_cache(self, task_id):
-        with self._data_rw_lock.gen_wlock():
-            if task_id in self._received_data:
-                del self._received_data[task_id]
-            if task_id in self._received_uuid:
-                del self._received_uuid[task_id]
-            if task_id in self._traffic_volume:
-                del self._traffic_volume[task_id]
-
-    def _is_new_data(self, model_request: ModelRequest) -> bool:
-        # 返回是否需要继续处理消息
-        task_id = model_request.task_id
-        uuid = f"{task_id}:{model_request.sender}:{model_request.key}:{model_request.seq}"
-        with self._data_rw_lock.gen_wlock():
-            if task_id in self._received_uuid and uuid in self._received_uuid[task_id]:
-                # 收到重复的消息
-                return False
-            elif task_id in self._received_uuid and uuid not in self._received_uuid[task_id]:
-                # 收到task_id的新消息
-                self._received_uuid[task_id].add(uuid)
-            else:
-                # 首次收到task_id的消息
-                self._received_uuid[task_id] = {uuid}
-            return True
-
-    def _handle_received_data(self, model_request: ModelRequest):
-        task_id = model_request.task_id
-        sender = model_request.sender
-        key = model_request.key
-        seq = model_request.seq
-        slice_num = model_request.slice_num
-        data = model_request.data
-        with self._data_rw_lock.gen_wlock():
-            if task_id not in self._received_data:
-                self._received_data[task_id] = {
-                    model_request.sender: {key: {seq: (slice_num, data)}}}
-            elif sender not in self._received_data[task_id]:
-                self._received_data[task_id][sender] = {
-                    key: {seq: (slice_num, data)}}
-            elif key not in self._received_data[task_id][sender]:
-                self._received_data[task_id][sender][key] = {
-                    seq: (slice_num, data)}
-            else:
-                self._received_data[task_id][sender][key][seq] = (
-                    slice_num, data)
-
-    def _is_all_data_ready(self, pull_request: PullRequest):
-        task_id = pull_request.task_id
-        sender = pull_request.sender
-        key = pull_request.key
-        with self._data_rw_lock.gen_rlock():
-            if task_id not in self._received_data:
-                return False
-            if sender not in self._received_data[task_id]:
-                return False
-            if key not in self._received_data[task_id][sender]:
-                return False
-            if len(self._received_data[task_id][sender][key]) == 0:
-                return False
-            _, first_value = next(
-                iter(self._received_data[task_id][sender][key].items()))
-            if first_value[0] != len(self._received_data[task_id][sender][key]):
-                return False
-            return True
-
-    def _send_with_retry(self, model_request: ModelRequest):
-        retry_times = 0
-        while retry_times <= self._send_retry_times:
-            model_response = self._rpc_client.send(model_request)
-            if model_response.base_response.error_code == 0:
-                return model_response.data
-            if retry_times <= self._send_retry_times:
-                retry_times += 1
-                time.sleep(self._retry_interval_s)
-            else:
-                raise PpcException(PpcErrorCode.NETWORK_ERROR.get_code(
-                ), model_response.base_response.message)
-
-    def _accumulate_traffic_volume(self, task_id, length):
-        with self._data_rw_lock.gen_wlock():
-            if task_id not in self._traffic_volume:
-                self._traffic_volume[task_id] = 0
-            self._traffic_volume[task_id] += length
diff --git a/python/ppc_model/network/test/stub_unittest.py b/python/ppc_model/network/test/stub_unittest.py
deleted file mode 100644
index 63e8e5c7..00000000
--- a/python/ppc_model/network/test/stub_unittest.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import unittest
-
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
-from ppc_model.network.stub import ModelStub, PushRequest, PullRequest
-
-
-class TestStub(unittest.TestCase):
-    def setUp(self):
-        super().__init__()
-        self._agency_id = 'TEST_AGENCY'
-        self._message_type = 'TEST_MESSAGE'
-        self._rpc_client = RpcClientMock()
-        self._thread_event_manager = ThreadEventManager()
-        self._stub = ModelStub(
-            agency_id=self._agency_id,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._rpc_client.set_message_handler(self._stub.on_message_received)
-
-    def test_push_pull(self):
-        task_id = '0x12345678'
-        byte_array = bytearray(31 * 1024 * 1024)
-        bytes_data = bytes(byte_array)
-        self._stub.push(PushRequest(
-            receiver=self._agency_id,
-            task_id=task_id,
-            key=self._message_type,
-            data=bytes_data
-        ))
-        self._stub.push(PushRequest(
-            receiver=self._agency_id,
-            task_id=task_id,
-            key=self._message_type + 'other',
-            data=bytes_data
-        ))
-        received_data = self._stub.pull(PullRequest(
-            sender=self._agency_id,
-            task_id=task_id,
-            key=self._message_type
-        ))
-        other_data = self._stub.pull(PullRequest(
-            sender=self._agency_id,
-            task_id=task_id,
-            key=self._message_type + 'other',
-        ))
-        self.assertEqual(bytes_data, received_data)
-        self.assertEqual(bytes_data, other_data)
-
-    def test_bad_client(self):
-        rpc_client = RpcClientMock(need_failed=True)
-        stub = ModelStub(
-            agency_id=self._agency_id,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        rpc_client.set_message_handler(stub.on_message_received)
-
-        task_id = '0x12345678'
-        byte_array = bytearray(3 * 1024 * 1024)
-        bytes_data = bytes(byte_array)
-        stub.push(PushRequest(
-            receiver=self._agency_id,
-            task_id=task_id,
-            key=self._message_type,
-            data=bytes_data
-        ))
-
-        received_data = stub.pull(PullRequest(
-            sender=self._agency_id,
-            task_id=task_id,
-            key=self._message_type
-        ))
-        self.assertEqual(bytes_data, received_data)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/python/ppc_model/network/wedpr_model_transport.py b/python/ppc_model/network/wedpr_model_transport.py
new file mode 100644
index 00000000..b260d9bc
--- /dev/null
+++ b/python/ppc_model/network/wedpr_model_transport.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+from wedpr_python_gateway_sdk.transport.impl.transport import Transport
+from wedpr_python_gateway_sdk.transport.impl.transport import RouteType
+from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
+from ppc_model.network.wedpr_model_transport_api import ModelTransportApi
+from ppc_model.network.wedpr_model_transport_api import ModelRouterApi
+from ppc_model.task.task_manager import TaskManager
+import time
+
+
+class ModelTransport(ModelTransportApi):
+    def __init__(self, transport: Transport, self_agency_id: str, task_manager: TaskManager,
+                 component_type,
+                 send_msg_timeout_ms: int = 5000,
+                 pop_msg_timeout_ms: int = 60000):
+        self.transport = transport
+        self.self_agency_id = self_agency_id
+        # default send msg timeout
+        self.send_msg_timeout = send_msg_timeout_ms
+        self.pop_msg_timeout = pop_msg_timeout_ms
+        self.task_manager = task_manager
+        self.component_type = component_type
+
+    @staticmethod
+    def get_topic(task_id: str, task_type: str, agency: str):
+        return f"{agency}_{task_id}_{task_type}"
+
+    def push_by_component(self, task_id: str, task_type: str, dst_inst: str, data):
+        self.transport.push_by_component(topic=self.get_topic(task_id, task_type, dst_inst),
+                                         dstInst=dst_inst,
+                                         component=self.component_type,
+                                         payload=data, timeout=self.send_msg_timeout)
+
+    def push_by_nodeid(self, task_id: str, task_type: str, dst_node: str, payload: bytes, seq: int = 0):
+        self.transport.push_by_nodeid(topic=self.get_topic(task_id, task_type, self.self_agency_id),
+                                      dstNode=bytes(
+                                          dst_node, encoding="utf-8"),
+                                      seq=seq, payload=payload,
+                                      timeout=self.send_msg_timeout)
+
+    def pop(self, task_id: str, task_type: str, dst_inst: str) -> MessageAPI:
+        while not self.task_manager.task_finished(task_id):
+            msg = self.transport.pop(topic=self.get_topic(
+                task_id, task_type, dst_inst), timeout_ms=self.pop_msg_timeout)
+            # wait for the msg if the task is running
+            if msg is None:
+                time.sleep(0.04)
+            else:
+                return msg
+        raise Exception(f"Not receive the message of topic:"
+                        f" {self.get_topic(task_id, task_type, dst_inst)} "
+                        f"even after the task has been killed!")
+
+    def get_component_type(self):
+        return self.component_type
+
+    def select_node(self, route_type: RouteType, dst_agency: str, dst_component: str) -> str:
+        return self.transport.select_node_by_route_policy(route_type=route_type,
+                                                          dst_inst=dst_agency, dst_component=dst_component)
+
+    def stop(self):
+        self.transport.stop()
+
+
+class ModelRouter(ModelRouterApi):
+    def __init__(self, logger, transport: ModelTransport, participant_id_list):
+        self.logger = logger
+        self.transport = transport
+        self.participant_id_list = participant_id_list
+        self.router_info = {}
+        for participant in self.participant_id_list:
+            self.__init_router__(participant)
+
+    def __init_router__(self, participant):
+        result = self.transport.select_node(route_type=RouteType.ROUTE_THROUGH_COMPONENT,
+                                            dst_agency=participant,
+                                            dst_component=self.transport.get_component_type())
+        self.logger.info(
+            f"__init_router__ for {participant}, result: {result}, component: {self.transport.get_component_type()}")
+        if result is None:
+            raise Exception(
+                f"No router can reach participant {participant}")
+        self.logger.info(
+            f"ModelRouter, select node {result} for participant {participant}, "
+            f"component: {self.transport.get_component_type()}")
+        self.router_info.update({participant: result})
+        return result
+
+    def __get_dstnode_by_participant(self, participant):
+        if participant in self.router_info.keys():
+            return self.router_info.get(participant)
+        return self.__init_router__(participant)
+
+    def push(self, task_id: str, task_type: str, dst_agency: str, payload: bytes, seq: int = 0):
+        dst_node = self.__get_dstnode_by_participant(dst_agency)
+        self.transport.push_by_nodeid(
+            task_id=task_id, task_type=task_type, dst_node=dst_node, payload=payload, seq=seq)
+
+    def pop(self, task_id: str, task_type: str, from_inst: str) -> bytes:
+        result = self.transport.pop(
+            task_id=task_id, task_type=task_type, dst_inst=from_inst)
+        self.logger.debug(f"Receive message from {result}")
+        return result.get_payload()
diff --git a/python/ppc_model/network/wedpr_model_transport_api.py b/python/ppc_model/network/wedpr_model_transport_api.py
new file mode 100644
index 00000000..c2f60172
--- /dev/null
+++ b/python/ppc_model/network/wedpr_model_transport_api.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+from abc import ABC, abstractmethod
+
+
+class ModelTransportApi(ABC):
+    @abstractmethod
+    def push_by_nodeid(self, task_id: str, task_type: str, dst_node: str, payload: bytes, seq: int = 0):
+        pass
+
+    @abstractmethod
+    def pop(self, task_id: str, task_type: str, dst_inst: str):
+        pass
+
+
+class ModelRouterApi:
+    @abstractmethod
+    def push(self, task_id: str, task_type: str, dst_agency: str, payload: bytes, seq: int = 0):
+        pass
+
+    @abstractmethod
+    def pop(self, task_id: str, task_type: str, from_inst: str) -> bytes:
+        pass
diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py
index 6c40bbc3..0a95bc9a 100644
--- a/python/ppc_model/ppc_model_app.py
+++ b/python/ppc_model/ppc_model_app.py
@@ -1,30 +1,26 @@
 # Note: here can't be refactored by autopep
 import sys
-sys.path.append("../")
-
-import multiprocessing
 import os
-from concurrent import futures
-from threading import Thread
-import grpc
-from cheroot.ssl.builtin import BuiltinSSLAdapter
-from cheroot.wsgi import Server as WSGIServer
-from flask import Flask, Blueprint
-from paste.translogger import TransLogger
-from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc
-from ppc_common.ppc_utils import utils
-from ppc_model.common.global_context import components
-from ppc_model.common.protocol import ModelTask
-from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine
-from ppc_model.network.grpc.grpc_server import ModelService
-from ppc_model.network.http.model_controller import ns as task_namespace
-from ppc_model.network.http.model_controller import ns2 as log_namespace
-from ppc_model.network.http.restx import api
-from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine
-from ppc_model.secure_lr.secure_lr_prediction_engine import SecureLRPredictionEngine
-from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine
-from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine
+root_path = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(root_path, "../"))
+# Note: here can't be refactored by autopep
+
 from ppc_model.secure_lgbm.secure_lgbm_training_engine import SecureLGBMTrainingEngine
+from ppc_model.secure_lgbm.secure_lgbm_prediction_engine import SecureLGBMPredictionEngine
+from ppc_model.secure_lr.secure_lr_training_engine import SecureLRTrainingEngine
+from ppc_model.secure_lr.secure_lr_prediction_engine import SecureLRPredictionEngine
+from ppc_model.preprocessing.preprocessing_engine import PreprocessingEngine
+from ppc_model.network.http.restx import api
+from ppc_model.network.http.model_controller import ns2 as log_namespace
+from ppc_model.network.http.model_controller import ns as task_namespace
+from ppc_model.feature_engineering.feature_engineering_engine import FeatureEngineeringEngine
+from ppc_model.common.protocol import ModelTask
+from ppc_model.common.global_context import components
+from paste.translogger import TransLogger
+from flask import Flask, Blueprint
+from cheroot.wsgi import Server as WSGIServer
+from cheroot.ssl.builtin import BuiltinSSLAdapter
+import multiprocessing
 
 
 app = Flask(__name__)
@@ -58,39 +54,6 @@ def register_task_handler():
         ModelTask.LR_PREDICTING, SecureLRPredictionEngine.run)
 
 
-def model_serve():
-    if app.config['SSL_SWITCH'] == 0:
-        ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)),
-                                options=components.grpc_options)
-        ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-            ModelService(), ppc_serve)
-        address = "[::]:{}".format(app.config['RPC_PORT'])
-        ppc_serve.add_insecure_port(address)
-    else:
-        grpc_root_crt = utils.load_credential_from_file(
-            os.path.abspath(app.config['SSL_CA']))
-        grpc_ssl_key = utils.load_credential_from_file(
-            os.path.abspath(app.config['SSL_KEY']))
-        grpc_ssl_crt = utils.load_credential_from_file(
-            os.path.abspath(app.config['SSL_CRT']))
-        server_credentials = grpc.ssl_server_credentials(((
-            grpc_ssl_key,
-            grpc_ssl_crt,
-        ),), grpc_root_crt, True)
-
-        ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)),
-                                options=components.grpc_options)
-        ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-            ModelService(), ppc_serve)
-        address = "[::]:{}".format(app.config['RPC_PORT'])
-        ppc_serve.add_secure_port(address, server_credentials)
-
-    ppc_serve.start()
-    components.logger().info(
-        f"Starting model grpc server at ://{app.config['HOST']}:{app.config['RPC_PORT']}")
-    ppc_serve.wait_for_termination()
-
-
 if __name__ == '__main__':
     initialize_app(app)
     register_task_handler()
@@ -98,22 +61,15 @@ def model_serve():
     # 启动子进程不继承父进程的锁状态，防止死锁
     multiprocessing.set_start_method('spawn')
 
-    Thread(target=model_serve).start()
-
     app.config['SECRET_KEY'] = os.urandom(24)
     server = WSGIServer((app.config['HOST'], app.config['HTTP_PORT']),
                         TransLogger(app, setup_console_handler=False), numthreads=2)
 
-    ssl_switch = app.config['SSL_SWITCH']
     protocol = 'http'
-    if ssl_switch == 1:
-        protocol = 'https'
-        server.ssl_adapter = BuiltinSSLAdapter(
-            certificate=app.config['SSL_CRT'],
-            private_key=app.config['SSL_KEY'],
-            certificate_chain=app.config['CA_CRT'])
-
-    message = f"Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']}"
+    message = f"Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']} successfully"
     print(message)
     components.logger().info(message)
     server.start()
+    # stop the nodes
+    components.transport.stop()
+    print("Stop ppc model server successfully")
diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
index 0c1925ca..4ba739a7 100644
--- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
+++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
@@ -5,12 +5,11 @@
 from ppc_model.common.initializer import Initializer
 from ppc_common.ppc_mock.mock_objects import MockLogger
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.network.stub import ModelStub
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.metrics.evaluation import Evaluation
 from ppc_model.metrics.model_plot import ModelPlot
 from ppc_model.common.model_result import ResultFileHandling
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
+from ppc_model.common.mock.mock_model_transport import MockModelRouterApi
 from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext
 from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty
 
@@ -67,33 +66,15 @@ def mock_args():
 class TestXgboostTraining(unittest.TestCase):
 
     def setUp(self):
-        self._active_rpc_client = RpcClientMock()
-        self._passive_rpc_client = RpcClientMock()
-        self._thread_event_manager = ThreadEventManager()
-        self._active_stub = ModelStub(
-            agency_id=ACTIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._active_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._passive_stub = ModelStub(
-            agency_id=PASSIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._passive_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._active_rpc_client.set_message_handler(
-            self._passive_stub.on_message_received)
-        self._passive_rpc_client.set_message_handler(
-            self._active_stub.on_message_received)
+        participants = [PASSIVE_PARTY, ACTIVE_PARTY]
+        self._active_transport = MockModelRouterApi(participants)
+        self._passive_transport = MockModelRouterApi(participants)
 
     def test_fit(self):
         args_a, args_b = mock_args()
 
         active_components = Initializer(log_config_path='', config_path='')
-        active_components.stub = self._active_stub
+        active_components.transport = self._active_transport
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
@@ -111,7 +92,7 @@ def test_fit(self):
         print(secure_dataset_a.test_y.shape)
 
         passive_components = Initializer(log_config_path='', config_path='')
-        passive_components.stub = self._passive_stub
+        passive_components.transport = self._passive_transport
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
index d8e20c90..826b89dd 100644
--- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
+++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
@@ -6,12 +6,11 @@
 from ppc_model.common.initializer import Initializer
 from ppc_common.ppc_mock.mock_objects import MockLogger
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.network.stub import ModelStub
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.metrics.evaluation import Evaluation
 from ppc_model.metrics.model_plot import ModelPlot
 from ppc_model.common.model_result import ResultFileHandling
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
+from ppc_model.common.mock.mock_model_transport import MockModelRouterApi
 from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext
 from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty
 
@@ -73,34 +72,18 @@ class TestXgboostTraining(unittest.TestCase):
     df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df)
 
     def setUp(self):
-        self._active_rpc_client = RpcClientMock()
-        self._passive_rpc_client = RpcClientMock()
         self._thread_event_manager = ThreadEventManager()
-        self._active_stub = ModelStub(
-            agency_id=ACTIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._active_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._passive_stub = ModelStub(
-            agency_id=PASSIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._passive_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._active_rpc_client.set_message_handler(
-            self._passive_stub.on_message_received)
-        self._passive_rpc_client.set_message_handler(
-            self._active_stub.on_message_received)
+        participants = [PASSIVE_PARTY, ACTIVE_PARTY]
+        self._active_transport = MockModelRouterApi(participants)
+        self._passive_transport = MockModelRouterApi(participants)
 
     def test_fit(self):
         args_a, args_b = mock_args()
         plot_lock = threading.Lock()
 
-        active_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock)
-        active_components.stub = self._active_stub
+        active_components = Initializer(
+            log_config_path='', config_path='', plot_lock=plot_lock)
+        active_components.transport = self._active_transport
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
@@ -115,9 +98,9 @@ def test_fit(self):
         print(secure_dataset_a.test_X.shape)
         print(secure_dataset_a.test_y.shape)
 
-        passive_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock)
-        passive_components.stub = self._passive_stub
-        passive_components.stub = self._passive_stub
+        passive_components = Initializer(
+            log_config_path='', config_path='', plot_lock=plot_lock)
+        passive_components.transport = self._passive_transport
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py
index 888c9618..1fa7a5b6 100644
--- a/python/ppc_model/secure_lgbm/vertical/booster.py
+++ b/python/ppc_model/secure_lgbm/vertical/booster.py
@@ -10,20 +10,18 @@
 from ppc_model.interface.model_base import VerticalModel
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.common.protocol import PheMessage
-from ppc_model.network.stub import PushRequest, PullRequest
 from ppc_model.common.model_result import ResultFileHandling
 from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning
 from ppc_model.secure_model_base.secure_model_booster import SecureModelBooster
 from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext, LGBMMessage
 
-
 # 抽离sgb的公共部分
+
+
 class VerticalBooster(SecureModelBooster):
     def __init__(self, ctx: SecureLGBMContext, dataset: SecureDataset) -> None:
         super().__init__(ctx)
         self.dataset = dataset
-        self._stub = ctx.components.stub
-
         self._tree_id = None
         self._leaf_id = None
         self._X_bin = None
@@ -134,21 +132,13 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal
         partner_id = ctx.participant_id_list[partner_index]
 
         if matrix_data:
-            self._stub.push(PushRequest(
-                receiver=partner_id,
-                task_id=ctx.task_id,
-                key=key_type,
-                data=PheMessage.packing_2dim_data(
-                    ctx.codec, ctx.phe.public_key, enc_data)
-            ))
+            payload = PheMessage.packing_2dim_data(
+                ctx.codec, ctx.phe.public_key, enc_data)
         else:
-            self._stub.push(PushRequest(
-                receiver=partner_id,
-                task_id=ctx.task_id,
-                key=key_type,
-                data=PheMessage.packing_data(
-                    ctx.codec, ctx.phe.public_key, enc_data)
-            ))
+            payload = PheMessage.packing_data(
+                ctx.codec, ctx.phe.public_key, enc_data)
+        self.ctx.model_router.push(
+            task_id=ctx.task_id, task_type=key_type, dst_agency=partner_id, payload=payload)
 
         self.logger.info(
             f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, "
@@ -157,13 +147,9 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal
     def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
-
-        byte_data = self._stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=ctx.task_id,
-            key=key_type
-        ))
-
+        # send without payload
+        byte_data = self.ctx.model_router.pop(
+            task_id=ctx.task_id, task_type=key_type, from_inst=partner_id)
         if matrix_data:
             public_key, enc_data = PheMessage.unpacking_2dim_data(
                 ctx.codec, byte_data)
@@ -180,13 +166,8 @@ def _send_byte_data(self, ctx, key_type, byte_data, partner_index):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
 
-        self._stub.push(PushRequest(
-            receiver=partner_id,
-            task_id=ctx.task_id,
-            key=key_type,
-            data=byte_data
-        ))
-
+        self.ctx.model_router.push(
+            task_id=ctx.task_id, task_type=key_type, dst_agency=partner_id, payload=byte_data)
         self.logger.info(
             f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, "
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
@@ -195,12 +176,8 @@ def _receive_byte_data(self, ctx, key_type, partner_index):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
 
-        byte_data = self._stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=ctx.task_id,
-            key=key_type
-        ))
-
+        byte_data = self.ctx.model_router.pop(
+            task_id=ctx.task_id, task_type=key_type, from_inst=partner_id)
         self.logger.info(
             f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, "
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
diff --git a/python/ppc_model/secure_lr/test/test_lr.py b/python/ppc_model/secure_lr/test/test_lr.py
index 7ed377ba..f6121fb6 100644
--- a/python/ppc_model/secure_lr/test/test_lr.py
+++ b/python/ppc_model/secure_lr/test/test_lr.py
@@ -1,3 +1,5 @@
+from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler
 import unittest
 import threading
 import traceback
@@ -6,16 +8,10 @@
 
 from ppc_model.common.initializer import Initializer
 from ppc_common.ppc_mock.mock_objects import MockLogger
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.network.stub import ModelStub
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.metrics.evaluation import Evaluation
-from ppc_model.metrics.model_plot import ModelPlot
 from ppc_model.metrics.loss import BinaryLoss
-from ppc_model.common.model_result import ResultFileHandling
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
 from ppc_model.secure_lr.secure_lr_context import SecureLRContext
-from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty
 from ppc_model.secure_lr.vertical.booster import VerticalBooster
 
 
@@ -90,7 +86,8 @@ def mock_args():
 
 # for _ in range(max_iter):
 for i in range(1):
-    idx = VerticalBooster._get_sample_idx(i, secure_dataset.train_X.shape[0], size = 8)
+    idx = VerticalBooster._get_sample_idx(
+        i, secure_dataset.train_X.shape[0], size=8)
     x_, y_ = secure_dataset.train_X[idx], secure_dataset.train_y[idx]
 
     g = BinaryLoss.dot_product(x_, train_weights) + bias
@@ -119,7 +116,8 @@ def mock_args():
 
 # for _ in range(max_iter):
 for i in range(2):
-    idx = VerticalBooster._get_sample_idx(i, secure_dataset.train_X.shape[0], size = 8)
+    idx = VerticalBooster._get_sample_idx(
+        i, secure_dataset.train_X.shape[0], size=8)
     x_, y_ = secure_dataset.train_X[idx], secure_dataset.train_y[idx]
 
     g = BinaryLoss.dot_product(x_, train_weights)
@@ -141,7 +139,6 @@ def mock_args():
 
 # df ---------------------------------------------
 # MinMaxScaler
-from sklearn.preprocessing import MinMaxScaler
 
 # 创建MinMaxScaler对象
 scaler = MinMaxScaler()
@@ -153,7 +150,7 @@ def mock_args():
 train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1])
 
 for i in range(2):
-    idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size = 8)
+    idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size=8)
     x_, y_ = train_X[idx], secure_dataset.train_y[idx]
 
     g = BinaryLoss.dot_product(x_, train_weights)
@@ -174,7 +171,6 @@ def mock_args():
 
 
 # StandardScaler
-from sklearn.preprocessing import StandardScaler 
 
 # 创建MinMaxScaler对象
 scaler = StandardScaler()
@@ -186,7 +182,7 @@ def mock_args():
 train_weights = VerticalBooster._init_weight(secure_dataset.train_X.shape[1])
 
 for i in range(2):
-    idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size = 8)
+    idx = VerticalBooster._get_sample_idx(i, train_X.shape[0], size=8)
     x_, y_ = train_X[idx], secure_dataset.train_y[idx]
 
     g = BinaryLoss.dot_product(x_, train_weights)
@@ -204,4 +200,3 @@ def mock_args():
 train_praba = BinaryLoss.sigmoid(g)
 auc = Evaluation.fevaluation(secure_dataset.train_y, train_praba)['auc']
 print(auc)
-
diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
index 3c99d12b..9aa985e9 100644
--- a/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
+++ b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
@@ -4,13 +4,10 @@
 
 from ppc_model.common.initializer import Initializer
 from ppc_common.ppc_mock.mock_objects import MockLogger
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.network.stub import ModelStub
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.metrics.evaluation import Evaluation
-from ppc_model.metrics.model_plot import ModelPlot
 from ppc_model.common.model_result import ResultFileHandling
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
+from ppc_model.common.mock.mock_model_transport import MockModelRouterApi
 from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext
 from ppc_model.secure_lgbm.vertical import VerticalLGBMActiveParty, VerticalLGBMPassiveParty
 
@@ -66,33 +63,15 @@ def mock_args():
 class TestXgboostTraining(unittest.TestCase):
 
     def setUp(self):
-        self._active_rpc_client = RpcClientMock()
-        self._passive_rpc_client = RpcClientMock()
-        self._thread_event_manager = ThreadEventManager()
-        self._active_stub = ModelStub(
-            agency_id=ACTIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._active_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._passive_stub = ModelStub(
-            agency_id=PASSIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._passive_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._active_rpc_client.set_message_handler(
-            self._passive_stub.on_message_received)
-        self._passive_rpc_client.set_message_handler(
-            self._active_stub.on_message_received)
+        participants = [PASSIVE_PARTY, ACTIVE_PARTY]
+        self._active_transport = MockModelRouterApi(participants)
+        self._passive_transport = MockModelRouterApi(participants)
 
     def test_fit(self):
         args_a, args_b = mock_args()
 
         active_components = Initializer(log_config_path='', config_path='')
-        active_components.stub = self._active_stub
+        active_components.transport = self._active_transport
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
@@ -110,7 +89,7 @@ def test_fit(self):
         print(secure_dataset_a.test_y.shape)
 
         passive_components = Initializer(log_config_path='', config_path='')
-        passive_components.stub = self._passive_stub
+        passive_components.transport = self._passive_transport
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_training.py
index d90e66a7..736f38f0 100644
--- a/python/ppc_model/secure_lr/test/test_secure_lr_training.py
+++ b/python/ppc_model/secure_lr/test/test_secure_lr_training.py
@@ -5,13 +5,10 @@
 
 from ppc_model.common.initializer import Initializer
 from ppc_common.ppc_mock.mock_objects import MockLogger
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_model.network.stub import ModelStub
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.metrics.evaluation import Evaluation
-from ppc_model.metrics.model_plot import ModelPlot
+from ppc_model.common.mock.mock_model_transport import MockModelRouterApi
 from ppc_model.common.model_result import ResultFileHandling
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
 from ppc_model.secure_lr.secure_lr_context import SecureLRContext
 from ppc_model.secure_lr.vertical import VerticalLRActiveParty, VerticalLRPassiveParty
 
@@ -68,7 +65,7 @@ class TestXgboostTraining(unittest.TestCase):
     y = cancer.target
 
     # MinMaxScaler
-    from sklearn.preprocessing import MinMaxScaler 
+    from sklearn.preprocessing import MinMaxScaler
     # 创建MinMaxScaler对象
     scaler = MinMaxScaler()
     # 拟合并转换数据
@@ -78,38 +75,22 @@ class TestXgboostTraining(unittest.TestCase):
     df_with_y, df_without_y = SecureDataset.hetero_split_dataset(df)
 
     def setUp(self):
-        self._active_rpc_client = RpcClientMock()
-        self._passive_rpc_client = RpcClientMock()
-        self._thread_event_manager = ThreadEventManager()
-        self._active_stub = ModelStub(
-            agency_id=ACTIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._active_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._passive_stub = ModelStub(
-            agency_id=PASSIVE_PARTY,
-            thread_event_manager=self._thread_event_manager,
-            rpc_client=self._passive_rpc_client,
-            send_retry_times=3,
-            retry_interval_s=0.1
-        )
-        self._active_rpc_client.set_message_handler(
-            self._passive_stub.on_message_received)
-        self._passive_rpc_client.set_message_handler(
-            self._active_stub.on_message_received)
+        participants = [PASSIVE_PARTY, ACTIVE_PARTY]
+        self._active_transport = MockModelRouterApi(participants)
+        self._passive_transport = MockModelRouterApi(participants)
 
     def test_fit(self):
         args_a, args_b = mock_args()
         plot_lock = threading.Lock()
 
-        active_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock)
-        active_components.stub = self._active_stub
+        active_components = Initializer(
+            log_config_path='', config_path='', plot_lock=plot_lock)
+        active_components.transport = self._active_transport
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
         task_info_a = SecureLRContext(args_a, active_components)
+
         secure_dataset_a = SecureDataset(task_info_a, self.df_with_y)
         booster_a = VerticalLRActiveParty(task_info_a, secure_dataset_a)
         print(secure_dataset_a.feature_name)
@@ -120,8 +101,9 @@ def test_fit(self):
         print(secure_dataset_a.test_X.shape)
         print(secure_dataset_a.test_y.shape)
 
-        passive_components = Initializer(log_config_path='', config_path='', plot_lock=plot_lock)
-        passive_components.stub = self._passive_stub
+        passive_components = Initializer(
+            log_config_path='', config_path='', plot_lock=plot_lock)
+        passive_components.transport = self._passive_transport
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py
index 537e3425..97400116 100644
--- a/python/ppc_model/secure_lr/vertical/booster.py
+++ b/python/ppc_model/secure_lr/vertical/booster.py
@@ -12,7 +12,6 @@
 from ppc_model.datasets.data_reduction.feature_selection import FeatureSelection
 from ppc_model.datasets.dataset import SecureDataset
 from ppc_model.common.protocol import PheMessage
-from ppc_model.network.stub import PushRequest, PullRequest
 from ppc_model.common.model_result import ResultFileHandling
 from ppc_model.datasets.feature_binning.feature_binning import FeatureBinning
 from ppc_model.secure_lr.secure_lr_context import SecureLRContext, LRMessage
@@ -25,7 +24,6 @@ class VerticalBooster(SecureModelBooster):
     def __init__(self, ctx: SecureLRContext, dataset: SecureDataset) -> None:
         super().__init__(ctx)
         self.dataset = dataset
-        self._stub = ctx.components.stub
 
         self._iter_id = None
 
@@ -178,22 +176,13 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal
         partner_id = ctx.participant_id_list[partner_index]
 
         if matrix_data:
-            self._stub.push(PushRequest(
-                receiver=partner_id,
-                task_id=ctx.task_id,
-                key=key_type,
-                data=PheMessage.packing_2dim_data(
-                    ctx.codec, ctx.phe.public_key, enc_data)
-            ))
+            payload = PheMessage.packing_2dim_data(
+                ctx.codec, ctx.phe.public_key, enc_data)
         else:
-            self._stub.push(PushRequest(
-                receiver=partner_id,
-                task_id=ctx.task_id,
-                key=key_type,
-                data=PheMessage.packing_data(
-                    ctx.codec, ctx.phe.public_key, enc_data)
-            ))
-
+            payload = PheMessage.packing_data(
+                ctx.codec, ctx.phe.public_key, enc_data)
+        self.ctx.model_router.push(
+            task_id=ctx.task_id, task_type=key_type, dst_agency=partner_id, payload=payload)
         self.logger.info(
             f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, "
             f"data_length: {len(enc_data)}, time_costs: {time.time() - start_time}s")
@@ -201,13 +190,8 @@ def _send_enc_data(self, ctx, key_type, enc_data, partner_index, matrix_data=Fal
     def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
-
-        byte_data = self._stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=ctx.task_id,
-            key=key_type
-        ))
-
+        byte_data = self.ctx.model_router.pop(
+            task_id=ctx.task_id, task_type=key_type, from_inst=partner_id)
         if matrix_data:
             public_key, enc_data = PheMessage.unpacking_2dim_data(
                 ctx.codec, byte_data)
@@ -223,14 +207,8 @@ def _receive_enc_data(self, ctx, key_type, partner_index, matrix_data=False):
     def _send_byte_data(self, ctx, key_type, byte_data, partner_index):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
-
-        self._stub.push(PushRequest(
-            receiver=partner_id,
-            task_id=ctx.task_id,
-            key=key_type,
-            data=byte_data
-        ))
-
+        self.ctx.model_router.push(
+            task_id=ctx.task_id, task_type=key_type, dst_agency=partner_id, payload=byte_data)
         self.logger.info(
             f"task {ctx.task_id}: Sending {key_type} to {partner_id} finished, "
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
@@ -239,11 +217,8 @@ def _receive_byte_data(self, ctx, key_type, partner_index):
         start_time = time.time()
         partner_id = ctx.participant_id_list[partner_index]
 
-        byte_data = self._stub.pull(PullRequest(
-            sender=partner_id,
-            task_id=ctx.task_id,
-            key=key_type
-        ))
+        byte_data = self.ctx.model_router.pop(
+            task_id=ctx.task_id, task_type=key_type, from_inst=partner_id)
 
         self.logger.info(
             f"task {ctx.task_id}: Received {key_type} from {partner_id} finished, "
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 49654e82..92764d4a 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -11,17 +11,14 @@
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
 from ppc_model.common.protocol import ModelTask, TaskStatus, LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER
-from ppc_model.network.stub import ModelStub
 
 
 class TaskManager:
     def __init__(self, logger,
                  thread_event_manager: ThreadEventManager,
-                 stub: ModelStub,
                  task_timeout_h: Union[int, float]):
         self.logger = logger
         self._thread_event_manager = thread_event_manager
-        self._stub = stub
         self._task_timeout_s = task_timeout_h * 3600
         self._rw_lock = rwlock.RWLockWrite()
         self._tasks: dict[str, list] = {}
@@ -90,6 +87,12 @@ def kill_one_task(self, task_id: str):
         with self._rw_lock.gen_wlock():
             self._tasks[task_id][0] = TaskStatus.FAILED.value
 
+    def task_finished(self, task_id: str) -> bool:
+        (status, _, _) = self.status(task_id)
+        if status == TaskStatus.RUNNING.value:
+            return False
+        return True
+
     def status(self, task_id: str) -> [str, float, float]:
         """
         返回: 任务状态, 通讯量(MB), 执行耗时(s)
@@ -100,9 +103,9 @@ def status(self, task_id: str) -> [str, float, float]:
                     PpcErrorCode.TASK_NOT_FOUND.get_code(),
                     PpcErrorCode.TASK_NOT_FOUND.get_msg())
             status = self._tasks[task_id][0]
-            traffic_volume = self._stub.traffic_volume(task_id)
             time_costs = self._tasks[task_id][2]
-            return status, traffic_volume, time_costs
+            # TODO: the traffic_volume
+            return status, 0, time_costs
 
     def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None):
         with self._rw_lock.gen_wlock():
@@ -154,7 +157,6 @@ def _cleanup_finished_tasks(self):
                 if job_id in self._jobs:
                     del self._jobs[job_id]
                 self._thread_event_manager.remove_event(task_id)
-                self._stub.cleanup_cache(task_id)
                 self.logger.info(
                     f"Cleanup task cache, task_id: {task_id}, job_id: {job_id}")
 
diff --git a/python/ppc_model/task/test/task_manager_unittest.py b/python/ppc_model/task/test/task_manager_unittest.py
index 060d0d6c..33d38e1d 100644
--- a/python/ppc_model/task/test/task_manager_unittest.py
+++ b/python/ppc_model/task/test/task_manager_unittest.py
@@ -4,20 +4,11 @@
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from ppc_common.ppc_mock.mock_objects import MockLogger
 from ppc_model.common.protocol import ModelTask
-from ppc_model.common.mock.rpc_client_mock import RpcClientMock
 from ppc_model.task.task_manager import TaskManager
-from ppc_model.network.stub import ModelStub, PushRequest, PullRequest
 
-rpc_client = RpcClientMock()
 thread_event_manager = ThreadEventManager()
-stub = ModelStub(
-    agency_id='TEST_AGENCY',
-    thread_event_manager=thread_event_manager,
-    rpc_client=rpc_client,
-    send_retry_times=3,
-    retry_interval_s=0.1
-)
-rpc_client.set_message_handler(stub.on_message_received)
+
+# TODO: complement here
 
 
 def my_send_task(args):
@@ -25,49 +16,25 @@ def my_send_task(args):
     time.sleep(1)
     byte_array = bytearray(31 * 1024 * 1024)
     bytes_data = bytes(byte_array)
-    stub.push(PushRequest(
-        receiver=args['receiver'],
-        task_id=args['task_id'],
-        key=args['key'],
-        data=bytes_data
-    ))
     time.sleep(1)
 
 
 def my_receive_task(args):
-    print("start my_receive_task")
-    stub.pull(PullRequest(
-        sender=args['sender'],
-        task_id=args['task_id'],
-        key=args['key'],
-    ))
-    time.sleep(1)
     print("finish my_receive_task")
 
 
 def my_failed_task(args):
     print("start my_failed_task")
-    time.sleep(1)
     raise Exception('For Test')
 
 
 def my_long_task(args):
     print("start my_long_task")
-    stub.pull(PullRequest(
-        sender=args['sender'],
-        task_id=args['task_id'],
-        key='not_ready',
-    ))
     print("finish my_receive_task")
 
 
 def my_timeout_task(args):
     print("start my_timeout_task")
-    stub.pull(PullRequest(
-        sender=args['sender'],
-        task_id=args['task_id'],
-        key='not_ready',
-    ))
     print("finish my_timeout_task")
 
 
@@ -77,7 +44,6 @@ def setUp(self):
         self._task_manager = TaskManager(
             logger=MockLogger(),
             thread_event_manager=thread_event_manager,
-            stub=stub,
             task_timeout_h=0.0005
         )
         self._task_manager.register_task_handler(
diff --git a/python/ppc_model/tools/start.sh b/python/ppc_model/tools/start.sh
index 477ac215..2eb4834f 100644
--- a/python/ppc_model/tools/start.sh
+++ b/python/ppc_model/tools/start.sh
@@ -1,36 +1,37 @@
 #!/bin/bash
+SHELL_FOLDER=$(cd $(dirname $0);pwd)
+LOG_ERROR() {
+    content=${1}
+    echo -e "\033[31m[ERROR] ${content}\033[0m"
+}
 
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
+LOG_INFO() {
+    content=${1}
+    echo -e "\033[32m[INFO] ${content}\033[0m"
+}
+binary_path=${SHELL_FOLDER}/ppc_model_app.py
+cd ${SHELL_FOLDER}
+node=$(basename ${SHELL_FOLDER})
+node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
 
-# kill crypto process
-crypto_pro_num=`ps -ef  | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | wc -l`
-for i in $( seq 1 $crypto_pro_num )
+if [ ! -z ${node_pid} ];then
+    echo " ${node} is running, pid is $node_pid."
+    exit 0
+else
+    nohup python ${binary_path} > start.out 2>&1 &
+    sleep 1.5
+fi
+try_times=4
+i=0
+while [ $i -lt ${try_times} ]
 do
-  crypto_pid=`ps -ef  | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | awk 'NR==1{print}'`
-  kill -9 $crypto_pid
+    node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
+    success_flag=$(tail -n20  start.out | grep successfully)
+    if [[ ! -z ${node_pid} && ! -z "${success_flag}" ]];then
+        echo -e "\033[32m ${node} start successfully pid=${node_pid}\033[0m"
+        exit 0
+    fi
+    sleep 0.5
+    ((i=i+1))
 done
-
-sleep 1
-
-nohup python ppc_model_app.py > start.out 2>&1 &
-
-check_service() {
-    try_times=5
-    i=0
-    while [ -z `ps -ef | grep ${1} | grep python | grep -v grep | awk '{print $2}'` ]; do
-        sleep 1
-        ((i = i + 1))
-        if [ $i -lt ${try_times} ]; then
-            echo -e "\033[32m.\033[0m\c"
-        else
-            echo -e "\033[31m\nServer ${1} isn't running. \033[0m"
-            return
-        fi
-    done
-
-    echo -e "\033[32mServer ${1} started \033[0m"
-}
-
-sleep 5
-check_service ppc_model_app.py
\ No newline at end of file
+echo -e "\033[31m  Exceed waiting time. Please try again to start ${node} \033[0m"
\ No newline at end of file
diff --git a/python/ppc_model/tools/stop.sh b/python/ppc_model/tools/stop.sh
index 3b290668..ae35743c 100644
--- a/python/ppc_model/tools/stop.sh
+++ b/python/ppc_model/tools/stop.sh
@@ -1,19 +1,46 @@
 #!/bin/bash
+SHELL_FOLDER=$(cd $(dirname $0);pwd)
 
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
+LOG_ERROR() {
+    content=${1}
+    echo -e "\033[31m[ERROR] ${content}\033[0m"
+}
 
-# kill crypto process
-crypto_pro_num=`ps -ef  | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | wc -l`
-for i in $( seq 1 $crypto_pro_num )
-do
-  crypto_pid=`ps -ef  | grep /ppc/scripts | grep j- | grep -v 'grep' | awk '{print $2}' | awk 'NR==1{print}'`
-  kill -9 $crypto_pid
-done
+LOG_INFO() {
+    content=${1}
+    echo -e "\033[32m[INFO] ${content}\033[0m"
+}
 
-sleep 1
+binary_path=${SHELL_FOLDER}/ppc_model_app.py
+node=$(basename ${SHELL_FOLDER})
+node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
+try_times=10
+i=0
+if [ -z ${node_pid} ];then
+    echo " ${node} isn't running."
+    exit 0
+fi
 
-ppc_model_app_pid=`ps aux |grep ppc_model_app.py |grep -v grep |awk '{print $2}'`
-kill -9 $ppc_model_app_pid
+#Stop monitor here
+dirs=($(ls -l ${SHELL_FOLDER} | awk '/^d/ {print $NF}'))
+for dir in ${dirs[*]}
+do
+    if [[ -f "${SHELL_FOLDER}/${dir}/node.mtail" && -f "${SHELL_FOLDER}/${dir}/stop_mtail_monitor.sh" ]];then
+        echo "try to start ${dir}"
+        bash ${SHELL_FOLDER}/${dir}/stop_mtail_monitor.sh &
+    fi
+done
 
-echo -e "\033[32mServer ppc_model_app.py killed. \033[0m"
+[ ! -z ${node_pid} ] && kill ${node_pid} > /dev/null
+while [ $i -lt ${try_times} ]
+do
+    sleep 1
+    node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
+    if [ -z ${node_pid} ];then
+        echo -e "\033[32m stop ${node} success.\033[0m"
+        exit 0
+    fi
+    ((i=i+1))
+done
+echo "  Exceed maximum number of retries. Please try again to stop ${node}"
+exit 1
diff --git a/python/ppc_model_gateway/__init__.py b/python/ppc_model_gateway/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_model_gateway/clients/__init__.py b/python/ppc_model_gateway/clients/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_model_gateway/clients/client_manager.py b/python/ppc_model_gateway/clients/client_manager.py
deleted file mode 100644
index e05f89fb..00000000
--- a/python/ppc_model_gateway/clients/client_manager.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import os
-
-import grpc
-
-from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceStub
-from ppc_common.ppc_utils import utils
-from ppc_model_gateway import config
-
-
-class ClientManager:
-    def __init__(self, config_data, grpc_options):
-        self._config_data = config_data
-        self._grpc_options = grpc_options
-        channel = grpc.insecure_channel(
-            self._config_data['NODE_ENDPOINT'], options=self._grpc_options)
-        self.node_stub = ModelServiceStub(channel)
-        self.agency_stub_dict = {}
-        self._create_partner_stubs()
-
-    def _create_partner_stubs(self):
-        for agency_id, endpoint in self._get_agency_dict().items():
-            if self._config_data['SSL_SWITCH'] == 0:
-                channel = grpc.insecure_channel(
-                    endpoint, options=self._grpc_options)
-                self.agency_stub_dict[agency_id] = ModelServiceStub(channel)
-            else:
-                channel = self._create_secure_channel(endpoint)
-                self.agency_stub_dict[agency_id] = ModelServiceStub(channel)
-
-    def _get_agency_dict(self) -> dict:
-        agency_dict = {}
-        for entry in self._config_data.get('AGENCY_LIST', []):
-            if ':' in entry:
-                key, value = entry.split(":", 1)
-                key = key.strip()
-                value = value.strip()
-                agency_dict[key] = value
-        return agency_dict
-
-    def _create_secure_channel(self, target):
-        grpc_root_crt = utils.load_credential_from_file(
-            os.path.abspath(self._config_data['SSL_CA']))
-        grpc_ssl_key = utils.load_credential_from_file(
-            os.path.abspath(self._config_data['SSL_KEY']))
-        grpc_ssl_crt = utils.load_credential_from_file(
-            os.path.abspath(self._config_data['SSL_CRT']))
-
-        credentials = grpc.ssl_channel_credentials(
-            root_certificates=grpc_root_crt,
-            private_key=grpc_ssl_key,
-            certificate_chain=grpc_ssl_crt
-        )
-
-        return grpc.secure_channel(target, credentials, options=self._grpc_options)
-
-
-client_manager = ClientManager(config.CONFIG_DATA, config.grpc_options)
diff --git a/python/ppc_model_gateway/conf/application-sample.yml b/python/ppc_model_gateway/conf/application-sample.yml
deleted file mode 100644
index 3c7abf8c..00000000
--- a/python/ppc_model_gateway/conf/application-sample.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-HOST: "0.0.0.0"
-
-# 0: off, 1: on
-SSL_SWITCH: 1
-SSL_CA: "cert/ca.crt"
-SSL_KEY: "cert/node.key"
-SSL_CRT: "cert/node.crt"
-
-# 0801配置
-NODE_TO_PARTNER_RPC_PORT: 43454
-PARTNER_TO_NODE_RPC_PORT: 43451
-
-NODE_ENDPOINT: "127.0.0.1:43472"
-
-AGENCY_LIST:
-  #- "SG: [@IDC_PPCS_SG_MODEL_GATEWAY_ENDPOINT]"
-  - "WeBank: 127.0.0.1:43451"
-
-MAX_MESSAGE_LENGTH_MB: 100
diff --git a/python/ppc_model_gateway/conf/logging.conf b/python/ppc_model_gateway/conf/logging.conf
deleted file mode 100644
index 4a7b0aa9..00000000
--- a/python/ppc_model_gateway/conf/logging.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-[loggers]
-keys=root,grpc
-
-[logger_root]
-level=INFO
-handlers=consoleHandler,fileHandler
-
-[logger_grpc]
-level = DEBUG
-handlers = accessHandler
-qualname = grpc
-propagate = 0
-
-[handlers]
-keys=fileHandler,consoleHandler,accessHandler
-
-[handler_accessHandler]
-class=handlers.TimedRotatingFileHandler
-args=('logs/appmonitor.log', 'D', 1, 30, 'utf-8')
-level=INFO
-formatter=simpleFormatter
-
-[handler_fileHandler]
-class=handlers.TimedRotatingFileHandler
-args=('logs/ppcs-modelgateway-gateway.log', 'D', 1, 30, 'utf-8')
-level=INFO
-formatter=simpleFormatter
-
-[handler_consoleHandler]
-class=StreamHandler
-args=(sys.stdout,)
-level=ERROR
-formatter=simpleFormatter
-
-[formatters]
-keys=simpleFormatter
-
-[formatter_simpleFormatter]
-format=[%(levelname)s][%(asctime)s %(msecs)03d][%(process)d][%(filename)s:%(lineno)d] %(message)s
-datefmt=%Y-%m-%d %H:%M:%S
diff --git a/python/ppc_model_gateway/config.py b/python/ppc_model_gateway/config.py
deleted file mode 100644
index 0e36903d..00000000
--- a/python/ppc_model_gateway/config.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# -*- coding: utf-8 -*-
-import logging
-import logging.config
-import os
-
-import yaml
-
-path = os.getcwd()
-log_dir = os.sep.join([path, 'logs'])
-chain_log_dir = os.sep.join([path, 'bin', 'logs'])
-print(f"log_dir: {log_dir}")
-print(f"chain_log_dir: {chain_log_dir}")
-if not os.path.exists(log_dir):
-    os.makedirs(log_dir)
-if not os.path.exists(chain_log_dir):
-    os.makedirs(chain_log_dir)
-logging_conf_path = os.path.normpath('logging.conf')
-logging.config.fileConfig(logging_conf_path)
-
-
-def get_logger(name=None):
-    log = logging.getLogger(name)
-    return log
-
-
-config_path = "application.yml"
-
-CONFIG_DATA = {}
-agency_dict = {}
-
-
-def read_config():
-    with open(config_path, 'rb') as f:
-        global CONFIG_DATA
-        CONFIG_DATA = yaml.safe_load(f.read())
-
-
-read_config()
-
-grpc_options = [
-    ('grpc.ssl_target_name_override', 'PPCS MODEL GATEWAY'),
-    ('grpc.max_send_message_length',
-     CONFIG_DATA['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024),
-    ('grpc.max_receive_message_length',
-     CONFIG_DATA['MAX_MESSAGE_LENGTH_MB'] * 1024 * 1024),
-    ('grpc.keepalive_time_ms', 15000),  # 每 15 秒发送一次心跳
-    ('grpc.keepalive_timeout_ms', 5000),  # 等待心跳回应的超时时间为 5 秒
-    ('grpc.keepalive_permit_without_calls', True),  # 即使没有调用也允许发送心跳
-    ('grpc.http2.min_time_between_pings_ms', 15000),  # 心跳之间最小时间间隔为 15 秒
-    ('grpc.http2.max_pings_without_data', 0),  # 在发送数据前不限制心跳次数
-    # 在没有数据传输的情况下，确保心跳包之间至少有 20 秒的间隔
-    ('grpc.http2.min_ping_interval_without_data_ms', 20000),
-    ("grpc.so_reuseport", 1),
-    ("grpc.use_local_subchannel_pool", 1),
-    ('grpc.enable_retries', 1),
-    ('grpc.service_config',
-     '{ "retryPolicy":{ "maxAttempts": 4, "initialBackoff": "0.1s", "maxBackoff": "1s", "backoffMutiplier": 2, "retryableStatusCodes": [ "UNAVAILABLE" ] } }')
-]
diff --git a/python/ppc_model_gateway/endpoints/__init__.py b/python/ppc_model_gateway/endpoints/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_model_gateway/endpoints/node_to_partner.py b/python/ppc_model_gateway/endpoints/node_to_partner.py
deleted file mode 100644
index 71f6aa07..00000000
--- a/python/ppc_model_gateway/endpoints/node_to_partner.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import time
-import traceback
-
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest
-from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceServicer
-from ppc_model_gateway import config
-from ppc_model_gateway.clients.client_manager import client_manager
-from ppc_model_gateway.endpoints.response_builder import build_error_model_response
-
-log = config.get_logger()
-
-
-class NodeToPartnerService(ModelServiceServicer):
-
-    def MessageInteraction(self, request: ModelRequest, context):
-        start_time = time.time()
-        try:
-            log.debug(
-                f"start sending data to {request.receiver}, task_id: {request.task_id}, "
-                f"key: {request.key}, seq: {request.seq}")
-            # 根据接收方的机构ID路由消息
-            client = client_manager.agency_stub_dict[request.receiver]
-            response = client.MessageInteraction(request)
-            end_time = time.time()
-            log.info(
-                f"finish sending data to {request.receiver}, task_id: {request.task_id}, "
-                f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, "
-                f"ret_code: {response.base_response.error_code}, time_costs: {str(end_time - start_time)}s")
-        except Exception:
-            end_time = time.time()
-            message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \
-                      f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \
-                      f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s"
-            log.warn(message)
-            response = build_error_model_response(message)
-        return response
diff --git a/python/ppc_model_gateway/endpoints/partner_to_node.py b/python/ppc_model_gateway/endpoints/partner_to_node.py
deleted file mode 100644
index e4aebfce..00000000
--- a/python/ppc_model_gateway/endpoints/partner_to_node.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import time
-import traceback
-
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest
-from ppc_common.ppc_protos.generated.ppc_model_pb2_grpc import ModelServiceServicer
-from ppc_model_gateway import config
-from ppc_model_gateway.clients.client_manager import client_manager
-from ppc_model_gateway.endpoints.response_builder import build_error_model_response
-
-log = config.get_logger()
-
-
-class PartnerToNodeService(ModelServiceServicer):
-
-    def MessageInteraction(self, request: ModelRequest, context):
-        start_time = time.time()
-        try:
-            log.debug(
-                f"start sending data to {request.receiver}, task_id: {request.task_id}, "
-                f"key: {request.key}, seq: {request.seq}")
-            response = client_manager.node_stub.MessageInteraction(request)
-            end_time = time.time()
-            log.info(
-                f"finish sending data to {request.receiver}, task_id: {request.task_id}, "
-                f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, "
-                f"ret_code: {response.base_response.error_code}, time_costs: {str(end_time - start_time)}s")
-        except Exception:
-            end_time = time.time()
-            message = f"[OnWarn]Send data to {request.receiver} failed, task_id: {request.task_id}, " \
-                      f"key: {request.key}, seq: {request.seq}, slice_num: {request.slice_num}, " \
-                      f"exception:{str(traceback.format_exc())}, time_costs: {str(end_time - start_time)}s"
-            log.warn(message)
-            response = build_error_model_response(message)
-        return response
diff --git a/python/ppc_model_gateway/endpoints/response_builder.py b/python/ppc_model_gateway/endpoints/response_builder.py
deleted file mode 100644
index f97fa21a..00000000
--- a/python/ppc_model_gateway/endpoints/response_builder.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelResponse
-
-
-def build_error_model_response(message: str):
-    model_response = ModelResponse()
-    model_response.base_response.error_code = -1
-    model_response.base_response.message = message
-    return model_response
diff --git a/python/ppc_model_gateway/ppc_model_gateway_app.py b/python/ppc_model_gateway/ppc_model_gateway_app.py
deleted file mode 100644
index 6d53286e..00000000
--- a/python/ppc_model_gateway/ppc_model_gateway_app.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Note: here can't be refactored by autopep
-import sys
-sys.path.append("../")
-
-import os
-from concurrent import futures
-from threading import Thread
-import grpc
-from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc
-from ppc_common.ppc_utils import utils
-from ppc_model_gateway import config
-from ppc_model_gateway.endpoints.node_to_partner import NodeToPartnerService
-from ppc_model_gateway.endpoints.partner_to_node import PartnerToNodeService
-
-
-
-log = config.get_logger()
-
-
-def node_to_partner_serve():
-    rpc_port = config.CONFIG_DATA['NODE_TO_PARTNER_RPC_PORT']
-
-    ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)),
-                            options=config.grpc_options)
-    ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-        NodeToPartnerService(), ppc_serve)
-    address = "[::]:{}".format(rpc_port)
-    ppc_serve.add_insecure_port(address)
-
-    ppc_serve.start()
-
-    start_message = f'Start ppc model gateway internal rpc server at {rpc_port}'
-    print(start_message)
-    log.info(start_message)
-    ppc_serve.wait_for_termination()
-
-
-def partner_to_node_serve():
-    rpc_port = config.CONFIG_DATA['PARTNER_TO_NODE_RPC_PORT']
-
-    if config.CONFIG_DATA['SSL_SWITCH'] == 0:
-        ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)),
-                                options=config.grpc_options)
-        ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-            PartnerToNodeService(), ppc_serve)
-        address = "[::]:{}".format(rpc_port)
-        ppc_serve.add_insecure_port(address)
-    else:
-        grpc_root_crt = utils.load_credential_from_file(
-            os.path.abspath(config.CONFIG_DATA['SSL_CA']))
-        grpc_ssl_key = utils.load_credential_from_file(
-            os.path.abspath(config.CONFIG_DATA['SSL_KEY']))
-        grpc_ssl_crt = utils.load_credential_from_file(
-            os.path.abspath(config.CONFIG_DATA['SSL_CRT']))
-        server_credentials = grpc.ssl_server_credentials(((
-            grpc_ssl_key,
-            grpc_ssl_crt,
-        ),), grpc_root_crt, True)
-
-        ppc_serve = grpc.server(futures.ThreadPoolExecutor(max_workers=max(1, os.cpu_count() - 1)),
-                                options=config.grpc_options)
-        ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-            PartnerToNodeService(), ppc_serve)
-        address = "[::]:{}".format(rpc_port)
-        ppc_serve.add_secure_port(address, server_credentials)
-
-    ppc_serve.start()
-
-    start_message = f'Start ppc model gateway external rpc server at {rpc_port}'
-    print(start_message)
-    log.info(start_message)
-    ppc_serve.wait_for_termination()
-
-
-if __name__ == '__main__':
-    log = config.get_logger()
-
-    # 设置守护线程
-    node_to_partner_serve_thread = Thread(target=node_to_partner_serve)
-    partner_to_node_serve_thread = Thread(target=partner_to_node_serve)
-
-    node_to_partner_serve_thread.daemon = True
-    partner_to_node_serve_thread.daemon = True
-
-    node_to_partner_serve_thread.start()
-    partner_to_node_serve_thread.start()
-
-    node_to_partner_serve_thread.join()
-    partner_to_node_serve_thread.join()
-
-    message = f'Start ppc model gateway successfully.'
-    print(message)
-    log.info(message)
diff --git a/python/ppc_model_gateway/test/__init__.py b/python/ppc_model_gateway/test/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_model_gateway/test/client.py b/python/ppc_model_gateway/test/client.py
deleted file mode 100644
index a941eb1b..00000000
--- a/python/ppc_model_gateway/test/client.py
+++ /dev/null
@@ -1,41 +0,0 @@
-
-import grpc
-import sys
-import os
-
-from ppc_model_gateway import config
-from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelRequest
-
-
-def generate_bytes(size_in_mb):
-    size_in_bytes = size_in_mb * 1024 * 1023
-    return os.urandom(size_in_bytes)
-
-
-def send_data():
-    channel = grpc.insecure_channel(
-        f'localhost:{port}', options=config.grpc_options)
-    stub = ppc_model_pb2_grpc.ModelServiceStub(channel)
-
-    request = ModelRequest()
-
-    request.task_id = "task_id"
-    request.receiver = receiver
-    request.key = 'key'
-    request.seq = 0
-    request.slice_num = 1
-    request.data = bytes(generate_bytes(
-        config.CONFIG_DATA['MAX_MESSAGE_LENGTH_MB']))
-
-    response = stub.MessageInteraction(request)
-    print("Received response:", response.base_response.message)
-
-
-if __name__ == '__main__':
-    if len(sys.argv) != 3:
-        print("Usage: python client.py  ")
-        sys.exit(1)
-    port = int(sys.argv[1])
-    receiver = sys.argv[2]
-    send_data()
diff --git a/python/ppc_model_gateway/test/server.py b/python/ppc_model_gateway/test/server.py
deleted file mode 100644
index 03d2a93a..00000000
--- a/python/ppc_model_gateway/test/server.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import os
-from concurrent import futures
-import grpc
-import sys
-
-from ppc_model_gateway import config
-from ppc_common.ppc_protos.generated import ppc_model_pb2_grpc
-from ppc_common.ppc_protos.generated.ppc_model_pb2 import ModelResponse
-
-
-class ModelService(ppc_model_pb2_grpc.ModelServiceServicer):
-    def MessageInteraction(self, request, context):
-        response = ModelResponse()
-        response.base_response.error_code = 0
-        response.base_response.message = "Data received successfully."
-        response.data = request.data
-        return response
-
-
-def serve():
-    server = grpc.server(futures.ThreadPoolExecutor(
-        max_workers=max(1, os.cpu_count() - 1)),
-        options=config.grpc_options)
-    ppc_model_pb2_grpc.add_ModelServiceServicer_to_server(
-        ModelService(), server)
-    server.add_insecure_port(f'[::]:{port}')
-    server.start()
-    print(f'Start serve successfully at {port}.')
-    server.wait_for_termination()
-
-
-if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        print("Usage: python server.py ")
-        sys.exit(1)
-    port = sys.argv[1]
-    serve()
diff --git a/python/ppc_model_gateway/tools/gen_cert.sh b/python/ppc_model_gateway/tools/gen_cert.sh
deleted file mode 100644
index 199b7844..00000000
--- a/python/ppc_model_gateway/tools/gen_cert.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/bash
-
-print_usage() {
-    echo "Usage: $0 [--ca] [--node ] [-h|--help]"
-    echo "  --ca                    Generate CA key and certificate"
-    echo "  --node          Generate  sets of certificates and place them in nodeX directories"
-    echo "  -h, --help              Display this help message"
-}
-
-generate_ca() {
-    echo "Generating CA key and certificate..."
-    # 生成CA私钥
-    openssl genrsa -out ca.key 2048
-    # 生成自签名的CA证书
-    openssl req -x509 -new -nodes -key ca.key -sha256 -days 36500 -out ca.crt -subj "/CN=PPCS CA"
-}
-
-generate_node_certificates() {
-    local number=$1
-    echo "Generating $number sets of node certificates..."
-    for ((i=1; i<=$number; i++)); do
-        # 生成节点私钥
-        openssl genrsa -out node.key 2048
-        # 生成证书签署请求(CSR)
-        openssl req -new -key node.key -out node.csr -subj "/CN=PPCS MODEL GATEWAY"
-        # 使用CA证书签署CSR以生成节点证书
-        openssl x509 -req -in node.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out node.crt -days 36500 -sha256
-        # 清理CSR文件
-        rm node.csr
-        # 创建节点目录并将证书移入
-        mkdir -p "node$i"
-        mv node.key node.crt "node$i/"
-        echo "Generated certificate set $i and placed in node$i directory"
-    done
-}
-
-# 检查参数
-if [[ $# -eq 0 ]]; then
-    echo "Error: No arguments provided."
-    print_usage
-    exit 1
-fi
-
-while [[ "$1" != "" ]]; do
-    case $1 in
-        --ca)
-            generate_ca
-            ;;
-        --node)
-            shift
-            if [[ $1 =~ ^[0-9]+$ ]]; then
-                generate_node_certificates $1
-            else
-                echo "Error: --node argument expects a number."
-                print_usage
-                exit 1
-            fi
-            ;;
-        -h | --help)
-            print_usage
-            exit 0
-            ;;
-        *)
-            echo "Error: Invalid argument: $1"
-            print_usage
-            exit 1
-            ;;
-    esac
-    shift
-done
diff --git a/python/ppc_model_gateway/tools/start.sh b/python/ppc_model_gateway/tools/start.sh
deleted file mode 100644
index a25aa05e..00000000
--- a/python/ppc_model_gateway/tools/start.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
-
-export PYTHONPATH=$dirpath/../
-source /data/app/ppcs-modelgateway/gateway_env/bin/deactivate
-source /data/app/ppcs-modelgateway/gateway_env/bin/activate
-sleep 1
-
-rm -rf $dirpath/../success
-nohup python $dirpath/ppc_model_gateway_app.py > start.out 2>&1 &
-
-check_service() {
-    try_times=5
-    i=0
-    while [ -z `ps -ef | grep ${1} | grep python | grep -v grep | awk '{print $2}'` ]; do
-        sleep 1
-        ((i = i + 1))
-        if [ $i -lt ${try_times} ]; then
-            echo -e "\033[32m.\033[0m\c"
-        else
-            echo -e "\033[31m\nServer ${1} isn't running. \033[0m"
-            return
-        fi
-    done
-    echo -e "\033[32mServer ${1} started \033[0m"
-	echo "success" > $dirpath/../success
-}
-
-sleep 5
-check_service ppc_model_gateway_app.py
diff --git a/python/ppc_model_gateway/tools/stop.sh b/python/ppc_model_gateway/tools/stop.sh
deleted file mode 100644
index fd17dcea..00000000
--- a/python/ppc_model_gateway/tools/stop.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
-
-sleep 1
-
-ppc_model_gateway_app_pid=`ps aux |grep ppc_model_gateway_app.py |grep -v grep |awk '{print $2}'`
-kill -9 $ppc_model_gateway_app_pid
-
-echo -e "\033[32mServer ppc_model_gateway_app.py killed. \033[0m"

From 9e62fb7182e525dcb96aa87dc46e9de48c21be40 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 25 Oct 2024 18:19:14 +0800
Subject: [PATCH 068/120] erase transport dependency to bcos-boostssl (#65)

* support windows compile

* optimize compile

* add build_wedpr_toolkit

* update cmake

* erase front dependency to bcos-boostssl

* recover removed methods of Message

* refactor workflow

* remove unused ppc-scheduler
---
 .github/workflows/cpp_full_node_workflow.yml  | 170 +++++++
 .github/workflows/cpp_sdk_workflow.yml        | 158 ++++++
 .github/workflows/cpp_toolkit_workflow.yml    | 149 ++++++
 .github/workflows/cpp_workflow.yml            | 227 ---------
 cpp/CMakeLists.txt                            |   1 +
 cpp/cmake/CompilerSettings.cmake              |  15 +-
 cpp/cmake/Dependencies.cmake                  |  16 +-
 cpp/cmake/Options.cmake                       |   7 +
 cpp/cmake/grpc.cmake                          |  47 ++
 cpp/cmake/python.cmake                        |  17 -
 cpp/cmake/{java.cmake => swig.cmake}          |   7 +-
 cpp/ppc-framework/protocol/Message.h          |  68 +--
 cpp/ppc-framework/protocol/P2PMessage.h       |  98 ++++
 cpp/tools/install_depends.sh                  |   9 +-
 cpp/vcpkg-configuration.json                  |   2 +-
 cpp/vcpkg.json                                |  42 +-
 cpp/wedpr-computing/ppc-mpc/CMakeLists.txt    |   1 -
 cpp/wedpr-computing/ppc-pir/CMakeLists.txt    |   1 -
 .../ppc-pir/tests/CMakeLists.txt              |   2 +-
 .../ppc-pir/tests/data/AysPreDataset.csv      |   3 +
 cpp/wedpr-computing/ppc-psi/CMakeLists.txt    |   1 -
 .../ppc-crypto-core/CMakeLists.txt            |   2 -
 .../ppc-crypto-core/src/CMakeLists.txt        |   3 +-
 cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt    |   2 -
 .../ppc-crypto/src/CMakeLists.txt             |   2 +-
 cpp/wedpr-crypto/ppc-homo/CMakeLists.txt      |   2 -
 cpp/wedpr-crypto/ppc-udf/CMakeLists.txt       |   2 -
 cpp/wedpr-crypto/sdk/CMakeLists.txt           |   2 -
 cpp/wedpr-helper/ppc-tools/CMakeLists.txt     |   2 -
 cpp/wedpr-main/CMakeLists.txt                 |   2 -
 cpp/wedpr-main/gateway/CMakeLists.txt         |   4 -
 cpp/wedpr-main/pro-node/CMakeLists.txt        |   4 -
 cpp/wedpr-protocol/grpc/CMakeLists.txt        |   8 +-
 cpp/wedpr-protocol/protobuf/CMakeLists.txt    |   2 -
 cpp/wedpr-protocol/protocol/CMakeLists.txt    |   2 -
 .../protocol/src/CMakeLists.txt               |   2 +-
 .../protocol/src/JsonTaskImpl.cpp             |   2 +-
 .../protocol/src/v1/MessageImpl.cpp           |  18 -
 .../protocol/src/v1/MessageImpl.h             |  25 +-
 .../protocol/tests/CMakeLists.txt             |   2 +-
 cpp/wedpr-protocol/tars/CMakeLists.txt        |   5 -
 cpp/wedpr-protocol/tars/TarsStruct.h          |  15 +-
 cpp/wedpr-storage/ppc-io/CMakeLists.txt       |   2 -
 cpp/wedpr-storage/ppc-storage/CMakeLists.txt  |   1 -
 .../ppc-front/ppc-front/CallbackManager.cpp   |   4 +
 .../ppc-gateway/CMakeLists.txt                |   4 -
 .../ppc-gateway/GatewayFactory.cpp            |   6 +-
 .../ppc-gateway/gateway/GatewayImpl.cpp       |  21 +-
 .../ppc-gateway/gateway/GatewayImpl.h         |   2 +-
 .../gateway/SendMessageWithRetry.cpp          |  12 +-
 .../gateway/SendMessageWithRetry.h            |   6 +-
 .../gateway/cache/MessageCache.cpp            |   4 +-
 .../ppc-gateway/gateway/cache/MessageCache.h  |   6 +-
 .../gateway/router/GatewayRouterManager.cpp   |   9 +-
 .../gateway/router/LocalRouter.cpp            |  11 +-
 .../ppc-gateway/gateway/router/LocalRouter.h  |   6 +-
 .../gateway/router/PeerRouterTable.cpp        |   4 +-
 .../gateway/router/PeerRouterTable.h          |   4 +-
 .../ppc-gateway/ppc-gateway/p2p/Service.cpp   |  23 +-
 .../ppc-gateway/ppc-gateway/p2p/Service.h     |   2 +-
 .../ppc-gateway/p2p/router/RouterManager.cpp  |   6 +-
 .../ppc-gateway/protocol/P2PMessageImpl.cpp   |  43 ++
 .../ppc-gateway/protocol/P2PMessageImpl.h     |  64 +++
 cpp/wedpr-transport/ppc-http/CMakeLists.txt   |   1 -
 cpp/wedpr-transport/ppc-rpc/CMakeLists.txt    |   1 -
 .../sdk-wrapper/CMakeLists.txt                |   3 -
 .../sdk-wrapper/java/CMakeLists.txt           |   7 +-
 .../wedpr/sdk/jni/generated/Message.java      |  31 +-
 .../jni/generated/wedpr_java_transport.java   |   5 -
 .../generated/wedpr_java_transportJNI.java    |  19 +-
 .../src/wedpr_java_transportJAVA_wrap.cxx     | 132 ++---
 .../sdk-wrapper/python/CMakeLists.txt         |   1 +
 .../generated/wedpr_python_transport.py       |  24 +-
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 459 +++++++----------
 cpp/wedpr-transport/sdk/src/CMakeLists.txt    |   2 -
 python/ppc_scheduler/__init__.py              |   0
 python/ppc_scheduler/common/__init__.py       |   0
 python/ppc_scheduler/common/global_context.py |   3 -
 .../common/global_job_manager.py              |  15 -
 python/ppc_scheduler/common/initializer.py    |  72 ---
 python/ppc_scheduler/common/log_utils.py      |  78 ---
 .../ppc_scheduler/conf/application-sample.yml |  13 -
 python/ppc_scheduler/conf/logging.conf        |  40 --
 python/ppc_scheduler/database/__init__.py     |   0
 .../database/computing_node_mapper.py         |  74 ---
 .../database/job_worker_mapper.py             |  52 --
 .../demo/default_flow_config_sample.py        | 224 ---------
 .../demo/prefect_workflow_sample.py           |  79 ---
 python/ppc_scheduler/demo/web_submit_job.json | 134 -----
 .../demo/workflow_args_sample.json            |  73 ---
 python/ppc_scheduler/endpoints/__init__.py    |   0
 python/ppc_scheduler/endpoints/body_schema.py |  18 -
 .../ppc_scheduler/endpoints/job_controller.py |  70 ---
 python/ppc_scheduler/endpoints/restx.py       |  35 --
 python/ppc_scheduler/job/__init__.py          |   0
 python/ppc_scheduler/job/job_manager.py       | 154 ------
 python/ppc_scheduler/job/job_status.py        |   5 -
 python/ppc_scheduler/job/job_type.py          |  12 -
 .../ppc_scheduler/mpc_generator/__init__.py   |   0
 .../ppc_scheduler/mpc_generator/generator.py  | 468 ------------------
 .../mpc_generator/mpc_func_str.py             |  64 ---
 .../mpc_sample/aggr_func_only.mpc             | 136 -----
 .../mpc_sample/aggr_func_with_group_by.mpc    | 234 ---------
 .../mpc_sample/basic_arith_ope.mpc            |  78 ---
 .../mpc_generator/test_generator.py           | 101 ----
 python/ppc_scheduler/node/__init__.py         |   0
 .../node/computing_node_client/__init__.py    |   0
 .../model_node_client.py                      |  95 ----
 .../computing_node_client/mpc_node_client.py  |  32 --
 .../node/computing_node_client/psi_client.py  |  74 ---
 .../node/computing_node_client/utils.py       |   8 -
 python/ppc_scheduler/node/node_manager.py     |  31 --
 python/ppc_scheduler/ppc_scheduler_app.py     | 125 -----
 python/ppc_scheduler/scripts/start.sh         |  69 ---
 python/ppc_scheduler/scripts/stop.sh          |  42 --
 python/ppc_scheduler/workflow/__init__.py     |   0
 .../workflow/builder/__init__.py              |   0
 .../workflow/builder/flow_builder.py          |  86 ----
 .../ppc_scheduler/workflow/common/__init__.py |   0
 python/ppc_scheduler/workflow/common/codec.py |  19 -
 .../workflow/common/flow_utils.py             |  34 --
 .../workflow/common/job_context.py            |  74 ---
 .../workflow/common/worker_status.py          |   7 -
 .../workflow/common/worker_type.py            |  14 -
 .../workflow/scheduler/__init__.py            |   0
 .../workflow/scheduler/scheduler.py           | 100 ----
 .../workflow/scheduler/scheduler_api.py       |   8 -
 .../ppc_scheduler/workflow/worker/__init__.py |   0
 .../workflow/worker/api_worker.py             |  10 -
 .../workflow/worker/default_worker.py         |  37 --
 .../workflow/worker/engine/__init__.py        |   0
 .../workflow/worker/engine/model_engine.py    |  52 --
 .../workflow/worker/engine/mpc_engine.py      |  31 --
 .../workflow/worker/engine/psi_engine.py      |  31 --
 .../workflow/worker/engine/shell_engine.py    |  16 -
 .../workflow/worker/engine/work_engine.py     |   6 -
 .../workflow/worker/model_worker.py           |  19 -
 .../workflow/worker/mpc_worker.py             |  19 -
 .../workflow/worker/psi_worker.py             |  21 -
 .../workflow/worker/python_worker.py          |  10 -
 .../workflow/worker/shell_worker.py           |  16 -
 .../ppc_scheduler/workflow/worker/worker.py   | 117 -----
 .../workflow/worker/worker_factory.py         |  34 --
 python/requirements.txt                       |   1 -
 144 files changed, 1149 insertions(+), 4395 deletions(-)
 create mode 100644 .github/workflows/cpp_full_node_workflow.yml
 create mode 100644 .github/workflows/cpp_sdk_workflow.yml
 create mode 100644 .github/workflows/cpp_toolkit_workflow.yml
 delete mode 100644 .github/workflows/cpp_workflow.yml
 create mode 100644 cpp/cmake/grpc.cmake
 rename cpp/cmake/{java.cmake => swig.cmake} (68%)
 create mode 100644 cpp/ppc-framework/protocol/P2PMessage.h
 create mode 100644 cpp/wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv
 create mode 100644 cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.cpp
 create mode 100644 cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.h
 delete mode 100644 python/ppc_scheduler/__init__.py
 delete mode 100644 python/ppc_scheduler/common/__init__.py
 delete mode 100644 python/ppc_scheduler/common/global_context.py
 delete mode 100644 python/ppc_scheduler/common/global_job_manager.py
 delete mode 100644 python/ppc_scheduler/common/initializer.py
 delete mode 100644 python/ppc_scheduler/common/log_utils.py
 delete mode 100644 python/ppc_scheduler/conf/application-sample.yml
 delete mode 100644 python/ppc_scheduler/conf/logging.conf
 delete mode 100644 python/ppc_scheduler/database/__init__.py
 delete mode 100644 python/ppc_scheduler/database/computing_node_mapper.py
 delete mode 100644 python/ppc_scheduler/database/job_worker_mapper.py
 delete mode 100644 python/ppc_scheduler/demo/default_flow_config_sample.py
 delete mode 100644 python/ppc_scheduler/demo/prefect_workflow_sample.py
 delete mode 100644 python/ppc_scheduler/demo/web_submit_job.json
 delete mode 100644 python/ppc_scheduler/demo/workflow_args_sample.json
 delete mode 100644 python/ppc_scheduler/endpoints/__init__.py
 delete mode 100644 python/ppc_scheduler/endpoints/body_schema.py
 delete mode 100644 python/ppc_scheduler/endpoints/job_controller.py
 delete mode 100644 python/ppc_scheduler/endpoints/restx.py
 delete mode 100644 python/ppc_scheduler/job/__init__.py
 delete mode 100644 python/ppc_scheduler/job/job_manager.py
 delete mode 100644 python/ppc_scheduler/job/job_status.py
 delete mode 100644 python/ppc_scheduler/job/job_type.py
 delete mode 100644 python/ppc_scheduler/mpc_generator/__init__.py
 delete mode 100644 python/ppc_scheduler/mpc_generator/generator.py
 delete mode 100644 python/ppc_scheduler/mpc_generator/mpc_func_str.py
 delete mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc
 delete mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc
 delete mode 100644 python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc
 delete mode 100644 python/ppc_scheduler/mpc_generator/test_generator.py
 delete mode 100644 python/ppc_scheduler/node/__init__.py
 delete mode 100644 python/ppc_scheduler/node/computing_node_client/__init__.py
 delete mode 100644 python/ppc_scheduler/node/computing_node_client/model_node_client.py
 delete mode 100644 python/ppc_scheduler/node/computing_node_client/mpc_node_client.py
 delete mode 100644 python/ppc_scheduler/node/computing_node_client/psi_client.py
 delete mode 100644 python/ppc_scheduler/node/computing_node_client/utils.py
 delete mode 100644 python/ppc_scheduler/node/node_manager.py
 delete mode 100644 python/ppc_scheduler/ppc_scheduler_app.py
 delete mode 100755 python/ppc_scheduler/scripts/start.sh
 delete mode 100755 python/ppc_scheduler/scripts/stop.sh
 delete mode 100644 python/ppc_scheduler/workflow/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/builder/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/builder/flow_builder.py
 delete mode 100644 python/ppc_scheduler/workflow/common/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/common/codec.py
 delete mode 100644 python/ppc_scheduler/workflow/common/flow_utils.py
 delete mode 100644 python/ppc_scheduler/workflow/common/job_context.py
 delete mode 100644 python/ppc_scheduler/workflow/common/worker_status.py
 delete mode 100644 python/ppc_scheduler/workflow/common/worker_type.py
 delete mode 100644 python/ppc_scheduler/workflow/scheduler/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/scheduler/scheduler.py
 delete mode 100644 python/ppc_scheduler/workflow/scheduler/scheduler_api.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/api_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/default_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/__init__.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/model_engine.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/mpc_engine.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/psi_engine.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/shell_engine.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/engine/work_engine.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/model_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/mpc_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/psi_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/python_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/shell_worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/worker.py
 delete mode 100644 python/ppc_scheduler/workflow/worker/worker_factory.py

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
new file mode 100644
index 00000000..7779352c
--- /dev/null
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -0,0 +1,170 @@
+name: WeDPR-Component ci for full node
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  release:
+    types: [published, push]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
+jobs:
+  build:
+    name: build all
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-12, ubuntu-20.04]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 5
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2022-07-28
+          override: true
+      - name: Prepare vcpkg
+        if: runner.os != 'Windows'
+        uses: friendlyanon/setup-vcpkg@v1
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
+      - uses: actions/cache@v2
+        id: deps_cache
+        with:
+          path: |
+            deps/
+            c:/vcpkg
+            !c:/vcpkg/.git
+            !c:/vcpkg/buildtrees
+            !c:/vcpkg/packages
+            !c:/vcpkg/downloads
+          key: build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          restore-keys: |
+            build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-all-${{ matrix.os }}-${{ github.base_ref }}-
+            build-all-${{ matrix.os }}-
+      - name: Build for linux
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt install -y lcov ccache wget libgmp-dev python3-dev
+          export GCC='gcc-10'
+          export CXX='g++-10'
+          bash -x cpp/tools/install_depends.sh -o ubuntu
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ || cat /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-osx-dbg-err.log
+      - name: Build for macOS
+        if: runner.os == 'macOS'
+        run: |
+          bash -x cpp/tools/install_depends.sh -o macos
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+            name: config-x64-linux-dbg-err.log
+            path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log
+      #- name: Test
+      #  if: runner.os != 'Windows'
+      #  run: |
+      #    cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest
+
+  build_centos:
+    name: build_centos full node
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+    container:
+      image: docker.io/centos:7
+      volumes:
+        - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 5
+      - uses: actions/cache@v3
+        id: deps_cache
+        with:
+          path: |
+            deps/
+            /usr/local/share/vcpkg/buildtrees
+            /usr/local/share/vcpkg/packages
+            /home/runner/.ccache
+            /Users/runner/.ccache/
+          key: centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          restore-keys: |
+            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
+            centos-notest-all-${{ matrix.os }}-
+      - name: Prepare centos tools
+        run: |
+          rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl-rh/2/3.el7.centos/noarch/centos-release-scl-rh-2-3.el7.centos.noarch.rpm
+          rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl/2/3.el7.centos/noarch/centos-release-scl-2-3.el7.centos.noarch.rpm
+          sed -i s/mirror.centos.org/mirrors.aliyun.com/g /etc/yum.repos.d/*.repo
+          sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+          sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+          cat /etc/yum.repos.d/*.repo
+          yum clean all
+          yum makecache
+          yum update -y
+          yum install -y epel-release centos-release-scl centos-release-scl-rh
+          yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+          yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
+          yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
+          yum install -y git
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2022-07-28
+          override: true
+      - name: Build
+        run: |
+          bash cpp/tools/install_depends.sh -o centos
+          alias cmake='cmake3'
+          . /opt/rh/devtoolset-11/enable 
+          mkdir -p cpp/build
+          cd cpp/build
+          cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+          cmake3 --build . --parallel 3
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+            name: vcpkg-manifest-install.log
+            path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
+      #- name: Test
+      #  run: |
+      #    export OMP_NUM_THREADS=1
+      #    cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ppc-air-node-centos-x64
+          path: ./cpp/build/bin/ppc-air-node
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ppc-pro-node-centos-x64
+          path: ./cpp/build/bin/ppc-pro-node
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ppc-gateway-service-centos-x64
+          path: ./cpp/build/bin/ppc-gateway-service
+      - uses: actions/upload-artifact@v3
+        with:
+          name: libppc-crypto-sdk-jni.so
+          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so
diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml
new file mode 100644
index 00000000..bc4252d1
--- /dev/null
+++ b/.github/workflows/cpp_sdk_workflow.yml
@@ -0,0 +1,158 @@
+name: WeDPR-Component ci for cpp-sdk
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  release:
+    types: [published, push]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
+jobs:
+  build_sdk:
+    name: build sdk
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, windows-2019, macos-12]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 5
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2022-07-28
+          override: true
+      - name: Prepare vcpkg
+        if: runner.os != 'Windows'
+        uses: friendlyanon/setup-vcpkg@v1
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
+      - uses: actions/cache@v2
+        id: deps_cache
+        with:
+          path: |
+            deps/
+            ${{ env.VCPKG_ROOT }}
+            ${{ env.VCPKG_ROOT }}/.git
+            ${{ env.VCPKG_ROOT }}/buildtrees
+            ${{ env.VCPKG_ROOT }}/packages
+            ${{ env.VCPKG_ROOT }}/downloads
+          key: build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          restore-keys: |
+            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-
+            build-sdk-${{ matrix.os }}-
+      - name: Add MSbuild to PATH
+        if: runner.os == 'Windows'
+        uses: microsoft/setup-msbuild@v1.1
+      - name: Build for windows
+        if: runner.os == 'Windows'
+        run: |
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_SDK=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ..
+          MSBuild /version
+          MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+      - name: Build for linux
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt install -y lcov ccache wget libgmp-dev python3-dev
+          export GCC='gcc-10'
+          export CXX='g++-10'
+          bash cpp/tools/install_depends.sh -o ubuntu
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
+      - name: Build for macos
+        if: runner.os == 'macOS'
+        run: |
+          brew install lcov
+          bash cpp/tools/install_depends.sh -o macos
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=OFF -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
+      - name: Test
+        if: runner.os == 'macOS'
+        run: |
+          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
+      - name: Test and upload coverage
+        if: runner.os == 'Linux'
+        run: |
+          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macos'
+        with:
+          name: libppc-crypto-sdk-jni.dylib
+          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'Windows'
+        with:
+          name: libppc-crypto-sdk-jni.dll
+          path: D:\a\WeDPR-Component\cpp\wedpr-component-sdk\bindings\java\src\main\resources\META-INF\native\Release\ppc-crypto-sdk-jni.dll
+
+  build_centos_for_sdk:
+      name: build_centos_for_sdk
+      runs-on: ${{ matrix.os }}
+      continue-on-error: true
+      strategy:
+        fail-fast: false
+        matrix:
+          os: [ubuntu-latest]
+      container:
+        image: docker.io/centos:7
+        volumes:
+          - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+      steps:
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 5
+        - uses: actions/cache@v3
+          id: deps_cache
+          with:
+            path: |
+              deps/
+              /usr/local/share/vcpkg/buildtrees
+              /usr/local/share/vcpkg/packages
+              /home/runner/.ccache
+              /Users/runner/.ccache/
+            key: centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            restore-keys: |
+              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
+              centos-sdk-notest-all-${{ matrix.os }}-
+        - name: Prepare centos tools
+          run: |
+            rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl-rh/2/3.el7.centos/noarch/centos-release-scl-rh-2-3.el7.centos.noarch.rpm
+            rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl/2/3.el7.centos/noarch/centos-release-scl-2-3.el7.centos.noarch.rpm
+            sed -i s/mirror.centos.org/mirrors.aliyun.com/g /etc/yum.repos.d/*.repo
+            sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+            sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+            cat /etc/yum.repos.d/*.repo
+            yum clean all
+            yum makecache
+            yum update -y
+            yum install -y epel-release centos-release-scl centos-release-scl-rh
+            yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+            yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
+            yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
+            yum install -y git
+        - name: Build
+          run: |
+            alias cmake='cmake3'
+            . /opt/rh/devtoolset-11/enable
+            mkdir -p cpp/build
+            cd cpp/build
+            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 --build . --parallel 3
\ No newline at end of file
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
new file mode 100644
index 00000000..fdcdc9a4
--- /dev/null
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -0,0 +1,149 @@
+name: WeDPR-Component ci for cpp-toolkit
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  release:
+    types: [published, push]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
+jobs:
+  build_wedpr_toolkit:
+    name: build wedpr toolkit
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, windows-2019, macos-12]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 5
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly-2022-07-28
+          override: true
+      - name: Prepare vcpkg
+        if: runner.os != 'Windows'
+        uses: friendlyanon/setup-vcpkg@v1
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
+      - uses: actions/cache@v2
+        id: deps_cache
+        with:
+          path: |
+            deps/
+            ${{ env.VCPKG_ROOT }}
+            ${{ env.VCPKG_ROOT }}/.git
+            ${{ env.VCPKG_ROOT }}/buildtrees
+            ${{ env.VCPKG_ROOT }}/packages
+            ${{ env.VCPKG_ROOT }}/downloads
+          key: build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          restore-keys: |
+            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-
+            build-toolkit-${{ matrix.os }}-
+      - name: Add MSbuild to PATH
+        if: runner.os == 'Windows'
+        uses: microsoft/setup-msbuild@v1.1
+      - name: Build for windows
+        if: runner.os == 'Windows'
+        run: |
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+      - name: Build for linux
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt install -y lcov ccache wget libgmp-dev python3-dev
+          export GCC='gcc-10'
+          export CXX='g++-10'
+          bash cpp/tools/install_depends.sh -o ubuntu
+          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
+      - name: Build for macos
+        if: runner.os == 'macOS'
+        run: |
+          brew install lcov
+          bash cpp/tools/install_depends.sh -o macos
+          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macos'
+        with:
+          name: libwedpr_java_transport_jni.dylib
+          path: ./cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native/libwedpr_java_transport_jni.dylib
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'Windows'
+        with:
+          name: libwedpr_java_transport_jni.dll
+          path: D:\a\WeDPR-Component\cpp\wedpr-transport\sdk-wrapper\java\bindings\src\main\resources\META-INF\native\Release\libwedpr_java_transport_jni.dll
+
+
+  build_centos_for_toolkit:
+      name: build_centos_for_toolkit
+      runs-on: ${{ matrix.os }}
+      continue-on-error: true
+      strategy:
+        fail-fast: false
+        matrix:
+          os: [ubuntu-latest]
+      container:
+        image: docker.io/centos:7
+        volumes:
+          - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+      steps:
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 5
+        - uses: actions/cache@v3
+          id: deps_cache
+          with:
+            path: |
+              deps/
+              /usr/local/share/vcpkg/buildtrees
+              /usr/local/share/vcpkg/packages
+              /home/runner/.ccache
+              /Users/runner/.ccache/
+            key: centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            restore-keys: |
+              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
+              centos-toolkit-notest-all-${{ matrix.os }}-
+        - name: Prepare centos tools
+          run: |
+            rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl-rh/2/3.el7.centos/noarch/centos-release-scl-rh-2-3.el7.centos.noarch.rpm
+            rpm -ivh https://cbs.centos.org/kojifiles/packages/centos-release-scl/2/3.el7.centos/noarch/centos-release-scl-2-3.el7.centos.noarch.rpm
+            sed -i s/mirror.centos.org/mirrors.aliyun.com/g /etc/yum.repos.d/*.repo
+            sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+            sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
+            cat /etc/yum.repos.d/*.repo
+            yum clean all
+            yum makecache
+            yum update -y
+            yum install -y epel-release centos-release-scl centos-release-scl-rh
+            yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+            yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
+            yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
+            yum install -y git
+        - name: Build
+          run: |
+            alias cmake='cmake3'
+            . /opt/rh/devtoolset-11/enable
+            mkdir -p cpp/build
+            cd cpp/build
+            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 --build . --parallel 3
\ No newline at end of file
diff --git a/.github/workflows/cpp_workflow.yml b/.github/workflows/cpp_workflow.yml
deleted file mode 100644
index 2e50590e..00000000
--- a/.github/workflows/cpp_workflow.yml
+++ /dev/null
@@ -1,227 +0,0 @@
-name: WeDPR-Component ci(cpp)
-on:
-  push:
-    paths-ignore:
-      - "docs/**"
-      - "Changelog.md"
-      - "README.md"
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "Changelog.md"
-      - "README.md"
-  release:
-    types: [published, push]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  RUST_BACKTRACE: 1
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
-
-jobs:
-  build:
-    name: build all
-    runs-on: ${{ matrix.os }}
-    continue-on-error: true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, ubuntu-20.04]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 5
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2022-07-28
-          override: true
-      - name: Prepare vcpkg
-        if: runner.os != 'Windows'
-        uses: friendlyanon/setup-vcpkg@v1
-        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
-      - uses: actions/cache@v2
-        id: deps_cache
-        with:
-          path: |
-            deps/
-            c:/vcpkg
-            !c:/vcpkg/.git
-            !c:/vcpkg/buildtrees
-            !c:/vcpkg/packages
-            !c:/vcpkg/downloads
-          key: build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-          restore-keys: |
-            build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-            build-${{ matrix.os }}-${{ github.base_ref }}-
-            build-${{ matrix.os }}-
-      - name: Build for linux
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt install -y lcov ccache wget libgmp-dev python3-dev
-          export GCC='gcc-10'
-          export CXX='g++-10'
-          bash cpp/tools/install_depends.sh -o ubuntu
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
-          make -j3
-      - name: Build for macos
-        if: runner.os == 'macOS'
-        run: |
-          bash cpp/tools/install_depends.sh -o macos
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
-          make -j3
-      - name: Test
-        if: runner.os != 'Windows'
-        run: |
-          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest
-
-  build_sdk:
-    name: build sdk
-    runs-on: ${{ matrix.os }}
-    continue-on-error: true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-22.04, windows-2019, macos-12]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 5
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2022-07-28
-          override: true
-      - name: Prepare vcpkg
-        if: runner.os != 'Windows'
-        uses: friendlyanon/setup-vcpkg@v1
-        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
-      - uses: actions/cache@v2
-        id: deps_cache
-        with:
-          path: |
-            deps/
-            c:/vcpkg
-            !c:/vcpkg/.git
-            !c:/vcpkg/buildtrees
-            !c:/vcpkg/packages
-            !c:/vcpkg/downloads
-          key: build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-          restore-keys: |
-            build-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-            build-${{ matrix.os }}-${{ github.base_ref }}-
-            build-${{ matrix.os }}-
-      - name: Build for windows
-        if: runner.os == 'Windows'
-        run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_SDK=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-          cmake --build . --parallel 3
-      - name: Build for linux
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt install -y lcov ccache wget libgmp-dev python3-dev
-          export GCC='gcc-10'
-          export CXX='g++-10'
-          bash cpp/tools/install_depends.sh -o ubuntu
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
-          make -j3
-      - name: Build for macos
-        if: runner.os == 'macOS'
-        run: |
-          brew install lcov
-          bash cpp/tools/install_depends.sh -o macos
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=OFF -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
-          make -j3
-      - name: Test
-        if: runner.os == 'macOS'
-        run: |
-          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
-      - name: Test and upload coverage
-        if: runner.os == 'Linux'
-        run: |
-          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
-      - uses: actions/upload-artifact@v4
-        if: runner.os == 'macos'
-        with:
-          name: libppc-crypto-sdk-jni.dylib
-          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.dylib
-      - uses: actions/upload-artifact@v4
-        if: runner.os == 'Windows'
-        with:
-          name: libppc-crypto-sdk-jni.dylib
-          path: D:\a\WeDPR-Component\cpp\wedpr-component-sdk\bindings\java\src\main\resources\META-INF\native\Release\ppc-crypto-sdk-jni.dll
-
-  build_centos:
-    name: build_centos full node
-    runs-on: ${{ matrix.os }}
-    continue-on-error: true
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04]
-    container: docker.io/centos:7
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 5
-      - uses: actions/cache@v2
-        id: deps_cache
-        with:
-          path: |
-            /home/runner/.ccache
-            /Users/runner/.ccache/
-            deps/
-          key: centos-notest-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-          restore-keys: |
-            centos-notest-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
-            centos-notest-${{ matrix.os }}-${{ github.base_ref }}-
-            centos-notest-${{ matrix.os }}-
-      - name: Prepare centos tools
-        run: |
-          sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
-          sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
-          sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-          yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
-          yum install -y devtoolset-10 devtoolset-11 llvm-toolset-7 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
-          yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y git
-      - name: Prepare vcpkg
-        if: runner.os != 'Windows'
-        uses: friendlyanon/setup-vcpkg@v1
-        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2022-07-28
-          override: true
-      - name: Build
-        run: |
-          alias cmake='cmake3'
-          . /opt/rh/devtoolset-10/enable
-          . /opt/rh/rh-perl530/enable
-          export LIBCLANG_PATH=/opt/rh/llvm-toolset-7/root/lib64/
-          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
-          . /opt/rh/llvm-toolset-7/enable
-          mkdir -p cpp/build
-          cd cpp/build
-          cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
-          cmake3 --build . --parallel 3
-      - name: Test
-        run: |
-          export OMP_NUM_THREADS=1
-          cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ppc-air-node-centos-x64
-          path: ./cpp/build/bin/ppc-air-node
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ppc-pro-node-centos-x64
-          path: ./cpp/build/bin/ppc-pro-node
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ppc-gateway-service-centos-x64
-          path: ./cpp/build/bin/ppc-gateway-service
-      - uses: actions/upload-artifact@v4
-        with:
-          name: libppc-crypto-sdk-jni.so
-          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 74642c67..ca080810 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -102,6 +102,7 @@ elseif(BUILD_SDK)
     add_sources("${SDK_SOURCE_LIST}")
 elseif(BUILD_WEDPR_TOOLKIT)
     add_sources("${TRANSPORT_SDK_TOOLKIT_SOURCE_LIST}")
+    include(swig)
 endif()
 ########### set the sources end ########### 
 
diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 8a068996..9596d8ff 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -138,14 +138,15 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
     endif()
 
     add_definitions(-DUSE_STD_RANGES)
-    add_compile_options(/std:c++latest)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17")
+    add_compile_definitions(NOMINMAX)
     add_compile_options(-bigobj)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
-    # set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
-    # set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
-    # set(CMAKE_CXX_FLAGS_RELEASE "/MT")
-    # set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
-    link_libraries(ws2_32 Crypt32 userenv)
+     # MSVC only support static build
+     set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
+     set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
+     set(CMAKE_CXX_FLAGS_RELEASE "/MT")
+     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
+     link_libraries(ws2_32 Crypt32 userenv)
 else ()
     message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.")
 endif ()
diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake
index d8e1bf62..3e9f1854 100644
--- a/cpp/cmake/Dependencies.cmake
+++ b/cpp/cmake/Dependencies.cmake
@@ -5,9 +5,7 @@
 #endif()
 
 ######## common dependencies ######## 
-find_package(OpenSSL REQUIRED)
 include(InstallBcosUtilities)
-
 if (TESTS)
     find_package(Boost COMPONENTS unit_test_framework)
 endif()
@@ -16,19 +14,23 @@ endif()
 if(ENABLE_CPU_FEATURES)
     find_package(CpuFeatures REQUIRED)
 endif()
-find_package(unofficial-sodium CONFIG REQUIRED)
 ######## common dependencies end ########
 
-
-##### the full-dependencies #####
+##### the wedpr_toolkit #####
 if(BUILD_ALL OR BUILD_WEDPR_TOOLKIT)
-    find_package(jsoncpp REQUIRED)
     find_package(TBB REQUIRED)
     find_package(gRPC REQUIRED)
-    find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED)
+    find_package(jsoncpp REQUIRED)
+endif()
+##### the sdk dependencies #####
+if(BUILD_ALL OR BUILD_SDK OR BUILD_UDF)
+    find_package(OpenSSL REQUIRED)
+    find_package(unofficial-sodium CONFIG REQUIRED)
 endif()
 
+##### the full-dependencies #####
 if(BUILD_ALL)
+    find_package(${BCOS_BOOSTSSL_TARGET} REQUIRED)
     # tcmalloc
     include(ProjectTCMalloc)
 
diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake
index cdc6eace..212ce9ca 100644
--- a/cpp/cmake/Options.cmake
+++ b/cpp/cmake/Options.cmake
@@ -66,6 +66,10 @@ macro(configure_project)
     endif ()
 
     set(VISIBILITY_FLAG " -fvisibility=hidden -fvisibility-inlines-hidden")
+    set(PROGRAM_POSTFIX "")
+    if(CMAKE_HOST_SYSTEM_NAME MATCHES "Windows")
+        set(PROGRAM_POSTFIX ".exe")
+    endif()
 
     set(MARCH_TYPE "-march=x86-64 -mtune=generic ${VISIBILITY_FLAG}")
     if (ARCH_NATIVE)
@@ -126,10 +130,12 @@ macro(configure_project)
     if (BUILD_UDF)
         set(VISIBILITY_FLAG "")
         set(BUILD_ALL OFF)
+        list(APPEND VCPKG_MANIFEST_FEATURES "sdk")
     endif()
     if (BUILD_SDK)
         set(VISIBILITY_FLAG "")
         set(BUILD_ALL OFF)
+        list(APPEND VCPKG_MANIFEST_FEATURES "sdk")
     endif()
     if (BUILD_WEDPR_TOOLKIT)
         set(VISIBILITY_FLAG "")
@@ -138,6 +144,7 @@ macro(configure_project)
     if (BUILD_ALL)
         # install all dependencies
         list(APPEND VCPKG_MANIFEST_FEATURES "all")
+        list(APPEND VCPKG_MANIFEST_FEATURES "sdk")
         if(ENABLE_SSE)
             # enable sse for libhdfs3
             list(APPEND VCPKG_MANIFEST_FEATURES "sse-libhdfs3")
diff --git a/cpp/cmake/grpc.cmake b/cpp/cmake/grpc.cmake
new file mode 100644
index 00000000..b88d316f
--- /dev/null
+++ b/cpp/cmake/grpc.cmake
@@ -0,0 +1,47 @@
+if(NOT GRPC_CPP_PLUGIN AND TARGET gRPC::grpc_cpp_plugin)
+  get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin
+    IMPORTED_LOCATION_RELEASE)
+  if(NOT EXISTS "${GRPC_CPP_PLUGIN}")
+    get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin
+      IMPORTED_LOCATION_RELWITHDEBINFO)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_PLUGIN}")
+    get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin
+      IMPORTED_LOCATION_MINSIZEREL)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_PLUGIN}")
+    get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin
+      IMPORTED_LOCATION_DEBUG)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_PLUGIN}")
+    get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin
+      IMPORTED_LOCATION_NOCONFIG)
+  endif()
+endif()
+
+
+if(NOT GRPC_CPP_REFLECTION AND TARGET gRPC::grpc++_reflection)
+  get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
+    IMPORTED_LOCATION_RELEASE)
+  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
+    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
+      IMPORTED_LOCATION_RELWITHDEBINFO)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
+    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
+      IMPORTED_LOCATION_MINSIZEREL)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
+    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
+      IMPORTED_LOCATION_DEBUG)
+  endif()
+  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
+    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
+      IMPORTED_LOCATION_NOCONFIG)
+  endif()
+endif()
+
+set(PROTOC_BINARY ${Protobuf_PROTOC_EXECUTABLE})
+message("# PROTOC_BINARY: ${Protobuf_PROTOC_EXECUTABLE}")
+message("# GRPC_CPP_PLUGIN: ${GRPC_CPP_PLUGIN}")
+message("# GRPC_CPP_REFLECTION: ${GRPC_CPP_REFLECTION}")
\ No newline at end of file
diff --git a/cpp/cmake/python.cmake b/cpp/cmake/python.cmake
index e941adfb..2fd2b776 100644
--- a/cpp/cmake/python.cmake
+++ b/cpp/cmake/python.cmake
@@ -1,23 +1,6 @@
 if(NOT BUILD_PYTHON)
   return()
 endif()
-
-# Use latest UseSWIG module (3.14) and Python3 module (3.18)
-cmake_minimum_required(VERSION 3.18)
-
-# Will need swig
-set(CMAKE_SWIG_FLAGS)
-find_package(SWIG REQUIRED)
-include(UseSWIG)
-
-if(${SWIG_VERSION} VERSION_GREATER_EQUAL 4)
-  list(APPEND CMAKE_SWIG_FLAGS "-doxygen")
-endif()
-
-if(UNIX AND NOT APPLE)
-  list(APPEND CMAKE_SWIG_FLAGS "-DSWIGWORDSIZE64")
-endif()
-
 # Find Python 3
 find_package(Python REQUIRED COMPONENTS Interpreter Development)
 message("Python_FOUND:${Python_FOUND}")
diff --git a/cpp/cmake/java.cmake b/cpp/cmake/swig.cmake
similarity index 68%
rename from cpp/cmake/java.cmake
rename to cpp/cmake/swig.cmake
index 53e15d19..d1e39a2f 100644
--- a/cpp/cmake/java.cmake
+++ b/cpp/cmake/swig.cmake
@@ -1,7 +1,10 @@
-if(NOT BUILD_JAVA)
+if(NOT AUTO_GENERATE)
   return()
 endif()
 
+# Use latest UseSWIG module (3.14) and Python3 module (3.18)
+cmake_minimum_required(VERSION 3.18)
+
 # Will need swig
 set(CMAKE_SWIG_FLAGS)
 find_package(SWIG REQUIRED)
@@ -13,4 +16,4 @@ endif()
 
 if(UNIX AND NOT APPLE)
   list(APPEND CMAKE_SWIG_FLAGS "-DSWIGWORDSIZE64")
-endif()
\ No newline at end of file
+endif()
diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h
index 55a73ddd..c3b1a470 100644
--- a/cpp/ppc-framework/protocol/Message.h
+++ b/cpp/ppc-framework/protocol/Message.h
@@ -22,7 +22,6 @@
 #include "RouteType.h"
 #include "ppc-framework/Helper.h"
 #include "ppc-framework/libwrapper/Buffer.h"
-#include 
 #include 
 #include 
 #include 
@@ -183,35 +182,28 @@ class MessageHeader
     uint16_t mutable m_length;
 };
 
-class Message : virtual public bcos::boostssl::MessageFace
+class Message
 {
 public:
     using Ptr = std::shared_ptr;
     Message() = default;
-    ~Message() override {}
+    virtual ~Message() {}
 
     virtual MessageHeader::Ptr header() const { return m_header; }
     virtual void setHeader(MessageHeader::Ptr header) { m_header = std::move(header); }
-
     /// the overloaed implementation ===
-    uint16_t version() const override { return m_header->version(); }
-    void setVersion(uint16_t version) override { m_header->setVersion(version); }
-    uint16_t packetType() const override { return m_header->packetType(); }
-    void setPacketType(uint16_t packetType) override { m_header->setPacketType(packetType); }
-    std::string const& seq() const override { return m_header->traceID(); }
-    void setSeq(std::string traceID) override { m_header->setTraceID(traceID); }
-    uint16_t ext() const override { return m_header->ext(); }
-    void setExt(uint16_t ext) override { m_header->setExt(ext); }
-
-    bool isRespPacket() const override { return m_header->isRespPacket(); }
-    void setRespPacket() override { m_header->setRespPacket(); }
-
-    virtual uint32_t length() const override
-    {
-        return m_header->length() + (m_payload ? m_payload->size() : 0);
-    }
-
-    std::shared_ptr payload() const override { return m_payload; }
+    uint16_t version() const { return m_header->version(); }
+    void setVersion(uint16_t version) { m_header->setVersion(version); }
+    uint16_t packetType() const { return m_header->packetType(); }
+    void setPacketType(uint16_t packetType) { m_header->setPacketType(packetType); }
+    std::string const& seq() const { return m_header->traceID(); }
+    void setSeq(std::string traceID) { m_header->setTraceID(traceID); }
+    uint16_t ext() const { return m_header->ext(); }
+    void setExt(uint16_t ext) { m_header->setExt(ext); }
+
+    bool isRespPacket() const { return m_header->isRespPacket(); }
+    void setRespPacket() { m_header->setRespPacket(); }
+    void setPayload(std::shared_ptr _payload) { m_payload = std::move(_payload); }
     // for swig wrapper
     OutputBuffer payloadBuffer() const
     {
@@ -222,11 +214,6 @@ class Message : virtual public bcos::boostssl::MessageFace
         return OutputBuffer{(unsigned char*)m_payload->data(), m_payload->size()};
     }
 
-    void setPayload(std::shared_ptr _payload) override
-    {
-        m_payload = std::move(_payload);
-    }
-
     void setFrontMessage(MessagePayload::Ptr frontMessage)
     {
         m_frontMessage = std::move(frontMessage);
@@ -236,10 +223,15 @@ class Message : virtual public bcos::boostssl::MessageFace
 
     // Note: swig wrapper require define all methods
     virtual bool encode(bcos::bytes& _buffer) = 0;
-    // encode and return the {header, payload}
-    virtual bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) = 0;
     virtual int64_t decode(bcos::bytesConstRef _buffer) = 0;
 
+    virtual uint32_t length() const
+    {
+        return m_header->length() + (m_payload ? m_payload->size() : 0);
+    }
+
+    virtual std::shared_ptr payload() const { return m_payload; }
+
 protected:
     MessageHeader::Ptr m_header;
     // Note: allocate here in case of wsService nullptr access caused coredump
@@ -259,12 +251,12 @@ class MessageHeaderBuilder
     virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader) = 0;
 };
 
-class MessageBuilder : public bcos::boostssl::MessageFaceFactory
+class MessageBuilder
 {
 public:
     using Ptr = std::shared_ptr;
     MessageBuilder() = default;
-    ~MessageBuilder() override = default;
+    virtual ~MessageBuilder() = default;
 
     virtual Message::Ptr build() = 0;
     virtual Message::Ptr build(bcos::bytesConstRef buffer) = 0;
@@ -320,20 +312,6 @@ inline std::string printMessage(Message::Ptr const& _msg)
     }
     return stringstream.str();
 }
-
-inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg)
-{
-    if (!_msg)
-    {
-        return "nullptr";
-    }
-    std::ostringstream stringstream;
-    stringstream << LOG_KV("rsp", _msg->isRespPacket()) << LOG_KV("traceID", _msg->seq())
-                 << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length())
-                 << LOG_KV("ext", _msg->ext());
-    return stringstream.str();
-}
-
 // function to send response
 using SendResponseFunction = std::function&& payload)>;
 using ReceiveMsgFunc = std::function;
diff --git a/cpp/ppc-framework/protocol/P2PMessage.h b/cpp/ppc-framework/protocol/P2PMessage.h
new file mode 100644
index 00000000..735668d6
--- /dev/null
+++ b/cpp/ppc-framework/protocol/P2PMessage.h
@@ -0,0 +1,98 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Message.h
+ * @author: yujiechen
+ * @date 2024-08-22
+ */
+#pragma once
+#include "Message.h"
+#include 
+namespace ppc::protocol
+{
+// the wrapper for override bcos::boostssl::MessageFace while use implementation for Message
+class P2PMessage : virtual public bcos::boostssl::MessageFace
+{
+public:
+    using Ptr = std::shared_ptr;
+    P2PMessage(Message::Ptr msg) : m_msg(std::move(msg)) {}
+    ~P2PMessage() override {}
+
+    /// the overloaed implementation ===
+    uint16_t version() const override { return m_msg->version(); }
+    void setVersion(uint16_t version) override { m_msg->setVersion(version); }
+    uint16_t packetType() const override { return m_msg->packetType(); }
+    void setPacketType(uint16_t packetType) override { m_msg->setPacketType(packetType); }
+    std::string const& seq() const override { return m_msg->seq(); }
+    void setSeq(std::string seq) override { m_msg->setSeq(seq); }
+    uint16_t ext() const override { return m_msg->ext(); }
+    void setExt(uint16_t ext) override { m_msg->setExt(ext); }
+    std::shared_ptr payload() const override { return m_msg->payload(); }
+    void setPayload(std::shared_ptr payload) override
+    {
+        m_msg->setPayload(std::move(payload));
+    }
+
+    bool encode(bcos::bytes& _buffer) override { return m_msg->encode(_buffer); }
+
+    int64_t decode(bcos::bytesConstRef _buffer) override { return m_msg->decode(_buffer); }
+
+    bool isRespPacket() const override { return m_msg->isRespPacket(); }
+    void setRespPacket() override { m_msg->setRespPacket(); }
+    uint32_t length() const override { return m_msg->length(); }
+
+    // encode and return the {header, payload}
+    virtual bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) = 0;
+
+    Message::Ptr const& msg() { return m_msg; }
+    MessageHeader::Ptr header() const { return m_msg->header(); }
+
+protected:
+    Message::Ptr m_msg;
+};
+
+
+class P2PMessageBuilder : virtual public bcos::boostssl::MessageFaceFactory
+{
+public:
+    using Ptr = std::shared_ptr;
+    P2PMessageBuilder() = default;
+    ~P2PMessageBuilder() override = default;
+
+    virtual P2PMessage::Ptr build() = 0;
+    virtual P2PMessage::Ptr build(ppc::protocol::RouteType routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) = 0;
+};
+inline std::string printWsMessage(bcos::boostssl::MessageFace::Ptr const& _msg)
+{
+    if (!_msg)
+    {
+        return "nullptr";
+    }
+    std::ostringstream stringstream;
+    stringstream << LOG_KV("rsp", _msg->isRespPacket()) << LOG_KV("traceID", _msg->seq())
+                 << LOG_KV("packetType", _msg->packetType()) << LOG_KV("length", _msg->length())
+                 << LOG_KV("ext", _msg->ext());
+    return stringstream.str();
+}
+inline std::string printP2PMessage(P2PMessage::Ptr const& _msg)
+{
+    if (_msg->msg() == nullptr)
+    {
+        return "";
+    }
+    return printMessage(_msg->msg());
+}
+}  // namespace ppc::protocol
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index b3e3f555..108a6aae 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -32,9 +32,13 @@ install_gsasl_depend()
     if [[ "${os_type}" == "centos" ]];then
         cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
     fi
+    # macos
+    if [[ "${os_type}" == "macos" ]];then
+        cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
+    fi
     # ubuntu
     if [[ "${os_type}" == "ubuntu" ]];then
-        cd libgsasl-1.8.0 && ./configure --with-pic --disable-ntlm --disable-gs2 --disable-gssapi --without-stringprep && make -j4 && make install
+        cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
     fi
     LOG_INFO "download and install gsasl success..."
 }
@@ -106,7 +110,8 @@ install_ubuntu_depends()
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
-    brew install clang cmake autoconf gsasl nasm lcov
+    brew install autoconf nasm lcov
+    install_gsasl_depend "macos"
     LOG_INFO "install depends for macos success ..."
 }
 
diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json
index 81446362..ade9fdc7 100644
--- a/cpp/vcpkg-configuration.json
+++ b/cpp/vcpkg-configuration.json
@@ -3,7 +3,7 @@
     {
       "kind": "git",
       "repository": "https://github.com/FISCO-BCOS/registry",
-      "baseline": "b483a0287c6ee7017983942af540e3ee672b0994",
+      "baseline": "489334d1f374134163a3d9719c3345fdd1d268d3",
       "packages": [
         "openssl",
         "bcos-utilities",
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index 7ff9a24e..d605e402 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -12,19 +12,11 @@
       "name": "vcpkg-cmake-config",
       "host": true
     },
-    {
-      "name": "openssl",
-      "version>=": "1.1.1-tassl"
-    },
-    "libsodium",
-    {
-      "name": "libsodium",
-      "version>=": "1.0.18"
-    },
     {
       "name": "bcos-utilities",
       "version>=": "1.0.0"
-    }
+    },
+    "zlib"
   ],
   "builtin-baseline": "51b14cd4e1230dd51c11ffeff6f7d53c61cc5297",
   "overrides": [
@@ -55,6 +47,10 @@
     { 
       "name": "grpc",
       "version": "1.51.1#1"
+    },
+    {
+      "name": "jsoncpp",
+      "version": "1.9.5"
     }
   ],
   "features": {
@@ -115,25 +111,39 @@
           "name": "grpc",
           "version>=": "1.51.1"
         },
+        {
+          "name": "bcos-boostssl",
+          "version>=": "3.2.3"
+        },
         "tarscpp",
         "tbb",
         "libxml2"
       ]
     },
-    "toolkit": {
-      "description": "tooklit dependencies",
+    "sdk": {
+      "description": "sdk dependencies",
       "dependencies": [
         {
-          "name": "bcos-boostssl",
-          "version>=": "3.2.3"
+          "name": "openssl",
+          "version>=": "1.1.1-tassl"
         },
+        "libsodium",
+        {
+          "name": "libsodium",
+          "version>=": "1.0.18"
+        }
+      ]
+    },
+    "toolkit": {
+      "description": "tooklit dependencies",
+      "dependencies": [
         {
           "name": "grpc",
           "version>=": "1.51.1"
         },
         "tarscpp",
-        "tbb",
-        "libxml2"
+        "jsoncpp",
+        "tbb"
       ]
     }
   }
diff --git a/cpp/wedpr-computing/ppc-mpc/CMakeLists.txt b/cpp/wedpr-computing/ppc-mpc/CMakeLists.txt
index 53720a81..be4f918f 100644
--- a/cpp/wedpr-computing/ppc-mpc/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-mpc/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-mpc VERSION ${VERSION})
 add_subdirectory(src)
 
 #if (TESTS)
diff --git a/cpp/wedpr-computing/ppc-pir/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/CMakeLists.txt
index bf3b53b3..0633d26f 100644
--- a/cpp/wedpr-computing/ppc-pir/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-pir VERSION ${VERSION})
 add_subdirectory(src)
 
 if (TESTS)
diff --git a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
index 6a969962..8af1b5f9 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
@@ -9,4 +9,4 @@ target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
 
 # target_link_libraries(${TEST_BINARY_NAME} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
 target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
-add_test(NAME test-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
+add_test(NAME test-pir WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv b/cpp/wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv
new file mode 100644
index 00000000..590b61f2
--- /dev/null
+++ b/cpp/wedpr-computing/ppc-pir/tests/data/AysPreDataset.csv
@@ -0,0 +1,3 @@
+id,x1
+1,test
+2,test2
diff --git a/cpp/wedpr-computing/ppc-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/CMakeLists.txt
index 06d0e477..8e5aa738 100644
--- a/cpp/wedpr-computing/ppc-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-psi VERSION ${VERSION})
 add_subdirectory(src/psi-framework)
 add_subdirectory(src/labeled-psi)
 add_subdirectory(src/ra2018-psi)
diff --git a/cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt
index 648242e1..2c156e1b 100644
--- a/cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto-core/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-crypto-core VERSION ${VERSION})
-
 add_subdirectory(src)
 if (TESTS)
     enable_testing()
diff --git a/cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt
index 3387e7a2..2a56797c 100644
--- a/cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto-core/src/CMakeLists.txt
@@ -5,5 +5,4 @@ find_package(OpenSSL REQUIRED)
 message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
 message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
-target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC unofficial-sodium::sodium 
-    unofficial-sodium::sodium_config_public ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB})
\ No newline at end of file
+target_link_libraries(${CRYPTO_CORE_TARGET} PUBLIC unofficial-sodium::sodium ${BCOS_UTILITIES_TARGET} OpenSSL::Crypto ${CPU_FEATURES_LIB})
\ No newline at end of file
diff --git a/cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt
index c17b68b1..9f27bc5f 100644
--- a/cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-crypto VERSION ${VERSION})
-
 add_subdirectory(src)
 if(DEMO)
     add_subdirectory(demo)
diff --git a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
index 954f1ca6..11a8de42 100644
--- a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
@@ -6,7 +6,7 @@ find_package(OpenSSL REQUIRED)
 message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
 message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
-target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium unofficial-sodium::sodium_config_public TBB::tbb ${CPU_FEATURES_LIB})
+target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB})
 
 if (ENABLE_IPP_CRYPTO)
     find_package(ipp-crypto REQUIRED)
diff --git a/cpp/wedpr-crypto/ppc-homo/CMakeLists.txt b/cpp/wedpr-crypto/ppc-homo/CMakeLists.txt
index fda32930..06f891dc 100644
--- a/cpp/wedpr-crypto/ppc-homo/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-homo/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-homo VERSION ${VERSION})
-
 add_subdirectory(codec)
 add_subdirectory(paillier)
 add_subdirectory(fahe)
diff --git a/cpp/wedpr-crypto/ppc-udf/CMakeLists.txt b/cpp/wedpr-crypto/ppc-udf/CMakeLists.txt
index e6fe5b4f..71e339b1 100644
--- a/cpp/wedpr-crypto/ppc-udf/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-udf/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-udf VERSION ${VERSION})
-
 # export windows dll symbol
 if(WIN32)
     message(STATUS "Compile on Windows")
diff --git a/cpp/wedpr-crypto/sdk/CMakeLists.txt b/cpp/wedpr-crypto/sdk/CMakeLists.txt
index fae969e6..5fb633c2 100644
--- a/cpp/wedpr-crypto/sdk/CMakeLists.txt
+++ b/cpp/wedpr-crypto/sdk/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(wedpr-component-sdk VERSION ${VERSION})
-
 # export windows dll symbol
 if(WIN32)
     message(STATUS "Compile on Windows")
diff --git a/cpp/wedpr-helper/ppc-tools/CMakeLists.txt b/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
index aa4ae9b6..26cb4e2f 100644
--- a/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
+++ b/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-tools VERSION ${VERSION})
-
 aux_source_directory(src/cuckoo SRCS)
 aux_source_directory(src/config SRCS)
 
diff --git a/cpp/wedpr-main/CMakeLists.txt b/cpp/wedpr-main/CMakeLists.txt
index 44e9a218..95d0d4b6 100644
--- a/cpp/wedpr-main/CMakeLists.txt
+++ b/cpp/wedpr-main/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-main VERSION ${VERSION})
-
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 add_subdirectory(air-node)
 add_subdirectory(pro-node)
diff --git a/cpp/wedpr-main/gateway/CMakeLists.txt b/cpp/wedpr-main/gateway/CMakeLists.txt
index 062d9c66..08a62d76 100644
--- a/cpp/wedpr-main/gateway/CMakeLists.txt
+++ b/cpp/wedpr-main/gateway/CMakeLists.txt
@@ -1,7 +1,3 @@
-cmake_minimum_required(VERSION 3.14)
-
-project(wedpr-gateway)
-
 include_directories(${CMAKE_SOURCE_DIR})
 
 
diff --git a/cpp/wedpr-main/pro-node/CMakeLists.txt b/cpp/wedpr-main/pro-node/CMakeLists.txt
index 40664ebc..f83f6c10 100644
--- a/cpp/wedpr-main/pro-node/CMakeLists.txt
+++ b/cpp/wedpr-main/pro-node/CMakeLists.txt
@@ -1,7 +1,3 @@
-cmake_minimum_required(VERSION 3.14)
-
-project(pro-psi-node)
-
 include_directories(${CMAKE_SOURCE_DIR})
 
 aux_source_directory(./ SRC_LIST)
diff --git a/cpp/wedpr-protocol/grpc/CMakeLists.txt b/cpp/wedpr-protocol/grpc/CMakeLists.txt
index a45b773d..fbd1fbc6 100644
--- a/cpp/wedpr-protocol/grpc/CMakeLists.txt
+++ b/cpp/wedpr-protocol/grpc/CMakeLists.txt
@@ -4,11 +4,7 @@ set(PROTO_INPUT_PATH ${CMAKE_SOURCE_DIR}/wedpr-protocol/proto/pb)
 file(GLOB_RECURSE MESSAGES_PROTOS "${PROTO_INPUT_PATH}/Service*.proto")
 
 # create PROTO_OUTPUT_PATH
-file(MAKE_DIRECTORY ${PROTO_OUTPUT_PATH})
-
-find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin REQUIRED)
-find_program(PROTOC_BINARY protoc REQUIRED)
-
+include(grpc)
 foreach(proto_file ${MESSAGES_PROTOS})
     get_filename_component(basename ${proto_file} NAME_WE)
     set(generated_file ${PROTO_OUTPUT_PATH}/${basename}.grpc.pb.cc)
@@ -27,7 +23,7 @@ foreach(proto_file ${MESSAGES_PROTOS})
 endforeach()
 
 add_library(${SERVICE_PB_TARGET} ${GRPC_MESSAGES_SRCS})
-target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure gRPC::grpc++_reflection)
+target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure ${GRPC_CPP_REFLECTION})
 
 add_subdirectory(client)
 add_subdirectory(server)
diff --git a/cpp/wedpr-protocol/protobuf/CMakeLists.txt b/cpp/wedpr-protocol/protobuf/CMakeLists.txt
index 615270ec..8ab6303d 100644
--- a/cpp/wedpr-protocol/protobuf/CMakeLists.txt
+++ b/cpp/wedpr-protocol/protobuf/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-protocol VERSION ${VERSION})
-
 add_subdirectory(src)
 if (TESTS)
     enable_testing()
diff --git a/cpp/wedpr-protocol/protocol/CMakeLists.txt b/cpp/wedpr-protocol/protocol/CMakeLists.txt
index cda79d6d..8ab6303d 100644
--- a/cpp/wedpr-protocol/protocol/CMakeLists.txt
+++ b/cpp/wedpr-protocol/protocol/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-protobuf VERSION ${VERSION})
-
 add_subdirectory(src)
 if (TESTS)
     enable_testing()
diff --git a/cpp/wedpr-protocol/protocol/src/CMakeLists.txt b/cpp/wedpr-protocol/protocol/src/CMakeLists.txt
index 095eeec4..60a80615 100644
--- a/cpp/wedpr-protocol/protocol/src/CMakeLists.txt
+++ b/cpp/wedpr-protocol/protocol/src/CMakeLists.txt
@@ -1,4 +1,4 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${PROTOCOL_TARGET} ${SRCS})
 
-target_link_libraries(${PROTOCOL_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${CPU_FEATURES_LIB} jsoncpp_static)
+target_link_libraries(${PROTOCOL_TARGET} PUBLIC ${CPU_FEATURES_LIB} jsoncpp_static)
diff --git a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
index f71e47ce..6e7af353 100644
--- a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
@@ -211,7 +211,7 @@ void JsonTaskImpl::decodeDataResourceDesc(DataResourceDesc::Ptr _desc, Json::Val
                 BOOST_THROW_EXCEPTION(
                     InvalidParameter() << errinfo_comment("The \"..\" cannot be in the path"));
             }
-            if (path.starts_with("/"))
+            if (path.rfind("/", 0) == 0)
             {
                 _desc->setPath(path);
             }
diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp
index 9b156941..637209e4 100644
--- a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.cpp
@@ -36,24 +36,6 @@ bool MessageImpl::encode(bcos::bytes& _buffer)
     return true;
 }
 
-bool MessageImpl::encode(bcos::boostssl::EncodedMsg& encodedMsg)
-{
-    try
-    {
-        // header
-        m_header->encode(encodedMsg.header);
-        // assign the payload back
-        encodedMsg.payload = m_payload;
-        return true;
-    }
-    catch (std::exception const& e)
-    {
-        PROTOCOL_LOG(WARNING) << LOG_DESC("encode message failed")
-                              << LOG_KV("error", boost::diagnostic_information(e));
-        return false;
-    }
-}
-
 int64_t MessageImpl::decode(bytesConstRef buffer)
 {
     if (buffer.size() > m_maxMessageLen)
diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h
index 8fb053fe..c16c040a 100644
--- a/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h
+++ b/cpp/wedpr-protocol/protocol/src/v1/MessageImpl.h
@@ -20,7 +20,6 @@
 #pragma once
 #include "ppc-framework/Common.h"
 #include "ppc-framework/protocol/Message.h"
-#include "ppc-utilities/Utilities.h"
 #include 
 #include 
 #include 
@@ -46,11 +45,9 @@ class MessageImpl : public Message
     ~MessageImpl() override = default;
 
     bool encode(bcos::bytes& _buffer) override;
-    // encode and return the {header, payload}
-    bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) override;
     int64_t decode(bcos::bytesConstRef _buffer) override;
 
-private:
+protected:
     MessageHeaderBuilder::Ptr m_headerBuilder;
 
     // default max message length is 100MB
@@ -82,6 +79,11 @@ class MessageBuilderImpl : public MessageBuilder
         return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen, buffer);
     }
 
+    virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader)
+    {
+        return m_msgHeaderBuilder->build(optionalHeader);
+    }
+
     Message::Ptr build(ppc::protocol::RouteType routeType,
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override
     {
@@ -92,20 +94,9 @@ class MessageBuilderImpl : public MessageBuilder
         return msg;
     }
 
-    bcos::boostssl::MessageFace::Ptr buildMessage() override
-    {
-        return std::make_shared(m_msgHeaderBuilder, m_maxMessageLen);
-    }
-
-    virtual MessageOptionalHeader::Ptr build(MessageOptionalHeader::Ptr const& optionalHeader)
-    {
-        return m_msgHeaderBuilder->build(optionalHeader);
-    }
-    std::string newSeq() override { return generateUUID(); }
-
-private:
+protected:
     MessageHeaderBuilder::Ptr m_msgHeaderBuilder;
     // default max message length is 100MB
     size_t m_maxMessageLen = 100 * 1024 * 1024;
 };
-}  // namespace ppc::protocol
\ No newline at end of file
+}  // namespace ppc::protocol
diff --git a/cpp/wedpr-protocol/protocol/tests/CMakeLists.txt b/cpp/wedpr-protocol/protocol/tests/CMakeLists.txt
index 4466ace3..3248c2ec 100644
--- a/cpp/wedpr-protocol/protocol/tests/CMakeLists.txt
+++ b/cpp/wedpr-protocol/protocol/tests/CMakeLists.txt
@@ -6,5 +6,5 @@ set(TEST_BINARY_NAME test-protocol)
 add_executable(${TEST_BINARY_NAME} ${SOURCES})
 target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
 
-target_link_libraries(${TEST_BINARY_NAME} ${PROTOCOL_TARGET} TBB::tbb ${BOOST_UNIT_TEST})
+target_link_libraries(${TEST_BINARY_NAME} ${PROTOCOL_TARGET} ${BCOS_UTILITIES_TARGET} TBB::tbb ${BOOST_UNIT_TEST})
 add_test(NAME test-protocol WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/tars/CMakeLists.txt b/cpp/wedpr-protocol/tars/CMakeLists.txt
index 7ae5a697..ebee8479 100644
--- a/cpp/wedpr-protocol/tars/CMakeLists.txt
+++ b/cpp/wedpr-protocol/tars/CMakeLists.txt
@@ -1,8 +1,3 @@
-cmake_minimum_required(VERSION 3.14)
-
-include(Version)
-project(wedpr-tars-protocol VERSION ${VERSION})
-
 # for tars generator
 set(TARS_HEADER_DIR ${CMAKE_BINARY_DIR}/generated/tars)
 find_program(TARS_TARS2CPP tars2cpp REQUIRED)
diff --git a/cpp/wedpr-protocol/tars/TarsStruct.h b/cpp/wedpr-protocol/tars/TarsStruct.h
index 395725cd..4818237d 100644
--- a/cpp/wedpr-protocol/tars/TarsStruct.h
+++ b/cpp/wedpr-protocol/tars/TarsStruct.h
@@ -19,22 +19,11 @@
  */
 
 #pragma once
+#include 
 #include 
-#include 
 
 namespace ppctars::serialize
 {
 template 
-concept TarsStruct = requires(TarsStructType tarsStruct)
-{
-    {
-        tarsStruct.className()
-    }
-    ->std::same_as;
-    {
-        tarsStruct.MD5()
-    }
-    ->std::same_as;
-    tarsStruct.resetDefautlt();
-};
+concept TarsStruct = std::derived_from;
 }  // namespace ppctars::serialize
diff --git a/cpp/wedpr-storage/ppc-io/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/CMakeLists.txt
index 94d81491..f4ffe9dd 100644
--- a/cpp/wedpr-storage/ppc-io/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-io/CMakeLists.txt
@@ -1,5 +1,3 @@
-project(ppc-io VERSION ${VERSION})
-
 add_subdirectory(src)
 if (DEMO)
     add_subdirectory(demo)
diff --git a/cpp/wedpr-storage/ppc-storage/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/CMakeLists.txt
index 36b28b6c..5e482dc1 100644
--- a/cpp/wedpr-storage/ppc-storage/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-storage/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-storage VERSION ${VERSION})
 add_subdirectory(src)
 
 if (DEMO)
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
index 1da25b26..e7a52fd7 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
@@ -176,6 +176,10 @@ void CallbackManager::onReceiveMessage(std::string const& topic, Message::Ptr ms
     }
     if (!callback)
     {
+        if (topic.empty())
+        {
+            return;
+        }
         FRONT_LOG(TRACE) << LOG_DESC("onReceiveMessage: not find the handler, put into the buffer")
                          << LOG_KV("topic", topic);
         addMsgCache(topic, msg);
diff --git a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
index dc81e28d..142780b5 100644
--- a/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
+++ b/cpp/wedpr-transport/ppc-gateway/CMakeLists.txt
@@ -1,8 +1,4 @@
 cmake_minimum_required(VERSION 3.14)
-
-include(Version)
-project(ppc-gateway VERSION ${VERSION})
-
 find_package(Boost COMPONENTS filesystem)
 
 file(GLOB_RECURSE SRCS ppc-gateway/*.cpp)
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
index a31a184d..db62f116 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
@@ -23,6 +23,7 @@
 #include "bcos-boostssl/websocket/WsInitializer.h"
 #include "ppc-gateway/p2p/Service.h"
 #include "ppc-gateway/p2p/router/RouterTableImpl.h"
+#include "ppc-gateway/protocol/P2PMessageImpl.h"
 #include "ppc-tools/src/config/PPCConfig.h"
 #include "protocol/src/v1/MessageHeaderImpl.h"
 #include "protocol/src/v1/MessageImpl.h"
@@ -54,8 +55,9 @@ Service::Ptr GatewayFactory::buildService() const
 
     auto wsInitializer = std::make_shared();
     // set the messageFactory
-    wsInitializer->setMessageFactory(
-        std::make_shared(std::make_shared()));
+    auto msgBuilder =
+        std::make_shared(std::make_shared());
+    wsInitializer->setMessageFactory(std::make_shared(msgBuilder));
     // set the config
     wsInitializer->setConfig(wsConfig);
     auto p2pService = std::make_shared(m_contextConfig->nodeID(),
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
index 273bb7fe..effe4ece 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
@@ -35,7 +35,7 @@ GatewayImpl::GatewayImpl(Service::Ptr const& service,
     std::shared_ptr ioService, std::string const& agency)
   : m_service(service),
     m_msgBuilder(
-        std::dynamic_pointer_cast(service->messageFactory())),
+        std::dynamic_pointer_cast(service->messageFactory())),
     m_frontBuilder(frontBuilder),
     m_agency(agency),
     m_p2pRouterManager(std::make_shared(service)),
@@ -155,13 +155,14 @@ void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType,
     auto p2pMessage = m_msgBuilder->build(routeType, routeInfo, std::move(payload));
     p2pMessage->setSeq(traceID);
     p2pMessage->setPacketType((uint16_t)GatewayPacketType::P2PMessage);
-    GATEWAY_LOG(TRACE) << LOG_DESC("asyncSendMessage") << LOG_KV("msg", printMessage(p2pMessage));
+    GATEWAY_LOG(TRACE) << LOG_DESC("asyncSendMessage")
+                       << LOG_KV("msg", printP2PMessage(p2pMessage));
     auto nodeList = m_localRouter->chooseReceiver(p2pMessage);
     // case send to the same agency
     if (!nodeList.empty())
     {
         GATEWAY_LOG(TRACE) << LOG_DESC("hit the local router, dispatch message directly")
-                           << LOG_KV("msg", printMessage(p2pMessage));
+                           << LOG_KV("msg", printP2PMessage(p2pMessage));
         m_localRouter->dispatcherMessage(p2pMessage, callback);
         return;
     }
@@ -171,7 +172,7 @@ void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType,
     if (selectedP2PNodes.empty())
     {
         GATEWAY_LOG(INFO) << LOG_DESC("can't find the gateway to send the message")
-                          << LOG_KV("detail", printMessage(p2pMessage));
+                          << LOG_KV("detail", printP2PMessage(p2pMessage));
         if (callback)
         {
             callback(std::make_shared(
@@ -189,7 +190,7 @@ void GatewayImpl::asyncSendMessage(ppc::protocol::RouteType routeType,
 void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr session)
 {
     // try to dispatcher to the front
-    auto p2pMessage = std::dynamic_pointer_cast(msg);
+    auto p2pMessage = std::dynamic_pointer_cast(msg);
     auto self = std::weak_ptr(shared_from_this());
     // Note: the callback can only been called once since it binds the callback seq
     auto callback = [p2pMessage, session, self](Error::Ptr error) {
@@ -209,7 +210,7 @@ void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr sessi
             GATEWAY_LOG(WARNING) << LOG_DESC("onReceiveP2PMessage: dispatcherMessage failed")
                                  << LOG_KV("code", error->errorCode())
                                  << LOG_KV("msg", error->errorMessage())
-                                 << printMessage(p2pMessage);
+                                 << printP2PMessage(p2pMessage);
             errorCode = std::to_string(error->errorCode());
         }
 
@@ -224,18 +225,18 @@ void GatewayImpl::onReceiveP2PMessage(MessageFace::Ptr msg, WsSession::Ptr sessi
         GATEWAY_LOG(ERROR)
             << LOG_DESC(
                    "onReceiveP2PMessage failed to find the node that can dispatch this message")
-            << LOG_KV("msg", printMessage(p2pMessage));
+            << LOG_KV("msg", printP2PMessage(p2pMessage));
         callback(std::make_shared(CommonError::NotFoundFrontServiceDispatchMsg,
             "unable to find the node to dispatcher this message, message detail: " +
-                printMessage(p2pMessage)));
+                printP2PMessage(p2pMessage)));
     }
 }
 
 void GatewayImpl::onReceiveBroadcastMessage(MessageFace::Ptr msg, WsSession::Ptr)
 {
-    auto p2pMessage = std::dynamic_pointer_cast(msg);
+    auto p2pMessage = std::dynamic_pointer_cast(msg);
     GATEWAY_LOG(TRACE) << LOG_DESC("onReceiveBroadcastMessage, dispatcher")
-                       << LOG_KV("msg", printMessage(p2pMessage));
+                       << LOG_KV("msg", printP2PMessage(p2pMessage));
     m_localRouter->dispatcherMessage(p2pMessage, nullptr);
 }
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
index aad2fb4f..e965bab3 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
@@ -86,7 +86,7 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this(
                 -1, "can't find the gateway to send the message, detail: " +
-                        printMessage(m_p2pMessage)));
+                        printP2PMessage(m_p2pMessage)));
         }
         return;
     }
@@ -66,7 +66,7 @@ void SendMessageWithRetry::trySendMessage()
         {
             GATEWAY_LOG(DEBUG) << LOG_BADGE("trySendMessage")
                                << LOG_DESC("send message failed, retry again")
-                               << LOG_KV("msg", printMessage(self->m_p2pMessage))
+                               << LOG_KV("msg", printP2PMessage(self->m_p2pMessage))
                                << LOG_KV("code", error->errorCode())
                                << LOG_KV("msg", error->errorMessage())
                                << LOG_KV("timeCost", (utcTime() - startT));
@@ -85,13 +85,13 @@ void SendMessageWithRetry::trySendMessage()
             {
                 GATEWAY_LOG(DEBUG)
                     << LOG_BADGE("trySendMessage again") << LOG_KV("respCode", respCode)
-                    << LOG_KV("msg", printMessage(self->m_p2pMessage));
+                    << LOG_KV("msg", printP2PMessage(self->m_p2pMessage));
                 // try again
                 self->trySendMessage();
                 return;
             }
             GATEWAY_LOG(TRACE) << LOG_BADGE("asyncSendMessageByNodeID success")
-                               << LOG_KV("msg", printMessage(self->m_p2pMessage));
+                               << LOG_KV("msg", printP2PMessage(self->m_p2pMessage));
             // send message successfully
             if (self->m_respFunc)
             {
@@ -102,7 +102,7 @@ void SendMessageWithRetry::trySendMessage()
         catch (const std::exception& e)
         {
             GATEWAY_LOG(ERROR) << LOG_BADGE("trySendMessage and receive response exception")
-                               << LOG_KV("msg", printMessage(self->m_p2pMessage))
+                               << LOG_KV("msg", printP2PMessage(self->m_p2pMessage))
                                << LOG_KV("error", boost::diagnostic_information(e));
 
             self->trySendMessage();
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h
index 7df3b6f3..bfd63a85 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/SendMessageWithRetry.h
@@ -18,7 +18,7 @@
  * @date 2024-08-26
  */
 #pragma once
-#include "ppc-framework/protocol/Message.h"
+#include "ppc-framework/protocol/P2PMessage.h"
 #include "ppc-gateway/gateway/router/GatewayNodeInfo.h"
 #include "ppc-gateway/p2p/Service.h"
 #include 
@@ -31,7 +31,7 @@ class SendMessageWithRetry : public std::enable_shared_from_this;
     SendMessageWithRetry(Service::Ptr const& service, GatewayNodeInfos&& dstNodeList,
-        ppc::protocol::Message::Ptr&& p2pMessage, ppc::protocol::ReceiveMsgFunc respFunc,
+        ppc::protocol::P2PMessage::Ptr&& p2pMessage, ppc::protocol::ReceiveMsgFunc respFunc,
         long timeout)
       : m_service(service),
         m_dstNodeList(std::move(dstNodeList)),
@@ -54,7 +54,7 @@ class SendMessageWithRetry : public std::enable_shared_from_this
 #include 
@@ -29,7 +29,7 @@ namespace ppc::gateway
 {
 struct MessageInfo
 {
-    ppc::protocol::Message::Ptr msg;
+    ppc::protocol::P2PMessage::Ptr msg;
     ppc::protocol::ReceiveMsgFunc callback;
 };
 struct HoldingMessageQueue
@@ -50,7 +50,7 @@ class MessageCache : public std::enable_shared_from_this
     {}
     virtual ~MessageCache() = default;
 
-    void insertCache(std::string const& topic, ppc::protocol::Message::Ptr const& msg,
+    void insertCache(std::string const& topic, ppc::protocol::P2PMessage::Ptr const& msg,
         ppc::protocol::ReceiveMsgFunc callback);
     HoldingMessageQueue::Ptr pop(std::string const& topic);
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
index b22b73a4..19553502 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
@@ -95,7 +95,7 @@ void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSessi
 {
     auto statusSeq =
         boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)msg->payload()->data()));
-    auto p2pMessage = std::dynamic_pointer_cast(msg);
+    auto p2pMessage = std::dynamic_pointer_cast(msg);
     auto const& from = (p2pMessage->header()->srcGwNode().size() > 0) ?
                            p2pMessage->header()->srcGwNode() :
                            session->nodeId();
@@ -128,7 +128,8 @@ bool GatewayRouterManager::statusChanged(std::string const& p2pNodeID, uint32_t
 void GatewayRouterManager::broadcastStatusSeq()
 {
     m_timer->restart();
-    auto message = std::dynamic_pointer_cast(m_service->messageFactory()->buildMessage());
+    auto message =
+        std::dynamic_pointer_cast(m_service->messageFactory()->buildMessage());
     message->setPacketType((uint16_t)GatewayPacketType::SyncNodeSeq);
     auto seq = m_localRouter->statusSeq();
     auto statusSeq = boost::asio::detail::socket_ops::host_to_network_long(seq);
@@ -142,7 +143,7 @@ void GatewayRouterManager::broadcastStatusSeq()
 void GatewayRouterManager::onReceiveRequestNodeStatusMsg(
     MessageFace::Ptr msg, WsSession::Ptr session)
 {
-    auto p2pMessage = std::dynamic_pointer_cast(msg);
+    auto p2pMessage = std::dynamic_pointer_cast(msg);
     auto const& from = (!p2pMessage->header()->srcGwNode().empty()) ?
                            p2pMessage->header()->srcGwNode() :
                            session->nodeId();
@@ -167,7 +168,7 @@ void GatewayRouterManager::onRecvResponseNodeStatusMsg(MessageFace::Ptr msg, WsS
     auto nodeStatus = m_nodeStatusFactory->build();
     nodeStatus->decode(bytesConstRef(msg->payload()->data(), msg->payload()->size()));
 
-    auto p2pMessage = std::dynamic_pointer_cast(msg);
+    auto p2pMessage = std::dynamic_pointer_cast(msg);
     auto const& from = (!p2pMessage->header()->srcGwNode().empty()) ?
                            p2pMessage->header()->srcGwNode() :
                            session->nodeId();
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
index 9b910408..ef9ce7ba 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
@@ -76,7 +76,8 @@ void LocalRouter::unRegisterTopic(bcos::bytesConstRef _nodeID, std::string const
     m_routerInfo->unRegisterTopic(_nodeID.toBytes(), topic);
 }
 
-bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc callback, bool holding)
+bool LocalRouter::dispatcherMessage(
+    P2PMessage::Ptr const& msg, ReceiveMsgFunc callback, bool holding)
 {
     auto frontList = chooseReceiver(msg);
     // find the front
@@ -88,11 +89,11 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call
         {
             if (i == 0)
             {
-                front->onReceiveMessage(msg, callback);
+                front->onReceiveMessage(msg->msg(), callback);
             }
             else
             {
-                front->onReceiveMessage(msg, [](bcos::Error::Ptr error) {
+                front->onReceiveMessage(msg->msg(), [](bcos::Error::Ptr error) {
                     if (!error || error->errorCode() == 0)
                     {
                         return;
@@ -126,7 +127,7 @@ bool LocalRouter::dispatcherMessage(Message::Ptr const& msg, ReceiveMsgFunc call
 }
 
 std::vector LocalRouter::chooseReceiver(
-    ppc::protocol::Message::Ptr const& msg)
+    ppc::protocol::P2PMessage::Ptr const& msg)
 {
     std::vector receivers;
     auto const& dstInst = msg->header()->optionalField()->dstInst();
@@ -168,6 +169,6 @@ std::vector LocalRouter::chooseReceiver(
     default:
         BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment(
                                   "chooseReceiver failed for unknown routeType, message detail: " +
-                                  printMessage(msg)));
+                                  printP2PMessage(msg)));
     }
 }
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
index 7c2e015e..e64aadb8 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.h
@@ -21,7 +21,7 @@
 #include "../cache/MessageCache.h"
 #include "GatewayNodeInfo.h"
 #include "ppc-framework/protocol/INodeInfo.h"
-#include "ppc-framework/protocol/Message.h"
+#include "ppc-framework/protocol/P2PMessage.h"
 #include "ppc-framework/protocol/RouteType.h"
 
 namespace ppc::gateway
@@ -51,9 +51,9 @@ class LocalRouter
     virtual void unRegisterTopic(bcos::bytesConstRef nodeID, std::string const& topic);
 
     virtual std::vector chooseReceiver(
-        ppc::protocol::Message::Ptr const& msg);
+        ppc::protocol::P2PMessage::Ptr const& msg);
 
-    virtual bool dispatcherMessage(ppc::protocol::Message::Ptr const& msg,
+    virtual bool dispatcherMessage(ppc::protocol::P2PMessage::Ptr const& msg,
         ppc::protocol::ReceiveMsgFunc callback, bool holding = true);
 
     std::shared_ptr generateNodeStatus()
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
index 6c7c1f4a..533578a0 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.cpp
@@ -284,7 +284,7 @@ void PeerRouterTable::selectRouterByComponent(GatewayNodeInfos& choosedGateway,
 }
 
 
-void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const
+void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::P2PMessage::Ptr const& msg) const
 {
     bcos::ReadGuard l(x_mutex);
     for (auto const& it : m_agency2GatewayInfos)
@@ -303,7 +303,7 @@ void PeerRouterTable::asyncBroadcastMessage(ppc::protocol::Message::Ptr const& m
         }
         PEER_ROUTER_LOG(TRACE) << LOG_DESC("asyncBroadcastMessage")
                                << LOG_KV("nodeID", printP2PIDElegantly(selectedNode->p2pNodeID()))
-                               << LOG_KV("msg", printMessage(msg));
+                               << LOG_KV("msg", printP2PMessage(msg));
         m_service->asyncSendMessageByNodeID(selectedNode->p2pNodeID(), msg);
     }
 }
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
index 330737f0..5600c46f 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/PeerRouterTable.h
@@ -19,7 +19,7 @@
  */
 #pragma once
 #include "GatewayNodeInfo.h"
-#include "ppc-framework/protocol/Message.h"
+#include "ppc-framework/protocol/P2PMessage.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-gateway/p2p/Service.h"
 #include 
@@ -44,7 +44,7 @@ class PeerRouterTable
     virtual std::vector selectTargetNodes(ppc::protocol::RouteType const& routeType,
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) const;
 
-    virtual void asyncBroadcastMessage(ppc::protocol::Message::Ptr const& msg) const;
+    virtual void asyncBroadcastMessage(ppc::protocol::P2PMessage::Ptr const& msg) const;
 
     std::set agencies(std::vector const& components) const;
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
index 12dcb06b..080d7aa9 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
@@ -206,7 +206,7 @@ WsSession::Ptr Service::getSessionByNodeID(std::string const& _nodeID)
 void Service::asyncSendMessageByNodeID(
     std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc)
 {
-    auto p2pMsg = std::dynamic_pointer_cast(msg);
+    auto p2pMsg = std::dynamic_pointer_cast(msg);
     p2pMsg->header()->setDstGwNode(dstNodeID);
     p2pMsg->header()->setSrcGwNode(m_nodeID);
     return asyncSendMessageWithForward(dstNodeID, msg, options, respFunc);
@@ -215,17 +215,17 @@ void Service::asyncSendMessageByNodeID(
 void Service::asyncSendMessageWithForward(
     std::string const& dstNodeID, MessageFace::Ptr msg, Options options, RespCallBack respFunc)
 {
-    auto p2pMsg = std::dynamic_pointer_cast(msg);
+    auto p2pMsg = std::dynamic_pointer_cast(msg);
     // without nextHop: maybe network unreachable or with distance equal to 1
     auto nextHop = m_routerTable->getNextHop(dstNodeID);
     if (nextHop.empty())
     {
-        SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessage directly") << printMessage(p2pMsg);
+        SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessage directly") << printP2PMessage(p2pMsg);
         return asyncSendMessage(dstNodeID, msg, options, respFunc);
     }
     // with nextHop, send the message to nextHop
     SERVICE_LOG(TRACE) << LOG_DESC("asyncSendMessageWithForward to nextHop")
-                       << printMessage(p2pMsg);
+                       << printP2PMessage(p2pMsg);
     return asyncSendMessage(nextHop, msg, options, respFunc);
 }
 
@@ -279,19 +279,19 @@ void Service::asyncSendMessage(
 
 void Service::onRecvMessage(MessageFace::Ptr _msg, std::shared_ptr _session)
 {
-    auto p2pMsg = std::dynamic_pointer_cast(_msg);
+    auto p2pMsg = std::dynamic_pointer_cast(_msg);
     // find the dstNode
     if (p2pMsg->header()->dstGwNode().empty() || p2pMsg->header()->dstGwNode() == m_nodeID)
     {
         SERVICE_LOG(TRACE) << LOG_DESC("onRecvMessage, dispatch for find the dst node")
-                           << printMessage(p2pMsg);
+                           << printP2PMessage(p2pMsg);
         WsService::onRecvMessage(_msg, _session);
         return;
     }
     // forward the message
     if (p2pMsg->header()->ttl() >= m_routerTable->unreachableDistance())
     {
-        SERVICE_LOG(WARNING) << LOG_DESC("onRecvMessage: ttl expired") << printMessage(p2pMsg);
+        SERVICE_LOG(WARNING) << LOG_DESC("onRecvMessage: ttl expired") << printP2PMessage(p2pMsg);
         return;
     }
     p2pMsg->header()->setTTL(p2pMsg->header()->ttl() + 1);
@@ -311,7 +311,7 @@ void Service::asyncBroadcastMessage(bcos::boostssl::MessageFace::Ptr msg, Option
         }
         for (auto const& node : reachableNodes)
         {
-            auto p2pMsg = std::dynamic_pointer_cast(msg);
+            auto p2pMsg = std::dynamic_pointer_cast(msg);
             p2pMsg->header()->setDstGwNode(node);
             asyncSendMessageByNodeID(node, msg, options);
         }
@@ -336,8 +336,8 @@ void Service::asyncSendMessageByP2PNodeID(uint16_t type, std::string const& dstN
 void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const& session,
     bcos::boostssl::MessageFace::Ptr msg, std::shared_ptr&& payload)
 {
-    auto respMessage = std::dynamic_pointer_cast(m_messageFactory->buildMessage());
-    auto requestMsg = std::dynamic_pointer_cast(msg);
+    auto respMessage = std::dynamic_pointer_cast(m_messageFactory->buildMessage());
+    auto requestMsg = std::dynamic_pointer_cast(msg);
     if (requestMsg->header() && requestMsg->header()->optionalField())
     {
         respMessage->header()->optionalField()->setDstNode(
@@ -360,7 +360,8 @@ void Service::sendRespMessageBySession(bcos::boostssl::ws::WsSession::Ptr const&
     WsSessions sessions;
     sessions.emplace_back(session);
     WsService::asyncSendMessage(sessions, respMessage);
-    SERVICE_LOG(TRACE) << "sendRespMessageBySession: " << LOG_KV("resp", printMessage(respMessage))
+    SERVICE_LOG(TRACE) << "sendRespMessageBySession: "
+                       << LOG_KV("resp", printP2PMessage(respMessage))
                        << LOG_KV("sessionNode", printP2PIDElegantly(session->nodeId()))
                        << LOG_KV("payloadSize",
                               respMessage->payload() ? respMessage->payload()->size() : 0);
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
index 20b525b1..64477b1e 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
@@ -20,7 +20,7 @@
 
 #pragma once
 #include "../Common.h"
-#include "ppc-framework/protocol/Message.h"
+#include "ppc-framework/protocol/P2PMessage.h"
 #include "router/RouterTableInterface.h"
 #include 
 namespace ppc::gateway
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
index b4970643..b4518cb2 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
@@ -20,7 +20,7 @@
 #include "RouterManager.h"
 #include "ppc-framework/Helper.h"
 #include "ppc-framework/gateway/GatewayProtocol.h"
-#include "ppc-framework/protocol/Message.h"
+#include "ppc-framework/protocol/P2PMessage.h"
 #include 
 
 using namespace bcos;
@@ -76,7 +76,7 @@ void RouterManager::onReceiveRouterSeq(MessageFace::Ptr msg, WsSession::Ptr sess
                              << LOG_KV("peer", printP2PIDElegantly(session->nodeId()))
                              << LOG_KV("seq", statusSeq);
     // request router table to peer
-    auto p2pMsg = std::dynamic_pointer_cast(msg);
+    auto p2pMsg = std::dynamic_pointer_cast(msg);
     auto dstP2PNodeID = (!p2pMsg->header()->srcGwNode().empty()) ? p2pMsg->header()->srcGwNode() :
                                                                    session->nodeId();
     m_service->asyncSendMessageByP2PNodeID((uint16_t)GatewayPacketType::RouterTableRequest,
@@ -117,7 +117,7 @@ void RouterManager::onReceiveRouterTableRequest(MessageFace::Ptr msg, WsSession:
 
     auto routerTableData = std::make_shared();
     m_service->routerTable()->encode(*routerTableData);
-    auto p2pMsg = std::dynamic_pointer_cast(msg);
+    auto p2pMsg = std::dynamic_pointer_cast(msg);
     auto dstP2PNodeID = (!p2pMsg->header()->srcGwNode().empty()) ? p2pMsg->header()->srcGwNode() :
                                                                    session->nodeId();
     m_service->asyncSendMessageByP2PNodeID(
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.cpp
new file mode 100644
index 00000000..daa711fb
--- /dev/null
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.cpp
@@ -0,0 +1,43 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file MessageImpl.h
+ * @author: yujiechen
+ * @date 2024-08-22
+ */
+#include "P2PMessageImpl.h"
+#include "../Common.h"
+
+using namespace bcos;
+using namespace ppc::protocol;
+
+
+bool P2PMessageImpl::encode(bcos::boostssl::EncodedMsg& encodedMsg)
+{
+    try
+    {
+        // header
+        m_msg->header()->encode(encodedMsg.header);
+        // assign the payload back
+        encodedMsg.payload = m_msg->payload();
+        return true;
+    }
+    catch (std::exception const& e)
+    {
+        GATEWAY_LOG(WARNING) << LOG_DESC("encode message failed")
+                             << LOG_KV("error", boost::diagnostic_information(e));
+        return false;
+    }
+}
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.h
new file mode 100644
index 00000000..07a29edc
--- /dev/null
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/protocol/P2PMessageImpl.h
@@ -0,0 +1,64 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file MessageImpl.h
+ * @author: yujiechen
+ * @date 2024-08-22
+ */
+#pragma once
+#include "ppc-framework/protocol/P2PMessage.h"
+#include "ppc-utilities/Utilities.h"
+
+namespace ppc::protocol
+{
+class P2PMessageImpl : public P2PMessage
+{
+public:
+    using Ptr = std::shared_ptr;
+    P2PMessageImpl(Message::Ptr msg) : P2PMessage(msg) {}
+    ~P2PMessageImpl() override = default;
+    // encode and return the {header, payload}
+    bool encode(bcos::boostssl::EncodedMsg& _encodedMsg) override;
+};
+
+class P2PMessageBuilderImpl : public P2PMessageBuilder
+{
+public:
+    using Ptr = std::shared_ptr;
+    P2PMessageBuilderImpl(MessageBuilder::Ptr msgBuilder) : m_msgBuilder(std::move(msgBuilder)) {}
+    ~P2PMessageBuilderImpl() override = default;
+
+    P2PMessage::Ptr build() override
+    {
+        return std::make_shared(m_msgBuilder->build());
+    }
+
+    P2PMessage::Ptr build(ppc::protocol::RouteType routeType,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload) override
+    {
+        return std::make_shared(
+            m_msgBuilder->build(routeType, routeInfo, std::move(payload)));
+    }
+
+    bcos::boostssl::MessageFace::Ptr buildMessage() override
+    {
+        return std::make_shared(m_msgBuilder->build());
+    }
+    std::string newSeq() override { return ppc::generateUUID(); }
+
+protected:
+    MessageBuilder::Ptr m_msgBuilder;
+};
+}  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-http/CMakeLists.txt b/cpp/wedpr-transport/ppc-http/CMakeLists.txt
index 6039e2ed..5e482dc1 100644
--- a/cpp/wedpr-transport/ppc-http/CMakeLists.txt
+++ b/cpp/wedpr-transport/ppc-http/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-http VERSION ${VERSION})
 add_subdirectory(src)
 
 if (DEMO)
diff --git a/cpp/wedpr-transport/ppc-rpc/CMakeLists.txt b/cpp/wedpr-transport/ppc-rpc/CMakeLists.txt
index f111a675..6fa01414 100644
--- a/cpp/wedpr-transport/ppc-rpc/CMakeLists.txt
+++ b/cpp/wedpr-transport/ppc-rpc/CMakeLists.txt
@@ -1,4 +1,3 @@
-project(ppc-rpc VERSION ${VERSION})
 add_subdirectory(src)
 
 if (TESTS)
diff --git a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt
index ab69fa97..b708386a 100644
--- a/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/CMakeLists.txt
@@ -1,8 +1,5 @@
 project(wedpr-transport-swig VERSION ${VERSION})
 
-find_package(SWIG REQUIRED)
-include(${SWIG_USE_FILE})
-
 # Add subdirectories for each language if desired
 option(BUILD_PYTHON "Build Python SWIG module" OFF)
 if(BUILD_PYTHON)
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
index c41394e7..f3665b1c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
@@ -1,13 +1,8 @@
 option(AUTO_GENERATE "Auto generate the targes" OFF)
 # Note: use SWIG Version 4.2.1
 if(AUTO_GENERATE)
-    if(WIN32)
-      message(STATUS "Getting SWIG for Windows: ...")
-      include(swig)
-      message(STATUS "Getting SWIG for Windows: ...DONE")
-    endif()
+    include(${SWIG_USE_FILE})
     # only auto-generate should find swig
-    include(java)
     file(GLOB_RECURSE SRCS wedpr_java_transport.i)
     set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON)
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
index 32ddfb70..dc50506c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
@@ -91,24 +91,15 @@ public void setRespPacket() {
         wedpr_java_transportJNI.Message_setRespPacket(swigCPtr, this);
     }
 
-    public long length() {
-        return wedpr_java_transportJNI.Message_length(swigCPtr, this);
-    }
-
-    public ubytes payload() {
-        long cPtr = wedpr_java_transportJNI.Message_payload(swigCPtr, this);
-        return (cPtr == 0) ? null : new ubytes(cPtr, true);
+    public void setPayload(ubytes _payload) {
+        wedpr_java_transportJNI.Message_setPayload(
+                swigCPtr, this, ubytes.getCPtr(_payload), _payload);
     }
 
     public byte[] payloadBuffer() {
         return wedpr_java_transportJNI.Message_payloadBuffer(swigCPtr, this);
     }
 
-    public void setPayload(ubytes _payload) {
-        wedpr_java_transportJNI.Message_setPayload(
-                swigCPtr, this, ubytes.getCPtr(_payload), _payload);
-    }
-
     public void setFrontMessage(MessagePayload frontMessage) {
         wedpr_java_transportJNI.Message_setFrontMessage(
                 swigCPtr, this, MessagePayload.getCPtr(frontMessage), frontMessage);
@@ -120,20 +111,24 @@ public MessagePayload frontMessage() {
     }
 
     public boolean encode(ubytes _buffer) {
-        return wedpr_java_transportJNI.Message_encode__SWIG_0(
+        return wedpr_java_transportJNI.Message_encode(
                 swigCPtr, this, ubytes.getCPtr(_buffer), _buffer);
     }
 
-    public boolean encode(SWIGTYPE_p_bcos__boostssl__EncodedMsg _encodedMsg) {
-        return wedpr_java_transportJNI.Message_encode__SWIG_1(
-                swigCPtr, this, SWIGTYPE_p_bcos__boostssl__EncodedMsg.getCPtr(_encodedMsg));
-    }
-
     public long decode(SWIGTYPE_p_bcos__bytesConstRef _buffer) {
         return wedpr_java_transportJNI.Message_decode(
                 swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(_buffer));
     }
 
+    public long length() {
+        return wedpr_java_transportJNI.Message_length(swigCPtr, this);
+    }
+
+    public ubytes payload() {
+        long cPtr = wedpr_java_transportJNI.Message_payload(swigCPtr, this);
+        return (cPtr == 0) ? null : new ubytes(cPtr, true);
+    }
+
     public void disOwnMemory() {
         swigSetCMemOwn(false);
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
index c8f1cdc8..d8b77685 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
@@ -21,9 +21,4 @@ public static String printOptionalField(MessageOptionalHeader optionalHeader) {
     public static String printMessage(Message _msg) {
         return wedpr_java_transportJNI.printMessage(Message.getCPtr(_msg), _msg);
     }
-
-    public static String printWsMessage(SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr _msg) {
-        return wedpr_java_transportJNI.printWsMessage(
-                SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr.getCPtr(_msg));
-    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index 82e57708..814c24f8 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -509,28 +509,25 @@ public static final native void Message_setHeader(
 
     public static final native void Message_setRespPacket(long jarg1, Message jarg1_);
 
-    public static final native long Message_length(long jarg1, Message jarg1_);
-
-    public static final native long Message_payload(long jarg1, Message jarg1_);
-
-    public static final native byte[] Message_payloadBuffer(long jarg1, Message jarg1_);
-
     public static final native void Message_setPayload(
             long jarg1, Message jarg1_, long jarg2, ubytes jarg2_);
 
+    public static final native byte[] Message_payloadBuffer(long jarg1, Message jarg1_);
+
     public static final native void Message_setFrontMessage(
             long jarg1, Message jarg1_, long jarg2, MessagePayload jarg2_);
 
     public static final native long Message_frontMessage(long jarg1, Message jarg1_);
 
-    public static final native boolean Message_encode__SWIG_0(
+    public static final native boolean Message_encode(
             long jarg1, Message jarg1_, long jarg2, ubytes jarg2_);
 
-    public static final native boolean Message_encode__SWIG_1(
-            long jarg1, Message jarg1_, long jarg2);
-
     public static final native long Message_decode(long jarg1, Message jarg1_, long jarg2);
 
+    public static final native long Message_length(long jarg1, Message jarg1_);
+
+    public static final native long Message_payload(long jarg1, Message jarg1_);
+
     public static final native void delete_MessageHeaderBuilder(long jarg1);
 
     public static final native long MessageHeaderBuilder_build__SWIG_0(
@@ -573,8 +570,6 @@ public static final native long MessageOptionalHeaderBuilder_build__SWIG_1(
 
     public static final native String printMessage(long jarg1, Message jarg1_);
 
-    public static final native String printWsMessage(long jarg1);
-
     public static final native int FrontMsgExtFlag_Response_get();
 
     public static final native void delete_MessagePayload(long jarg1);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 699684c9..807fb464 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -5247,39 +5247,22 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1length(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  uint32_t result;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
-  arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
-  jresult = (jlong)result; 
-  return jresult;
-}
-
-
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1payload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setPayload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  std::shared_ptr< bcos::bytes > result;
+  std::shared_ptr< bcos::bytes > arg2 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  std::shared_ptr< bcos::bytes > *argp2 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
+  (void)jarg2_;
   
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::Message const *)arg1)->payload();
-  *(std::shared_ptr< bcos::bytes > **)&jresult = result ? new std::shared_ptr< bcos::bytes >(result) : 0; 
-  return jresult;
+  argp2 = *(std::shared_ptr< bcos::bytes > **)&jarg2;
+  if (argp2) arg2 = *argp2; 
+  (arg1)->setPayload(arg2);
 }
 
 
@@ -5304,25 +5287,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setPayload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::shared_ptr< bcos::bytes > arg2 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  std::shared_ptr< bcos::bytes > *argp2 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg2_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
-  arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  argp2 = *(std::shared_ptr< bcos::bytes > **)&jarg2;
-  if (argp2) arg2 = *argp2; 
-  (arg1)->setPayload(arg2);
-}
-
-
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setFrontMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   ppc::protocol::MessagePayload::Ptr arg2 ;
@@ -5360,7 +5324,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
-SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1encode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1encode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
   jboolean jresult = 0 ;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   bcos::bytes *arg2 = 0 ;
@@ -5386,12 +5350,13 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
 }
 
 
-SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1encode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
-  jboolean jresult = 0 ;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
+  jlong jresult = 0 ;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  bcos::boostssl::EncodedMsg *arg2 = 0 ;
+  bcos::bytesConstRef arg2 ;
   std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  bool result;
+  bcos::bytesConstRef *argp2 ;
+  int64_t result;
   
   (void)jenv;
   (void)jcls;
@@ -5399,43 +5364,54 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = *(bcos::boostssl::EncodedMsg **)&jarg2;
-  if (!arg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::boostssl::EncodedMsg & is null");
+  argp2 = *(bcos::bytesConstRef **)&jarg2; 
+  if (!argp2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef");
     return 0;
-  } 
-  result = (bool)(arg1)->encode(*arg2);
-  jresult = (jboolean)result; 
+  }
+  arg2 = *argp2; 
+  result = (int64_t)(arg1)->decode(arg2);
+  jresult = (jlong)result; 
   return jresult;
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1decode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1length(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jlong jresult = 0 ;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  bcos::bytesConstRef arg2 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  bcos::bytesConstRef *argp2 ;
-  int64_t result;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  uint32_t result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
   
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  argp2 = *(bcos::bytesConstRef **)&jarg2; 
-  if (!argp2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef");
-    return 0;
-  }
-  arg2 = *argp2; 
-  result = (int64_t)(arg1)->decode(arg2);
+  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
   jresult = (jlong)result; 
   return jresult;
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1payload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< bcos::bytes > result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
+  arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::protocol::Message const *)arg1)->payload();
+  *(std::shared_ptr< bcos::bytes > **)&jresult = result ? new std::shared_ptr< bcos::bytes >(result) : 0; 
+  return jresult;
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
   std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
@@ -5697,24 +5673,6 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
 }
 
 
-SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printWsMessage(JNIEnv *jenv, jclass jcls, jlong jarg1) {
-  jstring jresult = 0 ;
-  bcos::boostssl::MessageFace::Ptr *arg1 = 0 ;
-  std::string result;
-  
-  (void)jenv;
-  (void)jcls;
-  arg1 = *(bcos::boostssl::MessageFace::Ptr **)&jarg1;
-  if (!arg1) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::boostssl::MessageFace::Ptr const & is null");
-    return 0;
-  } 
-  result = ppc::protocol::printWsMessage((bcos::boostssl::MessageFace::Ptr const &)*arg1);
-  jresult = jenv->NewStringUTF((&result)->c_str()); 
-  return jresult;
-}
-
-
 SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontMsgExtFlag_1Response_1get(JNIEnv *jenv, jclass jcls) {
   jint jresult = 0 ;
   ppc::protocol::FrontMsgExtFlag result;
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
index fbaa6b04..d78c693a 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
@@ -1,6 +1,7 @@
 option(AUTO_GENERATE "Auto generate the targes" OFF)
 # Note: use SWIG Version 4.2.1
 if(AUTO_GENERATE)
+    include(${SWIG_USE_FILE})
     file(GLOB_RECURSE SRCS *.i)
     set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON)
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 0b97ed83..9b02b5e6 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1199,30 +1199,30 @@ def isRespPacket(self):
     def setRespPacket(self):
         return _wedpr_python_transport.Message_setRespPacket(self)
 
-    def length(self):
-        return _wedpr_python_transport.Message_length(self)
-
-    def payload(self):
-        return _wedpr_python_transport.Message_payload(self)
+    def setPayload(self, _payload):
+        return _wedpr_python_transport.Message_setPayload(self, _payload)
 
     def payloadBuffer(self):
         return _wedpr_python_transport.Message_payloadBuffer(self)
 
-    def setPayload(self, _payload):
-        return _wedpr_python_transport.Message_setPayload(self, _payload)
-
     def setFrontMessage(self, frontMessage):
         return _wedpr_python_transport.Message_setFrontMessage(self, frontMessage)
 
     def frontMessage(self):
         return _wedpr_python_transport.Message_frontMessage(self)
 
-    def encode(self, *args):
-        return _wedpr_python_transport.Message_encode(self, *args)
+    def encode(self, _buffer):
+        return _wedpr_python_transport.Message_encode(self, _buffer)
 
     def decode(self, _buffer):
         return _wedpr_python_transport.Message_decode(self, _buffer)
 
+    def length(self):
+        return _wedpr_python_transport.Message_length(self)
+
+    def payload(self):
+        return _wedpr_python_transport.Message_payload(self)
+
 
 # Register Message in _wedpr_python_transport:
 _wedpr_python_transport.Message_swigregister(Message)
@@ -1288,10 +1288,6 @@ def printMessage(_msg):
     return _wedpr_python_transport.printMessage(_msg)
 
 
-def printWsMessage(_msg):
-    return _wedpr_python_transport.printWsMessage(_msg)
-
-
 FrontMsgExtFlag_Response = _wedpr_python_transport.FrontMsgExtFlag_Response
 
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index fef8030e..d93bbbfe 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -3654,87 +3654,85 @@ namespace Swig {
 #define SWIGTYPE_p_UniquePtr swig_types[6]
 #define SWIGTYPE_p_allocator_type swig_types[7]
 #define SWIGTYPE_p_bcos__Error swig_types[8]
-#define SWIGTYPE_p_bcos__boostssl__EncodedMsg swig_types[9]
-#define SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr swig_types[10]
-#define SWIGTYPE_p_bcos__bytesConstRef swig_types[11]
-#define SWIGTYPE_p_char swig_types[12]
-#define SWIGTYPE_p_difference_type swig_types[13]
-#define SWIGTYPE_p_int swig_types[14]
-#define SWIGTYPE_p_long_long swig_types[15]
-#define SWIGTYPE_p_p_PyObject swig_types[16]
-#define SWIGTYPE_p_ppc__front__ErrorCallback swig_types[17]
-#define SWIGTYPE_p_ppc__front__FrontConfig swig_types[18]
-#define SWIGTYPE_p_ppc__front__FrontConfigBuilder swig_types[19]
-#define SWIGTYPE_p_ppc__front__GetPeersInfoHandler swig_types[20]
-#define SWIGTYPE_p_ppc__front__IFront swig_types[21]
-#define SWIGTYPE_p_ppc__front__IFrontBuilder swig_types[22]
-#define SWIGTYPE_p_ppc__front__IFrontClient swig_types[23]
-#define SWIGTYPE_p_ppc__front__IMessageHandler swig_types[24]
-#define SWIGTYPE_p_ppc__front__MessageDispatcherHandler swig_types[25]
-#define SWIGTYPE_p_ppc__front__SendResponseHandler swig_types[26]
-#define SWIGTYPE_p_ppc__gateway__IGateway__Ptr swig_types[27]
-#define SWIGTYPE_p_ppc__protocol__EndPoint swig_types[28]
-#define SWIGTYPE_p_ppc__protocol__GrpcConfig swig_types[29]
-#define SWIGTYPE_p_ppc__protocol__GrpcServerConfig swig_types[30]
-#define SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr swig_types[31]
-#define SWIGTYPE_p_ppc__protocol__Message swig_types[32]
-#define SWIGTYPE_p_ppc__protocol__MessageBuilder swig_types[33]
-#define SWIGTYPE_p_ppc__protocol__MessageHeader swig_types[34]
-#define SWIGTYPE_p_ppc__protocol__MessageHeaderBuilder swig_types[35]
-#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeader swig_types[36]
-#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeaderBuilder swig_types[37]
-#define SWIGTYPE_p_ppc__protocol__MessagePayload swig_types[38]
-#define SWIGTYPE_p_ppc__protocol__MessagePayloadBuilder swig_types[39]
-#define SWIGTYPE_p_ppc__protocol__RouteType swig_types[40]
-#define SWIGTYPE_p_ppc__sdk__Transport swig_types[41]
-#define SWIGTYPE_p_ppc__sdk__TransportBuilder swig_types[42]
-#define SWIGTYPE_p_short swig_types[43]
-#define SWIGTYPE_p_signed_char swig_types[44]
-#define SWIGTYPE_p_size_type swig_types[45]
-#define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[46]
-#define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[47]
-#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[48]
-#define SWIGTYPE_p_std__exception swig_types[49]
-#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[50]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[51]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[52]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[53]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[54]
-#define SWIGTYPE_p_std__invalid_argument swig_types[55]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[56]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[57]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[58]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[59]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[60]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[61]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[62]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[63]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[64]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[65]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[66]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[67]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[68]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[69]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[70]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[71]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[72]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[73]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[74]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[75]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[76]
-#define SWIGTYPE_p_std__string swig_types[77]
-#define SWIGTYPE_p_std__string_view swig_types[78]
-#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[79]
-#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[80]
-#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[81]
-#define SWIGTYPE_p_swig__SwigPyIterator swig_types[82]
-#define SWIGTYPE_p_unsigned_char swig_types[83]
-#define SWIGTYPE_p_unsigned_int swig_types[84]
-#define SWIGTYPE_p_unsigned_long_long swig_types[85]
-#define SWIGTYPE_p_unsigned_short swig_types[86]
-#define SWIGTYPE_p_value_type swig_types[87]
-static swig_type_info *swig_types[89];
-static swig_module_info swig_module = {swig_types, 88, 0, 0, 0, 0};
+#define SWIGTYPE_p_bcos__bytesConstRef swig_types[9]
+#define SWIGTYPE_p_char swig_types[10]
+#define SWIGTYPE_p_difference_type swig_types[11]
+#define SWIGTYPE_p_int swig_types[12]
+#define SWIGTYPE_p_long_long swig_types[13]
+#define SWIGTYPE_p_p_PyObject swig_types[14]
+#define SWIGTYPE_p_ppc__front__ErrorCallback swig_types[15]
+#define SWIGTYPE_p_ppc__front__FrontConfig swig_types[16]
+#define SWIGTYPE_p_ppc__front__FrontConfigBuilder swig_types[17]
+#define SWIGTYPE_p_ppc__front__GetPeersInfoHandler swig_types[18]
+#define SWIGTYPE_p_ppc__front__IFront swig_types[19]
+#define SWIGTYPE_p_ppc__front__IFrontBuilder swig_types[20]
+#define SWIGTYPE_p_ppc__front__IFrontClient swig_types[21]
+#define SWIGTYPE_p_ppc__front__IMessageHandler swig_types[22]
+#define SWIGTYPE_p_ppc__front__MessageDispatcherHandler swig_types[23]
+#define SWIGTYPE_p_ppc__front__SendResponseHandler swig_types[24]
+#define SWIGTYPE_p_ppc__gateway__IGateway__Ptr swig_types[25]
+#define SWIGTYPE_p_ppc__protocol__EndPoint swig_types[26]
+#define SWIGTYPE_p_ppc__protocol__GrpcConfig swig_types[27]
+#define SWIGTYPE_p_ppc__protocol__GrpcServerConfig swig_types[28]
+#define SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr swig_types[29]
+#define SWIGTYPE_p_ppc__protocol__Message swig_types[30]
+#define SWIGTYPE_p_ppc__protocol__MessageBuilder swig_types[31]
+#define SWIGTYPE_p_ppc__protocol__MessageHeader swig_types[32]
+#define SWIGTYPE_p_ppc__protocol__MessageHeaderBuilder swig_types[33]
+#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeader swig_types[34]
+#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeaderBuilder swig_types[35]
+#define SWIGTYPE_p_ppc__protocol__MessagePayload swig_types[36]
+#define SWIGTYPE_p_ppc__protocol__MessagePayloadBuilder swig_types[37]
+#define SWIGTYPE_p_ppc__protocol__RouteType swig_types[38]
+#define SWIGTYPE_p_ppc__sdk__Transport swig_types[39]
+#define SWIGTYPE_p_ppc__sdk__TransportBuilder swig_types[40]
+#define SWIGTYPE_p_short swig_types[41]
+#define SWIGTYPE_p_signed_char swig_types[42]
+#define SWIGTYPE_p_size_type swig_types[43]
+#define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[44]
+#define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[45]
+#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[46]
+#define SWIGTYPE_p_std__exception swig_types[47]
+#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[48]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[49]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[50]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[51]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[52]
+#define SWIGTYPE_p_std__invalid_argument swig_types[53]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[54]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[55]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[56]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[57]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[58]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[59]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[60]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[61]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[62]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[63]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[64]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[65]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[66]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[67]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[68]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[69]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[70]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[71]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[72]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[73]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[74]
+#define SWIGTYPE_p_std__string swig_types[75]
+#define SWIGTYPE_p_std__string_view swig_types[76]
+#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[77]
+#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[78]
+#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[79]
+#define SWIGTYPE_p_swig__SwigPyIterator swig_types[80]
+#define SWIGTYPE_p_unsigned_char swig_types[81]
+#define SWIGTYPE_p_unsigned_int swig_types[82]
+#define SWIGTYPE_p_unsigned_long_long swig_types[83]
+#define SWIGTYPE_p_unsigned_short swig_types[84]
+#define SWIGTYPE_p_value_type swig_types[85]
+static swig_type_info *swig_types[87];
+static swig_module_info swig_module = {swig_types, 86, 0, 0, 0, 0};
 #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name)
 #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name)
 
@@ -19345,75 +19343,46 @@ SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  std::shared_ptr< bcos::bytes > arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  uint32_t result;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setPayload", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_length" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
       arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
-  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::shared_ptr< bcos::bytes > result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payload" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setPayload" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes >""'");
     }
+    if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
   }
-  result = ((ppc::protocol::Message const *)arg1)->payload();
-  {
-    std::shared_ptr<  bcos::bytes > *smartresult = result ? new std::shared_ptr<  bcos::bytes >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
-  }
+  (arg1)->setPayload(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
@@ -19458,52 +19427,6 @@ SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::shared_ptr< bcos::bytes > arg2 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setPayload", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setPayload" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes >""'");
-    }
-    if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
-  }
-  (arg1)->setPayload(arg2);
-  resultobj = SWIG_Py_Void();
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
@@ -19589,7 +19512,7 @@ SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_encode__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   bcos::bytes *arg2 = 0 ;
@@ -19600,10 +19523,11 @@ SWIGINTERN PyObject *_wrap_Message_encode__SWIG_0(PyObject *self, Py_ssize_t nob
   void *argp2 = 0 ;
   int res2 = 0 ;
   std::shared_ptr< bcos::bytes > tempshared2 ;
+  PyObject *swig_obj[2] ;
   bool result;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_encode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
@@ -19644,25 +19568,26 @@ SWIGINTERN PyObject *_wrap_Message_encode__SWIG_0(PyObject *self, Py_ssize_t nob
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_encode__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  bcos::boostssl::EncodedMsg *arg2 = 0 ;
+  bcos::bytesConstRef arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::protocol::Message > tempshared1 ;
   std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
+  void *argp2 ;
   int res2 = 0 ;
-  bool result;
+  PyObject *swig_obj[2] ;
+  int64_t result;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_decode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_encode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_decode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
@@ -19673,110 +19598,96 @@ SWIGINTERN PyObject *_wrap_Message_encode__SWIG_1(PyObject *self, Py_ssize_t nob
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__boostssl__EncodedMsg,  0 );
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::boostssl::EncodedMsg &""'"); 
-  }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::boostssl::EncodedMsg &""'"); 
+  {
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
   }
-  arg2 = reinterpret_cast< bcos::boostssl::EncodedMsg * >(argp2);
-  result = (bool)(arg1)->encode(*arg2);
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  result = (int64_t)(arg1)->decode(arg2);
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
+SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint32_t result;
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "Message_encode", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_Message_encode__SWIG_0(self, argc, argv);
-      }
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_length" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      void *vptr = 0;
-      int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_bcos__boostssl__EncodedMsg, SWIG_POINTER_NO_NULL);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_Message_encode__SWIG_1(self, argc, argv);
-      }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  
+  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
+  return resultobj;
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'Message_encode'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::Message::encode(bcos::bytes &)\n"
-    "    ppc::protocol::Message::encode(bcos::boostssl::EncodedMsg &)\n");
-  return 0;
+  return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  bcos::bytesConstRef arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  int64_t result;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::shared_ptr< bcos::bytes > result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_decode", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_decode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payload" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
       arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  result = ((ppc::protocol::Message const *)arg1)->payload();
   {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
-    } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
-    }
+    std::shared_ptr<  bcos::bytes > *smartresult = result ? new std::shared_ptr<  bcos::bytes >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
   }
-  result = (int64_t)(arg1)->decode(arg2);
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
   return NULL;
@@ -20542,33 +20453,6 @@ SWIGINTERN PyObject *_wrap_printMessage(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_printWsMessage(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  bcos::boostssl::MessageFace::Ptr *arg1 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_bcos__boostssl__MessageFace__Ptr,  0  | 0);
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printWsMessage" "', argument " "1"" of type '" "bcos::boostssl::MessageFace::Ptr const &""'"); 
-  }
-  if (!argp1) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "printWsMessage" "', argument " "1"" of type '" "bcos::boostssl::MessageFace::Ptr const &""'"); 
-  }
-  arg1 = reinterpret_cast< bcos::boostssl::MessageFace::Ptr * >(argp1);
-  result = ppc::protocol::printWsMessage((bcos::boostssl::MessageFace::Ptr const &)*arg1);
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_delete_MessagePayload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
@@ -24731,14 +24615,14 @@ static PyMethodDef SwigMethods[] = {
 	 { "Message_setExt", _wrap_Message_setExt, METH_VARARGS, NULL},
 	 { "Message_isRespPacket", _wrap_Message_isRespPacket, METH_O, NULL},
 	 { "Message_setRespPacket", _wrap_Message_setRespPacket, METH_O, NULL},
-	 { "Message_length", _wrap_Message_length, METH_O, NULL},
-	 { "Message_payload", _wrap_Message_payload, METH_O, NULL},
-	 { "Message_payloadBuffer", _wrap_Message_payloadBuffer, METH_O, NULL},
 	 { "Message_setPayload", _wrap_Message_setPayload, METH_VARARGS, NULL},
+	 { "Message_payloadBuffer", _wrap_Message_payloadBuffer, METH_O, NULL},
 	 { "Message_setFrontMessage", _wrap_Message_setFrontMessage, METH_VARARGS, NULL},
 	 { "Message_frontMessage", _wrap_Message_frontMessage, METH_O, NULL},
 	 { "Message_encode", _wrap_Message_encode, METH_VARARGS, NULL},
 	 { "Message_decode", _wrap_Message_decode, METH_VARARGS, NULL},
+	 { "Message_length", _wrap_Message_length, METH_O, NULL},
+	 { "Message_payload", _wrap_Message_payload, METH_O, NULL},
 	 { "Message_swigregister", Message_swigregister, METH_O, NULL},
 	 { "delete_MessageHeaderBuilder", _wrap_delete_MessageHeaderBuilder, METH_O, NULL},
 	 { "MessageHeaderBuilder_build", _wrap_MessageHeaderBuilder_build, METH_VARARGS, NULL},
@@ -24751,7 +24635,6 @@ static PyMethodDef SwigMethods[] = {
 	 { "MessageOptionalHeaderBuilder_swigregister", MessageOptionalHeaderBuilder_swigregister, METH_O, NULL},
 	 { "printOptionalField", _wrap_printOptionalField, METH_O, NULL},
 	 { "printMessage", _wrap_printMessage, METH_O, NULL},
-	 { "printWsMessage", _wrap_printWsMessage, METH_O, NULL},
 	 { "delete_MessagePayload", _wrap_delete_MessagePayload, METH_O, NULL},
 	 { "MessagePayload_encode", _wrap_MessagePayload_encode, METH_VARARGS, NULL},
 	 { "MessagePayload_decode", _wrap_MessagePayload_decode, METH_VARARGS, NULL},
@@ -24898,8 +24781,6 @@ static swig_type_info _swigt__p_UniqueConstPtr = {"_p_UniqueConstPtr", "UniqueCo
 static swig_type_info _swigt__p_UniquePtr = {"_p_UniquePtr", "UniquePtr *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_allocator_type = {"_p_allocator_type", "allocator_type *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_bcos__Error = {"_p_bcos__Error", "bcos::Error *", 0, 0, (void*)0, 0};
-static swig_type_info _swigt__p_bcos__boostssl__EncodedMsg = {"_p_bcos__boostssl__EncodedMsg", "bcos::boostssl::EncodedMsg *", 0, 0, (void*)0, 0};
-static swig_type_info _swigt__p_bcos__boostssl__MessageFace__Ptr = {"_p_bcos__boostssl__MessageFace__Ptr", "bcos::boostssl::MessageFace::Ptr *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_bcos__bytesConstRef = {"_p_bcos__bytesConstRef", "bcos::bytesConstRef *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_char = {"_p_char", "char *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_difference_type = {"_p_difference_type", "difference_type *", 0, 0, (void*)0, 0};
@@ -24988,8 +24869,6 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_UniquePtr,
   &_swigt__p_allocator_type,
   &_swigt__p_bcos__Error,
-  &_swigt__p_bcos__boostssl__EncodedMsg,
-  &_swigt__p_bcos__boostssl__MessageFace__Ptr,
   &_swigt__p_bcos__bytesConstRef,
   &_swigt__p_char,
   &_swigt__p_difference_type,
@@ -25078,8 +24957,6 @@ static swig_cast_info _swigc__p_UniqueConstPtr[] = {  {&_swigt__p_UniqueConstPtr
 static swig_cast_info _swigc__p_UniquePtr[] = {  {&_swigt__p_UniquePtr, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_allocator_type[] = {  {&_swigt__p_allocator_type, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_bcos__Error[] = {  {&_swigt__p_bcos__Error, 0, 0, 0},{0, 0, 0, 0}};
-static swig_cast_info _swigc__p_bcos__boostssl__EncodedMsg[] = {  {&_swigt__p_bcos__boostssl__EncodedMsg, 0, 0, 0},{0, 0, 0, 0}};
-static swig_cast_info _swigc__p_bcos__boostssl__MessageFace__Ptr[] = {  {&_swigt__p_bcos__boostssl__MessageFace__Ptr, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_bcos__bytesConstRef[] = {  {&_swigt__p_bcos__bytesConstRef, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_char[] = {  {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_difference_type[] = {  {&_swigt__p_difference_type, 0, 0, 0},{0, 0, 0, 0}};
@@ -25168,8 +25045,6 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_UniquePtr,
   _swigc__p_allocator_type,
   _swigc__p_bcos__Error,
-  _swigc__p_bcos__boostssl__EncodedMsg,
-  _swigc__p_bcos__boostssl__MessageFace__Ptr,
   _swigc__p_bcos__bytesConstRef,
   _swigc__p_char,
   _swigc__p_difference_type,
diff --git a/cpp/wedpr-transport/sdk/src/CMakeLists.txt b/cpp/wedpr-transport/sdk/src/CMakeLists.txt
index 1231ca68..18e585fb 100644
--- a/cpp/wedpr-transport/sdk/src/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk/src/CMakeLists.txt
@@ -1,6 +1,4 @@
 cmake_minimum_required(VERSION 3.14)
-project(ppc-transport-sdk VERSION ${VERSION})
-
 file(GLOB_RECURSE SRCS *.cpp)
 
 add_library(${WEDPR_TRANSPORT_SDK_TARGET} ${SRCS})
diff --git a/python/ppc_scheduler/__init__.py b/python/ppc_scheduler/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/common/__init__.py b/python/ppc_scheduler/common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/common/global_context.py b/python/ppc_scheduler/common/global_context.py
deleted file mode 100644
index 0d96ed01..00000000
--- a/python/ppc_scheduler/common/global_context.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from ppc_scheduler.common.initializer import Initializer
-
-components = Initializer()
\ No newline at end of file
diff --git a/python/ppc_scheduler/common/global_job_manager.py b/python/ppc_scheduler/common/global_job_manager.py
deleted file mode 100644
index 14d91556..00000000
--- a/python/ppc_scheduler/common/global_job_manager.py
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-from ppc_scheduler.job.job_manager import JobManager
-
-
-global_job_manager: JobManager = None
-
-def update_job_manager(job_manager):
-    global global_job_manager
-    global_job_manager = job_manager
-    
-def get_job_manager():
-    global global_job_manager
-    return global_job_manager
-
diff --git a/python/ppc_scheduler/common/initializer.py b/python/ppc_scheduler/common/initializer.py
deleted file mode 100644
index 162feebf..00000000
--- a/python/ppc_scheduler/common/initializer.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import logging
-import logging.config
-import os
-from contextlib import contextmanager
-
-import yaml
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-from ppc_common.deps_services import storage_loader
-from ppc_common.ppc_utils import common_func
-
-class Initializer:
-    def __init__(self):
-        self.config_data = None
-        self.job_cache_dir = None
-        self.config_data = None
-        self.sql_session = None
-        self.sql_engine = None
-        self.storage_client = None
-
-    def init_all(self, config_data):
-        self.config_data = config_data
-        
-        # self.init_log()
-        # self.init_config()
-        self.init_cache()
-        self.init_sql_client()
-        self.init_storage_client()
-    
-    @staticmethod    
-    def init_config(config_path: str):
-        with open(config_path, 'rb') as f:
-            config_data = yaml.safe_load(f.read())
-            return config_data
-        
-    @staticmethod
-    def init_log(log_config_path: str):
-        logging.config.fileConfig(log_config_path)
-
-    def init_cache(self):
-        self.job_cache_dir = common_func.get_config_value(
-            "JOB_TEMP_DIR", "/tmp", self.config_data, False)
-        if not os.path.exists(self.job_cache_dir):
-            os.makedirs(self.job_cache_dir)
-
-    def init_sql_client(self):
-        self.sql_engine = create_engine(self.config_data['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True)
-        self.sql_session = sessionmaker(bind=self.sql_engine)
-
-    @contextmanager
-    def create_sql_session(self):
-        session = self.sql_session()
-        try:
-            yield session
-            session.commit()
-        except:
-            session.rollback()
-            raise
-        finally:
-            session.close()
-
-    def init_storage_client(self):
-        self.storage_client = storage_loader.load(
-            self.config_data, self.logger())
-
-    def update_thread_event_manager(self, thread_event_manager):
-        self.thread_event_manager = thread_event_manager
-
-    @staticmethod
-    def logger(name=None):
-        return logging.getLogger(name)
diff --git a/python/ppc_scheduler/common/log_utils.py b/python/ppc_scheduler/common/log_utils.py
deleted file mode 100644
index 159b9e5c..00000000
--- a/python/ppc_scheduler/common/log_utils.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-
-from ppc_common.ppc_utils import utils, path
-
-
-def job_start_log_info(job_id):
-    return f"=====================start_job_{job_id}====================="
-
-def job_end_log_info(job_id):
-    return f"======================end_job_{job_id}======================"
-
-def worker_start_log_info(work_id):
-    return f"=====================start_work_{work_id}====================="
-
-def worker_end_log_info(work_id):
-    return f"======================end_work_{work_id}======================"
-
-def upload_job_log(storage_client, job_id, extra=None):
-    job_log_path = None
-    try:
-        file_path = path.get_path()
-        job_log_path = utils.get_log_temp_file_path(file_path, job_id)
-        origin_log_path = utils.get_log_file_path(file_path)
-        filter_job_log(job_id, origin_log_path, job_log_path)
-        if extra is not None:
-            job_log = open(job_log_path, 'a+')
-            job_log.write('\n' * 3)
-            job_log.write(extra)
-            job_log.close()
-        storage_client.upload_file(job_log_path, job_id + os.sep + utils.LOG_NAME)
-    except (Exception) as e:
-        print(e)
-    finally:
-        if os.path.exists(job_log_path):
-            os.remove(job_log_path)
-
-
-def read_line_inverse(file):
-    file.seek(0, 2)
-    current_position = file.tell()
-    position = 0
-    while position + current_position >= 0:
-        while True:
-            position -= 1
-            try:
-                file.seek(position, 2)
-                if file.read(1) == b'\n':
-                    break
-            except:
-                # point at file header
-                file.seek(0, 0)
-                break
-        line = file.readline()
-        yield line
-
-
-def filter_job_log(job_id, origin_log_path, job_log_path):
-    origin_log_file = open(origin_log_path, 'rb')
-    line_list = []
-    need_record = False
-
-    # search job log
-    for line in read_line_inverse(origin_log_file):
-        if need_record:
-            line_list.append(line)
-        if not need_record and str(line).__contains__(job_end_log_info(job_id)):
-            need_record = True
-            line_list.append(line)
-        elif str(line).__contains__(job_start_log_info(job_id)):
-            break
-    origin_log_file.close()
-
-    # save log lines into temp file
-    job_log_file = open(job_log_path, 'wb+')
-    job_log_file.writelines(list(reversed(line_list)))
-    job_log_file.close()
diff --git a/python/ppc_scheduler/conf/application-sample.yml b/python/ppc_scheduler/conf/application-sample.yml
deleted file mode 100644
index 6d43e9b3..00000000
--- a/python/ppc_scheduler/conf/application-sample.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-HTTP_HOST: "0.0.0.0"
-HTTP_PORT: 43471
-HTTP_THREAD_NUM: 8
-
-JOB_TIMEOUT_H: 1800
-WORKSPACE: "/data/app/files/job"
-STORAGE_TYPE: "HDFS"
-HDFS_URL: "http://127.0.0.1:50070"
-
-# mysql or dm
-DB_TYPE: "mysql"
-SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4"
-# SQLALCHEMY_DATABASE_URI: "dm+dmPython://ppcv16:ppc12345678@127.0.0.1:5236"
diff --git a/python/ppc_scheduler/conf/logging.conf b/python/ppc_scheduler/conf/logging.conf
deleted file mode 100644
index b472e2ab..00000000
--- a/python/ppc_scheduler/conf/logging.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-[loggers]
-keys=root,wsgi
-
-[logger_root]
-level=INFO
-handlers=consoleHandler,fileHandler
-
-[logger_wsgi]
-level = INFO
-handlers = accessHandler
-qualname = wsgi
-propagate = 0
-
-[handlers]
-keys=fileHandler,consoleHandler,accessHandler
-
-[handler_accessHandler]
-class=handlers.TimedRotatingFileHandler
-args=('./logs/appmonitor.log', 'D', 1, 30, 'utf-8')
-level=INFO
-formatter=simpleFormatter
-
-[handler_fileHandler]
-class=handlers.TimedRotatingFileHandler
-args=('./logs/scheduler.log', 'D', 1, 30, 'utf-8')
-level=INFO
-formatter=simpleFormatter
-
-[handler_consoleHandler]
-class=StreamHandler
-args=(sys.stdout,)
-level=ERROR
-formatter=simpleFormatter
-
-[formatters]
-keys=simpleFormatter
-
-[formatter_simpleFormatter]
-format=[%(levelname)s][%(asctime)s %(msecs)03d][%(process)d][%(filename)s:%(lineno)d] %(message)s
-datefmt=%Y-%m-%d %H:%M:%S
diff --git a/python/ppc_scheduler/database/__init__.py b/python/ppc_scheduler/database/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/database/computing_node_mapper.py b/python/ppc_scheduler/database/computing_node_mapper.py
deleted file mode 100644
index 5b6417a3..00000000
--- a/python/ppc_scheduler/database/computing_node_mapper.py
+++ /dev/null
@@ -1,74 +0,0 @@
-from sqlalchemy import update, and_, select, delete
-
-from ppc_common.db_models.computing_node import ComputingNodeRecord
-
-
-# def insert_computing_node(session, node_id: str, url: str, node_type: str, loading: int):
-#     new_node = ComputingNodeRecord(
-#         id=node_id,
-#         url=url,
-#         type=node_type,
-#         loading=loading
-#     )
-
-#     session.add(new_node)
-
-
-# def delete_computing_node(session, url: str, node_type: str):
-#     stmt = (
-#         delete(ComputingNodeRecord).where(
-#             and_(
-#                 ComputingNodeRecord.url == url,
-#                 ComputingNodeRecord.type == node_type
-#             )
-#         )
-#     )
-
-#     result = session.execute(stmt)
-
-#     return result.rowcount > 0
-
-
-def get_and_update_min_loading_url(session, node_type):
-    
-    # select min_loading node
-    min_loading_node_id = session.query(ComputingNodeRecord.id).filter(
-        ComputingNodeRecord.type == node_type
-    ).order_by(ComputingNodeRecord.loading.asc()).first()
-
-
-    # update loading
-    stmt = (
-        update(ComputingNodeRecord).where(
-            and_(
-                ComputingNodeRecord.id == min_loading_node_id.id,
-                ComputingNodeRecord.type == node_type
-            )
-        ).values(
-            loading=ComputingNodeRecord.loading + 1
-        )
-    )
-    session.execute(stmt)
-
-    # select min_loading node
-    record = session.query(ComputingNodeRecord.url, ComputingNodeRecord.token).filter(
-        ComputingNodeRecord.id == min_loading_node_id.id
-    ).order_by(ComputingNodeRecord.loading.asc()).first()
-    
-    return record
-
-def release_loading(session, url: str, node_type: str):
-    stmt = (
-        update(ComputingNodeRecord).where(
-            and_(
-                ComputingNodeRecord.url == url,
-                ComputingNodeRecord.type == node_type,
-                ComputingNodeRecord.loading > 0
-            )
-        ).values(
-            loading=ComputingNodeRecord.loading - 1
-        )
-    )
-    result = session.execute(stmt)
-
-    return result.rowcount > 0
diff --git a/python/ppc_scheduler/database/job_worker_mapper.py b/python/ppc_scheduler/database/job_worker_mapper.py
deleted file mode 100644
index 0d8a7e8b..00000000
--- a/python/ppc_scheduler/database/job_worker_mapper.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import datetime
-import json
-# from sqlalchemy.dialects.mysql import insert
-from sqlalchemy import and_, update, insert
-from sqlalchemy.exc import NoResultFound
-
-from ppc_common.db_models.job_worker_record import JobWorkerRecord
-from ppc_common.ppc_utils import utils
-from ppc_scheduler.workflow.common import codec
-from ppc_scheduler.workflow.common.worker_status import WorkerStatus
-
-def insert_job_worker(session, worker):
-    upstreams_str = json.dumps(worker['upstreams'])
-    inputs_statement_str = json.dumps(worker['inputs_statement'])
-    args_str=json.dumps(worker['args'])
-    
-    stmt = insert(JobWorkerRecord).prefix_with(" IGNORE").values(
-        worker_id=worker['worker_id'],
-        job_id=worker['job_id'],
-        type=worker['type'],
-        status=WorkerStatus.PENDING,
-        args=args_str,
-        upstreams=upstreams_str,
-        inputs_statement=inputs_statement_str
-    )
-    
-    result = session.execute(stmt)
-    return result.rowcount > 0
-
-def query_job_worker(session, job_id, worker_id):
-    try:
-        return session.query(JobWorkerRecord).filter(
-            and_(JobWorkerRecord.worker_id == worker_id,
-                 JobWorkerRecord.job_id == job_id)).one()
-    except NoResultFound:
-        return None
-
-
-def update_job_worker(session, job_id, worker_id, status, outputs):
-    stmt = (
-        update(JobWorkerRecord).where(
-            and_(
-                JobWorkerRecord.job_id == job_id,
-                JobWorkerRecord.worker_id == worker_id
-            )
-        ).values(
-            status=status,
-            outputs=codec.serialize_worker_outputs(outputs)
-        )
-    )
-    result = session.execute(stmt)
-    return result.rowcount > 0
diff --git a/python/ppc_scheduler/demo/default_flow_config_sample.py b/python/ppc_scheduler/demo/default_flow_config_sample.py
deleted file mode 100644
index 123df9c2..00000000
--- a/python/ppc_scheduler/demo/default_flow_config_sample.py
+++ /dev/null
@@ -1,224 +0,0 @@
-flow_dict = {
-
-    "SHELL": [
-        {
-            "index": 1,
-            "type": "T_SHELL"
-        },
-        {
-            "index": 2,
-            "type": "T_SHELL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        },
-        {
-            "index": 3,
-            "type": "T_SHELL",
-            "upstreams": [
-                {
-                    "index": 2
-                }
-            ]
-        }
-    ],
-
-    "PSI": [
-        {
-            "index": 1,
-            "type": "T_PSI"
-        }
-    ],
-
-    "MPC": [
-        {
-            "index": 1,
-            "type": "T_MPC"
-        }
-    ],
-
-    "PSI_MPC": [
-        {
-            "index": 1,
-            "type": "T_PSI"
-        },
-        {
-            "index": 2,
-            "type": "T_MPC",
-            "upstreams": [
-                {
-                    "index": 1,
-                    "output_input_map": [
-                        "0:0"
-                    ]
-                }
-            ]
-        }
-    ],
-
-    "PREPROCESSING": [
-        {
-            "index": 1,
-            "type": "T_MODEL"
-        }
-    ],
-
-    "FEATURE_ENGINEERING": [
-        {
-            "index": 1,
-            "type": "T_MODEL"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        }
-    ],
-
-    "TRAINING": [
-        {
-            "index": 1,
-            "type": "T_MODEL"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        }
-    ],
-
-    "PREDICTION": [
-        {
-            "index": 1,
-            "type": "T_MODEL"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        }
-    ],
-
-    "FEATURE_ENGINEERING_TRAINING": [
-        {
-            "index": 1,
-            "type": "T_MODEL"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        },
-        {
-            "index": 3,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 2
-                }
-            ]
-        }
-    ],
-
-    "PSI_FEATURE_ENGINEERING": [
-        {
-            "index": 1,
-            "type": "T_PSI"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        },
-        {
-            "index": 3,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 2
-                }
-            ]
-        }
-    ],
-
-    "PSI_TRAINING": [
-        {
-            "index": 1,
-            "type": "T_PSI"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        },
-        {
-            "index": 3,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 2
-                }
-            ]
-        }
-    ],
-
-    "PSI_FEATURE_ENGINEERING_TRAINING": [
-        {
-            "index": 1,
-            "type": "T_PSI"
-        },
-        {
-            "index": 2,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 1
-                }
-            ]
-        },
-        {
-            "index": 3,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 2
-                }
-            ]
-        },
-        {
-            "index": 4,
-            "type": "T_MODEL",
-            "upstreams": [
-                {
-                    "index": 3
-                }
-            ]
-        }
-    ]
-}
diff --git a/python/ppc_scheduler/demo/prefect_workflow_sample.py b/python/ppc_scheduler/demo/prefect_workflow_sample.py
deleted file mode 100644
index 9a146260..00000000
--- a/python/ppc_scheduler/demo/prefect_workflow_sample.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import time
-from prefect import Task, Flow
-from prefect.triggers import all_successful, any_failed
-from prefect import Flow
-from prefect.executors import LocalDaskExecutor
-
-class SuccessTask(Task):
-    def run(self):
-        print(" ===>>>> Success Task Ran")
-
-class FailTask(Task):
-    def run(self):
-        print(" ===>>> Fail Task Ran")
-
-class JobTask(Task):
-    def __init__(self, name):
-        super().__init__(name=name)
-        self.name = name
-    
-    def run(self, x, y):
-        print(" ==> " + self.name + " is running, " + str(x) + ", " + str(y))
-        return x + y
-
-# with Flow("My Flow") as f:
-#     t1 = a(1, 2) # t1 != a
-#     t2 = a(5, 7) # t2 != a
-
-with Flow("example_flow") as flow:
-    
-    task1 = JobTask("t1")
-    task2 = JobTask("t2")
-    task3 = JobTask("t3")
-    task4 = JobTask("t4")
-    
-    # task1=t1(1,2)
-    # task2=t2(3,4)
-    # task3=t3(4,5)
-    # task4=t4(6,7)
-    task1.bind(x = 1, y = 2)
-    task2.bind(x = 3, y = 4)
-    task3.bind(x = 5, y = 6)
-    task4.bind(x = 7, y = 8)
-
-    task3.set_upstream(task1, flow=flow)
-    task3.set_upstream(task2, flow=flow)
-    task4.set_upstream(task3, flow=flow)
-    
-    # 定义成功任务，仅当所有上游任务成功时才运行
-    success_task = SuccessTask(name="Success Task")
-    success_task.trigger = all_successful
-    
-    # 定义失败任务，只要任何一个上游任务失败就运行
-    fail_task = FailTask(name="Fail Task")
-    fail_task.trigger = any_failed
-    
-    # 设置依赖关系
-    task1.set_downstream(success_task, flow=flow)
-    task2.set_downstream(success_task, flow=flow)
-    task3.set_downstream(success_task, flow=flow)
-    task4.set_downstream(success_task, flow=flow)
-    
-    task1.set_downstream(fail_task, flow=flow)
-    task2.set_downstream(fail_task, flow=flow)
-    task3.set_downstream(fail_task, flow=flow)
-    task4.set_downstream(fail_task, flow=flow)
-
-
-# flow.executor = LocalDaskExecutor()
-
-start_time = time.time()
-# 运行工作流
-flow_state=flow.run()
-
-end_time = time.time()
-print(f" ## costs: {end_time - start_time}")
-
-print(flow_state.result)
-
-flow.visualize(flow_state, "./my_flow", 'svg')
\ No newline at end of file
diff --git a/python/ppc_scheduler/demo/web_submit_job.json b/python/ppc_scheduler/demo/web_submit_job.json
deleted file mode 100644
index f5b6d9cd..00000000
--- a/python/ppc_scheduler/demo/web_submit_job.json
+++ /dev/null
@@ -1,134 +0,0 @@
-[
-    {
-      "job": {
-        "jobType": "PSI",
-        "projectName": "octo-test",
-        "param": "{\"dataSetList\":[{\"idFields\":[\"x200\"],\"dataset\":{\"owner\":\"octo1\",\"ownerAgency\":\"WeBank\",\"path\":\"/user/ppc/webank/octo1/d-9323009025845253\",\"storageTypeStr\":\"HDFS\",\"datasetID\":\"d-9323009025845253\"},\"receiveResult\":false},{\"idFields\":[\" x106\"],\"dataset\":{\"owner\":\"octopuswang1\",\"ownerAgency\":\"SGD\",\"path\":\"/user/ppc/sgd/octopuswang1/d-9322966927878149\",\"storageTypeStr\":\"HDFS\",\"datasetID\":\"d-9322966927878149\"},\"receiveResult\":true}]}"
-      },
-      "taskParties": [
-        {
-          "userName": "octo1",
-          "agency": "WeBank"
-        },
-        {
-          "userName": "octopuswang1",
-          "agency": "SGD"
-        }
-      ],
-      "datasetList": [
-        "d-xxx",
-        "d-yyy"
-      ]
-    },
-    {
-      "dataSetList": [
-        {
-          "idFields": [
-            "x200"
-          ],
-          "dataset": {
-            "owner": "octo1",
-            "ownerAgency": "WeBank",
-            "path": "/user/ppc/webank/octo1/d-9323009025845253",
-            "storageTypeStr": "HDFS",
-            "datasetID": "d-9323009025845253"
-          },
-          "receiveResult": false
-        },
-        {
-          "idFields": [
-            " x106"
-          ],
-          "dataset": {
-            "owner": "octopuswang1",
-            "ownerAgency": "SGD",
-            "path": "/user/ppc/sgd/octopuswang1/d-9322966927878149",
-            "storageTypeStr": "HDFS",
-            "datasetID": "d-9322966927878149"
-          },
-          "receiveResult": true
-        }
-      ]
-    }
-  ]
-
-  {
-    "jsonrpc": "1",
-    "method": "asyncRunTask",
-    "token": "",
-    "id": 61480,
-    "params": {
-      "taskID": "9441012273129477-PSI",
-      "type": 0,
-      "algorithm": 0,
-      "syncResult": true,
-      "lowBandwidth": false,
-      "parties": [
-        {
-          "id": "WeBank",
-          "partyIndex": 0,
-          "data": {
-            "id": "9441012273129477",
-            "input": {
-              "datasetID": "d-9431392664627205",
-              "type": 2,
-              "storageTypeStr": "HDFS",
-              "path": "/user/ppc/webank/octo1/d-9323009025845253",
-              "owner": "octo1",
-              "ownerAgency": "WeBank",
-              "storagePath": {
-                "storageType": "HDFS",
-                "filePath": "/user/ppc/webank/octo1/d-9323009025845253"
-              }
-            },
-            "output": {
-              "datasetID": null,
-              "type": 2,
-              "storageTypeStr": "HDFS",
-              "path": "/user/wedpr//home/cache/octo1/9441012273129477/psi_result.csv",
-              "owner": "octo1",
-              "ownerAgency": "WeBank",
-              "storagePath": {
-                "storageType": "HDFS",
-                "filePath": "/user/wedpr/home/cache/octo1/9441012273129477/psi_result.csv"
-              }
-            }
-          }
-        },
-        {
-          "id": "SGD",
-          "partyIndex": 0,
-          "data": {
-            "id": "9441012273129477",
-            "input": {
-              "datasetID": "d-9431392714827781",
-              "type": 2,
-              "storageTypeStr": "HDFS",
-              "path": "/user/ppc/webank/octo1/d-9323009025845253",
-              "owner": "octopuswang1",
-              "ownerAgency": "SGD",
-              "storagePath": {
-                "storageType": "HDFS",
-                "filePath": "/user/ppc/webank/octo1/d-9323009025845253"
-              }
-            },
-            "output": {
-              "datasetID": null,
-              "type": 2,
-              "storageTypeStr": "HDFS",
-              "path": "/user/wedpr//home/cache/octopuswang1/9441012273129477/psi_result.csv",
-              "owner": "octopuswang1",
-              "ownerAgency": "SGD",
-              "storagePath": {
-                "storageType": "HDFS",
-                "filePath": "/user/wedpr/home/cache/octopuswang1/9441012273129477/psi_result.csv"
-              }
-            }
-          }
-        }
-      ],
-      "receiverList": [
-        "SGD"
-      ]
-    }
-  }
\ No newline at end of file
diff --git a/python/ppc_scheduler/demo/workflow_args_sample.json b/python/ppc_scheduler/demo/workflow_args_sample.json
deleted file mode 100644
index d77a9c1c..00000000
--- a/python/ppc_scheduler/demo/workflow_args_sample.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
-  "jobId": "x12345678",
-  "agency": "WeBank",
-  "workflow": [
-    {
-      "index": 1,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ]
-    },
-    {
-      "index": 2,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ],
-      "upstreams": [
-        {
-          "index": 1
-        }
-      ]
-    },
-    {
-      "index": 3,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ],
-      "upstreams": [
-        {
-          "index": 2
-        }
-      ]
-    },
-    {
-      "index": 4,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ],
-      "upstreams": [
-        {
-          "index": 3
-        }
-      ]
-    },
-    {
-      "index": 5,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ],
-      "upstreams": [
-        {
-          "index": 4
-        }
-      ]
-    },
-    {
-      "index": 6,
-      "type": "SHELL",
-      "args": [
-        "sleep 10"
-      ],
-      "upstreams": [
-        {
-          "index": 5
-        }
-      ]
-    }
-  ]
-}
\ No newline at end of file
diff --git a/python/ppc_scheduler/endpoints/__init__.py b/python/ppc_scheduler/endpoints/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/endpoints/body_schema.py b/python/ppc_scheduler/endpoints/body_schema.py
deleted file mode 100644
index 01ad24b8..00000000
--- a/python/ppc_scheduler/endpoints/body_schema.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-import json
-
-from flask_restx import fields
-
-from ppc_model.network.http.restx import api
-
-response_base = api.model('Response base info', {
-    'errorCode': fields.Integer(description='return code'),
-    'message': fields.String(description='return message')
-})
-
-response_job_status = api.inherit('Task status', response_base, {
-    'data': fields.Raw(description='Task status data as key-value dictionary', example={
-        'status': 'RUNNING',
-        'time_costs': '30s'
-    })
-})
diff --git a/python/ppc_scheduler/endpoints/job_controller.py b/python/ppc_scheduler/endpoints/job_controller.py
deleted file mode 100644
index dbf8d9cb..00000000
--- a/python/ppc_scheduler/endpoints/job_controller.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from flask import request
-from flask_restx import Resource
-
-from ppc_common.ppc_utils import utils
-from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException
-from ppc_scheduler.common.global_context import components
-from ppc_scheduler.endpoints.body_schema import response_job_status, response_base
-from ppc_scheduler.endpoints.restx import api
-from ppc_scheduler.common.global_job_manager import get_job_manager
-
-ns = api.namespace('/wedpr/v3/scheduler/job',
-                   description='Operations related to run job')
-
-
-@ns.route('/')
-class JobCollection(Resource):
-
-    @api.response(200, 'Job started successfully.', response_base)
-    def post(self, job_id):
-        """
-        Run a specific job by job_id.
-        """
-        request_body = request.get_json()
-        
-        if 'jobId' not in request_body:
-            raise PpcException(
-                    PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(),
-                    f"Missing 'jobId' in request")
-        # if 'agency' not in request_body:
-        #     raise PpcException(
-        #             PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(),
-        #             f"Missing 'agency' in request")
-        if 'workflow' not in request_body:
-            raise PpcException(
-                    PpcErrorCode.PARAMETER_CHECK_ERROR.get_code(),
-                    f"Missing 'workflow' in request")
-        
-        components.logger().info(f"Recv run job request, job_id: {job_id}, request: {request_body}")
-        
-        get_job_manager().run_task(job_id, request_body)
-        return utils.BASE_RESPONSE
-
-    @api.response(200, 'Job status retrieved successfully.', response_job_status)
-    def get(self, job_id):
-        """
-        Get the status of a specific job by job_id.
-        """
-        response = utils.BASE_RESPONSE
-        
-        components.logger().info(f"Recv query job request, job_id: {job_id}")
-        
-        status, time_costs = get_job_manager().status(job_id)
-        response['data'] = {
-            'status': status,
-            'time_costs': time_costs
-        }
-        return response
-
-    @api.response(200, 'Job killed successfully.', response_base)
-    def delete(self, job_id):
-        """
-        Kill a specific job by job_id.
-        """
-        
-        components.logger().info(f"receive kill job request, job_id: {job_id}")
-        
-        get_job_manager().kill_job(job_id)
-        return utils.BASE_RESPONSE
diff --git a/python/ppc_scheduler/endpoints/restx.py b/python/ppc_scheduler/endpoints/restx.py
deleted file mode 100644
index 28f9829f..00000000
--- a/python/ppc_scheduler/endpoints/restx.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from flask_restx import Api
-
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-from ppc_model.common.global_context import components
-
-authorizations = {
-    'apikey': {
-        'type': 'apiKey',
-        'in': 'header',
-        'name': 'Authorization'
-    }
-}
-
-api = Api(version='1.0', title='Ppc Scheduler Service',
-          authorizations=authorizations, security='apikey')
-
-
-@api.errorhandler(PpcException)
-def default_error_handler(e):
-    components.logger().exception(e)
-    info = e.to_dict()
-    response = {'errorCode': info['code'], 'message': info['message']}
-    components.logger().error(f"OnError: code: {info['code']}, message: {info['message']}")
-    return response, 500
-
-
-@api.errorhandler(BaseException)
-def default_error_handler(e):
-    components.logger().exception(e)
-    message = 'unknown error.'
-    response = {'errorCode': PpcErrorCode.INTERNAL_ERROR, 'message': message}
-    components.logger().error(f"OnError: unknown error")
-    return response, 500
diff --git a/python/ppc_scheduler/job/__init__.py b/python/ppc_scheduler/job/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/job/job_manager.py b/python/ppc_scheduler/job/job_manager.py
deleted file mode 100644
index 05e85a00..00000000
--- a/python/ppc_scheduler/job/job_manager.py
+++ /dev/null
@@ -1,154 +0,0 @@
-import datetime
-import threading
-import time
-from typing import Union
-
-from readerwriterlock import rwlock
-
-from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-from ppc_scheduler.common import log_utils
-from ppc_scheduler.job.job_status import JobStatus
-from ppc_scheduler.workflow.scheduler.scheduler import Scheduler
-from ppc_scheduler.workflow.builder.flow_builder import FlowBuilder
-from ppc_scheduler.workflow.common.job_context import JobContext
-
-
-class JobManager:
-    def __init__(self, logger,
-                 scheduler: Scheduler,
-                 thread_event_manager: ThreadEventManager,
-                 workspace: str,
-                 job_timeout_h: Union[int, float]):
-        self.logger = logger
-        self._thread_event_manager = thread_event_manager
-        self._scheduler = scheduler
-        self._workspace = workspace
-        self._job_timeout_s = job_timeout_h * 3600
-        self._rw_lock = rwlock.RWLockWrite()
-        self._jobs: dict[str, list] = {}
-        self._flow_builder = FlowBuilder(logger=logger)
-        self._async_executor = AsyncThreadExecutor(
-            event_manager=self._thread_event_manager, logger=logger)
-        self._cleanup_thread = threading.Thread(target=self._loop_action)
-        self._cleanup_thread.daemon = True
-        self._cleanup_thread.start()
-
-    def run_task(self, job_id, request_body):
-        """
-        run task
-        param job_id: job id
-        param request: job params
-        """
-        # TODO: The database persists job information
-        with self._rw_lock.gen_wlock():
-            if job_id in self._jobs:
-                self.logger.info(
-                    f"Job already exists, job_id: {job_id}, status: {self._jobs[job_id][0]}")
-                return
-            self._jobs[job_id] = [
-                JobStatus.RUNNING, datetime.datetime.now(), 0]
-        self.logger.info(log_utils.job_start_log_info(job_id))
-
-        # Create job context
-        job_context = JobContext.create_job_context(
-            request_body, self._workspace)
-        # Build job workflow
-        flow_context = self._flow_builder.build_flow_context(
-            job_id=job_context.job_id, workflow_configs=job_context.workflow_configs)
-        # Persistent workflow
-        self._flow_builder.save_flow_context(job_context.job_id, flow_context)
-        # Run workflow
-        self._async_executor.execute(
-            job_id, self._run_job_flow, self._on_task_finish, (job_context, flow_context))
-
-    def _run_job_flow(self, job_context, flow_context):
-        """
-        run job flow
-        """
-
-        # the scheduler module starts scheduling tasks
-        self._scheduler.run(job_context, flow_context)
-
-    def kill_job(self, job_id: str):
-        with self._rw_lock.gen_rlock():
-            if job_id not in self._jobs or self._jobs[job_id][0] != JobStatus.RUNNING:
-                return
-
-        self.logger.info(f"Kill job, job_id: {job_id}")
-        self._async_executor.kill(job_id)
-
-        with self._rw_lock.gen_wlock():
-            self._jobs[job_id][0] = JobStatus.FAILURE
-
-    def status(self, job_id: str) -> tuple[str, float]:
-        """
-        query task status
-        return: task status and task cost(s)
-        """
-        with self._rw_lock.gen_rlock():
-            if job_id not in self._jobs:
-                raise PpcException(
-                    PpcErrorCode.JOB_NOT_FOUND.get_code(),
-                    PpcErrorCode.JOB_NOT_FOUND.get_msg())
-            status = self._jobs[job_id][0]
-            time_costs = self._jobs[job_id][2]
-            return status, time_costs
-
-    def _on_task_finish(self, job_id: str, is_succeeded: bool, e: Exception = None):
-        with self._rw_lock.gen_wlock():
-            time_costs = (datetime.datetime.now() -
-                          self._jobs[job_id][1]).total_seconds()
-            self._jobs[job_id][2] = time_costs
-            if is_succeeded:
-                self._jobs[job_id][0] = JobStatus.SUCCESS
-                self.logger.info(
-                    f"Job {job_id} completed, time_costs: {time_costs}s")
-            else:
-                self._jobs[job_id][0] = JobStatus.FAILURE
-                self.logger.warn(
-                    f"Job {job_id} failed, time_costs: {time_costs}s, error: {e}")
-            self.logger.info(log_utils.job_end_log_info(job_id))
-
-    def _loop_action(self):
-        while True:
-            time.sleep(20)
-            self._terminate_timeout_jobs()
-            # TODO: store into the database
-            # self._cleanup_finished_jobs()
-            self._report_jobs()
-
-    def _terminate_timeout_jobs(self):
-        jobs_to_kill = []
-        with self._rw_lock.gen_rlock():
-            for job_id, value in self._jobs.items():
-                alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
-                if alive_time >= self._job_timeout_s and value[0] == JobStatus.RUNNING:
-                    jobs_to_kill.append(job_id)
-
-        for job_id in jobs_to_kill:
-            self.logger.warn(f"Job is timeout, job_id: {job_id}")
-            self.kill_job(job_id)
-
-    def _cleanup_finished_jobs(self):
-        jobs_to_cleanup = []
-        with self._rw_lock.gen_rlock():
-            for job_id, value in self._jobs.items():
-                alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
-                if alive_time >= self._job_timeout_s + 3600:
-                    jobs_to_cleanup.append((job_id, value[3]))
-                    self.logger.info(f"Job is finished, job_id: {job_id}")
-        with self._rw_lock.gen_wlock():
-            for job_id, job_id in jobs_to_cleanup:
-                if job_id in self._jobs:
-                    del self._jobs[job_id]
-                self._thread_event_manager.remove_event(job_id)
-                self.logger.info(f"Cleanup job cache, job_id: {job_id}")
-
-    def _report_jobs(self):
-        with self._rw_lock.gen_rlock():
-            job_count = len(self._jobs)
-            self.logger.info(f" ## Report job count: {job_count}")
diff --git a/python/ppc_scheduler/job/job_status.py b/python/ppc_scheduler/job/job_status.py
deleted file mode 100644
index c9cd8fa2..00000000
--- a/python/ppc_scheduler/job/job_status.py
+++ /dev/null
@@ -1,5 +0,0 @@
-class JobStatus:
-    RUNNING = 'RUNNING'
-    SUCCESS = 'SUCCESS'
-    FAILURE = 'FAILED'
-    KILLED = 'KILLED'
diff --git a/python/ppc_scheduler/job/job_type.py b/python/ppc_scheduler/job/job_type.py
deleted file mode 100644
index 22e67f02..00000000
--- a/python/ppc_scheduler/job/job_type.py
+++ /dev/null
@@ -1,12 +0,0 @@
-
-class JobType:
-    PSI = "PSI"
-    MPC = "MPC"
-    PREPROCESSING = 'PREPROCESSING'
-    FEATURE_ENGINEERING = 'FEATURE_ENGINEERING'
-    TRAINING = 'TRAINING'
-    PREDICTION = 'PREDICTION'
-    
-    SHELL = "SHELL"
-    PYTHON = "PYTHON"
-    API = "API"
diff --git a/python/ppc_scheduler/mpc_generator/__init__.py b/python/ppc_scheduler/mpc_generator/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/mpc_generator/generator.py b/python/ppc_scheduler/mpc_generator/generator.py
deleted file mode 100644
index 86466192..00000000
--- a/python/ppc_scheduler/mpc_generator/generator.py
+++ /dev/null
@@ -1,468 +0,0 @@
-from enum import Enum
-
-import sqlparse
-import sqlvalidator
-from sqlparse.exceptions import SQLParseError
-from sqlparse.sql import Comparison, Identifier, Function
-from sqlparse.tokens import Punctuation, Operator, Name, Token
-
-from ppc_scheduler.mpc_generator import mpc_func_str
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-from ppc_common.ppc_utils.utils import PPC_RESULT_FIELDS_FLAG, PPC_RESULT_VALUES_FLAG
-
-
-class SqlPattern(Enum):
-    BASIC_ARITH_OPE = 1
-    AGGR_FUNC_ONLY = 2
-    AGGR_FUNC_WITH_GROUP_BY = 3
-
-
-INDENT = "    "
-
-SUPPORTED_KEYWORDS = [
-    'SELECT',
-    'FROM',
-    'WHERE',
-    'JOIN',
-    'INNER JOIN',
-    'ON',
-    'AS',
-    'GROUP BY',
-    'COUNT',
-    'SUM',
-    'AVG',
-    'MAX',
-    'MIN',
-]
-
-VALUE_TYPE = 'pfix'
-
-GROUP_BY_COLUMN_CODE = 'group_indexes_key[i]'
-
-DISPLAY_FIELDS_FUNC = 'set_display_field_names'
-
-DISPLAY_RESULT_FUNC = 'display_data'
-
-
-class CodeGenerator:
-
-    def __init__(self, sql_str):
-        self.sql_str = sql_str
-
-        # three patterns supported
-        self.sql_pattern = SqlPattern.BASIC_ARITH_OPE
-
-        # based on ID only
-        self.need_run_psi = False
-
-        # record dataset sources
-        self.table_set = set()
-
-        # 0: table number, 1: column index
-        self.group_by_column = []
-
-        # filter conditions
-        self.condition_list = []
-
-        self.result_fields = []
-
-        self.max_column_index = []
-
-    def sql_to_mpc_code(self):
-        operators = self.pre_parsing()
-        format_sql_str = sqlparse.format(self.sql_str, reindent=True, keyword_case='upper')
-        mpc_str = self.generate_common_code(format_sql_str)
-        mpc_str = self.generate_function_code(mpc_str)
-        mpc_str = self.generate_result_calculation_code(operators, mpc_str)
-        mpc_str = self.generate_result_print_code(mpc_str)
-        mpc_str = self.generate_mpc_execution_code(mpc_str)
-        mpc_str = self.replace_max_filed(mpc_str)
-        return mpc_str
-
-    def pre_parsing(self):
-        try:
-            if not sqlvalidator.parse(self.sql_str).is_valid():
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement")
-
-            # format sql str
-            format_sql_str = sqlparse.format(self.sql_str, reindent=True, keyword_case='upper')
-
-            tokens = sqlparse.parse(format_sql_str)[0].tokens
-
-            # warning unsupported keywords
-            self.check_sql_tokens(tokens)
-
-            # parse table aliases
-            aliases = self.find_table_alias(tokens, {}, False)
-
-            # recover table aliases
-            new_sql_str = self.recover_table_name(tokens, aliases, '')
-            format_new_sql_str = sqlparse.format(new_sql_str, reindent=True, keyword_case='upper')
-            tokens = sqlparse.parse(format_new_sql_str)[0].tokens
-
-            # parse filters (only 'id' based column alignment is supported currently)
-            self.find_table_and_condition(tokens, False)
-
-            # check table suffix and number of participants
-            self.check_table()
-
-            # ensure that all tables participate in alignment
-            self.check_table_alignment(self.need_run_psi, len(self.table_set))
-
-            self.max_column_index = [0 for _ in range(len(self.table_set))]
-
-            self.check_sql_pattern(tokens)
-
-            operators = self.extract_operators(format_new_sql_str)
-
-            return operators
-        except SQLParseError:
-            raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement")
-
-    def check_sql_tokens(self, tokens):
-        for token in tokens:
-            if token.is_keyword and token.value not in SUPPORTED_KEYWORDS:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), f"keyword '{token.value}' not supported")
-            if hasattr(token, 'tokens'):
-                self.check_sql_tokens(token.tokens)
-
-    def find_table_alias(self, tokens, aliases, after_from):
-        end_current_round = False
-        for i in range(len(tokens)):
-            if after_from and tokens[i].value == 'AS':
-                # find a alias
-                end_current_round = True
-                aliases[tokens[i + 2].value] = tokens[i - 2].value
-        for i in range(len(tokens)):
-            if tokens[i].value == 'FROM':
-                after_from = True
-            if after_from and not end_current_round and hasattr(tokens[i], 'tokens'):
-                self.find_table_alias(tokens[i].tokens, aliases, after_from)
-        return aliases
-
-    def recover_table_name(self, tokens, aliases, format_sql_str):
-        for i in range(len(tokens)):
-            if tokens[i].value == 'AS' and tokens[i + 2].value in aliases:
-                break
-            elif not tokens[i].is_group:
-                if tokens[i].value in aliases:
-                    format_sql_str += aliases[tokens[i].value]
-                else:
-                    format_sql_str += tokens[i].value
-            elif hasattr(tokens[i], 'tokens'):
-                format_sql_str = self.recover_table_name(tokens[i].tokens, aliases, format_sql_str)
-        return format_sql_str
-
-    def find_table_and_condition(self, tokens, after_from):
-        for token in tokens:
-            if token.value == 'FROM':
-                after_from = True
-            if after_from:
-                if type(token) == Comparison:
-                    self.check_equal_comparison(token.tokens)
-                    self.condition_list.append(token.value)
-                if type(token) == Identifier and '.' not in token.value:
-                    self.table_set.add(token.value)
-                elif hasattr(token, 'tokens'):
-                    self.find_table_and_condition(token.tokens, after_from)
-
-    def check_equal_comparison(self, tokens):
-        for i in range(len(tokens)):
-            if tokens[i].value == '=':
-                self.need_run_psi = True
-            elif tokens[i].value == '.' and tokens[i + 1].value != 'id':
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(),
-                                   f"only 'id' based column alignment is supported currently")
-            elif hasattr(tokens[i], 'tokens'):
-                self.check_equal_comparison(tokens[i].tokens)
-
-    def check_table_alignment(self, has_equal_comparison, table_count):
-        if has_equal_comparison:
-            column = self.condition_list[0].split('=')[0].strip()
-            table = column[0:column.find('.')]
-
-            # all tables must be aligned
-            self.equal_comparison_dfs(table, [0 for _ in range(len(self.condition_list))])
-            if len(self.table_set) != table_count:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "all tables must be aligned")
-
-    def equal_comparison_dfs(self, table, flag_array):
-        for i in range(len(self.condition_list)):
-            if flag_array[i] == 0 and table in self.condition_list[i]:
-                flag_array[i] = 1
-                columns = self.condition_list[i].split('=')
-                for column in columns:
-                    table = column[0:column.find('.')].strip()
-                    self.table_set.add(table)
-                    self.equal_comparison_dfs(table, flag_array)
-
-    def check_table(self):
-        table_count = len(self.table_set)
-        if table_count > 5 or table_count < 2:
-            raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "source must be 2 to 5 parties")
-        suffixes = set()
-        for table in self.table_set:
-            suffixes.add(table[-1])
-        for i in range(table_count):
-            if str(i) not in suffixes:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "invalid suffix of table name")
-
-    def update_max_field(self, table_number, field_number):
-        if field_number > self.max_column_index[table_number]:
-            self.max_column_index[table_number] = field_number
-
-    def replace_max_filed(self, mpc_str):
-        for i in range(len(self.max_column_index)):
-            mpc_str = mpc_str.replace(f'$(source{i}_column_count)', str(self.max_column_index[i] + 1))
-        return mpc_str
-
-    def check_sql_pattern(self, tokens):
-        for i in range(len(tokens)):
-            if tokens[i].value == 'GROUP BY':
-                self.sql_pattern = SqlPattern.AGGR_FUNC_WITH_GROUP_BY
-                items = tokens[i + 2].value.split('.')
-                self.group_by_column = [int(items[0][-1]), get_column_number(items[1])]
-            elif type(tokens[i]) == Function:
-                self.sql_pattern = SqlPattern.AGGR_FUNC_ONLY
-            elif hasattr(tokens[i], 'tokens'):
-                self.check_sql_pattern(tokens[i].tokens)
-
-    def extract_operators(self, format_sql_str):
-        start = format_sql_str.find("SELECT")
-        end = format_sql_str.find("FROM")
-        operators = format_sql_str[start + 6:end].split(',')
-        for i in range(len(operators)):
-            if ' AS ' in operators[i]:
-                index = operators[i].find(' AS ')
-                self.result_fields.append(operators[i][index + 4:].strip().strip('\n').strip('\'').strip('\"').strip())
-                operators[i] = operators[i][:index].strip()
-            else:
-                self.result_fields.append(f"result{i}")
-                operators[i] = operators[i].strip()
-            if ' ' in self.result_fields[-1]:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "result field cannot contain space")
-        return operators
-
-    def generate_common_code(self, format_sql_str):
-        table_count = len(self.table_set)
-        result_column_count = len(self.result_fields)
-
-        if self.need_run_psi:
-            mpc_str = '# PSI_OPTION=True\n\n'
-        else:
-            mpc_str = ''
-        mpc_str += '# BIT_LENGTH = 128\n\n'
-        mpc_str += '# This file is generated automatically by ams\n'
-        mpc_str += f"'''\n{format_sql_str}\n'''\n\n"
-        mpc_str += "from ppc import *\n\n"
-        # mpc_str += "program.use_trunc_pr = True\n"
-        # mpc_str += "program.use_split(3)\n"
-        mpc_str += "n_threads = 8\n"
-        mpc_str += f"value_type = {VALUE_TYPE}\n\n"
-        if VALUE_TYPE == 'pfix':
-            mpc_str += f"pfix.set_precision(16, 47)\n\n"
-
-        for i in range(table_count):
-            mpc_str += f"SOURCE{i} = {i}\n"
-            mpc_str += f"source{i}_record_count = $(source{i}_record_count)\n"
-            mpc_str += f"source{i}_column_count = $(source{i}_column_count)\n"
-            mpc_str += f"source{i}_record = Matrix(source{i}_record_count, source{i}_column_count, value_type)\n\n"
-
-        if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE:
-            mpc_str += "# basic arithmetic operation means that all parties have same number of record\n"
-            mpc_str += "result_record = $(source0_record_count)\n"
-            mpc_str += f"results = Matrix(result_record, {result_column_count}, value_type)\n\n\n"
-        elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-            mpc_str += f"results = Matrix(1, {result_column_count}, value_type)\n\n\n"
-        elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-            mpc_str += "# group by means all parties have same number of record\n"
-            mpc_str += "source_record_count = $(source0_record_count)\n"
-            mpc_str += "result_record = cint(source_record_count)\n"
-            mpc_str += f"results = Matrix(source_record_count, {result_column_count}, value_type)\n\n\n"
-
-        mpc_str += "def read_data_collection(data_collection, party_id):\n"
-        mpc_str += f"{INDENT}if data_collection.sizes[0] > 0:\n"
-        mpc_str += f"{INDENT}{INDENT}data_collection.input_from(party_id)\n\n\n"
-
-        return mpc_str
-
-    def generate_function_code(self, mpc_str):
-        if self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-            mpc_str += mpc_func_str.FUNC_COMPUTE_SUM
-            mpc_str += mpc_func_str.FUNC_COMPUTE_COUNT
-            mpc_str += mpc_func_str.FUNC_COMPUTE_AVG
-            mpc_str += mpc_func_str.FUNC_COMPUTE_MAX
-            mpc_str += mpc_func_str.FUNC_COMPUTE_MIN
-        elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-            mpc_str += mpc_func_str.GROUP_BY_GLOBAL_VARIABLE
-            mpc_str += mpc_func_str.FUNC_COMPUTE_GROUP_BY_INDEXES
-            mpc_str += mpc_func_str.FUNC_COMPUTE_SUM_WITH_GROUP_BY
-            mpc_str += mpc_func_str.FUNC_COMPUTE_COUNT_WITH_GROUP_BY
-            mpc_str += mpc_func_str.FUNC_COMPUTE_AVG_WITH_GROUP_BY
-            mpc_str += mpc_func_str.FUNC_COMPUTE_MAX_WITH_GROUP_BY
-            mpc_str += mpc_func_str.FUNC_COMPUTE_MIN_WITH_GROUP_BY
-        return mpc_str
-
-    def generate_result_calculation_code(self, operators, mpc_str):
-        for i in range(len(operators)):
-            tokens = sqlparse.parse(operators[i])[0].tokens
-            participants_set = set()
-            formula_str = self.generate_formula(tokens, '', participants_set)
-            if len(participants_set) == 1:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "disabled query pattern")
-
-            if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE or self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-                mpc_str += f"def calculate_result_{i}():\n" \
-                           f"{INDENT}@for_range_opt_multithread(n_threads, result_record)\n" \
-                           f"{INDENT}def _(i):\n" \
-                           f"{INDENT}{INDENT}results[i][{i}] = {formula_str}\n\n\n"
-            elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-                mpc_str += f"def calculate_result_{i}():\n" \
-                           f"{INDENT}results[0][{i}] = {formula_str}\n\n\n"
-        return mpc_str
-
-    def generate_result_print_code(self, mpc_str):
-        field_print_str = f"{PPC_RESULT_FIELDS_FLAG} = ['{self.result_fields[0]}'"
-        for i in range(1, len(self.result_fields)):
-            field_print_str += f", '{self.result_fields[i]}'"
-        field_print_str += ']'
-
-        if self.sql_pattern == SqlPattern.BASIC_ARITH_OPE or self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-            result_print_str = f"{PPC_RESULT_VALUES_FLAG} = [results[i][0].reveal()"
-            for i in range(1, len(self.result_fields)):
-                result_print_str += f", results[i][{i}].reveal()"
-            result_print_str += ']'
-            mpc_str += f"def print_results():\n" \
-                       f"{INDENT}{field_print_str}\n" \
-                       f"{INDENT}{DISPLAY_FIELDS_FUNC}({PPC_RESULT_FIELDS_FLAG})\n\n" \
-                       f"{INDENT}@for_range_opt(result_record)\n" \
-                       f"{INDENT}def _(i):\n" \
-                       f"{INDENT}{INDENT}{result_print_str}\n" \
-                       f"{INDENT}{INDENT}{DISPLAY_RESULT_FUNC}({PPC_RESULT_VALUES_FLAG})\n\n\n"
-        elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-            result_print_str = f"{PPC_RESULT_VALUES_FLAG} = [results[0][0].reveal()"
-            for i in range(1, len(self.result_fields)):
-                result_print_str += f", results[0][{i}].reveal()"
-            result_print_str += ']'
-            mpc_str += f"def print_results():\n" \
-                       f"{INDENT}{field_print_str}\n" \
-                       f"{INDENT}{DISPLAY_FIELDS_FUNC}({PPC_RESULT_FIELDS_FLAG})\n\n" \
-                       f"{INDENT}{result_print_str}\n" \
-                       f"{INDENT}{DISPLAY_RESULT_FUNC}({PPC_RESULT_VALUES_FLAG})\n\n\n"
-        return mpc_str
-
-    def generate_mpc_execution_code(self, mpc_str):
-        mpc_str += 'def ppc_main():\n'
-        for i in range(len(self.table_set)):
-            mpc_str += f"{INDENT}read_data_collection(source{i}_record, SOURCE{i})\n"
-
-        if self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-            mpc_str += f"\n{INDENT}compute_group_by_indexes(source{self.group_by_column[0]}_record, " \
-                       f"{self.group_by_column[1]})\n\n"
-
-        for i in range(len(self.result_fields)):
-            mpc_str += f"{INDENT}calculate_result_{i}()\n"
-
-        mpc_str += f"\n{INDENT}print_results()\n\n\n"
-
-        mpc_str += "ppc_main()\n"
-        return mpc_str
-
-    def generate_formula(self, tokens, formula_str, participants_set):
-        for token in tokens:
-            if token.ttype == Punctuation \
-                    or token.ttype == Operator \
-                    or token.ttype == Token.Literal.Number.Integer \
-                    or token.ttype == Token.Operator.Comparison:
-                formula_str += token.value
-            elif type(token) == Function:
-                formula_str += self.handle_function(token)
-            elif type(token) == Identifier and token.tokens[0].ttype == Name and len(token.tokens) >= 3:
-                (table_number, field_number) = self.handle_basic_identifier(token)
-                if self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-                    if table_number != self.group_by_column[0] or field_number != self.group_by_column[1]:
-                        raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "bad sql statement")
-                    self.update_max_field(table_number, field_number)
-                    formula_str += GROUP_BY_COLUMN_CODE
-                elif self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-                    raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "disabled query pattern")
-                elif self.sql_pattern == SqlPattern.BASIC_ARITH_OPE:
-                    if token.value == token.parent.value:
-                        raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(),
-                                           "disabled query pattern")
-                    self.update_max_field(table_number, field_number)
-                    formula_str += f"source{table_number}_record[i][{field_number}]"
-                    participants_set.add(table_number)
-            elif hasattr(token, 'tokens'):
-                formula_str = self.generate_formula(token.tokens, formula_str, participants_set)
-        return formula_str
-
-    def handle_function(self, token):
-        tokens = token.tokens
-        func_name = tokens[0].value
-        (table_number, field_number) = self.handle_parenthesis(tokens[1])
-        self.update_max_field(table_number, field_number)
-        return self.func_to_formula(func_name, table_number, field_number)
-
-    def func_to_formula(self, func, table_number, field_number):
-        if self.sql_pattern == SqlPattern.AGGR_FUNC_ONLY:
-            formula = {
-                'COUNT': f"{mpc_func_str.FUNC_COMPUTE_COUNT_NAME}(source{table_number}_record_count)",
-                'SUM': f"{mpc_func_str.FUNC_COMPUTE_SUM_NAME}(source{table_number}_record, "
-                       f"source{table_number}_record_count, {field_number})",
-                'AVG': f"{mpc_func_str.FUNC_COMPUTE_AVG_NAME}(source{table_number}_record, "
-                       f"source{table_number}_record_count, {field_number})",
-                'MAX': f"{mpc_func_str.FUNC_COMPUTE_MAX_NAME}(source{table_number}_record, "
-                       f"source{table_number}_record_count, {field_number})",
-                'MIN': f"{mpc_func_str.FUNC_COMPUTE_MIN_NAME}(source{table_number}_record, "
-                       f"source{table_number}_record_count, {field_number})"
-            }
-        elif self.sql_pattern == SqlPattern.AGGR_FUNC_WITH_GROUP_BY:
-            formula = {
-                'COUNT': f"{mpc_func_str.FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME}(i)",
-                'SUM': f"{mpc_func_str.FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME}(source{table_number}_record,"
-                       f" {field_number}, i)",
-                'AVG': f"{mpc_func_str.FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME}(source{table_number}_record,"
-                       f" {field_number}, i)",
-                'MAX': f"{mpc_func_str.FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME}(source{table_number}_record,"
-                       f" {field_number}, i)",
-                'MIN': f"{mpc_func_str.FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME}(source{table_number}_record,"
-                       f" {field_number}, i)"
-            }
-        else:
-            formula = {}
-
-        return formula.get(func, '')
-
-    def handle_parenthesis(self, token):
-        for token in token.tokens:
-            if type(token) == Identifier:
-                (table_number, field_number) = self.handle_basic_identifier(token)
-                return table_number, field_number
-
-    def handle_basic_identifier(self, token):
-        tokens = token.tokens
-
-        if tokens[0].value not in self.table_set:
-            raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "table name not matched")
-        if tokens[1].value != '.':
-            raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), "invalid identifier")
-
-        field_num = get_column_number(tokens[2].value)
-
-        return int(tokens[0].value[-1]), field_num
-
-
-def get_column_number(field_name):
-    field_len = len(field_name)
-    field_num = 0
-    for i in range(field_len, 0, -1):
-        try:
-            int(field_name[i - 1:field_len])
-        except ValueError:
-            if i == field_len:
-                raise PpcException(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(),
-                                   f"invalid field suffix of table column '{field_name}'")
-            field_num = int(field_name[i:field_len])
-            break
-    return field_num
diff --git a/python/ppc_scheduler/mpc_generator/mpc_func_str.py b/python/ppc_scheduler/mpc_generator/mpc_func_str.py
deleted file mode 100644
index 87501124..00000000
--- a/python/ppc_scheduler/mpc_generator/mpc_func_str.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import os
-
-FILE_PATH = os.path.abspath(__file__)
-
-CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".")
-
-AGGR_FUNC_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_only.mpc"
-GROUP_BY_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_with_group_by.mpc"
-
-FUNC_COMPUTE_SUM_NAME = 'compute_sum'
-FUNC_COMPUTE_COUNT_NAME = 'compute_count'
-FUNC_COMPUTE_AVG_NAME = 'compute_avg'
-FUNC_COMPUTE_MAX_NAME = 'compute_max'
-FUNC_COMPUTE_MIN_NAME = 'compute_min'
-
-FUNC_COMPUTE_GROUP_BY_INDEXES_NAME = 'compute_group_by_indexes'
-FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME = 'compute_sum_with_group_by'
-FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME = 'compute_count_with_group_by'
-FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME = 'compute_avg_with_group_by'
-FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME = 'compute_max_with_group_by'
-FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME = 'compute_min_with_group_by'
-
-with open(AGGR_FUNC_SAMPLE_PATH, "r") as file:
-    AGGR_FUNC_SAMPLE_STR = file.read()
-
-with open(GROUP_BY_SAMPLE_PATH, "r") as file:
-    GROUP_BY_SAMPLE_STR = file.read()
-
-
-def get_body_str_by_name(start_str, end_str, sql_pattern):
-    if sql_pattern == 1:
-        source_str = AGGR_FUNC_SAMPLE_STR
-    elif sql_pattern == 2:
-        source_str = GROUP_BY_SAMPLE_STR
-    else:
-        return ''
-
-    start_index = source_str.find(start_str)
-    source_str = source_str[start_index:]
-
-    end_index = source_str.find(end_str) + len(end_str)
-    return source_str[:end_index]
-
-
-def get_func_str_by_name(func_name, sql_pattern):
-    start_str = f"def {func_name}"
-    end_str = "\n\n\n"
-    return get_body_str_by_name(start_str, end_str, sql_pattern)
-
-
-FUNC_COMPUTE_SUM = get_func_str_by_name(FUNC_COMPUTE_SUM_NAME, 1)
-FUNC_COMPUTE_COUNT = get_func_str_by_name(FUNC_COMPUTE_COUNT_NAME, 1)
-FUNC_COMPUTE_AVG = get_func_str_by_name(FUNC_COMPUTE_AVG_NAME, 1)
-FUNC_COMPUTE_MAX = get_func_str_by_name(FUNC_COMPUTE_MAX_NAME, 1)
-FUNC_COMPUTE_MIN = get_func_str_by_name(FUNC_COMPUTE_MIN_NAME, 1)
-
-GROUP_BY_GLOBAL_VARIABLE = get_body_str_by_name("# matrix of indexes", "\n\n\n", 2)
-
-FUNC_COMPUTE_GROUP_BY_INDEXES = get_func_str_by_name(FUNC_COMPUTE_GROUP_BY_INDEXES_NAME, 2)
-FUNC_COMPUTE_SUM_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_SUM_WITH_GROUP_BY_NAME, 2)
-FUNC_COMPUTE_COUNT_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_COUNT_WITH_GROUP_BY_NAME, 2)
-FUNC_COMPUTE_AVG_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_AVG_WITH_GROUP_BY_NAME, 2)
-FUNC_COMPUTE_MAX_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_MAX_WITH_GROUP_BY_NAME, 2)
-FUNC_COMPUTE_MIN_WITH_GROUP_BY = get_func_str_by_name(FUNC_COMPUTE_MIN_WITH_GROUP_BY_NAME, 2)
diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc
deleted file mode 100644
index 9ddb3c34..00000000
--- a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_only.mpc
+++ /dev/null
@@ -1,136 +0,0 @@
-# PSI_OPTION=True
-
-# BIT_LENGTH = 128
-
-# This file is generated automatically by ams
-'''
-SELECT COUNT(s1.field3) + COUNT(s2.field3) AS r0,
-       SUM(s1.field3) + COUNT(s0.field0) AS 'count',
-       (MAX(s0.field1) + MAX(s2.field1)) / 2 AS r1,
-       (AVG(s1.field2) + AVG(s2.field2)) / 2 AS r2,
-       MIN(s1.field0) - MIN(s0.field0) AS r3
-FROM (source0 AS s0
-      INNER JOIN source1 AS s1 ON s0.id = s1.id)
-INNER JOIN source2 AS s2 ON s0.id = s2.id;
-'''
-
-from ppc import *
-
-n_threads = 8
-value_type = pfix
-
-pfix.set_precision(16, 47)
-
-SOURCE0 = 0
-source0_record_count = $(source0_record_count)
-source0_column_count = 2
-source0_record = Matrix(source0_record_count, source0_column_count, value_type)
-
-SOURCE1 = 1
-source1_record_count = $(source1_record_count)
-source1_column_count = 4
-source1_record = Matrix(source1_record_count, source1_column_count, value_type)
-
-SOURCE2 = 2
-source2_record_count = $(source2_record_count)
-source2_column_count = 4
-source2_record = Matrix(source2_record_count, source2_column_count, value_type)
-
-results = Matrix(1, 5, value_type)
-
-
-def read_data_collection(data_collection, party_id):
-    if data_collection.sizes[0] > 0:
-        data_collection.input_from(party_id)
-
-
-def compute_sum(source, record_count, col_index):
-    records_sum = Array(1, value_type)
-    records_sum[0] = source[0][col_index]
-
-    @for_range(1, record_count)
-    def _(i):
-        records_sum[0] = records_sum[0] + source[i][col_index]
-
-    return records_sum[0]
-
-
-def compute_count(record_count):
-    return record_count
-
-
-def compute_avg(source, record_count, col_index):
-    records_sum = Array(1, value_type)
-    records_sum[0] = source[0][col_index]
-
-    @for_range(1, record_count)
-    def _(i):
-        records_sum[0] = records_sum[0] + source[i][col_index]
-
-    return records_sum[0] / record_count
-
-
-def compute_max(source, record_count, col_index):
-    max_record = Array(1, value_type)
-    max_record[0] = source[0][col_index]
-
-    @for_range(1, record_count)
-    def _(i):
-        max_record[0] = condition(max_record[0] < source[i][col_index], source[i][col_index], max_record[0])
-
-    return max_record[0]
-
-
-def compute_min(source, record_count, col_index):
-    min_record = Array(1, value_type)
-    min_record[0] = source[0][col_index]
-
-    @for_range(1, record_count)
-    def _(i):
-        min_record[0] = condition(min_record[0] > source[i][col_index], source[i][col_index], min_record[0])
-
-    return min_record[0]
-
-
-def calculate_result_0():
-    results[0][0] = compute_count(source1_record_count)+compute_count(source2_record_count)
-
-
-def calculate_result_1():
-    results[0][1] = compute_sum(source1_record, source1_record_count, 3)+compute_count(source0_record_count)
-
-
-def calculate_result_2():
-    results[0][2] = (compute_max(source0_record, source0_record_count, 1)+compute_max(source2_record, source2_record_count, 1))/2
-
-
-def calculate_result_3():
-    results[0][3] = (compute_avg(source1_record, source1_record_count, 2)+compute_avg(source2_record, source2_record_count, 2))/2
-
-
-def calculate_result_4():
-    results[0][4] = compute_min(source1_record, source1_record_count, 0)-compute_min(source0_record, source0_record_count, 0)
-
-
-def print_results():
-    result_fields = ['r0', 'count', 'r1', 'r2', 'r3']
-    set_display_field_names(result_fields)
-
-    result_values = [results[0][0].reveal(), results[0][1].reveal(), results[0][2].reveal(), results[0][3].reveal(), results[0][4].reveal()]
-    display_data(result_values)
-
-
-def ppc_main():
-    read_data_collection(source0_record, SOURCE0)
-    read_data_collection(source1_record, SOURCE1)
-    read_data_collection(source2_record, SOURCE2)
-    calculate_result_0()
-    calculate_result_1()
-    calculate_result_2()
-    calculate_result_3()
-    calculate_result_4()
-
-    print_results()
-
-
-ppc_main()
diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc
deleted file mode 100644
index f1a01339..00000000
--- a/python/ppc_scheduler/mpc_generator/mpc_sample/aggr_func_with_group_by.mpc
+++ /dev/null
@@ -1,234 +0,0 @@
-# PSI_OPTION=True
-
-# BIT_LENGTH = 128
-
-# This file is generated automatically by ams
-'''
-SELECT 3*s1.field4 AS r0,
-       COUNT(s1.field4) AS 'count',
-       AVG(s0.field1) * 2 + s1.field4 AS r1,
-       (SUM(s0.field2) + SUM(s1.field2))/(COUNT(s1.field3) + 100/(MIN(s0.field1)+MIN(s1.field1))) + 10,
-       MAX(s1.field1),
-       MIN(s2.field2)
-FROM (source0 AS s0
-      INNER JOIN source1 AS s1 ON s0.id = s1.id)
-INNER JOIN source2 AS s2 ON s0.id = s2.id
-GROUP BY s1.field4;
-'''
-
-from ppc import *
-
-n_threads = 8
-value_type = pfix
-
-pfix.set_precision(16, 47)
-
-SOURCE0 = 0
-source0_record_count = $(source0_record_count)
-source0_column_count = 3
-source0_record = Matrix(source0_record_count, source0_column_count, value_type)
-
-SOURCE1 = 1
-source1_record_count = $(source1_record_count)
-source1_column_count = 5
-source1_record = Matrix(source1_record_count, source1_column_count, value_type)
-
-SOURCE2 = 2
-source2_record_count = $(source2_record_count)
-source2_column_count = 3
-source2_record = Matrix(source2_record_count, source2_column_count, value_type)
-
-# group by means all parties have same number of record
-source_record_count = $(source0_record_count)
-result_record = cint(source_record_count)
-results = Matrix(source_record_count, 6, value_type)
-
-
-def read_data_collection(data_collection, party_id):
-    if data_collection.sizes[0] > 0:
-        data_collection.input_from(party_id)
-
-
-# matrix of indexes after group by:
-#     0         1         2
-# 0 count1 start_index end_index
-# 1 count2 start_index end_index
-# 2 count3 start_index end_index
-# ...
-# source_record_count - 1 ...
-group_indexes_key = Array(source_record_count, value_type)
-group_indexes_matrix = Matrix(source_record_count, 3, pint)
-group_column = Array(source_record_count, value_type)
-
-
-def compute_group_by_indexes(source, col_index):
-    # group_count group_index group_flag
-    group_states = Array(3, cint)
-
-    @for_range_opt(source_record_count)
-    def _(i):
-        group_column[i] = source[i][col_index]
-        group_states[1] = 0
-        group_states[2] = 0
-
-        @for_range(group_states[0])
-        def _(j):
-            @if_(pint(group_indexes_key[j] == source[i][col_index]).reveal())
-            def _():
-                group_states[1] = j
-                group_states[2] = 1
-
-        @if_e(group_states[2] == 0)
-        def _():
-            # new item
-            group_indexes_key[group_states[0]] = source[i][col_index]
-            group_indexes_matrix[group_states[0]][0] = 1
-            group_indexes_matrix[group_states[0]][1] = i
-            group_indexes_matrix[group_states[0]][2] = i
-            group_states[0] = group_states[0] + 1
-
-        @else_
-        def _():
-            group_indexes_matrix[group_states[1]][0] = group_indexes_matrix[group_states[1]][0] + 1
-            group_indexes_matrix[group_states[1]][2] = i
-
-    global result_record
-    result_record = group_states[0]
-
-
-def compute_sum_with_group_by(source, col_index, group_row_index):
-    records_sum = Array(1, value_type)
-
-    start_index = group_indexes_matrix[group_row_index][1].reveal()
-    end_index = group_indexes_matrix[group_row_index][2].reveal()
-
-    records_sum[0] = source[start_index][col_index]
-
-    @for_range(start_index + 1, end_index + 1)
-    def _(i):
-        @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal())
-        def _():
-            records_sum[0] = records_sum[0] + source[i][col_index]
-
-    return records_sum[0]
-
-
-def compute_count_with_group_by(group_row_index):
-    return group_indexes_matrix[group_row_index][0]
-
-
-def compute_avg_with_group_by(source, col_index, group_row_index):
-    records_sum = Array(1, value_type)
-
-    start_index = group_indexes_matrix[group_row_index][1].reveal()
-    end_index = group_indexes_matrix[group_row_index][2].reveal()
-
-    records_sum[0] = source[start_index][col_index]
-
-    @for_range(start_index + 1, end_index + 1)
-    def _(i):
-        @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal())
-        def _():
-            records_sum[0] = records_sum[0] + source[i][col_index]
-
-    return value_type(records_sum[0] / group_indexes_matrix[group_row_index][0])
-
-
-def compute_max_with_group_by(source, col_index, group_row_index):
-    max_records = Array(1, value_type)
-
-    start_index = group_indexes_matrix[group_row_index][1].reveal()
-    end_index = group_indexes_matrix[group_row_index][2].reveal()
-
-    max_records[0] = source[start_index][col_index]
-
-    @for_range(start_index + 1, end_index + 1)
-    def _(i):
-        @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal())
-        def _():
-            max_records[0] = condition(max_records[0] < source[i][col_index], source[i][col_index], max_records[0])
-
-    return max_records[0]
-
-
-def compute_min_with_group_by(source, col_index, group_row_index):
-    min_records = Array(1, value_type)
-
-    start_index = group_indexes_matrix[group_row_index][1].reveal()
-    end_index = group_indexes_matrix[group_row_index][2].reveal()
-
-    min_records[0] = source[start_index][col_index]
-
-    @for_range(start_index + 1, end_index + 1)
-    def _(i):
-        @if_(pint(group_indexes_key[group_row_index] == group_column[i]).reveal())
-        def _():
-            min_records[0] = condition(min_records[0] > source[i][col_index], source[i][col_index], min_records[0])
-
-    return min_records[0]
-
-
-def calculate_result_0():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][0] = 3*group_indexes_key[i]
-
-
-def calculate_result_1():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][1] = compute_count_with_group_by(i)
-
-
-def calculate_result_2():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][2] = compute_avg_with_group_by(source0_record, 1, i)*2+group_indexes_key[i]
-
-
-def calculate_result_3():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][3] = (compute_sum_with_group_by(source0_record, 2, i)+compute_sum_with_group_by(source1_record, 2, i))/(compute_count_with_group_by(i)+100/(compute_min_with_group_by(source0_record, 1, i)+compute_min_with_group_by(source1_record, 1, i)))+10
-
-
-def calculate_result_4():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][4] = compute_max_with_group_by(source1_record, 1, i)
-
-
-def calculate_result_5():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][5] = compute_min_with_group_by(source2_record, 2, i)
-
-
-def print_results():
-    result_fields = ['r0', 'count', 'r1', 'result3', 'result4', 'result5']
-    set_display_field_names(result_fields)
-
-    @for_range_opt(result_record)
-    def _(i):
-        result_values = [results[i][0].reveal(), results[i][1].reveal(), results[i][2].reveal(), results[i][3].reveal(), results[i][4].reveal(), results[i][5].reveal()]
-        display_data(result_values)
-
-
-def ppc_main():
-    read_data_collection(source0_record, SOURCE0)
-    read_data_collection(source1_record, SOURCE1)
-    read_data_collection(source2_record, SOURCE2)
-
-    compute_group_by_indexes(source1_record, 4)
-
-    calculate_result_0()
-    calculate_result_1()
-    calculate_result_2()
-    calculate_result_3()
-    calculate_result_4()
-    calculate_result_5()
-
-    print_results()
-
-
-ppc_main()
diff --git a/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc b/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc
deleted file mode 100644
index fa2fc97d..00000000
--- a/python/ppc_scheduler/mpc_generator/mpc_sample/basic_arith_ope.mpc
+++ /dev/null
@@ -1,78 +0,0 @@
-# PSI_OPTION=True
-
-# BIT_LENGTH = 128
-
-# This file is generated automatically by ams
-'''
-SELECT 3*(s1.field3 + s2.field3) - s0.field3 AS r0,
-       (s0.field1 + s2.field1) / 2 * s1.field1 AS r1
-FROM (source0 AS s0
-      INNER JOIN source1 AS s1 ON s0.id = s1.id)
-INNER JOIN source2 AS s2 ON s0.id = s2.id;
-'''
-
-from ppc import *
-
-n_threads = 8
-value_type = pfix
-
-pfix.set_precision(16, 47)
-
-SOURCE0 = 0
-source0_record_count = $(source0_record_count)
-source0_column_count = 4
-source0_record = Matrix(source0_record_count, source0_column_count, value_type)
-
-SOURCE1 = 1
-source1_record_count = $(source1_record_count)
-source1_column_count = 4
-source1_record = Matrix(source1_record_count, source1_column_count, value_type)
-
-SOURCE2 = 2
-source2_record_count = $(source2_record_count)
-source2_column_count = 4
-source2_record = Matrix(source2_record_count, source2_column_count, value_type)
-
-# basic arithmetic operation means that all parties have same number of record
-result_record = $(source0_record_count)
-results = Matrix(result_record, 2, value_type)
-
-
-def read_data_collection(data_collection, party_id):
-    if data_collection.sizes[0] > 0:
-        data_collection.input_from(party_id)
-
-
-def calculate_result_0():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][0] = 3*(source1_record[i][3]+source2_record[i][3])-source0_record[i][3]
-
-
-def calculate_result_1():
-    @for_range_opt_multithread(n_threads, result_record)
-    def _(i):
-        results[i][1] = (source0_record[i][1]+source2_record[i][1])/2*source1_record[i][1]
-
-
-def print_results():
-    result_fields = ['r0', 'r1']
-    set_display_field_names(result_fields)
-
-    @for_range_opt(result_record)
-    def _(i):
-        result_values = [results[i][0].reveal(), results[i][1].reveal()]
-        display_data(result_values)
-
-
-def ppc_main():
-    read_data_collection(source0_record, SOURCE0)
-    read_data_collection(source1_record, SOURCE1)
-    read_data_collection(source2_record, SOURCE2)
-    calculate_result_0()
-    calculate_result_1()
-
-    print_results()
-
-
-ppc_main()
diff --git a/python/ppc_scheduler/mpc_generator/test_generator.py b/python/ppc_scheduler/mpc_generator/test_generator.py
deleted file mode 100644
index 9a37a2e1..00000000
--- a/python/ppc_scheduler/mpc_generator/test_generator.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-import os
-import unittest
-
-from ppc_scheduler.mpc_generator.generator import CodeGenerator
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-
-FILE_PATH = os.path.abspath(__file__)
-
-CURRENT_PATH = os.path.abspath(os.path.dirname(FILE_PATH) + os.path.sep + ".")
-
-BASIC_ARITH_OPE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}basic_arith_ope.mpc"
-AGGR_FUNC_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_only.mpc"
-GROUP_BY_SAMPLE_PATH = f"{CURRENT_PATH}{os.sep}mpc_sample{os.sep}aggr_func_with_group_by.mpc"
-
-with open(BASIC_ARITH_OPE_PATH, "r") as file:
-    BASIC_ARITH_OPE_STR = file.read()
-
-with open(AGGR_FUNC_SAMPLE_PATH, "r") as file:
-    AGGR_FUNC_SAMPLE_STR = file.read()
-
-with open(GROUP_BY_SAMPLE_PATH, "r") as file:
-    GROUP_BY_SAMPLE_STR = file.read()
-
-
-class TestGenerator(unittest.TestCase):
-
-    def test_bad_sql(self):
-        try:
-            sql = "select from a from b where c = d"
-            code_generator = CodeGenerator(sql)
-            code_generator.sql_to_mpc_code()
-        except PpcException as e:
-            self.assertEqual(PpcErrorCode.ALGORITHM_BAD_SQL.get_code(), e.code)
-
-        sql = "select a0.f1 + b1.f1 from a0, b1 where a0.id=b1.id"
-        code_generator = CodeGenerator(sql)
-        self.assertIsNotNone(code_generator.sql_to_mpc_code())
-
-    def test_unsupported_keyword(self):
-        try:
-            sql = "select s0.f1 + s1.f1 from s0, s1 where s0.f1 > 1 and s0.f1 < 10"
-            code_generator = CodeGenerator(sql)
-            code_generator.sql_to_mpc_code()
-        except PpcException as e:
-            self.assertEqual("keyword 'AND' not supported", e.message)
-
-    def test_disabled_query_pattern(self):
-        try:
-            sql = "select s0.f1, 3 + s1.f1 from s0, s1"
-            code_generator = CodeGenerator(sql)
-            code_generator.sql_to_mpc_code()
-        except PpcException as e:
-            self.assertEqual("disabled query pattern", e.message)
-
-        try:
-            sql = "select s0.f1, s1.f1 + s1.f1 from s0, s1"
-            code_generator = CodeGenerator(sql)
-            code_generator.sql_to_mpc_code()
-        except PpcException as e:
-            self.assertEqual("disabled query pattern", e.message)
-
-    def test_basic_pattern(self):
-        sql = "SELECT 3*(s1.field3 + s2.field3) - s0.field3 AS r0, \
-                  (s0.field1 + s2.field1) / 2 * s1.field1 AS r1\
-              FROM (source0 AS s0\
-                  INNER JOIN source1 AS s1 ON s0.id = s1.id)\
-              INNER JOIN source2 AS s2 ON s0.id = s2.id;"
-        code_generator = CodeGenerator(sql)
-        self.assertEqual(BASIC_ARITH_OPE_STR, code_generator.sql_to_mpc_code())
-
-    def test_single_aggre_pattern(self):
-        sql = "SELECT COUNT(s1.field3) + COUNT(s2.field3) AS r0,\
-                  SUM(s1.field3) + COUNT(s0.field0) AS 'count',\
-                  (MAX(s0.field1) + MAX(s2.field1)) / 2 AS r1,\
-                  (AVG(s1.field2) + AVG(s2.field2)) / 2 AS r2,\
-                  MIN(s1.field0) - MIN(s0.field0) AS r3\
-               FROM (source0 AS s0\
-                  INNER JOIN source1 AS s1 ON s0.id = s1.id)\
-               INNER JOIN source2 AS s2 ON s0.id = s2.id;"
-        code_generator = CodeGenerator(sql)
-        self.assertEqual(AGGR_FUNC_SAMPLE_STR, code_generator.sql_to_mpc_code())
-
-    def test_group_by_pattern(self):
-        sql = "SELECT 3*s1.field4 AS r0,\
-                   COUNT(s1.field4) AS 'count', \
-                   AVG(s0.field1) * 2 + s1.field4 AS r1,\
-                   (SUM(s0.field2) + SUM(s1.field2))/(COUNT(s1.field3) + 100/(MIN(s0.field1)+MIN(s1.field1))) + 10,\
-                   MAX(s1.field1),\
-                   MIN(s2.field2)\
-               FROM (source0 AS s0\
-                   INNER JOIN source1 AS s1 ON s0.id = s1.id)\
-               INNER JOIN source2 AS s2 ON s0.id = s2.id\
-               GROUP BY s1.field4;"
-        code_generator = CodeGenerator(sql)
-        self.assertEqual(GROUP_BY_SAMPLE_STR, code_generator.sql_to_mpc_code())
-
-
-if __name__ == '__main__':
-    unittest.main(verbosity=1)
diff --git a/python/ppc_scheduler/node/__init__.py b/python/ppc_scheduler/node/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/node/computing_node_client/__init__.py b/python/ppc_scheduler/node/computing_node_client/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/node/computing_node_client/model_node_client.py b/python/ppc_scheduler/node/computing_node_client/model_node_client.py
deleted file mode 100644
index 0bb9d9fe..00000000
--- a/python/ppc_scheduler/node/computing_node_client/model_node_client.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import json
-import time
-
-from ppc_common.ppc_utils import http_utils
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-
-RUN_MODEL_API_PREFIX = "/api/ppc-model/pml/run-model-task/"
-GET_MODEL_LOG_API_PREFIX = "/api/ppc-model/pml/record-model-log/"
-
-
-class ModelClient:
-    def __init__(self, logger, endpoint, token, polling_interval_s: int = 5, max_retries: int = 5, retry_delay_s: int = 5):
-        self.logger = logger
-        self.endpoint = endpoint
-        self.token = token
-        self.polling_interval_s = polling_interval_s
-        self.max_retries = max_retries
-        self.retry_delay_s = retry_delay_s
-        self._completed_status = 'COMPLETED'
-        self._failed_status = 'FAILED'
-
-    def run(self, *args):
-        
-        params = args[0]
-        if type(params) == str:
-            params = json.loads(params) 
-        
-        task_id = params['task_id']
-        
-        try:
-            self.logger.info(f"model client begin to run model task {task_id}")
-            response = self._send_request_with_retry(http_utils.send_post_request,
-                                                     endpoint=self.endpoint,
-                                                     uri=RUN_MODEL_API_PREFIX + task_id,
-                                                     params=params)
-            check_response(response)
-            return self._poll_task_status(task_id)
-        except Exception as e:
-            self.logger.error(f"model client run model task exception, task: {task_id}, e: {e}")
-            raise e
-
-    def kill(self, task_id):
-        try:
-            self.logger.info(f"model client begin to kill model task {task_id}")
-            response = self._send_request_with_retry(http_utils.send_delete_request,
-                                                     endpoint=self.endpoint,
-                                                     uri=RUN_MODEL_API_PREFIX + task_id)
-            check_response(response)
-            self.logger.info(f"model client task {task_id} was killed")
-            return response
-        except Exception as e:
-            self.logger.warn(f"model client kill task {task_id} exception, e: {e}")
-            raise e
-
-    def _poll_task_status(self, task_id):
-        while True:
-            response = self._send_request_with_retry(http_utils.send_get_request,
-                                                     endpoint=self.endpoint,
-                                                     uri=RUN_MODEL_API_PREFIX + task_id)
-            check_response(response)
-            if response['data']['status'] == self._completed_status:
-                self.logger.info(f"model client task {task_id} completed, response: {response['data']}")
-                return response
-            elif response['data']['status'] == self._failed_status:
-                self.logger.warn(f"model client task {task_id} failed, response: {response['data']}")
-                raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data'])
-            else:
-                time.sleep(self.polling_interval_s)
-
-    def get_remote_log(self, remote_id):
-        response = self._send_request_with_retry(http_utils.send_get_request,
-                                                 endpoint=self.endpoint,
-                                                 uri=GET_MODEL_LOG_API_PREFIX + remote_id)
-        check_response(response)
-        return response['data']
-
-    def _send_request_with_retry(self, request_func, *args, **kwargs):
-        attempt = 0
-        while attempt < self.max_retries:
-            try:
-                response = request_func(*args, **kwargs)
-                return response
-            except Exception as e:
-                self.logger.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}")
-                attempt += 1
-                if attempt < self.max_retries:
-                    time.sleep(self.retry_delay_s)
-                else:
-                    self.logger.warn(f"Request failed after {self.max_retries} attempts")
-                    raise e
-
-
-def check_response(response):
-    if response['errorCode'] != 0:
-        raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['message'])
diff --git a/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py b/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py
deleted file mode 100644
index f64ece20..00000000
--- a/python/ppc_scheduler/node/computing_node_client/mpc_node_client.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import random
-
-from ppc_common.ppc_utils import http_utils, utils
-from ppc_scheduler.node.computing_node_client.utils import check_privacy_service_response
-
-
-class MpcClient:
-    def __init__(self, endpoint):
-        self.endpoint = endpoint
-
-    def run(self, job_info, token):
-        params = {
-            'jsonrpc': '2',
-            'method': 'run',
-            'token': token,
-            'id': random.randint(1, 65535),
-            'params': job_info
-        }
-        response = http_utils.send_post_request(self.endpoint, None, params)
-        check_privacy_service_response(response)
-        return response['result']
-
-    def kill(self, job_id, token):
-        params = {
-            'jsonrpc': '2',
-            'method': 'kill',
-            'token': token,
-            'id': random.randint(1, 65535),
-            'params': {'jobId': job_id}
-        }
-        http_utils.send_post_request(self.endpoint, None, params)
-        return utils.make_response(0, "success", None)
diff --git a/python/ppc_scheduler/node/computing_node_client/psi_client.py b/python/ppc_scheduler/node/computing_node_client/psi_client.py
deleted file mode 100644
index 7dcd5ba3..00000000
--- a/python/ppc_scheduler/node/computing_node_client/psi_client.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import json
-import random
-import time
-
-from ppc_common.ppc_utils import http_utils
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-from ppc_scheduler.node.computing_node_client.utils import check_privacy_service_response
-
-
-class PsiClient:
-    def __init__(self, logger, endpoint, token, polling_interval_s: int = 3, max_retries: int = 3, retry_delay_s: int = 3):
-        self.logger = logger
-        self.endpoint = endpoint
-        self.token = token
-        self.polling_interval_s = polling_interval_s
-        self.max_retries = max_retries
-        self.retry_delay_s = retry_delay_s
-        self._get_task_status_method = 'getTaskStatus'
-        self._completed_status = 'COMPLETED'
-        self._failed_status = 'FAILED'
-
-    def run(self, *args):
-        
-        params = args[0]
-        if type(params) == str:
-            params = json.loads(params) 
-        
-        task_id = params['taskID']
-        
-        json_rpc_request = {
-            'jsonrpc': '1',
-            'method': 'asyncRunTask',
-            'token': self.token,
-            'id': random.randint(1, 65535),
-            'params': params
-        }
-        response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, json_rpc_request)
-        check_privacy_service_response(response)
-        return self._poll_task_status(task_id)
-
-    def _poll_task_status(self, task_id: str):
-        while True:
-            params = {
-                'jsonrpc': '1',
-                'method': 'getTaskStatus',
-                'token': self.token,
-                'id': random.randint(1, 65535),
-                'params': {
-                    'taskID': task_id,
-                }
-            }
-            response = self._send_request_with_retry(http_utils.send_post_request, self.endpoint, None, params)
-            check_privacy_service_response(response)
-            if response['result']['status'] == self._completed_status:
-                return response['result']
-            elif response['result']['status'] == self._failed_status:
-                self.logger.warn(f"task {task_id} failed, response: {response['data']}")
-                raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['data'])
-            time.sleep(self.polling_interval_s)
-
-    def _send_request_with_retry(self, request_func, *args, **kwargs):
-        attempt = 0
-        while attempt < self.max_retries:
-            try:
-                response = request_func(*args, **kwargs)
-                return response
-            except Exception as e:
-                self.logger.warn(f"Request failed: {e}, attempt {attempt + 1}/{self.max_retries}")
-                attempt += 1
-                if attempt < self.max_retries:
-                    time.sleep(self.retry_delay_s)
-                else:
-                    self.logger.warn(f"Request failed after {self.max_retries} attempts")
-                    raise e
diff --git a/python/ppc_scheduler/node/computing_node_client/utils.py b/python/ppc_scheduler/node/computing_node_client/utils.py
deleted file mode 100644
index 3d3527c8..00000000
--- a/python/ppc_scheduler/node/computing_node_client/utils.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException
-
-
-def check_privacy_service_response(response):
-    if 'result' not in response.keys():
-        raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), "http request error")
-    elif 0 != response['result']['code'] or response['result']['status'] == 'FAILED':
-        raise PpcException(PpcErrorCode.CALL_SCS_ERROR.get_code(), response['result']['message'])
diff --git a/python/ppc_scheduler/node/node_manager.py b/python/ppc_scheduler/node/node_manager.py
deleted file mode 100644
index c180714b..00000000
--- a/python/ppc_scheduler/node/node_manager.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from ppc_scheduler.database import computing_node_mapper
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-
-
-class ComputingNodeManager:
-    type_map = {
-        WorkerType.T_PSI: 'PSI',
-        WorkerType.T_MPC: 'MPC',
-        WorkerType.T_MODEL: 'MODEL'
-    }
-
-    def __init__(self, components):
-        self.components = components
-
-    def add_node(self, node_id: str, url: str, worker_type: str):
-        with self.components.create_sql_session() as session:
-            computing_node_mapper.insert_computing_node(
-                session, node_id, url, self.type_map[worker_type], 0)
-
-    def remove_node(self, url: str, worker_type: str):
-        with self.components.create_sql_session() as session:
-            computing_node_mapper.delete_computing_node(
-                session, url, self.type_map[worker_type])
-
-    def get_node(self, worker_type: str):
-        with self.components.create_sql_session() as session:
-            return computing_node_mapper.get_and_update_min_loading_url(session, self.type_map[worker_type])
-
-    def release_node(self, url: str, worker_type: str):
-        with self.components.create_sql_session() as session:
-            return computing_node_mapper.release_loading(session, url, self.type_map[worker_type])
diff --git a/python/ppc_scheduler/ppc_scheduler_app.py b/python/ppc_scheduler/ppc_scheduler_app.py
deleted file mode 100644
index 0195236f..00000000
--- a/python/ppc_scheduler/ppc_scheduler_app.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Note: here can't be refactored by autopep
-
-import os
-import sys
-import argparse
-
-current_file_path = os.path.abspath(__file__)
-current_file_real_path = os.path.realpath(current_file_path)
-
-
-current_dir = os.path.dirname(current_file_real_path)
-parent_dir = os.path.dirname(current_dir)
-
-sys.path.append(current_dir)
-sys.path.append(parent_dir)
-print(sys.path)
-
-from ppc_scheduler.endpoints.restx import api
-from ppc_scheduler.endpoints.job_controller import ns as job_namespace
-from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_scheduler.job.job_manager import JobManager
-from ppc_scheduler.workflow.scheduler.scheduler import Scheduler
-from ppc_scheduler.common.global_job_manager import update_job_manager
-from ppc_scheduler.common.global_context import components
-from ppc_scheduler.workflow.scheduler.scheduler_api import SchedulerApi
-from paste.translogger import TransLogger
-from flask import Flask, Blueprint
-from cheroot.wsgi import Server as WSGIServer
-
-app = Flask(__name__)
-
-def init_thread_event_manager():
-    thread_event_manager = ThreadEventManager()
-    return thread_event_manager
-
-def init_scheduler(config_data, workspace: str, logger):
-    scheduler_api = Scheduler(workspace, logger=logger)
-    return scheduler_api
-
-def init_job_manager(config_data, workspace: str, thread_event_manager: ThreadEventManager, scheduler: SchedulerApi, logger):
-        
-    job_timeout_h = config_data['JOB_TIMEOUT_H']
-    
-    job_manager = JobManager(
-        logger=logger,
-        scheduler=scheduler,
-        thread_event_manager=thread_event_manager,
-        workspace=workspace,
-        job_timeout_h=job_timeout_h
-    )
-    
-    logger.info("Initialize job manager, job_timeout_h: %s", job_timeout_h)
-    
-    return job_manager
-
-def initialize_app(app, config_path, log_config_path):    
-    # init log first
-    components.init_log(log_config_path=log_config_path)
-    logger = components.logger()
-    
-    # init config
-    config_data = components.init_config(config_path=config_path)
-    # workspaces
-    workspace = config_data['WORKSPACE']
-    logger.info(f" ==> Initialize workspace: {workspace}")
-    
-    # event manager
-    thread_event_manager = init_thread_event_manager()
-    
-    # scheduler
-    scheduler = init_scheduler(config_data=config_data, workspace=workspace, logger=logger)
-    
-    # job manager
-    job_manager = init_job_manager(config_data=config_data, workspace=workspace, thread_event_manager=thread_event_manager, scheduler=scheduler, logger=logger)
-    
-    update_job_manager(job_manager)
-    
-    # initialize application components
-    components.init_all(config_data=config_data)
-    components.update_thread_event_manager(thread_event_manager)
-
-    app.config.update(config_data)
-    
-    blueprint = Blueprint('api', __name__, url_prefix='/api')
-    api.init_app(blueprint)
-    api.add_namespace(job_namespace)
-    app.register_blueprint(blueprint)
-    
-    components.logger().info(app.url_map)
-
-def main(config_path, log_config_path):
-    
-    print(f"Using config: {config_path}")
-    print(f"Using logging config: {log_config_path}")
-    
-    initialize_app(app, config_path, log_config_path)
-
-    app.config['SECRET_KEY'] = os.urandom(24)
-    
-    listen_ip = app.config['HTTP_HOST']
-    listen_port = app.config['HTTP_PORT']
-    thread_num = app.config['HTTP_THREAD_NUM']
-    
-    server = WSGIServer((listen_ip, listen_port),
-                        TransLogger(app, setup_console_handler=False), numthreads=thread_num)
-
-    protocol = 'http'
-    message = f"Starting wedpr scheduler server at {protocol}://{listen_ip}:{listen_port}"
-    print(message)
-    components.logger().info(message)
-    server.start()
-
-if __name__ == "__main__":
-    # Create ArgumentParser
-    parser = argparse.ArgumentParser(description='wedpr scheduler')
-    # Add argument
-    parser.add_argument('--config', default='./conf/application.yml', 
-                        help='Path to the configuration file')
-    parser.add_argument('--log_config', default='./conf/logging.conf',
-                        help='Path to the logging configuration file')
-    # Parser argument
-    args = parser.parse_args()
-
-    # Run main program
-    main(config_path=args.config, log_config_path=args.log_config)
diff --git a/python/ppc_scheduler/scripts/start.sh b/python/ppc_scheduler/scripts/start.sh
deleted file mode 100755
index 0dbd65b6..00000000
--- a/python/ppc_scheduler/scripts/start.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
-
-LANG=zh_CN.UTF-8
-##############################################################################
-##
-##  Wedpr scheduler service start up script for UN*X
-##
-##############################################################################
-
-# @function: output log with red color (error log)
-# @param: content: error message
-
-function LOG_ERROR() {
-    local content=${1}
-    echo -e "\033[31m"${content}"\033[0m"
-}
-
-# @function: output information log
-# @param: content: information message
-function LOG_INFO() {
-    local content=${1}
-    echo -e "\033[32m"${content}"\033[0m"
-}
-
-function Usage() {
-    LOG_INFO "Usage:start the console"
-    LOG_INFO "./start_scheduler.sh"
-}
-
-function check_python() {
-    # check python
-    if ! command -v python >/dev/null 2>&1; then
-        echo "Python is not installed."
-        exit 1
-    fi
-
-    # check python version, 3.9 support
-    python_version=$(python --version)
-    if [[ $python_version =~ Python\ 3.9.* ]]; then
-        echo "Python version is 3.9, as required."
-    else
-        echo "Python version is not 3.9. Installed version is: $python_version"
-        exit 2
-    fi
-}
-
-# source venv/bin/activate
-
-pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}')
-if [ ! -z ${pid} ]; then
-    echo " the scheduler has started, pid is ${pid}."
-    exit 0
-fi
-
-nohup python $dirpath/ppc_scheduler/ppc_scheduler_app.py --config $dirpath/conf/application.yml --log_config $dirpath/conf/logging.conf >/dev/null 2>&1 &
-
-sleep 3
-
-pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}')
-if [ ! -z ${pid} ]; then
-    echo " start scheduler successfully, pid is ${pid}."
-    exit 0
-else
-    echo " start scheduler failed."
-    exit 1
-fi
diff --git a/python/ppc_scheduler/scripts/stop.sh b/python/ppc_scheduler/scripts/stop.sh
deleted file mode 100755
index 686122f6..00000000
--- a/python/ppc_scheduler/scripts/stop.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-
-dirpath="$(cd "$(dirname "$0")" && pwd)"
-cd $dirpath
-
-LANG=zh_CN.UTF-8
-##############################################################################
-##
-##  Wedpr scheduler service start up script for UN*X
-##
-##############################################################################
-
-# @function: output log with red color (error log)
-# @param: content: error message
-
-function LOG_ERROR() {
-    local content=${1}
-    echo -e "\033[31m"${content}"\033[0m"
-}
-
-# @function: output information log
-# @param: content: information message
-function LOG_INFO() {
-    local content=${1}
-    echo -e "\033[32m"${content}"\033[0m"
-}
-
-function Usage() {
-    LOG_INFO "Usage:start the console"
-    LOG_INFO "./start_scheduler.sh"
-}
-
-pid=$(ps aux | grep $dirpath/ppc_scheduler/ppc_scheduler_app.py | grep -v grep | awk '{print $2}')
-
-if [ ! -z ${pid} ]; then
-    kill -9 ${pid}
-    echo " scheduler is running, pid is ${pid}, kill it."
-    exit 0
-else
-    echo " scheduler is not running."
-    exit 1
-fi
diff --git a/python/ppc_scheduler/workflow/__init__.py b/python/ppc_scheduler/workflow/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/builder/__init__.py b/python/ppc_scheduler/workflow/builder/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/builder/flow_builder.py b/python/ppc_scheduler/workflow/builder/flow_builder.py
deleted file mode 100644
index 207ff7fe..00000000
--- a/python/ppc_scheduler/workflow/builder/flow_builder.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# -*- coding: utf-8 -*-
-from datetime import datetime
-import json
-from ppc_common.db_models.job_worker_record import JobWorkerRecord
-from ppc_scheduler.common.global_context import components
-from ppc_scheduler.database import job_worker_mapper
-from ppc_scheduler.workflow.common import flow_utils
-from ppc_scheduler.workflow.common.worker_status import WorkerStatus
-
-
-class FlowBuilder:
-    def __init__(self, logger):
-        self.logger = logger
-
-    def build_flow_context(self, job_id, workflow_configs):
-        self.logger.info(f"## start build_flow_context, job_id: {job_id}")
-        flow_context = {}
-        index_type_map = {}
-        for workflow_config in workflow_configs:
-            workflow_type = workflow_config['type']
-            workflow_index = workflow_config['index']
-            workflow_args = workflow_config['args']
-            index_type_map[workflow_index] = workflow_type
-
-        for workflow_config in workflow_configs:
-            workflow_type = workflow_config['type']
-            workflow_index = workflow_config['index']
-            workflow_args = workflow_config['args']
-            worker_id = flow_utils.cat_worker_id(job_id, workflow_index, workflow_type)
-            upstreams = []
-            inputs_statement = []
-            inputs_statement_tuple = []
-            if 'upstreams' in workflow_config:
-                for upstream_config in workflow_config["upstreams"]:
-                    index = upstream_config['index']
-                    upstream_id = flow_utils.cat_worker_id(job_id, index, index_type_map[index])
-                    upstreams.append(upstream_id),
-                    if 'output_input_map' in upstream_config:
-                        for mapping in upstream_config.get("output_input_map", []):
-                            output_index, input_index = mapping.split(":")
-                            inputs_statement_tuple.append((upstream_id, int(output_index), int(input_index)))
-
-            inputs_statement_tuple.sort(key=lambda x: x[2])
-            for upstream_id, output_index, _ in inputs_statement_tuple:
-                inputs_statement.append(
-                    {
-                        'output_index': output_index,
-                        'upstream': upstream_id
-                    }
-                )
-                
-            worker_context = {
-                'job_id': job_id,
-                'worker_id': worker_id,
-                'type': workflow_type,
-                'status': WorkerStatus.PENDING,
-                'args': workflow_args,
-                'upstreams': upstreams,
-                'inputs_statement': inputs_statement
-                }
-            
-            self.logger.debug(f"## mid build_flow_context, work_context:\n{worker_context}")
-            
-            flow_context[worker_id] = worker_context
-        self.logger.info(f"## end build_flow_context, flow_context:\n{flow_context}")
-        
-        return flow_context
-
-    def save_flow_context(self, job_id, flow_context):
-        self.logger.info(f"## start save flow context, job_id: {job_id}")
-        with components.create_sql_session() as session:
-            for worker_id in flow_context:
-                worker_context = flow_context[worker_id]
-                insert_success = job_worker_mapper.insert_job_worker(session, worker_context)
-                if insert_success:
-                    # insert
-                    self.logger.info(f"## Save worker context successfully, job_id: {job_id}, worker_id: {worker_id}, work_context:\n{worker_context}")
-                    continue
-                # worker already exist
-                worker_record = job_worker_mapper.query_job_worker(session,job_id, worker_id)
-                worker_context['status'] = worker_record.status
-                worker_context['args'] = json.loads(worker_record.args)
-                worker_context['upstreams'] = json.loads(worker_record.upstreams)
-                worker_context['inputs_statement'] = json.loads(worker_record.inputs_statement)
-                self.logger.info(f"Load worker context successfully, job_id: {job_id}, worker_id: {worker_id}, work_context:\n{worker_context}")
-            self.logger.info(f"## end save flow context, job_id: {job_id}")
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/common/__init__.py b/python/ppc_scheduler/workflow/common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/common/codec.py b/python/ppc_scheduler/workflow/common/codec.py
deleted file mode 100644
index 45fa3e7d..00000000
--- a/python/ppc_scheduler/workflow/common/codec.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import json
-
-def deserialize_worker_outputs(outputs_str):
-    return json.loads(outputs_str)
-
-def serialize_worker_outputs(outputs):
-    return json.dumps(outputs)
-
-def deserialize_upstreams(upstreams_str):
-    return json.loads(upstreams_str)
-
-def serialize_upstreams(upstreams):
-    return json.dumps(upstreams)
-
-def deserialize_inputs_statement(inputs_statement_str):
-    return json.loads(inputs_statement_str)
-
-def serialize_inputs_statement(inputs_statement):
-    return json.dumps(inputs_statement)
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/common/flow_utils.py b/python/ppc_scheduler/workflow/common/flow_utils.py
deleted file mode 100644
index e2486e4b..00000000
--- a/python/ppc_scheduler/workflow/common/flow_utils.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-
-
-def cat_worker_id(job_id, index, worker_type):
-    return f"{job_id}_{index}_{worker_type}"
-
-
-def success_id(job_id):
-    return cat_worker_id(job_id, 0, WorkerType.T_ON_SUCCESS)
-
-
-def failure_id(job_id):
-    return cat_worker_id(job_id, 0, WorkerType.T_ON_FAILURE)
-
-
-def to_origin_inputs(worker_inputs):
-    inputs = []
-    for each in worker_inputs:
-        output_index = each['output_index']
-        upstream_outputs = each['upstream_outputs']
-        inputs.append(upstream_outputs[output_index])
-    return inputs
-
-
-def to_worker_inputs(job_workers, inputs_statement):
-    worker_inputs = []
-    for each in inputs_statement:
-        output_index = each['output_index']
-        upstream = each['upstream']
-        worker_inputs.append({
-            'output_index': output_index,
-            'upstream_outputs': job_workers[upstream]
-        })
-    return worker_inputs
diff --git a/python/ppc_scheduler/workflow/common/job_context.py b/python/ppc_scheduler/workflow/common/job_context.py
deleted file mode 100644
index 387af114..00000000
--- a/python/ppc_scheduler/workflow/common/job_context.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import os
-
-from ppc_scheduler.job.job_type import JobType
-from ppc_scheduler.mpc_generator.generator import CodeGenerator
-from ppc_scheduler.common.global_context import components
-
-
-class JobContext:
-    PSI_PREPARE_FILE = "psi_inputs"
-    PSI_RESULT_INDEX_FILE = "psi_result_index"
-    PSI_RESULT_FILE = "psi_result"
-    MPC_PREPARE_FILE = "mpc_prepare.csv"
-    MPC_RESULT_FILE = "mpc_result.csv"
-    MPC_OUTPUT_FILE = "mpc_output.txt"
-    HDFS_STORAGE_PATH = "/user/ppc/"
-
-    def __init__(self, job_id, workspace):
-        self.job_id = job_id
-        self.workspace = workspace
-
-        self.workflow_view_path = 'workflow_view'
-        self.job_cache_dir = "{}{}{}".format(self.workspace, os.sep, self.job_id)
-        # self.dataset_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.dataset_id)
-        self.psi_prepare_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_PREPARE_FILE)
-        self.psi_result_index_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_INDEX_FILE)
-        self.psi_result_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.PSI_RESULT_FILE)
-        self.mpc_file_name = "{}.mpc".format(self.job_id)
-        self.mpc_model_module_name = "{}.json".format(self.job_id)
-        self.mpc_file_path = "{}{}{}".format(self.job_cache_dir, os.sep, self.mpc_file_name)
-        self.mpc_prepare_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_PREPARE_FILE)
-        self.mpc_result_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_RESULT_FILE)
-        self.mpc_output_path = "{}{}{}".format(self.job_cache_dir, os.sep, JobContext.MPC_OUTPUT_FILE)
-
-    @staticmethod
-    def create_job_context(request: object, workspace: str):
-        job_context = JobContext(request, workspace)
-        
-        """
-        request format
-        {
-        "jobId": "job_id",
-        "agency": "WeBank",
-        "workflow": [
-            {
-            "index": 1,
-            "type": "WorkerType1",
-            "args": [
-                "arg1",
-                "arg2"
-            ]
-            },
-            {
-            "index": 2,
-            "type": "WorkerType1",
-            "args": [
-                "arg1",
-                "arg2"
-            ],
-            "upstreams": [
-                {
-                "index": 1
-                }
-            ]
-            }
-        ]
-        }
-        """
-        
-        job_context.request = request
-        job_context.job_id = request['jobId']
-        job_context.agency = request['agency']
-        job_context.workflow_configs = request['workflow']
-        
-        return job_context
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/common/worker_status.py b/python/ppc_scheduler/workflow/common/worker_status.py
deleted file mode 100644
index f915f473..00000000
--- a/python/ppc_scheduler/workflow/common/worker_status.py
+++ /dev/null
@@ -1,7 +0,0 @@
-class WorkerStatus:
-    PENDING = 'PENDING'
-    RUNNING = 'RUNNING'
-    SUCCESS = 'SUCCESS'
-    FAILURE = 'FAILURE'
-    TIMEOUT = 'TIMEOUT'
-    KILLED = 'KILLED'
diff --git a/python/ppc_scheduler/workflow/common/worker_type.py b/python/ppc_scheduler/workflow/common/worker_type.py
deleted file mode 100644
index 1a7314f2..00000000
--- a/python/ppc_scheduler/workflow/common/worker_type.py
+++ /dev/null
@@ -1,14 +0,0 @@
-class WorkerType:
-    # generic job worker
-    T_API = 'API'
-    T_PYTHON = 'PYTHON'
-    T_SHELL = 'SHELL'
-
-    # specific job worker
-    T_PSI = 'PSI'
-    T_MPC = 'MPC'
-    T_MODEL = "MODEL"
-
-    # finish job
-    T_ON_SUCCESS = 'T_ON_SUCCESS'
-    T_ON_FAILURE = 'T_ON_FAILURE'
diff --git a/python/ppc_scheduler/workflow/scheduler/__init__.py b/python/ppc_scheduler/workflow/scheduler/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/scheduler/scheduler.py b/python/ppc_scheduler/workflow/scheduler/scheduler.py
deleted file mode 100644
index 82f6262d..00000000
--- a/python/ppc_scheduler/workflow/scheduler/scheduler.py
+++ /dev/null
@@ -1,100 +0,0 @@
-import time
-from prefect import Flow
-from prefect.executors import LocalDaskExecutor
-from prefect.triggers import all_successful, any_failed
-
-from ppc_scheduler.workflow.common import flow_utils
-from ppc_scheduler.workflow.common.job_context import JobContext
-from ppc_scheduler.workflow.common.worker_status import WorkerStatus
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-from ppc_scheduler.workflow.builder.flow_builder import FlowBuilder
-from ppc_scheduler.workflow.worker.worker_factory import WorkerFactory
-from ppc_scheduler.workflow.scheduler.scheduler_api import SchedulerApi
-
-
-class Scheduler(SchedulerApi):
-    def __init__(self, workspace, logger):
-        self.workspace = workspace
-        self.logger = logger
-
-    def run(self, job_context: JobContext, flow_context: dict):
- 
-        job_id = job_context.job_id
-        job_flow = Flow(f"job_flow_{job_id}")
-
-        finish_job_on_success = WorkerFactory.build_worker(
-            job_context,
-            flow_utils.success_id(job_id),
-            WorkerType.T_ON_SUCCESS,
-            None
-            )
-
-        finish_job_on_success.trigger = all_successful
-        finish_job_on_success.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow)
-        job_flow.add_task(finish_job_on_success)
-
-        # set reference task to bind job flow status
-        job_flow.set_reference_tasks([finish_job_on_success])
-
-        # create a final job worker to handle failure
-        finish_job_on_failure = WorkerFactory.build_worker(
-            job_context,
-            flow_utils.failure_id(job_id),
-            WorkerType.T_ON_FAILURE,
-            None
-            )
-
-        # do finish_job_on_failure while any job worker failed
-        finish_job_on_failure.trigger = any_failed
-        finish_job_on_failure.bind(worker_status=WorkerStatus.PENDING, worker_inputs=[], flow=job_flow)
-        job_flow.add_task(finish_job_on_failure)
-        
-        job_workers = {}
-        # create main job workers
-        for worker_id in flow_context:
-            worker_type = flow_context[worker_id]['type']
-            worker_args = flow_context[worker_id]['args']
-            
-            job_worker = WorkerFactory.build_worker(job_context=job_context, worker_id=worker_id, worker_type=worker_type, worker_args=worker_args)
-            job_flow.add_task(job_worker)
-            job_workers[worker_id] = job_worker
-
-            # set upstream for final job
-            finish_job_on_success.set_upstream(job_worker, flow=job_flow)
-            finish_job_on_failure.set_upstream(job_worker, flow=job_flow)
-
-        # customize main job workers
-        for worker_id in flow_context:
-            # set upstream
-            upstreams = flow_context[worker_id]['upstreams']
-            status = flow_context[worker_id]['status']
-            for upstream in upstreams:
-                if upstream not in job_workers:
-                    raise Exception(-1, f"Not found upstream job worker : {upstream}, "
-                                        f"job_id: {job_context.job_id}")
-                job_workers[worker_id].set_upstream(job_workers[upstream], flow=job_flow)
-                
-            # bind worker inputs
-            inputs_statement = flow_context[worker_id]['inputs_statement']
-            worker_inputs = flow_utils.to_worker_inputs(job_workers, inputs_statement)
-            job_workers[worker_id].bind(worker_status=status,
-                                        worker_inputs=worker_inputs, flow=job_flow)
-
-        # enable parallel execution
-        job_flow.executor = LocalDaskExecutor()
-
-        #
-        start_time = time.time()
-
-        # run dag workflow
-        job_flow_state = job_flow.run()
-        
-        end_time = time.time()
-        
-        self.logger.info(f" ## Job worker result, job: {job_id}, success: {job_flow_state.is_successful()}, costs: {end_time - start_time}, flow_state: {job_flow_state}")
-
-        # save workflow view as file
-        job_flow.visualize(job_flow_state, job_context.workflow_view_path, 'svg')
-        
-        if not job_flow_state.is_successful():
-            raise Exception(-1, f"Job run failed, job_id: {job_id}")
diff --git a/python/ppc_scheduler/workflow/scheduler/scheduler_api.py b/python/ppc_scheduler/workflow/scheduler/scheduler_api.py
deleted file mode 100644
index 9ef5a25f..00000000
--- a/python/ppc_scheduler/workflow/scheduler/scheduler_api.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from abc import ABC, abstractmethod
-
-from ppc_scheduler.workflow.common.job_context import JobContext
-
-class SchedulerApi(ABC):
-    @abstractmethod
-    def run(self, job_context: JobContext, flow_context: dict):
-        pass
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/worker/__init__.py b/python/ppc_scheduler/workflow/worker/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/worker/api_worker.py b/python/ppc_scheduler/workflow/worker/api_worker.py
deleted file mode 100644
index 0160caf5..00000000
--- a/python/ppc_scheduler/workflow/worker/api_worker.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class ApiWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs):
-        ...
diff --git a/python/ppc_scheduler/workflow/worker/default_worker.py b/python/ppc_scheduler/workflow/worker/default_worker.py
deleted file mode 100644
index 02d2cf5f..00000000
--- a/python/ppc_scheduler/workflow/worker/default_worker.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import os
-import time
-
-from ppc_common.ppc_utils import utils
-from ppc_scheduler.common import log_utils
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class DefaultWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs):
-        # log_utils.upload_job_log(self.components.storage_client, self.job_context.job_id)
-        # self._save_workflow_view_file()
-        if self.worker_type == WorkerType.T_ON_FAILURE:
-            # notice job manager that this job has failed
-            raise Exception("Job run failed, job_id: " + self.job_context.job_id)
-
-    # TODO: 
-    # def _save_workflow_view_file(self):
-    #     file = f"{self.job_context.workflow_view_path}.svg"
-        
-    #     if not utils.file_exists(file):
-    #         return
-        
-    #     try_count = 10
-    #     while try_count > 0:
-    #         if utils.file_exists(file):
-    #             break
-    #         time.sleep(1)
-    #         try_count -= 1
-    #    self.components.storage_client.upload_file(file,
-    #                                                self.job_context.job_id + os.sep +
-    #                                                self.job_context.workflow_view_path)
diff --git a/python/ppc_scheduler/workflow/worker/engine/__init__.py b/python/ppc_scheduler/workflow/worker/engine/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/ppc_scheduler/workflow/worker/engine/model_engine.py b/python/ppc_scheduler/workflow/worker/engine/model_engine.py
deleted file mode 100644
index cd104310..00000000
--- a/python/ppc_scheduler/workflow/worker/engine/model_engine.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import time
-
-from ppc_scheduler.workflow.common.job_context import JobContext
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine
-
-
-class ModelWorkerEngine(WorkerEngine):
-    def __init__(self, model_client, worker_type, worker_id, components, job_context: JobContext):
-        self.model_client = model_client
-        self.worker_type = worker_type
-        self.worker_id = worker_id
-        self.components = components
-        self.job_context = job_context
-        self.logger = self.components.logger()
-
-    def run(self, *args) -> list:
-        if self.worker_type == WorkerType.T_MODEL:
-            pass
-        else:
-            raise ValueError(f"Unsupported worker type: {self.worker_type}")
-
-        job_id = self.job_context.job_id
-        start_time = time.time()
-
-        self.logger.info(
-            f"## model engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}")
-
-        # send job request to model node and wait for the job to finish
-        self.model_client.run(*args)
-
-        time_costs = time.time() - start_time
-        self.logger.info(
-            f"## model engine run finished, job_id={job_id}, timecost: {time_costs}s")
-
-        # args = {
-        #     'job_id': job_id,
-        #     'task_id': task_id,
-        #     'task_type': 'PREPROCESSING',
-        #     'dataset_id': self.job_context.dataset_id,
-        #     'dataset_storage_path': dataset_storage_path,
-        #     'job_algorithm_type': self.job_context.job_type,
-        #     'need_run_psi': self.job_context.need_run_psi,
-        #     'model_dict': self.job_context.model_config_dict
-        # }
-        # self.log.info(f"start prepare_xgb, job_id: {job_id}, task_id: {task_id}, args: {args}")
-        # self.model_client.run(args)
-        # self.log.info(
-        #     f"call compute_xgb_job service success, job: {job_id}, "
-        #     f"task_id: {task_id}, timecost: {time.time() - start}")
-
-        return []
diff --git a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py b/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py
deleted file mode 100644
index f55d091e..00000000
--- a/python/ppc_scheduler/workflow/worker/engine/mpc_engine.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-import os
-from time import time
-
-import pandas as pd
-
-from ppc_common.ppc_utils import utils
-from ppc_scheduler.mpc_generator.generator import CodeGenerator
-from ppc_scheduler.workflow.common.job_context import JobContext
-from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine
-
-
-class MpcWorkerEngine(WorkerEngine):
-    def __init__(self, mpc_client, worker_type, components, job_context: JobContext):
-        self.mpc_client = mpc_client
-        self.worker_type = worker_type
-        self.components = components
-        self.job_context = job_context
-        self.logger = self.components.logger()
-
-    def run(self, *args) -> list:
-        
-        job_id = self.job_context.job_id
-        start_time = time.time()
-
-        self.logger.info(f"computing mpc begin, job_id={job_id}")
-        
-        time_costs = time.time() - start_time
-        self.logger.info(f"computing mpc finished, job_id={job_id}, timecost: {time_costs}s")
-        
-        return [self.job_context.mpc_output_path]
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py b/python/ppc_scheduler/workflow/worker/engine/psi_engine.py
deleted file mode 100644
index b6e735cc..00000000
--- a/python/ppc_scheduler/workflow/worker/engine/psi_engine.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-import os
-import time
-
-from ppc_common.ppc_utils import utils, common_func
-from ppc_scheduler.workflow.common.job_context import JobContext
-from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine
-
-
-class PsiWorkerEngine(WorkerEngine):
-    def __init__(self, psi_client, worker_id, worker_type, components, job_context: JobContext):
-        self.psi_client = psi_client
-        self.worker_id = worker_id
-        self.worker_type = worker_type
-        self.components = components
-        self.job_context = job_context
-        self.logger = self.components.logger()
-
-    def run(self, *args) -> list:
-        job_id = self.job_context.job_id
-        start_time = time.time()
-
-        self.logger.info(f"## psi engine run begin, job_id={job_id}, worker_id={self.worker_id}, args: {args}")
-        
-        # send job request to psi node and wait for the job to finish
-        self.psi_client.run(*args)
-        
-        time_costs = time.time() - start_time
-        self.logger.info(f"## psi engine run finished, job_id={job_id}, timecost: {time_costs}s")
-        
-        return [JobContext.HDFS_STORAGE_PATH + job_id + os.sep + self.job_context.PSI_RESULT_INDEX_FILE]
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/worker/engine/shell_engine.py b/python/ppc_scheduler/workflow/worker/engine/shell_engine.py
deleted file mode 100644
index a406f8a7..00000000
--- a/python/ppc_scheduler/workflow/worker/engine/shell_engine.py
+++ /dev/null
@@ -1,16 +0,0 @@
-import os
-from ppc_scheduler.workflow.worker.engine.work_engine import WorkerEngine
-from ppc_scheduler.common.global_context import components
-
-class ShellEngine(WorkerEngine):
-    
-    def __init__(self, cmd: str) -> None:
-        super().__init__()
-        self.cmd = cmd
-    
-    def run(self, *args):
-        # print("shell engine is processing.")
-        components.logger().info(f"shell engine is processing, cmd: {self.cmd}")
-        
-        result = os.system(self.cmd)
-        return list(str(result))
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/worker/engine/work_engine.py b/python/ppc_scheduler/workflow/worker/engine/work_engine.py
deleted file mode 100644
index 6cff7e23..00000000
--- a/python/ppc_scheduler/workflow/worker/engine/work_engine.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from abc import ABC, abstractmethod
-
-class WorkerEngine(ABC):
-    @abstractmethod
-    def run(self, *args):
-        pass
\ No newline at end of file
diff --git a/python/ppc_scheduler/workflow/worker/model_worker.py b/python/ppc_scheduler/workflow/worker/model_worker.py
deleted file mode 100644
index 0268e705..00000000
--- a/python/ppc_scheduler/workflow/worker/model_worker.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from ppc_scheduler.node.computing_node_client.model_node_client import ModelClient
-from ppc_scheduler.workflow.worker.engine.model_engine import ModelWorkerEngine
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class ModelWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs):
-        model_client_node = self.node_manager.get_node(self.worker_type)
-        model_client = ModelClient(self.components.logger(), model_client_node[0], model_client_node[1])
-        model_engine = ModelWorkerEngine(model_client, self.worker_type, self.worker_id, self.components, self.job_context)
-        try:
-            outputs = model_engine.run(*self.worker_args)
-            return outputs
-        finally:
-            self.node_manager.release_node(model_client_node, self.worker_type)
diff --git a/python/ppc_scheduler/workflow/worker/mpc_worker.py b/python/ppc_scheduler/workflow/worker/mpc_worker.py
deleted file mode 100644
index c8fc64eb..00000000
--- a/python/ppc_scheduler/workflow/worker/mpc_worker.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from ppc_scheduler.node.computing_node_client.mpc_node_client import MpcClient
-from ppc_scheduler.workflow.worker.engine.mpc_engine import MpcWorkerEngine
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class MpcWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs) -> list:
-        node_endpoint = self.node_manager.get_node(self.worker_type)
-        mpc_client = MpcClient(node_endpoint)
-        mpc_engine = MpcWorkerEngine(mpc_client, self.worker_type, self.components, self.job_context)
-        try:
-            outputs = mpc_engine.run()
-            return outputs
-        finally:
-            self.node_manager.release_node(node_endpoint, self.worker_type)
diff --git a/python/ppc_scheduler/workflow/worker/psi_worker.py b/python/ppc_scheduler/workflow/worker/psi_worker.py
deleted file mode 100644
index bf596268..00000000
--- a/python/ppc_scheduler/workflow/worker/psi_worker.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from ppc_scheduler.node.computing_node_client.psi_client import PsiClient
-from ppc_scheduler.workflow.worker.engine.psi_engine import PsiWorkerEngine
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class PsiWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs) -> list:
-        psi_client_node = self.node_manager.get_node(self.worker_type)
-        logger = self.components.logger()
-        logger.info(f"## getting psi client : {psi_client_node}")
-        psi_client = PsiClient(logger, psi_client_node[0], psi_client_node[1])
-        psi_engine = PsiWorkerEngine(psi_client, self.worker_id, self.worker_type, self.components, self.job_context)
-        try:
-            outputs = psi_engine.run(*self.worker_args)
-            return outputs
-        finally:
-            self.node_manager.release_node(psi_client_node, self.worker_type)
diff --git a/python/ppc_scheduler/workflow/worker/python_worker.py b/python/ppc_scheduler/workflow/worker/python_worker.py
deleted file mode 100644
index 9d5e295b..00000000
--- a/python/ppc_scheduler/workflow/worker/python_worker.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from ppc_scheduler.workflow.worker.worker import Worker
-
-
-class PythonWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs):
-        ...
diff --git a/python/ppc_scheduler/workflow/worker/shell_worker.py b/python/ppc_scheduler/workflow/worker/shell_worker.py
deleted file mode 100644
index 3317593d..00000000
--- a/python/ppc_scheduler/workflow/worker/shell_worker.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from ppc_scheduler.workflow.worker.worker import Worker
-from ppc_scheduler.workflow.worker.engine.shell_engine import ShellEngine
-
-
-class ShellWorker(Worker):
-
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        super().__init__(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-
-    def engine_run(self, worker_inputs):   
-        
-        # self.log_worker()
-             
-        mpc_engine = ShellEngine(cmd=self.worker_args[0])
-        outputs = mpc_engine.run()
-        return outputs
diff --git a/python/ppc_scheduler/workflow/worker/worker.py b/python/ppc_scheduler/workflow/worker/worker.py
deleted file mode 100644
index 290179f6..00000000
--- a/python/ppc_scheduler/workflow/worker/worker.py
+++ /dev/null
@@ -1,117 +0,0 @@
-import time
-
-from func_timeout import FunctionTimedOut
-from prefect import Task
-from prefect.engine import signals
-
-from ppc_scheduler.common import log_utils
-from ppc_scheduler.database import job_worker_mapper
-from ppc_scheduler.node.node_manager import ComputingNodeManager
-from ppc_scheduler.workflow.common import codec, flow_utils
-from ppc_scheduler.workflow.common.worker_status import WorkerStatus
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-
-
-class Worker(Task):
-    def __init__(self, components, job_context, worker_id, worker_type, worker_args, retries=0, retry_delay_s=0, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.components = components
-        self.node_manager = ComputingNodeManager(components)
-        self.logger = components.logger()
-        self.job_context = job_context
-        self.worker_id = worker_id
-        self.worker_type = worker_type
-        self.worker_args = worker_args
-        self.retries = retries
-        self.retry_delay_s = retry_delay_s
-
-    def log_worker(self):
-        self.components.logger().info(" ## View worker, job_id = %s, work_id = %s", self.job_context.job_id, self.worker_id)
-
-    def engine_run(self, worker_inputs) -> list:
-        # this func should be implemented by subclass
-        ...
-
-    def run(self, worker_status, worker_inputs):
-        self.components.logger().info("Worker begin run, job_id = %s, work_id = %s, work_status = %s", self.job_context.job_id, self.worker_id, worker_status)
-        start_time = time.time()
-        try:
-            # job is killed
-            if self.components.thread_event_manager.event_status(self.job_context.job_id):
-                self._write_failed_status(WorkerStatus.KILLED)
-                self.logger.warn(
-                    f" worker was killed, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}")
-                raise signals.FAIL(message='killed!')
-
-            if worker_status == WorkerStatus.SUCCESS:
-                self.logger.info(
-                    f" Worker has been executed successfully, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}")
-                return self._load_output_from_db()
-
-            inputs = []
-            if self.worker_type != WorkerType.T_ON_SUCCESS \
-                    and self.worker_type != WorkerType.T_ON_FAILURE:
-                inputs = flow_utils.to_origin_inputs(worker_inputs)
-            
-            outputs = self._try_run_task(inputs)
-            if outputs is not None:
-                self._save_worker_result(outputs)
-        except FunctionTimedOut:
-            self._write_failed_status(WorkerStatus.FAILURE)
-            end_time = time.time()
-            self.logger.error(
-                f"[OnError] job worker was timeout, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}, elapsed time: {end_time - start_time}")
-            raise signals.FAIL(message='timeout!')
-        except BaseException as be:
-            self._write_failed_status(WorkerStatus.FAILURE)
-            self.logger.error(f"[OnError] job worker failed, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}")
-            self.logger.exception(be)
-            raise signals.FAIL(message='failed!')
-
-    def _try_run_task(self, inputs):
-        start_time = time.time()
-        # self.logger.info(f"Worker try run task, job_id: {self.job_context.job_id}, worker_id: {self.worker_id}, inputs: {inputs}")
-        # parse inputs for worker
-        if self.retries:
-            attempt = 0
-            while attempt <= self.retries:
-                try:
-                    self.logger.info(log_utils.worker_start_log_info(self.worker_id))
-                    outputs = self.engine_run(inputs)
-                    self.logger.info(log_utils.worker_end_log_info(self.worker_id))
-                    return outputs
-                except Exception as e:
-                    attempt += 1
-                    if attempt > self.retries:
-                        self.logger.error(
-                            f"worker failed after {self.retries} attempts, "
-                            f"job_id: {self.job_context.job_id}, worker_id: {self.worker_id}")
-                        raise e
-                    else:
-                        self.logger.warn(
-                            f"worker failed, attempts: {attempt}"
-                            f"job_id: {self.job_context.job_id}, worker_id: {self.worker_id}")
-                        self.logger.exception(e)
-                        time.sleep(self.retry_delay_s)
-        else:
-            # outputs = self.engine_run(inputs)
-            self.logger.info(log_utils.worker_start_log_info(self.worker_id))
-            outputs = self.engine_run(inputs)
-            self.logger.info(log_utils.worker_end_log_info(self.worker_id))
-            end_time = time.time()
-            # self.logger.info(f"Worker try run task end, job_id: {self.job_context.job_id}, worker: {self.worker_id}, elapsed time: {end_time - start_time}, outputs: {outputs}")
-            return outputs
-
-    def _load_output_from_db(self):
-        with self.components.create_sql_session() as session:
-            worker_record = job_worker_mapper.query_job_worker(session, self.job_context.job_id, self.worker_id)
-            return codec.deserialize_worker_outputs(worker_record.outputs)
-
-    def _save_worker_result(self, outputs):
-        with self.components.create_sql_session() as session:
-            job_worker_mapper.update_job_worker(session, self.job_context.job_id, self.worker_id,
-                                                WorkerStatus.SUCCESS, outputs)
-
-    def _write_failed_status(self, status):
-        with self.components.create_sql_session() as session:
-            job_worker_mapper.update_job_worker(session, self.job_context.job_id, self.worker_id, status, [])
diff --git a/python/ppc_scheduler/workflow/worker/worker_factory.py b/python/ppc_scheduler/workflow/worker/worker_factory.py
deleted file mode 100644
index c8573185..00000000
--- a/python/ppc_scheduler/workflow/worker/worker_factory.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from ppc_common.ppc_utils.exception import PpcErrorCode, PpcException
-from ppc_scheduler.common.global_context import components
-from ppc_scheduler.workflow.common.worker_type import WorkerType
-from ppc_scheduler.workflow.worker.api_worker import ApiWorker
-from ppc_scheduler.workflow.worker.default_worker import DefaultWorker
-from ppc_scheduler.workflow.worker.model_worker import ModelWorker
-from ppc_scheduler.workflow.worker.mpc_worker import MpcWorker
-from ppc_scheduler.workflow.worker.psi_worker import PsiWorker
-from ppc_scheduler.workflow.worker.python_worker import PythonWorker
-from ppc_scheduler.workflow.worker.shell_worker import ShellWorker
-
-
-class WorkerFactory:
-
-    @staticmethod
-    def build_worker(job_context, worker_id, worker_type, worker_args, *args, **kwargs):
-        if worker_type == WorkerType.T_API:
-            return ApiWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs)
-        elif worker_type == WorkerType.T_PYTHON:
-            return PythonWorker(components, job_context, worker_id, worker_type, worker_args, *args, *kwargs)
-        elif worker_type == WorkerType.T_SHELL:
-            return ShellWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-        elif worker_type == WorkerType.T_PSI:
-            return PsiWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-        elif worker_type == WorkerType.T_MPC:
-            return MpcWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-        elif worker_type == WorkerType.T_MODEL:
-            return ModelWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-        elif worker_type == WorkerType.T_ON_SUCCESS or \
-                worker_type == WorkerType.T_ON_FAILURE:
-            return DefaultWorker(components, job_context, worker_id, worker_type, worker_args, *args, **kwargs)
-        else:
-            raise PpcException(PpcErrorCode.UNSUPPORTED_WORK_TYPE,
-                               f"Unsupported worker type: {worker_type}")
diff --git a/python/requirements.txt b/python/requirements.txt
index e464227f..b92953e4 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -37,7 +37,6 @@ google~=3.0.0
 paste~=3.5.0
 func_timeout==4.3.0
 cheroot==8.5.2
-prefect==1.4.0
 gmssl~=3.2.1
 readerwriterlock~=1.0.4
 jsoncomment~=0.2.3

From c4cc6cd20a1e3833dbdf0683e35a748a218a2072 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 28 Oct 2024 10:31:09 +0800
Subject: [PATCH 069/120] ihc support negative (#66)

---
 cpp/cmake/CompilerSettings.cmake              | 35 +++++++---
 cpp/tools/install_depends.sh                  | 19 +++++-
 cpp/wedpr-protocol/grpc/client/GrpcClient.cpp |  1 +
 python/ppc_common/ppc_crypto/ihc_cipher.py    | 45 ++++++++++---
 .../ppc_crypto/test/phe_unittest.py           | 67 ++++++++++++++++---
 5 files changed, 137 insertions(+), 30 deletions(-)

diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 9596d8ff..61396933 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -1,3 +1,12 @@
+set(CMAKE_CXX_STANDARD 20)
+set(Boost_NO_WARN_NEW_VERSIONS ON)
+message(STATUS "COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
+
+# export windows dll symbol
+if(WIN32)
+    message(STATUS "Compile on Windows")
+    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON")
+endif()
 if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
     find_program(CCACHE_PROGRAM ccache)
     if(CCACHE_PROGRAM)
@@ -123,6 +132,7 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
     endif()
     endif ()
 elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
+    add_compile_definitions(NOMINMAX)
 
     # Only support visual studio 2017 and visual studio 2019
     set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support
@@ -137,16 +147,23 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
         message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
     endif()
 
-    add_definitions(-DUSE_STD_RANGES)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17")
-    add_compile_definitions(NOMINMAX)
+    add_compile_options(/std:c++latest)
     add_compile_options(-bigobj)
-     # MSVC only support static build
-     set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
-     set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
-     set(CMAKE_CXX_FLAGS_RELEASE "/MT")
-     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
-     link_libraries(ws2_32 Crypt32 userenv)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
+    if(BUILD_SHARED_LIBS)
+        if(CMAKE_BUILD_TYPE MATCHES "Debug")
+            add_compile_options(/MDd)
+        else()
+            add_compile_options(/MD)
+        endif()
+    else ()
+        if(CMAKE_BUILD_TYPE MATCHES "Debug")
+            add_compile_options(/MTd)
+        else()
+            add_compile_options(/MT)
+        endif ()
+    endif ()
+    link_libraries(ws2_32 Crypt32 userenv)
 else ()
     message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.")
 endif ()
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index 108a6aae..a4c467aa 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -26,6 +26,23 @@ install_gsasl_depend()
         return
     fi
     LOG_INFO "download and install gsasl..."
+    wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/gsasl-1.8.0.tar.gz && tar -xvf gsasl-1.8.0.tar.gz 
+    
+    # centos
+    if [[ "${os_type}" == "centos" ]];then
+        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
+    fi
+    # macos
+    if [[ "${os_type}" == "macos" ]];then
+        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
+    fi
+    # ubuntu
+    if [[ "${os_type}" == "ubuntu" ]];then
+        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
+    fi
+    LOG_INFO "download and install gsasl success..."
+
+    LOG_INFO "download and install libgsasl..."
     wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz && tar -xvf libgsasl-1.8.0.tar.gz 
     
     # centos
@@ -40,7 +57,7 @@ install_gsasl_depend()
     if [[ "${os_type}" == "ubuntu" ]];then
         cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
     fi
-    LOG_INFO "download and install gsasl success..."
+    LOG_INFO "download and install libgsasl success..."
 }
 
 install_nasm_depend()
diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
index c0b3d0a5..9eb3429c 100644
--- a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
@@ -19,6 +19,7 @@
  */
 #include "GrpcClient.h"
 #include "Common.h"
+#include 
 
 using namespace ppc::protocol;
 using namespace ppc::proto;
diff --git a/python/ppc_common/ppc_crypto/ihc_cipher.py b/python/ppc_common/ppc_crypto/ihc_cipher.py
index a0f7c132..4adfeb1c 100644
--- a/python/ppc_common/ppc_crypto/ihc_cipher.py
+++ b/python/ppc_common/ppc_crypto/ihc_cipher.py
@@ -7,21 +7,43 @@
 import secrets
 
 
+class NumberCodec:
+    def __init__(self, key_length):
+        self.key_length = key_length
+        self.max_mod = 1 << key_length
+        self.non_negative_range = int((1 << key_length) / 3)
+        self.negative_range = int((1 << int(key_length + 1)) / 3)
+
+    def encode(self, value):
+        if value > 0:
+            if value > self.non_negative_range:
+                raise Exception(f"The value {value} out of range")
+            return value
+        return value % self.max_mod
+
+    def decode(self, value):
+        if value > self.negative_range:
+            return value - self.max_mod
+        return value
+
+
 @dataclass
 class IhcCiphertext():
-    __slots__ = ['c_left', 'c_right']
+    __slots__ = ['c_left', 'c_right', 'number_codec']
 
-    def __init__(self, c_left: int, c_right: int) -> None:
+    def __init__(self, c_left: int, c_right: int, number_codec: NumberCodec) -> None:
         self.c_left = c_left
         self.c_right = c_right
+        self.number_codec = number_codec
 
     def __add__(self, other):
         cipher_left = self.c_left + other.c_left
         cipher_right = self.c_right + other.c_right
-        return IhcCiphertext(cipher_left, cipher_right)
-    
+        return IhcCiphertext(cipher_left, cipher_right, self.number_codec)
+
     def __mul__(self, num: int):
-        return IhcCiphertext(num * self.c_left, num * self.c_right)
+        num_value = self.number_codec.encode(num)
+        return IhcCiphertext(num_value * self.c_left, num_value * self.c_right, self.number_codec)
 
     def __eq__(self, other):
         return self.c_left == other.c_left and self.c_right == other.c_right
@@ -51,7 +73,7 @@ def decode(cls, encoded_data: bytes):
             encoded_data[8:8 + len_c_left], byteorder='big')
         c_right = int.from_bytes(
             encoded_data[8 + len_c_left:8 + len_c_left + len_c_right], byteorder='big')
-        return cls(c_left, c_right)
+        return cls(c_left, c_right, cls.number_codec)
 
 
 class IhcCipher(PheCipher):
@@ -64,17 +86,18 @@ def __init__(self, key_length: int = 256, iter_round: int = 16) -> None:
         self.key_length = key_length
 
         self.max_mod = 1 << key_length
+        self.number_codec = NumberCodec(self.key_length)
 
     def encrypt(self, number: int) -> IhcCiphertext:
         random_u = secrets.randbits(self.key_length)
-        x_this = number
+        x_this = self.number_codec.encode(number)
+        # print(f"###### x_this: {x_this}, number: {number}")
         x_last = random_u
         for i in range(0, self.iter_round):
             x_tmp = (self.private_key * x_this - x_last) % self.max_mod
             x_last = x_this
             x_this = x_tmp
-        # cipher = IhcCiphertext(x_this, x_last, self.max_mod)
-        cipher = IhcCiphertext(x_this, x_last)
+        cipher = IhcCiphertext(x_this, x_last, self.number_codec)
         return cipher
 
     def decrypt(self, cipher: IhcCiphertext) -> int:
@@ -84,7 +107,9 @@ def decrypt(self, cipher: IhcCiphertext) -> int:
             x_tmp = (self.private_key * x_this - x_last) % self.max_mod
             x_last = x_this
             x_this = x_tmp
-        return x_this
+        result = self.number_codec.decode(x_this)
+        # print(f"###### x_this: {x_this}, result: {result}")
+        return result
 
     def encrypt_batch(self, numbers) -> list:
         return [self.encrypt(num) for num in numbers]
diff --git a/python/ppc_common/ppc_crypto/test/phe_unittest.py b/python/ppc_common/ppc_crypto/test/phe_unittest.py
index a8ef12a7..b7381a1e 100644
--- a/python/ppc_common/ppc_crypto/test/phe_unittest.py
+++ b/python/ppc_common/ppc_crypto/test/phe_unittest.py
@@ -8,11 +8,13 @@
 from ppc_common.ppc_crypto.paillier_cipher import PaillierCipher
 
 
-class PaillierUtilsTest(unittest.TestCase):
+class PaillierTest:
+    def __init__(self, ut):
+        self.paillier = PaillierCipher(key_length=1024)
+        self.ut = ut
 
-    def test_enc_and_dec_parallel(self):
-        paillier = PaillierCipher(key_length=1024)
-        inputs = np.random.randint(1, 10001, size=10)
+    def test_enc_and_dec_parallel(self, test_size, start, end):
+        inputs = np.random.randint(start, end, size=test_size)
 
         # start_time = time.time()
         # paillier.encrypt_batch(inputs)
@@ -20,21 +22,66 @@ def test_enc_and_dec_parallel(self):
         # print("enc:", end_time - start_time, "seconds")
 
         start_time = time.time()
-        ciphers = paillier.encrypt_batch_parallel(inputs)
+        ciphers = self.paillier.encrypt_batch_parallel(inputs)
         end_time = time.time()
         print("enc_p:", end_time - start_time, "seconds")
 
         start_time = time.time()
-        outputs = paillier.decrypt_batch_parallel(ciphers)
+        outputs = self.paillier.decrypt_batch_parallel(ciphers)
         end_time = time.time()
         print("dec_p:", end_time - start_time, "seconds")
 
-        self.assertListEqual(list(inputs), list(outputs))
+        self.ut.assertListEqual(list(inputs), list(outputs))
+        self.test_ihc_mul_enc_and_dec(ciphers, inputs, 10)
+        # test add and enc dec
+        inputs2 = np.random.randint(start, end, size=test_size)
+        ciphers2 = self.paillier.encrypt_batch_parallel(inputs2)
+        self.test_ihc_add_enc_and_desc(ciphers, ciphers2, inputs, inputs2)
+
+    def test_ihc_mul_enc_and_dec(self, ciphers, inputs, mul_value):
+        start_time = time.time()
+        mul_ciphers = []
+        for cipher in ciphers:
+            cipher.__mul__(mul_value)
+            mul_ciphers.append(cipher * (mul_value))
+        # decrypt
+        outputs = self.paillier.decrypt_batch_parallel(mul_ciphers)
+        mul_result = []
+        for input in inputs:
+            mul_result.append(mul_value * input)
+        self.ut.assertListEqual(mul_result, list(outputs))
+        end_time = time.time()
+        print(
+            f"#### test_ihc_mul_enc_and_desc passed, time: {end_time - start_time} seconds")
+
+    def test_ihc_add_enc_and_desc(self, ciphers1, ciphers2, inputs1, inputs2):
+        start_time = time.time()
+        add_ciphers = []
+        i = 0
+        expected_result = []
+        for cipher in ciphers1:
+            add_ciphers.append(cipher + ciphers2[i])
+            expected_result.append(inputs1[i] + inputs2[i])
+            i += 1
+        outputs = self.paillier.decrypt_batch_parallel(add_ciphers)
+        self.ut.assertListEqual(expected_result, list(outputs))
+        end_time = time.time()
+        print(
+            f"#### test_ihc_add_enc_and_desc passed, time: {end_time - start_time} seconds, size: {len(inputs1)}")
+
+
+class PaillierUtilsTest(unittest.TestCase):
+
+    def test_enc_and_dec_parallel(self):
+        paillier_test = PaillierTest(self)
+        paillier_test.test_enc_and_dec_parallel(10, -20, -1)
+        paillier_test.test_enc_and_dec_parallel(10000, -20000, 20000)
+        paillier_test.test_enc_and_dec_parallel(10000, 0, 20000)
 
     def test_ihc_enc_and_dec_parallel(self):
         ihc = IhcCipher(key_length=256)
         try_size = 100000
-        inputs = np.random.randint(1, 10001, size=try_size)
+        inputs = np.random.randint(-10001, 10001, size=try_size)
         expected = np.sum(inputs)
 
         start_time = time.time()
@@ -50,7 +97,7 @@ def test_ihc_enc_and_dec_parallel(self):
             cipher_left = (cipher_start.c_left + ciphers[i].c_left)
             cipher_right = (cipher_start.c_right + ciphers[i].c_right)
             # IhcCiphertext(cipher_left, cipher_right, cipher_start.max_mod)
-            IhcCiphertext(cipher_left, cipher_right)
+            IhcCiphertext(cipher_left, cipher_right, ihc.number_codec)
         end_time = time.time()
         print(f"size:{try_size}, add_p raw with class: {end_time - start_time} seconds, average times: {(end_time - start_time)/try_size * 1000 * 1000} us")
 
@@ -86,7 +133,7 @@ def test_ihc_enc_and_dec_parallel(self):
     def test_ihc_code(self):
         ihc = IhcCipher(key_length=256)
         try_size = 100000
-        inputs = np.random.randint(1, 10001, size=try_size)
+        inputs = np.random.randint(-10001, 10001, size=try_size)
         start_time = time.time()
         ciphers = ihc.encrypt_batch_parallel(inputs)
         end_time = time.time()

From 40aab9f49760a945830acf13aee9a06ed6b16274 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 28 Oct 2024 15:46:53 +0800
Subject: [PATCH 070/120] decrease copy overhead (#67)

* upload cpp_toolkit

* decrease copy overhead

* update swig generated files
---
 .github/workflows/cpp_full_node_workflow.yml  |  33 +-
 .github/workflows/cpp_toolkit_workflow.yml    |   8 +-
 cpp/CMakeLists.txt                            |   2 +-
 cpp/cmake/python.cmake                        |   2 +-
 cpp/cmake/swig.cmake                          |   1 +
 cpp/ppc-framework/front/IFront.h              |  28 +-
 cpp/ppc-framework/protocol/Message.h          |  14 +-
 cpp/ppc-framework/protocol/MessagePayload.h   |  18 +-
 .../protocol/src/v1/MessageHeaderImpl.cpp     |   7 +-
 .../protocol/src/v1/MessagePayloadImpl.cpp    |   9 +-
 .../ppc-front/ppc-front/Front.cpp             |   8 +-
 .../ppc-front/ppc-front/FrontImpl.cpp         |  16 +-
 .../ppc-front/ppc-front/FrontImpl.h           |  12 +-
 .../sdk-wrapper/java/CMakeLists.txt           |   1 -
 .../wedpr/sdk/jni/generated/IFront.java       |  34 -
 .../jni/generated/MessageOptionalHeader.java  |  24 +-
 .../sdk/jni/generated/MessagePayload.java     |   9 -
 .../generated/wedpr_java_transportJNI.java    |  34 +-
 .../src/wedpr_java_transportJAVA_wrap.cxx     | 204 +-----
 .../java/swig/wedpr_java_transport.i          |  12 +
 .../sdk-wrapper/python/CMakeLists.txt         |   1 -
 .../generated/wedpr_python_transport.py       |  34 +-
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 640 +-----------------
 .../python/swig/wedpr_python_transport.i      |  12 +
 24 files changed, 136 insertions(+), 1027 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 7779352c..77136375 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -64,7 +64,7 @@ jobs:
           sudo apt install -y lcov ccache wget libgmp-dev python3-dev
           export GCC='gcc-10'
           export CXX='g++-10'
-          bash -x cpp/tools/install_depends.sh -o ubuntu
+          sudo bash -x cpp/tools/install_depends.sh -o ubuntu
           mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ || cat /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-osx-dbg-err.log
       - name: Build for macOS
         if: runner.os == 'macOS'
@@ -78,6 +78,21 @@ jobs:
         with:
             name: config-x64-linux-dbg-err.log
             path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macOS'      
+        with:
+          name: ppc-air-node-macOS
+          path: ./cpp/build/bin/ppc-air-node
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macOS'
+        with:
+          name: ppc-pro-node-macOS
+          path: ./cpp/build/bin/ppc-pro-node
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macOS'
+        with:
+          name: ppc-gateway-service-macOS
+          path: ./cpp/build/bin/ppc-gateway-service
       #- name: Test
       #  if: runner.os != 'Windows'
       #  run: |
@@ -126,7 +141,7 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
+          yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
           yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
           yum install -y git
       - uses: actions-rs/toolchain@v1
@@ -135,7 +150,7 @@ jobs:
           override: true
       - name: Build
         run: |
-          bash cpp/tools/install_depends.sh -o centos
+          bash -x cpp/tools/install_depends.sh -o centos
           alias cmake='cmake3'
           . /opt/rh/devtoolset-11/enable 
           mkdir -p cpp/build
@@ -154,17 +169,13 @@ jobs:
       #    cd build && CTEST_OUTPUT_ON_FAILURE=TRUE make test
       - uses: actions/upload-artifact@v3
         with:
-          name: ppc-air-node-centos-x64
+          name: ppc-air-node-x64
           path: ./cpp/build/bin/ppc-air-node
       - uses: actions/upload-artifact@v3
         with:
-          name: ppc-pro-node-centos-x64
+          name: ppc-pro-node-x64
           path: ./cpp/build/bin/ppc-pro-node
       - uses: actions/upload-artifact@v3
         with:
-          name: ppc-gateway-service-centos-x64
-          path: ./cpp/build/bin/ppc-gateway-service
-      - uses: actions/upload-artifact@v3
-        with:
-          name: libppc-crypto-sdk-jni.so
-          path: ./cpp/wedpr-component-sdk/bindings/java/src/main/resources/META-INF/native/libppc-crypto-sdk-jni.so
+          name: ppc-gateway-service-x64
+          path: ./cpp/build/bin/ppc-gateway-service
\ No newline at end of file
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index fdcdc9a4..fa1dc2e8 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -79,7 +79,7 @@ jobs:
         run: |
           brew install lcov
           bash cpp/tools/install_depends.sh -o macos
-          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          mkdir -p cpp/build && cd cpp/build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - uses: actions/upload-artifact@v4
         if: runner.os == 'macos'
@@ -146,4 +146,8 @@ jobs:
             mkdir -p cpp/build
             cd cpp/build
             cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-            cmake3 --build . --parallel 3
\ No newline at end of file
+            cmake3 --build . --parallel 3
+        - uses: actions/upload-artifact@v3
+          with:
+            name: libwedpr_java_transport_jni.so
+            path: ./cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native/libwedpr_java_transport_jni.so
\ No newline at end of file
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ca080810..d6f941b4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -101,8 +101,8 @@ elseif(BUILD_UDF)
 elseif(BUILD_SDK)
     add_sources("${SDK_SOURCE_LIST}")
 elseif(BUILD_WEDPR_TOOLKIT)
-    add_sources("${TRANSPORT_SDK_TOOLKIT_SOURCE_LIST}")
     include(swig)
+    add_sources("${TRANSPORT_SDK_TOOLKIT_SOURCE_LIST}")
 endif()
 ########### set the sources end ########### 
 
diff --git a/cpp/cmake/python.cmake b/cpp/cmake/python.cmake
index 2fd2b776..c726136e 100644
--- a/cpp/cmake/python.cmake
+++ b/cpp/cmake/python.cmake
@@ -9,7 +9,7 @@ message("Python_Development_FOUND:${Python_Development_FOUND}")
 message("Python_LIBRARIES:${Python_LIBRARIES}")
 message("Python_INCLUDE_DIRS:${Python_INCLUDE_DIRS}")
 
-list(APPEND CMAKE_SWIG_FLAGS "-py3" "-DPY3")
+list(APPEND CMAKE_SWIG_FLAGS "-DPY3")
 
 function(search_python_module)
   set(options NO_VERSION)
diff --git a/cpp/cmake/swig.cmake b/cpp/cmake/swig.cmake
index d1e39a2f..850d636b 100644
--- a/cpp/cmake/swig.cmake
+++ b/cpp/cmake/swig.cmake
@@ -7,6 +7,7 @@ cmake_minimum_required(VERSION 3.18)
 
 # Will need swig
 set(CMAKE_SWIG_FLAGS)
+
 find_package(SWIG REQUIRED)
 include(UseSWIG)
 
diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h
index b0fa8876..1512dfca 100644
--- a/cpp/ppc-framework/front/IFront.h
+++ b/cpp/ppc-framework/front/IFront.h
@@ -168,8 +168,8 @@ class IFront : virtual public IFrontClient
      * @param callback callback
      */
     virtual void asyncSendMessage(uint16_t routeType,
-        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
-        long timeout, ppc::protocol::ReceiveMsgFunc errorCallback,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytesConstRef payload,
+        int seq, long timeout, ppc::protocol::ReceiveMsgFunc errorCallback,
         ppc::protocol::MessageCallback callback) = 0;
 
     /////// to simplify SDK wrapper ////
@@ -181,14 +181,13 @@ class IFront : virtual public IFrontClient
         uint64_t payloadSize, int seq, long timeout, ErrorCallback::Ptr errorCallback,
         IMessageHandler::Ptr msgHandler)
     {
-        // TODO: optimize here
-        bcos::bytes copyedPayload(payload, payload + payloadSize);
-        asyncSendMessage(routeType, routeInfo, std::move(copyedPayload), seq, timeout,
+        asyncSendMessage(routeType, routeInfo,
+            bcos::bytesConstRef((bcos::byte*)payload, payloadSize), seq, timeout,
             populateErrorCallback(errorCallback), populateMsgCallback(msgHandler));
     }
 
-    virtual void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID,
-        bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0;
+    virtual void asyncSendResponse(bcos::bytesConstRef dstNode, std::string const& traceID,
+        bcos::bytesConstRef payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) = 0;
 
     /////// to simplify SDK wrapper  ////
 
@@ -198,19 +197,16 @@ class IFront : virtual public IFrontClient
         std::string const& traceID, char* payload, uint64_t payloadSize, int seq,
         ErrorCallback::Ptr errorCallback)
     {
-        // TODO: optimize here
-        bcos::bytes copiedDstNode(dstNode, dstNode + dstNodeSize);
-        bcos::bytes copyedPayload(payload, payload + payloadSize);
-        asyncSendResponse(copiedDstNode, traceID, std::move(copyedPayload), seq,
+        asyncSendResponse(bcos::bytesConstRef((bcos::byte*)dstNode, dstNodeSize), traceID,
+            bcos::bytesConstRef((bcos::byte*)payload, payloadSize), seq,
             populateErrorCallback(errorCallback));
     }
 
     // the sync interface for async_send_message
     virtual bcos::Error::Ptr push(uint16_t routeType,
-        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
-        long timeout) = 0;
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytesConstRef payload,
+        int seq, long timeout) = 0;
 
-    // TODO: optmize here
     // Note: the python not support function overload, for different interfaces with the same
     // functionality, it is best to define methods with different names the 'payload', 'payloadSize'
     // should not been changed any more, since the swig has defined by the name to convert python
@@ -219,8 +215,8 @@ class IFront : virtual public IFrontClient
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, char* payload,
         uint64_t payloadSize, int seq, long timeout)
     {
-        bcos::bytes copyedPayload(payload, payload + payloadSize);
-        return push(routeType, routeInfo, std::move(copyedPayload), seq, timeout);
+        return push(routeType, routeInfo, bcos::bytesConstRef((bcos::byte*)payload, payloadSize),
+            seq, timeout);
     }
     virtual ppc::protocol::Message::Ptr pop(std::string const& topic, long timeoutMs) = 0;
     virtual ppc::protocol::Message::Ptr peek(std::string const& topic) = 0;
diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h
index c3b1a470..4c83ad8a 100644
--- a/cpp/ppc-framework/protocol/Message.h
+++ b/cpp/ppc-framework/protocol/Message.h
@@ -57,7 +57,10 @@ class MessageOptionalHeader
 
     // !!! Note: the first paramater type should not been changed, for it's used for pass-in java
     // byte[] into c bytes
-    virtual void setSrcNode(char* data, uint64_t length) { m_srcNode.assign(data, data + length); }
+    virtual void setSrcNodeBuffer(char* data, uint64_t length)
+    {
+        m_srcNode.assign(data, data + length);
+    }
 
     // the target nodeID that should receive the message
     virtual bcos::bytes const& dstNode() const { return m_dstNode; }
@@ -68,8 +71,12 @@ class MessageOptionalHeader
         // Note: this will be copied to java through jni
         return OutputBuffer{(unsigned char*)m_dstNode.data(), m_dstNode.size()};
     }
-    virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; }
-
+    virtual void setDstNode(bcos::bytes const& dstNode)
+    {
+        m_dstNode = dstNode;
+        m_dstNodePtr = bcos::bytesConstRef((bcos::byte*)m_dstNode.data(), m_dstNode.size());
+    }
+    virtual void setDstNodePtr(bcos::bytesConstRef const& dstNodePtr) { m_dstNodePtr = dstNodePtr; }
     // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in
     // java byte[] into c bytes
     // Note: the python not support function override
@@ -100,6 +107,7 @@ class MessageOptionalHeader
     std::string m_srcInst;
     // the target nodeID that should receive the message
     bcos::bytes m_dstNode;
+    bcos::bytesConstRef m_dstNodePtr;  // to decrease the copy overhead
     // the target agency that need receive the message
     std::string m_dstInst;
 };
diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h
index 0a612a97..8d2194d9 100644
--- a/cpp/ppc-framework/protocol/MessagePayload.h
+++ b/cpp/ppc-framework/protocol/MessagePayload.h
@@ -46,10 +46,20 @@ class MessagePayload
     // for swig wrapper here
     virtual OutputBuffer dataBuffer() const
     {
-        return OutputBuffer{(unsigned char*)m_data.data(), m_data.size()};
+        return OutputBuffer{(unsigned char*)m_dataPtr.data(), m_dataPtr.size()};
     }
-    virtual void setData(bcos::bytes&& data) { m_data = std::move(data); }
-    virtual void setData(bcos::bytes const& data) { m_data = data; }
+    virtual void setData(bcos::bytes&& data)
+    {
+        m_data = std::move(data);
+        m_dataPtr = bcos::bytesConstRef((bcos::byte*)m_data.data(), m_data.size());
+    }
+    virtual void setData(bcos::bytes const& data)
+    {
+        m_data = data;
+        m_dataPtr = bcos::bytesConstRef((bcos::byte*)m_data.data(), m_data.size());
+    }
+    virtual void setDataPtr(bcos::bytesConstRef dataPtr) { m_dataPtr = dataPtr; }
+    virtual bcos::bytesConstRef const& dataPtr() const { return m_dataPtr; }
     // the seq
     virtual uint16_t seq() const { return m_seq; }
     virtual void setSeq(uint16_t seq) { m_seq = seq; }
@@ -76,6 +86,8 @@ class MessagePayload
     // the traceID
     std::string m_traceID;
     bcos::bytes m_data;
+    // used to decrease the copy-overhead
+    bcos::bytesConstRef m_dataPtr;
     uint16_t m_ext = 0;
     int64_t mutable m_length;
 };
diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
index 8dc17520..a97e505b 100644
--- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
@@ -42,10 +42,10 @@ void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const
     buffer.insert(buffer.end(), (byte*)&srcInstLen, (byte*)&srcInstLen + 2);
     buffer.insert(buffer.end(), m_srcInst.begin(), m_srcInst.end());
     // the target nodeID that should receive the message
-    uint16_t dstNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstNode.size());
+    uint16_t dstNodeLen =
+        boost::asio::detail::socket_ops::host_to_network_short(m_dstNodePtr.size());
     buffer.insert(buffer.end(), (byte*)&dstNodeLen, (byte*)&dstNodeLen + 2);
-    buffer.insert(buffer.end(), m_dstNode.begin(), m_dstNode.end());
-    bcos::bytes m_dstNode;
+    buffer.insert(buffer.end(), m_dstNodePtr.begin(), m_dstNodePtr.end());
     // the target agency that need receive the message
     uint16_t dstInstLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstInst.size());
     buffer.insert(buffer.end(), (byte*)&dstInstLen, (byte*)&dstInstLen + 2);
@@ -74,6 +74,7 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con
     offset = decodeNetworkBuffer(m_srcInst, data.data(), data.size(), offset, false);
     //  dstNode
     offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset, false);
+    m_dstNodePtr = bcos::bytesConstRef((bcos::byte*)m_dstNode.data(), m_dstNode.size());
     // dstInst
     offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset, false);
     // topic
diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp
index e43124d8..67653c3c 100644
--- a/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/v1/MessagePayloadImpl.cpp
@@ -42,9 +42,9 @@ int64_t MessagePayloadImpl::encode(bcos::bytes& buffer) const
     buffer.insert(buffer.end(), (byte*)&traceIDLen, (byte*)&traceIDLen + 2);
     buffer.insert(buffer.end(), m_traceID.begin(), m_traceID.end());
     // data
-    uint32_t dataLen = boost::asio::detail::socket_ops::host_to_network_long(m_data.size());
+    uint32_t dataLen = boost::asio::detail::socket_ops::host_to_network_long(m_dataPtr.size());
     buffer.insert(buffer.end(), (byte*)&dataLen, (byte*)&dataLen + 4);
-    buffer.insert(buffer.end(), m_data.begin(), m_data.end());
+    buffer.insert(buffer.end(), m_dataPtr.begin(), m_dataPtr.end());
     // update the length
     m_length = buffer.size();
     return m_length;
@@ -75,5 +75,8 @@ int64_t MessagePayloadImpl::decode(bcos::bytesConstRef buffer)
     auto offset =
         decodeNetworkBuffer(m_traceID, buffer.data(), buffer.size(), (pointer - buffer.data()));
     // data
-    return decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset, true);
+    auto ret = decodeNetworkBuffer(m_data, buffer.data(), buffer.size(), offset, true);
+    // reset the dataPtr
+    m_dataPtr = bcos::bytesConstRef((bcos::byte*)m_data.data(), m_data.size());
+    return ret;
 }
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
index 0fa9c7f5..73cd811d 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
@@ -135,8 +135,9 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace
         };
     }
     // ROUTE_THROUGH_TOPIC will hold the topic
-    m_front->asyncSendMessage((uint16_t)RouteType::ROUTE_THROUGH_TOPIC, routeInfo, std::move(data),
-        _message->seq(), _timeout, _callback, msgCallback);
+    m_front->asyncSendMessage((uint16_t)RouteType::ROUTE_THROUGH_TOPIC, routeInfo,
+        bcos::bytesConstRef((bcos::byte*)data.data(), data.size()), _message->seq(), _timeout,
+        _callback, msgCallback);
 }
 
 // send response when receiving message from given agencyID
@@ -145,7 +146,8 @@ void Front::asyncSendResponse(bcos::bytes const& dstNode, std::string const& tra
 {
     bcos::bytes data;
     message->encode(data);
-    m_front->asyncSendResponse(dstNode, traceID, std::move(data), 0, _callback);
+    m_front->asyncSendResponse(bcos::bytesConstRef((bcos::byte*)dstNode.data(), dstNode.size()),
+        traceID, bcos::bytesConstRef((bcos::byte*)data.data(), data.size()), 0, _callback);
 }
 
 /**
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index 96a4b00b..8c7aa8bb 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -106,18 +106,18 @@ void FrontImpl::stop()
     }
 }
 
-void FrontImpl::asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID,
-    bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback)
+void FrontImpl::asyncSendResponse(bcos::bytesConstRef dstNode, std::string const& traceID,
+    bcos::bytesConstRef payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback)
 {
     // generate the frontMessage
     auto frontMessage = m_messageFactory->build();
     frontMessage->setTraceID(traceID);
     frontMessage->setSeq(seq);
-    frontMessage->setData(std::move(payload));
+    frontMessage->setDataPtr(payload);
 
     auto routeInfo = m_routerInfoBuilder->build();
     routeInfo->setSrcNode(m_nodeID);
-    routeInfo->setDstNode(dstNode);
+    routeInfo->setDstNodePtr(dstNode);
 
     asyncSendMessageToGateway(true, std::move(frontMessage), RouteType::ROUTE_THROUGH_NODEID,
         traceID, routeInfo, -1, errorCallback);
@@ -139,7 +139,7 @@ void FrontImpl::asyncSendResponse(bcos::bytes const& dstNode, std::string const&
  * @param callback callback
  */
 void FrontImpl::asyncSendMessage(uint16_t routeType, MessageOptionalHeader::Ptr const& routeInfo,
-    bcos::bytes&& payload, int seq, long timeout, ReceiveMsgFunc errorCallback,
+    bcos::bytesConstRef payload, int seq, long timeout, ReceiveMsgFunc errorCallback,
     MessageCallback callback)
 {
     // generate the frontMessage
@@ -147,7 +147,7 @@ void FrontImpl::asyncSendMessage(uint16_t routeType, MessageOptionalHeader::Ptr
     auto traceID = ppc::generateUUID();
     frontMessage->setTraceID(traceID);
     frontMessage->setSeq(seq);
-    frontMessage->setData(std::move(payload));
+    frontMessage->setDataPtr(payload);
     m_callbackManager->addCallback(traceID, timeout, callback);
     auto self = weak_from_this();
     // send the message to the gateway
@@ -268,11 +268,11 @@ void FrontImpl::onReceiveMessage(Message::Ptr const& msg, ReceiveMsgFunc callbac
 
 // the sync interface for asyncSendMessage
 bcos::Error::Ptr FrontImpl::push(uint16_t routeType, MessageOptionalHeader::Ptr const& routeInfo,
-    bcos::bytes&& payload, int seq, long timeout)
+    bcos::bytesConstRef payload, int seq, long timeout)
 {
     auto promise = std::make_shared>();
     asyncSendMessage(
-        routeType, routeInfo, std::move(payload), seq, timeout,
+        routeType, routeInfo, payload, seq, timeout,
         [promise](bcos::Error::Ptr error) { promise->set_value(error); }, nullptr);
     return promise->get_future().get();
 }
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
index 3243535e..419afe53 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
@@ -53,8 +53,8 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
     void stop() override;
 
     bcos::Error::Ptr push(uint16_t routeType,
-        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
-        long timeout) override;
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytesConstRef payload,
+        int seq, long timeout) override;
     /**
      * @brief async send message
      *
@@ -71,8 +71,8 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
      * @param callback callback
      */
     void asyncSendMessage(uint16_t routeType,
-        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytes&& payload, int seq,
-        long timeout, ppc::protocol::ReceiveMsgFunc errorCallback,
+        ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo, bcos::bytesConstRef payload,
+        int seq, long timeout, ppc::protocol::ReceiveMsgFunc errorCallback,
         ppc::protocol::MessageCallback callback) override;
 
     /**
@@ -169,8 +169,8 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
         return m_messageFactory;
     }
 
-    void asyncSendResponse(bcos::bytes const& dstNode, std::string const& traceID,
-        bcos::bytes&& payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) override;
+    void asyncSendResponse(bcos::bytesConstRef dstNode, std::string const& traceID,
+        bcos::bytesConstRef payload, int seq, ppc::protocol::ReceiveMsgFunc errorCallback) override;
 
     ppc::protocol::INodeInfo::Ptr const& nodeInfo() override { return m_nodeInfo; }
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
index f3665b1c..132634ec 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/java/CMakeLists.txt
@@ -1,7 +1,6 @@
 option(AUTO_GENERATE "Auto generate the targes" OFF)
 # Note: use SWIG Version 4.2.1
 if(AUTO_GENERATE)
-    include(${SWIG_USE_FILE})
     # only auto-generate should find swig
     file(GLOB_RECURSE SRCS wedpr_java_transport.i)
     set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON)
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
index 35bad809..38dd9e9a 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
@@ -119,22 +119,6 @@ public void async_send_response(
                 errorCallback);
     }
 
-    public Error push(
-            int routeType, MessageOptionalHeader routeInfo, ubytes payload, int seq, int timeout) {
-        long cPtr =
-                wedpr_java_transportJNI.IFront_push(
-                        swigCPtr,
-                        this,
-                        routeType,
-                        MessageOptionalHeader.getCPtr(routeInfo),
-                        routeInfo,
-                        ubytes.swigRelease(payload),
-                        payload,
-                        seq,
-                        timeout);
-        return (cPtr == 0) ? null : new Error(cPtr, true);
-    }
-
     public Error push_msg(
             int routeType,
             MessageOptionalHeader routeInfo,
@@ -171,24 +155,6 @@ public void asyncGetPeers(GetPeersInfoHandler getPeersCallback) {
                 swigCPtr, this, GetPeersInfoHandler.getCPtr(getPeersCallback), getPeersCallback);
     }
 
-    /**
-     * register the nodeInfo to the gateway

-     *
-     * @param nodeInfo the nodeInfo
-     */
-    public Error registerNodeInfo(SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr nodeInfo) {
-        long cPtr =
-                wedpr_java_transportJNI.IFront_registerNodeInfo(
-                        swigCPtr, this, SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr.getCPtr(nodeInfo));
-        return (cPtr == 0) ? null : new Error(cPtr, true);
-    }
-
-    /** unRegister the nodeInfo to the gateway */
-    public Error unRegisterNodeInfo() {
-        long cPtr = wedpr_java_transportJNI.IFront_unRegisterNodeInfo(swigCPtr, this);
-        return (cPtr == 0) ? null : new Error(cPtr, true);
-    }
-
     public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr nodeInfo() {
         return new SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr(
                 wedpr_java_transportJNI.IFront_nodeInfo(swigCPtr, this), false);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java
index 892922bb..b0a472ed 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessageOptionalHeader.java
@@ -59,40 +59,20 @@ public void setComponentType(String componentType) {
                 swigCPtr, this, componentType);
     }
 
-    public ubytes srcNode() {
-        return new ubytes(
-                wedpr_java_transportJNI.MessageOptionalHeader_srcNode(swigCPtr, this), true);
-    }
-
     /** for swig-wrapper(pass the binary data) */
     public byte[] srcNodeBuffer() {
         return wedpr_java_transportJNI.MessageOptionalHeader_srcNodeBuffer(swigCPtr, this);
     }
 
-    public void setSrcNode(ubytes srcNode) {
-        wedpr_java_transportJNI.MessageOptionalHeader_setSrcNode__SWIG_0(
-                swigCPtr, this, ubytes.getCPtr(srcNode), srcNode);
-    }
-
-    public void setSrcNode(byte[] data, java.math.BigInteger length) {
-        wedpr_java_transportJNI.MessageOptionalHeader_setSrcNode__SWIG_1(
+    public void setSrcNodeBuffer(byte[] data, java.math.BigInteger length) {
+        wedpr_java_transportJNI.MessageOptionalHeader_setSrcNodeBuffer(
                 swigCPtr, this, data, length);
     }
 
-    public ubytes dstNode() {
-        return new ubytes(
-                wedpr_java_transportJNI.MessageOptionalHeader_dstNode(swigCPtr, this), true);
-    }
-
     public byte[] dstNodeBuffer() {
         return wedpr_java_transportJNI.MessageOptionalHeader_dstNodeBuffer(swigCPtr, this);
     }
 
-    public void setDstNode(ubytes dstNode) {
-        wedpr_java_transportJNI.MessageOptionalHeader_setDstNode(
-                swigCPtr, this, ubytes.getCPtr(dstNode), dstNode);
-    }
-
     public void setDstNodeBuffer(byte[] data, java.math.BigInteger length) {
         wedpr_java_transportJNI.MessageOptionalHeader_setDstNodeBuffer(
                 swigCPtr, this, data, length);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
index 082e6f3d..3998cdba 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
@@ -58,19 +58,10 @@ public void setVersion(short version) {
         wedpr_java_transportJNI.MessagePayload_setVersion(swigCPtr, this, version);
     }
 
-    public ubytes data() {
-        return new ubytes(wedpr_java_transportJNI.MessagePayload_data(swigCPtr, this), true);
-    }
-
     public byte[] dataBuffer() {
         return wedpr_java_transportJNI.MessagePayload_dataBuffer(swigCPtr, this);
     }
 
-    public void setData(ubytes data) {
-        wedpr_java_transportJNI.MessagePayload_setData__SWIG_0(
-                swigCPtr, this, ubytes.swigRelease(data), data);
-    }
-
     public int seq() {
         return wedpr_java_transportJNI.MessagePayload_seq(swigCPtr, this);
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index 814c24f8..cc5dfc17 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -369,27 +369,15 @@ public static final native String MessageOptionalHeader_componentType(
     public static final native void MessageOptionalHeader_setComponentType(
             long jarg1, MessageOptionalHeader jarg1_, String jarg2);
 
-    public static final native long MessageOptionalHeader_srcNode(
-            long jarg1, MessageOptionalHeader jarg1_);
-
     public static final native byte[] MessageOptionalHeader_srcNodeBuffer(
             long jarg1, MessageOptionalHeader jarg1_);
 
-    public static final native void MessageOptionalHeader_setSrcNode__SWIG_0(
-            long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_);
-
-    public static final native void MessageOptionalHeader_setSrcNode__SWIG_1(
+    public static final native void MessageOptionalHeader_setSrcNodeBuffer(
             long jarg1, MessageOptionalHeader jarg1_, byte[] jarg2, java.math.BigInteger jarg3);
 
-    public static final native long MessageOptionalHeader_dstNode(
-            long jarg1, MessageOptionalHeader jarg1_);
-
     public static final native byte[] MessageOptionalHeader_dstNodeBuffer(
             long jarg1, MessageOptionalHeader jarg1_);
 
-    public static final native void MessageOptionalHeader_setDstNode(
-            long jarg1, MessageOptionalHeader jarg1_, long jarg2, ubytes jarg2_);
-
     public static final native void MessageOptionalHeader_setDstNodeBuffer(
             long jarg1, MessageOptionalHeader jarg1_, byte[] jarg2, java.math.BigInteger jarg3);
 
@@ -585,13 +573,8 @@ public static final native long MessagePayload_decode(
     public static final native void MessagePayload_setVersion(
             long jarg1, MessagePayload jarg1_, short jarg2);
 
-    public static final native long MessagePayload_data(long jarg1, MessagePayload jarg1_);
-
     public static final native byte[] MessagePayload_dataBuffer(long jarg1, MessagePayload jarg1_);
 
-    public static final native void MessagePayload_setData__SWIG_0(
-            long jarg1, MessagePayload jarg1_, long jarg2, ubytes jarg2_);
-
     public static final native int MessagePayload_seq(long jarg1, MessagePayload jarg1_);
 
     public static final native void MessagePayload_setSeq(
@@ -732,17 +715,6 @@ public static final native void IFront_async_send_response(
             long jarg8,
             ErrorCallback jarg8_);
 
-    public static final native long IFront_push(
-            long jarg1,
-            IFront jarg1_,
-            int jarg2,
-            long jarg3,
-            MessageOptionalHeader jarg3_,
-            long jarg4,
-            ubytes jarg4_,
-            int jarg5,
-            int jarg6);
-
     public static final native long IFront_push_msg(
             long jarg1,
             IFront jarg1_,
@@ -761,10 +733,6 @@ public static final native long IFront_push_msg(
     public static final native void IFront_asyncGetPeers(
             long jarg1, IFront jarg1_, long jarg2, GetPeersInfoHandler jarg2_);
 
-    public static final native long IFront_registerNodeInfo(long jarg1, IFront jarg1_, long jarg2);
-
-    public static final native long IFront_unRegisterNodeInfo(long jarg1, IFront jarg1_);
-
     public static final native long IFront_nodeInfo(long jarg1, IFront jarg1_);
 
     public static final native long IFront_registerTopic(long jarg1, IFront jarg1_, String jarg2);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 807fb464..678deb89 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -4151,24 +4151,6 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1srcNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  bcos::bytes *result = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
-  arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNode();
-  *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); 
-  return jresult;
-}
-
-
 SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1srcNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jbyteArray jresult = 0 ;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -4190,29 +4172,7 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcNode_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg2_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeader > **)&jarg1;
-  arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  
-  arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0);
-  if (!arg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null");
-    return ;
-  } 
-  (arg1)->setSrcNode((bcos::bytes const &)*arg2);
-}
-
-
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcNode_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) {
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setSrcNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) {
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   char *arg2 = (char *) 0 ;
   uint64_t arg3 ;
@@ -4253,7 +4213,7 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  (arg1)->setSrcNode(arg2,arg3);
+  (arg1)->setSrcNodeBuffer(arg2,arg3);
   {
     jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); 
   }
@@ -4261,24 +4221,6 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1dstNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  bcos::bytes *result = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
-  arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNode();
-  *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); 
-  return jresult;
-}
-
-
 SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1dstNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jbyteArray jresult = 0 ;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -4300,28 +4242,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg2_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeader > **)&jarg1;
-  arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  
-  arg2 = (bcos::bytes *)((*(std::shared_ptr< const bcos::bytes > **)&jarg2) ? (*(std::shared_ptr< const bcos::bytes > **)&jarg2)->get() : 0);
-  if (!arg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes const & reference is null");
-    return ;
-  } 
-  (arg1)->setDstNode((bcos::bytes const &)*arg2);
-}
-
-
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessageOptionalHeader_1setDstNodeBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jbyteArray jarg2, jobject jarg3) {
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   char *arg2 = (char *) 0 ;
@@ -5784,24 +5704,6 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1data(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
-  bcos::bytes *result = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
-  arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bcos::bytes *) &((ppc::protocol::MessagePayload const *)arg1)->data();
-  *(std::shared_ptr< const bcos::bytes > **)&jresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0); 
-  return jresult;
-}
-
-
 SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1dataBuffer(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jbyteArray jresult = 0 ;
   ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
@@ -5823,29 +5725,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1setData_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
-  std::unique_ptr< bcos::bytes > rvrdeleter2 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg2_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
-  arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = *(bcos::bytes **)&jarg2;
-  if (!arg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null");
-    return ;
-  }
-  rvrdeleter2.reset(arg2); 
-  (arg1)->setData((bcos::bytes &&)*arg2);
-}
-
-
 SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1seq(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jint jresult = 0 ;
   ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
@@ -6716,43 +6595,6 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jlong jarg4, jobject jarg4_, jint jarg5, jint jarg6) {
-  jlong jresult = 0 ;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  uint16_t arg2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
-  bcos::bytes *arg4 = 0 ;
-  int arg5 ;
-  long arg6 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempnull3 ;
-  std::unique_ptr< bcos::bytes > rvrdeleter4 ;
-  bcos::Error::Ptr result;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg3_;
-  (void)jarg4_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
-  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = (uint16_t)jarg2; 
-  arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; 
-  arg4 = *(bcos::bytes **)&jarg4;
-  if (!arg4) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes && is null");
-    return 0;
-  }
-  rvrdeleter4.reset(arg4); 
-  arg5 = (int)jarg5; 
-  arg6 = (long)jarg6; 
-  result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6);
-  *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
-  return jresult;
-}
-
-
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1push_1msg(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_, jbyteArray jarg4, jobject jarg5, jint jarg6, jint jarg7) {
   jlong jresult = 0 ;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
@@ -6893,48 +6735,6 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1registerNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
-  jlong jresult = 0 ;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  ppc::protocol::INodeInfo::Ptr *arg2 = 0 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  bcos::Error::Ptr result;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
-  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = *(ppc::protocol::INodeInfo::Ptr **)&jarg2;
-  if (!arg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "ppc::protocol::INodeInfo::Ptr const & is null");
-    return 0;
-  } 
-  result = (arg1)->registerNodeInfo((ppc::protocol::INodeInfo::Ptr const &)*arg2);
-  *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
-  return jresult;
-}
-
-
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1unRegisterNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  bcos::Error::Ptr result;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
-  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->unRegisterNodeInfo();
-  *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
-  return jresult;
-}
-
-
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1nodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jlong jresult = 0 ;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index 91b8570c..bae976b4 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -221,6 +221,18 @@ WRAP(ppc::sdk::Transport)
 %ignore ppc::front::IFront::populateErrorCallback;
 %ignore ppc::front::IFront::populateMessageDispatcherCallback;
 %ignore ppc::front::IFront::populateMsgCallback;
+%ignore ppc::front::IFront::push;
+%ignore ppc::front::IFront::registerNodeInfo;
+%ignore ppc::front::IFront::unRegisterNodeInfo;
+%ignore ppc::protocol::MessageOptionalHeader::srcNode;
+%ignore ppc::protocol::MessageOptionalHeader::dstNode;
+%ignore ppc::protocol::MessageOptionalHeader::setDstNode;
+%ignore ppc::protocol::MessageOptionalHeader::setDstNodePtr;
+%ignore ppc::protocol::MessageOptionalHeader::setSrcNode;
+%ignore ppc::protocol::MessagePayload::data;
+%ignore ppc::protocol::MessagePayload::setData;
+%ignore ppc::protocol::MessagePayload::setDataPtr;
+%ignore ppc::protocol::MessagePayload::dataPtr;
 
 /*
 ///// tests  ///
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
index d78c693a..fbaa6b04 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
@@ -1,7 +1,6 @@
 option(AUTO_GENERATE "Auto generate the targes" OFF)
 # Note: use SWIG Version 4.2.1
 if(AUTO_GENERATE)
-    include(${SWIG_USE_FILE})
     file(GLOB_RECURSE SRCS *.i)
     set_source_files_properties(${SRCS} PROPERTIES CPLUSPLUS ON)
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 9b02b5e6..88c27637 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1015,25 +1015,16 @@ def componentType(self):
     def setComponentType(self, componentType):
         return _wedpr_python_transport.MessageOptionalHeader_setComponentType(self, componentType)
 
-    def srcNode(self):
-        return _wedpr_python_transport.MessageOptionalHeader_srcNode(self)
-
     def srcNodeBuffer(self):
         r""" for swig-wrapper(pass the binary data)"""
         return _wedpr_python_transport.MessageOptionalHeader_srcNodeBuffer(self)
 
-    def setSrcNode(self, *args):
-        return _wedpr_python_transport.MessageOptionalHeader_setSrcNode(self, *args)
-
-    def dstNode(self):
-        return _wedpr_python_transport.MessageOptionalHeader_dstNode(self)
+    def setSrcNodeBuffer(self, data):
+        return _wedpr_python_transport.MessageOptionalHeader_setSrcNodeBuffer(self, data)
 
     def dstNodeBuffer(self):
         return _wedpr_python_transport.MessageOptionalHeader_dstNodeBuffer(self)
 
-    def setDstNode(self, dstNode):
-        return _wedpr_python_transport.MessageOptionalHeader_setDstNode(self, dstNode)
-
     def setDstNodeBuffer(self, data):
         return _wedpr_python_transport.MessageOptionalHeader_setDstNodeBuffer(self, data)
 
@@ -1312,15 +1303,9 @@ def version(self):
     def setVersion(self, version):
         return _wedpr_python_transport.MessagePayload_setVersion(self, version)
 
-    def data(self):
-        return _wedpr_python_transport.MessagePayload_data(self)
-
     def dataBuffer(self):
         return _wedpr_python_transport.MessagePayload_dataBuffer(self)
 
-    def setData(self, *args):
-        return _wedpr_python_transport.MessagePayload_setData(self, *args)
-
     def seq(self):
         return _wedpr_python_transport.MessagePayload_seq(self)
 
@@ -1558,9 +1543,6 @@ def async_send_message(self, routeType, routeInfo, payload, seq, timeout, errorC
     def async_send_response(self, dstNode, dstNodeSize, traceID, payload, seq, errorCallback):
         return _wedpr_python_transport.IFront_async_send_response(self, dstNode, dstNodeSize, traceID, payload, seq, errorCallback)
 
-    def push(self, routeType, routeInfo, payload, seq, timeout):
-        return _wedpr_python_transport.IFront_push(self, routeType, routeInfo, payload, seq, timeout)
-
     def push_msg(self, routeType, routeInfo, payload, seq, timeout):
         return _wedpr_python_transport.IFront_push_msg(self, routeType, routeInfo, payload, seq, timeout)
 
@@ -1573,18 +1555,6 @@ def peek(self, topic):
     def asyncGetPeers(self, getPeersCallback):
         return _wedpr_python_transport.IFront_asyncGetPeers(self, getPeersCallback)
 
-    def registerNodeInfo(self, nodeInfo):
-        r"""
-        register the nodeInfo to the gateway
-        :type nodeInfo: ppc::protocol::INodeInfo::Ptr
-        :param nodeInfo: the nodeInfo
-        """
-        return _wedpr_python_transport.IFront_registerNodeInfo(self, nodeInfo)
-
-    def unRegisterNodeInfo(self):
-        r"""unRegister the nodeInfo to the gateway"""
-        return _wedpr_python_transport.IFront_unRegisterNodeInfo(self)
-
     def nodeInfo(self):
         return _wedpr_python_transport.IFront_nodeInfo(self)
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index d93bbbfe..51a14c06 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -16933,45 +16933,6 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setComponentType(PyObject *self
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNode(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  bcos::bytes *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNode();
-  {
-    std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0);
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -17010,63 +16971,7 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, P
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes const > tempshared2 ;
-  
-  (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "bcos::bytes const &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); 
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
-    } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get());
-    }
-  }
-  (arg1)->setSrcNode((bcos::bytes const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return resultobj;
-fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   char *arg2 = (char *) 0 ;
@@ -17075,14 +16980,15 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *se
   int res1 = 0 ;
   std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
   std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcNodeBuffer", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
@@ -17116,7 +17022,7 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *se
 #endif
     if (res < 0) {
       PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setSrcNode" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
     }
 #ifndef Py_LIMITED_API
     size = view.len;
@@ -17126,7 +17032,7 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *se
     arg2 = (char *) buf;
     arg3 = (uint64_t) (size / sizeof(char));
   }
-  (arg1)->setSrcNode(arg2,arg3);
+  (arg1)->setSrcNodeBuffer(arg2,arg3);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17134,96 +17040,6 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode__SWIG_1(PyObject *se
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNode(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcNode", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageOptionalHeader_setSrcNode__SWIG_0(self, argc, argv);
-      }
-    }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        if (argc <= 2) {
-          return _wrap_MessageOptionalHeader_setSrcNode__SWIG_1(self, argc, argv);
-        }
-        {
-          int res = SWIG_AsVal_unsigned_SS_long_SS_long(argv[2], NULL);
-          _v = SWIG_CheckState(res);
-        }
-        if (_v) {
-          return _wrap_MessageOptionalHeader_setSrcNode__SWIG_1(self, argc, argv);
-        }
-      }
-    }
-  }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeader_setSrcNode'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessageOptionalHeader::setSrcNode(bcos::bytes const &)\n"
-    "    ppc::protocol::MessageOptionalHeader::setSrcNode(char *,uint64_t)\n");
-  return 0;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNode(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  bcos::bytes *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = (bcos::bytes *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNode();
-  {
-    std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0);
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -17262,63 +17078,6 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, P
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNode(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes const > tempshared2 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstNode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstNode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setDstNode" "', argument " "2"" of type '" "bcos::bytes const &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setDstNode" "', argument " "2"" of type '" "bcos::bytes const &""'"); 
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
-    } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get());
-    }
-  }
-  (arg1)->setDstNode((bcos::bytes const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return resultobj;
-fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -20673,45 +20432,6 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_data(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  bcos::bytes *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_data" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = (bcos::bytes *) &((ppc::protocol::MessagePayload const *)arg1)->data();
-  {
-    std::shared_ptr< const bcos::bytes > *smartresult = new std::shared_ptr< const bcos::bytes >(result SWIG_NO_NULL_DELETER_0);
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
@@ -20750,155 +20470,6 @@ SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setData__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::unique_ptr< bcos::bytes > rvrdeleter2 ;
-  
-  (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setData" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE |  0 );
-  if (!SWIG_IsOK(res2)) {
-    if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "bcos::bytes &&""'");
-    } else {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes &&""'"); 
-    }
-  }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes &&""'"); 
-  }
-  arg2 = reinterpret_cast< bcos::bytes * >(argp2);
-  rvrdeleter2.reset(arg2);
-  (arg1)->setData((bcos::bytes &&)*arg2);
-  resultobj = SWIG_Py_Void();
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessagePayload_setData__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  bcos::bytes *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes const > tempshared2 ;
-  
-  (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setData" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes const &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setData" "', argument " "2"" of type '" "bcos::bytes const &""'"); 
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
-    } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr< const bcos::bytes > * >(argp2)->get());
-    }
-  }
-  (arg1)->setData((bcos::bytes const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return resultobj;
-fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessagePayload_setData(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessagePayload_setData", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      void *vptr = 0;
-      int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NO_NULL);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessagePayload_setData__SWIG_0(self, argc, argv);
-      }
-    }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessagePayload_setData__SWIG_1(self, argc, argv);
-      }
-    }
-  }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessagePayload_setData'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessagePayload::setData(bcos::bytes &&)\n"
-    "    ppc::protocol::MessagePayload::setData(bcos::bytes const &)\n");
-  return 0;
-}
-
-
 SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
@@ -22851,103 +22422,6 @@ SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_push(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  uint16_t arg2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
-  bcos::bytes *arg4 = 0 ;
-  int arg5 ;
-  long arg6 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
-  void *argp4 = 0 ;
-  int res4 = 0 ;
-  std::unique_ptr< bcos::bytes > rvrdeleter4 ;
-  int val5 ;
-  int ecode5 = 0 ;
-  long val6 ;
-  int ecode6 = 0 ;
-  PyObject *swig_obj[6] ;
-  bcos::Error::Ptr result;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_push", 6, 6, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_push" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_push" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_push" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      arg3 = &tempshared3;
-    } else {
-      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
-    }
-  }
-  res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE |  0 );
-  if (!SWIG_IsOK(res4)) {
-    if (res4 == SWIG_ERROR_RELEASE_NOT_OWNED) {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_push" "', cannot release ownership as memory is not owned for argument " "4"" of type '" "bcos::bytes &&""'");
-    } else {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_push" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
-    }
-  }
-  if (!argp4) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_push" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
-  }
-  arg4 = reinterpret_cast< bcos::bytes * >(argp4);
-  rvrdeleter4.reset(arg4);
-  ecode5 = SWIG_AsVal_int(swig_obj[4], &val5);
-  if (!SWIG_IsOK(ecode5)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "IFront_push" "', argument " "5"" of type '" "int""'");
-  } 
-  arg5 = static_cast< int >(val5);
-  ecode6 = SWIG_AsVal_long(swig_obj[5], &val6);
-  if (!SWIG_IsOK(ecode6)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_push" "', argument " "6"" of type '" "long""'");
-  } 
-  arg6 = static_cast< long >(val6);
-  result = (arg1)->push(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4,arg5,arg6);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
@@ -23223,94 +22697,6 @@ SWIGINTERN PyObject *_wrap_IFront_asyncGetPeers(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_registerNodeInfo(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  ppc::protocol::INodeInfo::Ptr *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  bcos::Error::Ptr result;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_registerNodeInfo", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerNodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr,  0  | 0);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerNodeInfo" "', argument " "2"" of type '" "ppc::protocol::INodeInfo::Ptr const &""'"); 
-  }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerNodeInfo" "', argument " "2"" of type '" "ppc::protocol::INodeInfo::Ptr const &""'"); 
-  }
-  arg2 = reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp2);
-  result = (arg1)->registerNodeInfo((ppc::protocol::INodeInfo::Ptr const &)*arg2);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_IFront_unRegisterNodeInfo(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  bcos::Error::Ptr result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterNodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = (arg1)->unRegisterNodeInfo();
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_IFront_nodeInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
@@ -24560,12 +23946,9 @@ static PyMethodDef SwigMethods[] = {
 	 { "MessageOptionalHeader_decode", _wrap_MessageOptionalHeader_decode, METH_VARARGS, NULL},
 	 { "MessageOptionalHeader_componentType", _wrap_MessageOptionalHeader_componentType, METH_O, NULL},
 	 { "MessageOptionalHeader_setComponentType", _wrap_MessageOptionalHeader_setComponentType, METH_VARARGS, NULL},
-	 { "MessageOptionalHeader_srcNode", _wrap_MessageOptionalHeader_srcNode, METH_O, NULL},
 	 { "MessageOptionalHeader_srcNodeBuffer", _wrap_MessageOptionalHeader_srcNodeBuffer, METH_O, " for swig-wrapper(pass the binary data)"},
-	 { "MessageOptionalHeader_setSrcNode", _wrap_MessageOptionalHeader_setSrcNode, METH_VARARGS, NULL},
-	 { "MessageOptionalHeader_dstNode", _wrap_MessageOptionalHeader_dstNode, METH_O, NULL},
+	 { "MessageOptionalHeader_setSrcNodeBuffer", _wrap_MessageOptionalHeader_setSrcNodeBuffer, METH_VARARGS, NULL},
 	 { "MessageOptionalHeader_dstNodeBuffer", _wrap_MessageOptionalHeader_dstNodeBuffer, METH_O, NULL},
-	 { "MessageOptionalHeader_setDstNode", _wrap_MessageOptionalHeader_setDstNode, METH_VARARGS, NULL},
 	 { "MessageOptionalHeader_setDstNodeBuffer", _wrap_MessageOptionalHeader_setDstNodeBuffer, METH_VARARGS, NULL},
 	 { "MessageOptionalHeader_dstInst", _wrap_MessageOptionalHeader_dstInst, METH_O, NULL},
 	 { "MessageOptionalHeader_setDstInst", _wrap_MessageOptionalHeader_setDstInst, METH_VARARGS, NULL},
@@ -24640,9 +24023,7 @@ static PyMethodDef SwigMethods[] = {
 	 { "MessagePayload_decode", _wrap_MessagePayload_decode, METH_VARARGS, NULL},
 	 { "MessagePayload_version", _wrap_MessagePayload_version, METH_O, NULL},
 	 { "MessagePayload_setVersion", _wrap_MessagePayload_setVersion, METH_VARARGS, NULL},
-	 { "MessagePayload_data", _wrap_MessagePayload_data, METH_O, NULL},
 	 { "MessagePayload_dataBuffer", _wrap_MessagePayload_dataBuffer, METH_O, NULL},
-	 { "MessagePayload_setData", _wrap_MessagePayload_setData, METH_VARARGS, NULL},
 	 { "MessagePayload_seq", _wrap_MessagePayload_seq, METH_O, NULL},
 	 { "MessagePayload_setSeq", _wrap_MessagePayload_setSeq, METH_VARARGS, NULL},
 	 { "MessagePayload_length", _wrap_MessagePayload_length, METH_O, NULL},
@@ -24708,17 +24089,10 @@ static PyMethodDef SwigMethods[] = {
 	 { "IFront_register_msg_handler", _wrap_IFront_register_msg_handler, METH_VARARGS, NULL},
 	 { "IFront_async_send_message", _wrap_IFront_async_send_message, METH_VARARGS, NULL},
 	 { "IFront_async_send_response", _wrap_IFront_async_send_response, METH_VARARGS, NULL},
-	 { "IFront_push", _wrap_IFront_push, METH_VARARGS, NULL},
 	 { "IFront_push_msg", _wrap_IFront_push_msg, METH_VARARGS, NULL},
 	 { "IFront_pop", _wrap_IFront_pop, METH_VARARGS, NULL},
 	 { "IFront_peek", _wrap_IFront_peek, METH_VARARGS, NULL},
 	 { "IFront_asyncGetPeers", _wrap_IFront_asyncGetPeers, METH_VARARGS, NULL},
-	 { "IFront_registerNodeInfo", _wrap_IFront_registerNodeInfo, METH_VARARGS, "\n"
-		"register the nodeInfo to the gateway\n"
-		":type nodeInfo: ppc::protocol::INodeInfo::Ptr\n"
-		":param nodeInfo: the nodeInfo\n"
-		""},
-	 { "IFront_unRegisterNodeInfo", _wrap_IFront_unRegisterNodeInfo, METH_O, "unRegister the nodeInfo to the gateway"},
 	 { "IFront_nodeInfo", _wrap_IFront_nodeInfo, METH_O, NULL},
 	 { "IFront_registerTopic", _wrap_IFront_registerTopic, METH_VARARGS, "\n"
 		"register the topic\n"
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index 588e2461..f238d32d 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -160,6 +160,18 @@ namespace bcos{
 %ignore ppc::front::IFront::populateErrorCallback;
 %ignore ppc::front::IFront::populateMessageDispatcherCallback;
 %ignore ppc::front::IFront::populateMsgCallback;
+%ignore ppc::front::IFront::push;
+%ignore ppc::front::IFront::registerNodeInfo;
+%ignore ppc::front::IFront::unRegisterNodeInfo;
+%ignore ppc::protocol::MessageOptionalHeader::srcNode;
+%ignore ppc::protocol::MessageOptionalHeader::dstNode;
+%ignore ppc::protocol::MessageOptionalHeader::setDstNode;
+%ignore ppc::protocol::MessageOptionalHeader::setDstNodePtr;
+%ignore ppc::protocol::MessageOptionalHeader::setSrcNode;
+%ignore ppc::protocol::MessagePayload::data;
+%ignore ppc::protocol::MessagePayload::setData;
+%ignore ppc::protocol::MessagePayload::setDataPtr;
+%ignore ppc::protocol::MessagePayload::dataPtr;
 
 /*
 ///// tests  ///

From 19613eb6a0d24d45238724be9e9926554ac32f2f Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 29 Oct 2024 01:02:21 +0800
Subject: [PATCH 071/120] deal with todo-lists (#69)

* fix centos ci no space left

* deal with todo-lists
---
 .github/workflows/cpp_full_node_workflow.yml  | 43 +++++++-------
 .github/workflows/cpp_sdk_workflow.yml        |  8 +--
 .github/workflows/cpp_toolkit_workflow.yml    |  8 +--
 cpp/cmake/CompilerSettings.cmake              | 58 +++++++------------
 cpp/cmake/Options.cmake                       |  2 +-
 cpp/cmake/grpc.cmake                          |  5 +-
 cpp/ppc-framework/protocol/GlobalConfig.h     |  2 +-
 cpp/ppc-framework/protocol/Protocol.h         | 24 +++-----
 cpp/ppc-framework/protocol/Task.h             |  2 +-
 cpp/test-utils/FakeFront.h                    | 12 ++--
 cpp/tools/install_depends.sh                  | 19 +-----
 .../src/controller/commandline_helper.py      |  2 +-
 cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h |  2 +-
 cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp |  8 +--
 cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h   |  2 -
 .../ppc-pir/tests/TestBaseOT.cpp              |  4 +-
 cpp/wedpr-computing/ppc-psi/src/Common.h      |  2 +-
 cpp/wedpr-computing/ppc-psi/src/PSIConfig.h   |  8 +--
 .../src/bs-ecdh-psi/BsEcdhPSIFactory.h        |  2 +-
 .../src/bs-ecdh-psi/core/BsEcdhCache.h        |  1 -
 .../src/bs-ecdh-psi/core/BsEcdhTaskState.h    |  2 +-
 .../bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h  | 34 +++++------
 .../src/bs-ecdh-psi/ffi/wedpr_utilities.h     | 36 ++++++------
 .../ppc-psi/src/cm2020-psi/CM2020PSIConfig.h  |  2 +-
 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp  |  2 +-
 .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 12 ++--
 .../src/cm2020-psi/core/CM2020PSISender.cpp   | 13 +++--
 .../src/cm2020-psi/protocol/CM2020PSIResult.h |  4 +-
 .../src/ecdh-conn-psi/EcdhConnPSIConfig.h     |  2 +-
 .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp     |  2 +-
 .../ecdh-conn-psi/core/EcdhConnPSIServer.cpp  |  4 +-
 .../protocol/PSIConnMessageFactory.cpp        |  4 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIConfig.h   |  2 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp   |  2 +-
 .../core/EcdhMultiPSIMaster.cpp               |  3 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIConfig.h      |  4 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp      |  6 +-
 .../src/labeled-psi/LabeledPSIConfig.h        |  4 +-
 .../src/labeled-psi/LabeledPSIImpl.cpp        |  2 +-
 .../labeled-psi/core/LabeledPSIReceiver.cpp   |  6 +-
 .../src/labeled-psi/core/LabeledPSISender.cpp |  6 +-
 .../ppc-psi/src/psi-framework/PSIFramework.h  |  2 +-
 .../ppc-psi/src/psi-framework/TaskGuarder.h   |  4 +-
 .../ppc-psi/src/ra2018-psi/RA2018PSIConfig.h  |  2 +-
 .../tests/cm2020-psi/TestCM2020Impl.cpp       |  4 +-
 .../tests/labeled-psi/TestLabeledPSIImpl.cpp  | 10 ++--
 .../tests/ra2018-psi/TestEcdhPSIImpl.cpp      |  4 +-
 .../ppc-homo/paillier/OpenSSLPaillier.cpp     |  1 -
 .../hive/udf/config/model/KeyConfig.java      |  1 -
 .../ppc-tools/src/config/PPCConfig.cpp        |  8 +++
 .../ppc-tools/src/config/PPCConfig.h          |  4 ++
 cpp/wedpr-initializer/Initializer.cpp         | 30 +++++-----
 .../grpc/client/FrontClient.cpp               |  4 +-
 .../grpc/server/GatewayServer.cpp             |  3 +-
 cpp/wedpr-protocol/grpc/server/GrpcServer.cpp |  2 +-
 .../protobuf/src/RequestConverter.h           |  5 +-
 .../protobuf/tests/NodeInfoImplTest.cpp       | 36 +++++++++++-
 .../protocol/tests/PPCMessageTest.cpp         |  2 +-
 .../test/unittests/PPCChannelTest.cpp         |  4 +-
 .../ppc-gateway/GatewayFactory.cpp            |  7 ++-
 .../ppc-gateway/gateway/GatewayImpl.cpp       |  5 +-
 .../ppc-gateway/gateway/GatewayImpl.h         |  3 +-
 .../gateway/router/GatewayRouterManager.cpp   |  7 ++-
 .../gateway/router/GatewayRouterManager.h     |  6 +-
 .../ppc-gateway/p2p/router/RouterManager.cpp  | 27 +++++++--
 .../ppc-gateway/p2p/router/RouterManager.h    |  4 +-
 .../ppc-gateway/test/demo/gateway_demo.cpp    |  4 +-
 cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp       |  4 +-
 .../sdk/jni/transport/impl/TransportImpl.java |  2 -
 69 files changed, 298 insertions(+), 263 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 77136375..fcea6730 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -53,9 +53,9 @@ jobs:
             !c:/vcpkg/buildtrees
             !c:/vcpkg/packages
             !c:/vcpkg/downloads
-          key: build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
           restore-keys: |
-            build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
             build-all-${{ matrix.os }}-${{ github.base_ref }}-
             build-all-${{ matrix.os }}-
       - name: Build for linux
@@ -72,12 +72,6 @@ jobs:
           bash -x cpp/tools/install_depends.sh -o macos
           mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
-      - name: Publish Error
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-            name: config-x64-linux-dbg-err.log
-            path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-linux-dbg-err.log
       - uses: actions/upload-artifact@v4
         if: runner.os == 'macOS'      
         with:
@@ -123,9 +117,9 @@ jobs:
             /usr/local/share/vcpkg/packages
             /home/runner/.ccache
             /Users/runner/.ccache/
-          key: centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
           restore-keys: |
-            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_full_node_workflow.yml') }}
             centos-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
             centos-notest-all-${{ matrix.os }}-
       - name: Prepare centos tools
@@ -141,28 +135,35 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
-          yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
+          yum install -y wget lcov java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
           yum install -y git
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Check disk space
+        run: df . -h
+      - name: Free disk space
+        run: |
+          du -sch /usr/share/*
+          du -sch /opt/*
+      - name: Check disk space
+        run: |
+          df . -h
+          du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
+          df /tmp
+          du /tmp | sort -n
+          ls -lh /tmp
       - name: Build
         run: |
           bash -x cpp/tools/install_depends.sh -o centos
           alias cmake='cmake3'
-          . /opt/rh/devtoolset-11/enable 
+          . /opt/rh/devtoolset-11/enable
+          rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
           cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-          cmake3 --build . --parallel 3
-      - name: Publish Error
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-            name: vcpkg-manifest-install.log
-            path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
+          #cmake3 --build . --parallel 3
       #- name: Test
       #  run: |
       #    export OMP_NUM_THREADS=1
@@ -178,4 +179,4 @@ jobs:
       - uses: actions/upload-artifact@v3
         with:
           name: ppc-gateway-service-x64
-          path: ./cpp/build/bin/ppc-gateway-service
\ No newline at end of file
+          path: ./cpp/build/bin/ppc-gateway-service
diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml
index bc4252d1..f4540e7e 100644
--- a/.github/workflows/cpp_sdk_workflow.yml
+++ b/.github/workflows/cpp_sdk_workflow.yml
@@ -53,9 +53,9 @@ jobs:
             ${{ env.VCPKG_ROOT }}/buildtrees
             ${{ env.VCPKG_ROOT }}/packages
             ${{ env.VCPKG_ROOT }}/downloads
-          key: build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
           restore-keys: |
-            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-sdk-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
             build-sdk-${{ matrix.os }}-${{ github.base_ref }}-
             build-sdk-${{ matrix.os }}-
       - name: Add MSbuild to PATH
@@ -127,9 +127,9 @@ jobs:
               /usr/local/share/vcpkg/packages
               /home/runner/.ccache
               /Users/runner/.ccache/
-            key: centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            key: centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
             restore-keys: |
-              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_sdk_workflow.yml') }}
               centos-sdk-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
               centos-sdk-notest-all-${{ matrix.os }}-
         - name: Prepare centos tools
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index fa1dc2e8..a1922156 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -53,9 +53,9 @@ jobs:
             ${{ env.VCPKG_ROOT }}/buildtrees
             ${{ env.VCPKG_ROOT }}/packages
             ${{ env.VCPKG_ROOT }}/downloads
-          key: build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+          key: build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
           restore-keys: |
-            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
             build-toolkit-${{ matrix.os }}-${{ github.base_ref }}-
             build-toolkit-${{ matrix.os }}-
       - name: Add MSbuild to PATH
@@ -118,9 +118,9 @@ jobs:
               /usr/local/share/vcpkg/packages
               /home/runner/.ccache
               /Users/runner/.ccache/
-            key: centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+            key: centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
             restore-keys: |
-              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/workflow.yml') }}
+              centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-${{ hashFiles('.github/workflows/cpp_toolkit_workflow.yml') }}
               centos-toolkit-notest-all-${{ matrix.os }}-${{ github.base_ref }}-
               centos-toolkit-notest-all-${{ matrix.os }}-
         - name: Prepare centos tools
diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 61396933..59c96030 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -1,12 +1,6 @@
 set(CMAKE_CXX_STANDARD 20)
 set(Boost_NO_WARN_NEW_VERSIONS ON)
 message(STATUS "COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
-
-# export windows dll symbol
-if(WIN32)
-    message(STATUS "Compile on Windows")
-    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON")
-endif()
 if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
     find_program(CCACHE_PROGRAM ccache)
     if(CCACHE_PROGRAM)
@@ -132,38 +126,28 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
     endif()
     endif ()
 elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
-    add_compile_definitions(NOMINMAX)
-
-    # Only support visual studio 2017 and visual studio 2019
-    set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support
+       # Only support visual studio 2017 and visual studio 2019
+       set(MSVC_MIN_VERSION "1914") # VS2017 15.7, for full-ish C++17 support
 
-    message(STATUS "Compile On Windows, MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
-
-    if (MSVC_TOOLSET_VERSION EQUAL 141)
-        message(STATUS "Compile On Visual Studio 2017")
-    elseif(MSVC_TOOLSET_VERSION EQUAL 142)
-        message(STATUS "Compile On Visual Studio 2019")
-    else()
-        message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
-    endif()
-
-    add_compile_options(/std:c++latest)
-    add_compile_options(-bigobj)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
-    if(BUILD_SHARED_LIBS)
-        if(CMAKE_BUILD_TYPE MATCHES "Debug")
-            add_compile_options(/MDd)
-        else()
-            add_compile_options(/MD)
-        endif()
-    else ()
-        if(CMAKE_BUILD_TYPE MATCHES "Debug")
-            add_compile_options(/MTd)
-        else()
-            add_compile_options(/MT)
-        endif ()
-    endif ()
-    link_libraries(ws2_32 Crypt32 userenv)
+       message(STATUS "Compile On Windows, MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
+   
+       if (MSVC_TOOLSET_VERSION EQUAL 141)
+           message(STATUS "Compile On Visual Studio 2017")
+       elseif(MSVC_TOOLSET_VERSION EQUAL 142)
+           message(STATUS "Compile On Visual Studio 2019")
+       else()
+           message(FATAL_ERROR "Unsupported Visual Studio, supported list: [2017, 2019]. Current MSVC_TOOLSET_VERSION: ${MSVC_TOOLSET_VERSION}")
+       endif()
+   
+       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17")
+       add_compile_definitions(NOMINMAX)
+       add_compile_options(-bigobj)
+        # MSVC only support static build
+        set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
+        set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MT")
+        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
+        link_libraries(ws2_32 Crypt32 userenv)
 else ()
     message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.")
 endif ()
diff --git a/cpp/cmake/Options.cmake b/cpp/cmake/Options.cmake
index 212ce9ca..9af80391 100644
--- a/cpp/cmake/Options.cmake
+++ b/cpp/cmake/Options.cmake
@@ -169,7 +169,7 @@ macro(print_config NAME)
     message("------------------------------------------------------------------------")
     message("-- Configuring ${NAME} ${PROJECT_VERSION}${VERSION_SUFFIX}")
     message("------------------------------------------------------------------------")
-    message("-- CMake               Cmake version and location   ${CMAKE_VERSION} (${CMAKE_COMMAND})")
+    message("-- CMake               CMake version and location   ${CMAKE_VERSION} (${CMAKE_COMMAND})")
     message("-- Compiler            C++ compiler version         ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
     message("-- CMAKE_BUILD_TYPE    Build type                   ${CMAKE_BUILD_TYPE}")
     message("-- VCPKG_MANIFEST_FEATURES VCPKG manifest features  ${VCPKG_MANIFEST_FEATURES}")
diff --git a/cpp/cmake/grpc.cmake b/cpp/cmake/grpc.cmake
index b88d316f..f95bbe1d 100644
--- a/cpp/cmake/grpc.cmake
+++ b/cpp/cmake/grpc.cmake
@@ -41,7 +41,10 @@ if(NOT GRPC_CPP_REFLECTION AND TARGET gRPC::grpc++_reflection)
   endif()
 endif()
 
+get_target_property(GRPC_CPP_REFLECTION_INCLUDE gRPC::grpc++_reflection INTERFACE_INCLUDE_DIRECTORIES)
+include_directories(${GRPC_CPP_REFLECTION_INCLUDE})
+
 set(PROTOC_BINARY ${Protobuf_PROTOC_EXECUTABLE})
 message("# PROTOC_BINARY: ${Protobuf_PROTOC_EXECUTABLE}")
 message("# GRPC_CPP_PLUGIN: ${GRPC_CPP_PLUGIN}")
-message("# GRPC_CPP_REFLECTION: ${GRPC_CPP_REFLECTION}")
\ No newline at end of file
+message("# GRPC_CPP_REFLECTION_INCLUDE: ${GRPC_CPP_REFLECTION_INCLUDE}")
\ No newline at end of file
diff --git a/cpp/ppc-framework/protocol/GlobalConfig.h b/cpp/ppc-framework/protocol/GlobalConfig.h
index ab02443e..e223a746 100644
--- a/cpp/ppc-framework/protocol/GlobalConfig.h
+++ b/cpp/ppc-framework/protocol/GlobalConfig.h
@@ -33,7 +33,7 @@ class GlobalConfig
     GlobalConfig()
     {
         // set the supported-curves for ecdh-psi
-        auto key = calculateKey((uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        auto key = calculateKey((uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
 #ifdef ENABLE_CRYPTO_MB
 #ifdef ENABLE_CPU_FEATURES
         if (ppc::CPU_FEATURES.avx512ifma)
diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h
index c9c6c850..ac86293b 100644
--- a/cpp/ppc-framework/protocol/Protocol.h
+++ b/cpp/ppc-framework/protocol/Protocol.h
@@ -57,8 +57,7 @@ inline std::ostream& operator<<(std::ostream& _out, TaskType const& _type)
 }
 
 // the PSIAlgorithm type
-// TODO: rename PSIAlgorithmType to TaskAlgorithmType
-enum class PSIAlgorithmType : uint8_t
+enum class TaskAlgorithmType : uint8_t
 {
     // PSI implementation for https://eprint.iacr.org/2020/729.pdf(Private Set Intersection in the
     // Internet Setting from Lightweight Oblivious PRF)
@@ -76,34 +75,29 @@ enum class PSIAlgorithmType : uint8_t
     BS_ECDH_PSI = 0x06,
 };
 
-// enum class PSIAlgorithmType : uint8_t
-// {
-//     OT_PIR_2PC = 0x10,
-// };
-
-inline std::ostream& operator<<(std::ostream& _out, PSIAlgorithmType const& _type)
+inline std::ostream& operator<<(std::ostream& _out, TaskAlgorithmType const& _type)
 {
     switch (_type)
     {
-    case PSIAlgorithmType::CM_PSI_2PC:
+    case TaskAlgorithmType::CM_PSI_2PC:
         _out << "CM_PSI_2PC";
         break;
-    case PSIAlgorithmType::RA_PSI_2PC:
+    case TaskAlgorithmType::RA_PSI_2PC:
         _out << "RA_PSI_2PC";
         break;
-    case PSIAlgorithmType::LABELED_PSI_2PC:
+    case TaskAlgorithmType::LABELED_PSI_2PC:
         _out << "LABELED_PSI_2PC";
         break;
-    case PSIAlgorithmType::ECDH_PSI_2PC:
+    case TaskAlgorithmType::ECDH_PSI_2PC:
         _out << "ECDH_PSI_2PC";
         break;
-    case PSIAlgorithmType::OT_PIR_2PC:
+    case TaskAlgorithmType::OT_PIR_2PC:
         _out << "OT_PIR_2PC";
         break;
-    case PSIAlgorithmType::ECDH_PSI_MULTI:
+    case TaskAlgorithmType::ECDH_PSI_MULTI:
         _out << "ECDH_PSI_MULTI";
         break;
-    case PSIAlgorithmType::ECDH_PSI_CONN:
+    case TaskAlgorithmType::ECDH_PSI_CONN:
         _out << "ECDH_PSI_CONN";
         break;
     default:
diff --git a/cpp/ppc-framework/protocol/Task.h b/cpp/ppc-framework/protocol/Task.h
index 727205db..a43dd4fb 100644
--- a/cpp/ppc-framework/protocol/Task.h
+++ b/cpp/ppc-framework/protocol/Task.h
@@ -162,7 +162,7 @@ inline std::string printTaskInfo(Task::ConstPtr _task)
     std::ostringstream stringstream;
     stringstream << LOG_KV("id", _task->id())
                  << LOG_KV("type", (ppc::protocol::TaskType)_task->type())
-                 << LOG_KV("algorithm", (ppc::protocol::PSIAlgorithmType)_task->algorithm())
+                 << LOG_KV("algorithm", (ppc::protocol::TaskAlgorithmType)_task->algorithm())
                  << LOG_KV("taskPtr", _task);
     if (_task->selfParty())
     {
diff --git a/cpp/test-utils/FakeFront.h b/cpp/test-utils/FakeFront.h
index ab1c9709..539c1238 100644
--- a/cpp/test-utils/FakeFront.h
+++ b/cpp/test-utils/FakeFront.h
@@ -89,7 +89,7 @@ class FakeFront : public FrontInterface
         uint8_t type = _message->algorithmType();
         switch (type)
         {
-        case uint8_t(PSIAlgorithmType::RA_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::RA_PSI_2PC):
         {
             if (!m_agencyToRA2018.count(_agencyID))
             {
@@ -103,7 +103,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::LABELED_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::LABELED_PSI_2PC):
         {
             if (!m_agencyToLabeledPSI.count(_agencyID))
             {
@@ -117,7 +117,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::CM_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::CM_PSI_2PC):
         {
             if (!m_agencyToCM2020.count(_agencyID))
             {
@@ -131,7 +131,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::ECDH_PSI_2PC):
+        case uint8_t(TaskAlgorithmType::ECDH_PSI_2PC):
         {
             if (!m_agencyToEcdhPSI.count(_agencyID))
             {
@@ -145,7 +145,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::ECDH_PSI_MULTI):
+        case uint8_t(TaskAlgorithmType::ECDH_PSI_MULTI):
         {
             if (!m_agencyToEcdhMutliPSI.count(_agencyID))
             {
@@ -159,7 +159,7 @@ class FakeFront : public FrontInterface
             _callback(nullptr);
             break;
         }
-        case uint8_t(PSIAlgorithmType::OT_PIR_2PC):
+        case uint8_t(TaskAlgorithmType::OT_PIR_2PC):
         {
             if (!m_agencyToOTPIR.count(_agencyID))
             {
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index a4c467aa..d7a83ca2 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -25,22 +25,6 @@ install_gsasl_depend()
         LOG_INFO "the libgasal.a has already exists!"
         return
     fi
-    LOG_INFO "download and install gsasl..."
-    wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/gsasl-1.8.0.tar.gz && tar -xvf gsasl-1.8.0.tar.gz 
-    
-    # centos
-    if [[ "${os_type}" == "centos" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    # macos
-    if [[ "${os_type}" == "macos" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    # ubuntu
-    if [[ "${os_type}" == "ubuntu" ]];then
-        cd gsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
-    fi
-    LOG_INFO "download and install gsasl success..."
 
     LOG_INFO "download and install libgsasl..."
     wget --no-check-certificate https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz && tar -xvf libgsasl-1.8.0.tar.gz 
@@ -57,6 +41,8 @@ install_gsasl_depend()
     if [[ "${os_type}" == "ubuntu" ]];then
         cd libgsasl-1.8.0 && ./configure --with-pic && make -j4 && make install
     fi
+    cd .. && rm -rf libgsasl-1.8.0 && rm -rf libgsasl-1.8.0.tar.gz
+    df -lh
     LOG_INFO "download and install libgsasl success..."
 }
 
@@ -74,6 +60,7 @@ install_nasm_depend()
     LOG_INFO "download and install nasm 2.15..."
     wget --no-check-certificate https://www.nasm.us/pub/nasm/releasebuilds/2.15/nasm-2.15.tar.gz && tar -xvf nasm-2.15.tar.gz
     cd nasm-2.15 && ./configure && make -j4 && make install
+    cd .. && rm -rf nasm-2.15.tar.gz && rm -rf nasm-2.15
     LOG_INFO "download and install nasm success..."
 }
 
diff --git a/cpp/tools/ppc-builder/src/controller/commandline_helper.py b/cpp/tools/ppc-builder/src/controller/commandline_helper.py
index 7873925c..bc995877 100644
--- a/cpp/tools/ppc-builder/src/controller/commandline_helper.py
+++ b/cpp/tools/ppc-builder/src/controller/commandline_helper.py
@@ -83,6 +83,6 @@ def execute_command(args):
     if command == utilities.CommandInfo.generate_config:
         generate_node_config(args, toml_config)
         return
-    # TODO: implement extend
+    # TODO: implement expand
     if command == utilities.CommandInfo.extend:
         utilities.log_error("unimplemented command %s" % command)
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h
index 3ecd2033..e51b1ade 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRConfig.h
@@ -44,7 +44,7 @@ class OtPIRConfig : public ppc::psi::PSIConfig
         uint16_t _parallelism = 3,
         const front::PPCMessageFactory::Ptr& _msgFactory =
             std::make_shared())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::OT_PIR_2PC, _selfPartyID, std::move(_front),
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::OT_PIR_2PC, _selfPartyID, std::move(_front),
             _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes),
         m_cryptoBox(std::move(_cryptoBox)),
         m_threadPool(std::move(_threadPool)),
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
index d0d2d383..6c390af9 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
@@ -39,7 +39,7 @@ using namespace ppc::psi;
 
 OtPIRImpl::OtPIRImpl(const OtPIRConfig::Ptr& _config, unsigned _idleTimeMs)
   : Worker("OT-PIR", _idleTimeMs),
-    TaskGuarder(_config, PSIAlgorithmType::OT_PIR_2PC, "OT-PIR-Timer"),
+    TaskGuarder(_config, TaskAlgorithmType::OT_PIR_2PC, "OT-PIR-Timer"),
     m_config(_config),
     m_msgQueue(std::make_shared()),
     m_ioService(std::make_shared()),
@@ -221,7 +221,6 @@ void OtPIRImpl::checkFinishedTask()
         removeReceiver(taskID);
         removeSender(taskID);
         removePendingTask(taskID);
-        // TODO: 为什么这里要再调用一下执行任务
         // asyncRunTask();
     }
 }
@@ -314,7 +313,7 @@ void OtPIRImpl::onHelloReceiver(const ppc::front::PPCMessageFace::Ptr& _message)
         // PIR_LOG(INFO) << LOG_BADGE("buildPPCMessage");
 
         auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR),
-            uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID,
+            uint8_t(protocol::TaskAlgorithmType::OT_PIR_2PC), m_taskID,
             std::make_shared());
         message->setMessageType(uint8_t(OTPIRMessageType::RESULTS));
         ppctars::serialize::encode(receiverMessageParams, *message->data());
@@ -510,7 +509,8 @@ void OtPIRImpl::runSenderGenerateCipher(PirTaskMessage taskMessage)
     // senderMessageParams.requestAgencyDataset = taskMessage.requestAgencyDataset;
     senderMessageParams.sendObfuscatedHash = senderMessage.sendObfuscatedHash;
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PIR),
-        uint8_t(protocol::PSIAlgorithmType::OT_PIR_2PC), m_taskID, std::make_shared());
+        uint8_t(protocol::TaskAlgorithmType::OT_PIR_2PC), m_taskID,
+        std::make_shared());
     message->setMessageType(uint8_t(OTPIRMessageType::HELLO_RECEIVER));
     ppctars::serialize::encode(senderMessageParams, *message->data());
     addSender(senderMessage);
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
index 971714f7..56aeac7d 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
@@ -184,8 +184,6 @@ class OtPIRImpl : public std::enable_shared_from_this,
     std::unordered_map m_receivers;
     mutable bcos::SharedMutex x_receivers;
 
-
-    // TODO: 改为prefix-message的map 预处理的数据集
     std::vector> messageKeypair;
     std::queue>
         m_taskQueue;
diff --git a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
index b76d867b..621a4255 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
+++ b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
@@ -197,7 +197,7 @@ void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, b
     senderPIRTask->setSelf(_senderParty);
     senderPIRTask->addParty(_receiverParty);
     senderPIRTask->setSyncResultToPeer(_syncResults);
-    senderPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC);
+    senderPIRTask->setAlgorithm((uint8_t)TaskAlgorithmType::OT_PIR_2PC);
     senderPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR);
 
     auto receiverPIRTask = std::make_shared(receiverAgencyName);
@@ -206,7 +206,7 @@ void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, b
     receiverPIRTask->setSelf(_receiverParty);
     receiverPIRTask->addParty(_senderParty);
     receiverPIRTask->setSyncResultToPeer(_syncResults);
-    receiverPIRTask->setAlgorithm((uint8_t)PSIAlgorithmType::OT_PIR_2PC);
+    receiverPIRTask->setAlgorithm((uint8_t)TaskAlgorithmType::OT_PIR_2PC);
     receiverPIRTask->setType((uint8_t)ppc::protocol::TaskType::PIR);
 
     testOTPIR(factory, senderPIR, receiverPIR, senderPIRTask, receiverPIRTask, _expectedPIRResult,
diff --git a/cpp/wedpr-computing/ppc-psi/src/Common.h b/cpp/wedpr-computing/ppc-psi/src/Common.h
index b2673d28..392e63d2 100644
--- a/cpp/wedpr-computing/ppc-psi/src/Common.h
+++ b/cpp/wedpr-computing/ppc-psi/src/Common.h
@@ -94,7 +94,7 @@ enum PSIRetCode : int
     TaskIsNotRunning = -1025,
     OnException = -1026,
     TaskExists = -1027,
-    TaskKilled= -1028,
+    TaskKilled = -1028,
     TaskCountReachMax = -1029,
     TaskTimeout = -1030
 };
diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
index 4b67e599..1b81b545 100644
--- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
@@ -34,12 +34,12 @@ class PSIConfig
 {
 public:
     using Ptr = std::shared_ptr;
-    PSIConfig(ppc::protocol::PSIAlgorithmType _algorithmType,
+    PSIConfig(ppc::protocol::TaskAlgorithmType _algorithmType,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader)
       : m_algorithmType(_algorithmType), m_dataResourceLoader(std::move(_dataResourceLoader))
     {}
 
-    PSIConfig(ppc::protocol::PSIAlgorithmType _algorithmType, const std::string& _selfParty,
+    PSIConfig(ppc::protocol::TaskAlgorithmType _algorithmType, const std::string& _selfParty,
         ppc::front::FrontInterface::Ptr _front,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes)
@@ -62,7 +62,7 @@ class PSIConfig
     ppc::front::FrontInterface::Ptr const& front() const { return m_front; }
     std::string selfParty() const { return m_selfParty; }
 
-    ppc::protocol::PSIAlgorithmType algorithmType() const { return m_algorithmType; }
+    ppc::protocol::TaskAlgorithmType algorithmType() const { return m_algorithmType; }
     ppc::front::PPCMessageFaceFactory::Ptr const& ppcMsgFactory() const { return m_ppcMsgFactory; }
     int networkTimeout() const { return m_networkTimeout; }
 
@@ -139,7 +139,7 @@ class PSIConfig
 
 protected:
     // the psi-alogrithm-type
-    ppc::protocol::PSIAlgorithmType m_algorithmType;
+    ppc::protocol::TaskAlgorithmType m_algorithmType;
     std::string m_selfParty;
     ppc::front::FrontInterface::Ptr m_front;
     // the front message factory
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
index 08c3bb30..272fbd1d 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
@@ -37,7 +37,7 @@ class BsEcdhPSIFactory
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _timeoutMinutes)
     {
         auto config = std::make_shared(
-            ppc::protocol::PSIAlgorithmType::BS_ECDH_PSI, std::move(_dataResourceLoader));
+            ppc::protocol::TaskAlgorithmType::BS_ECDH_PSI, std::move(_dataResourceLoader));
         return std::make_shared(
             std::move(config), std::move(_threadPool), _timeoutMinutes);
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h
index a939f8f2..035d3ba8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h
@@ -34,7 +34,6 @@
 
 namespace ppc::psi
 {
-
 enum TaskStep : int
 {
     Initializing = 1,
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h
index 9ae29a01..b69f235a 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/core/BsEcdhTaskState.h
@@ -162,7 +162,7 @@ class BsEcdhTaskState
     {
         m_status = protocol::TaskStatus::RUNNING;
         m_autoPauseThreshold = PAUSE_THRESHOLD;
-	m_latestActiveTime = bcos::utcSteadyTime();
+        m_latestActiveTime = bcos::utcSteadyTime();
     }
 
     void turnToPausing()
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h
index 8427bebe..5c689c40 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_ffi_edwards25519.h
@@ -1,21 +1,21 @@
 /**
-*  Copyright (C) 2023 WeDPR.
-*  SPDX-License-Identifier: Apache-2.0
-*  Licensed under the Apache License, Version 2.0 (the "License");
-*  you may not use this file except in compliance with the License.
-*  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-*  Unless required by applicable law or agreed to in writing, software
-*  distributed under the License is distributed on an "AS IS" BASIS,
-*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-*  See the License for the specific language governing permissions and
-*  limitations under the License.
-*
-* @file wedpr_ffi_edwards25519.h
-* @author: shawnhe
-* @date 2023-10-10
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file wedpr_ffi_edwards25519.h
+ * @author: shawnhe
+ * @date 2023-10-10
  */
 
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h
index c477d117..9fcf5da2 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/ffi/wedpr_utilities.h
@@ -1,22 +1,22 @@
 /**
-*  Copyright (C) 2023 WeDPR.
-*  SPDX-License-Identifier: Apache-2.0
-*  Licensed under the Apache License, Version 2.0 (the "License");
-*  you may not use this file except in compliance with the License.
-*  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-*  Unless required by applicable law or agreed to in writing, software
-*  distributed under the License is distributed on an "AS IS" BASIS,
-*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-*  See the License for the specific language governing permissions and
-*  limitations under the License.
-*
-* @file wedpr_utilities.h
-* @author: shawnhe
-* @date 2023-10-10
-*/
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file wedpr_utilities.h
+ * @author: shawnhe
+ * @date 2023-10-10
+ */
 
 #ifndef _WEDPR_UTILITIES_H_
 #define _WEDPR_UTILITIES_H_
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
index 2f07e2cc..bd89fb11 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
@@ -44,7 +44,7 @@ class CM2020PSIConfig : public PSIConfig
         uint16_t _parallelism = 3,
         const front::PPCMessageFactory::Ptr& _msgFactory =
             std::make_shared())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::CM_PSI_2PC, _selfPartyID, std::move(_front),
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::CM_PSI_2PC, _selfPartyID, std::move(_front),
             _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes),
         m_cryptoBox(std::move(_cryptoBox)),
         m_threadPool(std::move(_threadPool)),
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
index d4534b5c..e0b62a7e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
@@ -32,7 +32,7 @@ using namespace ppc::task;
 
 CM2020PSIImpl::CM2020PSIImpl(const CM2020PSIConfig::Ptr& _config, unsigned _idleTimeMs)
   : Worker("CM2020-PSI", _idleTimeMs),
-    TaskGuarder(_config, PSIAlgorithmType::CM_PSI_2PC, "CM2020-PSI-Timer"),
+    TaskGuarder(_config, TaskAlgorithmType::CM_PSI_2PC, "CM2020-PSI-Timer"),
     m_config(_config),
     m_msgQueue(std::make_shared()),
     m_ioService(std::make_shared()),
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
index 0f4c92d1..dfe83d24 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
@@ -95,7 +95,7 @@ void CM2020PSIReceiver::runReceiver()
         }
 
         auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-            uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID,
+            uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC), m_taskID,
             std::make_shared());
         message->setMessageType(uint8_t(CM2020PSIMessageType::HELLO_SENDER));
         ppctars::serialize::encode(cm2020Params, *message->data());
@@ -189,7 +189,7 @@ void CM2020PSIReceiver::syncInputsSize()
                           << LOG_KV("data", *bcos::toHexString(*data))
                           << LOG_KV("m_rInputSize", m_rInputSize);
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(
-        uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, data);
+        uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, data);
     message->setMessageType(uint8_t(CM2020PSIMessageType::RECEIVER_SIZE));
 
     m_config->front()->asyncSendMessage(
@@ -216,7 +216,7 @@ void CM2020PSIReceiver::runRandomOT()
     m_otState = m_ot->senderGeneratePointA();
 
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-        uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, m_otState.second);
+        uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC), m_taskID, m_otState.second);
     message->setMessageType(uint8_t(CM2020PSIMessageType::POINT_A));
 
     // send point A to ot receiver
@@ -506,7 +506,7 @@ void CM2020PSIReceiver::negotiateMatrix(uint32_t _bucketIndex, uint32_t _matrixI
         auto sendBuffer = std::make_shared(
             buffer->begin() + rLen, buffer->begin() + rLen + currentLen);
         auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-            uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, sendBuffer);
+            uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC), m_taskID, sendBuffer);
         message->setSeq(_bucketIndex * totalRound + round);
         message->setMessageType(uint8_t(CM2020PSIMessageType::MATRIX));
 
@@ -808,7 +808,7 @@ void CM2020PSIReceiver::syncResults(uint32_t _count)
     auto countData = std::make_shared();
     encodeUnsignedNum(countData, uint32_t(_count));
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(
-        uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, countData);
+        uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, countData);
     message->setMessageType(uint8_t(CM2020PSIMessageType::RESULTS_SIZE));
 
     auto error = m_config->sendMessage(m_taskState->peerID(), message);
@@ -837,7 +837,7 @@ void CM2020PSIReceiver::syncResults(uint32_t _count)
             if (count == currentLen)
             {
                 auto resMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI),
-                    uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, buffer);
+                    uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, buffer);
                 resMessage->setMessageType(uint8_t(CM2020PSIMessageType::RESULTS));
                 resMessage->setSeq(round);
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
index d4018dc4..99a9105f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
@@ -92,7 +92,7 @@ void CM2020PSISender::runSender()
     try
     {
         auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-            uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID,
+            uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC), m_taskID,
             std::make_shared());
         message->setMessageType(uint8_t(CM2020PSIMessageType::HELLO_RECEIVER));
 
@@ -176,7 +176,7 @@ void CM2020PSISender::syncInputsSize()
     auto data = std::make_shared();
     encodeUnsignedNum(data, m_sInputSize);
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(
-        uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, data);
+        uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, data);
     message->setMessageType(uint8_t(CM2020PSIMessageType::SENDER_SIZE));
 
     m_config->front()->asyncSendMessage(
@@ -239,8 +239,8 @@ void CM2020PSISender::onPointAReceived(front::PPCMessageFace::Ptr _message)
         auto retPair = m_ot->receiverGeneratePointsB(m_otChoices, pointA);
 
         // send batch point B to ot sender
-        auto message = m_config->ppcMsgFactory()->buildPPCMessage(
-            uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, retPair.first);
+        auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI),
+            uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, retPair.first);
         message->setMessageType(uint8_t(CM2020PSIMessageType::POINT_B_ARRAY));
 
         m_config->front()->asyncSendMessage(
@@ -413,7 +413,8 @@ void CM2020PSISender::noticeReceiverDoNextRound()
     CM2020_PSI_LOG(INFO) << LOG_BADGE("noticeReceiverDoNextRound") << LOG_KV("taskID", m_taskID)
                          << LOG_KV("round", m_oprfRound);
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-        uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC), m_taskID, std::make_shared());
+        uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC), m_taskID,
+        std::make_shared());
     message->setMessageType(uint8_t(CM2020PSIMessageType::DO_NEXT_ROUND));
     m_config->front()->asyncSendMessage(
         m_taskState->peerID(), message, m_config->networkTimeout(),
@@ -634,7 +635,7 @@ void CM2020PSISender::computeAndSendHash()
             }
 
             auto message = m_config->ppcMsgFactory()->buildPPCMessage(
-                uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC), m_taskID, buffer);
+                uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, buffer);
             message->setMessageType(uint8_t(CM2020PSIMessageType::HASHES));
             message->setSeq(round);
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
index b690883e..4d461183 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
@@ -76,8 +76,8 @@ class CM2020PSIResult : public protocol::TaskResult
             jsonData["communication"] = m_communication;
             jsonData["syncResult"] = m_enableSyncResults;
             jsonData["intersections"] = m_intersections;
-//            jsonData["party0Size"] = m_party0Size;
-//            jsonData["party1Size"] = m_party1Size;
+            //            jsonData["party0Size"] = m_party0Size;
+            //            jsonData["party1Size"] = m_party1Size;
 
             if (!m_outputs.empty())
             {
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
index d42c8bf8..147dbab5 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
@@ -41,7 +41,7 @@ class EcdhConnPSIConfig : public PSIConfig
         EcdhConnPSIMessageFactory::Ptr _psiMsgFactory,
         ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader,
         bcos::ThreadPool::Ptr _threadPool)
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN, _ppcConfig->agencyID(), _front,
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_CONN, _ppcConfig->agencyID(), _front,
             _ppcMsgFactory, _dataResourceLoader, _ppcConfig->holdingMessageMinutes()),
         m_msgFactory(_psiMsgFactory),
         m_threadPool(_threadPool),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
index 81b5339f..6e989d85 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
@@ -33,7 +33,7 @@ using namespace ppc::task;
 EcdhConnPSIImpl::EcdhConnPSIImpl(const EcdhConnPSIConfig::Ptr& _config, unsigned _idleTimeMs)
   : m_config(std::move(_config)),
     m_msgPool(std::make_shared()),
-    TaskGuarder(_config, PSIAlgorithmType::ECDH_PSI_CONN, "ECDH-CONN-PSI-Timer"),
+    TaskGuarder(_config, TaskAlgorithmType::ECDH_PSI_CONN, "ECDH-CONN-PSI-Timer"),
     m_ecdhConnTaskStateFactory(std::make_shared())
 {}
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp
index 9d342c9b..d615cdc5 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/core/EcdhConnPSIServer.cpp
@@ -50,9 +50,9 @@ void EcdhConnPSIServer::onHandShakeRequestHandler(const bcos::bytes& _msg)
     auto clientCurves = handRequestVo->GetCurve();
     auto clientHashList = handRequestVo->GetHash();
     auto supportedCurves = g_PPCConfig.supportedCurves(
-        (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        (uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     auto supportedHashList = g_PPCConfig.supportedHashList(
-        (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        (uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     std::set localCurves(supportedCurves.begin(), supportedCurves.end());
     std::set localHashTypes(supportedHashList.begin(), supportedHashList.end());
     bool curveSelected = false;
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp
index ec5ceb22..783cd4fb 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/protocol/PSIConnMessageFactory.cpp
@@ -53,8 +53,8 @@ bcos::bytes PSIConnMessageFactory::createHandshakeRequest(
     auto _hash = handShakeRequestVo->GetHash().cbegin();
 
 #ifdef ENABLE_CONN
-    eccSuit->set_curve(2); // SM2
-    eccSuit->set_hash(11); // SHA256
+    eccSuit->set_curve(2);  // SM2
+    eccSuit->set_hash(11);  // SHA256
 #else
     eccSuit->set_curve(*_curve);
     eccSuit->set_hash(*_hash);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
index 9b256a72..d24a8f2c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
@@ -20,7 +20,7 @@ class EcdhMultiPSIConfig : public PSIConfig
         int _holdingMessageMinutes, EcdhMultiPSIMessageFactory::Ptr const& _psiMsgFactory,
         const front::PPCMessageFactory::Ptr& _ppcMsgFactory =
             std::make_shared())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI, _selfPartyID, std::move(_front),
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_MULTI, _selfPartyID, std::move(_front),
             _ppcMsgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes),
         m_threadPool(std::move(_threadPool)),
         m_cryptoBox(std::move(_cryptoBox)),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
index e3e9b9bb..1f7ec3e1 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
@@ -13,7 +13,7 @@ using namespace ppc::task;
 EcdhMultiPSIImpl::EcdhMultiPSIImpl(const EcdhMultiPSIConfig::Ptr& _config, unsigned _idleTimeMs)
   : m_config(std::move(_config)),
     m_msgQueue(std::make_shared()),
-    TaskGuarder(_config, PSIAlgorithmType::ECDH_PSI_MULTI, "ECDH-MULTI-PSI-Timer")
+    TaskGuarder(_config, TaskAlgorithmType::ECDH_PSI_MULTI, "ECDH-MULTI-PSI-Timer")
 {}
 
 void EcdhMultiPSIImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index 690738f2..877e1516 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -19,7 +19,8 @@ EcdhMultiPSIMaster::EcdhMultiPSIMaster(EcdhMultiPSIConfig::Ptr _config, TaskStat
     m_taskID = task->id();
     m_masterCipherDataCache = std::make_shared();
     m_final_counts[m_taskID] = 0;
-    m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(), m_config->selfParty()) != receivers.end());
+    m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
+                                                    m_config->selfParty()) != receivers.end());
 }
 
 void EcdhMultiPSIMaster::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
index 41affc04..d2810d16 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
@@ -38,8 +38,8 @@ class EcdhPSIConfig : public PSIConfig
         ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader, uint32_t _dataBatchSize,
         int _holdingMessageMinutes, bcos::ThreadPool::Ptr const& _threadPool = nullptr,
         int _threadPoolSize = std::thread::hardware_concurrency())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC, _selfParty, _front, _ppcMsgFactory,
-            _dataResourceLoader, _holdingMessageMinutes),
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_2PC, _selfParty, _front,
+            _ppcMsgFactory, _dataResourceLoader, _holdingMessageMinutes),
         m_msgFactory(_msgFactory),
         m_threadPool(_threadPool),
         m_ecdhCryptoFactory(_ecdhCryptoFactory),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
index 3162328d..694dfe86 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
@@ -373,7 +373,7 @@ void EcdhPSIImpl::blindData(TaskState::Ptr const& _taskState)
             ECDH_LOG(WARNING) << LOG_DESC("blindData: send SyncDataBatchInfo request error")
                               << LOG_KV("code", _error->errorCode())
                               << LOG_KV("msg", _error->errorMessage());
-            // cancel the task(TODO:retry)
+            // cancel the task
             psi->cancelTask(std::move(_error), _taskState->task()->id());
         });
     ECDH_LOG(INFO) << LOG_DESC("blindData success") << printTaskInfo(_taskState->task())
@@ -653,9 +653,9 @@ void EcdhPSIImpl::onHandshakeRequest(PSIMessageInterface::Ptr const& _msg)
     std::set clientHashList(hashList.begin(), hashList.end());
 
     auto supportedCurves = g_PPCConfig.supportedCurves(
-        (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        (uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     auto supportedHashList = g_PPCConfig.supportedHashList(
-        (uint8_t)TaskType::PSI, (uint8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+        (uint8_t)TaskType::PSI, (uint8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     std::set localCurves(supportedCurves.begin(), supportedCurves.end());
     std::set localHashTypes(supportedHashList.begin(), supportedHashList.end());
     auto handshakeResponse =
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
index 01b7ebdd..8fe48aab 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
@@ -41,8 +41,8 @@ class LabeledPSIConfig : public PSIConfig
         ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
         front::PPCMessageFactory::Ptr _msgFactory = std::make_shared())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC, _selfPartyID, std::move(_front),
-            _msgFactory, _dataResourceLoader, _holdingMessageMinutes),
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::LABELED_PSI_2PC, _selfPartyID,
+            std::move(_front), _msgFactory, _dataResourceLoader, _holdingMessageMinutes),
         m_cryptoBox(std::move(_cryptoBox)),
         m_threadPool(std::move(_threadPool)),
         m_messageFactory(_msgFactory)
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
index 290b1f83..0ac28d7e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
@@ -37,7 +37,7 @@ using namespace ppc::task;
 
 LabeledPSIImpl::LabeledPSIImpl(LabeledPSIConfig::Ptr _config, unsigned _idleTimeMs)
   : Worker("LabeledPSI", _idleTimeMs),
-    TaskGuarder(_config, PSIAlgorithmType::LABELED_PSI_2PC, "Labeled-PSI-Timer"),
+    TaskGuarder(_config, TaskAlgorithmType::LABELED_PSI_2PC, "Labeled-PSI-Timer"),
     m_config(std::move(_config)),
     m_msgQueue(std::make_shared()),
     m_worker(std::make_shared("senderDB-worker", 1))
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp
index 37221243..ec399d4e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSIReceiver.cpp
@@ -158,7 +158,7 @@ void LabeledPSIReceiver::runReceiver()
                           << LOG_KV("taskID", m_taskID) << LOG_KV("itemsSize", m_items.size());
 
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-        uint8_t(protocol::PSIAlgorithmType::LABELED_PSI_2PC), m_taskID,
+        uint8_t(protocol::TaskAlgorithmType::LABELED_PSI_2PC), m_taskID,
         std::make_shared());
     message->setMessageType(uint8_t(LabeledPSIMessageType::PARAMS_REQUEST));
 
@@ -250,7 +250,7 @@ void LabeledPSIReceiver::runOprfAsClient()
     oprfData.data = m_oprfClient->blind(m_items);
 
     auto blindedMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI),
-        uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared());
+        uint8_t(TaskAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared());
     blindedMessage->setMessageType(uint8_t(LabeledPSIMessageType::OPRF_BLINDED_ITEMS));
     ppctars::serialize::encode(oprfData, *blindedMessage->data());
 
@@ -448,7 +448,7 @@ void LabeledPSIReceiver::requestQuery()
     }
 
     auto queryMessage = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI),
-        uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared());
+        uint8_t(TaskAlgorithmType::LABELED_PSI_2PC), m_taskID, std::make_shared());
     queryMessage->setMessageType(uint8_t(LabeledPSIMessageType::QUERY));
     ppctars::serialize::encode(queryRequest, *queryMessage->data());
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp
index 419140cf..dcfbe99c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/core/LabeledPSISender.cpp
@@ -50,7 +50,7 @@ void LabeledPSISender::handlePsiParamsRequest(const PPCMessageFace::Ptr& _messag
     {
         auto tarsParams = fromPSIParams(m_senderDB->getParams(), m_senderDB->getBinBundleCount());
         auto paramsMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI),
-            uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared());
+            uint8_t(TaskAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared());
         paramsMessage->setMessageType(uint8_t(LabeledPSIMessageType::PARAMS_RESPONSE));
         ppctars::serialize::encode(tarsParams, *paramsMessage->data());
 
@@ -91,7 +91,7 @@ void LabeledPSISender::handleBlindedItems(const front::PPCMessageFace::Ptr& _mes
         evaluatedData.data = m_config->oprfServer()->evaluate(blindedData.data);
 
         auto evaluatedMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI),
-            uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared());
+            uint8_t(TaskAlgorithmType::LABELED_PSI_2PC), taskID, std::make_shared());
         evaluatedMessage->setMessageType(uint8_t(LabeledPSIMessageType::OPRF_EVALUATED_ITEMS));
         ppctars::serialize::encode(evaluatedData, *evaluatedMessage->data());
 
@@ -432,7 +432,7 @@ void LabeledPSISender::sendResultPackage(const std::string& _taskID, const std::
     }
 
     auto resultMessage = m_messageFactory->buildPPCMessage(uint8_t(TaskType::PSI),
-        uint8_t(PSIAlgorithmType::LABELED_PSI_2PC), _taskID, std::make_shared());
+        uint8_t(TaskAlgorithmType::LABELED_PSI_2PC), _taskID, std::make_shared());
     resultMessage->setMessageType(uint8_t(LabeledPSIMessageType::RESPONSE));
     resultMessage->setSeq(_seq);
     ppctars::serialize::encode(queryResponse, *resultMessage->data());
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h
index 66238319..bea4974e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.h
@@ -32,7 +32,7 @@
 
 #define PSI_FRAMEWORK_LOG(LEVEL)        \
     BCOS_LOG(LEVEL) << LOG_BADGE("PSI") \
-                    << LOG_BADGE((ppc::protocol::PSIAlgorithmType)m_psiConfig->algorithmType())
+                    << LOG_BADGE((ppc::protocol::TaskAlgorithmType)m_psiConfig->algorithmType())
 namespace ppc::psi
 {
 using PSIMsgQueue = bcos::ConcurrentQueue;
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
index aea7e3bf..c7da3a68 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
@@ -35,7 +35,7 @@ class TaskGuarder
     using Ptr = std::shared_ptr;
     explicit TaskGuarder(PSIConfig::Ptr _config) : m_config(std::move(_config)) {}
     TaskGuarder(
-        PSIConfig::Ptr _config, protocol::PSIAlgorithmType _type, const std::string& _timerName)
+        PSIConfig::Ptr _config, protocol::TaskAlgorithmType _type, const std::string& _timerName)
       : m_config(std::move(_config)),
         m_type(_type),
         m_taskStateFactory(std::make_shared())
@@ -322,7 +322,7 @@ class TaskGuarder
 
 protected:
     PSIConfig::Ptr m_config;
-    protocol::PSIAlgorithmType m_type;
+    protocol::TaskAlgorithmType m_type;
     TaskStateFactory::Ptr m_taskStateFactory;
     // the timer used to check the activity of peer node
     std::shared_ptr m_pingTimer;
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
index 6954356e..9b4e2d5f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
@@ -42,7 +42,7 @@ class RA2018PSIConfig : public PSIConfig
         std::string const& _dbName = "ra2018", uint64_t _cuckooFilterCacheSize = 255 * 1024 * 1024,
         uint64_t _ra2018CacheCapacity = 1024 * 1024 * 1024, int _dataBatchSize = -1,
         int _threadPoolSize = std::thread::hardware_concurrency())
-      : PSIConfig(ppc::protocol::PSIAlgorithmType::RA_PSI_2PC, _selfParty, _front, _ppcMsgFactory,
+      : PSIConfig(ppc::protocol::TaskAlgorithmType::RA_PSI_2PC, _selfParty, _front, _ppcMsgFactory,
             _dataResourceLoader, _holdingMessageMinutes),
         m_oprf(std::move(_oprf)),
         m_binHashImpl(std::move(_binHashImpl)),
diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
index bb9f5bf6..bee77f70 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
@@ -153,7 +153,7 @@ void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _param
     senderPSITask->setSelf(_senderParty);
     senderPSITask->addParty(_receiverParty);
     senderPSITask->setSyncResultToPeer(_syncResults);
-    senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::CM_PSI_2PC);
+    senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::CM_PSI_2PC);
 
     auto receiverPSITask = std::make_shared(receiverAgencyName);
     receiverPSITask->setId(_taskID);
@@ -161,7 +161,7 @@ void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _param
     receiverPSITask->setSelf(_receiverParty);
     receiverPSITask->addParty(_senderParty);
     receiverPSITask->setSyncResultToPeer(_syncResults);
-    receiverPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::CM_PSI_2PC);
+    receiverPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::CM_PSI_2PC);
 
     testCM2020PSI(factory, senderPSI, receiverPSI, senderPSITask, receiverPSITask,
         _expectedPSIResult, _expectedSuccess, _expectedErrorCode);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
index 3a030eae..d7a75168 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
@@ -61,7 +61,7 @@ void runSetup(LabeledPSIImpl::Ptr _sender)
     senderPSITask->setId("0x00000000");
     senderPSITask->setParam(R"(["setup_sender_db","32"])");
     senderPSITask->setSelf(senderParty);
-    senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC);
+    senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
 
     std::atomic flag = 0;
     _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) {
@@ -97,7 +97,7 @@ void saveCache(LabeledPSIImpl::Ptr _sender)
     senderPSITask->setId("0x00000012");
     senderPSITask->setParam(R"(["save_sender_cache"])");
     senderPSITask->setSelf(senderParty);
-    senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC);
+    senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
 
     std::atomic flag = 0;
     _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) {
@@ -128,7 +128,7 @@ void loadCache(LabeledPSIImpl::Ptr _sender)
     senderPSITask->setId("0x00000034");
     senderPSITask->setParam(R"(["load_sender_cache"])");
     senderPSITask->setSelf(senderParty);
-    senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC);
+    senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
 
     std::atomic flag = 0;
     _sender->asyncRunTask(senderPSITask, [&flag](ppc::protocol::TaskResult::Ptr&& _response) {
@@ -236,13 +236,13 @@ void testLabeledPSIImplFunc(const std::string& _taskID, const std::string& _para
     senderPSITask->setId(_taskID);
     senderPSITask->setParam(_params);
     senderPSITask->setSelf(_senderParty);
-    senderPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC);
+    senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
     senderPSITask->addParty(_receiverParty);
 
     auto receiverPSITask = std::make_shared(receiverAgencyName);
     receiverPSITask->setId(_taskID);
     receiverPSITask->setSelf(_receiverParty);
-    receiverPSITask->setAlgorithm((uint8_t)PSIAlgorithmType::LABELED_PSI_2PC);
+    receiverPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
     receiverPSITask->addParty(_senderParty);
 
     testLabeledPSI(senderPSI, receiverPSI, senderPSITask, receiverPSITask, _expectedSuccess,
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
index 5c973bed..a100a4cd 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
@@ -87,7 +87,7 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS
     std::string taskID = "runPSI";
     clientPSITask->setId(taskID);
     clientPSITask->setType((int8_t)TaskType::PSI);
-    clientPSITask->setAlgorithm((int8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+    clientPSITask->setAlgorithm((int8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     clientPSITask->setSelf(clientParty);
     clientPSITask->addParty(serverParty);
 
@@ -103,7 +103,7 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS
         serverPSITask->setId(taskID);
     }
     serverPSITask->setType((int8_t)TaskType::PSI);
-    serverPSITask->setAlgorithm((int8_t)PSIAlgorithmType::ECDH_PSI_2PC);
+    serverPSITask->setAlgorithm((int8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     serverPSITask->addParty(clientParty);
     // run psi
     testPSI(factory, serverPSI, clientPSI, serverPSITask, clientPSITask, _expectPSISuccess,
diff --git a/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp
index 302b2f27..75a32d91 100644
--- a/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp
+++ b/cpp/wedpr-crypto/ppc-homo/paillier/OpenSSLPaillier.cpp
@@ -221,7 +221,6 @@ BigNum OpenSSLPaillier::decrypt(bcos::bytesConstRef const& _cipherData, void* _k
     result.div(result.bn().get(), NULL, pk->n.bn().get(), ctx.get());
 
     // result * lambdaInverse mod n
-    // TODO: check the overhead here
     auto lambdaInverse = sk->lambda.Invert(pk->n);
     result.modMul(result.bn().get(), lambdaInverse.bn().get(), pk->n.bn().get(), ctx.get());
     // decode result to support negative case
diff --git a/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java
index a3308d29..550fb158 100644
--- a/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java
+++ b/cpp/wedpr-crypto/ppc-udf/hive/src/main/java/com/webank/wedpr/hive/udf/config/model/KeyConfig.java
@@ -45,7 +45,6 @@ public class KeyConfig {
 
     public KeyConfig(ConfigProperty configProperty) throws ConfigException {
         Map keyConfig = configProperty.getKeys();
-        // TODO: use dpm key files
         /// load the aes private key
         aesPrivateKey = ConfigProperty.getValue(keyConfig, AES_PRIVATE_KEY_PROPERTY, null);
         if (aesPrivateKey == null) {
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index 1a18fad4..95fac22a 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -65,6 +65,14 @@ void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
     }
     // load the holdingMessageMinutes, in minutes
     m_holdingMessageMinutes = loadHoldingMessageMinutes(_pt, "gateway.holding_msg_minutes");
+    // load seqSyncPeriod
+    m_seqSyncPeriod = _pt.get("gateway.seq_sync_period_ms", 5000);
+    if (m_seqSyncPeriod < 3000)
+    {
+        BOOST_THROW_EXCEPTION(
+            InvalidConfig() << errinfo_comment(
+                "The value of gateway.seq_sync_period_ms must no little than 3s"));
+    }
     // load the grpcConfig
     m_grpcConfig = loadGrpcConfig("transport", _pt);
     // load the GrpcServerConfig
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index d66db91e..2971e259 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -241,6 +241,8 @@ class PPCConfig
     // for ut
     void setAgencyID(std::string const& agencyID) { m_agencyID = agencyID; }
 
+    uint16_t seqSyncPeriod() const { return m_seqSyncPeriod; }
+
 private:
     virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp,
         std::string const& sectionName, boost::property_tree::ptree const& pt);
@@ -296,6 +298,8 @@ class PPCConfig
     // the gateway holding message time, in minutes, default 30min
     int m_holdingMessageMinutes = 30;
 
+    uint16_t m_seqSyncPeriod = 5000;
+
     // the front config
     // TODO: parse the frontConfig
     ppc::front::FrontConfig::Ptr m_frontConfig;
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index d1e1acec..cfba5249 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -29,7 +29,7 @@
 #include "protocol/src/PPCMessage.h"
 #include "wedpr-transport/sdk/src/TransportBuilder.h"
 #if 0
-//TODO: optimize here
+//TODO: optimize here to implement EcdhConnPSI
 #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h"
 #endif
 #include "ppc-front/Front.h"
@@ -210,7 +210,7 @@ void Initializer::initMsgHandlers()
     // register msg-handlers for ra2018-psi
     auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::RA_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::RA_PSI_2PC,
         [weakRA2018PSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakRA2018PSI.lock();
             if (!psi)
@@ -224,7 +224,7 @@ void Initializer::initMsgHandlers()
     INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for labeledPSI");
     auto weakLabeledPSI = std::weak_ptr(m_labeledPSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::LABELED_PSI_2PC,
         [weakLabeledPSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakLabeledPSI.lock();
             if (!psi)
@@ -238,7 +238,7 @@ void Initializer::initMsgHandlers()
     INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for CM2020PSI");
     auto weakCM2020PSI = std::weak_ptr(m_cm2020PSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::CM_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::CM_PSI_2PC,
         [weakCM2020PSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakCM2020PSI.lock();
             if (!psi)
@@ -252,7 +252,7 @@ void Initializer::initMsgHandlers()
     // register msg-handlers for ecdh-psi
     auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_2PC,
         [weakEcdhPSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakEcdhPSI.lock();
             if (!psi)
@@ -266,7 +266,7 @@ void Initializer::initMsgHandlers()
     /*INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-conn-psi");
     auto weakEcdhConnPSI = std::weak_ptr(m_ecdhConnPSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_CONN,
         [weakEcdhConnPSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakEcdhConnPSI.lock();
             if (!psi)
@@ -280,7 +280,7 @@ void Initializer::initMsgHandlers()
     INIT_LOG(INFO) << LOG_DESC("initMsgHandlers for ecdh-multi-psi");
     auto weakEcdhMultiPSI = std::weak_ptr(m_ecdhMultiPSI);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_MULTI,
         [weakEcdhMultiPSI](ppc::front::PPCMessageFace::Ptr _msg) {
             auto psi = weakEcdhMultiPSI.lock();
             if (!psi)
@@ -294,7 +294,7 @@ void Initializer::initMsgHandlers()
     // register msg-handlers for ecdh-psi
     auto weakOtPIR = std::weak_ptr(m_otPIR);
     m_ppcFront->registerMessageHandler((uint8_t)ppc::protocol::TaskType::PIR,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::OT_PIR_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::OT_PIR_2PC,
         [weakOtPIR](ppc::front::PPCMessageFace::Ptr _msg) {
             auto pir = weakOtPIR.lock();
             if (!pir)
@@ -313,7 +313,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     // register task handler for ra2018-psi
     auto weakRA2018PSI = std::weak_ptr(m_ra2018PSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::RA_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::RA_PSI_2PC,
         [weakRA2018PSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto ra2018Impl = weakRA2018PSI.lock();
@@ -328,7 +328,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for labeledPSI");
     auto weakLabeledPSI = std::weak_ptr(m_labeledPSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::LABELED_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::LABELED_PSI_2PC,
         [weakLabeledPSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto labeledPSI = weakLabeledPSI.lock();
@@ -343,7 +343,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for cm2020PSI");
     auto weakCM2020PSI = std::weak_ptr(m_cm2020PSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::CM_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::CM_PSI_2PC,
         [weakCM2020PSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto cm2020PSI = weakCM2020PSI.lock();
@@ -358,7 +358,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhPSI");
     auto weakEcdhPSI = std::weak_ptr(m_ecdhPSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_2PC,
         [weakEcdhPSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto psi = weakEcdhPSI.lock();
@@ -372,7 +372,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhMultiPSI");
     auto weakEcdhMultiPSI = std::weak_ptr(m_ecdhMultiPSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_MULTI,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_MULTI,
         [weakEcdhMultiPSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto psi = weakEcdhMultiPSI.lock();
@@ -386,7 +386,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     /*INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for ecdhConnPSI");
     auto weakEcdhConnPSI = std::weak_ptr(m_ecdhConnPSI);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PSI,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::ECDH_PSI_CONN,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::ECDH_PSI_CONN,
         [weakEcdhConnPSI](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto psi = weakEcdhConnPSI.lock();
@@ -401,7 +401,7 @@ void Initializer::registerRpcHandler(ppc::rpc::RpcInterface::Ptr const& _rpc)
     INIT_LOG(INFO) << LOG_DESC("registerRpcHandler for otPIR");
     auto weakOtPIR = std::weak_ptr(m_otPIR);
     _rpc->registerTaskHandler(ppc::protocol::TaskType::PIR,
-        (uint8_t)ppc::protocol::PSIAlgorithmType::OT_PIR_2PC,
+        (uint8_t)ppc::protocol::TaskAlgorithmType::OT_PIR_2PC,
         [weakOtPIR](ppc::protocol::Task::ConstPtr _task,
             std::function _handler) {
             auto pir = weakOtPIR.lock();
diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
index cd3b9b39..d3b33818 100644
--- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
@@ -28,11 +28,11 @@ using namespace grpc;
 
 void FrontClient::onReceiveMessage(ppc::protocol::Message::Ptr const& msg, ReceiveMsgFunc callback)
 {
-    // TODO: optimize here
     std::unique_ptr request(new ReceivedMessage());
     bcos::bytes encodedData;
     msg->encode(encodedData);
-    request->set_data(encodedData.data(), encodedData.size());
+    *request->mutable_data() =
+        std::move(std::string_view((const char*)encodedData.data(), encodedData.size()));
 
     // The ClientContext instance used for creating an rpc must remain alive and valid for the
     // lifetime of the rpc
diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
index 4222d8af..a6b8b63b 100644
--- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
@@ -29,7 +29,8 @@ ServerUnaryReactor* GatewayServer::asyncSendMessage(CallbackServerContext* conte
     ServerUnaryReactor* reactor(context->DefaultReactor());
     try
     {
-        // TODO: optimize here
+        // TODO: optimize here (since bytes of protobuf is represented with string, no zero-copy
+        // method has been found yet, unless the payload is stored in string)
         bcos::bytes payloadData(sendedMsg->payload().begin(), sendedMsg->payload().end());
         auto routeInfo = generateRouteInfo(m_routeInfoBuilder, sendedMsg->routeinfo());
         m_gateway->asyncSendMessage((ppc::protocol::RouteType)sendedMsg->routetype(), routeInfo,
diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
index de95ab48..ee289ee1 100644
--- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
@@ -19,7 +19,7 @@
  */
 #include "GrpcServer.h"
 #include "Common.h"
-#include 
+#include "grpcpp/ext/proto_server_reflection_plugin.h"
 
 using namespace ppc::protocol;
 using namespace grpc;
diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
index 753b2aab..3bc8aebc 100644
--- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
+++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
@@ -24,6 +24,7 @@
 #include "ppc-framework/protocol/Protocol.h"
 #include 
 #include 
+#include 
 #include 
 
 namespace ppc::protocol
@@ -64,7 +65,9 @@ inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& trac
     // set the route information
     setRouteInfo(request->mutable_routeinfo(), routeInfo);
     // set the payload(TODO: optimize here)
-    request->set_payload(payload.data(), payload.size());
+    *request->mutable_payload() =
+        std::move(std::string_view((const char*)payload.data(), payload.size()));
+
     request->set_timeout(timeout);
     return request;
 }
diff --git a/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp b/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp
index a234a8e1..6d0de595 100644
--- a/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp
+++ b/cpp/wedpr-protocol/protobuf/tests/NodeInfoImplTest.cpp
@@ -17,12 +17,14 @@
  * @author: yujiechen
  * @date 2024-09-06
  */
-
 #include "protobuf/src/NodeInfoImpl.h"
+#include "protobuf/src/RequestConverter.h"
+#include 
 #include 
 #include 
 
 using namespace ppc;
+using namespace ppc::proto;
 using namespace ppc::protocol;
 using namespace bcos::test;
 
@@ -54,6 +56,31 @@ void testNodeInfoEncodeDecode(INodeInfoFactory::Ptr factory, INodeInfo::Ptr node
     }
 }
 
+void testPayloadMove(int payloadSize, int round)
+{
+    bcos::bytes payload;
+    char tmp = 'a';
+    for (int i = 0; i < payloadSize; i++)
+    {
+        payload.emplace_back((char)(tmp + 1));
+    }
+    auto startT = bcos::utcSteadyTime();
+    auto request = new ppc::proto::SendedMessageRequest();
+    for (int i = 0; i < round; i++)
+    {
+        std::unique_ptr request(
+            new ppc::proto::SendedMessageRequest());
+        //*request->mutable_payload() = std::move(std::string_view((const char*)payload.data(),
+        //payload.size()));
+        *request->mutable_payload() = std::string_view((const char*)payload.data(), payload.size());
+        // request->set_payload((const char*)payload.data(), payload.size());
+        // std::cout << "#### request size: " << request->payload().size() << ", origin payload
+        // size: " << payloadSize << std::endl;
+        BOOST_CHECK(*request->mutable_payload() == std::string(payload.begin(), payload.end()));
+    }
+    std::cout << "### testPayloadMove, timecost: " << (bcos::utcSteadyTime() - startT) << std::endl;
+}
+
 BOOST_AUTO_TEST_CASE(testNodeInfo)
 {
     auto nodeInfoFactory = std::make_shared();
@@ -68,4 +95,11 @@ BOOST_AUTO_TEST_CASE(testNodeInfo)
     testNodeInfoEncodeDecode(nodeInfoFactory, nodeInfo);
 }
 
+BOOST_AUTO_TEST_CASE(payloadMoveTest)
+{
+    testPayloadMove(10, 1);
+    testPayloadMove(100000, 10);
+    testPayloadMove(10000000, 10);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp b/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp
index 1a8fea15..f478eae2 100644
--- a/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp
+++ b/cpp/wedpr-protocol/protocol/tests/PPCMessageTest.cpp
@@ -113,7 +113,7 @@ BOOST_AUTO_TEST_CASE(test_ppcMesage)
 {
     int version = 1;
     int taskType = uint8_t(protocol::TaskType::PSI);
-    int algorithmType = uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC);
+    int algorithmType = uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC);
     int messageType = 4;
     std::string taskID = "12345678";
     int seq = 5;
diff --git a/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp b/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp
index e5f1ad9e..e5ae7866 100644
--- a/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp
+++ b/cpp/wedpr-transport/ppc-front/test/unittests/PPCChannelTest.cpp
@@ -47,14 +47,14 @@ BOOST_AUTO_TEST_CASE(test_ppcChannel)
 
     // register message handler
     channelManager->registerMsgHandlerForChannel(
-        uint8_t(protocol::TaskType::PSI), uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC));
+        uint8_t(protocol::TaskType::PSI), uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC));
 
     // receive message
     auto messageFactory = std::make_shared();
     auto message = messageFactory->buildPPCMessage();
     message->setVersion(1);
     message->setTaskType(uint8_t(protocol::TaskType::PSI));
-    message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC));
+    message->setAlgorithmType(uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC));
     message->setMessageType(4);
     message->setSeq(5);
     message->setTaskID("T_123456");
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
index db62f116..1c330acc 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/GatewayFactory.cpp
@@ -46,7 +46,7 @@ Service::Ptr GatewayFactory::buildService() const
     wsConfig->setSmSSL(gwConfig.networkConfig.enableSM);
     wsConfig->setMaxMsgSize(gwConfig.maxAllowedMsgSize);
     wsConfig->setReconnectPeriod(gwConfig.reconnectTime);
-    // TODO: setHeartbeatPeriod, setSendMsgTimeout
+    // default HeartbeatPeriod is 10s
     wsConfig->setThreadPoolSize(gwConfig.networkConfig.threadPoolSize);
     // connected peers
     wsConfig->setConnectPeers(m_gatewayConfig->nodeIPEndpointSetPtr());
@@ -73,6 +73,7 @@ Service::Ptr GatewayFactory::buildService() const
 IGateway::Ptr GatewayFactory::build(ppc::front::IFrontBuilder::Ptr const& frontBuilder) const
 {
     auto service = buildService();
-    return std::make_shared(
-        service, frontBuilder, std::make_shared(), m_config->agencyID());
+    return std::make_shared(service, frontBuilder,
+        std::make_shared(), m_config->agencyID(),
+        m_config->seqSyncPeriod());
 }
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
index effe4ece..bc6822b9 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.cpp
@@ -32,7 +32,8 @@ using namespace bcos::boostssl::ws;
 
 GatewayImpl::GatewayImpl(Service::Ptr const& service,
     ppc::front::IFrontBuilder::Ptr const& frontBuilder,
-    std::shared_ptr ioService, std::string const& agency)
+    std::shared_ptr ioService, std::string const& agency,
+    uint16_t seqSyncPeriod)
   : m_service(service),
     m_msgBuilder(
         std::dynamic_pointer_cast(service->messageFactory())),
@@ -52,7 +53,7 @@ GatewayImpl::GatewayImpl(Service::Ptr const& service,
         boost::bind(&GatewayImpl::onReceiveBroadcastMessage, this, boost::placeholders::_1,
             boost::placeholders::_2));
     m_gatewayRouterManager = std::make_shared(
-        m_service, m_gatewayInfoFactory, m_localRouter, m_peerRouter);
+        m_service, m_gatewayInfoFactory, m_localRouter, m_peerRouter, seqSyncPeriod);
 
     m_service->registerOnNewSession([this](WsSession::Ptr _session) {
         if (!_session)
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
index e965bab3..061dd567 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
@@ -33,7 +33,8 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_this;
     GatewayImpl(Service::Ptr const& service, ppc::front::IFrontBuilder::Ptr const& frontBuilder,
-        std::shared_ptr ioService, std::string const& agency);
+        std::shared_ptr ioService, std::string const& agency,
+        uint16_t seqSyncPeriod = 5000);
     ~GatewayImpl() override = default;
 
     void start() override;
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
index 19553502..ad482cb5 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
@@ -31,11 +31,12 @@ using namespace bcos::boostssl::ws;
 
 GatewayRouterManager::GatewayRouterManager(Service::Ptr service,
     GatewayNodeInfoFactory::Ptr nodeStatusFactory, LocalRouter::Ptr localRouter,
-    PeerRouterTable::Ptr peerRouter)
+    PeerRouterTable::Ptr peerRouter, uint16_t seqSyncPeriod)
   : m_service(std::move(service)),
     m_nodeStatusFactory(std::move(nodeStatusFactory)),
     m_localRouter(std::move(localRouter)),
-    m_peerRouter(std::move(peerRouter))
+    m_peerRouter(std::move(peerRouter)),
+    m_seqSyncPeriod(seqSyncPeriod)
 {
     m_service->registerMsgHandler((uint16_t)GatewayPacketType::SyncNodeSeq,
         boost::bind(&GatewayRouterManager::onReceiveNodeSeqMessage, this, boost::placeholders::_1,
@@ -49,7 +50,7 @@ GatewayRouterManager::GatewayRouterManager(Service::Ptr service,
         boost::bind(&GatewayRouterManager::onRecvResponseNodeStatusMsg, this,
             boost::placeholders::_1, boost::placeholders::_2));
 
-    m_timer = std::make_shared(SEQ_SYNC_PERIOD, "seqSync");
+    m_timer = std::make_shared(m_seqSyncPeriod, "seqSync");
     // broadcast seq periodically
     m_timer->registerTimeoutHandler([this]() { broadcastStatusSeq(); });
 }
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h
index 679eb08e..4224c290 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.h
@@ -32,7 +32,8 @@ class GatewayRouterManager
 public:
     using Ptr = std::shared_ptr;
     GatewayRouterManager(Service::Ptr service, GatewayNodeInfoFactory::Ptr nodeStatusFactory,
-        LocalRouter::Ptr localRouter, PeerRouterTable::Ptr peerRouter);
+        LocalRouter::Ptr localRouter, PeerRouterTable::Ptr peerRouter,
+        uint16_t seqSyncPeriod = 5000);
     virtual void start();
     virtual void stop();
 
@@ -66,7 +67,6 @@ class GatewayRouterManager
     std::map m_p2pID2Seq;
     mutable bcos::SharedMutex x_p2pID2Seq;
 
-    // TODO: make this configurable
-    unsigned const SEQ_SYNC_PERIOD = 3000;
+    uint16_t m_seqSyncPeriod;
 };
 }  // namespace ppc::gateway
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
index b4518cb2..3c5f3b94 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.cpp
@@ -172,14 +172,29 @@ void RouterManager::onP2PNodesUnreachable(std::set const& _p2pNodeI
             ReadGuard readGuard(x_unreachableHandlers);
             handlers = m_unreachableHandlers;
         }
-        // TODO: async here
-        for (auto const& node : _p2pNodeIDs)
-        {
-            for (auto const& it : m_unreachableHandlers)
+        auto self = weak_from_this();
+        m_service->threadPool()->enqueue([self, _p2pNodeIDs]() {
+            try
             {
-                it(node);
+                auto mgr = self.lock();
+                if (!mgr)
+                {
+                    return;
+                }
+                for (auto const& node : _p2pNodeIDs)
+                {
+                    for (auto const& it : mgr->m_unreachableHandlers)
+                    {
+                        it(node);
+                    }
+                }
             }
-        }
+            catch (std::exception const& e)
+            {
+                SERVICE_ROUTER_LOG(WARNING) << LOG_DESC("call unreachable handlers error for ")
+                                            << boost::diagnostic_information(e);
+            }
+        });
     }
     catch (std::exception const& e)
     {
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h
index ab50a887..d55841db 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/router/RouterManager.h
@@ -21,11 +21,12 @@
 #pragma once
 #include "../Service.h"
 #include "RouterTableInterface.h"
+#include 
 #include 
 
 namespace ppc::gateway
 {
-class RouterManager
+class RouterManager : public std::enable_shared_from_this
 {
 public:
     using Ptr = std::shared_ptr;
@@ -63,6 +64,7 @@ class RouterManager
 private:
     // for message forward
     Service::Ptr m_service;
+
     std::shared_ptr m_routerTimer;
 
     // called when the given node unreachable
diff --git a/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp b/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp
index bb8aca0e..3d702dc7 100644
--- a/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/test/demo/gateway_demo.cpp
@@ -41,7 +41,7 @@ inline static PPCMessageFace::Ptr buildMessage(std::string _taskID, uint32_t _se
     auto message = messageFactory->buildPPCMessage();
     message->setVersion(1);
     message->setTaskType(uint8_t(protocol::TaskType::PSI));
-    message->setAlgorithmType(uint8_t(protocol::PSIAlgorithmType::CM_PSI_2PC));
+    message->setAlgorithmType(uint8_t(protocol::TaskAlgorithmType::CM_PSI_2PC));
     message->setMessageType(4);
     message->setSeq(_seq);
     message->setTaskID(_taskID);
@@ -84,7 +84,7 @@ int main(int argc, char* argv[])
     front->start();
 
     std::atomic flag = 0;
-    front->registerMessageHandler(uint8_t(TaskType::PSI), uint8_t(PSIAlgorithmType::CM_PSI_2PC),
+    front->registerMessageHandler(uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC),
         [&flag](const PPCMessageFace::Ptr& _message) {
             std::cout << "message received\n"
                       << "type: " << unsigned(_message->messageType()) << std::endl;
diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
index f2a78a09..5b039131 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
@@ -191,7 +191,7 @@ void Rpc::runTask(Json::Value const& _req, RespFunc _respFunc)
         oss << "The specified task algorithm not been implemented or disabled, type: "
             << (ppc::protocol::TaskType)(task->type()) << "(" << std::to_string(task->type())
             << ")";
-        oss << ", algorithm: " << (ppc::protocol::PSIAlgorithmType)(task->algorithm()) << "("
+        oss << ", algorithm: " << (ppc::protocol::TaskAlgorithmType)(task->algorithm()) << "("
             << std::to_string(task->algorithm()) << ")";
         auto errorMsg = oss.str();
         RPC_LOG(WARNING) << errorMsg;
@@ -249,7 +249,7 @@ void Rpc::asyncRunTask(Json::Value const& _req, RespFunc _respFunc)
         oss << "The specified task algorithm not been implemented or disabled, type: "
             << (ppc::protocol::TaskType)(task->type()) << "(" << std::to_string(task->type())
             << ")";
-        oss << ", algorithm: " << (ppc::protocol::PSIAlgorithmType)(task->algorithm()) << "("
+        oss << ", algorithm: " << (ppc::protocol::TaskAlgorithmType)(task->algorithm()) << "("
             << std::to_string(task->algorithm()) << ")";
         auto errorMsg = oss.str();
         RPC_LOG(WARNING) << errorMsg;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index 7f3ceb28..e44fccec 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -69,7 +69,6 @@ public void stop() {
     }
 
     /**
-     * TODO: register the component
      *
      * @param component the component used to router
      * @throws Exception failed case
@@ -80,7 +79,6 @@ public void registerComponent(String component) {
     }
 
     /**
-     * TODO: unregister the component
      *
      * @param component the component used to route
      * @throws Exception failed case

From 3718216f5749ebc1e486bed50bad5396846dcbc5 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 29 Oct 2024 20:43:04 +0800
Subject: [PATCH 072/120] fix cpp toolkit windows compile (#70)

---
 .github/workflows/cpp_full_node_workflow.yml  | 17 ++++++++---
 .github/workflows/cpp_toolkit_workflow.yml    |  4 +--
 cpp/CMakeLists.txt                            | 14 ++++++----
 cpp/cmake/CompilerSettings.cmake              | 25 +++++++++++++----
 cpp/cmake/grpc.cmake                          | 28 +------------------
 cpp/vcpkg.json                                |  1 +
 cpp/wedpr-protocol/grpc/CMakeLists.txt        |  2 +-
 cpp/wedpr-protocol/proto/pb/Service.proto     |  1 +
 .../protobuf/src/NodeInfoImpl.h               |  6 ++--
 .../protobuf/src/RequestConverter.h           | 14 ++++++----
 .../ppc-front/ppc-front/CallbackManager.cpp   |  6 ++--
 .../ppc-front/ppc-front/Common.h              |  1 -
 .../ppc-front/ppc-front/FrontImpl.cpp         |  9 +++---
 13 files changed, 68 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index fcea6730..3a8b3631 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -135,8 +135,7 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y wget lcov java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
-          yum install -y git
+          yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
@@ -154,7 +153,7 @@ jobs:
           df /tmp
           du /tmp | sort -n
           ls -lh /tmp
-      - name: Build
+      - name: DependenciesBuild
         run: |
           bash -x cpp/tools/install_depends.sh -o centos
           alias cmake='cmake3'
@@ -163,7 +162,17 @@ jobs:
           mkdir -p cpp/build
           cd cpp/build
           cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-          #cmake3 --build . --parallel 3
+      - name: FreeDiskSpace
+        run: |
+          df -lh
+          du -sch /usr/local/share/vcpkg/buildtrees/wedprcrypto/src/*
+          du -sch /usr/local/share/vcpkg/buildtrees/grpc/src/*
+          rm -rf /usr/local/share/vcpkg/buildtrees/wedprcrypto/src
+          rm -rf /usr/local/share/vcpkg/buildtrees/grpc/src
+          rm -rf /usr/local/share/vcpkg/buildtrees/grpc/*-dbg
+      - name: Compile
+        run: |          
+          cd cpp/build && cmake3 --build . --parallel 3
       #- name: Test
       #  run: |
       #    export OMP_NUM_THREADS=1
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index a1922156..cb189e5c 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -64,7 +64,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |
@@ -90,7 +90,7 @@ jobs:
         if: runner.os == 'Windows'
         with:
           name: libwedpr_java_transport_jni.dll
-          path: D:\a\WeDPR-Component\cpp\wedpr-transport\sdk-wrapper\java\bindings\src\main\resources\META-INF\native\Release\libwedpr_java_transport_jni.dll
+          path: D:\a\WeDPR-Component\WeDPR-Component\cpp\wedpr-transport\sdk-wrapper\java\bindings\src\main\resources\META-INF\native\Release\wedpr_java_transport_jni.dll
 
 
   build_centos_for_toolkit:
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d6f941b4..a56790ce 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -29,13 +29,15 @@ if (NOT DEFINED URL_BASE)
     #set(URL_BASE "github.com.cnpmjs.org")
 endif ()
 
-# export windows dll symbol
-if(WIN32)
-    message(STATUS "Compile on Windows")
+if (WIN32)
+    set(VCPKG_C_FLAGS -D_WIN32_WINNT=0x0600 -DWINVER=0x0600)
+    set(VCPKG_CXX_FLAGS -D_WIN32_WINNT=0x0600 -DWINVER=0x0600)
+    add_definitions(-D_WIN32_WINNT=0x0600)
+    add_definitions(-DBOOST_USE_WINAPI_VERSION=BOOST_WINAPI_VERSION_WIN7)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON")
-endif()
-
-
+else ()
+    add_definitions(-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX)
+endif ()
 
 # basic settings
 include(Options)
diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 59c96030..9ae74f51 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -1,5 +1,11 @@
 set(CMAKE_CXX_STANDARD 20)
 set(Boost_NO_WARN_NEW_VERSIONS ON)
+# export windows dll symbol
+if(WIN32)
+    message(STATUS "Compile on Windows")
+    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS "ON")
+endif()
+
 message(STATUS "COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}")
 if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang"))
     find_program(CCACHE_PROGRAM ccache)
@@ -142,11 +148,20 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "MSVC")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17")
        add_compile_definitions(NOMINMAX)
        add_compile_options(-bigobj)
-        # MSVC only support static build
-        set(CMAKE_CXX_FLAGS_DEBUG "/MTd /DEBUG")
-        set(CMAKE_CXX_FLAGS_MINSIZEREL "/MT /Os")
-        set(CMAKE_CXX_FLAGS_RELEASE "/MT")
-        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MT /DEBUG")
+       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
+       if(BUILD_SHARED_LIBS)
+           if(CMAKE_BUILD_TYPE MATCHES "Debug")
+               add_compile_options(/MDd)
+           else()
+               add_compile_options(/MD)
+           endif()
+       else ()
+           if(CMAKE_BUILD_TYPE MATCHES "Debug")
+               add_compile_options(/MTd)
+           else()
+               add_compile_options(/MT)
+           endif ()
+       endif ()       
         link_libraries(ws2_32 Crypt32 userenv)
 else ()
     message(WARNING "Your compiler is not tested, if you run into any issues, we'd welcome any patches.")
diff --git a/cpp/cmake/grpc.cmake b/cpp/cmake/grpc.cmake
index f95bbe1d..f02b80b5 100644
--- a/cpp/cmake/grpc.cmake
+++ b/cpp/cmake/grpc.cmake
@@ -19,32 +19,6 @@ if(NOT GRPC_CPP_PLUGIN AND TARGET gRPC::grpc_cpp_plugin)
   endif()
 endif()
 
-
-if(NOT GRPC_CPP_REFLECTION AND TARGET gRPC::grpc++_reflection)
-  get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
-    IMPORTED_LOCATION_RELEASE)
-  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
-    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
-      IMPORTED_LOCATION_RELWITHDEBINFO)
-  endif()
-  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
-    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
-      IMPORTED_LOCATION_MINSIZEREL)
-  endif()
-  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
-    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
-      IMPORTED_LOCATION_DEBUG)
-  endif()
-  if(NOT EXISTS "${GRPC_CPP_REFLECTION}")
-    get_target_property(GRPC_CPP_REFLECTION gRPC::grpc++_reflection
-      IMPORTED_LOCATION_NOCONFIG)
-  endif()
-endif()
-
-get_target_property(GRPC_CPP_REFLECTION_INCLUDE gRPC::grpc++_reflection INTERFACE_INCLUDE_DIRECTORIES)
-include_directories(${GRPC_CPP_REFLECTION_INCLUDE})
-
 set(PROTOC_BINARY ${Protobuf_PROTOC_EXECUTABLE})
 message("# PROTOC_BINARY: ${Protobuf_PROTOC_EXECUTABLE}")
-message("# GRPC_CPP_PLUGIN: ${GRPC_CPP_PLUGIN}")
-message("# GRPC_CPP_REFLECTION_INCLUDE: ${GRPC_CPP_REFLECTION_INCLUDE}")
\ No newline at end of file
+message("# GRPC_CPP_PLUGIN: ${GRPC_CPP_PLUGIN}")
\ No newline at end of file
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index d605e402..69a47350 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -109,6 +109,7 @@
         },
         {
           "name": "grpc",
+          "features": ["codegen"],
           "version>=": "1.51.1"
         },
         {
diff --git a/cpp/wedpr-protocol/grpc/CMakeLists.txt b/cpp/wedpr-protocol/grpc/CMakeLists.txt
index fbd1fbc6..b72e308b 100644
--- a/cpp/wedpr-protocol/grpc/CMakeLists.txt
+++ b/cpp/wedpr-protocol/grpc/CMakeLists.txt
@@ -23,7 +23,7 @@ foreach(proto_file ${MESSAGES_PROTOS})
 endforeach()
 
 add_library(${SERVICE_PB_TARGET} ${GRPC_MESSAGES_SRCS})
-target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure ${GRPC_CPP_REFLECTION})
+target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::grpc++_unsecure gRPC::grpc++_reflection)
 
 add_subdirectory(client)
 add_subdirectory(server)
diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto
index 9b7834cb..47d2e069 100644
--- a/cpp/wedpr-protocol/proto/pb/Service.proto
+++ b/cpp/wedpr-protocol/proto/pb/Service.proto
@@ -46,6 +46,7 @@ message PeersInfo{
     string peersInfo = 2;
 };
 message Empty{
+    int32 status = 1;
 };
 
 message Condition{
diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
index 04f0496c..57da2dd2 100644
--- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
+++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
@@ -39,7 +39,8 @@ class NodeInfoImpl : public INodeInfo
 
     NodeInfoImpl(bcos::bytesConstRef const& nodeID, std::string const& endPoint) : NodeInfoImpl()
     {
-        m_rawNodeInfo->set_nodeid(nodeID.data(), nodeID.size());
+        *(m_rawNodeInfo->mutable_nodeid()) =
+            std::string_view((const char*)nodeID.data(), nodeID.size());
         m_rawNodeInfo->set_endpoint(endPoint);
     }
 
@@ -47,7 +48,8 @@ class NodeInfoImpl : public INodeInfo
 
     void setNodeID(bcos::bytesConstRef nodeID) override
     {
-        m_rawNodeInfo->set_nodeid(nodeID.data(), nodeID.size());
+        *(m_rawNodeInfo->mutable_nodeid()) =
+            std::string_view((const char*)nodeID.data(), nodeID.size());
     }
     void setEndPoint(std::string const& endPoint) override
     {
diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
index 3bc8aebc..065398c2 100644
--- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
+++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
@@ -50,9 +50,12 @@ inline void setRouteInfo(
     // set the route information
     route_info->set_topic(routeInfo->topic());
     route_info->set_componenttype(routeInfo->componentType());
-    route_info->set_srcnode(routeInfo->srcNode().data(), routeInfo->srcNode().size());
-    route_info->set_dstnode(routeInfo->dstNode().data(), routeInfo->dstNode().size());
-    route_info->set_dstinst(routeInfo->dstInst().data(), routeInfo->dstInst().size());
+    *(route_info->mutable_srcnode()) =
+        std::string_view((const char*)routeInfo->srcNode().data(), routeInfo->srcNode().size());
+    *(route_info->mutable_dstnode()) =
+        std::string_view((const char*)routeInfo->dstNode().data(), routeInfo->dstNode().size());
+    *(route_info->mutable_dstinst()) =
+        std::string_view((const char*)routeInfo->dstInst().data(), routeInfo->dstInst().size());
 }
 
 inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& traceID,
@@ -86,7 +89,7 @@ inline ppc::proto::NodeInfo* toNodeInfoRequest(
     bcos::bytesConstRef const& nodeID, std::string const& topic)
 {
     auto request = new ppc::proto::NodeInfo();
-    request->set_nodeid(nodeID.data(), nodeID.size());
+    *(request->mutable_nodeid()) = std::string_view((const char*)nodeID.data(), nodeID.size());
     request->set_topic(topic);
     return request;
 }
@@ -98,7 +101,8 @@ inline ppc::proto::NodeInfo* toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo)
     {
         return request;
     };
-    request->set_nodeid(nodeInfo->nodeID().data(), nodeInfo->nodeID().size());
+    *(request->mutable_nodeid()) =
+        std::string_view((const char*)nodeInfo->nodeID().data(), nodeInfo->nodeID().size());
     request->set_endpoint(nodeInfo->endPoint());
     auto const& components = nodeInfo->components();
     for (auto const& component : components)
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
index e7a52fd7..e32a1be6 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/CallbackManager.cpp
@@ -92,10 +92,10 @@ void CallbackManager::onMessageTimeout(
             [callback, error]() { callback->msgCallback(error, nullptr, nullptr); });
         FRONT_LOG(WARNING) << LOG_BADGE("onMessageTimeout") << LOG_KV("traceID", traceID);
     }
-    catch (std::exception& e)
+    catch (std::exception const& e)
     {
-        FRONT_LOG(ERROR) << "onMessageTimeout" << LOG_KV("traceID", traceID)
-                         << LOG_KV("error", boost::diagnostic_information(e));
+        FRONT_LOG(WARNING) << LOG_BADGE("onMessageTimeout") << LOG_KV("traceID", traceID)
+                           << LOG_KV("error", boost::diagnostic_information(e));
     }
 }
 
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Common.h b/cpp/wedpr-transport/ppc-front/ppc-front/Common.h
index a221d179..74b24cd2 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/Common.h
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/Common.h
@@ -19,7 +19,6 @@
  */
 
 #pragma once
-
 #include "ppc-framework/Common.h"
 #include 
 #include 
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index 8c7aa8bb..664ee1c6 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -163,10 +163,11 @@ void FrontImpl::asyncSendMessage(uint16_t routeType, MessageOptionalHeader::Ptr
             if (error && error->errorCode() != 0)
             {
                 // send failed
-                FRONT_LOG(ERROR) << LOG_DESC("asyncSendMessage failed")
-                                 << LOG_KV("routeInfo", printOptionalField(routeInfo))
-                                 << LOG_KV("traceID", traceID) << LOG_KV("code", error->errorCode())
-                                 << LOG_KV("msg", error->errorMessage());
+                FRONT_LOG(WARNING)
+                    << LOG_DESC("asyncSendMessage failed")
+                    << LOG_KV("routeInfo", printOptionalField(routeInfo))
+                    << LOG_KV("traceID", traceID) << LOG_KV("code", error->errorCode())
+                    << LOG_KV("msg", error->errorMessage());
                 // try to trigger the callback
                 front->handleCallback(error, traceID, nullptr);
             }

From 77c9f3f9b42930888e4005ab5ecb659d3cba8a7d Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 30 Oct 2024 20:06:00 +0800
Subject: [PATCH 073/120] fix dstNode empty bug when asyncSendResponse (#71)

* fix full node centos ci

* upload python toolkit related artifacts

* fix dstNode empty bug when asyncSendResponse

* add initLog for transport sdk

* add log init
---
 .github/workflows/cpp_full_node_workflow.yml  | 19 +++++++-----
 .github/workflows/cpp_toolkit_workflow.yml    | 16 +++++++++-
 cpp/ppc-framework/protocol/Message.h          |  8 +----
 .../grpc/client/FrontClient.cpp               |  1 -
 .../grpc/server/FrontServer.cpp               |  1 -
 .../protobuf/src/RequestConverter.h           | 11 +++----
 .../protocol/src/v1/MessageHeaderImpl.cpp     |  6 ++--
 .../ppc-front/ppc-front/FrontImpl.cpp         |  5 +--
 .../gateway/router/LocalRouter.cpp            |  2 ++
 .../sdk/jni/generated/TransportBuilder.java   |  4 +++
 .../generated/wedpr_java_transportJNI.java    |  2 ++
 .../sdk/jni/transport/TransportConfig.java    |  2 ++
 .../sdk/jni/transport/impl/TransportImpl.java |  2 --
 .../src/wedpr_java_transportJAVA_wrap.cxx     | 18 +++++++++++
 .../java/swig/wedpr_java_transport.i          |  1 -
 .../generated/wedpr_python_transport.py       |  4 +++
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 31 +++++++++++++++++++
 .../python/swig/wedpr_python_transport.i      |  1 -
 .../sdk/src/TransportBuilder.cpp              |  3 ++
 .../sdk/src/TransportBuilder.h                | 25 +++++++++++++++
 20 files changed, 127 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 3a8b3631..339a6922 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -65,7 +65,8 @@ jobs:
           export GCC='gcc-10'
           export CXX='g++-10'
           sudo bash -x cpp/tools/install_depends.sh -o ubuntu
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../ || cat /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/libhdfs3/config-x64-osx-dbg-err.log
+          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          make -j3
       - name: Build for macOS
         if: runner.os == 'macOS'
         run: |
@@ -87,10 +88,10 @@ jobs:
         with:
           name: ppc-gateway-service-macOS
           path: ./cpp/build/bin/ppc-gateway-service
-      #- name: Test
-      #  if: runner.os != 'Windows'
-      #  run: |
-      #    cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE ctest
+      - name: Test
+        if: runner.os != 'Windows'
+        run: |
+          cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test ARGS="--output-on-failure"
 
   build_centos:
     name: build_centos full node
@@ -165,13 +166,15 @@ jobs:
       - name: FreeDiskSpace
         run: |
           df -lh
-          du -sch /usr/local/share/vcpkg/buildtrees/wedprcrypto/src/*
+          du -sch /usr/local/share/vcpkg/buildtrees/*
           du -sch /usr/local/share/vcpkg/buildtrees/grpc/src/*
-          rm -rf /usr/local/share/vcpkg/buildtrees/wedprcrypto/src
           rm -rf /usr/local/share/vcpkg/buildtrees/grpc/src
           rm -rf /usr/local/share/vcpkg/buildtrees/grpc/*-dbg
+          df -lh
       - name: Compile
-        run: |          
+        run: |   
+          alias cmake='cmake3'
+          . /opt/rh/devtoolset-11/enable       
           cd cpp/build && cmake3 --build . --parallel 3
       #- name: Test
       #  run: |
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index cb189e5c..4e5bf001 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -91,6 +91,16 @@ jobs:
         with:
           name: libwedpr_java_transport_jni.dll
           path: D:\a\WeDPR-Component\WeDPR-Component\cpp\wedpr-transport\sdk-wrapper\java\bindings\src\main\resources\META-INF\native\Release\wedpr_java_transport_jni.dll
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'macos'
+        with:
+          name: libwedpr_python_transport.dylib
+          path: ./cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/libwedpr_python_transport.dylib
+      - uses: actions/upload-artifact@v4
+        if: runner.os == 'Windows'
+        with:
+          name: libwedpr_python_transport.dll
+          path: D:\a\WeDPR-Component\WeDPR-Component\cpp\wedpr-transport\sdk-wrapper\python\bindings\wedpr_python_gateway_sdk\libs\Release\wedpr_python_transport.dll
 
 
   build_centos_for_toolkit:
@@ -150,4 +160,8 @@ jobs:
         - uses: actions/upload-artifact@v3
           with:
             name: libwedpr_java_transport_jni.so
-            path: ./cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native/libwedpr_java_transport_jni.so
\ No newline at end of file
+            path: ./cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/resources/META-INF/native/libwedpr_java_transport_jni.so
+        - uses: actions/upload-artifact@v3
+          with:
+            name: libwedpr_python_transport.so
+            path: ./cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/libwedpr_python_transport.so
\ No newline at end of file
diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h
index 4c83ad8a..0317b875 100644
--- a/cpp/ppc-framework/protocol/Message.h
+++ b/cpp/ppc-framework/protocol/Message.h
@@ -71,12 +71,7 @@ class MessageOptionalHeader
         // Note: this will be copied to java through jni
         return OutputBuffer{(unsigned char*)m_dstNode.data(), m_dstNode.size()};
     }
-    virtual void setDstNode(bcos::bytes const& dstNode)
-    {
-        m_dstNode = dstNode;
-        m_dstNodePtr = bcos::bytesConstRef((bcos::byte*)m_dstNode.data(), m_dstNode.size());
-    }
-    virtual void setDstNodePtr(bcos::bytesConstRef const& dstNodePtr) { m_dstNodePtr = dstNodePtr; }
+    virtual void setDstNode(bcos::bytes const& dstNode) { m_dstNode = dstNode; }
     // !!! Note: the first paramater type(char*) should not been changed, for it's used for pass-in
     // java byte[] into c bytes
     // Note: the python not support function override
@@ -107,7 +102,6 @@ class MessageOptionalHeader
     std::string m_srcInst;
     // the target nodeID that should receive the message
     bcos::bytes m_dstNode;
-    bcos::bytesConstRef m_dstNodePtr;  // to decrease the copy overhead
     // the target agency that need receive the message
     std::string m_dstInst;
 };
diff --git a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
index d3b33818..48c5167f 100644
--- a/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/FrontClient.cpp
@@ -33,7 +33,6 @@ void FrontClient::onReceiveMessage(ppc::protocol::Message::Ptr const& msg, Recei
     msg->encode(encodedData);
     *request->mutable_data() =
         std::move(std::string_view((const char*)encodedData.data(), encodedData.size()));
-
     // The ClientContext instance used for creating an rpc must remain alive and valid for the
     // lifetime of the rpc
     auto context = std::make_shared();
diff --git a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp
index 90e7cebe..975ff850 100644
--- a/cpp/wedpr-protocol/grpc/server/FrontServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/FrontServer.cpp
@@ -36,7 +36,6 @@ ServerUnaryReactor* FrontServer::onReceiveMessage(
         auto msg = m_msgBuilder->build(bcos::bytesConstRef(
             (bcos::byte*)receivedMsg->data().data(), receivedMsg->data().size()));
         m_front->onReceiveMessage(msg, [reactor, reply](bcos::Error::Ptr error) {
-            FRONT_SERVER_LOG(TRACE) << LOG_DESC("onReceiveMessage");
             toSerializedError(reply, error);
             reactor->Finish(Status::OK);
         });
diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
index 065398c2..9079bbd5 100644
--- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
+++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
@@ -13,7 +13,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  *
- * @file Common.h
+ * @file RequestConverter.h
  * @author: yujiechen
  * @date 2021-04-12
  */
@@ -51,11 +51,10 @@ inline void setRouteInfo(
     route_info->set_topic(routeInfo->topic());
     route_info->set_componenttype(routeInfo->componentType());
     *(route_info->mutable_srcnode()) =
-        std::string_view((const char*)routeInfo->srcNode().data(), routeInfo->srcNode().size());
+        std::string((const char*)routeInfo->srcNode().data(), routeInfo->srcNode().size());
     *(route_info->mutable_dstnode()) =
-        std::string_view((const char*)routeInfo->dstNode().data(), routeInfo->dstNode().size());
-    *(route_info->mutable_dstinst()) =
-        std::string_view((const char*)routeInfo->dstInst().data(), routeInfo->dstInst().size());
+        std::string((const char*)routeInfo->dstNode().data(), routeInfo->dstNode().size());
+    *(route_info->mutable_dstinst()) = routeInfo->dstInst();
 }
 
 inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& traceID,
@@ -67,10 +66,8 @@ inline ppc::proto::SendedMessageRequest* generateRequest(std::string const& trac
     request->set_routetype(uint16_t(routeType));
     // set the route information
     setRouteInfo(request->mutable_routeinfo(), routeInfo);
-    // set the payload(TODO: optimize here)
     *request->mutable_payload() =
         std::move(std::string_view((const char*)payload.data(), payload.size()));
-
     request->set_timeout(timeout);
     return request;
 }
diff --git a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
index a97e505b..00d0c78d 100644
--- a/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/v1/MessageHeaderImpl.cpp
@@ -42,10 +42,9 @@ void MessageOptionalHeaderImpl::encode(bcos::bytes& buffer) const
     buffer.insert(buffer.end(), (byte*)&srcInstLen, (byte*)&srcInstLen + 2);
     buffer.insert(buffer.end(), m_srcInst.begin(), m_srcInst.end());
     // the target nodeID that should receive the message
-    uint16_t dstNodeLen =
-        boost::asio::detail::socket_ops::host_to_network_short(m_dstNodePtr.size());
+    uint16_t dstNodeLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstNode.size());
     buffer.insert(buffer.end(), (byte*)&dstNodeLen, (byte*)&dstNodeLen + 2);
-    buffer.insert(buffer.end(), m_dstNodePtr.begin(), m_dstNodePtr.end());
+    buffer.insert(buffer.end(), m_dstNode.begin(), m_dstNode.end());
     // the target agency that need receive the message
     uint16_t dstInstLen = boost::asio::detail::socket_ops::host_to_network_short(m_dstInst.size());
     buffer.insert(buffer.end(), (byte*)&dstInstLen, (byte*)&dstInstLen + 2);
@@ -74,7 +73,6 @@ int64_t MessageOptionalHeaderImpl::decode(bcos::bytesConstRef data, uint64_t con
     offset = decodeNetworkBuffer(m_srcInst, data.data(), data.size(), offset, false);
     //  dstNode
     offset = decodeNetworkBuffer(m_dstNode, data.data(), data.size(), offset, false);
-    m_dstNodePtr = bcos::bytesConstRef((bcos::byte*)m_dstNode.data(), m_dstNode.size());
     // dstInst
     offset = decodeNetworkBuffer(m_dstInst, data.data(), data.size(), offset, false);
     // topic
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index 664ee1c6..91dbddc6 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -117,7 +117,7 @@ void FrontImpl::asyncSendResponse(bcos::bytesConstRef dstNode, std::string const
 
     auto routeInfo = m_routerInfoBuilder->build();
     routeInfo->setSrcNode(m_nodeID);
-    routeInfo->setDstNodePtr(dstNode);
+    routeInfo->setDstNode(dstNode.toBytes());
 
     asyncSendMessageToGateway(true, std::move(frontMessage), RouteType::ROUTE_THROUGH_NODEID,
         traceID, routeInfo, -1, errorCallback);
@@ -226,7 +226,8 @@ void FrontImpl::asyncSendMessageToGateway(bool responsePacket, MessagePayload::P
     auto payload = std::make_shared();
     frontMessage->encode(*payload);
     FRONT_LOG(TRACE) << LOG_DESC("asyncSendMessageToGateway") << LOG_KV("routeType", routeType)
-                     << LOG_KV("traceID", traceID) << printOptionalField(routeInfo)
+                     << LOG_KV("response", responsePacket) << LOG_KV("traceID", traceID)
+                     << printOptionalField(routeInfo)
                      << LOG_KV("payloadSize", frontMessage->length());
     m_gatewayClient->asyncSendMessage(
         routeType, routeInfo, traceID, std::move(*payload), timeout, callback);
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
index ef9ce7ba..68af4f24 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
@@ -111,6 +111,8 @@ bool LocalRouter::dispatcherMessage(
     if (msg->header() && msg->header()->optionalField() &&
         msg->header()->optionalField()->topic().empty())
     {
+        LOCAL_ROUTER_LOG(WARNING) << LOG_DESC("dispatcherMessage failed for no target found!")
+                                  << printP2PMessage(msg);
         return false;
     }
     if (!holding)
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
index 5a77b230..c329cf05 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
@@ -70,4 +70,8 @@ public SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t frontConfigB
         return new SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t(
                 wedpr_java_transportJNI.TransportBuilder_frontConfigBuilder(swigCPtr, this), false);
     }
+
+    public static void initLog(String configPath) {
+        wedpr_java_transportJNI.TransportBuilder_initLog(configPath);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index cc5dfc17..bf581fb4 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -770,6 +770,8 @@ public static final native long TransportBuilder_buildConfig(
     public static final native long TransportBuilder_frontConfigBuilder(
             long jarg1, TransportBuilder jarg1_);
 
+    public static final native void TransportBuilder_initLog(String jarg1);
+
     public static final native long new_Transport(long jarg1, FrontConfig jarg1_);
 
     public static final native void delete_Transport(long jarg1);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
index 68f8232a..21c5f026 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
@@ -38,6 +38,8 @@ public class TransportConfig {
     }
 
     private static synchronized void createTransportBuilder() {
+        // TODO: make log path configuration here
+        TransportBuilder.initLog("conf/wedpr_sdk_log_config.ini");
         logger.info("init transportBuilder");
         transportBuilder = new TransportBuilder();
         logger.info("init transportBuilder success");
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index e44fccec..e507f418 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -69,7 +69,6 @@ public void stop() {
     }
 
     /**
-     *
      * @param component the component used to router
      * @throws Exception failed case
      */
@@ -79,7 +78,6 @@ public void registerComponent(String component) {
     }
 
     /**
-     *
      * @param component the component used to route
      * @throws Exception failed case
      */
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 678deb89..708a29b3 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -7032,6 +7032,24 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1initLog(JNIEnv *jenv, jclass jcls, jstring jarg1) {
+  std::string *arg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  if(!jarg1) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg1_pstr = (const char *)jenv->GetStringUTFChars(jarg1, 0); 
+  if (!arg1_pstr) return ;
+  std::string arg1_str(arg1_pstr);
+  arg1 = &arg1_str;
+  jenv->ReleaseStringUTFChars(jarg1, arg1_pstr); 
+  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1Transport(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jlong jresult = 0 ;
   ppc::front::FrontConfig::Ptr arg1 ;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index bae976b4..e661a855 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -227,7 +227,6 @@ WRAP(ppc::sdk::Transport)
 %ignore ppc::protocol::MessageOptionalHeader::srcNode;
 %ignore ppc::protocol::MessageOptionalHeader::dstNode;
 %ignore ppc::protocol::MessageOptionalHeader::setDstNode;
-%ignore ppc::protocol::MessageOptionalHeader::setDstNodePtr;
 %ignore ppc::protocol::MessageOptionalHeader::setSrcNode;
 %ignore ppc::protocol::MessagePayload::data;
 %ignore ppc::protocol::MessagePayload::setData;
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 88c27637..9318c761 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1628,6 +1628,10 @@ def buildConfig(self, threadPoolSize, nodeID):
     def frontConfigBuilder(self):
         return _wedpr_python_transport.TransportBuilder_frontConfigBuilder(self)
 
+    @staticmethod
+    def initLog(configPath):
+        return _wedpr_python_transport.TransportBuilder_initLog(configPath)
+
 
 # Register TransportBuilder in _wedpr_python_transport:
 _wedpr_python_transport.TransportBuilder_swigregister(TransportBuilder)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index 51a14c06..cab7c01b 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -23238,6 +23238,36 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, P
 }
 
 
+SWIGINTERN PyObject *_wrap_TransportBuilder_initLog(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::string *arg1 = 0 ;
+  int res1 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    std::string *ptr = (std::string *)0;
+    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+    }
+    arg1 = ptr;
+  }
+  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res1)) delete arg1;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res1)) delete arg1;
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *TransportBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@@ -24118,6 +24148,7 @@ static PyMethodDef SwigMethods[] = {
 	 { "TransportBuilder_buildProTransport", _wrap_TransportBuilder_buildProTransport, METH_VARARGS, NULL},
 	 { "TransportBuilder_buildConfig", _wrap_TransportBuilder_buildConfig, METH_VARARGS, NULL},
 	 { "TransportBuilder_frontConfigBuilder", _wrap_TransportBuilder_frontConfigBuilder, METH_O, NULL},
+	 { "TransportBuilder_initLog", _wrap_TransportBuilder_initLog, METH_O, NULL},
 	 { "TransportBuilder_swigregister", TransportBuilder_swigregister, METH_O, NULL},
 	 { "TransportBuilder_swiginit", TransportBuilder_swiginit, METH_VARARGS, NULL},
 	 { "new_Transport", _wrap_new_Transport, METH_O, NULL},
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index f238d32d..e6e082c9 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -166,7 +166,6 @@ namespace bcos{
 %ignore ppc::protocol::MessageOptionalHeader::srcNode;
 %ignore ppc::protocol::MessageOptionalHeader::dstNode;
 %ignore ppc::protocol::MessageOptionalHeader::setDstNode;
-%ignore ppc::protocol::MessageOptionalHeader::setDstNodePtr;
 %ignore ppc::protocol::MessageOptionalHeader::setSrcNode;
 %ignore ppc::protocol::MessagePayload::data;
 %ignore ppc::protocol::MessagePayload::setData;
diff --git a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
index f0cc6a5b..c6ebcf32 100644
--- a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
+++ b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
@@ -29,6 +29,9 @@
 using namespace ppc::sdk;
 using namespace ppc::front;
 
+bcos::BoostLogInitializer::Ptr TransportBuilder::g_logInitializer;
+std::once_flag TransportBuilder::g_flag;
+
 TransportBuilder::TransportBuilder()
 {
     m_frontConfigBuilder = std::make_shared(
diff --git a/cpp/wedpr-transport/sdk/src/TransportBuilder.h b/cpp/wedpr-transport/sdk/src/TransportBuilder.h
index d8ffa95b..e579abc8 100644
--- a/cpp/wedpr-transport/sdk/src/TransportBuilder.h
+++ b/cpp/wedpr-transport/sdk/src/TransportBuilder.h
@@ -21,6 +21,7 @@
 #include "Transport.h"
 #include "ppc-framework/front/FrontConfig.h"
 #include "ppc-framework/gateway/IGateway.h"
+#include 
 #include 
 namespace ppc::sdk
 {
@@ -49,7 +50,31 @@ class TransportBuilder
 
     ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder() { return m_frontConfigBuilder; }
 
+
+    static void initLog(const std::string& configPath)
+    {
+        std::call_once(g_flag, [configPath]() {
+            // init the log
+            boost::property_tree::ptree pt;
+            try
+            {
+                boost::property_tree::read_ini(configPath, pt);
+            }
+            catch (std::exception const& e)
+            {
+                // disable the log when the configPath not exists
+                pt.put("log.enable", false);
+            }
+
+            g_logInitializer = std::make_shared();
+            g_logInitializer->initLog(pt, bcos::FileLogger, "gateway_sdk_log");
+        });
+    }
+
 private:
     ppc::front::FrontConfigBuilder::Ptr m_frontConfigBuilder;
+
+    static std::once_flag g_flag;
+    static bcos::BoostLogInitializer::Ptr g_logInitializer;
 };
 }  // namespace ppc::sdk
\ No newline at end of file

From 1399960016eb525789de1715f262252fe0777582 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Thu, 31 Oct 2024 20:51:31 +0800
Subject: [PATCH 074/120] fix ihc bug && persistent task result (#73)

* fix ihc bug

* add python action

* return log result

* persistent task result
---
 .github/workflows/cpp_toolkit_workflow.yml    |   8 +-
 .../sdk-wrapper/python/CMakeLists.txt         |  11 +-
 python/ppc_common/db_models/computing_node.py |  11 -
 python/ppc_common/db_models/config.sql        |  40 ----
 .../ppc_common/db_models/job_worker_record.py |  17 --
 .../ppc_common/deps_services/hdfs_storage.py  |   2 +-
 .../async_thread_executor.py                  |   2 +-
 python/ppc_common/ppc_crypto/ihc_cipher.py    |   4 +-
 .../ppc_crypto/test/phe_unittest.py           |  60 +++--
 python/ppc_model/common/initializer.py        |  18 +-
 python/ppc_model/common/protocol.py           |   7 +-
 python/ppc_model/conf/application-sample.yml  |   5 +-
 .../test/test_feature_binning.py              |   4 +-
 .../ppc_model/datasets/test/test_dataset.py   |  10 +-
 python/ppc_model/db/__init__.py               |   0
 python/ppc_model/db/job_worker_record.py      |  34 +++
 .../feature_engineering_context.py            |   3 +-
 .../feature_engineering_engine.py             |   4 +-
 .../test/feature_engineering_unittest.py      |   2 +
 python/ppc_model/interface/task_engine.py     |   2 +-
 python/ppc_model/log/__init__.py              |   0
 python/ppc_model/log/log_retriever.py         |  92 ++++++++
 python/ppc_model/metrics/test/test_metrics.py |   6 +-
 .../model_result/task_result_handler.py       |  12 +-
 .../network/http/model_controller.py          |  13 +-
 .../preprocessing/preprocessing_engine.py     |   3 +-
 .../preprocessing/processing_context.py       |   3 +-
 .../secure_lgbm/secure_lgbm_context.py        |   3 +-
 .../secure_lgbm_prediction_engine.py          |   4 +-
 .../secure_lgbm_training_engine.py            |   4 +-
 .../secure_lgbm/test/test_save_load_model.py  |   2 +-
 .../test/test_secure_lgbm_context.py          |   4 +-
 .../test_secure_lgbm_performance_training.py  |   4 +-
 .../test/test_secure_lgbm_training.py         |   4 +-
 .../ppc_model/secure_lr/secure_lr_context.py  |   3 +-
 .../secure_lr/secure_lr_prediction_engine.py  |   4 +-
 .../secure_lr/secure_lr_training_engine.py    |   4 +-
 .../secure_lr/test/test_enc_matmul.py         |   2 +-
 python/ppc_model/secure_lr/test/test_lr.py    |   2 +-
 .../test_secure_lr_performance_training.py    |   4 +-
 .../secure_lr/test/test_secure_lr_training.py |   4 +-
 .../secure_model_base/secure_model_context.py |   3 +-
 python/ppc_model/task/task_manager.py         | 219 ++++++++++++------
 43 files changed, 412 insertions(+), 231 deletions(-)
 delete mode 100644 python/ppc_common/db_models/computing_node.py
 delete mode 100644 python/ppc_common/db_models/config.sql
 delete mode 100644 python/ppc_common/db_models/job_worker_record.py
 create mode 100644 python/ppc_model/db/__init__.py
 create mode 100644 python/ppc_model/db/job_worker_record.py
 create mode 100644 python/ppc_model/log/__init__.py
 create mode 100644 python/ppc_model/log/log_retriever.py

diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index 4e5bf001..5189171a 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -39,6 +39,12 @@ jobs:
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Prepare python
+        if: runner.os == 'Windows'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
+          cache: 'pip'
       - name: Prepare vcpkg
         if: runner.os != 'Windows'
         uses: friendlyanon/setup-vcpkg@v1
@@ -64,7 +70,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
index fbaa6b04..dffb7cdf 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
+++ b/cpp/wedpr-transport/sdk-wrapper/python/CMakeLists.txt
@@ -39,11 +39,12 @@ endif()
 # which won't be interpreted inside a generator expression.
 # i.e. we can't use: $<$:${PYTHON_LIBRARIES}>
 # see: https://cmake.org/cmake/help/git-stage/command/target_link_libraries.html#command:target_link_libraries
-if(MSVC)
-  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${Python3_LIBRARIES})
-else()
-  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET})
-endif()
+#if(MSVC)
+#  message("## Python3_LIBRARIES: ${Python3_LIBRARIES}")
+#  target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET} ${Python3_LIBRARIES})
+#else()
+target_link_libraries(${WEDPR_PYTHON_TRANSPORT} PRIVATE ${WEDPR_TRANSPORT_SDK_TARGET})
+#endif()
 SET(LIBRARY_OUTPUT_PATH ${WEDPR_PYTHON_TRANSPORT_LIB_DIR})
 
 message(STATUS "CMAKE_INSTALL_INCLUDEDIR => ${CMAKE_INSTALL_INCLUDEDIR}")
diff --git a/python/ppc_common/db_models/computing_node.py b/python/ppc_common/db_models/computing_node.py
deleted file mode 100644
index e8fa5543..00000000
--- a/python/ppc_common/db_models/computing_node.py
+++ /dev/null
@@ -1,11 +0,0 @@
-
-from ppc_common.db_models import db
-
-
-class ComputingNodeRecord(db.Model):
-    __tablename__ = 'wedpr_computing_node'
-    id = db.Column(db.String(255), primary_key=True)
-    url = db.Column(db.String(255))
-    type = db.Column(db.String(255))
-    token = db.Column(db.String(255))
-    loading = db.Column(db.Integer)
diff --git a/python/ppc_common/db_models/config.sql b/python/ppc_common/db_models/config.sql
deleted file mode 100644
index 36d8406e..00000000
--- a/python/ppc_common/db_models/config.sql
+++ /dev/null
@@ -1,40 +0,0 @@
-
-CREATE TABLE wedpr_scheduler_job_worker_table (
-    worker_id VARCHAR(100),
-    job_id VARCHAR(255),
-    type VARCHAR(255),
-    status VARCHAR(255),
-    args TEXT,
-    upstreams TEXT,
-    inputs_statement TEXT,
-    outputs TEXT,
-    create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-    PRIMARY KEY (worker_id),
-    INDEX job_id_idx (job_id)
-)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC;
-
-CREATE TABLE wedpr_scheduler_job_table (
-    job_id VARCHAR(255),
-    request TEXT,
-    status TEXT,
-    create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-    PRIMARY KEY (job_id)
-)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC;
-
-CREATE TABLE wedpr_computing_node (
-    id VARCHAR(255),
-    url VARCHAR(255),
-    type VARCHAR(255),
-    loading INT,
-    token VARCHAR(255) DEFAULT '',
-    PRIMARY KEY (id)
-)ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC;
-
-
-INSERT INTO wedpr_computing_node (id, url, type, loading, token)
-VALUES
-    ("001", '127.0.0.1:10200', 'PSI', 0, ''),
-    ("002", '127.0.0.1:10201', 'MPC', 0, ''),
-    ("003", '127.0.0.1:10202', 'MODEL', 0, '');
diff --git a/python/ppc_common/db_models/job_worker_record.py b/python/ppc_common/db_models/job_worker_record.py
deleted file mode 100644
index af9b4c26..00000000
--- a/python/ppc_common/db_models/job_worker_record.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-from datetime import datetime
-from ppc_common.db_models import db
-
-
-class JobWorkerRecord(db.Model):
-    __tablename__ = 'wedpr_scheduler_job_worker_table'
-    worker_id = db.Column(db.String(127), primary_key=True)
-    job_id = db.Column(db.String(255), index=True)
-    type = db.Column(db.String(255))
-    status = db.Column(db.String(255))
-    upstreams = db.Column(db.Text)
-    inputs_statement = db.Column(db.Text)
-    args = db.Column(db.Text)
-    outputs = db.Column(db.Text)
-    create_time = db.Column(db.DateTime, default=datetime.now)
-    update_time = db.Column(db.DateTime, onupdate=datetime.now)
diff --git a/python/ppc_common/deps_services/hdfs_storage.py b/python/ppc_common/deps_services/hdfs_storage.py
index d48fbc77..472c9ea4 100644
--- a/python/ppc_common/deps_services/hdfs_storage.py
+++ b/python/ppc_common/deps_services/hdfs_storage.py
@@ -70,7 +70,7 @@ def save_data(self, data: AnyStr, hdfs_path):
                           hdfs_path), data, overwrite=True)
         return
 
-    def get_data(self, hdfs_path) -> AnyStr:
+    def get_data(self, hdfs_path):
         with self.client.read(os.path.join(self._hdfs_storage_path, hdfs_path)) as reader:
             content = reader.read()
         return content
diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py
index 15b1f5ca..2c14c308 100644
--- a/python/ppc_common/ppc_async_executor/async_thread_executor.py
+++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py
@@ -21,7 +21,7 @@ def execute(self, target_id: str, target: Callable, on_target_finish: Callable[[
                 args=()):
         def thread_target(logger, on_finish, *args):
             try:
-                target(*args)
+                target(target_id, *args)
                 on_finish(target_id, True)
             except Exception as e:
                 logger.warn(traceback.format_exc())
diff --git a/python/ppc_common/ppc_crypto/ihc_cipher.py b/python/ppc_common/ppc_crypto/ihc_cipher.py
index 4adfeb1c..0ac4723f 100644
--- a/python/ppc_common/ppc_crypto/ihc_cipher.py
+++ b/python/ppc_common/ppc_crypto/ihc_cipher.py
@@ -64,7 +64,7 @@ def encode(self) -> bytes:
         return len_bytes + c_left_bytes + c_right_bytes
 
     @classmethod
-    def decode(cls, encoded_data: bytes):
+    def decode(cls, encoded_data: bytes, key_length: int = 256):
         # 解码整数的长度
         len_c_left, len_c_right = struct.unpack('>II', encoded_data[:8])
 
@@ -73,7 +73,7 @@ def decode(cls, encoded_data: bytes):
             encoded_data[8:8 + len_c_left], byteorder='big')
         c_right = int.from_bytes(
             encoded_data[8 + len_c_left:8 + len_c_left + len_c_right], byteorder='big')
-        return cls(c_left, c_right, cls.number_codec)
+        return cls(c_left, c_right, NumberCodec(key_length))
 
 
 class IhcCipher(PheCipher):
diff --git a/python/ppc_common/ppc_crypto/test/phe_unittest.py b/python/ppc_common/ppc_crypto/test/phe_unittest.py
index b7381a1e..db610243 100644
--- a/python/ppc_common/ppc_crypto/test/phe_unittest.py
+++ b/python/ppc_common/ppc_crypto/test/phe_unittest.py
@@ -8,12 +8,13 @@
 from ppc_common.ppc_crypto.paillier_cipher import PaillierCipher
 
 
-class PaillierTest:
+class HomoTest:
     def __init__(self, ut):
         self.paillier = PaillierCipher(key_length=1024)
+        self.ihc_cipher = IhcCipher(key_length=1024)
         self.ut = ut
 
-    def test_enc_and_dec_parallel(self, test_size, start, end):
+    def test_enc_and_dec_parallel(self,  homo_impl, test_size, start, end):
         inputs = np.random.randint(start, end, size=test_size)
 
         # start_time = time.time()
@@ -22,39 +23,39 @@ def test_enc_and_dec_parallel(self, test_size, start, end):
         # print("enc:", end_time - start_time, "seconds")
 
         start_time = time.time()
-        ciphers = self.paillier.encrypt_batch_parallel(inputs)
+        ciphers = homo_impl.encrypt_batch_parallel(inputs)
         end_time = time.time()
         print("enc_p:", end_time - start_time, "seconds")
 
         start_time = time.time()
-        outputs = self.paillier.decrypt_batch_parallel(ciphers)
+        outputs = homo_impl.decrypt_batch_parallel(ciphers)
         end_time = time.time()
         print("dec_p:", end_time - start_time, "seconds")
 
         self.ut.assertListEqual(list(inputs), list(outputs))
-        self.test_ihc_mul_enc_and_dec(ciphers, inputs, 10)
+        self.test_homo_mul_enc_and_dec(homo_impl, ciphers, inputs, 10)
         # test add and enc dec
         inputs2 = np.random.randint(start, end, size=test_size)
-        ciphers2 = self.paillier.encrypt_batch_parallel(inputs2)
-        self.test_ihc_add_enc_and_desc(ciphers, ciphers2, inputs, inputs2)
+        ciphers2 = homo_impl.encrypt_batch_parallel(inputs2)
+        self.test_homo_add_enc_and_desc(
+            homo_impl, ciphers, ciphers2, inputs, inputs2)
 
-    def test_ihc_mul_enc_and_dec(self, ciphers, inputs, mul_value):
+    def test_homo_mul_enc_and_dec(self, homo_impl, ciphers, inputs, mul_value):
         start_time = time.time()
         mul_ciphers = []
         for cipher in ciphers:
-            cipher.__mul__(mul_value)
-            mul_ciphers.append(cipher * (mul_value))
+            mul_ciphers.append(cipher * mul_value)
         # decrypt
-        outputs = self.paillier.decrypt_batch_parallel(mul_ciphers)
+        outputs = homo_impl.decrypt_batch_parallel(mul_ciphers)
         mul_result = []
         for input in inputs:
             mul_result.append(mul_value * input)
         self.ut.assertListEqual(mul_result, list(outputs))
         end_time = time.time()
         print(
-            f"#### test_ihc_mul_enc_and_desc passed, time: {end_time - start_time} seconds")
+            f"#### test_homo_mul_enc_and_dec passed, time: {end_time - start_time} seconds")
 
-    def test_ihc_add_enc_and_desc(self, ciphers1, ciphers2, inputs1, inputs2):
+    def test_homo_add_enc_and_desc(self, homo_impl, ciphers1, ciphers2, inputs1, inputs2):
         start_time = time.time()
         add_ciphers = []
         i = 0
@@ -63,22 +64,36 @@ def test_ihc_add_enc_and_desc(self, ciphers1, ciphers2, inputs1, inputs2):
             add_ciphers.append(cipher + ciphers2[i])
             expected_result.append(inputs1[i] + inputs2[i])
             i += 1
-        outputs = self.paillier.decrypt_batch_parallel(add_ciphers)
+        outputs = homo_impl.decrypt_batch_parallel(add_ciphers)
         self.ut.assertListEqual(expected_result, list(outputs))
         end_time = time.time()
         print(
-            f"#### test_ihc_add_enc_and_desc passed, time: {end_time - start_time} seconds, size: {len(inputs1)}")
+            f"#### test_homo_add_enc_and_desc passed, time: {end_time - start_time} seconds, size: {len(inputs1)}")
 
 
 class PaillierUtilsTest(unittest.TestCase):
-
-    def test_enc_and_dec_parallel(self):
-        paillier_test = PaillierTest(self)
-        paillier_test.test_enc_and_dec_parallel(10, -20, -1)
-        paillier_test.test_enc_and_dec_parallel(10000, -20000, 20000)
-        paillier_test.test_enc_and_dec_parallel(10000, 0, 20000)
+    def test_paillier_enc_and_dec(self):
+        print("######## test paillier case: ####")
+        homo_test = HomoTest(self)
+        homo_test.test_enc_and_dec_parallel(homo_test.paillier, 10, -20, -1)
+        homo_test.test_enc_and_dec_parallel(
+            homo_test.paillier, 10000, -20000, 20000)
+        homo_test.test_enc_and_dec_parallel(
+            homo_test.paillier, 10000, 0, 20000)
+        print("######## test paillier case end ####")
+
+    def test_ihc_enc_and_dec(self):
+        print("######## test ihc case: ####")
+        homo_test = HomoTest(self)
+        homo_test.test_enc_and_dec_parallel(homo_test.ihc_cipher, 10, -20, -1)
+        homo_test.test_enc_and_dec_parallel(
+            homo_test.ihc_cipher, 10000, -20000, 20000)
+        homo_test.test_enc_and_dec_parallel(
+            homo_test.ihc_cipher, 10000, 0, 20000)
+        print("######## test ihc case end ####")
 
     def test_ihc_enc_and_dec_parallel(self):
+        print("##### test_ihc_enc_and_dec_parallel")
         ihc = IhcCipher(key_length=256)
         try_size = 100000
         inputs = np.random.randint(-10001, 10001, size=try_size)
@@ -130,7 +145,8 @@ def test_ihc_enc_and_dec_parallel(self):
         self.assertListEqual(list(inputs), list(outputs))
         assert decrypted == expected
 
-    def test_ihc_code(self):
+    def test_ihc_decode(self):
+        print("##### test_ihc_decode")
         ihc = IhcCipher(key_length=256)
         try_size = 100000
         inputs = np.random.randint(-10001, 10001, size=try_size)
diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py
index 8a89404e..7e5d11c0 100644
--- a/python/ppc_model/common/initializer.py
+++ b/python/ppc_model/common/initializer.py
@@ -9,8 +9,12 @@
 from ppc_common.ppc_utils import common_func
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from wedpr_python_gateway_sdk.transport.impl.transport_loader import TransportLoader
+from ppc_common.deps_services.mysql_storage import MySQLStorage
+from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader
 from ppc_model.network.wedpr_model_transport import ModelTransport
 from ppc_model.task.task_manager import TaskManager
+from ppc_model.task.task_manager import TaskPersistent
+from ppc_model.log.log_retriever import LogRetriever
 
 
 class Initializer:
@@ -27,13 +31,23 @@ def __init__(self, log_config_path, config_path, plot_lock=None):
         self.job_cache_dir = common_func.get_config_value(
             "JOB_TEMP_DIR", "/tmp", self.config_data, False)
         self.thread_event_manager = ThreadEventManager()
+
+        task_persistent = TaskPersistent(self.logger(), MySQLStorage(
+            storage_config=SQLStorageConfigLoader.load(self.config_data)))
+
+        self.storage_client = storage_loader.load(
+            self.config_data, self.logger())
+        log_path = common_func.get_config_value(
+            "LOG_PATH", "logs/wedpr-model.log", self.config_data, False)
+        self.log_retriever = LogRetriever(
+            self.logger(), self.job_cache_dir, self.storage_client, log_path)
         self.task_manager = TaskManager(
             logger=self.logger(),
+            task_persistent=task_persistent,
+            log_retriever=self.log_retriever,
             thread_event_manager=self.thread_event_manager,
             task_timeout_h=self.config_data['TASK_TIMEOUT_H']
         )
-        self.storage_client = storage_loader.load(
-            self.config_data, self.logger())
         # default send msg timeout
         self.MODEL_COMPONENT = "WEDPR_MODEL"
         self.send_msg_timeout_ms = 5000
diff --git a/python/ppc_model/common/protocol.py b/python/ppc_model/common/protocol.py
index 31bc4ded..cd6a2b9e 100644
--- a/python/ppc_model/common/protocol.py
+++ b/python/ppc_model/common/protocol.py
@@ -20,9 +20,12 @@ class ModelTask(Enum):
 
 
 class TaskStatus(Enum):
+    NotFound = "NotFound"
+    PENDING = "PENDING"
     RUNNING = "RUNNING"
-    COMPLETED = "COMPLETED"
-    FAILED = "FAILED"
+    FAILURE = "FAILURE"
+    KILLED = "KILLED"
+    SUCCESS = "SUCCESS"
 
 
 class RpcType(Enum):
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index cf3cbd11..c88b9591 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -40,4 +40,7 @@ transport_threadpool_size: 4
 transport_node_id: "MODEL_WeBank_NODE"
 transport_gateway_targets: "ipv4:127.0.0.1:40600,127.0.0.1:40601"
 transport_host_ip: "127.0.0.1"
-transport_listen_port: 6500
\ No newline at end of file
+transport_listen_port: 6500
+
+# the log retriever config
+LOG_PATH: "logs/wedpr-model.log"
\ No newline at end of file
diff --git a/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py
index 20592c87..88bb637c 100644
--- a/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py
+++ b/python/ppc_model/datasets/feature_binning/test/test_feature_binning.py
@@ -39,7 +39,7 @@ def test_train_feature_binning(self):
             }
         }
 
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label)
         secure_dataset = SecureDataset(task_info, model_data)
@@ -79,7 +79,7 @@ def test_test_feature_binning(self):
             }
         }
 
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label)
         secure_dataset = SecureDataset(task_info, model_data)
diff --git a/python/ppc_model/datasets/test/test_dataset.py b/python/ppc_model/datasets/test/test_dataset.py
index 8fec5b8e..a4f1ec7b 100644
--- a/python/ppc_model/datasets/test/test_dataset.py
+++ b/python/ppc_model/datasets/test/test_dataset.py
@@ -72,7 +72,7 @@ def test_random_split_dataset(self):
                 'random_state': 2024
             }
         }
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         print(task_info.model_params.get_all_params())
 
         # 模拟构造主动方数据集
@@ -98,7 +98,7 @@ def test_random_split_dataset(self):
                 'random_state': 2024
             }
         }
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         print(task_info.model_params.get_all_params())
 
         # 模拟构造被动方数据集
@@ -127,7 +127,7 @@ def test_customized_split_dataset(self):
                 'eval_set_value': 'OOS'
             }
         }
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         print(task_info.model_params.get_all_params())
 
         # 模拟构造主动方数据集
@@ -157,7 +157,7 @@ def test_predict_dataset(self):
             'algorithm_subtype': None,
             'model_dict': {}
         }
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         print(task_info.model_params.get_all_params())
 
         # 模拟构造主动方数据集
@@ -183,7 +183,7 @@ def test_iv_selected_dataset(self):
             'algorithm_subtype': None,
             'model_dict': {}
         }
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         print(task_info.model_params.get_all_params())
 
         # 模拟构造主动方数据集
diff --git a/python/ppc_model/db/__init__.py b/python/ppc_model/db/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/ppc_model/db/job_worker_record.py b/python/ppc_model/db/job_worker_record.py
new file mode 100644
index 00000000..c3d4b0d9
--- /dev/null
+++ b/python/ppc_model/db/job_worker_record.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from datetime import datetime
+from ppc_common.db_models import db
+
+
+class JobWorkerRecord(db.Model):
+    """
+    CREATE TABLE if not exists wedpr_job_worker_table (
+    worker_id VARCHAR(100),
+    job_id VARCHAR(255),
+    type VARCHAR(255),
+    status VARCHAR(255),
+    args LONGTEXT,
+    upstreams TEXT,
+    inputs_statement TEXT,
+    outputs TEXT,
+    create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    PRIMARY KEY (worker_id),
+    INDEX job_id_idx (job_id)
+    )ENGINE='InnoDB' DEFAULT CHARSET='utf8mb4' COLLATE='utf8mb4_bin' ROW_FORMAT=DYNAMIC;
+    """
+    __tablename__ = 'wedpr_job_worker_table'
+    worker_id = db.Column(db.String(100), primary_key=True)
+    job_id = db.Column(db.String(255), index=True)
+    type = db.Column(db.String(255))
+    status = db.Column(db.String(255))
+    upstreams = db.Column(db.Text)
+    inputs_statement = db.Column(db.Text)
+    args = db.Column(db.Text)
+    outputs = db.Column(db.Text)
+    exec_result = db.Column(db.Text)
+    create_time = db.Column(db.DateTime, default=datetime.now)
+    update_time = db.Column(db.DateTime, onupdate=datetime.now)
diff --git a/python/ppc_model/feature_engineering/feature_engineering_context.py b/python/ppc_model/feature_engineering/feature_engineering_context.py
index c3a793ea..e8df2299 100644
--- a/python/ppc_model/feature_engineering/feature_engineering_context.py
+++ b/python/ppc_model/feature_engineering/feature_engineering_context.py
@@ -19,6 +19,7 @@ class FeMessage(Enum):
 class FeatureEngineeringContext(Context):
 
     def __init__(self,
+                 task_id,
                  args,
                  components: Initializer,
                  role: TaskRole,
@@ -26,7 +27,7 @@ def __init__(self,
                  feature_name_list: list,
                  label: np.ndarray = None):
         super().__init__(args['job_id'],
-                         args['task_id'],
+                         task_id,
                          components,
                          role)
         self.feature_name_list = feature_name_list
diff --git a/python/ppc_model/feature_engineering/feature_engineering_engine.py b/python/ppc_model/feature_engineering/feature_engineering_engine.py
index 0b3bb88f..b882203f 100644
--- a/python/ppc_model/feature_engineering/feature_engineering_engine.py
+++ b/python/ppc_model/feature_engineering/feature_engineering_engine.py
@@ -12,7 +12,7 @@ class FeatureEngineeringEngine(TaskEngine):
     task_type = ModelTask.FEATURE_ENGINEERING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
         input_path = BaseContext.feature_engineering_input_path(
             args['job_id'], components.config_data['JOB_TEMP_DIR'])
 
@@ -20,6 +20,7 @@ def run(args):
             field_list, label, feature = SecureDataset.read_dataset(
                 input_path, True)
             context = FeatureEngineeringContext(
+                task_id=task_id,
                 args=args,
                 components=components,
                 role=TaskRole.ACTIVE_PARTY,
@@ -32,6 +33,7 @@ def run(args):
             field_list, _, feature = SecureDataset.read_dataset(
                 input_path, False)
             context = FeatureEngineeringContext(
+                task_id = task_id,
                 args=args,
                 components=components,
                 role=TaskRole.PASSIVE_PARTY,
diff --git a/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
index 4dd89171..09340a8b 100644
--- a/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
+++ b/python/ppc_model/feature_engineering/test/feature_engineering_unittest.py
@@ -82,6 +82,7 @@ def test_fit(self):
         active_components.mock_logger = MockLogger()
         active_components.storage_client = MockStorageClient()
         active_context = FeatureEngineeringContext(
+            task_id=args_a['task_id'],
             args=args_a,
             components=active_components,
             role=TaskRole.ACTIVE_PARTY,
@@ -98,6 +99,7 @@ def test_fit(self):
         passive_components.mock_logger = MockLogger()
         passive_components.storage_client = MockStorageClient()
         passive_context = FeatureEngineeringContext(
+            task_id=args_b['task_id'],
             args=args_b,
             components=passive_components,
             role=TaskRole.PASSIVE_PARTY,
diff --git a/python/ppc_model/interface/task_engine.py b/python/ppc_model/interface/task_engine.py
index 123b0eb9..0f55887d 100644
--- a/python/ppc_model/interface/task_engine.py
+++ b/python/ppc_model/interface/task_engine.py
@@ -5,5 +5,5 @@ class TaskEngine(ABC):
     task_type: str
 
     @staticmethod
-    def run(args: dict):
+    def run(task_id: str, args: dict):
         ...
diff --git a/python/ppc_model/log/__init__.py b/python/ppc_model/log/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/python/ppc_model/log/log_retriever.py b/python/ppc_model/log/log_retriever.py
new file mode 100644
index 00000000..fd2a4f70
--- /dev/null
+++ b/python/ppc_model/log/log_retriever.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+import subprocess
+import logging
+from ppc_model.common.protocol import LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER
+from ppc_common.deps_services.storage_api import StorageApi
+import os
+
+
+class LogRetriever:
+    # max retrieve 500k
+    max_retrieved_log_size = 500 * 1024
+
+    def __init__(self, logger, job_cache_dir, storage_client: StorageApi, log_path):
+        self.logger = logger
+        self.job_cache_dir = job_cache_dir
+        self.storage_client = storage_client
+        self.relative_log_path = log_path
+
+    def retrieve_log(self, job_id):
+        try:
+            # get the log size
+            log_size_byte = self.storage_client.get_data(
+                self._get_remote_log_size_path(job_id))
+            log_size = int.from_bytes(log_size_byte, 'big')
+            remote_log_path = os.path.join(
+                self.storage_client.get_home_path(), self._get_remote_log_path(job_id))
+            if log_size > LogRetriever.max_retrieved_log_size:
+                return (log_size, remote_log_path, None)
+            # get the log data
+            log_data = self.storage_client.get_data(
+                self._get_remote_log_path(job_id)).decode(encoding='utf-8')
+            return (log_size, remote_log_path, log_data)
+        except:
+            return (None, None, None)
+
+    def _get_remote_log_path(self, job_id):
+        return os.path.join(job_id, f'{job_id}.log')
+
+    def _get_remote_log_size_path(self, job_id):
+        return f"{self._get_remote_log_path(job_id)}.size"
+
+    def _get_local_job_log_path(self, job_id):
+        return os.path.join(self.job_cache_dir, job_id, f'{job_id}.log')
+
+    def upload_log(self, job_id):
+        ret = self.record_model_job_log(job_id)
+        if ret is False:
+            return
+        # the log size
+        log_size = os.path.getsize(self._get_local_job_log_path(job_id))
+        # store the log size
+        self.storage_client.save_data(log_size.to_bytes((log_size.bit_length() + 7) // 8, 'big'),
+                                      self._get_remote_log_size_path(job_id))
+        # store the log path
+        self.storage_client.upload_file(self._get_local_job_log_path(
+            job_id), self._get_remote_log_path(job_id))
+
+    def _make_local_log_dir(self, job_id):
+        local_log_path = self._get_local_job_log_path(job_id)
+        if os.path.exists(local_log_path):
+            return
+        parent_dir = os.path.dirname(local_log_path)
+        if os.path.exists(parent_dir):
+            return
+        os.makedirs(parent_dir)
+
+    def record_model_job_log(self, job_id):
+        log_file = self._get_log_file_path()
+        if log_file is None or log_file == "":
+            current_working_dir = os.getcwd()
+            log_file = os.path.join(
+                current_working_dir, self.relative_log_path)
+        start_keyword = LOG_START_FLAG_FORMATTER.format(job_id=job_id)
+        end_keyword = LOG_END_FLAG_FORMATTER.format(job_id=job_id)
+        self._make_local_log_dir(job_id)
+
+        command = f"grep -i {job_id} {log_file} > {self._get_local_job_log_path(job_id)}"
+        result = subprocess.run(
+            command, shell=True, text=True, capture_output=True)
+        if result.stderr:
+            self.logger.warn(
+                f"record_model_job_log for job {job_id} failed, error: {result.stderr}")
+            return False
+        return True
+
+    def _get_log_file_path(self):
+        log_file_path = None
+        for handler in self.logger.handlers:
+            if isinstance(handler, logging.FileHandler):
+                log_file_path = handler.baseFilename
+                break
+        return log_file_path
diff --git a/python/ppc_model/metrics/test/test_metrics.py b/python/ppc_model/metrics/test/test_metrics.py
index 5382d5d8..76a2d92a 100644
--- a/python/ppc_model/metrics/test/test_metrics.py
+++ b/python/ppc_model/metrics/test/test_metrics.py
@@ -71,7 +71,7 @@ def test_active_metrics(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLGBMContext(self.args_a, active_components)
+        task_info_a = SecureLGBMContext(self.args_a['task_id'], self.args_a, active_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=True)
         secure_dataset_a = SecureDataset(task_info_a, model_data)
@@ -96,7 +96,7 @@ def test_passive_metrics(self):
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
-        task_info_b = SecureLGBMContext(self.args_b, passive_components)
+        task_info_b = SecureLGBMContext(self.args_b['task_id'], self.args_b, passive_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=False)
         secure_dataset_b = SecureDataset(task_info_b, model_data)
@@ -119,7 +119,7 @@ def test_model_plot(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLGBMContext(self.args_a, active_components)
+        task_info_a = SecureLGBMContext(self.args_a['task_id'], self.args_a, active_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=True)
         secure_dataset_a = SecureDataset(task_info_a, model_data)
diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py
index d5ef6ca6..ba85b616 100644
--- a/python/ppc_model/model_result/task_result_handler.py
+++ b/python/ppc_model/model_result/task_result_handler.py
@@ -7,6 +7,7 @@
 from ppc_model.common.model_result import ResultFileHandling
 from ppc_common.ppc_ml.model.algorithm_info import EvaluationType
 from ppc_model.common.base_context import BaseContext
+from ppc_model.common.initializer import Initializer
 from enum import Enum
 
 
@@ -317,7 +318,7 @@ def to_dict(self):
 
 
 class TaskResultHandler:
-    def __init__(self, task_result_request: TaskResultRequest, components):
+    def __init__(self, task_result_request: TaskResultRequest, components: Initializer):
         self.task_result_request = task_result_request
         self.components = components
         self.logger = components.logger()
@@ -338,12 +339,19 @@ def get_response(self):
         merged_result = dict()
         for result in self.result_list:
             merged_result.update(result.to_dict())
-
         if self.model_data is None:
             response = {"jobPlanetResult":  merged_result}
         else:
             response = {"jobPlanetResult":  merged_result,
                         "modelData": self.model_data}
+        # record the log
+        log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
+            self.task_result_request.job_id)
+        log_result = {}
+        log_result.update({"logSize": log_size})
+        log_result.update({"logPath": log_path})
+        log_result.update({"logContent": log_content})
+        response.update({"logDetail": log_result})
         return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response)
 
     def _get_evaluation_result(self):
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index f8b0130f..6e985b47 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -44,12 +44,12 @@ def get(self, model_id):
         """
         response = utils.BASE_RESPONSE
         task_id = model_id
-        status, traffic_volume, time_costs = components.task_manager.status(
+        status, traffic_volume, exec_result = components.task_manager.status(
             task_id)
         response['data'] = {
             'status': status,
             'traffic_volume': traffic_volume,
-            'time_costs': time_costs
+            'exec_result': exec_result,
         }
         return response
 
@@ -64,15 +64,6 @@ def delete(self, model_id):
         return utils.BASE_RESPONSE
 
 
-@ns2.route('/')
-class ModelLogCollection(Resource):
-    @api.response(200, 'Task status retrieved successfully.', response_task_status)
-    def get(self, job_id):
-        log_content = components.task_manager.record_model_job_log(job_id)
-        return utils.make_response(utils.PpcErrorCode.SUCCESS.get_code(),
-                                   utils.PpcErrorCode.SUCCESS.get_msg(), log_content)
-
-
 @ns_get_job_result.route('/')
 class ModelResultCollection(Resource):
     @api.response(201, 'Get task result successfully.', response_base)
diff --git a/python/ppc_model/preprocessing/preprocessing_engine.py b/python/ppc_model/preprocessing/preprocessing_engine.py
index 3ca7c6e3..8d9a5ee5 100644
--- a/python/ppc_model/preprocessing/preprocessing_engine.py
+++ b/python/ppc_model/preprocessing/preprocessing_engine.py
@@ -10,8 +10,9 @@ class PreprocessingEngine(TaskEngine):
     task_type = ModelTask.PREPROCESSING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
         context = ProcessingContext(
+            task_id=task_id,
             args=args,
             components=components,
         )
diff --git a/python/ppc_model/preprocessing/processing_context.py b/python/ppc_model/preprocessing/processing_context.py
index 5e597ffe..bc56bcd5 100644
--- a/python/ppc_model/preprocessing/processing_context.py
+++ b/python/ppc_model/preprocessing/processing_context.py
@@ -8,10 +8,11 @@
 
 class ProcessingContext(Context):
     def __init__(self,
+                 task_id,
                  args,
                  components: Initializer):
         super().__init__(args['job_id'],
-                         args['task_id'],
+                         task_id,
                          components,
                          role=None)
         self.dataset_path = args['dataset_path']
diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_context.py b/python/ppc_model/secure_lgbm/secure_lgbm_context.py
index 804c5b97..a6d7d301 100644
--- a/python/ppc_model/secure_lgbm/secure_lgbm_context.py
+++ b/python/ppc_model/secure_lgbm/secure_lgbm_context.py
@@ -120,10 +120,11 @@ def _get_params(self):
 class SecureLGBMContext(SecureModelContext):
 
     def __init__(self,
+                 task_id,
                  args,
                  components: Initializer
                  ):
-        super().__init__(args, components)
+        super().__init__(task_id, args, components)
 
         self.phe = PheCipherFactory.build_phe(
             components.homo_algorithm, components.public_key_length)
diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py
index 203f71e1..09912342 100644
--- a/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py
+++ b/python/ppc_model/secure_lgbm/secure_lgbm_prediction_engine.py
@@ -13,9 +13,9 @@ class SecureLGBMPredictionEngine(TaskEngine):
     task_type = ModelTask.XGB_PREDICTING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
 
-        task_info = SecureLGBMContext(args, components)
+        task_info = SecureLGBMContext(task_id, args, components)
         secure_dataset = SecureDataset(task_info)
 
         if task_info.role == TaskRole.ACTIVE_PARTY:
diff --git a/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py b/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py
index 4fe502c8..5c9c78e2 100644
--- a/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py
+++ b/python/ppc_model/secure_lgbm/secure_lgbm_training_engine.py
@@ -14,9 +14,9 @@ class SecureLGBMTrainingEngine(TaskEngine):
     task_type = ModelTask.XGB_TRAINING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
 
-        task_info = SecureLGBMContext(args, components)
+        task_info = SecureLGBMContext(task_id, args, components)
         secure_dataset = SecureDataset(task_info)
 
         if task_info.role == TaskRole.ACTIVE_PARTY:
diff --git a/python/ppc_model/secure_lgbm/test/test_save_load_model.py b/python/ppc_model/secure_lgbm/test/test_save_load_model.py
index e42c4369..dbf8a433 100644
--- a/python/ppc_model/secure_lgbm/test/test_save_load_model.py
+++ b/python/ppc_model/secure_lgbm/test/test_save_load_model.py
@@ -48,7 +48,7 @@ def test_save_load_model(self):
             trees.append(tree)
         # print(trees)
 
-        task_info = SecureLGBMContext(self.args, self.components)
+        task_info = SecureLGBMContext(self.args['task_id'], self.args, self.components)
         booster = VerticalBooster(task_info, dataset=None)
         booster._X_split = x_split
         booster._trees = trees
diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py
index b04b9218..8ad9be7c 100644
--- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py
+++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_context.py
@@ -25,7 +25,7 @@ def test_get_model_params(self):
             'model_dict': {}
         }
 
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         model_params = task_info.get_model_params()
         # 打印LGBMModel默认参数
         print(model_params._get_params())
@@ -54,7 +54,7 @@ def test_set_model_params(self):
             }
         }
 
-        task_info = SecureLGBMContext(args, self.components)
+        task_info = SecureLGBMContext(args['task_id'], args, self.components)
         model_params = task_info.get_model_params()
         # 打印SecureLGBMParams自定义参数
         print(model_params.get_params())
diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
index 4ba739a7..bde18cdb 100644
--- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
+++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_performance_training.py
@@ -78,7 +78,7 @@ def test_fit(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLGBMContext(args_a, active_components)
+        task_info_a = SecureLGBMContext(args_a['task_id'], args_a, active_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=True)
         secure_dataset_a = SecureDataset(task_info_a, model_data)
@@ -96,7 +96,7 @@ def test_fit(self):
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
-        task_info_b = SecureLGBMContext(args_b, passive_components)
+        task_info_b = SecureLGBMContext(args_b['task_id'], args_b, passive_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=False)
         secure_dataset_b = SecureDataset(task_info_b, model_data)
diff --git a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
index 826b89dd..d0812e1e 100644
--- a/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
+++ b/python/ppc_model/secure_lgbm/test/test_secure_lgbm_training.py
@@ -87,7 +87,7 @@ def test_fit(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLGBMContext(args_a, active_components)
+        task_info_a = SecureLGBMContext(args_a['task_id'], args_a, active_components)
         secure_dataset_a = SecureDataset(task_info_a, self.df_with_y)
         booster_a = VerticalLGBMActiveParty(task_info_a, secure_dataset_a)
         print(secure_dataset_a.feature_name)
@@ -104,7 +104,7 @@ def test_fit(self):
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
-        task_info_b = SecureLGBMContext(args_b, passive_components)
+        task_info_b = SecureLGBMContext(args_b['task_id'], args_b, passive_components)
         secure_dataset_b = SecureDataset(task_info_b, self.df_without_y)
         booster_b = VerticalLGBMPassiveParty(task_info_b, secure_dataset_b)
         print(secure_dataset_b.feature_name)
diff --git a/python/ppc_model/secure_lr/secure_lr_context.py b/python/ppc_model/secure_lr/secure_lr_context.py
index d4e74a01..a3fa20a6 100644
--- a/python/ppc_model/secure_lr/secure_lr_context.py
+++ b/python/ppc_model/secure_lr/secure_lr_context.py
@@ -73,10 +73,11 @@ def _get_params(self):
 class SecureLRContext(SecureModelContext):
 
     def __init__(self,
+                 task_id,
                  args,
                  components: Initializer
                  ):
-        super().__init__(args, components)
+        super().__init__(task_id, args, components)
 
         self.phe = PheCipherFactory.build_phe(
             components.homo_algorithm, components.public_key_length)
diff --git a/python/ppc_model/secure_lr/secure_lr_prediction_engine.py b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py
index 92776269..853b698c 100644
--- a/python/ppc_model/secure_lr/secure_lr_prediction_engine.py
+++ b/python/ppc_model/secure_lr/secure_lr_prediction_engine.py
@@ -13,9 +13,9 @@ class SecureLRPredictionEngine(TaskEngine):
     task_type = ModelTask.LR_PREDICTING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
 
-        task_info = SecureLRContext(args, components)
+        task_info = SecureLRContext(task_id, args, components)
         secure_dataset = SecureDataset(task_info)
 
         if task_info.role == TaskRole.ACTIVE_PARTY:
diff --git a/python/ppc_model/secure_lr/secure_lr_training_engine.py b/python/ppc_model/secure_lr/secure_lr_training_engine.py
index 66497f23..21cef8d2 100644
--- a/python/ppc_model/secure_lr/secure_lr_training_engine.py
+++ b/python/ppc_model/secure_lr/secure_lr_training_engine.py
@@ -14,9 +14,9 @@ class SecureLRTrainingEngine(TaskEngine):
     task_type = ModelTask.LR_TRAINING
 
     @staticmethod
-    def run(args):
+    def run(task_id, args):
 
-        task_info = SecureLRContext(args, components)
+        task_info = SecureLRContext(task_id, args, components)
         secure_dataset = SecureDataset(task_info)
 
         if task_info.role == TaskRole.ACTIVE_PARTY:
diff --git a/python/ppc_model/secure_lr/test/test_enc_matmul.py b/python/ppc_model/secure_lr/test/test_enc_matmul.py
index 6dc921b7..4b78b0eb 100644
--- a/python/ppc_model/secure_lr/test/test_enc_matmul.py
+++ b/python/ppc_model/secure_lr/test/test_enc_matmul.py
@@ -40,7 +40,7 @@ def test_enc_matmul(self):
         active_components = Initializer(log_config_path='', config_path='')
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
-        task_info = SecureLRContext(args, active_components)
+        task_info = SecureLRContext(args['task_id'], args, active_components)
         
         # 15个特征，batch_size: 8
         arr = np.array([2, 4, -5, 0, 9, -7, 12, 3])
diff --git a/python/ppc_model/secure_lr/test/test_lr.py b/python/ppc_model/secure_lr/test/test_lr.py
index f6121fb6..397d0b07 100644
--- a/python/ppc_model/secure_lr/test/test_lr.py
+++ b/python/ppc_model/secure_lr/test/test_lr.py
@@ -73,7 +73,7 @@ def mock_args():
 active_components.config_data = {
     'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
 active_components.mock_logger = MockLogger()
-task_info_a = SecureLRContext(args_a, active_components)
+task_info_a = SecureLRContext(args_a['task_id'], args_a, active_components)
 
 # df ---------------------------------------------
 secure_dataset = SecureDataset(task_info_a, df)
diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
index 9aa985e9..5052289f 100644
--- a/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
+++ b/python/ppc_model/secure_lr/test/test_secure_lr_performance_training.py
@@ -75,7 +75,7 @@ def test_fit(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLGBMContext(args_a, active_components)
+        task_info_a = SecureLGBMContext(args_a['task_id'], args_a, active_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=True)
         secure_dataset_a = SecureDataset(task_info_a, model_data)
@@ -93,7 +93,7 @@ def test_fit(self):
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
-        task_info_b = SecureLGBMContext(args_b, passive_components)
+        task_info_b = SecureLGBMContext(args_b['task_id'], args_b, passive_components)
         model_data = SecureDataset.simulate_dataset(
             data_size, feature_dim, has_label=False)
         secure_dataset_b = SecureDataset(task_info_b, model_data)
diff --git a/python/ppc_model/secure_lr/test/test_secure_lr_training.py b/python/ppc_model/secure_lr/test/test_secure_lr_training.py
index 736f38f0..6492f38f 100644
--- a/python/ppc_model/secure_lr/test/test_secure_lr_training.py
+++ b/python/ppc_model/secure_lr/test/test_secure_lr_training.py
@@ -89,7 +89,7 @@ def test_fit(self):
         active_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/active', 'AGENCY_ID': ACTIVE_PARTY}
         active_components.mock_logger = MockLogger()
-        task_info_a = SecureLRContext(args_a, active_components)
+        task_info_a = SecureLRContext(args_a['task_id'], args_a, active_components)
 
         secure_dataset_a = SecureDataset(task_info_a, self.df_with_y)
         booster_a = VerticalLRActiveParty(task_info_a, secure_dataset_a)
@@ -107,7 +107,7 @@ def test_fit(self):
         passive_components.config_data = {
             'JOB_TEMP_DIR': '/tmp/passive', 'AGENCY_ID': PASSIVE_PARTY}
         passive_components.mock_logger = MockLogger()
-        task_info_b = SecureLRContext(args_b, passive_components)
+        task_info_b = SecureLRContext(args_b['task_id'], args_b, passive_components)
         secure_dataset_b = SecureDataset(task_info_b, self.df_without_y)
         booster_b = VerticalLRPassiveParty(task_info_b, secure_dataset_b)
         print(secure_dataset_b.feature_name)
diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py
index 8292d0cb..50ed9455 100644
--- a/python/ppc_model/secure_model_base/secure_model_context.py
+++ b/python/ppc_model/secure_model_base/secure_model_context.py
@@ -98,6 +98,7 @@ def get_all_params(self):
 
 class SecureModelContext(Context):
     def __init__(self,
+                 task_id,
                  args,
                  components: Initializer):
 
@@ -107,7 +108,7 @@ def __init__(self,
             role = TaskRole.PASSIVE_PARTY
 
         super().__init__(args['job_id'],
-                         args['task_id'],
+                         task_id,
                          components,
                          role)
         self.is_label_holder = args['is_label_holder']
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 92764d4a..995194ab 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -1,4 +1,5 @@
 import datetime
+import json
 import logging
 import os
 import threading
@@ -10,19 +11,99 @@
 from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
+from ppc_common.deps_services.sql_storage_api import SQLStorageAPI
 from ppc_model.common.protocol import ModelTask, TaskStatus, LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER
+from ppc_model.db.job_worker_record import JobWorkerRecord
+from ppc_model.log.log_retriever import LogRetriever
+
+
+class TaskResult:
+    def __init__(self, job_id, task_id):
+        self.task_status = TaskStatus.RUNNING.value
+        self.start_time = datetime.datetime.now()
+        self.job_id = job_id
+        self.task_id = task_id
+        self.diagnosis_msg = ""
+        self.exec_result = None
+
+    def get_timecost(self):
+        return (datetime.datetime.now() - self.start_time).total_seconds()
+
+    def finalize(self):
+        # generate the exec_result
+        exec_result_dict = {}
+        exec_result_dict.update({"timecost": self.get_timecost()})
+        exec_result_dict.update({"diagnosis_msg": self.diagnosis_msg})
+        self.exec_result = json.dumps(exec_result_dict)
+
+    def __repr__(self):
+        return f"job_id: {self.job_id}, task_id: {self.task_id}, " \
+               f"task_status: {self.task_status}, start_time: {self.start_time}"
+
+
+class TaskPersistent:
+    def __init__(self, logger, sql_storage: SQLStorageAPI):
+        self.logger = logger
+        self.sql_storage = sql_storage
+
+    def query_tasks(self, job_id) -> []:
+        task_recorder_list = self.sql_storage.query(JobWorkerRecord,
+                                                    JobWorkerRecord.job_id == job_id)
+        tasks = []
+        if task_recorder_list is None or task_recorder_list.count() == 0:
+            return tasks
+        self.logger.info(
+            f"query_tasks, job_id: {job_id}, count: {task_recorder_list.count()}")
+        for task in task_recorder_list:
+            tasks.append(task)
+        return tasks
+
+    def query_task(self, worker_id) -> JobWorkerRecord:
+        task = self.sql_storage.query(JobWorkerRecord,
+                                      JobWorkerRecord.worker_id == worker_id)
+        if task is None or task.count() == 0:
+            return None
+        return task.first()
+
+    def job_finished(self, job_id):
+        task_recorders = self.query_tasks(job_id)
+        if task_recorders is None:
+            return True
+        for task in task_recorders:
+            if task.status == TaskStatus.RUNNING.value or task.status == TaskStatus.PENDING.value:
+                return False
+        return True
+
+    def on_task_finished(self, task_result: TaskResult):
+        """
+        :param task_result: the task result
+        :return:
+        """
+        # query the task
+        worker_recorder = self.query_task(task_result.task_id)
+        if worker_recorder is None:
+            self.logger.warn(
+                f"TaskPersistent error, the task not found! task_result: {task_result}")
+            return
+        # update the task result
+        worker_recorder.status = task_result.task_status
+        worker_recorder.exec_result = task_result.exec_result
+        self.sql_storage.merge(worker_recorder)
 
 
 class TaskManager:
     def __init__(self, logger,
+                 task_persistent: TaskPersistent,
+                 log_retriever: LogRetriever,
                  thread_event_manager: ThreadEventManager,
                  task_timeout_h: Union[int, float]):
         self.logger = logger
+        self.task_persistent = task_persistent
+        self.log_retriever = log_retriever
         self._thread_event_manager = thread_event_manager
         self._task_timeout_s = task_timeout_h * 3600
         self._rw_lock = rwlock.RWLockWrite()
-        self._tasks: dict[str, list] = {}
-        self._jobs: dict[str, set] = {}
+        self._tasks: dict[str, TaskResult] = {}
         self._handlers = {}
         self._async_executor = AsyncThreadExecutor(
             event_manager=self._thread_event_manager, logger=logger)
@@ -46,17 +127,18 @@ def run_task(self, task_id: str, task_type: ModelTask, args=()):
         param args: 各任务参数
         """
         job_id = args[0]['job_id']
+        task_recorder = self.task_persistent.query_task(task_id)
+        if task_recorder is not None and task_recorder.status == TaskStatus.RUNNING.value:
+            self.logger.info(
+                f"Task already exists and Running, task_id: {task_id}")
+            return
         with self._rw_lock.gen_wlock():
             if task_id in self._tasks:
                 self.logger.info(
-                    f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id][0]}")
+                    f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id].task_id}")
                 return
-            self._tasks[task_id] = [TaskStatus.RUNNING.value,
-                                    datetime.datetime.now(), 0, args[0]['job_id']]
-            if job_id in self._jobs:
-                self._jobs[job_id].add(task_id)
-            else:
-                self._jobs[job_id] = {task_id}
+            self._tasks[task_id] = TaskResult(
+                job_id=args[0]['job_id'], task_id=task_id)
         self.logger.info(LOG_START_FLAG_FORMATTER.format(job_id=job_id))
         self.logger.info(f"Run task, job_id: {job_id}, task_id: {task_id}")
         self._async_executor.execute(
@@ -66,62 +148,81 @@ def kill_task(self, job_id: str):
         """
         终止任务
         """
-        task_ids = []
-        with self._rw_lock.gen_rlock():
-            if job_id not in self._jobs:
-                return
-            for task_id in self._jobs[job_id]:
-                task_ids.append(task_id)
-
-        for task_id in task_ids:
-            self.kill_one_task(task_id)
+        tasks = self.task_persistent.query_tasks(job_id)
+        for task in tasks:
+            self.kill_one_task(task.worker_id)
+        self.logger.info(LOG_END_FLAG_FORMATTER.format(
+            job_id=job_id))
+        # upload the log
+        self.logger.info(f"kill_task, job {job_id} killed, upload the log")
+        self.log_retriever.upload_log(job_id)
 
     def kill_one_task(self, task_id: str):
         with self._rw_lock.gen_rlock():
-            if task_id not in self._tasks or self._tasks[task_id][0] != TaskStatus.RUNNING.value:
+            if task_id not in self._tasks or self._tasks[task_id].task_status != TaskStatus.RUNNING.value:
                 return
 
         self.logger.info(f"Kill task, task_id: {task_id}")
         self._async_executor.kill(task_id)
 
+        # persistent the status to killed
         with self._rw_lock.gen_wlock():
-            self._tasks[task_id][0] = TaskStatus.FAILED.value
+            if task_id not in self._tasks.keys():
+                return
+            task_result = self._tasks[task_id]
+            task_result.task_status = TaskStatus.KILLED.value
+            self.task_persistent.on_task_finished(task_result)
+            self._tasks.pop(task_id)
 
     def task_finished(self, task_id: str) -> bool:
         (status, _, _) = self.status(task_id)
-        if status == TaskStatus.RUNNING.value:
+        if status == TaskStatus.RUNNING.value or status == TaskStatus.PENDING.value:
             return False
         return True
 
-    def status(self, task_id: str) -> [str, float, float]:
+    def status(self, task_id: str) -> [str, float, str]:
         """
         返回: 任务状态, 通讯量(MB), 执行耗时(s)
         """
-        with self._rw_lock.gen_rlock():
-            if task_id not in self._tasks:
-                raise PpcException(
-                    PpcErrorCode.TASK_NOT_FOUND.get_code(),
-                    PpcErrorCode.TASK_NOT_FOUND.get_msg())
-            status = self._tasks[task_id][0]
-            time_costs = self._tasks[task_id][2]
-            # TODO: the traffic_volume
-            return status, 0, time_costs
+        result = self.task_persistent.query_task(task_id)
+        if result is None:
+            return TaskStatus.NotFound.value, 0.0, None
+        return result.status, 0, result.exec_result
 
     def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None):
+        task_result = None
         with self._rw_lock.gen_wlock():
-            time_costs = (datetime.datetime.now() -
-                          self._tasks[task_id][1]).total_seconds()
-            self._tasks[task_id][2] = time_costs
-            if is_succeeded:
-                self._tasks[task_id][0] = TaskStatus.COMPLETED.value
-                self.logger.info(f"Task {task_id} completed, job_id: {self._tasks[task_id][3]}, "
-                                 f"time_costs: {time_costs}s")
-            else:
-                self._tasks[task_id][0] = TaskStatus.FAILED.value
-                self.logger.warn(f"Task {task_id} failed, job_id: {self._tasks[task_id][3]}, "
-                                 f"time_costs: {time_costs}s, error: {e}")
-            self.logger.info(LOG_END_FLAG_FORMATTER.format(
-                job_id=self._tasks[task_id][3]))
+            if task_id not in self._tasks.keys():
+                self.logger.warn(
+                    f"_on_task_finish: the task {task_id} not Found!")
+                return
+            task_result = self._tasks[task_id]
+        # update the task result
+        if is_succeeded:
+            task_result.task_status = TaskStatus.SUCCESS.value
+            self.logger.info(f"Task {task_id} completed, job_id: {task_result.job_id}, "
+                             f"time_costs: {task_result.get_timecost()}s")
+        else:
+            task_result.task_status = TaskStatus.FAILURE.value
+            task_result.diagnosis_msg = str(e)
+            self.logger.warn(f"Task {task_id} failed, job_id: {task_result.job_id}, "
+                             f"time_costs: {self._tasks[task_id].get_timecost()}s, error: {e}")
+
+        self.logger.info(LOG_END_FLAG_FORMATTER.format(
+            job_id=task_result.job_id))
+        # finalize the task result
+        task_result.finalize()
+        self.task_persistent.on_task_finished(task_result)
+
+        with self._rw_lock.gen_wlock():
+            # erase from the queue
+            self._tasks.pop(task_id)
+
+            # record and upload the log if all task finished
+        if self.task_persistent.job_finished(task_result.job_id):
+            self.logger.info(
+                f"_on_task_finish: all sub tasks finished, upload the log, task_info: {task_result}")
+            self.log_retriever.upload_log(task_result.job_id)
 
     def _loop_cleanup(self):
         while True:
@@ -154,36 +255,6 @@ def _cleanup_finished_tasks(self):
             for task_id, job_id in tasks_to_cleanup:
                 if task_id in self._tasks:
                     del self._tasks[task_id]
-                if job_id in self._jobs:
-                    del self._jobs[job_id]
                 self._thread_event_manager.remove_event(task_id)
                 self.logger.info(
                     f"Cleanup task cache, task_id: {task_id}, job_id: {job_id}")
-
-    def record_model_job_log(self, job_id):
-        log_file = self._get_log_file_path()
-        if log_file is None or log_file == "":
-            current_working_dir = os.getcwd()
-            relative_log_path = "logs/wedpr-model.log"
-            log_file = os.path.join(current_working_dir, relative_log_path)
-
-        start_keyword = LOG_START_FLAG_FORMATTER.format(job_id=job_id)
-        end_keyword = LOG_END_FLAG_FORMATTER.format(job_id=job_id)
-        with open(log_file, 'r') as file:
-            log_data = file.read()
-        start_index = log_data.find(start_keyword)
-        end_index = log_data.rfind(end_keyword)
-
-        if start_index == -1 or end_index == -1:
-            return f"{job_id} not found in log data"
-
-        end_index += len(end_keyword)
-        return log_data[start_index:end_index]
-
-    def _get_log_file_path(self):
-        log_file_path = None
-        for handler in self.logger.handlers:
-            if isinstance(handler, logging.FileHandler):
-                log_file_path = handler.baseFilename
-                break
-        return log_file_path

From 9e02df70d0d1538b869e5dfd0faf16a098c1c123 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 1 Nov 2024 17:10:18 +0800
Subject: [PATCH 075/120] support task-kill (#74)

---
 .../async_thread_executor.py                  | 24 ++++++++++++++++++-
 python/ppc_model/task/task_manager.py         | 12 +++++-----
 python/tools/fake_ml_train_data.py            |  4 ++--
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py
index 2c14c308..1d4ea7c3 100644
--- a/python/ppc_common/ppc_async_executor/async_thread_executor.py
+++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py
@@ -1,4 +1,6 @@
 import threading
+import inspect
+import ctypes
 import time
 import traceback
 from typing import Callable
@@ -37,15 +39,35 @@ def thread_target(logger, on_finish, *args):
         stop_event = threading.Event()
         self.event_manager.add_event(target_id, stop_event)
 
+    def _thread_exit_(self, thread, exit_type):
+        """Raises an exception in the threads with id tid"""
+        thread_id = thread.ident
+        if not inspect.isclass(exit_type):
+            raise TypeError("Only types can be raised (not instances)")
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
+            ctypes.c_long(thread_id), ctypes.py_object(exit_type))
+        if res == 0:
+            self.logger.warn("Invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, None)
+            self.logger.warn("PyThreadState_SetAsyncExc failed")
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+        self.logger.info(f"thread_exit success, thread_id: {thread_id}")
+
     def kill(self, target_id: str):
         with self.lock:
             if target_id not in self.threads:
                 return False
             else:
                 thread = self.threads[target_id]
-
         self.event_manager.set_event(target_id)
+        self._thread_exit_(thread, SystemExit)
         thread.join()
+        # clear the thread information
+        with self.lock:
+            self.threads.pop(target_id)
         self.logger.info(f"Target {target_id} has been stopped!")
         return True
 
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 995194ab..27dc1b11 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -158,21 +158,20 @@ def kill_task(self, job_id: str):
         self.log_retriever.upload_log(job_id)
 
     def kill_one_task(self, task_id: str):
+        task_result = None
         with self._rw_lock.gen_rlock():
-            if task_id not in self._tasks or self._tasks[task_id].task_status != TaskStatus.RUNNING.value:
+            if task_id not in self._tasks.keys() or self._tasks[task_id].task_status != TaskStatus.RUNNING.value:
                 return
-
+            task_result = self._tasks[task_id]
         self.logger.info(f"Kill task, task_id: {task_id}")
         self._async_executor.kill(task_id)
 
         # persistent the status to killed
         with self._rw_lock.gen_wlock():
-            if task_id not in self._tasks.keys():
-                return
-            task_result = self._tasks[task_id]
             task_result.task_status = TaskStatus.KILLED.value
             self.task_persistent.on_task_finished(task_result)
             self._tasks.pop(task_id)
+        self.logger.info(f"Kill task success, task_id: {task_id}")
 
     def task_finished(self, task_id: str) -> bool:
         (status, _, _) = self.status(task_id)
@@ -194,9 +193,10 @@ def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None)
         with self._rw_lock.gen_wlock():
             if task_id not in self._tasks.keys():
                 self.logger.warn(
-                    f"_on_task_finish: the task {task_id} not Found!")
+                    f"_on_task_finish: the task {task_id} not Found! maybe killed!")
                 return
             task_result = self._tasks[task_id]
+
         # update the task result
         if is_succeeded:
             task_result.task_status = TaskStatus.SUCCESS.value
diff --git a/python/tools/fake_ml_train_data.py b/python/tools/fake_ml_train_data.py
index 65a152d6..c0b145e0 100644
--- a/python/tools/fake_ml_train_data.py
+++ b/python/tools/fake_ml_train_data.py
@@ -33,7 +33,7 @@ def parse_args():
     parser.add_argument("-f", '--feature_size',
                         help='the feature size', required=True)
     parser.add_argument("-s", '--sample_capacity',
-                        help='the faked data size(in GB, default 1GB)', default=1, required=False)
+                        help='the faked data size(in MB, default 1MB)', default=1, required=False)
     parser.add_argument("-d", '--id_file',
                         help='the id file', required=False)
     parser.add_argument("-S", '--sample_file',
@@ -108,7 +108,7 @@ def fake_data(args):
     data_type = DataType.value_of(args.data_type)
     sample_capacity_bytes = None
     if args.sample_capacity is not None:
-        sample_capacity_bytes = int(args.sample_capacity) * 1024
+        sample_capacity_bytes = int(args.sample_capacity) * 1024 * 1024
     feature_size = int(args.feature_size)
     granularity = 100
     id = 0

From 419522bed4608afe520f7a459c9a4c6777ebf76f Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 1 Nov 2024 17:43:03 +0800
Subject: [PATCH 076/120] support only fetch log (#76)

---
 .../model_result/task_result_handler.py       | 21 +++++++++++--------
 .../network/http/model_controller.py          |  5 ++++-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py
index ba85b616..0a707504 100644
--- a/python/ppc_model/model_result/task_result_handler.py
+++ b/python/ppc_model/model_result/task_result_handler.py
@@ -12,9 +12,10 @@
 
 
 class TaskResultRequest:
-    def __init__(self, job_id, task_type):
+    def __init__(self, job_id, task_type, only_fetch_log):
         self.job_id = job_id
         self.task_type = task_type
+        self.only_fetch_log = only_fetch_log
 
 
 class DataType(Enum):
@@ -336,14 +337,16 @@ def __init__(self, task_result_request: TaskResultRequest, components: Initializ
         self._get_feature_processing_result()
 
     def get_response(self):
-        merged_result = dict()
-        for result in self.result_list:
-            merged_result.update(result.to_dict())
-        if self.model_data is None:
-            response = {"jobPlanetResult":  merged_result}
-        else:
-            response = {"jobPlanetResult":  merged_result,
-                        "modelData": self.model_data}
+        response = dict()
+        if not self.task_result_request.only_fetch_log:
+            merged_result = dict()
+            for result in self.result_list:
+                merged_result.update(result.to_dict())
+            if self.model_data is None:
+                response = {"jobPlanetResult":  merged_result}
+            else:
+                response = {"jobPlanetResult":  merged_result,
+                            "modelData": self.model_data}
         # record the log
         log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
             self.task_result_request.job_id)
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index 6e985b47..89f69e1d 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -77,9 +77,12 @@ def post(self, task_id):
             f"run task request, task_id: {task_id}, args: {args}")
         user_name = args['user']
         task_type = args['jobType']
+        only_fetch_log = {'True': True, 'False': False}.get(
+            args['onlyFetchLog'])
         components.logger().info(
             f"get_job_direct_result_response, job: {task_id}")
-        task_result_request = TaskResultRequest(task_id, task_type)
+        task_result_request = TaskResultRequest(
+            task_id, task_type, only_fetch_log)
         job_result_handler = TaskResultHandler(
             task_result_request=task_result_request, components=components)
         response = job_result_handler.get_response()

From 7d4be4b59d0e3dcbf3a3e1b4950bb88f1c66bc80 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 1 Nov 2024 18:45:23 +0800
Subject: [PATCH 077/120] optimize log (#77)

---
 python/ppc_common/ppc_async_executor/async_executor.py |  1 +
 .../ppc_async_executor/async_thread_executor.py        | 10 +++++++---
 .../ppc_async_executor/test/async_executor_unittest.py |  4 ++--
 python/ppc_model/metrics/evaluation.py                 |  2 +-
 python/ppc_model/metrics/model_plot.py                 |  2 +-
 python/ppc_model/network/http/model_controller.py      |  4 ++--
 .../secure_lgbm/monitor/evaluation_monitor.py          |  2 +-
 python/ppc_model/task/task_manager.py                  |  2 +-
 8 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/python/ppc_common/ppc_async_executor/async_executor.py b/python/ppc_common/ppc_async_executor/async_executor.py
index 50b62b7c..3b151b3e 100644
--- a/python/ppc_common/ppc_async_executor/async_executor.py
+++ b/python/ppc_common/ppc_async_executor/async_executor.py
@@ -4,6 +4,7 @@
 
 class AsyncExecutor(ABC):
     def execute(self,
+                job_id: str,
                 task_id: str,
                 target: Callable,
                 on_target_finish: Callable[[str, bool, Exception], None],
diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py
index 1d4ea7c3..2975f9fe 100644
--- a/python/ppc_common/ppc_async_executor/async_thread_executor.py
+++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py
@@ -19,18 +19,22 @@ def __init__(self, event_manager: ThreadEventManager, logger):
         self._cleanup_thread.daemon = True
         self._cleanup_thread.start()
 
-    def execute(self, target_id: str, target: Callable, on_target_finish: Callable[[str, bool, Exception], None],
+    def execute(self, job_id: str, target_id: str,
+                target: Callable, on_target_finish: Callable[[str, bool, Exception], None],
                 args=()):
         def thread_target(logger, on_finish, *args):
             try:
                 target(target_id, *args)
                 on_finish(target_id, True)
             except Exception as e:
-                logger.warn(traceback.format_exc())
+                error_detail = traceback.format_exc().replace('\n',
+                                                              f'\n{job_id}.{target_id}')
+                logger.warn(
+                    f"Execute task: {target_id} error, job: {job_id}, error: {e}, traceback: {error_detail}")
                 on_finish(target_id, False, e)
 
         thread = threading.Thread(target=thread_target, args=(
-            self.logger, on_target_finish,) + args)
+            self.logger, on_target_finish) + args)
         thread.start()
 
         with self.lock:
diff --git a/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py b/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py
index f4a91e7b..b59122ab 100644
--- a/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py
+++ b/python/ppc_common/ppc_async_executor/test/async_executor_unittest.py
@@ -88,9 +88,9 @@ def test_kill(self):
         task_id_1 = '0xaa'
         task_id_2 = '0xbb'
 
-        executor.execute(task_id_1, thread_task1, on_target_finish,
+        executor.execute("job1", task_id_1, thread_task1, on_target_finish,
                          (shared_status1, task_id_1, key, 1))
-        executor.execute(task_id_2, thread_task2,
+        executor.execute("job2", task_id_2, thread_task2,
                          on_target_finish, (shared_status2, key, 1))
 
         time.sleep(2)
diff --git a/python/ppc_model/metrics/evaluation.py b/python/ppc_model/metrics/evaluation.py
index b2f5dac2..33802c7c 100644
--- a/python/ppc_model/metrics/evaluation.py
+++ b/python/ppc_model/metrics/evaluation.py
@@ -163,7 +163,7 @@ def evaluation_file(self, ctx, data_index: np.ndarray,
                         f'y_true = {len(y_true)}, {y_true[0:2]}')
                     ctx.components.logger().info(
                         f'y_praba = {len(y_praba)}, {y_praba[0:2]}')
-                    err = traceback.format_exc()
+                    err = traceback.format_exc().replace('\n', ' ; error: ')
                     # ctx.components.logger().exception(err)
                     ctx.components.logger().info(
                         f'plot metrics in times-{retry_num} failed, traceback: {err}.')
diff --git a/python/ppc_model/metrics/model_plot.py b/python/ppc_model/metrics/model_plot.py
index dd3632a2..976d42ab 100644
--- a/python/ppc_model/metrics/model_plot.py
+++ b/python/ppc_model/metrics/model_plot.py
@@ -56,7 +56,7 @@ def plot_tree(self):
                         self.ctx.components.logger().info(
                             f'tree_id = {i}, tree = {tree}')
                         self.ctx.components.logger().info(f'G = {self._G}')
-                        err = traceback.format_exc()
+                        err = traceback.format_exc().replace('\n', ' ; error: ')
                         # self.ctx.components.logger().exception(err)
                         self.ctx.components.logger().info(
                             f'plot tree-{i} in times-{retry_num} failed, traceback: {err}.')
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index 89f69e1d..0c565c0d 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -31,7 +31,7 @@ def post(self, model_id):
         args = request.get_json()
         task_id = model_id
         components.logger().info(
-            f"run task request, task_id: {task_id}, args: {args}")
+            f"run task request, task_id: {task_id}")
         task_type = args['task_type']
         components.task_manager.run_task(
             task_id, ModelTask(task_type), (args,))
@@ -74,7 +74,7 @@ def post(self, task_id):
         start_t = time.time()
         args = request.get_json()
         components.logger().info(
-            f"run task request, task_id: {task_id}, args: {args}")
+            f"get task result, task_id: {task_id}, args: {args}")
         user_name = args['user']
         task_type = args['jobType']
         only_fetch_log = {'True': True, 'False': False}.get(
diff --git a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
index edda3ba8..29787b1e 100644
--- a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
+++ b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
@@ -113,7 +113,7 @@ def after_training(self, model: _Model) -> _Model:
             except:
                 self.logger.info(f'scores = {model.get_history()}')
                 self.logger.info(f'path = {model.get_workspace()}')
-                err = traceback.format_exc()
+                err = traceback.format_exc().replace('\n', ' ; error ')
                 # self.logger.exception(err)
                 self.logger.info(
                     f'plot moniter in times-{retry_num} failed, traceback: {err}.')
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 27dc1b11..4dd23084 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -142,7 +142,7 @@ def run_task(self, task_id: str, task_type: ModelTask, args=()):
         self.logger.info(LOG_START_FLAG_FORMATTER.format(job_id=job_id))
         self.logger.info(f"Run task, job_id: {job_id}, task_id: {task_id}")
         self._async_executor.execute(
-            task_id, self._handlers[task_type.value], self._on_task_finish, args)
+            args[0]['job_id'], task_id, self._handlers[task_type.value], self._on_task_finish, args)
 
     def kill_task(self, job_id: str):
         """

From 9f9118bd2fa7afddb08c4f3157ef894f2a7c01c8 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 4 Nov 2024 09:42:14 +0800
Subject: [PATCH 078/120] optimize task manager log (#78)

---
 .../ppc_common/ppc_async_executor/async_thread_executor.py  | 2 +-
 python/ppc_model/task/task_manager.py                       | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/ppc_common/ppc_async_executor/async_thread_executor.py b/python/ppc_common/ppc_async_executor/async_thread_executor.py
index 2975f9fe..04fa1756 100644
--- a/python/ppc_common/ppc_async_executor/async_thread_executor.py
+++ b/python/ppc_common/ppc_async_executor/async_thread_executor.py
@@ -31,7 +31,7 @@ def thread_target(logger, on_finish, *args):
                                                               f'\n{job_id}.{target_id}')
                 logger.warn(
                     f"Execute task: {target_id} error, job: {job_id}, error: {e}, traceback: {error_detail}")
-                on_finish(target_id, False, e)
+                on_finish(target_id, False, error_detail)
 
         thread = threading.Thread(target=thread_target, args=(
             self.logger, on_target_finish) + args)
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 4dd23084..abc817c8 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -188,7 +188,7 @@ def status(self, task_id: str) -> [str, float, str]:
             return TaskStatus.NotFound.value, 0.0, None
         return result.status, 0, result.exec_result
 
-    def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None):
+    def _on_task_finish(self, task_id: str, is_succeeded: bool, error_msg: str = None):
         task_result = None
         with self._rw_lock.gen_wlock():
             if task_id not in self._tasks.keys():
@@ -204,9 +204,9 @@ def _on_task_finish(self, task_id: str, is_succeeded: bool, e: Exception = None)
                              f"time_costs: {task_result.get_timecost()}s")
         else:
             task_result.task_status = TaskStatus.FAILURE.value
-            task_result.diagnosis_msg = str(e)
+            task_result.diagnosis_msg = error_msg
             self.logger.warn(f"Task {task_id} failed, job_id: {task_result.job_id}, "
-                             f"time_costs: {self._tasks[task_id].get_timecost()}s, error: {e}")
+                             f"time_costs: {self._tasks[task_id].get_timecost()}s, error: {error_msg}")
 
         self.logger.info(LOG_END_FLAG_FORMATTER.format(
             job_id=task_result.job_id))

From 937ea69ac621902fc61b4335793b118ccb0e0ad7 Mon Sep 17 00:00:00 2001
From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com>
Date: Mon, 4 Nov 2024 11:15:46 +0800
Subject: [PATCH 079/120] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=93=E5=AE=B6?=
 =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=BB=A3=E7=A0=81=E5=92=8C=E6=B5=8B=E8=AF=95?=
 =?UTF-8?q?=E7=94=A8=E4=BE=8B=20(#72)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* update secure lr

* update model and predict

* update ppc_dev

* update model setting

* Update booster.py

* update wedpr_ml_toolkit

* update predict feature selection

* update jupyter task

* update test ml toolkit
---
 python/wedpr_ml_toolkit/test/UserGuide.md     |  42 +++
 .../wedpr_ml_toolkit/test/config.properties   |   1 -
 .../wedpr_ml_toolkit/test/test_dataset.ipynb  | 209 ++++++++++++
 python/wedpr_ml_toolkit/test/test_psi.ipynb   | 142 ++++++++
 .../wedpr_ml_toolkit/test/test_xgboost.ipynb  | 311 ++++++++++++++++++
 .../config/wedpr_ml_config.py                 |   7 +
 .../wedpr_ml_toolkit/context/data_context.py  |  27 +-
 .../wedpr_ml_toolkit/context/job_context.py   | 224 +++++++++++--
 .../toolkit/dataset_toolkit.py                |   9 +-
 .../transport/storage_entrypoint.py           |   4 +-
 .../transport/wedpr_remote_job_client.py      |  43 ++-
 .../wedpr_ml_toolkit/wedpr_ml_toolkit.py      |  32 +-
 12 files changed, 991 insertions(+), 60 deletions(-)
 create mode 100644 python/wedpr_ml_toolkit/test/UserGuide.md
 create mode 100644 python/wedpr_ml_toolkit/test/test_dataset.ipynb
 create mode 100644 python/wedpr_ml_toolkit/test/test_psi.ipynb
 create mode 100644 python/wedpr_ml_toolkit/test/test_xgboost.ipynb

diff --git a/python/wedpr_ml_toolkit/test/UserGuide.md b/python/wedpr_ml_toolkit/test/UserGuide.md
new file mode 100644
index 00000000..47a72092
--- /dev/null
+++ b/python/wedpr_ml_toolkit/test/UserGuide.md
@@ -0,0 +1,42 @@
+# wedpr专家模式用户手册
+
+## 配置
+
+1. 左侧用户目录中新建配置文件，文件命名为：config.properties
+2. 配置信息参考：
+
+```
+access_key_id=
+access_key_secret=
+remote_entrypoints=http://139.159.202.235:8005,http://139.159.202.235:8006
+
+agency_name=SGD
+workspace_path=/user/ppc/milestone2/sgd/
+user=test_user
+storage_endpoint=http://192.168.0.18:50070
+```
+
+3. 通过前端页面登录，例如：http://139.159.202.235:8005/
+4. 创建个人项目空间，通过【打开jupyter】按钮进入专家模式
+
+## 基础功能
+
+1. 支持通过launcher启动python，jupyter，终端，文本编辑等功能
+2. 支持在用户目录空间创建/修改/删除配置文件，文本文件，bash，python notebook等格式文件
+3. 通过launcher启动python，jupyter，终端后可以正常执行对应的代码功能
+
+## hdfs数据功能
+
+1. 支持注册dataset，支持两种方式: pd.Dataframe, hdfs_path
+2. 支持更新dataset
+
+* 详细使用说明参考示例文件：【test_dataset.ipynb】
+
+## wedpr任务功能
+
+1. 支持配置任务参数
+2. 支持提交psi，建模训练，预测等任务
+3. 支持获取任务结果
+4. 支持对任务结果进行明文处理
+
+* 详细使用说明参考示例文件：【test_psi.ipynb】和【test_xgboost.ipynb】
diff --git a/python/wedpr_ml_toolkit/test/config.properties b/python/wedpr_ml_toolkit/test/config.properties
index 358d170d..ac88fcad 100644
--- a/python/wedpr_ml_toolkit/test/config.properties
+++ b/python/wedpr_ml_toolkit/test/config.properties
@@ -6,4 +6,3 @@ agency_name=SGD
 workspace_path=/user/wedpr/milestone2/sgd/
 user=test_user
 storage_endpoint=http://127.0.0.1:50070
-
diff --git a/python/wedpr_ml_toolkit/test/test_dataset.ipynb b/python/wedpr_ml_toolkit/test/test_dataset.ipynb
new file mode 100644
index 00000000..e6fcef07
--- /dev/null
+++ b/python/wedpr_ml_toolkit/test/test_dataset.ipynb
@@ -0,0 +1,209 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['/usr/lib/python3/dist-packages/wedpr_ml_toolkit/', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python\\\\wedpr_ml_toolkit', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\python38.zip', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\DLLs', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib', 'c:\\\\Users\\\\yanxi\\\\anaconda3', '', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\win32', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\win32\\\\lib', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\Pythonwin', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\IPython\\\\extensions', 'C:\\\\Users\\\\yanxi\\\\.ipython']\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
+    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
+    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 读取配置文件\n",
+    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
+    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "http://139.159.202.235:50070 /user/ppc/milestone2/sgd/test_user SGD\n",
+      "/user/ppc/milestone2/sgd/test_user\\d-101\n",
+      "   id  y        x1        x2        x3        x4        x5        x6  \\\n",
+      "0   0  1  0.954183  0.652034  0.704070  0.180889  0.025025  0.511596   \n",
+      "1   1  1  0.302088  0.462222  0.435542  0.029966  0.931294  0.848483   \n",
+      "2   2  1  0.468104  0.430161  0.239322  0.588153  0.470668  0.225856   \n",
+      "3   3  0  0.152269  0.811666  0.834451  0.354288  0.635447  0.062092   \n",
+      "4   4  0  0.841470  0.800512  0.451507  0.118651  0.748845  0.557916   \n",
+      "\n",
+      "         x7        x8        x9       x10  \n",
+      "0  0.529848  0.759689  0.159081  0.556419  \n",
+      "1  0.962787  0.224096  0.464418  0.208487  \n",
+      "2  0.564879  0.730366  0.394245  0.299081  \n",
+      "3  0.424057  0.202234  0.577448  0.636958  \n",
+      "4  0.030906  0.514350  0.340864  0.123303  \n"
+     ]
+    }
+   ],
+   "source": [
+    "# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path\n",
+    "# 1. pd.Dataframe\n",
+    "df = pd.DataFrame({\n",
+    "    'id': np.arange(0, 100),  # id列，顺序整数\n",
+    "    'y': np.random.randint(0, 2, size=100),\n",
+    "    # x1到x10列，随机数\n",
+    "    **{f'x{i}': np.random.rand(100) for i in range(1, 11)}\n",
+    "})\n",
+    "\n",
+    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
+    "                          dataset_owner='flyhuang1',\n",
+    "                          agency=wedpr_config.user_config.agency_name,\n",
+    "                          values=df,\n",
+    "                          is_label_holder=True)\n",
+    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
+    "dataset1.storage_client = None  # 本地测试时跳过hdfs上传/下载过程\n",
+    "dataset1.save_values(path='d-101')\n",
+    "print(dataset1.dataset_path)\n",
+    "print(dataset1.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "http://139.159.202.235:50070 /user/ppc/milestone2/sgd/test_user WeBank\n",
+      "/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\n",
+      "/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\n",
+      "   id        z1        z2        z3        z4        z5        z6        z7  \\\n",
+      "0   0  0.597205  0.942475  0.886443  0.560584  0.254432  0.370152  0.076031   \n",
+      "1   1  0.778616  0.607374  0.616211  0.602282  0.385989  0.816963  0.756814   \n",
+      "2   2  0.999795  0.596794  0.240741  0.241070  0.857676  0.342412  0.066459   \n",
+      "3   3  0.968410  0.895163  0.636140  0.978791  0.237098  0.095272  0.938806   \n",
+      "4   4  0.921513  0.454901  0.004514  0.769216  0.627185  0.676253  0.184952   \n",
+      "\n",
+      "         z8        z9       z10  \n",
+      "0  0.587627  0.851390  0.864929  \n",
+      "1  0.661537  0.865674  0.050091  \n",
+      "2  0.473916  0.080120  0.477873  \n",
+      "3  0.452399  0.953515  0.405465  \n",
+      "4  0.877475  0.316322  0.139290  \n"
+     ]
+    }
+   ],
+   "source": [
+    "# 2. hdfs_path\n",
+    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
+    "                          dataset_owner='flyhuang',\n",
+    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
+    "                          agency=\"WeBank\")\n",
+    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
+    "print(dataset2.dataset_path)\n",
+    "dataset2.storage_client = None  # 本地测试时跳过hdfs上传/下载过程\n",
+    "\n",
+    "# 提供本地测试数据\n",
+    "if dataset2.storage_client is None:\n",
+    "    # 支持更新dataset的values数据\n",
+    "    df2 = pd.DataFrame({\n",
+    "        'id': np.arange(0, 100),  # id列，顺序整数\n",
+    "        **{f'z{i}': np.random.rand(100) for i in range(1, 11)}  # x1到x10列，随机数\n",
+    "    })\n",
+    "    dataset2.update_values(values=df2)\n",
+    "    dataset2.save_values()\n",
+    "    print(dataset2.dataset_path)\n",
+    "    print(dataset2.values.head())\n",
+    "\n",
+    "# 对于己方数据集支持load_values，其他方数据集无需load_values，可直接使用\n",
+    "if dataset2.storage_client is not None:\n",
+    "    # 仅支持load本机构hdfs的数据集\n",
+    "    dataset2.load_values(header=0)\n",
+    "    print(dataset2.dataset_path)\n",
+    "    print(dataset2.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/user/ppc/milestone2/sgd/test_user\\d-101\n",
+      "   id  y        x1        x2        x3        x4        x5        x6  \\\n",
+      "0   0  1  0.954183  0.652034  0.704070  0.180889  0.025025  0.511596   \n",
+      "1   1  1  0.302088  0.462222  0.435542  0.029966  0.931294  0.848483   \n",
+      "2   2  1  0.468104  0.430161  0.239322  0.588153  0.470668  0.225856   \n",
+      "3   3  0  0.152269  0.811666  0.834451  0.354288  0.635447  0.062092   \n",
+      "4   4  0  0.841470  0.800512  0.451507  0.118651  0.748845  0.557916   \n",
+      "\n",
+      "         x7        x8        x9       x10  \n",
+      "0  0.529848  0.759689  0.159081  0.556419  \n",
+      "1  0.962787  0.224096  0.464418  0.208487  \n",
+      "2  0.564879  0.730366  0.394245  0.299081  \n",
+      "3  0.424057  0.202234  0.577448  0.636958  \n",
+      "4  0.030906  0.514350  0.340864  0.123303  \n"
+     ]
+    }
+   ],
+   "source": [
+    "# 更新数据集\n",
+    "if dataset1.storage_client is not None:\n",
+    "    dataset1.update_values(\n",
+    "        path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485')\n",
+    "    dataset1.load_values(header=0)\n",
+    "print(dataset1.dataset_path)\n",
+    "print(dataset1.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/python/wedpr_ml_toolkit/test/test_psi.ipynb b/python/wedpr_ml_toolkit/test/test_psi.ipynb
new file mode 100644
index 00000000..435b3586
--- /dev/null
+++ b/python/wedpr_ml_toolkit/test/test_psi.ipynb
@@ -0,0 +1,142 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
+    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
+    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit\n",
+    "from wedpr_ml_toolkit.context.data_context import DataContext\n",
+    "from wedpr_ml_toolkit.context.job_context import JobType"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 读取配置文件\n",
+    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
+    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 注册 dataset1\n",
+    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
+    "                          dataset_owner='flyhuang1',\n",
+    "                          agency=wedpr_config.user_config.agency_name,\n",
+    "                          dataset_id = 'd-9606704699156485',\n",
+    "                          dataset_path=\"/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485\",\n",
+    "                          is_label_holder=True)\n",
+    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
+    "dataset1.load_values(header=0)\n",
+    "print(dataset1.dataset_path)\n",
+    "print(dataset1.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 注册 dataset2\n",
+    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
+    "                          dataset_owner='flyhuang',\n",
+    "                          dataset_id = 'd-9606695119693829',\n",
+    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
+    "                          agency=\"WeBank\")\n",
+    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
+    "dataset2.load_values(header=0)\n",
+    "print(dataset2.dataset_path)\n",
+    "print(dataset2.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 构建 dataset context\n",
+    "dataset = DataContext(dataset1, dataset2)\n",
+    "print(dataset.datasets)\n",
+    "\n",
+    "# init the job context\n",
+    "project_id = \"9606702107011078\"\n",
+    "\n",
+    "# 构造psi任务配置\n",
+    "psi_job_context = wedpr_ml_toolkit.build_job_context(\n",
+    "    JobType.PSI, project_id, dataset, None, \"id\")\n",
+    "print(psi_job_context.participant_id_list, psi_job_context.result_receiver_id_list)\n",
+    "print(psi_job_context.project_id)\n",
+    "\n",
+    "psi_job_param = psi_job_context.build()\n",
+    "print(psi_job_param.taskParties)\n",
+    "print(psi_job_param.datasetList)\n",
+    "print(psi_job_param.job)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 执行psi任务\n",
+    "# psi_job_id = '9670241574201350'  # 测试时跳过创建新任务过程\n",
+    "psi_job_id = psi_job_context.submit()\n",
+    "print(psi_job_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 获取psi任务结果\n",
+    "# psi_job_id = '9670241574201350'  # 测试时跳过创建新任务过程\n",
+    "print(psi_job_id)\n",
+    "psi_result = psi_job_context.parse_result(psi_job_id, True)\n",
+    "psi_result.load_values()\n",
+    "print(psi_result.values.shape)\n",
+    "print(psi_result.values.head())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/python/wedpr_ml_toolkit/test/test_xgboost.ipynb b/python/wedpr_ml_toolkit/test/test_xgboost.ipynb
new file mode 100644
index 00000000..84082084
--- /dev/null
+++ b/python/wedpr_ml_toolkit/test/test_xgboost.ipynb
@@ -0,0 +1,311 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
+    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
+    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit\n",
+    "from wedpr_ml_toolkit.context.data_context import DataContext\n",
+    "from wedpr_ml_toolkit.context.job_context import JobType"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 读取配置文件\n",
+    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
+    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 注册 dataset1\n",
+    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
+    "                          dataset_owner='flyhuang1',\n",
+    "                          agency=wedpr_config.user_config.agency_name,\n",
+    "                          dataset_id = 'd-9606704699156485',\n",
+    "                          dataset_path=\"/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485\",\n",
+    "                          is_label_holder=True)\n",
+    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
+    "dataset1.load_values(header=0)\n",
+    "print(dataset1.dataset_path)\n",
+    "print(dataset1.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 注册 dataset2\n",
+    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
+    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
+    "                          dataset_owner='flyhuang',\n",
+    "                          dataset_id = 'd-9606695119693829',\n",
+    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
+    "                          agency=\"WeBank\")\n",
+    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
+    "dataset2.load_values(header=0)\n",
+    "print(dataset2.dataset_path)\n",
+    "print(dataset2.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 构建 dataset context\n",
+    "dataset = DataContext(dataset1, dataset2)\n",
+    "print(dataset.datasets)\n",
+    "\n",
+    "# init the job context\n",
+    "project_id = \"9606702107011078\"\n",
+    "\n",
+    "# 构造xgb任务配置\n",
+    "model_setting = {'use_psi': 0, 'fillna': 0, 'na_select': 1, 'filloutlier': 0, 'normalized': 0, 'standardized': 0, 'categorical': '', 'psi_select_col': '', 'psi_select_base': '', 'psi_select_thresh': 0.3, 'psi_select_bins': 4, 'corr_select': 0, 'use_iv': 0, 'group_num': 4, 'iv_thresh': 0.1, 'use_goss': 0, 'test_dataset_percentage': 0.3, 'learning_rate': 0.1, 'num_trees': 6, 'max_depth': 3, 'max_bin': 4, 'silent': 0, 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'reg_alpha': 0, 'reg_lambda': 1, 'gamma': 0, 'min_child_weight': 0, 'min_child_samples': 10, 'seed': 2024, 'early_stopping_rounds': 0, 'eval_metric': 'auc', 'verbose_eval': 1, 'eval_set_column': '', 'train_set_value': '', 'eval_set_value': '', 'train_features': ''}\n",
+    "\n",
+    "xgb_job_context = wedpr_ml_toolkit.build_job_context(\n",
+    "    JobType.XGB_TRAINING, project_id, dataset, model_setting, \"id\")\n",
+    "print(xgb_job_context.participant_id_list, xgb_job_context.result_receiver_id_list)\n",
+    "print(xgb_job_context.project_id)\n",
+    "\n",
+    "xgb_job_param = xgb_job_context.build()\n",
+    "print(xgb_job_param.taskParties)\n",
+    "print(xgb_job_param.datasetList)\n",
+    "print(xgb_job_param.job)\n",
+    "# import json\n",
+    "# print(json.dumps(xgb_job_param.__dict__))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 执行xgb任务\n",
+    "# xgb_job_id = '9707983191943174'  # 测试时跳过创建新任务过程\n",
+    "xgb_job_id = xgb_job_context.submit()\n",
+    "print(xgb_job_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 获取xgb任务结果\n",
+    "# xgb_job_id = '9707983191943174'  # 测试时跳过创建新任务过程\n",
+    "print(xgb_job_id)\n",
+    "xgb_result = xgb_job_context.parse_result(xgb_job_id, True)\n",
+    "xgb_result.train_result.load_values(header = 0)\n",
+    "xgb_result.test_result.load_values(header = 0)\n",
+    "print(xgb_result.train_result.values.head())\n",
+    "print(xgb_result.test_result.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 明文处理预测结果\n",
+    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "data = xgb_result.test_result.values\n",
+    "\n",
+    "# 提取真实标签和预测概率\n",
+    "y_true = data['class_label']\n",
+    "y_pred_proba = data['class_pred']\n",
+    "y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5\n",
+    "\n",
+    "# 计算评估指标\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "precision = precision_score(y_true, y_pred)\n",
+    "recall = recall_score(y_true, y_pred)\n",
+    "f1 = f1_score(y_true, y_pred)\n",
+    "auc = roc_auc_score(y_true, y_pred_proba)\n",
+    "\n",
+    "print(f\"Accuracy: {accuracy:.2f}\")\n",
+    "print(f\"Precision: {precision:.2f}\")\n",
+    "print(f\"Recall: {recall:.2f}\")\n",
+    "print(f\"F1 Score: {f1:.2f}\")\n",
+    "print(f\"AUC: {auc:.2f}\")\n",
+    "\n",
+    "# ROC 曲线\n",
+    "fpr, tpr, _ = roc_curve(y_true, y_pred_proba)\n",
+    "plt.figure(figsize=(12, 5))\n",
+    "\n",
+    "# ROC 曲线\n",
+    "plt.subplot(1, 2, 1)\n",
+    "plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')\n",
+    "plt.plot([0, 1], [0, 1], 'k--')\n",
+    "plt.xlabel('False Positive Rate')\n",
+    "plt.ylabel('True Positive Rate')\n",
+    "plt.title('ROC Curve')\n",
+    "plt.legend()\n",
+    "\n",
+    "# 精确率-召回率曲线\n",
+    "precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)\n",
+    "plt.subplot(1, 2, 2)\n",
+    "plt.plot(recall_vals, precision_vals)\n",
+    "plt.xlabel('Recall')\n",
+    "plt.ylabel('Precision')\n",
+    "plt.title('Precision-Recall Curve')\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 构造xgb预测任务配置\n",
+    "model_setting = {'use_psi': 0, 'use_iv': 0}\n",
+    "\n",
+    "xgb_job_context = wedpr_ml_toolkit.build_job_context(\n",
+    "    JobType.XGB_PREDICTING, project_id, dataset, model_setting, \"id\", xgb_result.model)\n",
+    "print(xgb_job_context.participant_id_list, xgb_job_context.result_receiver_id_list)\n",
+    "print(xgb_job_context.project_id)\n",
+    "\n",
+    "xgb_job_param = xgb_job_context.build()\n",
+    "print(xgb_job_param.taskParties)\n",
+    "print(xgb_job_param.datasetList)\n",
+    "# import json\n",
+    "# print(json.dumps(xgb_job_param.__dict__))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 执行xgb预测任务\n",
+    "# xgb_job_id = '9708824062994438'  # 测试时跳过创建新任务过程\n",
+    "xgb_job_id = xgb_job_context.submit()\n",
+    "print(xgb_job_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 获取xgb预测任务结果\n",
+    "# xgb_job_id = '9708824062994438'  # 测试时跳过创建新任务过程\n",
+    "print(xgb_job_id)\n",
+    "xgb_result = xgb_job_context.parse_result(xgb_job_id, True)\n",
+    "xgb_result.test_result.load_values(header = 0)\n",
+    "print(xgb_result.test_result.values.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 明文处理预测结果\n",
+    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "data = xgb_result.test_result.values\n",
+    "\n",
+    "# 提取真实标签和预测概率\n",
+    "y_true = data['class_label']\n",
+    "y_pred_proba = data['class_pred']\n",
+    "y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5\n",
+    "\n",
+    "# 计算评估指标\n",
+    "accuracy = accuracy_score(y_true, y_pred)\n",
+    "precision = precision_score(y_true, y_pred)\n",
+    "recall = recall_score(y_true, y_pred)\n",
+    "f1 = f1_score(y_true, y_pred)\n",
+    "auc = roc_auc_score(y_true, y_pred_proba)\n",
+    "\n",
+    "print(f\"Accuracy: {accuracy:.2f}\")\n",
+    "print(f\"Precision: {precision:.2f}\")\n",
+    "print(f\"Recall: {recall:.2f}\")\n",
+    "print(f\"F1 Score: {f1:.2f}\")\n",
+    "print(f\"AUC: {auc:.2f}\")\n",
+    "\n",
+    "# ROC 曲线\n",
+    "fpr, tpr, _ = roc_curve(y_true, y_pred_proba)\n",
+    "plt.figure(figsize=(12, 5))\n",
+    "\n",
+    "# ROC 曲线\n",
+    "plt.subplot(1, 2, 1)\n",
+    "plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')\n",
+    "plt.plot([0, 1], [0, 1], 'k--')\n",
+    "plt.xlabel('False Positive Rate')\n",
+    "plt.ylabel('True Positive Rate')\n",
+    "plt.title('ROC Curve')\n",
+    "plt.legend()\n",
+    "\n",
+    "# 精确率-召回率曲线\n",
+    "precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)\n",
+    "plt.subplot(1, 2, 2)\n",
+    "plt.plot(recall_vals, precision_vals)\n",
+    "plt.xlabel('Recall')\n",
+    "plt.ylabel('Precision')\n",
+    "plt.title('Precision-Recall Curve')\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
index 3e8e8830..7edd83b9 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
@@ -51,6 +51,11 @@ def __init__(self, timeout_seconds=3):
         self.timeout_seconds = timeout_seconds
 
 
+class AgencyConfig(BaseObject):
+    def __init__(self, agency_name=None):
+        self.agency_name = agency_name
+
+
 class WeDPRMlConfig:
     def __init__(self, config_dict):
         self.auth_config = AuthConfig()
@@ -63,6 +68,8 @@ def __init__(self, config_dict):
         self.user_config.set_params(**config_dict)
         self.http_config = HttpConfig()
         self.http_config.set_params(**config_dict)
+        self.agency_config = AgencyConfig()
+        self.agency_config.set_params(**config_dict)
 
 
 class WeDPRMlConfigBuilder:
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
index ade5b8bb..90b6119c 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
@@ -7,7 +7,6 @@ class DataContext:
 
     def __init__(self, *datasets):
         self.datasets = list(datasets)
-        self.ctx = self.datasets[0].ctx
 
         self._check_datasets()
 
@@ -28,13 +27,13 @@ def _check_datasets(self):
     def to_psi_format(self, merge_filed, result_receiver_id_list):
         dataset_psi = []
         for dataset in self.datasets:
-            if dataset.agency.agency_id in result_receiver_id_list:
+            if dataset.agency in result_receiver_id_list:
                 result_receiver = "true"
             else:
                 result_receiver = "false"
             dataset_psi_info = {"idFields": [merge_filed],
-                                "dataset": {"owner": dataset.ctx.user_name,
-                                            "ownerAgency": dataset.agency.agency_id,
+                                "dataset": {"owner": dataset.dataset_owner,
+                                            "ownerAgency": dataset.agency,
                                             "path": dataset.dataset_path,
                                             "storageTypeStr": "HDFS",
                                             "datasetID": dataset.dataset_id},
@@ -42,8 +41,24 @@ def to_psi_format(self, merge_filed, result_receiver_id_list):
             dataset_psi.append(dataset_psi_info)
         return dataset_psi
 
-    def to_model_formort(self):
+    def to_model_formort(self, merge_filed, result_receiver_id_list):
         dataset_model = []
         for dataset in self.datasets:
-            dataset_model.append(dataset.dataset_path)
+            if dataset.agency in result_receiver_id_list:
+                result_receiver = "true"
+            else:
+                result_receiver = "false"
+            if dataset.is_label_holder:
+                label_provider = "true"
+            else:
+                label_provider = "false"
+            dataset_psi_info = {"idFields": [merge_filed],
+                                "dataset": {"owner": dataset.dataset_owner,
+                                            "ownerAgency": dataset.agency,
+                                            "path": dataset.dataset_path,
+                                            "storageTypeStr": "HDFS",
+                                            "datasetID": dataset.dataset_id},
+                                "labelProvider": label_provider,
+                                "receiveResult": result_receiver}
+            dataset_model.append(dataset_psi_info)
         return dataset_model
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
index 90355003..7bcfb109 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
@@ -1,21 +1,25 @@
 # -*- coding: utf-8 -*-
 import json
 
+from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit
+from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobParam
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo
 from abc import abstractmethod
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient
-from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobType
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobType, ModelType
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import ModelResult
 
 
 class JobContext:
 
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, dataset: DataContext = None, my_agency=None):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, dataset: DataContext = None, my_agency=None):
         if dataset is None:
             raise Exception("Must define the job related datasets!")
         self.remote_job_client = remote_job_client
-        self.project_name = project_name
+        self.storage_entry_point = storage_entry_point
+        self.project_id = project_id
         self.dataset = dataset
         self.create_agency = my_agency
         self.participant_id_list = []
@@ -42,10 +46,10 @@ def __init_participant__(self):
         participant_id_list = []
         dataset_id_list = []
         for dataset in self.dataset.datasets:
-            participant_id_list.append(dataset.agency.agency_id)
+            participant_id_list.append(dataset.agency)
             dataset_id_list.append(dataset.dataset_id)
-            self.task_parties.append({'userName': dataset.ctx.user_name,
-                                      'agency': dataset.agency.agency_id})
+            self.task_parties.append({'userName': dataset.dataset_owner,
+                                      'agency': dataset.agency})
         self.participant_id_list = participant_id_list
         self.dataset_id_list = dataset_id_list
 
@@ -54,7 +58,7 @@ def __init_label_information__(self):
         label_columns = None
         for dataset in self.dataset.datasets:
             if dataset.is_label_holder:
-                label_holder_agency = dataset.agency.agency_id
+                label_holder_agency = dataset.agency
                 label_columns = 'y'
         self.label_holder_agency = label_holder_agency
         self.label_columns = label_columns
@@ -71,18 +75,30 @@ def submit(self):
         return self.remote_job_client.submit_job(self.build())
 
     @abstractmethod
-    def parse_result(self, result_detail):
+    def parse_result(self, job_id, block_until_success):
         pass
 
     def fetch_job_result(self, job_id, block_until_success):
-        job_result = self.query_job_status(job_id, block_until_success)
-        # TODO: determine success or not here
-        return self.parse_result(self.remote_job_client.query_job_detail(job_id))
+        # job_result = self.query_job_status(job_id, block_until_success)
+        # # TODO: determine success or not here
+        # return self.parse_result(self.remote_job_client.query_job_detail(job_id, block_until_success))
+
+        # # query_job_status
+        # job_result = self.remote_job_client.poll_job_result(job_id, block_until_success)
+        # # failed case
+        # if job_result == None or job_result.job_status == None or (not job_result.job_status.run_success()):
+        #     raise Exception(f'job {job_id} running failed!')
+        # # success case
+        # ...
+
+        # query_job_detail
+        result_detail = self.remote_job_client.query_job_detail(job_id, block_until_success)
+        return result_detail
 
 
 class PSIJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, project_name, dataset, my_agency)
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.merge_field = merge_field
 
     def get_job_type(self) -> JobType:
@@ -91,59 +107,205 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_psi_format(
             self.merge_field, self.result_receiver_id_list)
-        job_info = JobInfo(job_type=self.get_job_type(), project_name=self.project_name, param=json.dumps(
-            {'dataSetList': self.dataset_list}).replace('"', '\\"'))
+        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+
+        psi_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                storage_workspace=None, 
+                                dataset_owner=self.storage_entry_point.user_config.user,
+                                dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
+
+        return psi_result
+
 
 class PreprocessingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None):
-        super().__init__(remote_job_client, project_name, dataset, my_agency)
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
+        self.merge_field = merge_field
 
     def get_job_type(self) -> JobType:
         return JobType.PREPROCESSING
 
-    # TODO: build the request
     def build(self) -> JobParam:
-        return None
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+
+        pre_result = result_detail
+        # pre_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+        #                         storage_workspace=None, 
+        #                         dataset_owner=self.storage_entry_point.user_config.user,
+        #                         dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
+
+        return pre_result
 
 
 class FeatureEngineeringJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None):
-        super().__init__(remote_job_client, project_name, dataset, my_agency)
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
+        self.merge_field = merge_field
 
     def get_job_type(self) -> JobType:
         return JobType.FEATURE_ENGINEERING
 
-    # TODO: build the jobParam
     def build(self) -> JobParam:
-        return None
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+
+        fe_result = result_detail
+        # fe_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+        #                         storage_workspace=None, 
+        #                         dataset_owner=self.storage_entry_point.user_config.user,
+        #                         dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
+
+        return fe_result
 
 
 class SecureLGBMTrainingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None):
-        super().__init__(remote_job_client, project_name, dataset, my_agency)
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
+        self.merge_field = merge_field
 
     def get_job_type(self) -> JobType:
         return JobType.XGB_TRAINING
 
-    # TODO: build the jobParam
     def build(self) -> JobParam:
-        return None
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+        # result_detail.modelResultDetail['ModelResult']
+        train_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
+        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+
+        xgb_result = ModelResult(job_id, train_result, test_result, result_detail.model, ModelType.XGB_MODEL_SETTING.name)
+        return xgb_result
 
 
 class SecureLGBMPredictJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, project_name: str, model_setting, dataset: DataContext = None, my_agency=None):
-        super().__init__(remote_job_client, project_name, dataset, my_agency)
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, predict_algorithm, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
+        self.merge_field = merge_field
+        self.predict_algorithm = predict_algorithm
 
     def get_job_type(self) -> JobType:
         return JobType.XGB_PREDICTING
 
-    # TODO: build the jobParam
     def build(self) -> JobParam:
-        return None
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting, 'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+
+        xgb_result = ModelResult(job_id, test_result=test_result)
+        return xgb_result
+
+
+class SecureLRTrainingJobContext(JobContext):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+        self.model_setting = model_setting
+        self.merge_field = merge_field
+
+    def get_job_type(self) -> JobType:
+        return JobType.LR_TRAINING
+
+    def build(self) -> JobParam:
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+        # result_detail.modelResultDetail['ModelResult']
+        train_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
+        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+
+        lr_result = ModelResult(job_id, train_result, test_result, result_detail.model, ModelType.LR_MODEL_SETTING.name)
+        return lr_result
+
+
+class SecureLRPredictJobContext(JobContext):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, predict_algorithm, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+        self.model_setting = model_setting
+        self.merge_field = merge_field
+        self.predict_algorithm = predict_algorithm
+
+    def get_job_type(self) -> JobType:
+        return JobType.LR_PREDICTING
+
+    def build(self) -> JobParam:
+        self.dataset_list = self.dataset.to_model_formort(
+            self.merge_field, self.result_receiver_id_list)
+        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting, 'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
+        job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
+        return job_param
+
+    def parse_result(self, job_id, block_until_success):
+        result_detail = self.fetch_job_result(job_id, block_until_success)
+        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
+                                    storage_workspace=None, 
+                                    dataset_owner=self.storage_entry_point.user_config.user,
+                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+
+        lr_result = ModelResult(job_id, test_result=test_result)
+        return lr_result
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
index 2e58a3c7..cf9f3327 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
@@ -10,11 +10,13 @@ def __init__(self,
                  storage_workspace,
                  dataset_id=None,
                  dataset_path=None,
+                 dataset_owner=None,
                  agency=None,
                  values=None,
                  is_label_holder=False):
         self.dataset_id = dataset_id
         self.dataset_path = dataset_path
+        self.dataset_owner = dataset_owner
         self.agency = agency
         self.values = values
         self.is_label_holder = is_label_holder
@@ -28,10 +30,10 @@ def __init__(self,
             self.columns = self.values.columns
             self.shape = self.values.shape
 
-    def load_values(self):
+    def load_values(self, header = None):
         # 加载hdfs的数据集
         if self.storage_client is not None:
-            self.values = self.storage_client.download(self.dataset_path)
+            self.values = self.storage_client.download(self.dataset_path, header=header)
             self.columns = self.values.columns
             self.shape = self.values.shape
 
@@ -39,7 +41,8 @@ def save_values(self, path=None):
         # 保存数据到hdfs目录
         if path is not None:
             self.dataset_path = path
-        if not self.dataset_path.startswith(self.storage_workspace):
+        if self.storage_workspace is not None and \
+            not self.dataset_path.startswith(self.storage_workspace):
             self.dataset_path = os.path.join(
                 self.storage_workspace, self.dataset_path)
         if self.storage_client is not None:
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
index 8ba01aa9..371ef823 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
@@ -26,14 +26,14 @@ def upload(self, dataframe, hdfs_path):
         self.storage_client.save_data(csv_buffer.getvalue(), hdfs_path)
         return
 
-    def download(self, hdfs_path):
+    def download(self, hdfs_path, header=None):
         """
         从HDFS下载数据并返回为Pandas DataFrame
         :param hdfs_path: HDFS文件路径
         :return: Pandas DataFrame
         """
         content = self.storage_client.get_data(hdfs_path)
-        dataframe = pd.read_csv(io.BytesIO(content))
+        dataframe = pd.read_csv(io.BytesIO(content), header=header)
         return dataframe
 
     def download_byte(self, hdfs_path):
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
index 8b3795a3..c04523d2 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
@@ -16,6 +16,13 @@ class JobType(Enum):
     FEATURE_ENGINEERING = "FEATURE_ENGINEERING",
     XGB_TRAINING = "XGB_TRAINING",
     XGB_PREDICTING = "XGB_PREDICTING"
+    LR_TRAINING = "LR_TRAINING",
+    LR_PREDICTING = "LR_PREDICTING"
+
+
+class ModelType(Enum):
+    XGB_MODEL_SETTING = "XGB_MODEL_SETTING",
+    LR_MODEL_SETTING = "LR_MODEL_SETTING",
 
 
 class JobStatus(Enum):
@@ -53,8 +60,9 @@ def get_job_status(job_status_tr: str):
 
 
 class JobInfo(BaseObject):
-    def __init__(self, job_id: str = None, job_type: JobType = None, project_name: str = None, param: str = None, **params: Any):
-        self.id = job_id
+    def __init__(self, job_id: str = None, job_type: JobType = None, project_id: str = None, param: str = None, **params: Any):
+        if job_id is not None:
+            self.id = job_id
         self.name = None
         self.owner = None
         self.ownerAgency = None
@@ -63,7 +71,7 @@ def __init__(self, job_id: str = None, job_type: JobType = None, project_name: s
         else:
             self.jobType = None
         self.parties = None
-        self.projectName = project_name
+        self.projectId = project_id
         self.param = param
         self.status = None
         self.result = None
@@ -76,9 +84,34 @@ def __repr__(self):
         return f"job_id: {self.id}, owner: {self.owner}, ownerAgency: {self.ownerAgency}, jobType: {self.jobType}, status: {self.status}"
 
 
+class ModelInfo(BaseObject):
+    def __init__(self, model, model_type, **params: Any):
+        
+        self.type = model_type
+        # self.setting = json.loads(model)
+        self.setting = model
+        self.startTime = None
+        self.endTime = None
+        self.step = None
+        self.id = None
+        self.name = None
+        self.agency = None
+        self.owner = None
+
+        self.set_params(**params)
+
+
+class ModelResult:
+    def __init__(self, job_id: str, train_result = None, test_result = None, model = None, model_type = None):
+        self.job_id = job_id
+        self.train_result = train_result
+        self.test_result = test_result
+        self.model = ModelInfo(model, model_type).__dict__
+
+
 class JobParam:
     def __init__(self, job_info: JobInfo, task_parities, dataset_list):
-        self.job = job_info
+        self.job = job_info.__dict__
         self.taskParties = task_parities
         self.datasetList = dataset_list
 
@@ -159,7 +192,7 @@ def get_job_config(self):
 
     def submit_job(self, job_params: JobParam) -> WeDPRResponse:
         wedpr_response = self.send_request(True,
-                                           self.job_config._submit_job_uri, None, None, json.dumps(job_params))
+                                           self.job_config.submit_job_uri, None, None, json.dumps(job_params.__dict__))
         submit_result = WeDPRResponse(**wedpr_response)
         # return the job_id
         if submit_result.success():
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
index 6d8b008e..ac999867 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
@@ -11,6 +11,8 @@
 from wedpr_ml_toolkit.context.job_context import FeatureEngineeringJobContext
 from wedpr_ml_toolkit.context.job_context import SecureLGBMPredictJobContext
 from wedpr_ml_toolkit.context.job_context import SecureLGBMTrainingJobContext
+from wedpr_ml_toolkit.context.job_context import SecureLRPredictJobContext
+from wedpr_ml_toolkit.context.job_context import SecureLRTrainingJobContext
 from wedpr_ml_toolkit.context.data_context import DataContext
 
 
@@ -40,20 +42,26 @@ def query_job_status(self, job_id, block_until_finish=False) -> JobInfo:
     def query_job_detail(self, job_id, block_until_finish=False) -> JobDetailResponse:
         return self.remote_job_client.query_job_detail(job_id, block_until_finish)
 
-    def build_job_context(self, job_type: JobType, project_name: str, dataset: DataContext, model_setting=None,
-                          id_fields='id'):
+    def build_job_context(self, job_type: JobType, project_id: str, dataset: DataContext, model_setting=None,
+                          id_fields='id', predict_algorithm=None):
         if job_type == JobType.PSI:
-            return PSIJobContext(self.remote_job_client, project_name, dataset, self.config.agency_config.agency_name,
-                                 id_fields)
+            return PSIJobContext(self.remote_job_client, self.storage_entry_point, project_id, dataset, 
+                                 self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.PREPROCESSING:
-            return PreprocessingJobContext(self.remote_job_client, project_name, model_setting, dataset,
-                                           self.config.agency_config.agency_name)
+            return PreprocessingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                           model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.FEATURE_ENGINEERING:
-            return FeatureEngineeringJobContext(self.remote_job_client, project_name, model_setting, dataset,
-                                                self.config.agency_config.agency_name)
+            return FeatureEngineeringJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.XGB_TRAINING:
-            return SecureLGBMTrainingJobContext(self.remote_job_client, project_name, model_setting, dataset,
-                                                self.config.agency_config.agency_name)
+            return SecureLGBMTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.XGB_PREDICTING:
-            return SecureLGBMPredictJobContext(self.remote_job_client, project_name, model_setting, dataset,
-                                               self.config.agency_config.agency_name)
+            return SecureLGBMPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                               model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)
+        if job_type == JobType.LR_TRAINING:
+            return SecureLRTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+        if job_type == JobType.LR_PREDICTING:
+            return SecureLRPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+                                               model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)

From fa96b320da1624fa2ca3ab2a9111185702923377 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 5 Nov 2024 18:28:43 +0800
Subject: [PATCH 080/120] update dependency install (#79)

* update dependency install

* download missed model_prepare.csv && dataset, support multiple-model
worker

* add handshake before establish a job

* add push_by_topic

* fix push_by_topic

* fix ecdh-psi not return status
---
 cpp/ppc-framework/protocol/Task.h             | 12 +--
 cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp |  2 +-
 .../src/bs-ecdh-psi/protocol/BsEcdhResult.h   |  2 +-
 .../src/cm2020-psi/core/CM2020PSIReceiver.cpp |  2 +-
 .../src/cm2020-psi/core/CM2020PSISender.cpp   |  2 +-
 .../src/cm2020-psi/protocol/CM2020PSIResult.h |  2 +-
 .../labeled-psi/protocol/LabeledPSIResult.h   |  2 +-
 .../ppc-psi/src/psi-framework/TaskState.h     | 12 +++
 .../gateway/router/LocalRouter.cpp            | 28 ++++---
 cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp       |  1 -
 .../transport/api/transport_api.py            | 10 ++-
 .../transport/impl/transport.py               |  7 ++
 python/ppc_model/common/base_context.py       |  6 ++
 python/ppc_model/common/initializer.py        |  7 +-
 python/ppc_model/conf/application-sample.yml  |  6 --
 python/ppc_model/datasets/dataset.py          |  6 ++
 .../feature_engineering_engine.py             |  9 +-
 python/ppc_model/interface/model_base.py      | 33 +++++++-
 .../network/wedpr_model_transport.py          | 84 +++++++++++++------
 python/ppc_model/ppc_model_app.py             |  4 +-
 .../local_processing_party.py                 | 11 ++-
 .../secure_lgbm/vertical/active_party.py      |  1 +
 .../secure_model_base/secure_model_context.py | 21 +++++
 python/ppc_model/task/task_manager.py         | 69 ++++++++-------
 python/requirements.txt                       |  2 -
 python/tools/install.sh                       |  1 +
 26 files changed, 240 insertions(+), 102 deletions(-)
 create mode 100644 python/tools/install.sh

diff --git a/cpp/ppc-framework/protocol/Task.h b/cpp/ppc-framework/protocol/Task.h
index a43dd4fb..1c9afccf 100644
--- a/cpp/ppc-framework/protocol/Task.h
+++ b/cpp/ppc-framework/protocol/Task.h
@@ -63,14 +63,10 @@ class TaskResult
     virtual void setFileInfo(FileInfo::Ptr _fileInfo) { m_fileInfo = std::move(_fileInfo); }
 
     // serialize the taskResult to json
-    virtual Json::Value serializeToJson() const
+    virtual Json::Value serializeToJson()
     {
         Json::Value response;
         response["taskID"] = m_taskID;
-        if (!m_status.empty())
-        {
-            response["status"] = m_status;
-        }
         if (m_timeCost)
         {
             response["timeCost"] = std::to_string(m_timeCost) + "ms";
@@ -81,7 +77,7 @@ class TaskResult
             response["fileID"] = m_fileInfo->fileID;
             response["fileMd5"] = m_fileInfo->fileMd5;
         }
-        if (m_error)
+        if (m_error && m_error->errorCode() != 0)
         {
             response["code"] = m_error->errorCode();
             response["message"] = m_error->errorMessage();
@@ -91,6 +87,10 @@ class TaskResult
             response["code"] = 0;
             response["message"] = "success";
         }
+        if (!m_status.empty())
+        {
+            response["status"] = m_status;
+        }
         return response;
     }
 
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
index 6c390af9..e344408e 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
@@ -540,7 +540,7 @@ void OtPIRImpl::runSenderGenerateCipher(PirTaskMessage taskMessage)
 
 void OtPIRImpl::onReceiverTaskDone(bcos::Error::Ptr _error)
 {
-    if (m_taskState->taskDone())
+    if (m_taskState->taskDone() && (!_error || _error->errorCode() == 0))
     {
         return;
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h
index e132f198..204a515a 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/protocol/BsEcdhResult.h
@@ -43,7 +43,7 @@ class BsEcdhResult : public protocol::TaskResult
     [[nodiscard]] Json::Value const& data() const { return m_data; }
 
     // serialize the taskResult to json
-    [[nodiscard]] Json::Value serializeToJson() const override
+    [[nodiscard]] Json::Value serializeToJson() override
     {
         Json::Value response;
         if (m_error && error()->errorCode())
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
index dfe83d24..2525882f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
@@ -931,7 +931,7 @@ void CM2020PSIReceiver::onReceiverException(const std::string& _module, const st
 
 void CM2020PSIReceiver::onReceiverTaskDone(bcos::Error::Ptr _error)
 {
-    if (m_taskState->taskDone())
+    if (m_taskState->taskDone() && (!_error || _error->errorCode() == 0))
     {
         return;
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
index 99a9105f..c7f9a947 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
@@ -832,7 +832,7 @@ void CM2020PSISender::onSenderException(const std::string& _message, const std::
 
 void CM2020PSISender::onSenderTaskDone(bcos::Error::Ptr _error)
 {
-    if (m_taskState->taskDone())
+    if (m_taskState->taskDone() && (!_error || _error->errorCode() == 0))
     {
         return;
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
index 4d461183..a17934a5 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/protocol/CM2020PSIResult.h
@@ -44,7 +44,7 @@ class CM2020PSIResult : public protocol::TaskResult
     }
 
     // serialize the taskResult to json
-    [[nodiscard]] Json::Value serializeToJson() const override
+    [[nodiscard]] Json::Value serializeToJson() override
     {
         Json::Value response;
         response["taskID"] = taskID();
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h
index 13dd3bf6..bc0dc1de 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h
@@ -47,7 +47,7 @@ class LabeledPSIResult : public protocol::TaskResult
     const std::vector>& getOutputs() { return m_outputs; }
 
     // serialize the taskResult to json
-    Json::Value serializeToJson() const override
+    Json::Value serializeToJson() override
     {
         Json::Value response;
         response["taskID"] = taskID();
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
index 24c2b3dc..f6184cde 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
@@ -225,6 +225,11 @@ class TaskState : public std::enable_shared_from_this
                     -1, "task " + m_task->id() + " failed for " +
                             boost::lexical_cast(m_failedCount) + " error!");
                 result->setError(std::move(error));
+                result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
+            }
+            else
+            {
+                result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::COMPLETED));
             }
 
             // clear file
@@ -239,6 +244,7 @@ class TaskState : public std::enable_shared_from_this
                              << LOG_KV("msg", boost::diagnostic_information(e));
             auto error = std::make_shared(-1, boost::diagnostic_information(e));
             result->setError(std::move(error));
+            result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
         }
         if (m_callback)
         {
@@ -262,6 +268,11 @@ class TaskState : public std::enable_shared_from_this
         }
         try
         {
+            _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::COMPLETED));
+            if (_result->error() && _result->error()->errorCode() != 0)
+            {
+                _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
+            }
             // Note: we consider that the task success even if the handler exception
             if (_noticePeer && !m_onlySelfRun && _result->error() &&
                 _result->error()->errorCode() && m_notifyPeerFinishHandler)
@@ -288,6 +299,7 @@ class TaskState : public std::enable_shared_from_this
                              << LOG_KV("msg", boost::diagnostic_information(e));
             auto error = std::make_shared(-1, boost::diagnostic_information(e));
             _result->setError(std::move(error));
+            _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
         }
         if (m_callback)
         {
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
index 68af4f24..cb6c6a4c 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
@@ -64,6 +64,8 @@ void LocalRouter::registerTopic(bcos::bytesConstRef _nodeID, std::string const&
     }
     for (auto const& msgInfo : msgQueue->messages)
     {
+        LOCAL_ROUTER_LOG(INFO) << LOG_DESC("registerTopic, dispatcher the holding msg queue")
+                               << LOG_KV("topic", topic) << LOG_KV("nodeID", printNodeID(_nodeID));
         dispatcherMessage(msgInfo.msg, msgInfo.callback, false);
     }
 }
@@ -80,6 +82,15 @@ bool LocalRouter::dispatcherMessage(
     P2PMessage::Ptr const& msg, ReceiveMsgFunc callback, bool holding)
 {
     auto frontList = chooseReceiver(msg);
+    auto commonCallback = [](bcos::Error::Ptr error) {
+        if (!error || error->errorCode() == 0)
+        {
+            return;
+        }
+        LOCAL_ROUTER_LOG(WARNING) << LOG_DESC("dispatcherMessage to front failed")
+                                  << LOG_KV("code", error->errorCode())
+                                  << LOG_KV("msg", error->errorMessage());
+    };
     // find the front
     if (!frontList.empty())
     {
@@ -93,15 +104,7 @@ bool LocalRouter::dispatcherMessage(
             }
             else
             {
-                front->onReceiveMessage(msg->msg(), [](bcos::Error::Ptr error) {
-                    if (!error || error->errorCode() == 0)
-                    {
-                        return;
-                    }
-                    LOCAL_ROUTER_LOG(WARNING) << LOG_DESC("dispatcherMessage to front failed")
-                                              << LOG_KV("code", error->errorCode())
-                                              << LOG_KV("msg", error->errorMessage());
-                });
+                front->onReceiveMessage(msg->msg(), commonCallback);
             }
             i++;
         }
@@ -122,7 +125,12 @@ bool LocalRouter::dispatcherMessage(
     // no connection found, cache the topic message and dispatcher later
     if (msg->header()->routeType() == (uint16_t)RouteType::ROUTE_THROUGH_TOPIC && m_cache)
     {
-        m_cache->insertCache(msg->header()->optionalField()->topic(), msg, callback);
+        // send response when hodling the message
+        if (callback)
+        {
+            callback(nullptr);
+        }
+        m_cache->insertCache(msg->header()->optionalField()->topic(), msg, commonCallback);
         return true;
     }
     return false;
diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
index 5b039131..a33e4acc 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
@@ -226,7 +226,6 @@ void Rpc::runTask(Json::Value const& _req, RespFunc _respFunc)
             _respFunc(result->error(), result->serializeToJson());
             return;
         }
-
         _respFunc(_result->error(), _result->serializeToJson());
     });
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
index a61510a6..fceac195 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
@@ -14,15 +14,19 @@ def stop(self):
         pass
 
     @abstractmethod
-    def push_by_nodeid(topic: str, dstNode: bytes, seq: int, payload: bytes, timeout: int):
+    def push_by_nodeid(self, topic: str, dstNode: bytes, seq: int, payload: bytes, timeout: int):
         pass
 
     @abstractmethod
-    def push_by_inst(topic: str, dstInst: str, seq: int, payload: bytes, timeout: int):
+    def push_by_inst(self, topic: str, dstInst: str, seq: int, payload: bytes, timeout: int):
         pass
 
     @abstractmethod
-    def push_by_component(topic: str, dstInst: str,  component: str, seq: int, payload: bytes, timeout: int):
+    def push_by_component(self, topic: str, dstInst: str,  component: str, seq: int, payload: bytes, timeout: int):
+        pass
+
+    @abstractmethod
+    def push_by_topic(self, topic: str, dstInst: str, seq: int, payload: bytes, timeout: int):
         pass
 
     @abstractmethod
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
index 2de8ba0d..ae4e0154 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
@@ -74,6 +74,13 @@ def push_by_nodeid(self, topic: str, dstNode: bytes, seq: int, payload: bytes, t
             RouteType.ROUTE_THROUGH_NODEID.value, route_info, payload, seq, timeout)
         Transport.check_result("push_by_nodeid", result)
 
+    def push_by_topic(self, topic: str, dstInst: str, seq: int, payload: bytes, timeout: int):
+        route_info = self.__route_info_builder.build(
+            topic=topic, dst_node=None, dst_inst=dstInst, component=None)
+        result = self._push_msg(
+            RouteType.ROUTE_THROUGH_TOPIC.value, route_info, payload, seq, timeout)
+        Transport.check_result("push_by_topic", result)
+
     def push_by_inst(self, topic: str, dstInst: str, seq: int, payload: bytes, timeout: int):
         route_info = self.__route_info_builder.build(
             topic=topic, dst_node=None, dst_inst=dstInst, component=None)
diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py
index bd19d2c7..c663bea8 100644
--- a/python/ppc_model/common/base_context.py
+++ b/python/ppc_model/common/base_context.py
@@ -139,6 +139,12 @@ def __init__(self, job_id: str, job_temp_dir: str):
         # self.get_key_pair()
         self.load_key('aes_key.bin')
 
+    @staticmethod
+    def load_file(storage_client, remote_path, local_path, logger):
+        if not os.path.exists(local_path):
+            logger.info(f"Download file from: {remote_path} to {local_path}")
+            storage_client.download_file(remote_path, local_path)
+
     @staticmethod
     def feature_engineering_input_path(job_id: str, job_temp_dir: str):
         return os.path.join(job_temp_dir, job_id, BaseContext.MODEL_PREPARE_FILE)
diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py
index 7e5d11c0..3b629552 100644
--- a/python/ppc_model/common/initializer.py
+++ b/python/ppc_model/common/initializer.py
@@ -9,6 +9,7 @@
 from ppc_common.ppc_utils import common_func
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
 from wedpr_python_gateway_sdk.transport.impl.transport_loader import TransportLoader
+from ppc_model.network.wedpr_model_transport import ModelRouter
 from ppc_common.deps_services.mysql_storage import MySQLStorage
 from ppc_common.ppc_config.sql_storage_config_loader import SQLStorageConfigLoader
 from ppc_model.network.wedpr_model_transport import ModelTransport
@@ -54,6 +55,7 @@ def __init__(self, log_config_path, config_path, plot_lock=None):
         self.pop_msg_timeout_ms = 60000
         # for UT
         self.transport = None
+        self.model_router = None
         # matplotlib 线程不安全，并行任务绘图增加全局锁
         self.plot_lock = plot_lock
         if plot_lock is None:
@@ -90,15 +92,14 @@ def init_transport(self, task_manager: TaskManager,
         transport.start()
         self.logger().info(
             f"Start transport success, config: {transport.get_config().desc()}")
-        transport.register_component(component_type)
-        self.logger().info(
-            f"Register the component {component_type} success")
         self.transport = ModelTransport(transport=transport,
                                         self_agency_id=self_agency_id,
                                         task_manager=task_manager,
                                         component_type=component_type,
                                         send_msg_timeout_ms=send_msg_timeout_ms,
                                         pop_msg_timeout_ms=pop_msg_timeout_ms)
+        self.model_router = ModelRouter(logger=self.logger(),
+                                        transport=self.transport)
 
     def logger(self, name=None):
         if self.mock_logger is None:
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index c88b9591..b2762275 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -8,10 +8,6 @@ PUBLIC_KEY_LENGTH: 2048
 MAX_MESSAGE_LENGTH_MB: 100
 TASK_TIMEOUT_H: 1800
 
-
-PEM_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/server.pem"
-SHARE_PATH: "/data/app/wedpr-model/wedpr-model-node/ppc_model_service/dataset_share/"
-
 DB_TYPE: "mysql"
 SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@4346-TDSQL_VIP]:[@4346-TDSQL_PORT]/ppcsmodeladm?autocommit=true&charset=utf8mb4"
 
@@ -33,8 +29,6 @@ gm_public_key: ""
 UPLOAD_FOLDER: "./upload_data_folder"
 JOB_TEMP_DIR: ".cache/job"
 
-FE_TIMEOUT_S: 5400
-
 # the transport config
 transport_threadpool_size: 4
 transport_node_id: "MODEL_WeBank_NODE"
diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py
index 994cef06..ee90721d 100644
--- a/python/ppc_model/datasets/dataset.py
+++ b/python/ppc_model/datasets/dataset.py
@@ -8,6 +8,7 @@
 from ppc_model.common.protocol import TaskRole
 from ppc_model.common.model_result import ResultFileHandling, CommonMessage, SendMessage
 from ppc_model.secure_lgbm.secure_lgbm_context import SecureLGBMContext
+from ppc_model.common.base_context import BaseContext
 
 
 class SecureDataset:
@@ -34,6 +35,11 @@ def __init__(self, ctx: SecureLGBMContext, model_data=None, delimiter: str = ' '
         self.feature_name = None
 
         if model_data is None:
+            # try to download the model_prepare_file
+            BaseContext.load_file(ctx.components.storage_client,
+                                  os.path.join(
+                                      ctx.job_id, BaseContext.MODEL_PREPARE_FILE),
+                                  ctx.model_prepare_file, ctx.components.logger())
             self.model_data = pd.read_csv(
                 ctx.model_prepare_file, header=0, delimiter=delimiter)
         else:
diff --git a/python/ppc_model/feature_engineering/feature_engineering_engine.py b/python/ppc_model/feature_engineering/feature_engineering_engine.py
index b882203f..3f0d1778 100644
--- a/python/ppc_model/feature_engineering/feature_engineering_engine.py
+++ b/python/ppc_model/feature_engineering/feature_engineering_engine.py
@@ -6,6 +6,7 @@
 from ppc_model.feature_engineering.vertical.active_party import VerticalFeatureEngineeringActiveParty
 from ppc_model.feature_engineering.vertical.passive_party import VerticalFeatureEngineeringPassiveParty
 from ppc_model.interface.task_engine import TaskEngine
+import os
 
 
 class FeatureEngineeringEngine(TaskEngine):
@@ -15,7 +16,11 @@ class FeatureEngineeringEngine(TaskEngine):
     def run(task_id, args):
         input_path = BaseContext.feature_engineering_input_path(
             args['job_id'], components.config_data['JOB_TEMP_DIR'])
-
+        # try to download the model_prepare_file
+        BaseContext.load_file(components.storage_client,
+                              os.path.join(
+                                  args['job_id'], BaseContext.MODEL_PREPARE_FILE),
+                              input_path, components.logger())
         if args['is_label_holder']:
             field_list, label, feature = SecureDataset.read_dataset(
                 input_path, True)
@@ -33,7 +38,7 @@ def run(task_id, args):
             field_list, _, feature = SecureDataset.read_dataset(
                 input_path, False)
             context = FeatureEngineeringContext(
-                task_id = task_id,
+                task_id=task_id,
                 args=args,
                 components=components,
                 role=TaskRole.PASSIVE_PARTY,
diff --git a/python/ppc_model/interface/model_base.py b/python/ppc_model/interface/model_base.py
index e755b0b3..5906f446 100644
--- a/python/ppc_model/interface/model_base.py
+++ b/python/ppc_model/interface/model_base.py
@@ -1,7 +1,7 @@
 from abc import ABC
 
 from pandas import DataFrame
-from ppc_model.network.wedpr_model_transport import ModelRouter
+from ppc_model.common.protocol import TaskRole
 
 
 class ModelBase(ABC):
@@ -9,9 +9,34 @@ class ModelBase(ABC):
 
     def __init__(self, ctx):
         self.ctx = ctx
-        self.ctx.model_router = ModelRouter(logger=self.ctx.components.logger(),
-                                            transport=self.ctx.components.transport,
-                                            participant_id_list=self.ctx.participant_id_list)
+        self.ctx.model_router = self.ctx.components.model_router
+        if self.ctx.role == TaskRole.ACTIVE_PARTY:
+            self.__active_handshake__()
+        else:
+            self.__passive_handshake__()
+
+    def __active_handshake__(self):
+        # handshake with all passive parties
+        for i in range(1, len(self.ctx.participant_id_list)):
+            participant = self.ctx.participant_id_list[i]
+            self.ctx.components.logger().info(
+                f"Active: send handshake to passive party: {participant}")
+            self.ctx.model_router.handshake(self.ctx.task_id, participant)
+            # wait for handshake response from the passive parties
+            self.ctx.components.logger().info(
+                f"Active: wait for handshake from passive party: {participant}")
+            self.ctx.model_router.wait_for_handshake(self.ctx.task_id)
+
+    def __passive_handshake__(self):
+        self.ctx.components.logger().info(
+            f"Passive: send handshake to active party: {self.ctx.participant_id_list[0]}")
+        # send handshake to the active party
+        self.ctx.model_router.handshake(
+            self.ctx.task_id, self.ctx.participant_id_list[0])
+        # wait for handshake for the active party
+        self.ctx.components.logger().info(
+            f"Passive: wait for Handshake from active party: {self.ctx.participant_id_list[0]}")
+        self.ctx.model_router.wait_for_handshake(self.ctx.task_id)
 
     def fit(
         self,
diff --git a/python/ppc_model/network/wedpr_model_transport.py b/python/ppc_model/network/wedpr_model_transport.py
index b260d9bc..6d93371c 100644
--- a/python/ppc_model/network/wedpr_model_transport.py
+++ b/python/ppc_model/network/wedpr_model_transport.py
@@ -6,6 +6,8 @@
 from ppc_model.network.wedpr_model_transport_api import ModelRouterApi
 from ppc_model.task.task_manager import TaskManager
 import time
+from readerwriterlock import rwlock
+from enum import Enum
 
 
 class ModelTransport(ModelTransportApi):
@@ -25,6 +27,10 @@ def __init__(self, transport: Transport, self_agency_id: str, task_manager: Task
     def get_topic(task_id: str, task_type: str, agency: str):
         return f"{agency}_{task_id}_{task_type}"
 
+    @staticmethod
+    def get_topic_without_agency(task_id: str, task_type: str):
+        return f"{task_id}_{task_type}"
+
     def push_by_component(self, task_id: str, task_type: str, dst_inst: str, data):
         self.transport.push_by_component(topic=self.get_topic(task_id, task_type, dst_inst),
                                          dstInst=dst_inst,
@@ -38,19 +44,22 @@ def push_by_nodeid(self, task_id: str, task_type: str, dst_node: str, payload: b
                                       seq=seq, payload=payload,
                                       timeout=self.send_msg_timeout)
 
-    def pop(self, task_id: str, task_type: str, dst_inst: str) -> MessageAPI:
+    def pop_by_topic(self, topic, task_id) -> MessageAPI:
         while not self.task_manager.task_finished(task_id):
-            msg = self.transport.pop(topic=self.get_topic(
-                task_id, task_type, dst_inst), timeout_ms=self.pop_msg_timeout)
+            msg = self.transport.pop(
+                topic=topic, timeout_ms=self.pop_msg_timeout)
             # wait for the msg if the task is running
             if msg is None:
                 time.sleep(0.04)
             else:
                 return msg
         raise Exception(f"Not receive the message of topic:"
-                        f" {self.get_topic(task_id, task_type, dst_inst)} "
+                        f" {topic} "
                         f"even after the task has been killed!")
 
+    def pop(self, task_id: str, task_type: str, dst_inst: str) -> MessageAPI:
+        return self.pop_by_topic(topic=self.get_topic(task_id, task_type, dst_inst), task_id=task_id)
+
     def get_component_type(self):
         return self.component_type
 
@@ -62,37 +71,62 @@ def stop(self):
         self.transport.stop()
 
 
+class BaseMessage(Enum):
+    Handshake = "Handshake"
+
+
 class ModelRouter(ModelRouterApi):
-    def __init__(self, logger, transport: ModelTransport, participant_id_list):
+    def __init__(self, logger, transport: ModelTransport):
         self.logger = logger
         self.transport = transport
-        self.participant_id_list = participant_id_list
+        # task_id=>{agency=>selectedNode}
         self.router_info = {}
-        for participant in self.participant_id_list:
-            self.__init_router__(participant)
+        self._rw_lock = rwlock.RWLockWrite()
+
+    def handshake(self, task_id, participant):
+        self.logger.info(f"handshake with {participant}")
+        topic = ModelTransport.get_topic_without_agency(
+            task_id, BaseMessage.Handshake.value)
+        self.transport.transport.register_topic(topic)
+        self.transport.transport.push_by_topic(topic=topic,
+                                               dstInst=participant,
+                                               seq=0, payload=bytes(),
+                                               timeout=self.transport.send_msg_timeout)
+
+    def wait_for_handshake(self, task_id):
+        topic = ModelTransport.get_topic_without_agency(
+            task_id, BaseMessage.Handshake.value)
+        self.transport.transport.register_topic(topic)
+        result = self.transport.pop_by_topic(topic=topic, task_id=task_id)
 
-    def __init_router__(self, participant):
-        result = self.transport.select_node(route_type=RouteType.ROUTE_THROUGH_COMPONENT,
-                                            dst_agency=participant,
-                                            dst_component=self.transport.get_component_type())
-        self.logger.info(
-            f"__init_router__ for {participant}, result: {result}, component: {self.transport.get_component_type()}")
         if result is None:
-            raise Exception(
-                f"No router can reach participant {participant}")
+            raise Exception(f"wait_for_handshake failed!")
         self.logger.info(
-            f"ModelRouter, select node {result} for participant {participant}, "
-            f"component: {self.transport.get_component_type()}")
-        self.router_info.update({participant: result})
-        return result
+            f"wait_for_handshake success, task: {task_id}, detail: {result}")
+        with self._rw_lock.gen_wlock():
+            from_inst = result.get_header().get_src_inst()
+            if task_id not in self.router_info.keys():
+                self.router_info.update({task_id: dict()})
+            self.router_info.get(task_id).update(
+                {from_inst: result.get_header().get_src_node().decode("utf-8")})
+
+    def on_task_finish(self, task_id):
+        topic = ModelTransport.get_topic_without_agency(
+            task_id, BaseMessage.Handshake.value)
+        self.transport.transport.unregister_topic(topic)
+        with self._rw_lock.gen_wlock():
+            if task_id in self.router_info.keys():
+                self.router_info.pop(task_id)
 
-    def __get_dstnode_by_participant(self, participant):
-        if participant in self.router_info.keys():
-            return self.router_info.get(participant)
-        return self.__init_router__(participant)
+    def __get_dstnode__(self, task_id, dst_agency):
+        with self._rw_lock.gen_rlock():
+            if task_id in self.router_info.keys() and dst_agency in self.router_info.get(task_id).keys():
+                return self.router_info.get(task_id).get(dst_agency)
+        raise Exception(
+            f"No Router  found for task {task_id}, dst_agency: {dst_agency}")
 
     def push(self, task_id: str, task_type: str, dst_agency: str, payload: bytes, seq: int = 0):
-        dst_node = self.__get_dstnode_by_participant(dst_agency)
+        dst_node = self.__get_dstnode__(task_id, dst_agency)
         self.transport.push_by_nodeid(
             task_id=task_id, task_type=task_type, dst_node=dst_node, payload=payload, seq=seq)
 
diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py
index 0a95bc9a..c2780600 100644
--- a/python/ppc_model/ppc_model_app.py
+++ b/python/ppc_model/ppc_model_app.py
@@ -52,7 +52,9 @@ def register_task_handler():
         ModelTask.LR_TRAINING, SecureLRTrainingEngine.run)
     task_manager.register_task_handler(
         ModelTask.LR_PREDICTING, SecureLRPredictionEngine.run)
-
+    # register clear handlers
+    task_manager.register_task_clear_handler(
+        components.model_router.on_task_finish)
 
 if __name__ == '__main__':
     initialize_app(app)
diff --git a/python/ppc_model/preprocessing/local_processing/local_processing_party.py b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
index 8a7aea0b..99c538a0 100644
--- a/python/ppc_model/preprocessing/local_processing/local_processing_party.py
+++ b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
@@ -7,6 +7,7 @@
 from ppc_common.ppc_utils import utils
 from ppc_model.preprocessing.local_processing.preprocessing import process_dataframe
 from ppc_model.preprocessing.processing_context import ProcessingContext
+from ppc_model.common.base_context import BaseContext
 
 
 class LocalProcessingParty(ABC):
@@ -31,13 +32,14 @@ def processing(self):
         if need_psi and (not utils.file_exists(psi_result_path)):
             storage_client.download_file(
                 self.ctx.remote_psi_result_path, psi_result_path)
-            self.handle_local_psi_result(psi_result_path)
+            self.handle_local_psi_result(
+                self.ctx.remote_psi_result_path, psi_result_path)
             log.info(
                 f"prepare_xgb_after_psi, make_dataset_to_xgb_data_plus_psi_data, dataset_file_path={dataset_file_path}, "
                 f"psi_result_path={dataset_file_path}, model_prepare_file={model_prepare_file}")
         self.make_dataset_to_xgb_data()
         storage_client.upload_file(
-            model_prepare_file, job_id + os.sep + self.ctx.model_prepare_file)
+            model_prepare_file, job_id + os.sep + BaseContext.MODEL_PREPARE_FILE)
         log.info(f"upload model_prepare_file to hdfs, job_id={job_id}")
         if job_algorithm_type == utils.AlgorithmType.Train.name:
             log.info(f"upload column_info to hdfs, job_id={job_id}")
@@ -46,7 +48,7 @@ def processing(self):
         log.info(
             f"call prepare_xgb_after_psi success, job_id={job_id}, timecost: {time.time() - start}")
 
-    def handle_local_psi_result(self, local_psi_result_path):
+    def handle_local_psi_result(self, remote_psi_result_path, local_psi_result_path):
         try:
             log = self.ctx.components.logger()
             log.info(
@@ -57,6 +59,9 @@ def handle_local_psi_result(self, local_psi_result_path):
                 psi_result_file.write('id\n' + content)
             log.info(
                 f"handle_local_psi_result: call handle_local_psi_result success, psi_result_path={local_psi_result_path}")
+            # upload to remote
+            self.ctx.components.storage_client.upload_file(
+                local_psi_result_path, remote_psi_result_path)
         except BaseException as e:
             log.exception(
                 f"handle_local_psi_result: handle_local_psi_result, psi_result_path={local_psi_result_path}, error:{e}")
diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py
index fe4c2ff7..ff6eff58 100644
--- a/python/ppc_model/secure_lgbm/vertical/active_party.py
+++ b/python/ppc_model/secure_lgbm/vertical/active_party.py
@@ -46,6 +46,7 @@ def fit(
     ) -> None:
         self.log.info(
             f'task {self.ctx.task_id}: Starting the lgbm on the active party.')
+
         self._init_active_data()
         self._init_valid_data()
         self._init_early_stop()
diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py
index 50ed9455..8db479e4 100644
--- a/python/ppc_model/secure_model_base/secure_model_context.py
+++ b/python/ppc_model/secure_model_base/secure_model_context.py
@@ -9,6 +9,7 @@
 from ppc_common.ppc_utils import common_func
 from ppc_common.ppc_utils.utils import AlgorithmType
 from ppc_model.common.model_setting import ModelSetting
+from ppc_model.common.base_context import BaseContext
 
 from sklearn.base import BaseEstimator
 
@@ -133,6 +134,26 @@ def __init__(self,
                 self.workspace, args['dataset_id'])
         else:
             self.dataset_file_path = None
+        # the remote dataset_file_path
+        if 'dataset_path' in args:
+            self.remote_dataset_path = args['dataset_path']
+        if self.remote_dataset_path is None:
+            raise f"Must define the dataset_path!"
+        # the remote psi_path
+        if 'psi_result_path' in args:
+            self.remote_psi_path = args['psi_result_path']
+        if self.remote_psi_path is None:
+            raise f"Must define the psi_result_path"
+        # prepare the dataset and psi file
+        BaseContext.load_file(storage_client=self.components.storage_client,
+                              remote_path=self.remote_dataset_path,
+                              local_path=self.dataset_file_path,
+                              logger=self.components.logger())
+        BaseContext.load_file(storage_client=self.components.storage_client,
+                              remote_path=self.remote_psi_path,
+                              local_path=self.psi_result_path,
+                              logger=self.components.logger())
+
         self.model_params = self.create_model_param()
         self.reset_model_params(ModelSetting(args['model_dict']))
         self.sync_file_list = {}
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index abc817c8..3ded3756 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -105,6 +105,7 @@ def __init__(self, logger,
         self._rw_lock = rwlock.RWLockWrite()
         self._tasks: dict[str, TaskResult] = {}
         self._handlers = {}
+        self._task_clear_handler = []
         self._async_executor = AsyncThreadExecutor(
             event_manager=self._thread_event_manager, logger=logger)
         self._cleanup_thread = threading.Thread(target=self._loop_cleanup)
@@ -119,6 +120,10 @@ def register_task_handler(self, task_type: ModelTask, task_handler: Callable):
         """
         self._handlers[task_type.value] = task_handler
 
+    # handler called when task finished
+    def register_task_clear_handler(self, handler: Callable[[str], None]):
+        self._task_clear_handler.append(handler)
+
     def run_task(self, task_id: str, task_type: ModelTask, args=()):
         """
         发起任务
@@ -189,40 +194,44 @@ def status(self, task_id: str) -> [str, float, str]:
         return result.status, 0, result.exec_result
 
     def _on_task_finish(self, task_id: str, is_succeeded: bool, error_msg: str = None):
-        task_result = None
-        with self._rw_lock.gen_wlock():
-            if task_id not in self._tasks.keys():
-                self.logger.warn(
-                    f"_on_task_finish: the task {task_id} not Found! maybe killed!")
-                return
-            task_result = self._tasks[task_id]
+        try:
+            task_result = None
+            with self._rw_lock.gen_wlock():
+                if task_id not in self._tasks.keys():
+                    self.logger.warn(
+                        f"_on_task_finish: the task {task_id} not Found! maybe killed!")
+                    return
+                task_result = self._tasks[task_id]
 
-        # update the task result
-        if is_succeeded:
-            task_result.task_status = TaskStatus.SUCCESS.value
-            self.logger.info(f"Task {task_id} completed, job_id: {task_result.job_id}, "
-                             f"time_costs: {task_result.get_timecost()}s")
-        else:
-            task_result.task_status = TaskStatus.FAILURE.value
-            task_result.diagnosis_msg = error_msg
-            self.logger.warn(f"Task {task_id} failed, job_id: {task_result.job_id}, "
-                             f"time_costs: {self._tasks[task_id].get_timecost()}s, error: {error_msg}")
+            # update the task result
+            if is_succeeded:
+                task_result.task_status = TaskStatus.SUCCESS.value
+                self.logger.info(f"Task {task_id} completed, job_id: {task_result.job_id}, "
+                                 f"time_costs: {task_result.get_timecost()}s")
+            else:
+                task_result.task_status = TaskStatus.FAILURE.value
+                task_result.diagnosis_msg = error_msg
+                self.logger.warn(f"Task {task_id} failed, job_id: {task_result.job_id}, "
+                                 f"time_costs: {self._tasks[task_id].get_timecost()}s, error: {error_msg}")
 
-        self.logger.info(LOG_END_FLAG_FORMATTER.format(
-            job_id=task_result.job_id))
-        # finalize the task result
-        task_result.finalize()
-        self.task_persistent.on_task_finished(task_result)
+            self.logger.info(LOG_END_FLAG_FORMATTER.format(
+                job_id=task_result.job_id))
+            # finalize the task result
+            task_result.finalize()
+            self.task_persistent.on_task_finished(task_result)
 
-        with self._rw_lock.gen_wlock():
-            # erase from the queue
-            self._tasks.pop(task_id)
+            with self._rw_lock.gen_wlock():
+                # erase from the queue
+                self._tasks.pop(task_id)
 
-            # record and upload the log if all task finished
-        if self.task_persistent.job_finished(task_result.job_id):
-            self.logger.info(
-                f"_on_task_finish: all sub tasks finished, upload the log, task_info: {task_result}")
-            self.log_retriever.upload_log(task_result.job_id)
+                # record and upload the log if all task finished
+            if self.task_persistent.job_finished(task_result.job_id):
+                self.logger.info(
+                    f"_on_task_finish: all sub tasks finished, upload the log, task_info: {task_result}")
+                self.log_retriever.upload_log(task_result.job_id)
+        finally:
+            for handler in self._task_clear_handler:
+                handler(task_id)
 
     def _loop_cleanup(self):
         while True:
diff --git a/python/requirements.txt b/python/requirements.txt
index b92953e4..c72f5f73 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -53,8 +53,6 @@ pydot
 snowland-smx
 numpy==1.23.1
 graphviz
-grpcio==1.62.1
-grpcio-tools==1.62.1
 xlrd~=1.0.0
 MarkupSafe>=2.1.1
 urllib3==1.26.18
diff --git a/python/tools/install.sh b/python/tools/install.sh
new file mode 100644
index 00000000..b03bb7fc
--- /dev/null
+++ b/python/tools/install.sh
@@ -0,0 +1 @@
+apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential

From 2c1aebb4f6deb2b90ec848448ae678e680f2969e Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 5 Nov 2024 21:55:17 +0800
Subject: [PATCH 081/120] fix handshake bug (#80)

* remove handle_local_psi

* fix handshake bug
---
 .../ppc-psi/src/cm2020-psi/Common.h           | 10 ++++-
 python/ppc_model/common/context.py            |  1 +
 python/ppc_model/interface/model_base.py      | 30 +++++--------
 .../network/wedpr_model_transport.py          | 45 ++++++++++++-------
 .../local_processing_party.py                 | 24 +---------
 .../secure_lr/vertical/active_party.py        | 22 +++++----
 python/tools/install.sh                       |  1 +
 7 files changed, 66 insertions(+), 67 deletions(-)

diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h
index ccfffdbe..6a357406 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/Common.h
@@ -72,8 +72,14 @@ inline uint32_t dedupDataBatch(ppc::io::DataBatch::Ptr dataBatch)
         return 0;
     }
     auto& data = dataBatch->mutableData();
-    tbb::parallel_sort(data->begin(), data->end());
-    auto unique_end = std::unique(data->begin(), data->end());
+    // Note: the header field should not been sorted
+    auto it = data->begin() + 1;
+    if (it >= data->end())
+    {
+        return data->size();
+    }
+    tbb::parallel_sort(it, data->end());
+    auto unique_end = std::unique(it, data->end());
     data->erase(unique_end, data->end());
     return data->size();
 }
diff --git a/python/ppc_model/common/context.py b/python/ppc_model/common/context.py
index 551b9f83..7a9ee774 100644
--- a/python/ppc_model/common/context.py
+++ b/python/ppc_model/common/context.py
@@ -8,6 +8,7 @@ class Context(BaseContext):
 
     def __init__(self, job_id: str, task_id: str, components: Initializer, role: TaskRole = None):
         super().__init__(job_id, components.config_data['JOB_TEMP_DIR'])
+        self.my_agency_id = components.config_data['AGENCY_ID']
         self.task_id = task_id
         self.components = components
         self.role = role
diff --git a/python/ppc_model/interface/model_base.py b/python/ppc_model/interface/model_base.py
index 5906f446..6ff8c6a1 100644
--- a/python/ppc_model/interface/model_base.py
+++ b/python/ppc_model/interface/model_base.py
@@ -10,33 +10,23 @@ class ModelBase(ABC):
     def __init__(self, ctx):
         self.ctx = ctx
         self.ctx.model_router = self.ctx.components.model_router
-        if self.ctx.role == TaskRole.ACTIVE_PARTY:
-            self.__active_handshake__()
-        else:
-            self.__passive_handshake__()
+        self.__handshake__()
 
-    def __active_handshake__(self):
+    def __handshake__(self):
         # handshake with all passive parties
-        for i in range(1, len(self.ctx.participant_id_list)):
+        for i in range(0, len(self.ctx.participant_id_list)):
             participant = self.ctx.participant_id_list[i]
+            if participant == self.ctx.my_agency_id:
+                continue
             self.ctx.components.logger().info(
-                f"Active: send handshake to passive party: {participant}")
+                f"Send handshake to party: {participant}")
             self.ctx.model_router.handshake(self.ctx.task_id, participant)
-            # wait for handshake response from the passive parties
-            self.ctx.components.logger().info(
-                f"Active: wait for handshake from passive party: {participant}")
-            self.ctx.model_router.wait_for_handshake(self.ctx.task_id)
 
-    def __passive_handshake__(self):
-        self.ctx.components.logger().info(
-            f"Passive: send handshake to active party: {self.ctx.participant_id_list[0]}")
-        # send handshake to the active party
-        self.ctx.model_router.handshake(
-            self.ctx.task_id, self.ctx.participant_id_list[0])
-        # wait for handshake for the active party
+        # wait for handshake response from the passive parties
         self.ctx.components.logger().info(
-            f"Passive: wait for Handshake from active party: {self.ctx.participant_id_list[0]}")
-        self.ctx.model_router.wait_for_handshake(self.ctx.task_id)
+            f"Wait for handshake from all parities")
+        self.ctx.model_router.wait_for_handshake(
+            self.ctx.task_id, self.ctx.participant_id_list, self.ctx.my_agency_id)
 
     def fit(
         self,
diff --git a/python/ppc_model/network/wedpr_model_transport.py b/python/ppc_model/network/wedpr_model_transport.py
index 6d93371c..a2dc50bd 100644
--- a/python/ppc_model/network/wedpr_model_transport.py
+++ b/python/ppc_model/network/wedpr_model_transport.py
@@ -93,22 +93,37 @@ def handshake(self, task_id, participant):
                                                seq=0, payload=bytes(),
                                                timeout=self.transport.send_msg_timeout)
 
-    def wait_for_handshake(self, task_id):
-        topic = ModelTransport.get_topic_without_agency(
-            task_id, BaseMessage.Handshake.value)
-        self.transport.transport.register_topic(topic)
-        result = self.transport.pop_by_topic(topic=topic, task_id=task_id)
-
-        if result is None:
-            raise Exception(f"wait_for_handshake failed!")
-        self.logger.info(
-            f"wait_for_handshake success, task: {task_id}, detail: {result}")
-        with self._rw_lock.gen_wlock():
-            from_inst = result.get_header().get_src_inst()
+    def __all_connected__(self, task_id, participant_id_list, self_agency_id):
+        with self._rw_lock.gen_rlock():
             if task_id not in self.router_info.keys():
-                self.router_info.update({task_id: dict()})
-            self.router_info.get(task_id).update(
-                {from_inst: result.get_header().get_src_node().decode("utf-8")})
+                return False
+            for participant in participant_id_list:
+                if participant == self_agency_id:
+                    continue
+                if participant not in self.router_info.get(task_id).keys():
+                    return False
+            self.logger.info(
+                f"__all_connected__, task: {task_id}, participant_id_list: {participant_id_list}")
+            return True
+
+    def wait_for_handshake(self, task_id, participant_id_list: list, self_agency_id):
+        while not self.__all_connected__(task_id, participant_id_list, self_agency_id):
+            time.sleep(0.04)
+            topic = ModelTransport.get_topic_without_agency(
+                task_id, BaseMessage.Handshake.value)
+            self.transport.transport.register_topic(topic)
+            result = self.transport.pop_by_topic(topic=topic, task_id=task_id)
+
+            if result is None:
+                raise Exception(f"wait_for_handshake failed!")
+            self.logger.info(
+                f"wait_for_handshake success, task: {task_id}, detail: {result}")
+            with self._rw_lock.gen_wlock():
+                from_inst = result.get_header().get_src_inst()
+                if task_id not in self.router_info.keys():
+                    self.router_info.update({task_id: dict()})
+                self.router_info.get(task_id).update(
+                    {from_inst: result.get_header().get_src_node().decode("utf-8")})
 
     def on_task_finish(self, task_id):
         topic = ModelTransport.get_topic_without_agency(
diff --git a/python/ppc_model/preprocessing/local_processing/local_processing_party.py b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
index 99c538a0..568f7cf6 100644
--- a/python/ppc_model/preprocessing/local_processing/local_processing_party.py
+++ b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
@@ -32,11 +32,10 @@ def processing(self):
         if need_psi and (not utils.file_exists(psi_result_path)):
             storage_client.download_file(
                 self.ctx.remote_psi_result_path, psi_result_path)
-            self.handle_local_psi_result(
-                self.ctx.remote_psi_result_path, psi_result_path)
             log.info(
                 f"prepare_xgb_after_psi, make_dataset_to_xgb_data_plus_psi_data, dataset_file_path={dataset_file_path}, "
-                f"psi_result_path={dataset_file_path}, model_prepare_file={model_prepare_file}")
+                f"psi_result_path={psi_result_path}, model_prepare_file={model_prepare_file}, "
+                f"remote_psi_result_path: {self.ctx.remote_psi_result_path}")
         self.make_dataset_to_xgb_data()
         storage_client.upload_file(
             model_prepare_file, job_id + os.sep + BaseContext.MODEL_PREPARE_FILE)
@@ -48,25 +47,6 @@ def processing(self):
         log.info(
             f"call prepare_xgb_after_psi success, job_id={job_id}, timecost: {time.time() - start}")
 
-    def handle_local_psi_result(self, remote_psi_result_path, local_psi_result_path):
-        try:
-            log = self.ctx.components.logger()
-            log.info(
-                f"handle_local_psi_result: start handle_local_psi_result, psi_result_path={local_psi_result_path}")
-            with open(local_psi_result_path, 'r+', encoding='utf-8') as psi_result_file:
-                content = psi_result_file.read()
-                psi_result_file.seek(0, 0)
-                psi_result_file.write('id\n' + content)
-            log.info(
-                f"handle_local_psi_result: call handle_local_psi_result success, psi_result_path={local_psi_result_path}")
-            # upload to remote
-            self.ctx.components.storage_client.upload_file(
-                local_psi_result_path, remote_psi_result_path)
-        except BaseException as e:
-            log.exception(
-                f"handle_local_psi_result: handle_local_psi_result, psi_result_path={local_psi_result_path}, error:{e}")
-            raise e
-
     def make_dataset_to_xgb_data(self):
         log = self.ctx.components.logger()
         dataset_file_path = self.ctx.dataset_file_path
diff --git a/python/ppc_model/secure_lr/vertical/active_party.py b/python/ppc_model/secure_lr/vertical/active_party.py
index cd7c731f..2040a359 100644
--- a/python/ppc_model/secure_lr/vertical/active_party.py
+++ b/python/ppc_model/secure_lr/vertical/active_party.py
@@ -41,9 +41,10 @@ def fit(
         self.log.info(
             f'task {self.ctx.task_id}: Starting the lr on the active party.')
         self._init_active_data()
-        
-        max_iter = self._init_iter(self.dataset.train_X.shape[0], 
+
+        max_iter = self._init_iter(self.dataset.train_X.shape[0],
                                    self.params.epochs, self.params.batch_size)
+        self.log.info(f"task: {self.ctx.task_id}, max_iter: {max_iter}")
         for _ in range(max_iter):
             self._iter_id += 1
             start_time = time.time()
@@ -59,7 +60,8 @@ def fit(
             self._build_iter(feature_select, idx)
 
         # 预测
-        self._train_praba = self._predict_tree(self.dataset.train_X, LRMessage.PREDICT_LEAF_MASK.value)
+        self._train_praba = self._predict_tree(
+            self.dataset.train_X, LRMessage.PREDICT_LEAF_MASK.value)
         # print('train_praba', set(self._train_praba))
 
         # 评估
@@ -69,10 +71,11 @@ def fit(
             self.log.info(
                 f'task {self.ctx.task_id}: iter-{self._iter_id}, auc: {auc}.')
         self.log.info(f'task {self.ctx.task_id}: Ending iter-{self._iter_id}, '
-                        f'time_costs: {time.time() - start_time}s.')
+                      f'time_costs: {time.time() - start_time}s.')
 
         # 预测验证集
-        self._test_praba = self._predict_tree(self.dataset.test_X, LRMessage.TEST_LEAF_MASK.value)
+        self._test_praba = self._predict_tree(
+            self.dataset.test_X, LRMessage.TEST_LEAF_MASK.value)
         if not self.params.silent and self.dataset.test_y is not None:
             auc = Evaluation.fevaluation(
                 self.dataset.test_y, self._test_praba)['auc']
@@ -89,7 +92,8 @@ def predict(self, dataset: SecureDataset = None) -> np.ndarray:
         if dataset is None:
             dataset = self.dataset
 
-        test_praba = self._predict_tree(dataset.test_X, LRMessage.VALID_LEAF_MASK.value)
+        test_praba = self._predict_tree(
+            dataset.test_X, LRMessage.VALID_LEAF_MASK.value)
         self._test_praba = test_praba
 
         if dataset.test_y is not None:
@@ -139,8 +143,10 @@ def _build_iter(self, feature_select, idx):
         public_key_list, d_other_list, partner_index_list = self._receive_d_instance_list()
         deriv = self._calculate_deriv(x_, d, partner_index_list, d_other_list)
 
-        self._train_weights -= self.params.learning_rate * deriv.astype('float')
-        self._train_weights[~np.isin(np.arange(len(self._train_weights)), feature_select)] = 0
+        self._train_weights -= self.params.learning_rate * \
+            deriv.astype('float')
+        self._train_weights[~np.isin(
+            np.arange(len(self._train_weights)), feature_select)] = 0
 
     def _predict_tree(self, X, key_type):
         train_g = self._loss_func.dot_product(X, self._train_weights)
diff --git a/python/tools/install.sh b/python/tools/install.sh
index b03bb7fc..143a5093 100644
--- a/python/tools/install.sh
+++ b/python/tools/install.sh
@@ -1 +1,2 @@
 apt-get install pkg-config python3-dev default-libmysqlclient-dev build-essential
+apt-get install graphviz
\ No newline at end of file

From 2dfb5ce49ff0bf4bf8adce0dd9cbd7799699beb6 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 8 Nov 2024 09:31:30 +0800
Subject: [PATCH 082/120] add NodeDiscovery to fetch alive node information
 (#81)

* add NodeDiscovery to fetch alive node information

* update generated java and python code

* psi node register serviceMeta to gateway

* model service register service inforamtion

* support update-meta at runtime

* fix fetch psi file when use_psi is false
---
 .github/workflows/cpp_toolkit_workflow.yml    |     8 +-
 cpp/ppc-framework/front/FrontConfig.h         |     4 +
 cpp/ppc-framework/front/IFront.h              |     5 +
 cpp/ppc-framework/front/INodeDiscovery.h      |    39 +
 cpp/ppc-framework/gateway/IGateway.h          |     2 +
 cpp/ppc-framework/protocol/INodeInfo.h        |     8 +-
 cpp/ppc-framework/protocol/Protocol.h         |    14 -
 cpp/ppc-framework/protocol/RouteType.h        |     2 +-
 cpp/ppc-framework/protocol/ServiceType.h      |    29 +
 .../ppc-tools/src/config/PPCConfig.cpp        |     1 -
 .../ppc-tools/src/config/PPCConfig.h          |     5 +
 cpp/wedpr-initializer/Initializer.cpp         |    10 +
 cpp/wedpr-initializer/Initializer.h           |     2 +
 .../air-node/AirNodeInitializer.cpp           |     5 +-
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp    |     2 +-
 .../pro-node/ProNodeInitializer.cpp           |     5 +-
 .../grpc/client/GatewayClient.cpp             |    33 +-
 .../grpc/client/GatewayClient.h               |     6 +-
 .../grpc/server/GatewayServer.cpp             |    22 +
 .../grpc/server/GatewayServer.h               |     2 +
 cpp/wedpr-protocol/proto/pb/NodeInfo.proto    |     2 +
 cpp/wedpr-protocol/proto/pb/Service.proto     |     7 +
 .../protobuf/src/NodeInfoImpl.h               |    14 +
 .../protobuf/src/RequestConverter.h           |    29 +-
 .../protocol/src/ServiceConfig.cpp            |    84 +
 .../protocol/src/ServiceConfig.h              |    87 +
 .../ppc-front/ppc-front/FrontFactory.cpp      |    10 +-
 .../ppc-front/ppc-front/FrontImpl.cpp         |    22 +-
 .../ppc-front/ppc-front/FrontImpl.h           |     5 +
 .../ppc-front/ppc-front/NodeDiscovery.cpp     |    69 +
 .../ppc-front/ppc-front/NodeDiscovery.h       |    49 +
 .../ppc-gateway/gateway/GatewayImpl.h         |    11 +
 .../gateway/router/GatewayNodeInfoImpl.cpp    |    17 +
 .../sdk-wrapper/java/bindings/build.gradle    |     2 +
 .../webank/wedpr/sdk/jni/common/Common.java   |     7 +
 .../webank/wedpr/sdk/jni/common/Constant.java |     1 +
 .../sdk/jni/common/ObjectMapperFactory.java   |    47 +
 .../wedpr/sdk/jni/demo/TransportDemo.java     |    19 +
 .../wedpr/sdk/jni/generated/FrontConfig.java  |    14 +-
 .../wedpr/sdk/jni/generated/IFront.java       |    15 +-
 .../sdk/jni/generated/INodeDiscovery.java     |    47 +
 .../wedpr/sdk/jni/generated/INodeInfo.java    |    85 +
 .../wedpr/sdk/jni/generated/NodeInfoVec.java  |   175 +
 .../jni/generated/SharedNodeDiscovery.java    |    54 +
 .../sdk/jni/generated/SharedNodeInfo.java     |    54 +
 .../jni/generated/wedpr_java_transport.java   |     4 +
 .../generated/wedpr_java_transportJNI.java    |    86 +
 .../sdk/jni/transport/TransportConfig.java    |    20 +
 .../sdk/jni/transport/WeDPRTransport.java     |     7 +
 .../sdk/jni/transport/impl/TransportImpl.java |    51 +
 .../sdk/jni/transport/model/ServiceMeta.java  |    69 +
 .../src/wedpr_java_transportJAVA_wrap.cxx     |   775 +-
 .../java/swig/wedpr_java_transport.i          |    25 +
 .../python/bindings/demo/transport_demo.py    |    19 +-
 .../transport/api/transport_api.py            |    61 +
 .../generated/wedpr_python_transport.py       |   218 +
 .../transport/impl/transport.py               |    24 +-
 .../transport/impl/transport_config.py        |    29 +-
 .../transport/impl/transport_loader.py        |    26 +-
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 14039 ++++++++++------
 .../python/swig/wedpr_python_transport.i      |    26 +-
 .../sdk/src/ProTransportImpl.cpp              |     9 +-
 python/ppc_model/common/initializer.py        |    14 +-
 .../secure_model_base/secure_model_context.py |    14 +-
 64 files changed, 11054 insertions(+), 5592 deletions(-)
 create mode 100644 cpp/ppc-framework/front/INodeDiscovery.h
 create mode 100644 cpp/ppc-framework/protocol/ServiceType.h
 create mode 100644 cpp/wedpr-protocol/protocol/src/ServiceConfig.cpp
 create mode 100644 cpp/wedpr-protocol/protocol/src/ServiceConfig.h
 create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
 create mode 100644 cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/ObjectMapperFactory.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeDiscovery.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeInfo.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/NodeInfoVec.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeDiscovery.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeInfo.java
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java

diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index 5189171a..f8056e66 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -39,12 +39,6 @@ jobs:
         with:
           toolchain: nightly-2022-07-28
           override: true
-      - name: Prepare python
-        if: runner.os == 'Windows'
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.8'
-          cache: 'pip'
       - name: Prepare vcpkg
         if: runner.os != 'Windows'
         uses: friendlyanon/setup-vcpkg@v1
@@ -70,7 +64,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=OFF -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |
diff --git a/cpp/ppc-framework/front/FrontConfig.h b/cpp/ppc-framework/front/FrontConfig.h
index 8af6ca05..40e0a48a 100644
--- a/cpp/ppc-framework/front/FrontConfig.h
+++ b/cpp/ppc-framework/front/FrontConfig.h
@@ -77,6 +77,9 @@ class FrontConfig
 
     std::vector& mutableComponents() { return m_components; }
 
+    std::string const meta() const { return m_meta; }
+    void setMeta(std::string meta) { m_meta = std::move(meta); }
+
 protected:
     ppc::protocol::GrpcConfig::Ptr m_grpcConfig;
     ppc::protocol::EndPoint m_selfEndPoint;
@@ -84,6 +87,7 @@ class FrontConfig
     std::string m_nodeID;
     std::string m_gatewayGrpcTarget;
     std::vector m_components;
+    std::string m_meta;
 };
 
 class FrontConfigBuilder
diff --git a/cpp/ppc-framework/front/IFront.h b/cpp/ppc-framework/front/IFront.h
index 1512dfca..9470686b 100644
--- a/cpp/ppc-framework/front/IFront.h
+++ b/cpp/ppc-framework/front/IFront.h
@@ -19,6 +19,7 @@
  */
 #pragma once
 #include "FrontConfig.h"
+#include "INodeDiscovery.h"
 #include "ppc-framework/protocol/Handler.h"
 #include "ppc-framework/protocol/INodeInfo.h"
 #include "ppc-framework/protocol/Message.h"
@@ -256,10 +257,14 @@ class IFront : virtual public IFrontClient
     virtual void registerComponent(std::string const& component) = 0;
     virtual void unRegisterComponent(std::string const& component) = 0;
 
+    virtual void updateMetaInfo(std::string const& meta) = 0;
+
     // get the target nodeList according to the routeInfo
     virtual std::vector selectNodesByRoutePolicy(
         int16_t routeType, ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) = 0;
 
+    virtual INodeDiscovery::Ptr const getNodeDiscovery() = 0;
+
 private:
     ppc::protocol::ReceiveMsgFunc populateErrorCallback(ErrorCallback::Ptr errorCallback)
     {
diff --git a/cpp/ppc-framework/front/INodeDiscovery.h b/cpp/ppc-framework/front/INodeDiscovery.h
new file mode 100644
index 00000000..b0c4792c
--- /dev/null
+++ b/cpp/ppc-framework/front/INodeDiscovery.h
@@ -0,0 +1,39 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file INodeDiscovery.h
+ * @author: yujiechen
+ * @date 2024-11-06
+ */
+#pragma once
+#include "ppc-framework/protocol/INodeInfo.h"
+#include 
+
+namespace ppc::front
+{
+class INodeDiscovery
+{
+public:
+    using Ptr = std::shared_ptr;
+    INodeDiscovery() = default;
+    virtual ~INodeDiscovery() = default;
+
+    virtual void start() = 0;
+    virtual void stop() = 0;
+
+    // Note: use std::shared_ptr here for swig wrapper
+    virtual std::vector> getAliveNodeList() const = 0;
+};
+}  // namespace ppc::front
\ No newline at end of file
diff --git a/cpp/ppc-framework/gateway/IGateway.h b/cpp/ppc-framework/gateway/IGateway.h
index 6b43d101..e917ce20 100644
--- a/cpp/ppc-framework/gateway/IGateway.h
+++ b/cpp/ppc-framework/gateway/IGateway.h
@@ -79,6 +79,8 @@ class IGateway
     // get the target nodeList according to the routeInfo
     virtual std::vector selectNodesByRoutePolicy(ppc::protocol::RouteType routeType,
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) = 0;
+
+    virtual std::vector getAliveNodeList() const = 0;
 };
 
 }  // namespace ppc::gateway
diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h
index 6dbf5281..68b0f862 100644
--- a/cpp/ppc-framework/protocol/INodeInfo.h
+++ b/cpp/ppc-framework/protocol/INodeInfo.h
@@ -61,10 +61,12 @@ class INodeInfo
 
     virtual bool equal(INodeInfo::Ptr const& info)
     {
-        return (nodeID().toBytes() == info->nodeID().toBytes()) &&
-               (copiedComponents() == info->copiedComponents());
+        return (nodeID().toBytes() == info->nodeID().toBytes()) && (endPoint() == info->endPoint());
     }
 
+    virtual std::string meta() const = 0;
+    // the node meta information
+    virtual void setMeta(std::string const& meta) = 0;
     virtual void toJson(Json::Value& jsonObject) const = 0;
 };
 class INodeInfoFactory
@@ -93,7 +95,7 @@ inline std::string printNodeInfo(INodeInfo::Ptr const& nodeInfo)
     {
         components = components + it + ",";
     }
-    stringstream << LOG_KV("components", components);
+    stringstream << LOG_KV("components", components) << LOG_KV("meta", nodeInfo->meta());
     return stringstream.str();
 }
 }  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h
index ac86293b..37909ef0 100644
--- a/cpp/ppc-framework/protocol/Protocol.h
+++ b/cpp/ppc-framework/protocol/Protocol.h
@@ -481,18 +481,4 @@ struct FileInfo
         return oss.str();
     }
 };
-
-const std::string FRONT_NAME = "Front";
-const std::string FRONT_SERVANT_NAME = "FrontServiceObj";
-const std::string FRONT_SERVICE_NAME = "FrontService";
-
-const std::string GATEWAY_NAME = "Gateway";
-const std::string GATEWAY_SERVANT_NAME = "GatewayServiceObj";
-const std::string GATEWAY_SERVICE_NAME = "GatewayService";
-
-inline std::string getPrxDesc(std::string const& _serviceName, std::string const& _objName)
-{
-    return _serviceName + "." + _objName;
-}
-
 }  // namespace ppc::protocol
diff --git a/cpp/ppc-framework/protocol/RouteType.h b/cpp/ppc-framework/protocol/RouteType.h
index 0d481345..3997931a 100644
--- a/cpp/ppc-framework/protocol/RouteType.h
+++ b/cpp/ppc-framework/protocol/RouteType.h
@@ -54,4 +54,4 @@ inline std::ostream& operator<<(std::ostream& _out, RouteType const& _type)
     }
     return _out;
 }
-}  // namespace ppc::protocol
\ No newline at end of file
+}  // namespace ppc::protocol
diff --git a/cpp/ppc-framework/protocol/ServiceType.h b/cpp/ppc-framework/protocol/ServiceType.h
new file mode 100644
index 00000000..4222064a
--- /dev/null
+++ b/cpp/ppc-framework/protocol/ServiceType.h
@@ -0,0 +1,29 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file ServiceType.h
+ * @author: yujiechen
+ * @date 2024-11-07
+ */
+
+#pragma once
+#include 
+#include 
+
+namespace ppc::protocol
+{
+const static std::string PSI_SERVICE_TYPE = "PSI";
+const static std::string MPC_SERVICE_TYPE = "MPC";
+}  // namespace ppc::protocol
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index 95fac22a..fb2aad73 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -156,7 +156,6 @@ void PPCConfig::loadFrontConfig(bool requireTransport,
     // the components
     auto components = pt.get("transport.components", "");
     boost::split(m_frontConfig->mutableComponents(), components, boost::is_any_of(","));
-
     PPCConfig_LOG(INFO) << LOG_DESC("loadFrontConfig") << printFrontDesc(m_frontConfig);
 }
 
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index 2971e259..85b993bd 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -243,6 +243,11 @@ class PPCConfig
 
     uint16_t seqSyncPeriod() const { return m_seqSyncPeriod; }
 
+    std::string accessEntrypoint() const
+    {
+        return m_frontConfig->selfEndPoint().host() + ":" + std::to_string(m_rpcConfig.listenPort);
+    }
+
 private:
     virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp,
         std::string const& sectionName, boost::property_tree::ptree const& pt);
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index cfba5249..531018db 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -32,6 +32,7 @@
 //TODO: optimize here to implement EcdhConnPSI
 #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h"
 #endif
+#include "ppc-framework/protocol/ServiceType.h"
 #include "ppc-front/Front.h"
 #include "ppc-front/FrontConfigImpl.h"
 #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h"
@@ -50,6 +51,7 @@ using namespace ppc::pir;
 using namespace ppc::tools;
 using namespace ppc::crypto;
 using namespace ppc::sdk;
+using namespace ppc::protocol;
 
 Initializer::Initializer(ppc::protocol::NodeArch _arch, std::string const& _configPath)
   : m_arch((uint16_t)_arch), m_configPath(_configPath)
@@ -82,6 +84,14 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
 
     // Note: must set the  m_holdingMessageMinutes before init the node
     TransportBuilder transportBuilder;
+    // register the serviceInfo
+    auto serviceConfig = m_serviceConfigBuilder.buildServiceConfig();
+    auto entryPoint =
+        m_serviceConfigBuilder.buildEntryPoint(PSI_SERVICE_TYPE, m_config->accessEntrypoint());
+    serviceConfig.addEntryPoint(entryPoint);
+    auto serviceMeta = serviceConfig.encode();
+    m_config->frontConfig()->setMeta(serviceMeta);
+    INIT_LOG(INFO) << LOG_DESC("register serviceMeta") << LOG_KV("serviceMeta", serviceMeta);
     if (m_arch == (uint16_t)ppc::protocol::NodeArch::AIR)
     {
         m_transport = transportBuilder.build(SDKMode::AIR, m_config->frontConfig(), gateway);
diff --git a/cpp/wedpr-initializer/Initializer.h b/cpp/wedpr-initializer/Initializer.h
index 348d1040..76d186d3 100644
--- a/cpp/wedpr-initializer/Initializer.h
+++ b/cpp/wedpr-initializer/Initializer.h
@@ -24,6 +24,7 @@
 #include "ppc-framework/rpc/RpcInterface.h"
 #include "ppc-framework/rpc/RpcTypeDef.h"
 #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.h"
+#include "wedpr-protocol/protocol/src/ServiceConfig.h"
 #include "wedpr-transport/sdk/src/Transport.h"
 #include 
 #include 
@@ -86,6 +87,7 @@ class Initializer : public std::enable_shared_from_this
     std::string m_configPath;
     std::shared_ptr m_config;
     ProtocolInitializer::Ptr m_protocolInitializer;
+    ppc::protocol::ServiceConfigBuilder m_serviceConfigBuilder;
 
     std::shared_ptr m_transportBuilder;
     ppc::sdk::Transport::Ptr m_transport;
diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
index af88181d..4c4ad2f5 100644
--- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
+++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
@@ -48,6 +48,8 @@ void AirNodeInitializer::init(std::string const& _configPath)
 
     // init the node
     m_nodeInitializer = std::make_shared(ppc::protocol::NodeArch::AIR, _configPath);
+    // load the rpc config
+    m_nodeInitializer->config()->loadRpcConfig(pt);
 
     // init the gateway
     initGateway(_configPath);
@@ -59,9 +61,6 @@ void AirNodeInitializer::init(std::string const& _configPath)
     m_gateway->registerNodeInfo(m_nodeInitializer->config()->frontConfig()->generateNodeInfo());
 
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
-    // load the rpc config
-    // not specify the certPath in air-mode
-    m_nodeInitializer->config()->loadRpcConfig(pt);
     // init RpcStatusInterface
     RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared();
 
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index 6998876a..31a720bd 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -26,6 +26,7 @@ using namespace bcos;
 using namespace ppc::mpc;
 using namespace ppc::tools;
 
+/// TODO: mpc support the gateway
 void MPCInitializer::init(std::string const& _configPath)
 {
     // init the log
@@ -40,7 +41,6 @@ void MPCInitializer::init(std::string const& _configPath)
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
     // load the rpc config
     auto ppcConfig = std::make_shared();
-    // not specify the certPath in air-mode
     ppcConfig->loadRpcConfig(pt);
     ppcConfig->loadMPCConfig(pt);
     // bool useMysql = pt.get("mpc.use_mysql", false);
diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
index ec0ce5cc..4d54658e 100644
--- a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
+++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
@@ -44,15 +44,14 @@ void ProNodeInitializer::init(std::string const& _configPath)
 
     // init the node
     m_nodeInitializer = std::make_shared(ppc::protocol::NodeArch::PRO, _configPath);
+    // load the rpc config
+    m_nodeInitializer->config()->loadRpcConfig(pt);
 
     // init the node(no need to set the gateway)
     m_nodeInitializer->init(nullptr);
 
 
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
-    // load the rpc config
-    // not specify the certPath in air-mode
-    m_nodeInitializer->config()->loadRpcConfig(pt);
     // init RpcStatusInterface
     RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared();
 
diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
index 850d70f9..01ff278d 100644
--- a/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.cpp
@@ -28,9 +28,11 @@ using namespace grpc;
 using namespace ppc::gateway;
 using namespace ppc::protocol;
 
-GatewayClient::GatewayClient(
-    ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints)
-  : GrpcClient(grpcConfig, endPoints), m_stub(ppc::proto::Gateway::NewStub(m_channel))
+GatewayClient::GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig,
+    std::string const& endPoints, INodeInfoFactory::Ptr nodeInfoFactory)
+  : GrpcClient(grpcConfig, endPoints),
+    m_stub(ppc::proto::Gateway::NewStub(m_channel)),
+    m_nodeInfoFactory(std::move(nodeInfoFactory))
 {
     for (auto const& channel : m_broadcastChannels)
     {
@@ -77,6 +79,28 @@ std::vector GatewayClient::selectNodesByRoutePolicy(
     return std::vector(response->nodelist().begin(), response->nodelist().end());
 }
 
+std::vector GatewayClient::getAliveNodeList() const
+{
+    auto request = std::make_shared();
+    auto response = std::make_shared();
+    auto context = std::make_shared();
+    // lambda keeps the lifecycle for clientContext
+    auto status = m_stub->getAliveNodeList(context.get(), *request, response.get());
+    if (!status.ok())
+    {
+        throw std::runtime_error(
+            "getAliveNodeList failed, code: " + std::to_string(status.error_code()) +
+            ", msg: " + status.error_message());
+    }
+    if (response->error().errorcode() != 0)
+    {
+        throw std::runtime_error(
+            "getAliveNodeList failed, code: " + std::to_string(response->error().errorcode()) +
+            ", msg: " + response->error().errormessage());
+    }
+    return toNodeInfoList(m_nodeInfoFactory, *response);
+}
+
 void GatewayClient::asyncGetPeers(std::function callback)
 {
     auto response = std::make_shared();
@@ -113,7 +137,8 @@ void GatewayClient::asyncGetAgencies(std::vector const& components,
 
 bcos::Error::Ptr GatewayClient::registerNodeInfo(INodeInfo::Ptr const& nodeInfo)
 {
-    std::unique_ptr request(toNodeInfoRequest(nodeInfo));
+    std::unique_ptr request(new ppc::proto::NodeInfo());
+    toNodeInfoRequest(request.get(), nodeInfo);
     return broadCast([&](ChannelInfo const& channel) {
         if (!m_broadcastStubs.count(channel.endPoint))
         {
diff --git a/cpp/wedpr-protocol/grpc/client/GatewayClient.h b/cpp/wedpr-protocol/grpc/client/GatewayClient.h
index 05fe0d2b..149a9052 100644
--- a/cpp/wedpr-protocol/grpc/client/GatewayClient.h
+++ b/cpp/wedpr-protocol/grpc/client/GatewayClient.h
@@ -27,7 +27,8 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient
 {
 public:
     using Ptr = std::shared_ptr;
-    GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints);
+    GatewayClient(ppc::protocol::GrpcConfig::Ptr const& grpcConfig, std::string const& endPoints,
+        INodeInfoFactory::Ptr nodeInfoFactory);
 
     ~GatewayClient() override = default;
 
@@ -69,8 +70,11 @@ class GatewayClient : public ppc::gateway::IGateway, public GrpcClient
     std::vector selectNodesByRoutePolicy(ppc::protocol::RouteType routeType,
         ppc::protocol::MessageOptionalHeader::Ptr const& routeInfo) override;
 
+    std::vector getAliveNodeList() const override;
+
 private:
     std::unique_ptr m_stub;
     std::map> m_broadcastStubs;
+    INodeInfoFactory::Ptr m_nodeInfoFactory;
 };
 }  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
index a6b8b63b..e4eb141c 100644
--- a/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.cpp
@@ -161,6 +161,28 @@ ServerUnaryReactor* GatewayServer::registerNodeInfo(CallbackServerContext* conte
     return reactor;
 }
 
+grpc::ServerUnaryReactor* GatewayServer::getAliveNodeList(grpc::CallbackServerContext* context,
+    const ppc::proto::Empty* request, ppc::proto::NodeInfoList* reply)
+{
+    ServerUnaryReactor* reactor(context->DefaultReactor());
+    try
+    {
+        auto result = m_gateway->getAliveNodeList();
+        toRawNodeInfoList(reply, result);
+        reactor->Finish(Status::OK);
+    }
+    catch (std::exception const& e)
+    {
+        GATEWAY_SERVER_LOG(WARNING) << LOG_DESC("getAliveNodeList exception")
+                                    << LOG_KV("error", boost::diagnostic_information(e));
+        toSerializedError(reply->mutable_error(),
+            std::make_shared(-1,
+                "getAliveNodeList failed for : " + std::string(boost::diagnostic_information(e))));
+        reactor->Finish(Status::OK);
+    }
+    return reactor;
+}
+
 ServerUnaryReactor* GatewayServer::unRegisterNodeInfo(
     CallbackServerContext* context, const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply)
 {
diff --git a/cpp/wedpr-protocol/grpc/server/GatewayServer.h b/cpp/wedpr-protocol/grpc/server/GatewayServer.h
index c4ab5828..53436d4c 100644
--- a/cpp/wedpr-protocol/grpc/server/GatewayServer.h
+++ b/cpp/wedpr-protocol/grpc/server/GatewayServer.h
@@ -48,6 +48,8 @@ class GatewayServer : public ppc::proto::Gateway::CallbackService
     grpc::ServerUnaryReactor* asyncGetAgencies(grpc::CallbackServerContext* context,
         const ppc::proto::Condition* request, ppc::proto::AgenciesInfo* reply) override;
 
+    grpc::ServerUnaryReactor* getAliveNodeList(grpc::CallbackServerContext* context,
+        const ppc::proto::Empty* request, ppc::proto::NodeInfoList* reply) override;
 
     grpc::ServerUnaryReactor* registerNodeInfo(grpc::CallbackServerContext* context,
         const ppc::proto::NodeInfo* nodeInfo, ppc::proto::Error* reply) override;
diff --git a/cpp/wedpr-protocol/proto/pb/NodeInfo.proto b/cpp/wedpr-protocol/proto/pb/NodeInfo.proto
index f238ee1e..7746697b 100644
--- a/cpp/wedpr-protocol/proto/pb/NodeInfo.proto
+++ b/cpp/wedpr-protocol/proto/pb/NodeInfo.proto
@@ -9,6 +9,8 @@ message NodeInfo{
     // the components
     repeated string components = 3;
     string topic = 4;
+    // the meta information
+    string meta = 5;
 };
 
 message GatewayNodeInfo{
diff --git a/cpp/wedpr-protocol/proto/pb/Service.proto b/cpp/wedpr-protocol/proto/pb/Service.proto
index 47d2e069..8980aa74 100644
--- a/cpp/wedpr-protocol/proto/pb/Service.proto
+++ b/cpp/wedpr-protocol/proto/pb/Service.proto
@@ -57,12 +57,19 @@ message NodeList{
     repeated string nodeList = 1;
     Error error = 2;
 };
+
+message NodeInfoList{
+    repeated NodeInfo nodeList = 1;
+    Error error = 2;
+};
+
 service Front {
     rpc onReceiveMessage (ReceivedMessage) returns (Error) {}
 }
 service Gateway{
     rpc asyncSendMessage(SendedMessageRequest) returns(Error){}
     rpc asyncGetPeers(Empty)returns(PeersInfo){}
+    rpc getAliveNodeList(Empty)returns(NodeInfoList){}
     rpc selectNodesByRoutePolicy(SelectRouteRequest)returns(NodeList){}
     rpc asyncGetAgencies(Condition)returns(AgenciesInfo){}
     rpc registerNodeInfo(NodeInfo) returns(Error){}
diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
index 57da2dd2..c7fc72b0 100644
--- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
+++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
@@ -123,6 +123,18 @@ class NodeInfoImpl : public INodeInfo
 
     void toJson(Json::Value& jsonObject) const override;
 
+    std::string meta() const override
+    {
+        bcos::ReadGuard l(x_rawNodeInfo);
+        return m_rawNodeInfo->meta();
+    }
+    // the node meta information
+    void setMeta(std::string const& meta) override
+    {
+        bcos::WriteGuard l(x_rawNodeInfo);
+        m_rawNodeInfo->set_meta(meta);
+    }
+
     virtual void encodeFields() const;
 
 protected:
@@ -132,7 +144,9 @@ class NodeInfoImpl : public INodeInfo
     std::shared_ptr m_front;
     std::set m_components;
     mutable bcos::SharedMutex x_components;
+
     std::shared_ptr m_rawNodeInfo;
+    mutable bcos::SharedMutex x_rawNodeInfo;
 };
 
 class NodeInfoFactory : public INodeInfoFactory
diff --git a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
index 9079bbd5..61394dde 100644
--- a/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
+++ b/cpp/wedpr-protocol/protobuf/src/RequestConverter.h
@@ -91,12 +91,11 @@ inline ppc::proto::NodeInfo* toNodeInfoRequest(
     return request;
 }
 
-inline ppc::proto::NodeInfo* toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo)
+inline void toNodeInfoRequest(ppc::proto::NodeInfo* request, INodeInfo::Ptr const& nodeInfo)
 {
-    auto request = new ppc::proto::NodeInfo();
     if (!nodeInfo)
     {
-        return request;
+        return;
     };
     *(request->mutable_nodeid()) =
         std::string_view((const char*)nodeInfo->nodeID().data(), nodeInfo->nodeID().size());
@@ -106,7 +105,17 @@ inline ppc::proto::NodeInfo* toNodeInfoRequest(INodeInfo::Ptr const& nodeInfo)
     {
         request->add_components(component);
     }
-    return request;
+    request->set_meta(nodeInfo->meta());
+}
+
+inline void toRawNodeInfoList(
+    ppc::proto::NodeInfoList* rawNodeInfoList, std::vector const& nodeInfoList)
+{
+    for (auto const& it : nodeInfoList)
+    {
+        auto rawNodeInfo = rawNodeInfoList->add_nodelist();
+        toNodeInfoRequest(rawNodeInfo, it);
+    }
 }
 
 inline INodeInfo::Ptr toNodeInfo(
@@ -122,9 +131,21 @@ inline INodeInfo::Ptr toNodeInfo(
         componentTypeList.insert(serializedNodeInfo.components(i));
     }
     nodeInfo->setComponents(componentTypeList);
+    nodeInfo->setMeta(serializedNodeInfo.meta());
     return nodeInfo;
 }
 
+inline std::vector toNodeInfoList(INodeInfoFactory::Ptr const& nodeInfoFactory,
+    ppc::proto::NodeInfoList const& serializedNodeListInfo)
+{
+    std::vector result;
+    for (int i = 0; i < serializedNodeListInfo.nodelist_size(); i++)
+    {
+        result.emplace_back(toNodeInfo(nodeInfoFactory, serializedNodeListInfo.nodelist(i)));
+    }
+    return result;
+}
+
 inline bcos::Error::Ptr toError(grpc::Status const& status, ppc::proto::Error const& error)
 {
     if (!status.ok())
diff --git a/cpp/wedpr-protocol/protocol/src/ServiceConfig.cpp b/cpp/wedpr-protocol/protocol/src/ServiceConfig.cpp
new file mode 100644
index 00000000..bf5367cb
--- /dev/null
+++ b/cpp/wedpr-protocol/protocol/src/ServiceConfig.cpp
@@ -0,0 +1,84 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file ServiceConfig.cpp
+ * @author: yujiechen
+ * @date 2022-11-4
+ */
+#include "ServiceConfig.h"
+#include "ppc-framework/Common.h"
+
+using namespace ppc::protocol;
+
+
+void EntryPointInfo::encode(Json::Value& json) const
+{
+    json["serviceName"] = m_serviceName;
+    json["entryPoint"] = m_entryPoint;
+}
+
+void EntryPointInfo::decode(Json::Value const& jsonObject)
+{
+    if (jsonObject.isMember("serviceName"))
+    {
+        m_serviceName = jsonObject["serviceName"].asString();
+    }
+    if (jsonObject.isMember("entryPoint"))
+    {
+        m_entryPoint = jsonObject["entryPoint"].asString();
+    }
+}
+
+
+void ServiceConfig::addEntryPoint(EntryPointInfo entryPoint)
+{
+    m_serviceInfos.emplace_back(std::move(entryPoint));
+}
+
+// encode the serviceConfig
+std::string ServiceConfig::encode() const
+{
+    Json::Value result;
+    Json::Value serviceInfos(Json::arrayValue);
+    for (auto const& it : m_serviceInfos)
+    {
+        Json::Value entryPointInfo;
+        it.encode(entryPointInfo);
+        serviceInfos.append(entryPointInfo);
+    }
+    result["serviceInfos"] = serviceInfos;
+    Json::FastWriter fastWriter;
+    return fastWriter.write(result);
+}
+
+// decode the serviceConfig
+void ServiceConfig::decode(std::string const& data)
+{
+    Json::Value root;
+    Json::Reader jsonReader;
+    if (!jsonReader.parse(std::string(data), root))
+    {
+        BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                  "The task information must be valid json string."));
+    }
+    if (!root.isMember("serviceInfos"))
+    {
+        return;
+    }
+    for (const auto& serviceInfo : root["serviceInfos"])
+    {
+        m_serviceInfos.emplace_back(EntryPointInfo(serviceInfo));
+    }
+}
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/protocol/src/ServiceConfig.h b/cpp/wedpr-protocol/protocol/src/ServiceConfig.h
new file mode 100644
index 00000000..ae6d84b3
--- /dev/null
+++ b/cpp/wedpr-protocol/protocol/src/ServiceConfig.h
@@ -0,0 +1,87 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file ServiceConfig.h
+ * @author: yujiechen
+ * @date 2022-11-4
+ */
+#pragma once
+#include 
+#include 
+#include 
+#include 
+
+namespace ppc::protocol
+{
+class EntryPointInfo
+{
+public:
+    EntryPointInfo(std::string const& serviceName, std::string const& entryPoint)
+      : m_serviceName(serviceName), m_entryPoint(entryPoint)
+    {}
+
+    EntryPointInfo(Json::Value const& jsonObject) { decode(jsonObject); }
+
+    virtual ~EntryPointInfo() = default;
+
+    virtual void encode(Json::Value& json) const;
+
+    virtual void decode(Json::Value const& jsonObject);
+
+    std::string const& serviceName() const { return m_serviceName; }
+    std::string const& entryPoint() const { return m_entryPoint; }
+
+private:
+    std::string m_serviceName;
+    std::string m_entryPoint;
+};
+
+class ServiceConfig
+{
+public:
+    using Ptr = std::shared_ptr;
+    ServiceConfig() = default;
+    ServiceConfig(std::string const& data) { decode(data); }
+    virtual ~ServiceConfig() = default;
+
+    void addEntryPoint(EntryPointInfo entryPoint);
+
+    // encode the serviceConfig
+    virtual std::string encode() const;
+
+    // decode the serviceConfig
+    virtual void decode(std::string const& data);
+    std::vector const& serviceInfos() const { return m_serviceInfos; }
+
+private:
+    std::vector m_serviceInfos;
+};
+
+class ServiceConfigBuilder
+{
+public:
+    using Ptr = std::shared_ptr;
+    ServiceConfigBuilder() = default;
+    virtual ~ServiceConfigBuilder() = default;
+
+    EntryPointInfo buildEntryPoint(std::string const& serviceName, std::string const& entryPoint)
+    {
+        return EntryPointInfo(serviceName, entryPoint);
+    }
+
+    ServiceConfig buildServiceConfig() const { return ServiceConfig(); }
+    ServiceConfig populateServiceConfig(std::string const& data) { return ServiceConfig(data); }
+};
+}  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp
index eaac6f7a..d2359786 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontFactory.cpp
@@ -31,8 +31,16 @@ IFront::Ptr FrontFactory::build(INodeInfoFactory::Ptr nodeInfoFactory,
     auto nodeInfo = nodeInfoFactory->build(
         bcos::bytesConstRef((bcos::byte*)config->nodeID().data(), config->nodeID().size()),
         config->selfEndPoint().entryPoint());
+    // set the component information
+    auto configuredComponents = config->getComponents();
+    nodeInfo->setComponents(
+        std::set(configuredComponents.begin(), configuredComponents.end()));
+    // set the meta
+    nodeInfo->setMeta(config->meta());
+
     FRONT_LOG(INFO) << LOG_DESC("build front") << LOG_KV("nodeID", config->nodeID())
-                    << LOG_KV("endPoint", config->selfEndPoint().entryPoint());
+                    << LOG_KV("endPoint", config->selfEndPoint().entryPoint())
+                    << LOG_KV("meta", config->meta());
     return std::make_shared(threadPool, nodeInfo, messageFactory, routerInfoBuilder,
         gateway, std::make_shared());
 }
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index 91dbddc6..cb976b08 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -18,6 +18,7 @@
  * @date 2024-08-30
  */
 #include "FrontImpl.h"
+#include "NodeDiscovery.h"
 #include "ppc-utilities/Utilities.h"
 
 using namespace bcos;
@@ -34,12 +35,14 @@ FrontImpl::FrontImpl(std::shared_ptr threadPool,
     m_messageFactory(std::move(messageFactory)),
     m_routerInfoBuilder(std::move(routerInfoBuilder)),
     m_ioService(std::move(ioService)),
-    m_gatewayClient(gateway)
+    m_gatewayClient(gateway),
+    m_nodeDiscovery(std::make_shared(gateway))
 {
     m_nodeID = m_nodeInfo->nodeID().toBytes();
     m_callbackManager = std::make_shared(m_threadPool, m_ioService);
 }
 
+
 /**
  * @brief start the IFront
  *
@@ -74,7 +77,13 @@ void FrontImpl::start()
         }
         FRONT_LOG(INFO) << "Front exit";
     });
+    if (m_nodeDiscovery)
+    {
+        m_nodeDiscovery->start();
+    }
 }
+
+
 /**
  * @brief stop the IFront
  *
@@ -104,6 +113,10 @@ void FrontImpl::stop()
             m_thread->detach();
         }
     }
+    if (m_nodeDiscovery)
+    {
+        m_nodeDiscovery->stop();
+    }
 }
 
 void FrontImpl::asyncSendResponse(bcos::bytesConstRef dstNode, std::string const& traceID,
@@ -295,6 +308,13 @@ void FrontImpl::registerComponent(std::string const& component)
                     << LOG_KV("insert", ret);
 }
 
+void FrontImpl::updateMetaInfo(std::string const& meta)
+{
+    // Note: the node will report the latest components
+    m_nodeInfo->setMeta(meta);
+    FRONT_LOG(INFO) << LOG_DESC("updateMetaInfo") << LOG_KV("meta", meta);
+}
+
 void FrontImpl::unRegisterComponent(std::string const& component)
 {
     // Note: the node will report the latest components
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
index 419afe53..2977d16d 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.h
@@ -21,6 +21,7 @@
 #include "CallbackManager.h"
 #include "Common.h"
 #include "ppc-framework/front/IFront.h"
+#include "ppc-framework/front/INodeDiscovery.h"
 #include "ppc-framework/gateway/IGateway.h"
 #include 
 #include 
@@ -184,6 +185,9 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
             (ppc::protocol::RouteType)routeType, routeInfo);
     }
 
+    INodeDiscovery::Ptr const getNodeDiscovery() override { return m_nodeDiscovery; }
+    void updateMetaInfo(std::string const& meta) override;
+
 private:
     void asyncSendMessageToGateway(bool responsePacket,
         ppc::protocol::MessagePayload::Ptr&& frontMessage, ppc::protocol::RouteType routeType,
@@ -197,6 +201,7 @@ class FrontImpl : public IFront, public IFrontClient, public std::enable_shared_
     bcos::bytes m_nodeID;
     std::shared_ptr m_threadPool;
     ppc::protocol::INodeInfo::Ptr m_nodeInfo;
+    INodeDiscovery::Ptr m_nodeDiscovery;
     ppc::protocol::MessagePayloadBuilder::Ptr m_messageFactory;
     ppc::protocol::MessageOptionalHeaderBuilder::Ptr m_routerInfoBuilder;
 
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
new file mode 100644
index 00000000..b1627d79
--- /dev/null
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
@@ -0,0 +1,69 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file NodeDiscovery.cpp
+ * @author: yujiechen
+ * @date 2024-11-06
+ */
+#include "NodeDiscovery.h"
+#include "Common.h"
+
+using namespace ppc::front;
+
+NodeDiscovery::NodeDiscovery(ppc::gateway::IGateway::Ptr gatewayClient)
+  : m_gatewayClient(std::move(gatewayClient))
+{
+    // fetch the meta information every 10s
+    m_metaFetcher = std::make_shared(10 * 1000, "metaFetcher");
+    m_metaFetcher->registerTimeoutHandler([this]() { fetchMetaInfoFromGateway(); });
+}
+
+std::vector NodeDiscovery::getAliveNodeList() const
+{
+    bcos::ReadGuard l(x_aliveNodeList);
+    return m_aliveNodeList;
+}
+
+void NodeDiscovery::start()
+{
+    if (m_metaFetcher)
+    {
+        m_metaFetcher->start();
+    }
+}
+
+void NodeDiscovery::stop()
+{
+    if (m_metaFetcher)
+    {
+        m_metaFetcher->stop();
+    }
+}
+
+void NodeDiscovery::fetchMetaInfoFromGateway()
+{
+    try
+    {
+        auto nodeList = m_gatewayClient->getAliveNodeList();
+        bcos::WriteGuard l(x_aliveNodeList);
+        m_aliveNodeList = nodeList;
+    }
+    catch (std::exception const& e)
+    {
+        FRONT_LOG(WARNING) << LOG_DESC("fetchMetaInfoFromGateway information failed")
+                           << LOG_KV("error", boost::diagnostic_information(e));
+    }
+    m_metaFetcher->restart();
+}
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
new file mode 100644
index 00000000..067460d4
--- /dev/null
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
@@ -0,0 +1,49 @@
+/**
+ *  Copyright (C) 2023 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file NodeDiscovery.h
+ * @author: yujiechen
+ * @date 2024-11-06
+ */
+#pragma once
+#include "ppc-framework/front/INodeDiscovery.h"
+#include "ppc-framework/gateway/IGateway.h"
+#include 
+#include 
+
+namespace ppc::front
+{
+class NodeDiscovery : public INodeDiscovery
+{
+public:
+    using Ptr = std::shared_ptr;
+    NodeDiscovery(ppc::gateway::IGateway::Ptr gatewayClient);
+    ~NodeDiscovery() override = default;
+
+    std::vector> getAliveNodeList() const override;
+    void start() override;
+    void stop() override;
+
+protected:
+    virtual void fetchMetaInfoFromGateway();
+
+private:
+    std::shared_ptr m_metaFetcher;
+    ppc::gateway::IGateway::Ptr m_gatewayClient;
+
+    std::vector m_aliveNodeList;
+    mutable bcos::SharedMutex x_aliveNodeList;
+};
+}  // namespace ppc::front
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
index 061dd567..b3fd2664 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/GatewayImpl.h
@@ -78,6 +78,17 @@ class GatewayImpl : public IGateway, public std::enable_shared_from_thisselectTargetNodes(routeType, routeInfo);
     }
 
+    std::vector getAliveNodeList() const override
+    {
+        auto aliveNodeList = m_localRouter->routerInfo()->nodeList();
+        std::vector result;
+        for (auto const& it : aliveNodeList)
+        {
+            result.emplace_back(it.second);
+        }
+        return result;
+    }
+
 protected:
     virtual void onReceiveP2PMessage(
         bcos::boostssl::MessageFace::Ptr msg, bcos::boostssl::ws::WsSession::Ptr session);
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
index 00a1e503..3b3bb35f 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
@@ -95,6 +95,23 @@ bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info)
     // the node info has not been updated
     if (existedNodeInfo != nullptr && existedNodeInfo->equal(info))
     {
+        auto meta = info->meta();
+        // update the meta
+        if (meta != existedNodeInfo->meta())
+        {
+            existedNodeInfo->setMeta(meta);
+            GATEWAY_LOG(INFO) << LOG_DESC("tryAddNodeInfo, update the meta, updated nodeInfo")
+                              << printNodeInfo(existedNodeInfo);
+        }
+        // update the components
+        auto components = info->copiedComponents();
+        if (components != existedNodeInfo->copiedComponents())
+        {
+            existedNodeInfo->setComponents(
+                std::set(components.begin(), components.end()));
+            GATEWAY_LOG(INFO) << LOG_DESC("tryAddNodeInfo, update the components, updated nodeInfo")
+                              << printNodeInfo(existedNodeInfo);
+        }
         return false;
     }
     {
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle
index 5010ee85..40ccf763 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/build.gradle
@@ -20,6 +20,7 @@ ext {
     if (!project.hasProperty("ossrhPassword")) {
         ossrhPassword="xxx"
     }
+    jacksonVersion = "2.17.1"
     lombokVersion = "1.18.32"
     junitVersion = "4.13.2"
     log4jVersion = "2.23.0"
@@ -83,6 +84,7 @@ dependencies {
 
     implementation("org.projectlombok:lombok:${lombokVersion}")
     implementation("org.apache.commons:commons-lang3:${apacheCommonLangVersion}")
+	implementation("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
     jmhImplementation ("org.openjdk.jmh:jmh-core:${jmhVersion}")
     jmhAnnotationProcessor ("org.openjdk.jmh:jmh-generator-annprocess:${jmhVersion}")
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java
index 64bb8321..3f121c96 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Common.java
@@ -33,4 +33,11 @@ public static void checkResult(String interfaceName, Error error) throws WeDPRSD
                 error.errorCode(),
                 "call " + interfaceName + " failed for: " + error.errorMessage());
     }
+
+    public static String getUrl(String url) {
+        if (url.startsWith("http://")) {
+            return url;
+        }
+        return String.format("http://%s", url);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java
index df0ec8e2..4c258d0c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/Constant.java
@@ -21,4 +21,5 @@ public class Constant {
     public static int FAILED = -1;
 
     public static String SUCCESS_MSG = "success";
+    public static final String URI_SPLITER = "/";
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/ObjectMapperFactory.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/ObjectMapperFactory.java
new file mode 100644
index 00000000..a80145e0
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/common/ObjectMapperFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017-2025  [webank-wedpr]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+package com.webank.wedpr.sdk.jni.common;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+/** Factory for managing our ObjectMapper instances. */
+public class ObjectMapperFactory {
+
+    private static final ObjectMapper DEFAULT_OBJECT_MAPPER = new ObjectMapper();
+
+    static {
+        configureObjectMapper();
+    }
+
+    private ObjectMapperFactory() {}
+
+    public static ObjectMapper getObjectMapper() {
+        return DEFAULT_OBJECT_MAPPER;
+    }
+
+    public static ObjectReader getObjectReader() {
+        return DEFAULT_OBJECT_MAPPER.reader();
+    }
+
+    private static void configureObjectMapper() {
+        ObjectMapperFactory.DEFAULT_OBJECT_MAPPER.configure(
+                JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+        ObjectMapperFactory.DEFAULT_OBJECT_MAPPER.configure(
+                DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
index 87482370..0305335e 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/demo/TransportDemo.java
@@ -25,6 +25,7 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
 import com.webank.wedpr.sdk.jni.transport.impl.RouteType;
 import com.webank.wedpr.sdk.jni.transport.impl.TransportImpl;
+import com.webank.wedpr.sdk.jni.transport.model.ServiceMeta;
 import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint;
 import java.util.List;
 import lombok.SneakyThrows;
@@ -128,6 +129,10 @@ public static void main(String[] args) throws Exception {
         String listenIp = "0.0.0.0";
         TransportEndPoint endPoint = new TransportEndPoint(hostIp, listenIp, listenPort);
         transportConfig.setSelfEndPoint(endPoint);
+        String serviceName = "Service_Transport_DEMO";
+        String entrypPoint = hostIp + ":" + listenPort;
+        transportConfig.registerService(serviceName, entrypPoint);
+
         String grpcTarget = "ipv4:127.0.0.1:40600,127.0.0.1:40601";
         if (args.length > 3) {
             grpcTarget = args[3];
@@ -157,8 +162,22 @@ public static void main(String[] args) throws Exception {
         // every 2s send a message
         Integer i = 0;
         String syncTopic = "sync_" + topic;
+        // update the service information
+        String serviceName2 = "Service_Transport_DEMO2";
+        transport.registerService(serviceName2, entrypPoint);
         while (true) {
             try {
+                // fetch the alive service information
+                List result =
+                        transport.getAliveEntryPoints(serviceName);
+                System.out.println(
+                        "#### getAliveEntryPoints1, result: " + StringUtils.join(result));
+
+                List result2 =
+                        transport.getAliveEntryPoints(serviceName2);
+                System.out.println(
+                        "#### getAliveEntryPoints2, result2: " + StringUtils.join(result2));
+
                 String payLoad = "testPayload" + i;
                 // send Message by nodeID
                 transport.asyncSendMessageByNodeID(
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
index af3b8bcb..472a74c0 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/FrontConfig.java
@@ -89,9 +89,9 @@ public GrpcConfig grpcConfig() {
         return (cPtr == 0) ? null : new GrpcConfig(cPtr, true);
     }
 
-    public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr generateNodeInfo() {
-        return new SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr(
-                wedpr_java_transportJNI.FrontConfig_generateNodeInfo(swigCPtr, this), true);
+    public INodeInfo generateNodeInfo() {
+        long cPtr = wedpr_java_transportJNI.FrontConfig_generateNodeInfo(swigCPtr, this);
+        return (cPtr == 0) ? null : new INodeInfo(cPtr, true);
     }
 
     public StringVec getComponents() {
@@ -113,6 +113,14 @@ public StringVec mutableComponents() {
                 wedpr_java_transportJNI.FrontConfig_mutableComponents(swigCPtr, this), false);
     }
 
+    public String meta() {
+        return wedpr_java_transportJNI.FrontConfig_meta(swigCPtr, this);
+    }
+
+    public void setMeta(String meta) {
+        wedpr_java_transportJNI.FrontConfig_setMeta(swigCPtr, this, meta);
+    }
+
     public void disOwnMemory() {
         swigSetCMemOwn(false);
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
index 38dd9e9a..a401a47e 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/IFront.java
@@ -155,9 +155,9 @@ public void asyncGetPeers(GetPeersInfoHandler getPeersCallback) {
                 swigCPtr, this, GetPeersInfoHandler.getCPtr(getPeersCallback), getPeersCallback);
     }
 
-    public SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr nodeInfo() {
-        return new SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr(
-                wedpr_java_transportJNI.IFront_nodeInfo(swigCPtr, this), false);
+    public INodeInfo nodeInfo() {
+        long cPtr = wedpr_java_transportJNI.IFront_nodeInfo(swigCPtr, this);
+        return (cPtr == 0) ? null : new INodeInfo(cPtr, true);
     }
 
     /**
@@ -190,6 +190,10 @@ public void unRegisterComponent(String component) {
         wedpr_java_transportJNI.IFront_unRegisterComponent(swigCPtr, this, component);
     }
 
+    public void updateMetaInfo(String meta) {
+        wedpr_java_transportJNI.IFront_updateMetaInfo(swigCPtr, this, meta);
+    }
+
     public StringVec selectNodesByRoutePolicy(short routeType, MessageOptionalHeader routeInfo) {
         return new StringVec(
                 wedpr_java_transportJNI.IFront_selectNodesByRoutePolicy(
@@ -200,4 +204,9 @@ public StringVec selectNodesByRoutePolicy(short routeType, MessageOptionalHeader
                         routeInfo),
                 true);
     }
+
+    public INodeDiscovery getNodeDiscovery() {
+        long cPtr = wedpr_java_transportJNI.IFront_getNodeDiscovery(swigCPtr, this);
+        return (cPtr == 0) ? null : new INodeDiscovery(cPtr, true);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeDiscovery.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeDiscovery.java
new file mode 100644
index 00000000..2b35ff23
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeDiscovery.java
@@ -0,0 +1,47 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class INodeDiscovery {
+    private transient long swigCPtr;
+    private transient boolean swigCMemOwn;
+
+    protected INodeDiscovery(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(INodeDiscovery obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected void swigSetCMemOwn(boolean own) {
+        swigCMemOwn = own;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_INodeDiscovery(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public NodeInfoVec getAliveNodeList() {
+        return new NodeInfoVec(
+                wedpr_java_transportJNI.INodeDiscovery_getAliveNodeList(swigCPtr, this), true);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeInfo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeInfo.java
new file mode 100644
index 00000000..f7b791c9
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/INodeInfo.java
@@ -0,0 +1,85 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class INodeInfo {
+    private transient long swigCPtr;
+    private transient boolean swigCMemOwn;
+
+    protected INodeInfo(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(INodeInfo obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected void swigSetCMemOwn(boolean own) {
+        swigCMemOwn = own;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_INodeInfo(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public String endPoint() {
+        return wedpr_java_transportJNI.INodeInfo_endPoint(swigCPtr, this);
+    }
+
+    public SWIGTYPE_p_bcos__bytesConstRef nodeID() {
+        return new SWIGTYPE_p_bcos__bytesConstRef(
+                wedpr_java_transportJNI.INodeInfo_nodeID(swigCPtr, this), true);
+    }
+
+    public void setNodeID(SWIGTYPE_p_bcos__bytesConstRef nodeID) {
+        wedpr_java_transportJNI.INodeInfo_setNodeID(
+                swigCPtr, this, SWIGTYPE_p_bcos__bytesConstRef.getCPtr(nodeID));
+    }
+
+    public void setEndPoint(String endPoint) {
+        wedpr_java_transportJNI.INodeInfo_setEndPoint(swigCPtr, this, endPoint);
+    }
+
+    public boolean addComponent(String component) {
+        return wedpr_java_transportJNI.INodeInfo_addComponent(swigCPtr, this, component);
+    }
+
+    public boolean eraseComponent(String component) {
+        return wedpr_java_transportJNI.INodeInfo_eraseComponent(swigCPtr, this, component);
+    }
+
+    public boolean componentExist(String component) {
+        return wedpr_java_transportJNI.INodeInfo_componentExist(swigCPtr, this, component);
+    }
+
+    public StringVec copiedComponents() {
+        return new StringVec(
+                wedpr_java_transportJNI.INodeInfo_copiedComponents(swigCPtr, this), true);
+    }
+
+    public String meta() {
+        return wedpr_java_transportJNI.INodeInfo_meta(swigCPtr, this);
+    }
+
+    public void setMeta(String meta) {
+        wedpr_java_transportJNI.INodeInfo_setMeta(swigCPtr, this, meta);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/NodeInfoVec.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/NodeInfoVec.java
new file mode 100644
index 00000000..c2a45062
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/NodeInfoVec.java
@@ -0,0 +1,175 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class NodeInfoVec extends java.util.AbstractList
+        implements java.util.RandomAccess {
+    private transient long swigCPtr;
+    protected transient boolean swigCMemOwn;
+
+    protected NodeInfoVec(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(NodeInfoVec obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected static long swigRelease(NodeInfoVec obj) {
+        long ptr = 0;
+        if (obj != null) {
+            if (!obj.swigCMemOwn)
+                throw new RuntimeException("Cannot release ownership as memory is not owned");
+            ptr = obj.swigCPtr;
+            obj.swigCMemOwn = false;
+            obj.delete();
+        }
+        return ptr;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_NodeInfoVec(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public NodeInfoVec(INodeInfo[] initialElements) {
+        this();
+        reserve(initialElements.length);
+
+        for (INodeInfo element : initialElements) {
+            add(element);
+        }
+    }
+
+    public NodeInfoVec(Iterable initialElements) {
+        this();
+        for (INodeInfo element : initialElements) {
+            add(element);
+        }
+    }
+
+    public INodeInfo get(int index) {
+        return doGet(index);
+    }
+
+    public INodeInfo set(int index, INodeInfo e) {
+        return doSet(index, e);
+    }
+
+    public boolean add(INodeInfo e) {
+        modCount++;
+        doAdd(e);
+        return true;
+    }
+
+    public void add(int index, INodeInfo e) {
+        modCount++;
+        doAdd(index, e);
+    }
+
+    public INodeInfo remove(int index) {
+        modCount++;
+        return doRemove(index);
+    }
+
+    protected void removeRange(int fromIndex, int toIndex) {
+        modCount++;
+        doRemoveRange(fromIndex, toIndex);
+    }
+
+    public int size() {
+        return doSize();
+    }
+
+    public int capacity() {
+        return doCapacity();
+    }
+
+    public void reserve(int n) {
+        doReserve(n);
+    }
+
+    public NodeInfoVec() {
+        this(wedpr_java_transportJNI.new_NodeInfoVec__SWIG_0(), true);
+    }
+
+    public NodeInfoVec(NodeInfoVec other) {
+        this(
+                wedpr_java_transportJNI.new_NodeInfoVec__SWIG_1(NodeInfoVec.getCPtr(other), other),
+                true);
+    }
+
+    public boolean isEmpty() {
+        return wedpr_java_transportJNI.NodeInfoVec_isEmpty(swigCPtr, this);
+    }
+
+    public void clear() {
+        wedpr_java_transportJNI.NodeInfoVec_clear(swigCPtr, this);
+    }
+
+    public NodeInfoVec(int count, INodeInfo value) {
+        this(
+                wedpr_java_transportJNI.new_NodeInfoVec__SWIG_2(
+                        count, INodeInfo.getCPtr(value), value),
+                true);
+    }
+
+    private int doCapacity() {
+        return wedpr_java_transportJNI.NodeInfoVec_doCapacity(swigCPtr, this);
+    }
+
+    private void doReserve(int n) {
+        wedpr_java_transportJNI.NodeInfoVec_doReserve(swigCPtr, this, n);
+    }
+
+    private int doSize() {
+        return wedpr_java_transportJNI.NodeInfoVec_doSize(swigCPtr, this);
+    }
+
+    private void doAdd(INodeInfo x) {
+        wedpr_java_transportJNI.NodeInfoVec_doAdd__SWIG_0(swigCPtr, this, INodeInfo.getCPtr(x), x);
+    }
+
+    private void doAdd(int index, INodeInfo x) {
+        wedpr_java_transportJNI.NodeInfoVec_doAdd__SWIG_1(
+                swigCPtr, this, index, INodeInfo.getCPtr(x), x);
+    }
+
+    private INodeInfo doRemove(int index) {
+        long cPtr = wedpr_java_transportJNI.NodeInfoVec_doRemove(swigCPtr, this, index);
+        return (cPtr == 0) ? null : new INodeInfo(cPtr, true);
+    }
+
+    private INodeInfo doGet(int index) {
+        long cPtr = wedpr_java_transportJNI.NodeInfoVec_doGet(swigCPtr, this, index);
+        return (cPtr == 0) ? null : new INodeInfo(cPtr, true);
+    }
+
+    private INodeInfo doSet(int index, INodeInfo val) {
+        long cPtr =
+                wedpr_java_transportJNI.NodeInfoVec_doSet(
+                        swigCPtr, this, index, INodeInfo.getCPtr(val), val);
+        return (cPtr == 0) ? null : new INodeInfo(cPtr, true);
+    }
+
+    private void doRemoveRange(int fromIndex, int toIndex) {
+        wedpr_java_transportJNI.NodeInfoVec_doRemoveRange(swigCPtr, this, fromIndex, toIndex);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeDiscovery.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeDiscovery.java
new file mode 100644
index 00000000..4b6e0677
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeDiscovery.java
@@ -0,0 +1,54 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class SharedNodeDiscovery {
+    private transient long swigCPtr;
+    protected transient boolean swigCMemOwn;
+
+    protected SharedNodeDiscovery(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(SharedNodeDiscovery obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected static long swigRelease(SharedNodeDiscovery obj) {
+        long ptr = 0;
+        if (obj != null) {
+            if (!obj.swigCMemOwn)
+                throw new RuntimeException("Cannot release ownership as memory is not owned");
+            ptr = obj.swigCPtr;
+            obj.swigCMemOwn = false;
+            obj.delete();
+        }
+        return ptr;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_SharedNodeDiscovery(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public SharedNodeDiscovery() {
+        this(wedpr_java_transportJNI.new_SharedNodeDiscovery(), true);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeInfo.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeInfo.java
new file mode 100644
index 00000000..09fb7ce3
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/SharedNodeInfo.java
@@ -0,0 +1,54 @@
+/* ----------------------------------------------------------------------------
+ * This file was automatically generated by SWIG (https://www.swig.org).
+ * Version 4.2.1
+ *
+ * Do not make changes to this file unless you know what you are doing - modify
+ * the SWIG interface file instead.
+ * ----------------------------------------------------------------------------- */
+
+package com.webank.wedpr.sdk.jni.generated;
+
+public class SharedNodeInfo {
+    private transient long swigCPtr;
+    protected transient boolean swigCMemOwn;
+
+    protected SharedNodeInfo(long cPtr, boolean cMemoryOwn) {
+        swigCMemOwn = cMemoryOwn;
+        swigCPtr = cPtr;
+    }
+
+    protected static long getCPtr(SharedNodeInfo obj) {
+        return (obj == null) ? 0 : obj.swigCPtr;
+    }
+
+    protected static long swigRelease(SharedNodeInfo obj) {
+        long ptr = 0;
+        if (obj != null) {
+            if (!obj.swigCMemOwn)
+                throw new RuntimeException("Cannot release ownership as memory is not owned");
+            ptr = obj.swigCPtr;
+            obj.swigCMemOwn = false;
+            obj.delete();
+        }
+        return ptr;
+    }
+
+    @SuppressWarnings({"deprecation", "removal"})
+    protected void finalize() {
+        delete();
+    }
+
+    public synchronized void delete() {
+        if (swigCPtr != 0) {
+            if (swigCMemOwn) {
+                swigCMemOwn = false;
+                wedpr_java_transportJNI.delete_SharedNodeInfo(swigCPtr);
+            }
+            swigCPtr = 0;
+        }
+    }
+
+    public SharedNodeInfo() {
+        this(wedpr_java_transportJNI.new_SharedNodeInfo(), true);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
index d8b77685..802d61d5 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
@@ -21,4 +21,8 @@ public static String printOptionalField(MessageOptionalHeader optionalHeader) {
     public static String printMessage(Message _msg) {
         return wedpr_java_transportJNI.printMessage(Message.getCPtr(_msg), _msg);
     }
+
+    public static String printNodeInfo(INodeInfo nodeInfo) {
+        return wedpr_java_transportJNI.printNodeInfo(INodeInfo.getCPtr(nodeInfo), nodeInfo);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index bf581fb4..371a44b1 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -25,6 +25,10 @@ public class wedpr_java_transportJNI {
 
     public static final native void delete_SharedFront(long jarg1);
 
+    public static final native long new_SharedNodeDiscovery();
+
+    public static final native void delete_SharedNodeDiscovery(long jarg1);
+
     public static final native long new_SharedFrontClient();
 
     public static final native void delete_SharedFrontClient(long jarg1);
@@ -81,6 +85,10 @@ public class wedpr_java_transportJNI {
 
     public static final native void delete_SharedRouteInfoBuilder(long jarg1);
 
+    public static final native long new_SharedNodeInfo();
+
+    public static final native void delete_SharedNodeInfo(long jarg1);
+
     public static final native long new_ubytes__SWIG_0();
 
     public static final native long new_ubytes__SWIG_1(long jarg1, ubytes jarg1_);
@@ -180,6 +188,42 @@ public static final native void StringVec_doRemoveRange(
 
     public static final native void delete_StringVec(long jarg1);
 
+    public static final native long new_NodeInfoVec__SWIG_0();
+
+    public static final native long new_NodeInfoVec__SWIG_1(long jarg1, NodeInfoVec jarg1_);
+
+    public static final native boolean NodeInfoVec_isEmpty(long jarg1, NodeInfoVec jarg1_);
+
+    public static final native void NodeInfoVec_clear(long jarg1, NodeInfoVec jarg1_);
+
+    public static final native long new_NodeInfoVec__SWIG_2(
+            int jarg1, long jarg2, INodeInfo jarg2_);
+
+    public static final native int NodeInfoVec_doCapacity(long jarg1, NodeInfoVec jarg1_);
+
+    public static final native void NodeInfoVec_doReserve(
+            long jarg1, NodeInfoVec jarg1_, int jarg2);
+
+    public static final native int NodeInfoVec_doSize(long jarg1, NodeInfoVec jarg1_);
+
+    public static final native void NodeInfoVec_doAdd__SWIG_0(
+            long jarg1, NodeInfoVec jarg1_, long jarg2, INodeInfo jarg2_);
+
+    public static final native void NodeInfoVec_doAdd__SWIG_1(
+            long jarg1, NodeInfoVec jarg1_, int jarg2, long jarg3, INodeInfo jarg3_);
+
+    public static final native long NodeInfoVec_doRemove(long jarg1, NodeInfoVec jarg1_, int jarg2);
+
+    public static final native long NodeInfoVec_doGet(long jarg1, NodeInfoVec jarg1_, int jarg2);
+
+    public static final native long NodeInfoVec_doSet(
+            long jarg1, NodeInfoVec jarg1_, int jarg2, long jarg3, INodeInfo jarg3_);
+
+    public static final native void NodeInfoVec_doRemoveRange(
+            long jarg1, NodeInfoVec jarg1_, int jarg2, int jarg3);
+
+    public static final native void delete_NodeInfoVec(long jarg1);
+
     public static final native long Error_buildError__SWIG_0(
             byte[] jarg1, byte[] jarg2, int jarg3, int jarg4, String jarg5);
 
@@ -280,6 +324,11 @@ public static final native void FrontConfig_addComponent(
 
     public static final native long FrontConfig_mutableComponents(long jarg1, FrontConfig jarg1_);
 
+    public static final native String FrontConfig_meta(long jarg1, FrontConfig jarg1_);
+
+    public static final native void FrontConfig_setMeta(
+            long jarg1, FrontConfig jarg1_, String jarg2);
+
     public static final native void delete_FrontConfigBuilder(long jarg1);
 
     public static final native long FrontConfigBuilder_build__SWIG_0(
@@ -605,6 +654,34 @@ public static final native long MessagePayloadBuilder_build__SWIG_0(
     public static final native long MessagePayloadBuilder_build__SWIG_1(
             long jarg1, MessagePayloadBuilder jarg1_, long jarg2);
 
+    public static final native void delete_INodeInfo(long jarg1);
+
+    public static final native String INodeInfo_endPoint(long jarg1, INodeInfo jarg1_);
+
+    public static final native long INodeInfo_nodeID(long jarg1, INodeInfo jarg1_);
+
+    public static final native void INodeInfo_setNodeID(long jarg1, INodeInfo jarg1_, long jarg2);
+
+    public static final native void INodeInfo_setEndPoint(
+            long jarg1, INodeInfo jarg1_, String jarg2);
+
+    public static final native boolean INodeInfo_addComponent(
+            long jarg1, INodeInfo jarg1_, String jarg2);
+
+    public static final native boolean INodeInfo_eraseComponent(
+            long jarg1, INodeInfo jarg1_, String jarg2);
+
+    public static final native boolean INodeInfo_componentExist(
+            long jarg1, INodeInfo jarg1_, String jarg2);
+
+    public static final native long INodeInfo_copiedComponents(long jarg1, INodeInfo jarg1_);
+
+    public static final native String INodeInfo_meta(long jarg1, INodeInfo jarg1_);
+
+    public static final native void INodeInfo_setMeta(long jarg1, INodeInfo jarg1_, String jarg2);
+
+    public static final native String printNodeInfo(long jarg1, INodeInfo jarg1_);
+
     public static final native void delete_IFrontClient(long jarg1);
 
     public static final native void IFrontClient_onReceiveMessage(
@@ -745,14 +822,23 @@ public static final native void IFront_registerComponent(
     public static final native void IFront_unRegisterComponent(
             long jarg1, IFront jarg1_, String jarg2);
 
+    public static final native void IFront_updateMetaInfo(long jarg1, IFront jarg1_, String jarg2);
+
     public static final native long IFront_selectNodesByRoutePolicy(
             long jarg1, IFront jarg1_, short jarg2, long jarg3, MessageOptionalHeader jarg3_);
 
+    public static final native long IFront_getNodeDiscovery(long jarg1, IFront jarg1_);
+
     public static final native void delete_IFrontBuilder(long jarg1);
 
     public static final native long IFrontBuilder_buildClient(
             long jarg1, IFrontBuilder jarg1_, String jarg2, long jarg3, boolean jarg4);
 
+    public static final native void delete_INodeDiscovery(long jarg1);
+
+    public static final native long INodeDiscovery_getAliveNodeList(
+            long jarg1, INodeDiscovery jarg1_);
+
     public static final native int SDKMode_AIR_get();
 
     public static final native int SDKMode_PRO_get();
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
index 21c5f026..fe04a528 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
@@ -16,8 +16,10 @@
 package com.webank.wedpr.sdk.jni.transport;
 
 import com.webank.wedpr.sdk.jni.common.JniLibLoader;
+import com.webank.wedpr.sdk.jni.common.ObjectMapperFactory;
 import com.webank.wedpr.sdk.jni.generated.FrontConfig;
 import com.webank.wedpr.sdk.jni.generated.TransportBuilder;
+import com.webank.wedpr.sdk.jni.transport.model.ServiceMeta;
 import com.webank.wedpr.sdk.jni.transport.model.TransportEndPoint;
 import com.webank.wedpr.sdk.jni.transport.model.TransportGrpcConfig;
 import java.util.List;
@@ -52,6 +54,7 @@ public static TransportBuilder getTransportBuilder() {
     private final FrontConfig frontConfig;
     private List components;
     private TransportEndPoint selfEndPoint;
+    private ServiceMeta serviceMeta = new ServiceMeta();
 
     public TransportConfig(Integer threadPoolSize, String nodeID) {
         this.frontConfig = transportBuilder.buildConfig(threadPoolSize, nodeID);
@@ -86,6 +89,23 @@ public void setComponents(List components) {
         }
     }
 
+    public void registerService(ServiceMeta serviceMeta) {
+        if (serviceMeta != null) {
+            this.serviceMeta = serviceMeta;
+        }
+    }
+
+    public void registerService(String serviceName, String entryPoint) throws Exception {
+        serviceMeta.addEntryPoint(new ServiceMeta.EntryPointMeta(serviceName, entryPoint));
+        // update the meta
+        this.frontConfig.setMeta(
+                ObjectMapperFactory.getObjectMapper().writeValueAsString(serviceMeta));
+    }
+
+    public ServiceMeta getServiceMeta() {
+        return this.serviceMeta;
+    }
+
     public FrontConfig getFrontConfig() {
         return frontConfig;
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
index b1e03756..93dc1fa9 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/WeDPRTransport.java
@@ -21,6 +21,7 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
 import com.webank.wedpr.sdk.jni.transport.impl.RouteType;
+import com.webank.wedpr.sdk.jni.transport.model.ServiceMeta;
 import java.util.List;
 
 public interface WeDPRTransport {
@@ -187,4 +188,10 @@ void pushByInst(String topic, String dstInst, int seq, byte[] payload, int timeo
 
     List selectNodeListByPolicy(
             RouteType routeType, String dstInst, String dstComponent, String dstNode);
+
+    List getAliveEntryPoints(String serviceName);
+
+    void registerService(String serviceName, String entryPoint) throws Exception;
+
+    public TransportConfig getTransportConfig();
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index e507f418..022c396d 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -17,6 +17,7 @@
 
 import com.webank.wedpr.sdk.jni.common.Common;
 import com.webank.wedpr.sdk.jni.common.Constant;
+import com.webank.wedpr.sdk.jni.common.ObjectMapperFactory;
 import com.webank.wedpr.sdk.jni.common.WeDPRSDKException;
 import com.webank.wedpr.sdk.jni.generated.*;
 import com.webank.wedpr.sdk.jni.generated.Error;
@@ -28,6 +29,7 @@
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageDispatcherCallback;
 import com.webank.wedpr.sdk.jni.transport.handlers.MessageErrorCallback;
+import com.webank.wedpr.sdk.jni.transport.model.ServiceMeta;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
@@ -450,4 +452,53 @@ public List selectNodeListByPolicy(
         }
         return nodeList;
     }
+
+    private void parseServiceMeta(
+            List entryPointInfos, String serviceName, String meta) {
+        try {
+            if (StringUtils.isBlank(meta)) {
+                return;
+            }
+            ServiceMeta serviceMeta =
+                    ObjectMapperFactory.getObjectMapper().readValue(meta, ServiceMeta.class);
+            if (serviceMeta.getServiceInfos() == null || serviceMeta.getServiceInfos().isEmpty()) {
+                return;
+            }
+            for (ServiceMeta.EntryPointMeta entryPointMeta : serviceMeta.getServiceInfos()) {
+                if (entryPointMeta.getServiceName().equalsIgnoreCase(serviceName)) {
+                    entryPointInfos.add(entryPointMeta);
+                }
+            }
+        } catch (Exception e) {
+            logger.warn("parseServiceMeta exception, meta: {}", meta, e);
+        }
+    }
+
+    @Override
+    public List getAliveEntryPoints(String serviceName) {
+        NodeInfoVec nodeInfoList = this.transport.getFront().getNodeDiscovery().getAliveNodeList();
+        List result = new ArrayList<>();
+        for (int i = 0; i < nodeInfoList.size(); i++) {
+            parseServiceMeta(result, serviceName, nodeInfoList.get(i).meta());
+        }
+        return result;
+    }
+
+    @Override
+    public void registerService(String serviceName, String entryPoint) throws Exception {
+        transportConfig
+                .getServiceMeta()
+                .addEntryPoint(new ServiceMeta.EntryPointMeta(serviceName, entryPoint));
+        // update the meta
+        this.transport
+                .getFront()
+                .updateMetaInfo(
+                        ObjectMapperFactory.getObjectMapper()
+                                .writeValueAsString(transportConfig.getServiceMeta()));
+    }
+
+    @Override
+    public TransportConfig getTransportConfig() {
+        return transportConfig;
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java
new file mode 100644
index 00000000..c99702a3
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2017-2025  [webank-wedpr]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package com.webank.wedpr.sdk.jni.transport.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.webank.wedpr.sdk.jni.common.Common;
+import com.webank.wedpr.sdk.jni.common.Constant;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.apache.commons.lang3.StringUtils;
+
+@Data
+@NoArgsConstructor
+@ToString
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ServiceMeta {
+    @Data
+    @NoArgsConstructor
+    @ToString
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class EntryPointMeta {
+        private String serviceName;
+        private String entryPoint;
+
+        public EntryPointMeta(String serviceName, String entryPoint) {
+            this.serviceName = serviceName;
+            this.entryPoint = entryPoint;
+        }
+
+        public String getUrl(String uriPath) {
+            if (StringUtils.isBlank(uriPath)) {
+                return Common.getUrl(entryPoint);
+            }
+            if (uriPath.startsWith(Constant.URI_SPLITER)) {
+                return Common.getUrl(entryPoint + uriPath);
+            }
+            return Common.getUrl(entryPoint + Constant.URI_SPLITER + uriPath);
+        }
+    }
+    // Note: here can't been changed
+    private List serviceInfos = new ArrayList<>();
+
+    public void setServiceInfos(List serviceInfos) {
+        if (serviceInfos == null) {
+            return;
+        }
+        this.serviceInfos = serviceInfos;
+    }
+
+    public void addEntryPoint(EntryPointMeta entryPointMeta) {
+        this.serviceInfos.add(entryPointMeta);
+    }
+}
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 708a29b3..f5e146b2 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -909,6 +909,8 @@ SWIGINTERN void SWIG_JavaException(JNIEnv *jenv, int code, const char *msg) {
 #include "wedpr-transport/sdk/src/Transport.h"
 #include "ppc-framework/libwrapper/Buffer.h"
 #include "ppc-framework/front/IFront.h"
+#include "ppc-framework/protocol/INodeInfo.h"
+#include "ppc-framework/front/INodeDiscovery.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-framework/front/FrontConfig.h"
 #include "ppc-framework/protocol/GrpcConfig.h"
@@ -1118,6 +1120,68 @@ SWIGINTERN void std_vector_Sl_std_string_Sg__doRemoveRange(std::vector< std::str
           throw std::out_of_range("vector index out of range");
         }
       }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *new_std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg___SWIG_2(jint count,std::shared_ptr< ppc::protocol::INodeInfo > const &value){
+        if (count < 0)
+          throw std::out_of_range("vector count must be positive");
+        return new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >(static_cast >::size_type>(count), value);
+      }
+SWIGINTERN jint std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doCapacity(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self){
+        return SWIG_VectorSize(self->capacity());
+      }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doReserve(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint n){
+        if (n < 0)
+          throw std::out_of_range("vector reserve size must be positive");
+        self->reserve(n);
+      }
+SWIGINTERN jint std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doSize(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *self){
+        return SWIG_VectorSize(self->size());
+      }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doAdd__SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){
+        self->push_back(x);
+      }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doAdd__SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint index,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){
+        jint size = static_cast(self->size());
+        if (0 <= index && index <= size) {
+          self->insert(self->begin() + index, x);
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doRemove(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint index){
+        jint size = static_cast(self->size());
+        if (0 <= index && index < size) {
+          std::shared_ptr< ppc::protocol::INodeInfo > const old_value = (*self)[index];
+          self->erase(self->begin() + index);
+          return old_value;
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doGet(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint index){
+        jint size = static_cast(self->size());
+        if (index >= 0 && index < size)
+          return (*self)[index];
+        else
+          throw std::out_of_range("vector index out of range");
+      }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doSet(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint index,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &val){
+        jint size = static_cast(self->size());
+        if (index >= 0 && index < size) {
+          std::shared_ptr< ppc::protocol::INodeInfo > const old_value = (*self)[index];
+          (*self)[index] = val;
+          return old_value;
+        }
+        else
+          throw std::out_of_range("vector index out of range");
+      }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doRemoveRange(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,jint fromIndex,jint toIndex){
+        jint size = static_cast(self->size());
+        if (0 <= fromIndex && fromIndex <= toIndex && toIndex <= size) {
+          self->erase(self->begin() + fromIndex, self->begin() + toIndex);
+        } else {
+          throw std::out_of_range("vector index out of range");
+        }
+      }
 
 struct SWIG_null_deleter {
   void operator() (void const *) const {
@@ -1479,6 +1543,30 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedNodeDiscovery(JNIEnv *jenv, jclass jcls) {
+  jlong jresult = 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+  *(std::shared_ptr< ppc::front::INodeDiscovery > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::INodeDiscovery >(*result) : 0;
+  if (1) delete result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedNodeDiscovery(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  std::shared_ptr< ppc::front::INodeDiscovery > *arg1 = (std::shared_ptr< ppc::front::INodeDiscovery > *) 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery > tempnull1 ;
+  
+  (void)jenv;
+  (void)jcls;
+  arg1 = jarg1 ? *(std::shared_ptr< ppc::front::INodeDiscovery > **)&jarg1 : &tempnull1; 
+  delete arg1;
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedFrontClient(JNIEnv *jenv, jclass jcls) {
   jlong jresult = 0 ;
   std::shared_ptr< ppc::front::IFrontClient > *result = 0 ;
@@ -1815,6 +1903,30 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1SharedNodeInfo(JNIEnv *jenv, jclass jcls) {
+  jlong jresult = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+  *(std::shared_ptr< ppc::protocol::INodeInfo > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::INodeInfo >(*result) : 0;
+  if (1) delete result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1SharedNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  std::shared_ptr< ppc::protocol::INodeInfo > *arg1 = (std::shared_ptr< ppc::protocol::INodeInfo > *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempnull1 ;
+  
+  (void)jenv;
+  (void)jcls;
+  arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::INodeInfo > **)&jarg1 : &tempnull1; 
+  delete arg1;
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1ubytes_1_1SWIG_10(JNIEnv *jenv, jclass jcls) {
   jlong jresult = 0 ;
   std::vector< uint8_t > *result = 0 ;
@@ -2673,6 +2785,284 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1NodeInfoVec_1_1SWIG_10(JNIEnv *jenv, jclass jcls) {
+  jlong jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >();
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1NodeInfoVec_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1;
+  if (!arg1) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const & is null");
+    return 0;
+  } 
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const &)*arg1);
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1isEmpty(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jboolean jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  bool result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  result = (bool)((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->empty();
+  jresult = (jboolean)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1clear(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  (arg1)->clear();
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1NodeInfoVec_1_1SWIG_12(JNIEnv *jenv, jclass jcls, jint jarg1, jlong jarg2, jobject jarg2_) {
+  jlong jresult = 0 ;
+  jint arg1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempnull2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg2_;
+  arg1 = jarg1; 
+  arg2 = jarg2 ? *(std::shared_ptr< ppc::protocol::INodeInfo > **)&jarg2 : &tempnull2; 
+  try {
+    result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new_std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg___SWIG_2(SWIG_STD_MOVE(arg1),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doCapacity(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  try {
+    result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doCapacity(arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doReserve(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doReserve(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::length_error &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  try {
+    result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doSize((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doAdd_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempnull2 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  (void)jarg2_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2 ? *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jarg2 : &tempnull2; 
+  std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doAdd__SWIG_0(arg1,(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg2);
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doAdd_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempnull3 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  (void)jarg3_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  arg3 = jarg3 ? *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jarg3 : &tempnull3; 
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doAdd__SWIG_1(arg1,SWIG_STD_MOVE(arg2),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doRemove(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  jlong jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doRemove(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jresult = result ? new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type(result) : 0; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doGet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  jlong jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  try {
+    result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *) &std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doGet(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jresult = *result ? new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type(*result) : 0; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doSet(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jlong jarg3, jobject jarg3_) {
+  jlong jresult = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempnull3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  (void)jarg3_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  arg3 = jarg3 ? *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jarg3 : &tempnull3; 
+  try {
+    result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doSet(arg1,SWIG_STD_MOVE(arg2),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return 0;
+  }
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type **)&jresult = result ? new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type(result) : 0; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_NodeInfoVec_1doRemoveRange(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2, jint jarg3) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  jint arg2 ;
+  jint arg3 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  arg2 = jarg2; 
+  arg3 = jarg3; 
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__doRemoveRange(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaIndexOutOfBoundsException, (&_e)->what());
+    return ;
+  }
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1NodeInfoVec(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
+  delete arg1;
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Error_1buildError_1_1SWIG_10(JNIEnv *jenv, jclass jcls, jbyteArray jarg1, jbyteArray jarg2, jint jarg3, jint jarg4, jstring jarg5) {
   jlong jresult = 0 ;
   char *arg1 = (char *) 0 ;
@@ -3419,7 +3809,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
   result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
-  *(ppc::protocol::INodeInfo::Ptr **)&jresult = new ppc::protocol::INodeInfo::Ptr(result); 
+  *(ppc::protocol::INodeInfo::Ptr **)&jresult = result ? new ppc::protocol::INodeInfo::Ptr(result) : 0; 
   return jresult;
 }
 
@@ -3487,11 +3877,46 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1mutableComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1mutableComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  std::vector< std::string > *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
+  arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (std::vector< std::string > *) &(arg1)->mutableComponents();
+  *(std::vector< std::string > **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1meta(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  std::string result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
+  arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::front::FrontConfig const *)arg1)->meta();
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_FrontConfig_1setMeta(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
   ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::string arg2 ;
   std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  std::vector< std::string > *result = 0 ;
   
   (void)jenv;
   (void)jcls;
@@ -3499,9 +3924,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::vector< std::string > *) &(arg1)->mutableComponents();
-  *(std::vector< std::string > **)&jresult = result; 
-  return jresult;
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  } 
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  (&arg2)->assign(arg2_pstr);
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  (arg1)->setMeta(arg2);
 }
 
 
@@ -5942,6 +6373,261 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1INodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  (void)arg1; delete smartarg1;
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1endPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  std::string *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+  jresult = jenv->NewStringUTF(result->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1nodeID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  bcos::bytesConstRef result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+  *(bcos::bytesConstRef **)&jresult = new bcos::bytesConstRef(result); 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1setNodeID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2) {
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  bcos::bytesConstRef *argp2 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  argp2 = *(bcos::bytesConstRef **)&jarg2; 
+  if (!argp2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null bcos::bytesConstRef");
+    return ;
+  }
+  arg2 = *argp2; 
+  (arg1)->setNodeID(arg2);
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1setEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  (arg1)->setEndPoint((std::string const &)*arg2);
+}
+
+
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1addComponent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  jboolean jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  bool result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return 0;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return 0;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+  jresult = (jboolean)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1eraseComponent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  jboolean jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  bool result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return 0;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return 0;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+  jresult = (jboolean)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1componentExist(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  jboolean jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  bool result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return 0;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return 0;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+  jresult = (jboolean)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1copiedComponents(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  std::vector< std::string > result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+  *(std::vector< std::string > **)&jresult = new std::vector< std::string >(result); 
+  return jresult;
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1meta(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  std::string result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeInfo_1setMeta(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
+  arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  (arg1)->setMeta((std::string const &)*arg2);
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printNodeInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::INodeInfo::Ptr *arg1 = 0 ;
+  ppc::protocol::INodeInfo::Ptr tempnull1 ;
+  std::string result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = jarg1 ? *(ppc::protocol::INodeInfo::Ptr **)&jarg1 : &tempnull1; 
+  result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontClient(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ;
   std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ;
@@ -6748,7 +7434,7 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
   result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
-  *(ppc::protocol::INodeInfo::Ptr **)&jresult = result; 
+  *(ppc::protocol::INodeInfo::Ptr **)&jresult = *result ? new ppc::protocol::INodeInfo::Ptr(*result) : 0; 
   return jresult;
 }
 
@@ -6857,6 +7543,30 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1updateMetaInfo(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
+  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  (arg1)->updateMetaInfo((std::string const &)*arg2);
+}
+
+
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1selectNodesByRoutePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jshort jarg2, jlong jarg3, jobject jarg3_) {
   jlong jresult = 0 ;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
@@ -6881,6 +7591,24 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_IFront_1getNodeDiscovery(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  ppc::front::INodeDiscovery::Ptr result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
+  arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (arg1)->getNodeDiscovery();
+  *(ppc::front::INodeDiscovery::Ptr **)&jresult = result ? new ppc::front::INodeDiscovery::Ptr(result) : 0; 
+  return jresult;
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1IFrontBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
   
@@ -6925,6 +7653,37 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1INodeDiscovery(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+  ppc::front::INodeDiscovery *arg1 = (ppc::front::INodeDiscovery *) 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::front::INodeDiscovery > **)&jarg1;
+  arg1 = (ppc::front::INodeDiscovery *)(smartarg1 ? smartarg1->get() : 0); 
+  (void)arg1; delete smartarg1;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_INodeDiscovery_1getAliveNodeList(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::front::INodeDiscovery *arg1 = (ppc::front::INodeDiscovery *) 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery const > *smartarg1 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::front::INodeDiscovery > **)&jarg1;
+  arg1 = (ppc::front::INodeDiscovery *)(smartarg1 ? smartarg1->get() : 0); 
+  result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+  *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jresult = new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >(result); 
+  return jresult;
+}
+
+
 SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_SDKMode_1AIR_1get(JNIEnv *jenv, jclass jcls) {
   jint jresult = 0 ;
   ppc::sdk::SDKMode result;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index e661a855..902409c5 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -36,6 +36,8 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long);
 
 // shared_ptr definition
 %shared_ptr(ppc::front::FrontConfig);
+%shared_ptr(ppc::front::INodeDiscovery);
+%shared_ptr(ppc::protocol::INodeInfo);
 %shared_ptr(ppc::front::IFront);
 %shared_ptr(ppc::front::IFrontClient);
 // the callbacks
@@ -68,6 +70,8 @@ PRIMITIVE_TYPEMAP(unsigned long int, long long);
 #include "wedpr-transport/sdk/src/Transport.h"
 #include "ppc-framework/libwrapper/Buffer.h"
 #include "ppc-framework/front/IFront.h"
+#include "ppc-framework/protocol/INodeInfo.h"
+#include "ppc-framework/front/INodeDiscovery.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-framework/front/FrontConfig.h"
 #include "ppc-framework/protocol/GrpcConfig.h"
@@ -98,11 +102,13 @@ namespace ppc::protocol{
     class EndPoint;
     class GrpcConfig;
     class RouteType;
+    class INodeInfo;
 }
 
 namespace ppc::front{
     class FrontConfig;
     class IFront;
+    class INodeDiscovery;
     class IFrontClient;
     class FrontImpl;
     class FrontBuilderImpl;
@@ -131,6 +137,8 @@ namespace bcos{
 %template(SharedGrpcConfig) std::shared_ptr;
 
 %template(SharedFront) std::shared_ptr;
+%template(SharedNodeDiscovery) std::shared_ptr;
+
 %template(SharedFrontClient) std::shared_ptr;
 
 %template(SharedErrorCallback) std::shared_ptr;
@@ -149,10 +157,12 @@ namespace bcos{
 %template(SharedMessageHeaderBuilder) std::shared_ptr;
 %template(SharedMessagePayloadBuilder) std::shared_ptr;
 %template(SharedRouteInfoBuilder) std::shared_ptr;
+%template(SharedNodeInfo) std::shared_ptr;
 
 %template(ubytes) std::vector;
 %template(ibytes) std::vector;
 %template(StringVec) std::vector;
+%template(NodeInfoVec) std::vector>;
 
 /// callbacks
 %feature("director") ppc::front::ErrorCallback;
@@ -232,6 +242,19 @@ WRAP(ppc::sdk::Transport)
 %ignore ppc::protocol::MessagePayload::setData;
 %ignore ppc::protocol::MessagePayload::setDataPtr;
 %ignore ppc::protocol::MessagePayload::dataPtr;
+%ignore ppc::front::INodeDiscovery::start;
+%ignore ppc::front::INodeDiscovery::stop;
+%ignore ppc::protocol::INodeInfo::INodeInfo;
+%ignore ppc::protocol::INodeInfo::setFront;
+%ignore ppc::protocol::INodeInfo::getFront;
+%ignore ppc::protocol::INodeInfo::components;
+%ignore ppc::protocol::INodeInfo::encode;
+%ignore ppc::protocol::INodeInfo::decode;
+%ignore ppc::protocol::INodeInfo::equal;
+%ignore ppc::protocol::INodeInfo::toJson;
+%ignore ppc::protocol::INodeInfo::setComponents;
+%ignore ppc::protocol::INodeInfoFactory;
+
 
 /*
 ///// tests  ///
@@ -248,8 +271,10 @@ WRAP(ppc::sdk::Transport)
 %include "ppc-framework/protocol/GrpcConfig.h"
 %include "ppc-framework/protocol/Message.h"
 %include "ppc-framework/protocol/MessagePayload.h"
+%include "ppc-framework/protocol/INodeInfo.h"
 
 %include "ppc-framework/front/IFront.h"
+%include "ppc-framework/front/INodeDiscovery.h"
 
 %include "wedpr-transport/sdk/src/TransportBuilder.h"
 %include "wedpr-transport/sdk/src/Transport.h"
\ No newline at end of file
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
index 898f3f76..061b7a24 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/demo/transport_demo.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-# Note: here can't be refactored by autopep
 import sys
 import os
+# Note: here can't be refactored by autopep
 root_path = os.path.abspath(os.path.dirname(__file__))
 sys.path.append(os.path.join(root_path, "../"))
 # Note: here can't be refactored by autopep
@@ -40,6 +40,13 @@ def message_event_loop(args):
         int(args.threadpool_size), args.node_id, args.gateway_targets)
     transport_config.set_self_endpoint(
         args.host_ip, int(args.listen_port), "0.0.0.0")
+    # def register_service_info(self, service, entrypoint):
+    service_name = "python_service_demo"
+    entrypoint = f"{args.host_ip}:{args.listen_port}"
+    transport_config.register_service_info(service_name, entrypoint)
+    service_name2 = "python_service_demo2"
+    transport_config.register_service_info(service_name2, entrypoint)
+
     transport = TransportLoader.load(transport_config)
     print(f"Create transport success, config: {transport_config.desc()}")
     transport.start()
@@ -47,8 +54,18 @@ def message_event_loop(args):
     transport.register_component(component)
     print(f"Start transport success")
     test_topic = "sync_message_event_loop_test"
+    # register service after start
+    service_name3 = "python_service_demo3"
+    transport.register_service_info(service_name3, entrypoint)
     while Transport.should_exit is False:
         try:
+            # fetch the alive entrypoints
+            alive_endpoint = transport.get_alive_entrypoints(service_name)
+            print(f"##### alive_endpoint: {alive_endpoint}")
+            alive_endpoint2 = transport.get_alive_entrypoints(service_name2)
+            print(f"##### alive_endpoint2: {alive_endpoint2}")
+            alive_endpoint3 = transport.get_alive_entrypoints(service_name3)
+            print(f"##### alive_endpoint3: {alive_endpoint3}")
             payload = b"test"
             transport.push_by_nodeid(topic=test_topic, dstNode=bytes(args.dst_node, encoding="utf-8"),
                                      seq=0, payload=payload, timeout=6000)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
index fceac195..949ae351 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/api/transport_api.py
@@ -1,7 +1,60 @@
 # -*- coding: utf-8 -*-
 
 from abc import ABC, abstractmethod
+import json
 from wedpr_python_gateway_sdk.transport.api.message_api import MessageAPI
+from wedpr_python_gateway_sdk.utils.base_object import BaseObject
+
+
+class EntryPointInfo(BaseObject):
+    def __init__(self, service_name: str = None, entrypoint: str = None):
+        self.serviceName = service_name
+        self.entryPoint = entrypoint
+
+    def __repr__(self):
+        return f"serviceName: {self.serviceName}, entryPoint: {self.entryPoint}"
+
+
+class ServiceMeta(BaseObject):
+    SERVICE_INFOS_KEY = "serviceInfos"
+
+    def __init__(self, entrypoints_info: [EntryPointInfo]):
+        self.serviceInfos = entrypoints_info
+
+    def encode(self) -> str:
+        service_meta_dict = {}
+        if self.serviceInfos is None:
+            return ""
+        service_info_list = []
+        for service_info in self.serviceInfos:
+            service_info_list.append(service_info.as_dict())
+        service_meta_dict.update(
+            {ServiceMeta.SERVICE_INFOS_KEY: service_info_list})
+        return json.dumps(service_meta_dict)
+
+    @staticmethod
+    def decode(json_str):
+        if json_str is None or len(json_str) == 0:
+            return None
+        service_meta = ServiceMeta([])
+        service_meta_dict = json.loads(json_str)
+        if ServiceMeta.SERVICE_INFOS_KEY in service_meta_dict.keys():
+            service_info_list = service_meta_dict.get(
+                ServiceMeta.SERVICE_INFOS_KEY)
+            for service_info in service_info_list:
+                entrypoint_info = EntryPointInfo()
+                entrypoint_info.set_params(**service_info)
+                service_meta.serviceInfos.append(entrypoint_info)
+        return service_meta
+
+    def __repr__(self):
+        repr_str = "{"
+        if self.serviceInfos is None:
+            return repr_str
+        for service_info in self.serviceInfos:
+            repr_str = f"{repr_str}, [{service_info}]"
+        repr_str = f"{repr_str}" + "}"
+        return repr_str
 
 
 class TransportAPI(ABC):
@@ -52,3 +105,11 @@ def register_component(self, component):
     @abstractmethod
     def unregister_component(self, component):
         pass
+
+    @abstractmethod
+    def get_alive_entrypoints(self, service_name: str) -> [EntryPointInfo]:
+        pass
+
+    @abstractmethod
+    def register_service_info(self, service, entrypoint):
+        pass
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 9318c761..2ec2e4c0 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -186,6 +186,36 @@ def __init__(self):
 _wedpr_python_transport.SharedFront_swigregister(SharedFront)
 
 
+class SharedNodeDiscovery(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _wedpr_python_transport.SharedNodeDiscovery_swiginit(
+            self, _wedpr_python_transport.new_SharedNodeDiscovery())
+    __swig_destroy__ = _wedpr_python_transport.delete_SharedNodeDiscovery
+
+
+# Register SharedNodeDiscovery in _wedpr_python_transport:
+_wedpr_python_transport.SharedNodeDiscovery_swigregister(SharedNodeDiscovery)
+
+
+class SharedNodeInfo(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _wedpr_python_transport.SharedNodeInfo_swiginit(
+            self, _wedpr_python_transport.new_SharedNodeInfo())
+    __swig_destroy__ = _wedpr_python_transport.delete_SharedNodeInfo
+
+
+# Register SharedNodeInfo in _wedpr_python_transport:
+_wedpr_python_transport.SharedNodeInfo_swigregister(SharedNodeInfo)
+
+
 class SharedFrontClient(object):
     thisown = property(lambda x: x.this.own(), lambda x,
                        v: x.this.own(v), doc="The membership flag")
@@ -735,6 +765,117 @@ def capacity(self):
 _wedpr_python_transport.StringVec_swigregister(StringVec)
 
 
+class NodeInfoVec(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def iterator(self):
+        return _wedpr_python_transport.NodeInfoVec_iterator(self)
+
+    def __iter__(self):
+        return self.iterator()
+
+    def __nonzero__(self):
+        return _wedpr_python_transport.NodeInfoVec___nonzero__(self)
+
+    def __bool__(self):
+        return _wedpr_python_transport.NodeInfoVec___bool__(self)
+
+    def __len__(self):
+        return _wedpr_python_transport.NodeInfoVec___len__(self)
+
+    def __getslice__(self, i, j):
+        return _wedpr_python_transport.NodeInfoVec___getslice__(self, i, j)
+
+    def __setslice__(self, *args):
+        return _wedpr_python_transport.NodeInfoVec___setslice__(self, *args)
+
+    def __delslice__(self, i, j):
+        return _wedpr_python_transport.NodeInfoVec___delslice__(self, i, j)
+
+    def __delitem__(self, *args):
+        return _wedpr_python_transport.NodeInfoVec___delitem__(self, *args)
+
+    def __getitem__(self, *args):
+        return _wedpr_python_transport.NodeInfoVec___getitem__(self, *args)
+
+    def __setitem__(self, *args):
+        return _wedpr_python_transport.NodeInfoVec___setitem__(self, *args)
+
+    def pop(self):
+        return _wedpr_python_transport.NodeInfoVec_pop(self)
+
+    def append(self, x):
+        return _wedpr_python_transport.NodeInfoVec_append(self, x)
+
+    def empty(self):
+        return _wedpr_python_transport.NodeInfoVec_empty(self)
+
+    def size(self):
+        return _wedpr_python_transport.NodeInfoVec_size(self)
+
+    def swap(self, v):
+        return _wedpr_python_transport.NodeInfoVec_swap(self, v)
+
+    def begin(self):
+        return _wedpr_python_transport.NodeInfoVec_begin(self)
+
+    def end(self):
+        return _wedpr_python_transport.NodeInfoVec_end(self)
+
+    def rbegin(self):
+        return _wedpr_python_transport.NodeInfoVec_rbegin(self)
+
+    def rend(self):
+        return _wedpr_python_transport.NodeInfoVec_rend(self)
+
+    def clear(self):
+        return _wedpr_python_transport.NodeInfoVec_clear(self)
+
+    def get_allocator(self):
+        return _wedpr_python_transport.NodeInfoVec_get_allocator(self)
+
+    def pop_back(self):
+        return _wedpr_python_transport.NodeInfoVec_pop_back(self)
+
+    def erase(self, *args):
+        return _wedpr_python_transport.NodeInfoVec_erase(self, *args)
+
+    def __init__(self, *args):
+        _wedpr_python_transport.NodeInfoVec_swiginit(
+            self, _wedpr_python_transport.new_NodeInfoVec(*args))
+
+    def push_back(self, x):
+        return _wedpr_python_transport.NodeInfoVec_push_back(self, x)
+
+    def front(self):
+        return _wedpr_python_transport.NodeInfoVec_front(self)
+
+    def back(self):
+        return _wedpr_python_transport.NodeInfoVec_back(self)
+
+    def assign(self, n, x):
+        return _wedpr_python_transport.NodeInfoVec_assign(self, n, x)
+
+    def resize(self, *args):
+        return _wedpr_python_transport.NodeInfoVec_resize(self, *args)
+
+    def insert(self, *args):
+        return _wedpr_python_transport.NodeInfoVec_insert(self, *args)
+
+    def reserve(self, n):
+        return _wedpr_python_transport.NodeInfoVec_reserve(self, n)
+
+    def capacity(self):
+        return _wedpr_python_transport.NodeInfoVec_capacity(self)
+    __swig_destroy__ = _wedpr_python_transport.delete_NodeInfoVec
+
+
+# Register NodeInfoVec in _wedpr_python_transport:
+_wedpr_python_transport.NodeInfoVec_swigregister(NodeInfoVec)
+
+
 class Error(object):
     thisown = property(lambda x: x.this.own(), lambda x,
                        v: x.this.own(v), doc="The membership flag")
@@ -861,6 +1002,12 @@ def addComponent(self, component):
     def mutableComponents(self):
         return _wedpr_python_transport.FrontConfig_mutableComponents(self)
 
+    def meta(self):
+        return _wedpr_python_transport.FrontConfig_meta(self)
+
+    def setMeta(self, meta):
+        return _wedpr_python_transport.FrontConfig_setMeta(self, meta)
+
 
 # Register FrontConfig in _wedpr_python_transport:
 _wedpr_python_transport.FrontConfig_swigregister(FrontConfig)
@@ -1582,9 +1729,15 @@ def registerComponent(self, component):
     def unRegisterComponent(self, component):
         return _wedpr_python_transport.IFront_unRegisterComponent(self, component)
 
+    def updateMetaInfo(self, meta):
+        return _wedpr_python_transport.IFront_updateMetaInfo(self, meta)
+
     def selectNodesByRoutePolicy(self, routeType, routeInfo):
         return _wedpr_python_transport.IFront_selectNodesByRoutePolicy(self, routeType, routeInfo)
 
+    def getNodeDiscovery(self):
+        return _wedpr_python_transport.IFront_getNodeDiscovery(self)
+
 
 # Register IFront in _wedpr_python_transport:
 _wedpr_python_transport.IFront_swigregister(IFront)
@@ -1605,6 +1758,23 @@ def buildClient(self, endPoint, onUnHealthHandler, removeHandlerOnUnhealth):
 
 # Register IFrontBuilder in _wedpr_python_transport:
 _wedpr_python_transport.IFrontBuilder_swigregister(IFrontBuilder)
+
+
+class INodeDiscovery(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+
+    def __init__(self, *args, **kwargs):
+        raise AttributeError("No constructor defined - class is abstract")
+    __repr__ = _swig_repr
+    __swig_destroy__ = _wedpr_python_transport.delete_INodeDiscovery
+
+    def getAliveNodeList(self):
+        return _wedpr_python_transport.INodeDiscovery_getAliveNodeList(self)
+
+
+# Register INodeDiscovery in _wedpr_python_transport:
+_wedpr_python_transport.INodeDiscovery_swigregister(INodeDiscovery)
 SDKMode_AIR = _wedpr_python_transport.SDKMode_AIR
 SDKMode_PRO = _wedpr_python_transport.SDKMode_PRO
 
@@ -1677,3 +1847,51 @@ def getConfig(self):
 
 # Register Transport in _wedpr_python_transport:
 _wedpr_python_transport.Transport_swigregister(Transport)
+
+
+class INodeInfo(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+
+    def __init__(self, *args, **kwargs):
+        raise AttributeError("No constructor defined - class is abstract")
+    __repr__ = _swig_repr
+    __swig_destroy__ = _wedpr_python_transport.delete_INodeInfo
+
+    def endPoint(self):
+        return _wedpr_python_transport.INodeInfo_endPoint(self)
+
+    def nodeID(self):
+        return _wedpr_python_transport.INodeInfo_nodeID(self)
+
+    def setNodeID(self, nodeID):
+        return _wedpr_python_transport.INodeInfo_setNodeID(self, nodeID)
+
+    def setEndPoint(self, endPoint):
+        return _wedpr_python_transport.INodeInfo_setEndPoint(self, endPoint)
+
+    def addComponent(self, component):
+        return _wedpr_python_transport.INodeInfo_addComponent(self, component)
+
+    def eraseComponent(self, component):
+        return _wedpr_python_transport.INodeInfo_eraseComponent(self, component)
+
+    def componentExist(self, component):
+        return _wedpr_python_transport.INodeInfo_componentExist(self, component)
+
+    def copiedComponents(self):
+        return _wedpr_python_transport.INodeInfo_copiedComponents(self)
+
+    def meta(self):
+        return _wedpr_python_transport.INodeInfo_meta(self)
+
+    def setMeta(self, meta):
+        return _wedpr_python_transport.INodeInfo_setMeta(self, meta)
+
+
+# Register INodeInfo in _wedpr_python_transport:
+_wedpr_python_transport.INodeInfo_swigregister(INodeInfo)
+
+
+def printNodeInfo(nodeInfo):
+    return _wedpr_python_transport.printNodeInfo(nodeInfo)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
index ae4e0154..2b9260a3 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport.py
@@ -11,7 +11,8 @@
 from wedpr_python_gateway_sdk.transport.api.transport_api import TransportAPI
 from wedpr_python_gateway_sdk.transport.impl.transport_config import TransportConfig
 import random
-
+from wedpr_python_gateway_sdk.transport.api.transport_api import EntryPointInfo
+from wedpr_python_gateway_sdk.transport.api.transport_api import ServiceMeta
 from enum import Enum
 import signal
 
@@ -31,6 +32,7 @@ def __init__(self, transport: Transport, transport_config: TransportConfig):
         self.__transport_config = transport_config
         self.__route_info_builder = RouteInfoBuilder(
             self.__transport.routeInfoBuilder())
+        self.service_meta = transport_config.service_meta
 
     def get_config(self) -> TransportConfig:
         return self.__transport_config
@@ -125,6 +127,26 @@ def unregister_component(self, component):
         result = self.__transport.getFront().unRegisterComponent(component)
         Transport.check_result("unregister_component", result)
 
+    def get_alive_entrypoints(self, service_name: str) -> [EntryPointInfo]:
+        alive_entrypoints = []
+        alive_node_list = self.__transport.getFront().getNodeDiscovery().getAliveNodeList()
+        if alive_node_list is None or len(alive_node_list) == 0:
+            return alive_entrypoints
+        for i in range(0, len(alive_node_list)):
+            alive_node_info = alive_node_list[i]
+            service_meta = ServiceMeta.decode(alive_node_info.meta())
+            if service_meta is None or service_meta.serviceInfos is None or len(service_meta.serviceInfos) == 0:
+                continue
+            for entrypoint_info in service_meta.serviceInfos:
+                if entrypoint_info.serviceName.lower() == service_name.lower():
+                    alive_entrypoints.append(entrypoint_info)
+        return alive_entrypoints
+
+    def register_service_info(self, service, entrypoint):
+        self.service_meta.serviceInfos.append(EntryPointInfo(
+            service_name=service, entrypoint=entrypoint))
+        self.__transport.getFront().updateMetaInfo(self.service_meta.encode())
+
 
 def signal_handler(sig, frame):
     print('You pressed Ctrl+C! Exiting gracefully.')
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
index 0c15b0b8..d8f5e3ea 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_config.py
@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 
+import json
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import FrontConfig
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import TransportBuilder
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import EndPoint
 from wedpr_python_gateway_sdk.transport.generated.wedpr_python_transport import GrpcConfig
+from wedpr_python_gateway_sdk.transport.api.transport_api import ServiceMeta
+from wedpr_python_gateway_sdk.transport.api.transport_api import EntryPointInfo
 
 
 class TransportConfig:
@@ -16,19 +19,34 @@ def __init__(self, threadpool_size: int, nodeID: str, gateway_targets: str):
         self.__front_config = self.__transport_builder.buildConfig(
             threadpool_size, nodeID)
         self.__front_config.setGatewayGrpcTarget(gateway_targets)
+        self.service_meta = ServiceMeta(entrypoints_info=[])
 
     def get_front_config(self) -> FrontConfig:
         return self.__front_config
 
+    def register_service_info(self, service, entrypoint):
+        self.service_meta.serviceInfos.append(EntryPointInfo(
+            service_name=service, entrypoint=entrypoint))
+        self.__front_config.setMeta(self.service_meta.encode())
+
+    def register_service_meta(self, service_meta: ServiceMeta):
+        if service_meta is None:
+            return
+        self.service_meta = service_meta
+        self.__front_config.setMeta(self.service_meta.encode())
+
+    def get_service_meta(self) -> ServiceMeta:
+        return self.service_meta
+
     def set_self_endpoint(self, host: str, port: int, listen_ip: str = None):
-        endPoint = EndPoint(host, port)
+        endpoint = EndPoint(host, port)
         if listen_ip is None:
             listen_ip = "0.0.0.0"
-        endPoint.setListenIp(listen_ip)
-        self.__front_config.setSelfEndPoint(endPoint)
+        endpoint.setListenIp(listen_ip)
+        self.__front_config.setSelfEndPoint(endpoint)
 
     def set_grpc_config(self, grpc_config: GrpcConfig):
-        self.__front_config.set_grpc_config(grpc_config)
+        self.__front_config.setGrpcConfig(grpc_config)
 
     def get_thread_pool_size(self) -> int:
         return self.__front_config.threadPoolSize()
@@ -46,4 +64,5 @@ def desc(self):
         return f"thread_pool_size: {self.get_thread_pool_size()}, \
                 nodeID: {self.get_node_id()}, \
                 gatewayTargets: {self.get_gateway_targets()}, \
-                endPoint: {self.get_self_endpoint().entryPoint()}"
+                endPoint: {self.get_self_endpoint().entryPoint()}," \
+               f"serviceMeta: {self.service_meta}"
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
index 7ee50de4..81f756a3 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
@@ -55,12 +55,12 @@ def load(transport_config: TransportConfig) -> Transport:
         return Transport(transport, transport_config)
 
     @staticmethod
-    def build(transport_threadpool_size: int = 4,
-              transport_node_id: str = None,
-              transport_gateway_targets: str = None,
-              transport_host_ip: str = None,
-              transport_listen_port: int = None,
-              **params):
+    def build_config(transport_threadpool_size: int = 4,
+                     transport_node_id: str = None,
+                     transport_gateway_targets: str = None,
+                     transport_host_ip: str = None,
+                     transport_listen_port: int = None,
+                     **params) -> TransportConfig:
         config = TransportConfigObject(transport_threadpool_size,
                                        transport_node_id,
                                        transport_gateway_targets,
@@ -74,5 +74,19 @@ def build(transport_threadpool_size: int = 4,
         transport_config.set_self_endpoint(config.transport_host_ip,
                                            config.transport_listen_port,
                                            config.transport_listen_ip)
+        return transport_config
 
+    @staticmethod
+    def build(transport_threadpool_size: int = 4,
+              transport_node_id: str = None,
+              transport_gateway_targets: str = None,
+              transport_host_ip: str = None,
+              transport_listen_port: int = None,
+              **params):
+        transport_config = TransportLoader.build_config(transport_threadpool_size,
+                                                        transport_node_id,
+                                                        transport_gateway_targets,
+                                                        transport_host_ip,
+                                                        transport_listen_port,
+                                                        **params)
         return TransportLoader.load(transport_config)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index cab7c01b..8b07830c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -3668,71 +3668,76 @@ namespace Swig {
 #define SWIGTYPE_p_ppc__front__IFrontBuilder swig_types[20]
 #define SWIGTYPE_p_ppc__front__IFrontClient swig_types[21]
 #define SWIGTYPE_p_ppc__front__IMessageHandler swig_types[22]
-#define SWIGTYPE_p_ppc__front__MessageDispatcherHandler swig_types[23]
-#define SWIGTYPE_p_ppc__front__SendResponseHandler swig_types[24]
-#define SWIGTYPE_p_ppc__gateway__IGateway__Ptr swig_types[25]
-#define SWIGTYPE_p_ppc__protocol__EndPoint swig_types[26]
-#define SWIGTYPE_p_ppc__protocol__GrpcConfig swig_types[27]
-#define SWIGTYPE_p_ppc__protocol__GrpcServerConfig swig_types[28]
-#define SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr swig_types[29]
-#define SWIGTYPE_p_ppc__protocol__Message swig_types[30]
-#define SWIGTYPE_p_ppc__protocol__MessageBuilder swig_types[31]
-#define SWIGTYPE_p_ppc__protocol__MessageHeader swig_types[32]
-#define SWIGTYPE_p_ppc__protocol__MessageHeaderBuilder swig_types[33]
-#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeader swig_types[34]
-#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeaderBuilder swig_types[35]
-#define SWIGTYPE_p_ppc__protocol__MessagePayload swig_types[36]
-#define SWIGTYPE_p_ppc__protocol__MessagePayloadBuilder swig_types[37]
-#define SWIGTYPE_p_ppc__protocol__RouteType swig_types[38]
-#define SWIGTYPE_p_ppc__sdk__Transport swig_types[39]
-#define SWIGTYPE_p_ppc__sdk__TransportBuilder swig_types[40]
-#define SWIGTYPE_p_short swig_types[41]
-#define SWIGTYPE_p_signed_char swig_types[42]
-#define SWIGTYPE_p_size_type swig_types[43]
-#define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[44]
-#define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[45]
-#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[46]
-#define SWIGTYPE_p_std__exception swig_types[47]
-#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[48]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[49]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[50]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[51]
-#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[52]
-#define SWIGTYPE_p_std__invalid_argument swig_types[53]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[54]
-#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[55]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[56]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[57]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[58]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[59]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[60]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[61]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[62]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[63]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[64]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[65]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[66]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[67]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[68]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[69]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[70]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[71]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[72]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[73]
-#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[74]
-#define SWIGTYPE_p_std__string swig_types[75]
-#define SWIGTYPE_p_std__string_view swig_types[76]
-#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[77]
-#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[78]
-#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[79]
-#define SWIGTYPE_p_swig__SwigPyIterator swig_types[80]
-#define SWIGTYPE_p_unsigned_char swig_types[81]
-#define SWIGTYPE_p_unsigned_int swig_types[82]
-#define SWIGTYPE_p_unsigned_long_long swig_types[83]
-#define SWIGTYPE_p_unsigned_short swig_types[84]
-#define SWIGTYPE_p_value_type swig_types[85]
-static swig_type_info *swig_types[87];
-static swig_module_info swig_module = {swig_types, 86, 0, 0, 0, 0};
+#define SWIGTYPE_p_ppc__front__INodeDiscovery swig_types[23]
+#define SWIGTYPE_p_ppc__front__MessageDispatcherHandler swig_types[24]
+#define SWIGTYPE_p_ppc__front__SendResponseHandler swig_types[25]
+#define SWIGTYPE_p_ppc__gateway__IGateway__Ptr swig_types[26]
+#define SWIGTYPE_p_ppc__protocol__EndPoint swig_types[27]
+#define SWIGTYPE_p_ppc__protocol__GrpcConfig swig_types[28]
+#define SWIGTYPE_p_ppc__protocol__GrpcServerConfig swig_types[29]
+#define SWIGTYPE_p_ppc__protocol__INodeInfo swig_types[30]
+#define SWIGTYPE_p_ppc__protocol__Message swig_types[31]
+#define SWIGTYPE_p_ppc__protocol__MessageBuilder swig_types[32]
+#define SWIGTYPE_p_ppc__protocol__MessageHeader swig_types[33]
+#define SWIGTYPE_p_ppc__protocol__MessageHeaderBuilder swig_types[34]
+#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeader swig_types[35]
+#define SWIGTYPE_p_ppc__protocol__MessageOptionalHeaderBuilder swig_types[36]
+#define SWIGTYPE_p_ppc__protocol__MessagePayload swig_types[37]
+#define SWIGTYPE_p_ppc__protocol__MessagePayloadBuilder swig_types[38]
+#define SWIGTYPE_p_ppc__protocol__RouteType swig_types[39]
+#define SWIGTYPE_p_ppc__sdk__Transport swig_types[40]
+#define SWIGTYPE_p_ppc__sdk__TransportBuilder swig_types[41]
+#define SWIGTYPE_p_short swig_types[42]
+#define SWIGTYPE_p_signed_char swig_types[43]
+#define SWIGTYPE_p_size_type swig_types[44]
+#define SWIGTYPE_p_std__allocatorT_signed_char_t swig_types[45]
+#define SWIGTYPE_p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t swig_types[46]
+#define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[47]
+#define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[48]
+#define SWIGTYPE_p_std__exception swig_types[49]
+#define SWIGTYPE_p_std__functionT_void_fF_t swig_types[50]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t swig_types[51]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_t_std__shared_ptrT_ppc__protocol__Message_t_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_tF_t swig_types[52]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t swig_types[53]
+#define SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_ppc__protocol__Message_tF_t swig_types[54]
+#define SWIGTYPE_p_std__invalid_argument swig_types[55]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__Error_t swig_types[56]
+#define SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t swig_types[57]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t swig_types[58]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t swig_types[59]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t swig_types[60]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t swig_types[61]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t swig_types[62]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t swig_types[63]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t swig_types[64]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t swig_types[65]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t swig_types[66]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t swig_types[67]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t swig_types[68]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t swig_types[69]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t swig_types[70]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t swig_types[71]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t swig_types[72]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t swig_types[73]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t swig_types[74]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t swig_types[75]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t swig_types[76]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t swig_types[77]
+#define SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t swig_types[78]
+#define SWIGTYPE_p_std__string swig_types[79]
+#define SWIGTYPE_p_std__string_view swig_types[80]
+#define SWIGTYPE_p_std__vectorT_int8_t_t swig_types[81]
+#define SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t swig_types[82]
+#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[83]
+#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[84]
+#define SWIGTYPE_p_swig__SwigPyIterator swig_types[85]
+#define SWIGTYPE_p_unsigned_char swig_types[86]
+#define SWIGTYPE_p_unsigned_int swig_types[87]
+#define SWIGTYPE_p_unsigned_long_long swig_types[88]
+#define SWIGTYPE_p_unsigned_short swig_types[89]
+#define SWIGTYPE_p_value_type swig_types[90]
+static swig_type_info *swig_types[92];
+static swig_module_info swig_module = {swig_types, 91, 0, 0, 0, 0};
 #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name)
 #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name)
 
@@ -4419,6 +4424,8 @@ SWIGINTERNINLINE PyObject*
 #include "wedpr-transport/sdk/src/Transport.h"
 #include "ppc-framework/libwrapper/Buffer.h"
 #include "ppc-framework/front/IFront.h"
+#include "ppc-framework/protocol/INodeInfo.h"
+#include "ppc-framework/front/INodeDiscovery.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-framework/front/FrontConfig.h"
 #include "ppc-framework/protocol/GrpcConfig.h"
@@ -6070,6 +6077,115 @@ SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__era
 SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__insert__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::value_type const &x){ return self->insert(pos, x); }
 SWIGINTERN void std_vector_Sl_std_string_Sg__insert__SWIG_1(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::size_type n,std::vector< std::string >::value_type const &x){ self->insert(pos, n, x); }
 
+  namespace swig {
+    template <>  struct traits< std::shared_ptr< ppc::protocol::INodeInfo > > {
+      typedef pointer_category category;
+      static const char* type_name() { return"std::shared_ptr< ppc::protocol::INodeInfo >"; }
+    };
+  }
+
+
+      namespace swig {
+	template <>  struct traits, std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > > {
+	  typedef pointer_category category;
+	  static const char* type_name() {
+	    return "std::vector<" "std::shared_ptr< ppc::protocol::INodeInfo >" "," "std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > >" " >";
+	  }
+	};
+      }
+    
+SWIGINTERN swig::SwigPyIterator *std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__iterator(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,PyObject **PYTHON_SELF){
+      return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF);
+    }
+SWIGINTERN bool std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____nonzero__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *self){
+      return !(self->empty());
+    }
+SWIGINTERN bool std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____bool__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *self){
+      return !(self->empty());
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____len__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *self){
+      return self->size();
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getslice__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type j){
+      return swig::getslice(self, i, j, 1);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setslice____SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type j){
+      swig::setslice(self, i, j, 1, std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >());
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setslice____SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type j,std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &v){
+      swig::setslice(self, i, j, 1, v);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delslice__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type j){
+      swig::delslice(self, i, j, 1);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delitem____SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i){
+      swig::erase(self, swig::getpos(self, i));
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getitem____SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return NULL;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type id = i;
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type jd = j;
+      return swig::getslice(self, id, jd, step);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,SWIGPY_SLICEOBJECT *slice,std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &v){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type id = i;
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type jd = j;
+      swig::setslice(self, id, jd, step, v);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type id = i;
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type jd = j;
+      swig::delslice(self, id, jd, step);
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delitem____SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,SWIGPY_SLICEOBJECT *slice){
+      Py_ssize_t i, j, step;
+      if( !PySlice_Check(slice) ) {
+        SWIG_Error(SWIG_TypeError, "Slice object expected.");
+        return;
+      }
+      PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step);
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type id = i;
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::difference_type jd = j;
+      swig::delslice(self, id, jd, step);
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getitem____SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i){
+      return *(swig::cgetpos(self, i));
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_2(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type i,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){
+      *(swig::getpos(self,i)) = x;
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__pop(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self){
+      if (self->size() == 0)
+	throw std::out_of_range("pop from empty container");
+      std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >::value_type x = self->back();
+      self->pop_back();
+      return x;
+    }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__append(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){
+      self->push_back(x);
+    }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__erase__SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator pos){ return self->erase(pos); }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__erase__SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator first,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator last){ return self->erase(first, last); }
+SWIGINTERN std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__insert__SWIG_0(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator pos,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){ return self->insert(pos, x); }
+SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__insert__SWIG_1(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *self,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator pos,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type n,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &x){ self->insert(pos, n, x); }
+
 
 
 
@@ -7437,6 +7553,130 @@ SWIGINTERN PyObject *SharedFront_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObje
   return SWIG_Python_InitShadowInstance(args);
 }
 
+SWIGINTERN PyObject *_wrap_new_SharedNodeDiscovery(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::shared_ptr< ppc::front::INodeDiscovery > *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "new_SharedNodeDiscovery", 0, 0, 0)) SWIG_fail;
+  result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+  {
+    std::shared_ptr<  ppc::front::INodeDiscovery > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::INodeDiscovery >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_POINTER_OWN);
+    if (SWIG_POINTER_NEW) delete result;
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_SharedNodeDiscovery(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::shared_ptr< ppc::front::INodeDiscovery > *arg1 = (std::shared_ptr< ppc::front::INodeDiscovery > *) 0 ;
+  void *argp1 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery > tempshared1 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t,  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedNodeDiscovery" "', argument " "1"" of type '" "std::shared_ptr< ppc::front::INodeDiscovery > *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::front::INodeDiscovery > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< ppc::front::INodeDiscovery > * >(argp1);
+      arg1 = &tempshared1;
+    } else {
+      arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::INodeDiscovery > * >(argp1) : &tempshared1;
+    }
+  }
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *SharedNodeDiscovery_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *SharedNodeDiscovery_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_SharedNodeInfo(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::shared_ptr< ppc::protocol::INodeInfo > *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "new_SharedNodeInfo", 0, 0, 0)) SWIG_fail;
+  result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+  {
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
+    if (SWIG_POINTER_NEW) delete result;
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_SharedNodeInfo(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::shared_ptr< ppc::protocol::INodeInfo > *arg1 = (std::shared_ptr< ppc::protocol::INodeInfo > *) 0 ;
+  void *argp1 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SharedNodeInfo" "', argument " "1"" of type '" "std::shared_ptr< ppc::protocol::INodeInfo > *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp1) tempshared1 = *reinterpret_cast< std::shared_ptr< ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = &tempshared1;
+    } else {
+      arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::INodeInfo > * >(argp1) : &tempshared1;
+    }
+  }
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *SharedNodeInfo_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *SharedNodeInfo_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
 SWIGINTERN PyObject *_wrap_new_SharedFrontClient(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   std::shared_ptr< ppc::front::IFrontClient > *result = 0 ;
@@ -13939,343 +14179,297 @@ SWIGINTERN PyObject *StringVec_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject
   return SWIG_Python_InitShadowInstance(args);
 }
 
-SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_iterator(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  char *arg1 = (char *) 0 ;
-  char *arg2 = (char *) 0 ;
-  int arg3 ;
-  int32_t arg4 ;
-  std::string *arg5 = 0 ;
-  int res1 ;
-  char *buf1 = 0 ;
-  int alloc1 = 0 ;
-  int res2 ;
-  char *buf2 = 0 ;
-  int alloc2 = 0 ;
-  int val3 ;
-  int ecode3 = 0 ;
-  int val4 ;
-  int ecode4 = 0 ;
-  int res5 = SWIG_OLDOBJ ;
-  bcos::Error result;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  PyObject **arg2 = (PyObject **) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  swig::SwigPyIterator *result = 0 ;
   
+  arg2 = &swig_obj[0];
   (void)self;
-  if ((nobjs < 5) || (nobjs > 5)) SWIG_fail;
-  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_iterator" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< char * >(buf1);
-  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
-  }
-  arg2 = reinterpret_cast< char * >(buf2);
-  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
-  } 
-  arg3 = static_cast< int >(val3);
-  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
-  } 
-  arg4 = static_cast< int32_t >(val4);
-  {
-    std::string *ptr = (std::string *)0;
-    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
-    if (!SWIG_IsOK(res5)) {
-      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
-    }
-    arg5 = ptr;
-  }
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (swig::SwigPyIterator *)std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__iterator(arg1,arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___nonzero__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___nonzero__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (bool)std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____nonzero__((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___bool__(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  char *arg1 = (char *) 0 ;
-  char *arg2 = (char *) 0 ;
-  int arg3 ;
-  int32_t arg4 ;
-  std::string *arg5 = 0 ;
-  bcos::Error *arg6 = 0 ;
-  int res1 ;
-  char *buf1 = 0 ;
-  int alloc1 = 0 ;
-  int res2 ;
-  char *buf2 = 0 ;
-  int alloc2 = 0 ;
-  int val3 ;
-  int ecode3 = 0 ;
-  int val4 ;
-  int ecode4 = 0 ;
-  int res5 = SWIG_OLDOBJ ;
-  void *argp6 = 0 ;
-  int res6 = 0 ;
-  std::shared_ptr< bcos::Error const > tempshared6 ;
-  bcos::Error result;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
   
   (void)self;
-  if ((nobjs < 6) || (nobjs > 6)) SWIG_fail;
-  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___bool__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  arg1 = reinterpret_cast< char * >(buf1);
-  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (bool)std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____bool__((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___len__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___len__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  arg2 = reinterpret_cast< char * >(buf2);
-  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____len__((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1);
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___getslice__(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec___getslice__", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___getslice__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___getslice__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
   } 
-  arg3 = static_cast< int >(val3);
-  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "NodeInfoVec___getslice__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
   } 
-  arg4 = static_cast< int32_t >(val4);
-  {
-    std::string *ptr = (std::string *)0;
-    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
-    if (!SWIG_IsOK(res5)) {
-      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
-    }
-    arg5 = ptr;
+  arg3 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val3);
+  try {
+    result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *)std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
-  {
-    int newmem = 0;
-    res6 = SWIG_ConvertPtrAndOwn(swig_obj[5], &argp6, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
-    if (!SWIG_IsOK(res6)) {
-      SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'");
-    }
-    if (!argp6) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'"); 
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared6 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6);
-      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6);
-      arg6 = const_cast< bcos::Error * >(tempshared6.get());
-    } else {
-      arg6 = const_cast< bcos::Error * >(reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6)->get());
-    }
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___setslice__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___setslice__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "NodeInfoVec___setslice__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val3);
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  char *arg1 = (char *) 0 ;
-  char *arg2 = (char *) 0 ;
-  int arg3 ;
-  int32_t arg4 ;
-  std::string *arg5 = 0 ;
-  std::exception *arg6 = 0 ;
-  int res1 ;
-  char *buf1 = 0 ;
-  int alloc1 = 0 ;
-  int res2 ;
-  char *buf2 = 0 ;
-  int alloc2 = 0 ;
-  int val3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *arg4 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
   int ecode3 = 0 ;
-  int val4 ;
-  int ecode4 = 0 ;
-  int res5 = SWIG_OLDOBJ ;
-  void *argp6 = 0 ;
-  int res6 = 0 ;
-  bcos::Error result;
+  int res4 = SWIG_OLDOBJ ;
   
   (void)self;
-  if ((nobjs < 6) || (nobjs > 6)) SWIG_fail;
-  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
+  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
-  }
-  arg1 = reinterpret_cast< char * >(buf1);
-  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___setslice__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg2 = reinterpret_cast< char * >(buf2);
-  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___setslice__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
   } 
-  arg3 = static_cast< int >(val3);
-  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "NodeInfoVec___setslice__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
   } 
-  arg4 = static_cast< int32_t >(val4);
+  arg3 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val3);
   {
-    std::string *ptr = (std::string *)0;
-    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
-    if (!SWIG_IsOK(res5)) {
-      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *ptr = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *)0;
+    res4 = swig::asptr(swig_obj[3], &ptr);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "NodeInfoVec___setslice__" "', argument " "4"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "NodeInfoVec___setslice__" "', argument " "4"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &""'"); 
     }
-    arg5 = ptr;
-  }
-  res6 = SWIG_ConvertPtr(swig_obj[5], &argp6, SWIGTYPE_p_std__exception,  0  | 0);
-  if (!SWIG_IsOK(res6)) {
-    SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); 
-  }
-  if (!argp6) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); 
+    arg4 = ptr;
   }
-  arg6 = reinterpret_cast< std::exception * >(argp6);
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &)*arg4);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res4)) delete arg4;
   return resultobj;
 fail:
-  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res5)) delete arg5;
+  if (SWIG_IsNewObj(res4)) delete arg4;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_buildError(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setslice__(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
-  PyObject *argv[7] = {
+  PyObject *argv[5] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "Error_buildError", 0, 6, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec___setslice__", 0, 4, argv))) SWIG_fail;
   --argc;
-  if (argc == 5) {
+  if (argc == 3) {
     int _v = 0;
-    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
-      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
-      _v = SWIG_CheckState(res);
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
       if (_v) {
         {
-          int res = SWIG_AsVal_int(argv[2], NULL);
+          int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL);
           _v = SWIG_CheckState(res);
         }
         if (_v) {
-          {
-            int res = SWIG_AsVal_int(argv[3], NULL);
-            _v = SWIG_CheckState(res);
-          }
-          if (_v) {
-            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
-            _v = SWIG_CheckState(res);
-            if (_v) {
-              return _wrap_Error_buildError__SWIG_0(self, argc, argv);
-            }
-          }
+          return _wrap_NodeInfoVec___setslice____SWIG_0(self, argc, argv);
         }
       }
     }
   }
-  if (argc == 6) {
+  if (argc == 4) {
     int _v = 0;
-    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
     _v = SWIG_CheckState(res);
     if (_v) {
-      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        {
-          int res = SWIG_AsVal_int(argv[2], NULL);
-          _v = SWIG_CheckState(res);
-        }
-        if (_v) {
-          {
-            int res = SWIG_AsVal_int(argv[3], NULL);
-            _v = SWIG_CheckState(res);
-          }
-          if (_v) {
-            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
-            _v = SWIG_CheckState(res);
-            if (_v) {
-              int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0);
-              _v = SWIG_CheckState(res);
-              if (_v) {
-                return _wrap_Error_buildError__SWIG_1(self, argc, argv);
-              }
-            }
-          }
-        }
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
       }
-    }
-  }
-  if (argc == 6) {
-    int _v = 0;
-    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
-      _v = SWIG_CheckState(res);
       if (_v) {
         {
-          int res = SWIG_AsVal_int(argv[2], NULL);
+          int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL);
           _v = SWIG_CheckState(res);
         }
         if (_v) {
-          {
-            int res = SWIG_AsVal_int(argv[3], NULL);
-            _v = SWIG_CheckState(res);
-          }
+          int res = swig::asptr(argv[3], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+          _v = SWIG_CheckState(res);
           if (_v) {
-            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
-            _v = SWIG_CheckState(res);
-            if (_v) {
-              int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__exception, SWIG_POINTER_NO_NULL | 0);
-              _v = SWIG_CheckState(res);
-              if (_v) {
-                return _wrap_Error_buildError__SWIG_2(self, argc, argv);
-              }
-            }
+            return _wrap_NodeInfoVec___setslice____SWIG_1(self, argc, argv);
           }
         }
       }
@@ -14283,471 +14477,657 @@ SWIGINTERN PyObject *_wrap_Error_buildError(PyObject *self, PyObject *args) {
   }
   
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'Error_buildError'.\n"
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec___setslice__'.\n"
     "  Possible C/C++ prototypes are:\n"
-    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &)\n"
-    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,bcos::Error const &)\n"
-    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,std::exception const &)\n");
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__setslice__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__setslice__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &)\n");
   return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_Error__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___delslice__(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  bcos::Error *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  ptrdiff_t val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
   
   (void)self;
-  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (bcos::Error *)new bcos::Error();
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec___delslice__", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___delslice__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___delslice__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "NodeInfoVec___delslice__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val3);
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_Error__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  int64_t arg1 ;
-  std::string arg2 ;
-  long long val1 ;
-  int ecode1 = 0 ;
-  bcos::Error *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
   
   (void)self;
   if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  ecode1 = SWIG_AsVal_long_SS_long(swig_obj[0], &val1);
-  if (!SWIG_IsOK(ecode1)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_Error" "', argument " "1"" of type '" "int64_t""'");
-  } 
-  arg1 = static_cast< int64_t >(val1);
-  {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "new_Error" "', argument " "2"" of type '" "std::string""'"); 
-    }
-    arg2 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___delitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  result = (bcos::Error *)new bcos::Error(arg1,arg2);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___delitem__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_Error(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
+SWIGINTERN PyObject *_wrap_NodeInfoVec___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *result = 0 ;
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "new_Error", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 0) {
-    return _wrap_new_Error__SWIG_0(self, argc, argv);
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___getitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  if (argc == 2) {
-    int _v = 0;
-    {
-      int res = SWIG_AsVal_long_SS_long(argv[0], NULL);
-      _v = SWIG_CheckState(res);
-    }
-    if (_v) {
-      int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0));
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_new_Error__SWIG_1(self, argc, argv);
-      }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  {
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
     }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
   }
-  
+  try {
+    result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *)std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getitem____SWIG_0(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_OWN |  0 );
+  return resultobj;
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_Error'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    bcos::Error::Error()\n"
-    "    bcos::Error::Error(int64_t,std::string)\n");
-  return 0;
+  return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_Error(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  bcos::Error *arg1 = (bcos::Error *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< bcos::Error > tempshared1 ;
-  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  int res3 = SWIG_OLDOBJ ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___setitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Error" "', argument " "1"" of type '" "bcos::Error *""'");
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
+  }
+  {
+    std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *ptr = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *)0;
+    res3 = swig::asptr(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "NodeInfoVec___setitem__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "NodeInfoVec___setitem__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &""'"); 
     }
+    arg3 = ptr;
+  }
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_0(arg1,arg2,(std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
   }
-  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_errorCode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  bcos::Error *arg1 = (bcos::Error *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< bcos::Error const > tempshared1 ;
-  std::shared_ptr< bcos::Error const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  int64_t result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___setitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorCode" "', argument " "1"" of type '" "bcos::Error const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
     }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
   }
-  result = (int64_t)((bcos::Error const *)arg1)->errorCode();
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_1(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_errorMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  bcos::Error *arg1 = (bcos::Error *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  SWIGPY_SLICEOBJECT *arg2 = (SWIGPY_SLICEOBJECT *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< bcos::Error const > tempshared1 ;
-  std::shared_ptr< bcos::Error const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___delitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorMessage" "', argument " "1"" of type '" "bcos::Error const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    if (!PySlice_Check(swig_obj[1])) {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'");
     }
+    arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1];
   }
-  result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____delitem____SWIG_1(arg1,arg2);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  } catch(std::invalid_argument &_e) {
+    SWIG_exception_fail(SWIG_ValueError, (&_e)->what());
+  }
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_setErrorCode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___delitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec___delitem__", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec___delitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec___delitem____SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec___delitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__delitem__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__delitem__(SWIGPY_SLICEOBJECT *)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  bcos::Error *arg1 = (bcos::Error *) 0 ;
-  int64_t arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< bcos::Error > tempshared1 ;
-  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
-  long long val2 ;
+  ptrdiff_t val2 ;
   int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Error_setErrorCode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorCode" "', argument " "1"" of type '" "bcos::Error *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___getitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  ecode2 = SWIG_AsVal_long_SS_long(swig_obj[1], &val2);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Error_setErrorCode" "', argument " "2"" of type '" "int64_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___getitem__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
   } 
-  arg2 = static_cast< int64_t >(val2);
-  (arg1)->setErrorCode(arg2);
-  resultobj = SWIG_Py_Void();
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
+  try {
+    result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *) &std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____getitem____SWIG_1((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1,SWIG_STD_MOVE(arg2));
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  }
+  {
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
+  }
+  (void)swig::container_owner >::value_type>::category>::back_reference(resultobj, swig_obj[0]);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Error_setErrorMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec___getitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec___getitem__", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec___getitem____SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec___getitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec___getitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__getitem__(SWIGPY_SLICEOBJECT *)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__getitem__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type) const\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  bcos::Error *arg1 = (bcos::Error *) 0 ;
-  std::string *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< bcos::Error > tempshared1 ;
-  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  ptrdiff_t val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared3 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Error_setErrorMessage", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec___setitem__" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec___setitem__" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type >(val2);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorMessage" "', argument " "1"" of type '" "bcos::Error *""'");
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "NodeInfoVec___setitem__" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+      if (argp3) tempshared3 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      arg3 = &tempshared3;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
-      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+      arg3 = (argp3) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3) : &tempshared3;
     }
   }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+  try {
+    std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg3);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
   }
-  (arg1)->setErrorMessage((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *Error_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
+SWIGINTERN PyObject *_wrap_NodeInfoVec___setitem__(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec___setitem__", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec___setitem____SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        _v = PySlice_Check(argv[1]);
+      }
+      if (_v) {
+        int res = swig::asptr(argv[2], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_NodeInfoVec___setitem____SWIG_0(self, argc, argv);
+        }
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0);
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_NodeInfoVec___setitem____SWIG_2(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec___setitem__'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > const &)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__setitem__(SWIGPY_SLICEOBJECT *)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::__setitem__(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::difference_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)\n");
+  return 0;
 }
 
-SWIGINTERN PyObject *Error_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
 
-SWIGINTERN PyObject *_wrap_InputBuffer_data_set(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_pop(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  InputBuffer *arg1 = (InputBuffer *) 0 ;
-  unsigned char *arg2 = (unsigned char *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "InputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_set" "', argument " "1"" of type '" "InputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_pop" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "InputBuffer_data_set" "', argument " "2"" of type '" "unsigned char const *""'"); 
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  try {
+    result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__pop(arg1);
+  } catch(std::out_of_range &_e) {
+    SWIG_exception_fail(SWIG_IndexError, (&_e)->what());
+  }
+  {
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
   }
-  arg2 = reinterpret_cast< unsigned char * >(argp2);
-  if (arg1) (arg1)->data = (unsigned char const *)arg2;
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_InputBuffer_data_get(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_append(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  InputBuffer *arg1 = (InputBuffer *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  unsigned char *result = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared2 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec_append", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_get" "', argument " "1"" of type '" "InputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_append" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  result = (unsigned char *) ((arg1)->data);
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 |  0 );
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "NodeInfoVec_append" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp2) tempshared2 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      arg2 = &tempshared2;
+    } else {
+      arg2 = (argp2) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2) : &tempshared2;
+    }
+  }
+  std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__append(arg1,(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_InputBuffer_len_set(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_NodeInfoVec__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
   PyObject *resultobj = 0;
-  InputBuffer *arg1 = (InputBuffer *) 0 ;
-  uint64_t arg2 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  unsigned long long val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "InputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_set" "', argument " "1"" of type '" "InputBuffer *""'"); 
-  }
-  arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "InputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'");
-  } 
-  arg2 = static_cast< uint64_t >(val2);
-  if (arg1) (arg1)->len = arg2;
-  resultobj = SWIG_Py_Void();
+  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_InputBuffer_len_get(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_NodeInfoVec__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  InputBuffer *arg1 = (InputBuffer *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  uint64_t result;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = 0 ;
+  int res1 = SWIG_OLDOBJ ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_get" "', argument " "1"" of type '" "InputBuffer *""'"); 
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  {
+    std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *ptr = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *)0;
+    res1 = swig::asptr(swig_obj[0], &ptr);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const &""'"); 
+    }
+    arg1 = ptr;
   }
-  arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  result = (uint64_t) ((arg1)->len);
-  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const &)*arg1);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_NEW |  0 );
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_InputBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_empty(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  InputBuffer *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "new_InputBuffer", 0, 0, 0)) SWIG_fail;
-  result = (InputBuffer *)new InputBuffer();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_InputBuffer, SWIG_POINTER_NEW |  0 );
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_empty" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (bool)((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->empty();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_InputBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_size(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  InputBuffer *arg1 = (InputBuffer *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, SWIG_POINTER_DISOWN |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_InputBuffer" "', argument " "1"" of type '" "InputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_size" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  delete arg1;
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = ((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->size();
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *InputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_InputBuffer, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *InputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_OutputBuffer_data_set(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_swap(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
-  unsigned char *arg2 = (unsigned char *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   void *argp2 = 0 ;
@@ -14755,18 +15135,21 @@ SWIGINTERN PyObject *_wrap_OutputBuffer_data_set(PyObject *self, PyObject *args)
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec_swap", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_set" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_swap" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN |  0 );
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t,  0 );
   if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "OutputBuffer_data_set" "', argument " "2"" of type '" "unsigned char *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "NodeInfoVec_swap" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > &""'"); 
   }
-  arg2 = reinterpret_cast< unsigned char * >(argp2);
-  if (arg1) (arg1)->data = arg2;
+  if (!argp2) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "NodeInfoVec_swap" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > &""'"); 
+  }
+  arg2 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp2);
+  (arg1)->swap(*arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -14774,101 +15157,109 @@ SWIGINTERN PyObject *_wrap_OutputBuffer_data_set(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_OutputBuffer_data_get(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_begin(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  unsigned char *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_get" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_begin" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  result = (unsigned char *) ((arg1)->data);
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 |  0 );
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (arg1)->begin();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_OutputBuffer_len_set(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_end(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
-  uint64_t arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  unsigned long long val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_set" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_end" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "OutputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'");
-  } 
-  arg2 = static_cast< uint64_t >(val2);
-  if (arg1) (arg1)->len = arg2;
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (arg1)->end();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_OutputBuffer_len_get(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_rbegin(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint64_t result;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::reverse_iterator result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_get" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_rbegin" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  result = (uint64_t) ((arg1)->len);
-  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (arg1)->rbegin();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::reverse_iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_OutputBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_rend(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::reverse_iterator result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "new_OutputBuffer", 0, 0, 0)) SWIG_fail;
-  result = (OutputBuffer *)new OutputBuffer();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_OutputBuffer, SWIG_POINTER_NEW |  0 );
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_rend" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (arg1)->rend();
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::reverse_iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_clear(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
@@ -14876,12 +15267,12 @@ SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) {
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, SWIG_POINTER_DISOWN |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_OutputBuffer" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_clear" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  delete arg1;
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  (arg1)->clear();
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -14889,123 +15280,97 @@ SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *OutputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_OutputBuffer, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *OutputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_delete_FrontConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_get_allocator(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  SwigValueWrapper< std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_get_allocator" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = ((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->get_allocator();
+  resultobj = SWIG_NewPointerObj((new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_threadPoolSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_NodeInfoVec__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg1 ;
+  size_t val1 ;
+  int ecode1 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1);
+  if (!SWIG_IsOK(ecode1)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg1 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val1);
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >(arg1);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec_pop_back(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  int result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_threadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_pop_back" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
-  resultobj = SWIG_From_int(static_cast< int >(result));
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  (arg1)->pop_back();
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setThreadPoolSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  int arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  int val2 ;
+  size_t val2 ;
   int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setThreadPoolSize", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_resize" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "2"" of type '" "int""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec_resize" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
   } 
-  arg2 = static_cast< int >(val2);
-  (arg1)->setThreadPoolSize(arg2);
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val2);
+  (arg1)->resize(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -15013,332 +15378,422 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setThreadPoolSize(PyObject *self, PyObjec
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_nodeID(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_nodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_erase" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T >::iterator > *iter_t = dynamic_cast >::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
     }
   }
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2));
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setNodeID(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  swig::SwigPyIterator *iter3 = 0 ;
+  int res3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setNodeID", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setNodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_erase" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T >::iterator > *iter_t = dynamic_cast >::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
     }
   }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); 
+  res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res3) || !iter3) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T >::iterator > *iter_t = dynamic_cast >::iterator > *>(iter3);
+    if (iter_t) {
+      arg3 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_erase" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
     }
-    arg2 = ptr;
   }
-  (arg1)->setNodeID((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_selfEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_erase(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec_erase", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast >::iterator > *>(iter) != 0));
+      if (_v) {
+        return _wrap_NodeInfoVec_erase__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast >::iterator > *>(iter) != 0));
+      if (_v) {
+        swig::SwigPyIterator *iter = 0;
+        int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+        _v = (SWIG_IsOK(res) && iter && (dynamic_cast >::iterator > *>(iter) != 0));
+        if (_v) {
+          return _wrap_NodeInfoVec_erase__SWIG_1(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec_erase'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::erase(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::erase(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_NodeInfoVec__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg1 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg2 = 0 ;
+  size_t val1 ;
+  int ecode1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *result = 0 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1);
+  if (!SWIG_IsOK(ecode1)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg1 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val1);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_selfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_NodeInfoVec" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp2) tempshared2 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      arg2 = &tempshared2;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      arg2 = (argp2) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2) : &tempshared2;
     }
   }
-  result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *)new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >(arg1,(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)*arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_mutableSelfEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_NodeInfoVec(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_NodeInfoVec", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 0) {
+    return _wrap_new_NodeInfoVec__SWIG_0(self, argc, argv);
+  }
+  if (argc == 1) {
+    int _v = 0;
+    {
+      int res = SWIG_AsVal_size_t(argv[0], NULL);
+      _v = SWIG_CheckState(res);
+    }
+    if (_v) {
+      return _wrap_new_NodeInfoVec__SWIG_2(self, argc, argv);
+    }
+  }
+  if (argc == 1) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_new_NodeInfoVec__SWIG_1(self, argc, argv);
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    {
+      int res = SWIG_AsVal_size_t(argv[0], NULL);
+      _v = SWIG_CheckState(res);
+    }
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_new_NodeInfoVec__SWIG_3(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_NodeInfoVec'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::vector()\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::vector(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const &)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::vector(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::vector(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec_push_back(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared2 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec_push_back", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_push_back" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "NodeInfoVec_push_back" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp2) tempshared2 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2);
+      arg2 = &tempshared2;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      arg2 = (argp2) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp2) : &tempshared2;
     }
   }
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  (arg1)->push_back((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)*arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setSelfEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_front(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  ppc::protocol::EndPoint *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setSelfEndPoint", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); 
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_front" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); 
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *) &((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->front();
+  {
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
   }
-  arg2 = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
-  (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
-  resultobj = SWIG_Py_Void();
+  (void)swig::container_owner >::value_type>::category>::back_reference(resultobj, swig_obj[0]);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setGatewayGrpcTarget(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_back(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGatewayGrpcTarget", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_back" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *) &((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->back();
   {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
   }
-  (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  (void)swig::container_owner >::value_type>::category>::back_reference(resultobj, swig_obj[0]);
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_gatewayGrpcTarget(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_assign(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared3 ;
+  PyObject *swig_obj[3] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec_assign", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_assign" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec_assign" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val2);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_gatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "NodeInfoVec_assign" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp3) tempshared3 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      arg3 = &tempshared3;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      arg3 = (argp3) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3) : &tempshared3;
     }
   }
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  (arg1)->assign(arg2,(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)*arg3);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setGrpcConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  ppc::protocol::GrpcConfig::Ptr arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared3 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGrpcConfig", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_resize" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec_resize" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val2);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGrpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "NodeInfoVec_resize" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp3) tempshared3 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      arg3 = &tempshared3;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGrpcConfig" "', argument " "2"" of type '" "ppc::protocol::GrpcConfig::Ptr""'");
+      arg3 = (argp3) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3) : &tempshared3;
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2);
   }
-  (arg1)->setGrpcConfig(arg2);
+  (arg1)->resize(arg2,(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)*arg3);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -15346,538 +15801,786 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setGrpcConfig(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_grpcConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_resize(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[4] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec_resize", 0, 3, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_size_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_NodeInfoVec_resize__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_size_t(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0);
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_NodeInfoVec_resize__SWIG_1(self, argc, argv);
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec_resize'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::resize(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::resize(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_NodeInfoVec_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg3 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::GrpcConfig::Ptr *result = 0 ;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_insert" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_insert" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T >::iterator > *iter_t = dynamic_cast >::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_insert" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+    }
+  }
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_grpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "NodeInfoVec_insert" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp3) tempshared3 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3);
+      arg3 = &tempshared3;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      arg3 = (argp3) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp3) : &tempshared3;
     }
   }
-  result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
-  {
-    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN);
-  }
+  result = std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg3);
+  resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator & >(result)),
+    swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN);
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_generateNodeInfo(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator arg2 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg3 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type *arg4 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::INodeInfo::Ptr result;
+  swig::SwigPyIterator *iter2 = 0 ;
+  int res2 ;
+  size_t val3 ;
+  int ecode3 = 0 ;
+  void *argp4 ;
+  int res4 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type tempshared4 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_insert" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
+  }
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0);
+  if (!SWIG_IsOK(res2) || !iter2) {
+    SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_insert" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+  } else {
+    swig::SwigPyIterator_T >::iterator > *iter_t = dynamic_cast >::iterator > *>(iter2);
+    if (iter_t) {
+      arg2 = iter_t->get_current();
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "NodeInfoVec_insert" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator""'");
+    }
+  }
+  ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "NodeInfoVec_insert" "', argument " "3"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg3 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val3);
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_generateNodeInfo" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
+    res4 = SWIG_ConvertPtrAndOwn(swig_obj[3], &argp4, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "NodeInfoVec_insert" "', argument " "4"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+      if (argp4) tempshared4 = *reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp4);
+      delete reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp4);
+      arg4 = &tempshared4;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+      arg4 = (argp4) ? reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type * >(argp4) : &tempshared4;
     }
   }
-  result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
-  resultobj = SWIG_NewPointerObj((new ppc::protocol::INodeInfo::Ptr(result)), SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr, SWIG_POINTER_OWN |  0 );
+  std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg4);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_getComponents(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
+SWIGINTERN PyObject *_wrap_NodeInfoVec_insert(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[5] = {
+    0
+  };
   
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_getComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
+  if (!(argc = SWIG_Python_UnpackTuple(args, "NodeInfoVec_insert", 0, 4, argv))) SWIG_fail;
+  --argc;
+  if (argc == 3) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast >::iterator > *>(iter) != 0));
+      if (_v) {
+        int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0);
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_NodeInfoVec_insert__SWIG_0(self, argc, argv);
+        }
+      }
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
+  }
+  if (argc == 4) {
+    int _v = 0;
+    int res = swig::asptr(argv[0], (std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > >**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      swig::SwigPyIterator *iter = 0;
+      int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0);
+      _v = (SWIG_IsOK(res) && iter && (dynamic_cast >::iterator > *>(iter) != 0));
+      if (_v) {
+        {
+          int res = SWIG_AsVal_size_t(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          int res = SWIG_ConvertPtr(argv[3], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0);
+          _v = SWIG_CheckState(res);
+          if (_v) {
+            return _wrap_NodeInfoVec_insert__SWIG_1(self, argc, argv);
+          }
+        }
+      }
     }
   }
-  result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
-  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(*result));
-  return resultobj;
+  
 fail:
-  return NULL;
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'NodeInfoVec_insert'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::insert(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)\n"
+    "    std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::insert(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::iterator,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type,std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::value_type const &)\n");
+  return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_reserve(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  std::vector< std::string,std::allocator< std::string > > *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  size_t val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setComponents", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
-    res2 = swig::asptr(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
-    }
-    arg2 = ptr;
+  if (!SWIG_Python_UnpackTuple(args, "NodeInfoVec_reserve", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_reserve" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2);
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "NodeInfoVec_reserve" "', argument " "2"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type""'");
+  } 
+  arg2 = static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type >(val2);
+  (arg1)->reserve(arg2);
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_addComponent(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_NodeInfoVec_capacity(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::size_type result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_addComponent", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_addComponent" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "NodeInfoVec_capacity" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *""'"); 
   }
-  (arg1)->addComponent((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  result = ((std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > const *)arg1)->capacity();
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfig_mutableComponents(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_NodeInfoVec(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *arg1 = (std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
-  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
-      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
-  result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
+  arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *FrontConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *NodeInfoVec_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *_wrap_delete_FrontConfigBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *NodeInfoVec_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
+  char *arg1 = (char *) 0 ;
+  char *arg2 = (char *) 0 ;
+  int arg3 ;
+  int32_t arg4 ;
+  std::string *arg5 = 0 ;
+  int res1 ;
+  char *buf1 = 0 ;
+  int alloc1 = 0 ;
+  int res2 ;
+  char *buf2 = 0 ;
+  int alloc2 = 0 ;
+  int val3 ;
+  int ecode3 = 0 ;
+  int val4 ;
+  int ecode4 = 0 ;
+  int res5 = SWIG_OLDOBJ ;
+  bcos::Error result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_POINTER_DISOWN |  0 );
+  if ((nobjs < 5) || (nobjs > 5)) SWIG_fail;
+  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfigBuilder" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
   }
-  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
-  delete arg1;
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< char * >(buf1);
+  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
+  }
+  arg2 = reinterpret_cast< char * >(buf2);
+  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
+  } 
+  arg3 = static_cast< int >(val3);
+  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+  if (!SWIG_IsOK(ecode4)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
+  } 
+  arg4 = static_cast< int32_t >(val4);
+  {
+    std::string *ptr = (std::string *)0;
+    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
+    if (!SWIG_IsOK(res5)) {
+      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    }
+    arg5 = ptr;
+  }
+  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  }
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return resultobj;
 fail:
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  ppc::front::FrontConfig::Ptr result;
-  
-  (void)self;
-  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 |  0 );
+  char *arg1 = (char *) 0 ;
+  char *arg2 = (char *) 0 ;
+  int arg3 ;
+  int32_t arg4 ;
+  std::string *arg5 = 0 ;
+  bcos::Error *arg6 = 0 ;
+  int res1 ;
+  char *buf1 = 0 ;
+  int alloc1 = 0 ;
+  int res2 ;
+  char *buf2 = 0 ;
+  int alloc2 = 0 ;
+  int val3 ;
+  int ecode3 = 0 ;
+  int val4 ;
+  int ecode4 = 0 ;
+  int res5 = SWIG_OLDOBJ ;
+  void *argp6 = 0 ;
+  int res6 = 0 ;
+  std::shared_ptr< bcos::Error const > tempshared6 ;
+  bcos::Error result;
+  
+  (void)self;
+  if ((nobjs < 6) || (nobjs > 6)) SWIG_fail;
+  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
   }
-  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
+  arg1 = reinterpret_cast< char * >(buf1);
+  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
+  }
+  arg2 = reinterpret_cast< char * >(buf2);
+  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
+  } 
+  arg3 = static_cast< int >(val3);
+  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+  if (!SWIG_IsOK(ecode4)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
+  } 
+  arg4 = static_cast< int32_t >(val4);
   {
-    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
+    std::string *ptr = (std::string *)0;
+    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
+    if (!SWIG_IsOK(res5)) {
+      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    }
+    arg5 = ptr;
+  }
+  {
+    int newmem = 0;
+    res6 = SWIG_ConvertPtrAndOwn(swig_obj[5], &argp6, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
+    if (!SWIG_IsOK(res6)) {
+      SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'");
+    }
+    if (!argp6) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "bcos::Error const &""'"); 
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared6 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6);
+      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6);
+      arg6 = const_cast< bcos::Error * >(tempshared6.get());
+    } else {
+      arg6 = const_cast< bcos::Error * >(reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6)->get());
+    }
+  }
+  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
   }
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return resultobj;
 fail:
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
-  int arg2 ;
-  std::string arg3 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  int val2 ;
-  int ecode2 = 0 ;
-  ppc::front::FrontConfig::Ptr result;
+  char *arg1 = (char *) 0 ;
+  char *arg2 = (char *) 0 ;
+  int arg3 ;
+  int32_t arg4 ;
+  std::string *arg5 = 0 ;
+  std::exception *arg6 = 0 ;
+  int res1 ;
+  char *buf1 = 0 ;
+  int alloc1 = 0 ;
+  int res2 ;
+  char *buf2 = 0 ;
+  int alloc2 = 0 ;
+  int val3 ;
+  int ecode3 = 0 ;
+  int val4 ;
+  int ecode4 = 0 ;
+  int res5 = SWIG_OLDOBJ ;
+  void *argp6 = 0 ;
+  int res6 = 0 ;
+  bcos::Error result;
   
   (void)self;
-  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 |  0 );
+  if ((nobjs < 6) || (nobjs > 6)) SWIG_fail;
+  res1 = SWIG_AsCharPtrAndSize(swig_obj[0], &buf1, NULL, &alloc1);
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_buildError" "', argument " "1"" of type '" "char const *""'");
   }
-  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
-  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfigBuilder_build" "', argument " "2"" of type '" "int""'");
+  arg1 = reinterpret_cast< char * >(buf1);
+  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_buildError" "', argument " "2"" of type '" "char const *""'");
+  }
+  arg2 = reinterpret_cast< char * >(buf2);
+  ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Error_buildError" "', argument " "3"" of type '" "int""'");
   } 
-  arg2 = static_cast< int >(val2);
+  arg3 = static_cast< int >(val3);
+  ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+  if (!SWIG_IsOK(ecode4)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "Error_buildError" "', argument " "4"" of type '" "int32_t""'");
+  } 
+  arg4 = static_cast< int32_t >(val4);
   {
     std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "FrontConfigBuilder_build" "', argument " "3"" of type '" "std::string""'"); 
+    res5 = SWIG_AsPtr_std_string(swig_obj[4], &ptr);
+    if (!SWIG_IsOK(res5)) {
+      SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
     }
-    arg3 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "5"" of type '" "std::string const &""'"); 
+    }
+    arg5 = ptr;
   }
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+  res6 = SWIG_ConvertPtr(swig_obj[5], &argp6, SWIGTYPE_p_std__exception,  0  | 0);
+  if (!SWIG_IsOK(res6)) {
+    SWIG_exception_fail(SWIG_ArgError(res6), "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); 
+  }
+  if (!argp6) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); 
+  }
+  arg6 = reinterpret_cast< std::exception * >(argp6);
+  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
   {
-    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
+    std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
   }
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return resultobj;
 fail:
+  if (alloc1 == SWIG_NEWOBJ) delete[] buf1;
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res5)) delete arg5;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Error_buildError(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
-  PyObject *argv[4] = {
+  PyObject *argv[7] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "FrontConfigBuilder_build", 0, 3, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "Error_buildError", 0, 6, argv))) SWIG_fail;
   --argc;
-  if (argc == 1) {
+  if (argc == 5) {
     int _v = 0;
-    void *vptr = 0;
-    int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0);
+    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
     _v = SWIG_CheckState(res);
     if (_v) {
-      return _wrap_FrontConfigBuilder_build__SWIG_0(self, argc, argv);
+      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        {
+          int res = SWIG_AsVal_int(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          {
+            int res = SWIG_AsVal_int(argv[3], NULL);
+            _v = SWIG_CheckState(res);
+          }
+          if (_v) {
+            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
+            _v = SWIG_CheckState(res);
+            if (_v) {
+              return _wrap_Error_buildError__SWIG_0(self, argc, argv);
+            }
+          }
+        }
+      }
     }
   }
-  if (argc == 3) {
+  if (argc == 6) {
     int _v = 0;
-    void *vptr = 0;
-    int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0);
+    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
     _v = SWIG_CheckState(res);
     if (_v) {
-      {
-        int res = SWIG_AsVal_int(argv[1], NULL);
-        _v = SWIG_CheckState(res);
-      }
+      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
+      _v = SWIG_CheckState(res);
       if (_v) {
-        int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0));
-        _v = SWIG_CheckState(res);
+        {
+          int res = SWIG_AsVal_int(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
         if (_v) {
-          return _wrap_FrontConfigBuilder_build__SWIG_1(self, argc, argv);
+          {
+            int res = SWIG_AsVal_int(argv[3], NULL);
+            _v = SWIG_CheckState(res);
+          }
+          if (_v) {
+            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
+            _v = SWIG_CheckState(res);
+            if (_v) {
+              int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0);
+              _v = SWIG_CheckState(res);
+              if (_v) {
+                return _wrap_Error_buildError__SWIG_1(self, argc, argv);
+              }
+            }
+          }
         }
       }
     }
   }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'FrontConfigBuilder_build'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::front::FrontConfigBuilder::build() const\n"
-    "    ppc::front::FrontConfigBuilder::build(int,std::string) const\n");
-  return 0;
-}
-
-
-SWIGINTERN PyObject *FrontConfigBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_printFrontDesc(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::FrontConfig::Ptr *arg1 = 0 ;
-  void *argp1 ;
-  int res1 = 0 ;
-  ppc::front::FrontConfig::Ptr tempshared1 ;
-  PyObject *swig_obj[1] ;
-  std::string result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printFrontDesc" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr const &""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp1) tempshared1 = *reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
-      delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
-      arg1 = &tempshared1;
-    } else {
-      arg1 = (argp1) ? reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1) : &tempshared1;
+  if (argc == 6) {
+    int _v = 0;
+    int res = SWIG_AsCharPtrAndSize(argv[0], 0, NULL, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        {
+          int res = SWIG_AsVal_int(argv[2], NULL);
+          _v = SWIG_CheckState(res);
+        }
+        if (_v) {
+          {
+            int res = SWIG_AsVal_int(argv[3], NULL);
+            _v = SWIG_CheckState(res);
+          }
+          if (_v) {
+            int res = SWIG_AsPtr_std_string(argv[4], (std::string**)(0));
+            _v = SWIG_CheckState(res);
+            if (_v) {
+              int res = SWIG_ConvertPtr(argv[5], 0, SWIGTYPE_p_std__exception, SWIG_POINTER_NO_NULL | 0);
+              _v = SWIG_CheckState(res);
+              if (_v) {
+                return _wrap_Error_buildError__SWIG_2(self, argc, argv);
+              }
+            }
+          }
+        }
+      }
     }
   }
-  result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
-  return resultobj;
+  
 fail:
-  return NULL;
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'Error_buildError'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &)\n"
+    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,bcos::Error const &)\n"
+    "    bcos::Error::buildError(char const *,char const *,int,int32_t,std::string const &,std::exception const &)\n");
+  return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
+SWIGINTERN PyObject *_wrap_new_Error__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *result = 0 ;
+  bcos::Error *result = 0 ;
   
   (void)self;
   if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
+  result = (bcos::Error *)new bcos::Error();
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_new_Error__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  std::string *arg1 = 0 ;
-  uint16_t arg2 ;
-  int res1 = SWIG_OLDOBJ ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
-  ppc::protocol::EndPoint *result = 0 ;
+  int64_t arg1 ;
+  std::string arg2 ;
+  long long val1 ;
+  int ecode1 = 0 ;
+  bcos::Error *result = 0 ;
   
   (void)self;
   if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  ecode1 = SWIG_AsVal_long_SS_long(swig_obj[0], &val1);
+  if (!SWIG_IsOK(ecode1)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_Error" "', argument " "1"" of type '" "int64_t""'");
+  } 
+  arg1 = static_cast< int64_t >(val1);
   {
     std::string *ptr = (std::string *)0;
-    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); 
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "new_Error" "', argument " "2"" of type '" "std::string""'"); 
     }
-    arg1 = ptr;
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  result = (bcos::Error *)new bcos::Error(arg1,arg2);
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_EndPoint" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
-  if (SWIG_IsNewObj(res1)) delete arg1;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res1)) delete arg1;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_EndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_Error(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
   PyObject *argv[3] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "new_EndPoint", 0, 2, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_Error", 0, 2, argv))) SWIG_fail;
   --argc;
   if (argc == 0) {
-    return _wrap_new_EndPoint__SWIG_0(self, argc, argv);
+    return _wrap_new_Error__SWIG_0(self, argc, argv);
   }
   if (argc == 2) {
     int _v = 0;
-    int res = SWIG_AsPtr_std_string(argv[0], (std::string**)(0));
-    _v = SWIG_CheckState(res);
+    {
+      int res = SWIG_AsVal_long_SS_long(argv[0], NULL);
+      _v = SWIG_CheckState(res);
+    }
     if (_v) {
-      {
-        int res = SWIG_AsVal_unsigned_SS_short(argv[1], NULL);
-        _v = SWIG_CheckState(res);
-      }
+      int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0));
+      _v = SWIG_CheckState(res);
       if (_v) {
-        return _wrap_new_EndPoint__SWIG_1(self, argc, argv);
+        return _wrap_new_Error__SWIG_1(self, argc, argv);
       }
     }
   }
   
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_EndPoint'.\n"
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_Error'.\n"
     "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::EndPoint::EndPoint()\n"
-    "    ppc::protocol::EndPoint::EndPoint(std::string const &,uint16_t)\n");
+    "    bcos::Error::Error()\n"
+    "    bcos::Error::Error(int64_t,std::string)\n");
   return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_EndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_Error(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  bcos::Error *arg1 = (bcos::Error *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< bcos::Error > tempshared1 ;
+  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_EndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Error" "', argument " "1"" of type '" "bcos::Error *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  delete arg1;
+  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -15885,79 +16588,113 @@ SWIGINTERN PyObject *_wrap_delete_EndPoint(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_host(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Error_errorCode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  bcos::Error *arg1 = (bcos::Error *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< bcos::Error const > tempshared1 ;
+  std::shared_ptr< bcos::Error const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  int64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_host" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorCode" "', argument " "1"" of type '" "bcos::Error const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  result = (int64_t)((bcos::Error const *)arg1)->errorCode();
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_port(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Error_errorMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  bcos::Error *arg1 = (bcos::Error *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< bcos::Error const > tempshared1 ;
+  std::shared_ptr< bcos::Error const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_port" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_errorMessage" "', argument " "1"" of type '" "bcos::Error const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_setHost(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Error_setErrorCode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
-  std::string arg2 ;
+  bcos::Error *arg1 = (bcos::Error *) 0 ;
+  int64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< bcos::Error > tempshared1 ;
+  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
+  long long val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setHost", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setHost" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  if (!SWIG_Python_UnpackTuple(args, "Error_setErrorCode", 2, 2, swig_obj)) SWIG_fail;
   {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "EndPoint_setHost" "', argument " "2"" of type '" "std::string""'"); 
-    }
-    arg2 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorCode" "', argument " "1"" of type '" "bcos::Error *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  (arg1)->setHost(arg2);
+  ecode2 = SWIG_AsVal_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Error_setErrorCode" "', argument " "2"" of type '" "int64_t""'");
+  } 
+  arg2 = static_cast< int64_t >(val2);
+  (arg1)->setErrorCode(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -15965,328 +16702,299 @@ SWIGINTERN PyObject *_wrap_EndPoint_setHost(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_setPort(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Error_setErrorMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
-  uint16_t arg2 ;
+  bcos::Error *arg1 = (bcos::Error *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
+  std::shared_ptr< bcos::Error > tempshared1 ;
+  std::shared_ptr< bcos::Error > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setPort", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setPort" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
+  if (!SWIG_Python_UnpackTuple(args, "Error_setErrorMessage", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Error_setErrorMessage" "', argument " "1"" of type '" "bcos::Error *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  bcos::Error > * >(argp1);
+      arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "EndPoint_setPort" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPort(arg2);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_setErrorMessage" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setErrorMessage((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_setListenIp(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *Error_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *Error_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_InputBuffer_data_set(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
-  std::string *arg2 = 0 ;
+  InputBuffer *arg1 = (InputBuffer *) 0 ;
+  unsigned char *arg2 = (unsigned char *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setListenIp", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "InputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setListenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_set" "', argument " "1"" of type '" "InputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+  arg1 = reinterpret_cast< InputBuffer * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "InputBuffer_data_set" "', argument " "2"" of type '" "unsigned char const *""'"); 
   }
-  (arg1)->setListenIp((std::string const &)*arg2);
+  arg2 = reinterpret_cast< unsigned char * >(argp2);
+  if (arg1) (arg1)->data = (unsigned char const *)arg2;
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_entryPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_InputBuffer_data_get(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  InputBuffer *arg1 = (InputBuffer *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string result;
+  unsigned char *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_entryPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_data_get" "', argument " "1"" of type '" "InputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  arg1 = reinterpret_cast< InputBuffer * >(argp1);
+  result = (unsigned char *) ((arg1)->data);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_listenEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_InputBuffer_len_set(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  InputBuffer *arg1 = (InputBuffer *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "InputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_set" "', argument " "1"" of type '" "InputBuffer *""'"); 
+  }
+  arg1 = reinterpret_cast< InputBuffer * >(argp1);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "InputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
+  if (arg1) (arg1)->len = arg2;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_InputBuffer_len_get(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  InputBuffer *arg1 = (InputBuffer *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string result;
+  uint64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "InputBuffer_len_get" "', argument " "1"" of type '" "InputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  arg1 = reinterpret_cast< InputBuffer * >(argp1);
+  result = (uint64_t) ((arg1)->len);
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_EndPoint_listenIp(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_InputBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  InputBuffer *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "new_InputBuffer", 0, 0, 0)) SWIG_fail;
+  result = (InputBuffer *)new InputBuffer();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_InputBuffer, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_InputBuffer(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  InputBuffer *arg1 = (InputBuffer *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_InputBuffer, SWIG_POINTER_DISOWN |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_InputBuffer" "', argument " "1"" of type '" "InputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  arg1 = reinterpret_cast< InputBuffer * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *EndPoint_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *InputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_InputBuffer, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *EndPoint_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *InputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   return SWIG_Python_InitShadowInstance(args);
 }
 
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
+SWIGINTERN PyObject *_wrap_OutputBuffer_data_set(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
+  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  unsigned char *arg2 = (unsigned char *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_data_set", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_set" "', argument " "1"" of type '" "OutputBuffer *""'"); 
+  }
+  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "OutputBuffer_data_set" "', argument " "2"" of type '" "unsigned char *""'"); 
+  }
+  arg2 = reinterpret_cast< unsigned char * >(argp2);
+  if (arg1) (arg1)->data = arg2;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_OutputBuffer_data_get(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint arg1 ;
-  bool arg2 ;
-  void *argp1 ;
+  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
-  bool val2 ;
-  int ecode2 = 0 ;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
-  
-  (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  {
-    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); 
-    }  
-    if (!argp1) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'");
-    } else {
-      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-      arg1 = *temp;
-      if (SWIG_IsNewObj(res1)) delete temp;
-    }
-  }
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'");
-  } 
-  arg2 = static_cast< bool >(val2);
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "new_GrpcServerConfig", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 0) {
-    return _wrap_new_GrpcServerConfig__SWIG_0(self, argc, argv);
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NO_NULL | 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      {
-        int res = SWIG_AsVal_bool(argv[1], NULL);
-        _v = SWIG_CheckState(res);
-      }
-      if (_v) {
-        return _wrap_new_GrpcServerConfig__SWIG_1(self, argc, argv);
-      }
-    }
-  }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_GrpcServerConfig'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig()\n"
-    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig(ppc::protocol::EndPoint,bool)\n");
-  return 0;
-}
-
-
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string result;
+  PyObject *swig_obj[1] ;
+  unsigned char *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  ppc::protocol::EndPoint arg2 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEndPoint", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'");
-    } else {
-      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
-    }
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_data_get" "', argument " "1"" of type '" "OutputBuffer *""'"); 
   }
-  (arg1)->setEndPoint(arg2);
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
+  result = (unsigned char *) ((arg1)->data);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_unsigned_char, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_OutputBuffer_len_set(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  bool arg2 ;
+  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
+  uint64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  bool val2 ;
+  unsigned long long val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_Python_UnpackTuple(args, "OutputBuffer_len_set", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_set" "', argument " "1"" of type '" "OutputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "OutputBuffer_len_set" "', argument " "2"" of type '" "uint64_t""'");
   } 
-  arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableHealthCheck(arg2);
+  arg2 = static_cast< uint64_t >(val2);
+  if (arg1) (arg1)->len = arg2;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16294,81 +17002,47 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self,
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_OutputBuffer_len_get(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
+  uint64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "OutputBuffer_len_get" "', argument " "1"" of type '" "OutputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
+  result = (uint64_t) ((arg1)->len);
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_OutputBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  bool result;
+  OutputBuffer *result = 0 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  if (!SWIG_Python_UnpackTuple(args, "new_OutputBuffer", 0, 0, 0)) SWIG_fail;
+  result = (OutputBuffer *)new OutputBuffer();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_OutputBuffer, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  OutputBuffer *arg1 = (OutputBuffer *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject *swig_obj[1] ;
@@ -16376,11 +17050,11 @@ SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *arg
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_DISOWN |  0 );
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_OutputBuffer, SWIG_POINTER_DISOWN |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_OutputBuffer" "', argument " "1"" of type '" "OutputBuffer *""'"); 
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  arg1 = reinterpret_cast< OutputBuffer * >(argp1);
   delete arg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
@@ -16389,41 +17063,24 @@ SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *arg
 }
 
 
-SWIGINTERN PyObject *GrpcServerConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *OutputBuffer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_OutputBuffer, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *GrpcServerConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *OutputBuffer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   return SWIG_Python_InitShadowInstance(args);
 }
 
-SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *result = 0 ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail;
-  result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
-  {
-    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_FrontConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
@@ -16431,17 +17088,17 @@ SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   (void)arg1; delete smartarg1;
@@ -16452,286 +17109,279 @@ SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_threadPoolSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  int result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_loadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_threadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
+  resultobj = SWIG_From_int(static_cast< int >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setThreadPoolSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  int arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  int val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setLoadBalancePolicy", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setThreadPoolSize", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg2 = ptr;
   }
-  (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "2"" of type '" "int""'");
+  } 
+  arg2 = static_cast< int >(val2);
+  (arg1)->setThreadPoolSize(arg2);
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_nodeID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_nodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setNodeID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  bool arg2 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  bool val2 ;
-  int ecode2 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setNodeID", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setNodeID" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
-  } 
-  arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableHealthCheck(arg2);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setNodeID" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setNodeID((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_selfEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  bool arg2 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  bool val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableDnslookup", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_selfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'");
-  } 
-  arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableDnslookup(arg2);
-  resultobj = SWIG_Py_Void();
+  result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_mutableSelfEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setSelfEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  ppc::protocol::EndPoint *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setSelfEndPoint", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeader" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); 
+  }
+  if (!argp2) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); 
+  }
+  arg2 = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
+  (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16739,231 +17389,208 @@ SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_encode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setGatewayGrpcTarget(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes > tempshared2 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_encode", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGatewayGrpcTarget", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
-    } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setGatewayGrpcTarget" "', argument " "2"" of type '" "std::string const &""'"); 
     }
+    arg2 = ptr;
   }
-  ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+  (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_decode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_gatewayGrpcTarget(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  bcos::bytesConstRef arg2 ;
-  uint64_t arg3 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  unsigned long long val3 ;
-  int ecode3 = 0 ;
-  PyObject *swig_obj[3] ;
-  int64_t result;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_decode", 3, 3, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_gatewayGrpcTarget" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "MessageOptionalHeader_decode" "', argument " "3"" of type '" "uint64_t""'");
-  } 
-  arg3 = static_cast< uint64_t >(val3);
-  result = (int64_t)(arg1)->decode(arg2,arg3);
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_componentType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setGrpcConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  ppc::protocol::GrpcConfig::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string result;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setGrpcConfig", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_componentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setGrpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setGrpcConfig" "', argument " "2"" of type '" "ppc::protocol::GrpcConfig::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2);
+  }
+  (arg1)->setGrpcConfig(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setComponentType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_grpcConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::string arg2 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::GrpcConfig::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setComponentType", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_grpcConfig" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
   {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "2"" of type '" "std::string""'"); 
-    }
-    arg2 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
+    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN);
   }
-  (arg1)->setComponentType(arg2);
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_generateNodeInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  OutputBuffer result;
+  ppc::protocol::INodeInfo::Ptr result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_generateNodeInfo" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+  result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
   {
-    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -16971,346 +17598,279 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, P
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNodeBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_getComponents(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  char *arg2 = (char *) 0 ;
-  uint64_t arg3 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcNodeBuffer", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_getComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int res; Py_ssize_t size = 0; const void *buf = 0;
-#ifndef Py_LIMITED_API
-    Py_buffer view;
-    res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO);
-#else
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
-    res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size);
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-#endif
-    if (res < 0) {
-      PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
-#ifndef Py_LIMITED_API
-    size = view.len;
-    buf = view.buf;
-    PyBuffer_Release(&view);
-#endif
-    arg2 = (char *) buf;
-    arg3 = (uint64_t) (size / sizeof(char));
   }
-  (arg1)->setSrcNodeBuffer(arg2,arg3);
-  resultobj = SWIG_Py_Void();
+  result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
+  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::vector< std::string,std::allocator< std::string > > *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  OutputBuffer result;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setComponents", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
   {
-    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+    std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0;
+    res2 = swig::asptr(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setComponents" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > > const &""'"); 
+    }
+    arg2 = ptr;
   }
+  (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNodeBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_addComponent(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  char *arg2 = (char *) 0 ;
-  uint64_t arg3 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstNodeBuffer", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_addComponent", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_addComponent" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int res; Py_ssize_t size = 0; const void *buf = 0;
-#ifndef Py_LIMITED_API
-    Py_buffer view;
-    res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO);
-#else
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
-    res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size);
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-#endif
-    if (res < 0) {
-      PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-#ifndef Py_LIMITED_API
-    size = view.len;
-    buf = view.buf;
-    PyBuffer_Release(&view);
-#endif
-    arg2 = (char *) buf;
-    arg3 = (uint64_t) (size / sizeof(char));
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
-  (arg1)->setDstNodeBuffer(arg2,arg3);
+  (arg1)->addComponent((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstInst(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_mutableComponents(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  std::vector< std::string,std::allocator< std::string > > *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_mutableComponents" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstInst(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_meta(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::front::FrontConfig const > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstInst", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_meta" "', argument " "1"" of type '" "ppc::front::FrontConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); 
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg2 = ptr;
   }
-  (arg1)->setDstInst((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  result = ((ppc::front::FrontConfig const *)arg1)->meta();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_topic(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfig_setMeta(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  ppc::front::FrontConfig *arg1 = (ppc::front::FrontConfig *) 0 ;
+  std::string arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  std::shared_ptr< ppc::front::FrontConfig > tempshared1 ;
+  std::shared_ptr< ppc::front::FrontConfig > *smartarg1 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "FrontConfig_setMeta", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_topic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfig_setMeta" "', argument " "1"" of type '" "ppc::front::FrontConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::FrontConfig > * >(argp1);
+      arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  {
+    std::string *ptr = (std::string *)0;
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "FrontConfig_setMeta" "', argument " "2"" of type '" "std::string""'"); 
+    }
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  (arg1)->setMeta(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *FrontConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_FrontConfigBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::unique_ptr< std::string > rvrdeleter2 ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__string, SWIG_POINTER_RELEASE |  0 );
-  if (!SWIG_IsOK(res2)) {
-    if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::string &&""'");
-    } else {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); 
-    }
-  }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); 
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfigBuilder" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder *""'"); 
   }
-  arg2 = reinterpret_cast< std::string * >(argp2);
-  rvrdeleter2.reset(arg2);
-  (arg1)->setTopic((std::string &&)*arg2);
+  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
+  delete arg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17318,104 +17878,133 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_0(PyObject *self
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  ppc::front::FrontConfig::Ptr result;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
+  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::front::FrontConfigBuilder *arg1 = (ppc::front::FrontConfigBuilder *) 0 ;
+  int arg2 ;
+  std::string arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int val2 ;
+  int ecode2 = 0 ;
+  ppc::front::FrontConfig::Ptr result;
+  
+  (void)self;
+  if ((nobjs < 3) || (nobjs > 3)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
+  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfigBuilder_build" "', argument " "2"" of type '" "int""'");
+  } 
+  arg2 = static_cast< int >(val2);
   {
     std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+    int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "FrontConfigBuilder_build" "', argument " "3"" of type '" "std::string""'"); 
     }
-    arg2 = ptr;
+    arg3 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+  {
+    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
   }
-  (arg1)->setTopic((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build(PyObject *self, PyObject *args) {
   Py_ssize_t argc;
-  PyObject *argv[3] = {
+  PyObject *argv[4] = {
     0
   };
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setTopic", 0, 2, argv))) SWIG_fail;
+  if (!(argc = SWIG_Python_UnpackTuple(args, "FrontConfigBuilder_build", 0, 3, argv))) SWIG_fail;
   --argc;
-  if (argc == 2) {
+  if (argc == 1) {
     int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+    void *vptr = 0;
+    int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0);
     _v = SWIG_CheckState(res);
     if (_v) {
-      void *vptr = 0;
-      int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_std__string, SWIG_POINTER_NO_NULL);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageOptionalHeader_setTopic__SWIG_0(self, argc, argv);
-      }
+      return _wrap_FrontConfigBuilder_build__SWIG_0(self, argc, argv);
     }
   }
-  if (argc == 2) {
+  if (argc == 3) {
     int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+    void *vptr = 0;
+    int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_ppc__front__FrontConfigBuilder, 0);
     _v = SWIG_CheckState(res);
     if (_v) {
-      int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0));
-      _v = SWIG_CheckState(res);
+      {
+        int res = SWIG_AsVal_int(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
       if (_v) {
-        return _wrap_MessageOptionalHeader_setTopic__SWIG_1(self, argc, argv);
+        int res = SWIG_AsPtr_std_string(argv[2], (std::string**)(0));
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          return _wrap_FrontConfigBuilder_build__SWIG_1(self, argc, argv);
+        }
       }
     }
   }
   
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeader_setTopic'.\n"
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'FrontConfigBuilder_build'.\n"
     "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessageOptionalHeader::setTopic(std::string &&)\n"
-    "    ppc::protocol::MessageOptionalHeader::setTopic(std::string const &)\n");
+    "    ppc::front::FrontConfigBuilder::build() const\n"
+    "    ppc::front::FrontConfigBuilder::build(int,std::string) const\n");
   return 0;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *FrontConfigBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__FrontConfigBuilder, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_printFrontDesc(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  void *argp1 = 0 ;
+  ppc::front::FrontConfig::Ptr *arg1 = 0 ;
+  void *argp1 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
+  ppc::front::FrontConfig::Ptr tempshared1 ;
   PyObject *swig_obj[1] ;
   std::string result;
   
@@ -17424,20 +18013,19 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObjec
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printFrontDesc" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
+      if (argp1) tempshared1 = *reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
+      delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
+      arg1 = &tempshared1;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
+      arg1 = (argp1) ? reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+  result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -17445,145 +18033,108 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObjec
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcInst(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
-  std::string *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcInst", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
-  }
-  (arg1)->setSrcInst((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *MessageOptionalHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessageHeader(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::string *arg1 = 0 ;
+  uint16_t arg2 ;
+  int res1 = SWIG_OLDOBJ ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    std::string *ptr = (std::string *)0;
+    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeader" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); 
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_EndPoint" "', argument " "1"" of type '" "std::string const &""'"); 
     }
+    arg1 = ptr;
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_EndPoint" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_encode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_EndPoint(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_EndPoint", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 0) {
+    return _wrap_new_EndPoint__SWIG_0(self, argc, argv);
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_AsPtr_std_string(argv[0], (std::string**)(0));
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_unsigned_SS_short(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_new_EndPoint__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_EndPoint'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::EndPoint::EndPoint()\n"
+    "    ppc::protocol::EndPoint::EndPoint(std::string const &,uint16_t)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_EndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  bcos::bytes *arg2 = 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes > tempshared2 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_encode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
-    } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_EndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
   }
-  ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  delete arg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17591,208 +18142,79 @@ SWIGINTERN PyObject *_wrap_MessageHeader_encode(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_decode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_host(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  bcos::bytesConstRef arg2 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  int64_t result;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_decode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
-    } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_host" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
-  result = (int64_t)(arg1)->decode(arg2);
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_version(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_port(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint8_t result;
+  uint16_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_version" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_port" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
-  result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
-  resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setVersion(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_setHost(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  std::string arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setVersion", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setVersion" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setVersion(arg2);
-  resultobj = SWIG_Py_Void();
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessageHeader_traceID(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_traceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessageHeader_setTraceID(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  std::string arg2 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTraceID", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
+  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setHost", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setHost" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
   {
     std::string *ptr = (std::string *)0;
     int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageHeader_setTraceID" "', argument " "2"" of type '" "std::string""'"); 
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "EndPoint_setHost" "', argument " "2"" of type '" "std::string""'"); 
     }
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setTraceID(arg2);
+  (arg1)->setHost(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17800,82 +18222,64 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setTraceID(PyObject *self, PyObject *ar
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_srcGwNode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_setPort(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setPort", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setPort" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
   }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "EndPoint_setPort" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setPort(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setSrcGwNode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_setListenIp(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
   int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setSrcGwNode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setSrcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!SWIG_Python_UnpackTuple(args, "EndPoint_setListenIp", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_setListenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
   {
     std::string *ptr = (std::string *)0;
     res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "EndPoint_setListenIp" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     arg2 = ptr;
   }
-  (arg1)->setSrcGwNode((std::string const &)*arg2);
+  (arg1)->setListenIp((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -17885,240 +18289,261 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setSrcGwNode(PyObject *self, PyObject *
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_dstGwNode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_entryPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_entryPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setDstGwNode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_listenEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  std::string result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setDstGwNode", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setDstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
-  (arg1)->setDstGwNode((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_packetType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_EndPoint_listenIp(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::EndPoint *arg1 = (ppc::protocol::EndPoint *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_packetType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setPacketType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *EndPoint_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *EndPoint_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  uint16_t arg2 ;
-  void *argp1 = 0 ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::protocol::EndPoint arg1 ;
+  bool arg2 ;
+  void *argp1 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  unsigned short val2 ;
+  bool val2 ;
   int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setPacketType", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setPacketType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); 
+    }  
+    if (!argp1) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'");
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+      arg1 = *temp;
+      if (SWIG_IsNewObj(res1)) delete temp;
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'");
   } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPacketType(arg2);
-  resultobj = SWIG_Py_Void();
+  arg2 = static_cast< bool >(val2);
+  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_ttl(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_GrpcServerConfig", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 0) {
+    return _wrap_new_GrpcServerConfig__SWIG_0(self, argc, argv);
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NO_NULL | 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_bool(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_new_GrpcServerConfig__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_GrpcServerConfig'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig()\n"
+    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig(ppc::protocol::EndPoint,bool)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  int16_t result;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::EndPoint arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEndPoint", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ttl" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'");
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
-  resultobj = SWIG_From_short(static_cast< short >(result));
+  (arg1)->setEndPoint(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setTTL(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  unsigned short val2 ;
+  bool val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTTL", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTTL" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setTTL" "', argument " "2"" of type '" "uint16_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
   } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setTTL(arg2);
+  arg2 = static_cast< bool >(val2);
+  (arg1)->setEnableHealthCheck(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18126,427 +18551,362 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setTTL(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_ext(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ext" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setExt(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setExt", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setExt" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setExt" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setExt(arg2);
-  resultobj = SWIG_Py_Void();
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_optionalField(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::MessageOptionalHeader::Ptr result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_optionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
-  {
-    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setOptionalField(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setOptionalField", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *GrpcServerConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *GrpcServerConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail;
+  result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setOptionalField" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'");
-    }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
+    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
   }
-  (arg1)->setOptionalField(arg2);
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_length(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_length" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_isRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_loadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setLoadBalancePolicy", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_srcP2PNodeIDView(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string_view result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
-  resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_MessageHeader_dstP2PNodeIDView(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string_view result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
-  resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
+  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_routeType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  uint16_t result;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  bool val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_routeType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+  } 
+  arg2 = static_cast< bool >(val2);
+  (arg1)->setEnableHealthCheck(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_setRouteType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
-  ppc::protocol::RouteType arg2 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  bool val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setRouteType", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableDnslookup", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRouteType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'");
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRouteType(arg2);
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'");
+  } 
+  arg2 = static_cast< bool >(val2);
+  (arg1)->setEnableDnslookup(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18554,13 +18914,13 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setRouteType(PyObject *self, PyObject *
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   bool result;
   
@@ -18569,20 +18929,20 @@ SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObje
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_hasOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -18590,20 +18950,24 @@ SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObje
 }
 
 
-SWIGINTERN PyObject *MessageHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
@@ -18611,17 +18975,17 @@ SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) {
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Message" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeader" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   (void)arg1; delete smartarg1;
@@ -18632,162 +18996,193 @@ SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_header(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_encode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  bcos::bytes *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::MessageHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  std::shared_ptr< bcos::bytes > tempshared2 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_encode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_header" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->header();
   {
-    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
+    }
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
+    } else {
+      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
+    }
   }
+  ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setHeader(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_decode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  ppc::protocol::MessageHeader::Ptr arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  uint64_t arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setHeader", 2, 2, swig_obj)) SWIG_fail;
+  unsigned long long val3 ;
+  int ecode3 = 0 ;
+  PyObject *swig_obj[3] ;
+  int64_t result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_decode", 3, 3, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setHeader" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t,  0 , &newmem);
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setHeader" "', argument " "2"" of type '" "ppc::protocol::MessageHeader::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2);
   }
-  (arg1)->setHeader(arg2);
-  resultobj = SWIG_Py_Void();
+  ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "MessageOptionalHeader_decode" "', argument " "3"" of type '" "uint64_t""'");
+  } 
+  arg3 = static_cast< uint64_t >(val3);
+  result = (int64_t)(arg1)->decode(arg2,arg3);
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_version(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_componentType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_version" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_componentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setVersion(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setComponentType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  std::string arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setVersion", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setComponentType", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setVersion" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setVersion" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setVersion(arg2);
+  {
+    std::string *ptr = (std::string *)0;
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageOptionalHeader_setComponentType" "', argument " "2"" of type '" "std::string""'"); 
+    }
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  (arg1)->setComponentType(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18795,77 +19190,106 @@ SWIGINTERN PyObject *_wrap_Message_setVersion(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_packetType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  OutputBuffer result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_packetType" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+  {
+    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setPacketType(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  char *arg2 = (char *) 0 ;
+  uint64_t arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setPacketType", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcNodeBuffer", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPacketType" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPacketType(arg2);
+  {
+    int res; Py_ssize_t size = 0; const void *buf = 0;
+#ifndef Py_LIMITED_API
+    Py_buffer view;
+    res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO);
+#else
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated"
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size);
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#endif
+    if (res < 0) {
+      PyErr_Clear();
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setSrcNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
+    }
+#ifndef Py_LIMITED_API
+    size = view.len;
+    buf = view.buf;
+    PyBuffer_Release(&view);
+#endif
+    arg2 = (char *) buf;
+    arg3 = (uint64_t) (size / sizeof(char));
+  }
+  (arg1)->setSrcNodeBuffer(arg2,arg3);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18873,79 +19297,106 @@ SWIGINTERN PyObject *_wrap_Message_setPacketType(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_seq(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  OutputBuffer result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_seq" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
+  {
+    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setSeq(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNodeBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::string arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  char *arg2 = (char *) 0 ;
+  uint64_t arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setSeq", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstNodeBuffer", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setSeq" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "Message_setSeq" "', argument " "2"" of type '" "std::string""'"); 
+    int res; Py_ssize_t size = 0; const void *buf = 0;
+#ifndef Py_LIMITED_API
+    Py_buffer view;
+    res = PyObject_GetBuffer(swig_obj[1], &view, PyBUF_CONTIG_RO);
+#else
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated"
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    res = PyObject_AsReadBuffer(swig_obj[1], &buf, &size);
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#endif
+    if (res < 0) {
+      PyErr_Clear();
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "MessageOptionalHeader_setDstNodeBuffer" "', argument " "2"" of type '" "(char* data, uint64_t length)""'");
     }
-    arg2 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
+#ifndef Py_LIMITED_API
+    size = view.len;
+    buf = view.buf;
+    PyBuffer_Release(&view);
+#endif
+    arg2 = (char *) buf;
+    arg3 = (uint64_t) (size / sizeof(char));
   }
-  (arg1)->setSeq(arg2);
+  (arg1)->setDstNodeBuffer(arg2,arg3);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18953,148 +19404,170 @@ SWIGINTERN PyObject *_wrap_Message_setSeq(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_ext(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstInst(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint16_t result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_ext" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_dstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
-  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setExt(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstInst(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  uint16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setExt", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setDstInst", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setExt" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setExt" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setExt(arg2);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setDstInst" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setDstInst((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_isRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_topic(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_topic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  std::unique_ptr< std::string > rvrdeleter2 ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__string, SWIG_POINTER_RELEASE |  0 );
+  if (!SWIG_IsOK(res2)) {
+    if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::string &&""'");
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); 
+    }
+  }
+  if (!argp2) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string &&""'"); 
+  }
+  arg2 = reinterpret_cast< std::string * >(argp2);
+  rvrdeleter2.reset(arg2);
+  (arg1)->setTopic((std::string &&)*arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19102,214 +19575,262 @@ SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  std::shared_ptr< bcos::bytes > arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setPayload", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setTopic" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setPayload" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes >""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
-  (arg1)->setPayload(arg2);
+  (arg1)->setTopic((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setTopic", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      void *vptr = 0;
+      int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_std__string, SWIG_POINTER_NO_NULL);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageOptionalHeader_setTopic__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_AsPtr_std_string(argv[1], (std::string**)(0));
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageOptionalHeader_setTopic__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeader_setTopic'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::MessageOptionalHeader::setTopic(std::string &&)\n"
+    "    ppc::protocol::MessageOptionalHeader::setTopic(std::string const &)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  OutputBuffer result;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payloadBuffer" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_srcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
-  {
-    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
-  }
+  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcInst(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  ppc::protocol::MessagePayload::Ptr arg2 ;
+  ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setFrontMessage", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageOptionalHeader_setSrcInst", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setFrontMessage" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t,  0 , &newmem);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setFrontMessage" "', argument " "2"" of type '" "ppc::protocol::MessagePayload::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2);
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageOptionalHeader_setSrcInst" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
-  (arg1)->setFrontMessage(arg2);
+  (arg1)->setSrcInst((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *MessageOptionalHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_MessageHeader(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::MessagePayload::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_frontMessage" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeader" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
-  {
-    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
-  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_encode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   bcos::bytes *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   void *argp2 = 0 ;
   int res2 = 0 ;
   std::shared_ptr< bcos::bytes > tempshared2 ;
   PyObject *swig_obj[2] ;
-  bool result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_encode", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_encode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_encode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_encode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
     int newmem = 0;
     res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
     }
     if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
@@ -19319,51 +19840,51 @@ SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
       arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  result = (bool)(arg1)->encode(*arg2);
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_decode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   bcos::bytesConstRef arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
   PyObject *swig_obj[2] ;
   int64_t result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_decode", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_decode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_decode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_decode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
     res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
     }  
     if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
     } else {
       bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
       arg2 = *temp;
@@ -19378,686 +19899,597 @@ SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_version(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint32_t result;
+  uint8_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_length" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_version" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
-  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
+  result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
+  resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setVersion(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::shared_ptr< bcos::bytes > result;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setVersion", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payload" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->payload();
-  {
-    std::shared_ptr<  bcos::bytes > *smartresult = result ? new std::shared_ptr<  bcos::bytes >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
-  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setVersion" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setVersion(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *Message_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessageHeaderBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_traceID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_traceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setTraceID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
-  bcos::bytesConstRef arg2 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  std::string arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  ppc::protocol::MessageHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTraceID", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
-    } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+    std::string *ptr = (std::string *)0;
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "MessageHeader_setTraceID" "', argument " "2"" of type '" "std::string""'"); 
     }
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
   }
-  result = (arg1)->build(arg2);
-  {
-    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
-  }
+  (arg1)->setTraceID(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_srcGwNode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
-  ppc::protocol::MessageHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
   (void)self;
-  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
-  {
-    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
-  }
+  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setSrcGwNode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setSrcGwNode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setSrcGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
-      arg2 = &tempshared2;
-    } else {
-      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setSrcGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
     }
+    arg2 = ptr;
   }
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
-  {
-    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
-  }
+  (arg1)->setSrcGwNode((std::string const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
+SWIGINTERN PyObject *_wrap_MessageHeader_dstGwNode(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageHeaderBuilder_build", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 1) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      return _wrap_MessageHeaderBuilder_build__SWIG_1(self, argc, argv);
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageHeaderBuilder_build__SWIG_0(self, argc, argv);
-      }
+  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageHeader_setDstGwNode(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setDstGwNode", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setDstGwNode" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageHeaderBuilder_build__SWIG_2(self, argc, argv);
-      }
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setDstGwNode" "', argument " "2"" of type '" "std::string const &""'"); 
     }
+    arg2 = ptr;
   }
-  
+  (arg1)->setDstGwNode((std::string const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageHeaderBuilder_build'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessageHeaderBuilder::build(bcos::bytesConstRef)\n"
-    "    ppc::protocol::MessageHeaderBuilder::build()\n"
-    "    ppc::protocol::MessageHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n");
-  return 0;
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
 }
 
 
-SWIGINTERN PyObject *MessageHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessageBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_packetType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  uint16_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_packetType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setPacketType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
-  ppc::protocol::Message::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setPacketType", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setPacketType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
-  {
-    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
-  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setPacketType(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_ttl(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
-  bcos::bytesConstRef arg2 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  ppc::protocol::Message::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  int16_t result;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ttl" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build(arg2);
-  {
-    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
-  }
+  result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
+  resultobj = SWIG_From_short(static_cast< short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setTTL(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
-  ppc::protocol::RouteType arg2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
-  bcos::bytes *arg4 = 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
-  void *argp4 = 0 ;
-  int res4 = 0 ;
-  std::unique_ptr< bcos::bytes > rvrdeleter4 ;
-  ppc::protocol::Message::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setTTL", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setTTL" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'");
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "MessageBuilder_build" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      arg3 = &tempshared3;
-    } else {
-      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
-    }
-  }
-  res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE |  0 );
-  if (!SWIG_IsOK(res4)) {
-    if (res4 == SWIG_ERROR_RELEASE_NOT_OWNED) {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', cannot release ownership as memory is not owned for argument " "4"" of type '" "bcos::bytes &&""'");
-    } else {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
-    }
-  }
-  if (!argp4) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
-  }
-  arg4 = reinterpret_cast< bcos::bytes * >(argp4);
-  rvrdeleter4.reset(arg4);
-  result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
-  {
-    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
-  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setTTL" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setTTL(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageBuilder_build(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[5] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageBuilder_build", 0, 4, argv))) SWIG_fail;
-  --argc;
-  if (argc == 1) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      return _wrap_MessageBuilder_build__SWIG_0(self, argc, argv);
-    }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageBuilder_build__SWIG_1(self, argc, argv);
-      }
-    }
-  }
-  if (argc == 4) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_ppc__protocol__RouteType, SWIG_POINTER_NO_NULL | 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
-        _v = SWIG_CheckState(res);
-        if (_v) {
-          void *vptr = 0;
-          int res = SWIG_ConvertPtr(argv[3], &vptr, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NO_NULL);
-          _v = SWIG_CheckState(res);
-          if (_v) {
-            return _wrap_MessageBuilder_build__SWIG_2(self, argc, argv);
-          }
-        }
-      }
-    }
-  }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageBuilder_build'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessageBuilder::build()\n"
-    "    ppc::protocol::MessageBuilder::build(bcos::bytesConstRef)\n"
-    "    ppc::protocol::MessageBuilder::build(ppc::protocol::RouteType,ppc::protocol::MessageOptionalHeader::Ptr const &,bcos::bytes &&)\n");
-  return 0;
-}
-
-
-SWIGINTERN PyObject *MessageBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeaderBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_ext(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  uint16_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_ext" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setExt(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setExt", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setExt" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setExt" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setExt(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageHeader_optionalField(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::MessageOptionalHeader::Ptr result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_optionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
-      arg2 = &tempshared2;
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -20068,363 +20500,388 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_0(PyObject *
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setOptionalField(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr result;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setOptionalField", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
   {
-    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setOptionalField" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
   }
+  (arg1)->setOptionalField(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeaderBuilder_build", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 1) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      return _wrap_MessageOptionalHeaderBuilder_build__SWIG_1(self, argc, argv);
+SWIGINTERN PyObject *_wrap_MessageHeader_length(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint16_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_length" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessageOptionalHeaderBuilder_build__SWIG_0(self, argc, argv);
-      }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  
+  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  return resultobj;
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeaderBuilder_build'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessageOptionalHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n"
-    "    ppc::protocol::MessageOptionalHeaderBuilder::build()\n");
-  return 0;
+  return NULL;
 }
 
 
-SWIGINTERN PyObject *MessageOptionalHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_printOptionalField(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_isRespPacket(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessageOptionalHeader::Ptr arg1 ;
-  void *argp1 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
-    if (argp1) arg1 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1);
   }
-  result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_printMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setRespPacket(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::Message::Ptr *arg1 = 0 ;
-  void *argp1 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
-  ppc::protocol::Message::Ptr tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printMessage" "', argument " "1"" of type '" "ppc::protocol::Message::Ptr const &""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp1) tempshared1 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1);
-      delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1);
-      arg1 = &tempshared1;
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      arg1 = (argp1) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1) : &tempshared1;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  (arg1)->setRespPacket();
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_MessagePayload(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_srcP2PNodeIDView(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  std::string_view result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayload" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_srcP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
+  resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_encode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_dstP2PNodeIDView(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  bcos::bytes *arg2 = 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
-  void *argp2 = 0 ;
-  int res2 = 0 ;
-  std::shared_ptr< bcos::bytes > tempshared2 ;
-  PyObject *swig_obj[2] ;
-  int64_t result;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string_view result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_encode", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_encode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_dstP2PNodeIDView" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
+  resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageHeader_routeType(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint16_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
-    }
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_routeType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
-      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_decode(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_setRouteType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  bcos::bytesConstRef arg2 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
+  ppc::protocol::RouteType arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
   PyObject *swig_obj[2] ;
-  int64_t result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_decode", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "MessageHeader_setRouteType", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_decode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_setRouteType" "', argument " "1"" of type '" "ppc::protocol::MessageHeader *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType,  0  | 0);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); 
     }  
     if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeader_setRouteType" "', argument " "2"" of type '" "ppc::protocol::RouteType""'");
     } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2);
       arg2 = *temp;
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (int64_t)(arg1)->decode(arg2);
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  (arg1)->setRouteType(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_version(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::MessageHeader *arg1 = (ppc::protocol::MessageHeader *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeader const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  uint8_t result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_version" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeader_hasOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageHeader const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessageHeader > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
-  resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
+  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *MessageHeader_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  uint8_t arg2 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
-  unsigned char val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setVersion", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Message" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setVersion" "', argument " "2"" of type '" "uint8_t""'");
-  } 
-  arg2 = static_cast< uint8_t >(val2);
-  (arg1)->setVersion(arg2);
+  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20432,37 +20889,38 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_header(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  OutputBuffer result;
+  ppc::protocol::MessageHeader::Ptr result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_dataBuffer" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_header" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+  result = ((ppc::protocol::Message const *)arg1)->header();
   {
-    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -20470,13 +20928,59 @@ SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setHeader(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessageHeader::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setHeader", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setHeader" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setHeader" "', argument " "2"" of type '" "ppc::protocol::MessageHeader::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2);
+  }
+  (arg1)->setHeader(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_Message_version(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   uint16_t result;
   
@@ -20485,20 +20989,20 @@ SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_seq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_version" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+  result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -20506,41 +21010,41 @@ SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setSeq(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setVersion(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
   unsigned short val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setSeq", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setVersion", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setSeq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setVersion" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setSeq" "', argument " "2"" of type '" "uint16_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setVersion" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setSeq(arg2);
+  (arg1)->setVersion(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20548,134 +21052,171 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setSeq(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_length(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_packetType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  int64_t result;
+  uint16_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_length" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_packetType" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
-  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_traceID(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setPacketType(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setPacketType", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_traceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPacketType" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
-  return resultobj;
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setPacketType(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setTraceID(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_seq(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_seq" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_Message_setSeq(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  std::string arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setTraceID", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setSeq", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setSeq" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
     std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); 
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "Message_setSeq" "', argument " "2"" of type '" "std::string""'"); 
     }
-    arg2 = ptr;
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setTraceID((std::string const &)*arg2);
+  (arg1)->setSeq(arg2);
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_ext(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   uint16_t result;
   
@@ -20684,20 +21225,20 @@ SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) {
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_ext" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_ext" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+  result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -20705,38 +21246,38 @@ SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setExt(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setExt(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   uint16_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
   unsigned short val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setExt", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setExt", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setExt" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setExt" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setExt" "', argument " "2"" of type '" "uint16_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setExt" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
   (arg1)->setExt(arg2);
@@ -20747,112 +21288,116 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setExt(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_setRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_isRespPacket(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
-  resultobj = SWIG_Py_Void();
+  result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayload_isRespPacket(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)(arg1)->isRespPacket();
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  (arg1)->setRespPacket();
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *MessagePayload_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_MessagePayloadBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  std::shared_ptr< bcos::bytes > arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setPayload", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayloadBuilder" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setPayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setPayload" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes >""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
+  }
+  (arg1)->setPayload(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20860,36 +21405,37 @@ SWIGINTERN PyObject *_wrap_delete_MessagePayloadBuilder(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
-  ppc::protocol::MessagePayload::Ptr result;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  OutputBuffer result;
   
   (void)self;
-  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payloadBuffer" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
+  result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
   {
-    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
+    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
   }
   return resultobj;
 fail:
@@ -20897,342 +21443,287 @@ SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_0(PyObject *self, P
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
-  bcos::bytesConstRef arg2 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  ppc::protocol::MessagePayload::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
-  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
-  ppc::protocol::MessagePayload::Ptr result;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_setFrontMessage", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setFrontMessage" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
-      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t,  0 , &newmem);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
-    } else {
-      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setFrontMessage" "', argument " "2"" of type '" "ppc::protocol::MessagePayload::Ptr""'");
     }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2);
   }
-  result = (arg1)->build(arg2);
-  {
-    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
-  }
+  (arg1)->setFrontMessage(arg2);
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
-  
-  if (!(argc = SWIG_Python_UnpackTuple(args, "MessagePayloadBuilder_build", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 1) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      return _wrap_MessagePayloadBuilder_build__SWIG_0(self, argc, argv);
-    }
-  }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
-      _v = SWIG_CheckState(res);
-      if (_v) {
-        return _wrap_MessagePayloadBuilder_build__SWIG_1(self, argc, argv);
-      }
-    }
-  }
-  
-fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessagePayloadBuilder_build'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::MessagePayloadBuilder::build()\n"
-    "    ppc::protocol::MessagePayloadBuilder::build(bcos::bytesConstRef)\n");
-  return 0;
-}
-
-
-SWIGINTERN PyObject *MessagePayloadBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_IFrontClient(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFrontClient > tempshared1 ;
-  std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  ppc::protocol::MessagePayload::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontClient" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_frontMessage" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
+  {
+    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFrontClient_onReceiveMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ;
-  ppc::protocol::Message::Ptr *arg2 = 0 ;
-  SwigValueWrapper< std::function< void (std::shared_ptr< bcos::Error >) > > arg3 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  bcos::bytes *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFrontClient > tempshared1 ;
-  std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ;
-  void *argp2 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  void *argp2 = 0 ;
   int res2 = 0 ;
-  ppc::protocol::Message::Ptr tempshared2 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  PyObject *swig_obj[3] ;
+  std::shared_ptr< bcos::bytes > tempshared2 ;
+  PyObject *swig_obj[2] ;
+  bool result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFrontClient_onReceiveMessage", 3, 3, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_encode", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontClient_onReceiveMessage" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_encode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
-      arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFrontClient_onReceiveMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr const &""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
-      delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
-      arg2 = &tempshared2;
-    } else {
-      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2) : &tempshared2;
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
     }
-  }
-  {
-    res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t,  0  | 0);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'"); 
-    }  
-    if (!argp3) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'");
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
     } else {
-      ppc::protocol::ReceiveMsgFunc * temp = reinterpret_cast< ppc::protocol::ReceiveMsgFunc * >(argp3);
-      arg3 = *temp;
-      if (SWIG_IsNewObj(res3)) delete temp;
+      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
-  resultobj = SWIG_Py_Void();
+  result = (bool)(arg1)->encode(*arg2);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *IFrontClient_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_new_ErrorCallback(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  PyObject *arg1 = (PyObject *) 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::front::ErrorCallback *result = 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  int64_t result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "Message_decode", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_decode" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  
   {
-    std::shared_ptr<  ppc::front::ErrorCallback > *smartresult = result ? new std::shared_ptr<  ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Message_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
   }
+  result = (int64_t)(arg1)->decode(arg2);
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_ErrorCallback(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
-  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  uint32_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_length" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_ErrorCallback_onError(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
-  bcos::Error::Ptr arg2 ;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
-  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  Swig::Director *director = 0;
-  bool upcall = false;
+  std::shared_ptr< ppc::protocol::Message const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::shared_ptr< bcos::bytes > result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "ErrorCallback_onError", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ErrorCallback_onError" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_payload" "', argument " "1"" of type '" "ppc::protocol::Message const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  result = ((ppc::protocol::Message const *)arg1)->payload();
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "ErrorCallback_onError" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
-    }
-    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
-  }
-  director = SWIG_DIRECTOR_CAST(arg1);
-  upcall = (director && (director->swig_get_self()==swig_obj[0]));
-  try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::ErrorCallback::onError");
-    } else {
-      (arg1)->onError(arg2);
-    }
-  } catch (Swig::DirectorException&) {
-    SWIG_fail;
+    std::shared_ptr<  bcos::bytes > *smartresult = result ? new std::shared_ptr<  bcos::bytes >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
   }
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_disown_ErrorCallback(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *Message_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_MessageHeaderBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
+  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
-  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
@@ -21240,24 +21731,20 @@ SWIGINTERN PyObject *_wrap_disown_ErrorCallback(PyObject *self, PyObject *args)
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
-      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  {
-    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
-    if (director) director->swig_disown();
-  }
-  
+  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21265,38 +21752,52 @@ SWIGINTERN PyObject *_wrap_disown_ErrorCallback(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *ErrorCallback_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *ErrorCallback_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_new_MessageDispatcherHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  PyObject *arg1 = (PyObject *) 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::front::MessageDispatcherHandler *result = 0 ;
+  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  ppc::protocol::MessageHeader::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  
   {
-    std::shared_ptr<  ppc::front::MessageDispatcherHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
+  }
+  result = (arg1)->build(arg2);
+  {
+    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -21304,410 +21805,443 @@ SWIGINTERN PyObject *_wrap_new_MessageDispatcherHandler(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_MessageDispatcherHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
+  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
+  ppc::protocol::MessageHeader::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = (arg1)->build();
+  {
+    std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_MessageDispatcherHandler_onMessage(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
-  ppc::protocol::Message::Ptr arg2 ;
+  ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  Swig::Director *director = 0;
-  bool upcall = false;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "MessageDispatcherHandler_onMessage", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageHeaderBuilder *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
-  }
-  director = SWIG_DIRECTOR_CAST(arg1);
-  upcall = (director && (director->swig_get_self()==swig_obj[0]));
-  try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::MessageDispatcherHandler::onMessage");
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
+      arg2 = &tempshared2;
     } else {
-      (arg1)->onMessage(arg2);
+      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
     }
-  } catch (Swig::DirectorException&) {
-    SWIG_fail;
   }
-  resultobj = SWIG_Py_Void();
+  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_disown_MessageDispatcherHandler(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
   
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageHeaderBuilder_build", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 1) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_MessageHeaderBuilder_build__SWIG_1(self, argc, argv);
     }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageHeaderBuilder_build__SWIG_0(self, argc, argv);
+      }
     }
   }
-  {
-    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
-    if (director) director->swig_disown();
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageHeaderBuilder_build__SWIG_2(self, argc, argv);
+      }
+    }
   }
   
-  resultobj = SWIG_Py_Void();
-  return resultobj;
 fail:
-  return NULL;
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageHeaderBuilder_build'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::MessageHeaderBuilder::build(bcos::bytesConstRef)\n"
+    "    ppc::protocol::MessageHeaderBuilder::build()\n"
+    "    ppc::protocol::MessageHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n");
+  return 0;
 }
 
 
-SWIGINTERN PyObject *MessageDispatcherHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *MessageHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *MessageDispatcherHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_new_SendResponseHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_MessageBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  SwigValueWrapper< std::function< void (std::shared_ptr< bcos::bytes > &&) > > arg1 ;
-  void *argp1 ;
+  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::front::SendResponseHandler *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
-    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t,  0  | 0);
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'"); 
-    }  
-    if (!argp1) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
     } else {
-      ppc::protocol::SendResponseFunction * temp = reinterpret_cast< ppc::protocol::SendResponseFunction * >(argp1);
-      arg1 = *temp;
-      if (SWIG_IsNewObj(res1)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_NEW |  0 );
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_SendResponseHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ;
+  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
+  ppc::protocol::Message::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SendResponseHandler" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); 
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (arg1)->build();
+  {
+    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
   }
-  arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1);
-  delete arg1;
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_SendResponseHandler_sendResponse(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ;
-  std::shared_ptr< bcos::bytes > *arg2 = 0 ;
+  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
+  bcos::bytesConstRef arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  void *argp2 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
+  void *argp2 ;
   int res2 = 0 ;
-  std::unique_ptr< std::shared_ptr< bcos::bytes > > rvrdeleter2 ;
-  PyObject *swig_obj[2] ;
+  ppc::protocol::Message::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "SendResponseHandler_sendResponse", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SendResponseHandler_sendResponse" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); 
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1);
-  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_RELEASE |  0 );
-  if (!SWIG_IsOK(res2)) {
-    if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'");
+  {
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
     } else {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); 
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  if (!argp2) {
-    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); 
+  result = (arg1)->build(arg2);
+  {
+    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
   }
-  arg2 = reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
-  rvrdeleter2.reset(arg2);
-  (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *SendResponseHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *SendResponseHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_new_IMessageHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  PyObject *arg1 = (PyObject *) 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::front::IMessageHandler *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
-  }
-  
-  {
-    std::shared_ptr<  ppc::front::IMessageHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
-  }
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_delete_IMessageHandler(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_IMessageHandler_onMessage(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
-  bcos::Error::Ptr arg2 ;
-  ppc::protocol::Message::Ptr arg3 ;
-  SwigValueWrapper< ppc::front::SendResponseHandler > arg4 ;
+  ppc::protocol::MessageBuilder *arg1 = (ppc::protocol::MessageBuilder *) 0 ;
+  ppc::protocol::RouteType arg2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  bcos::bytes *arg4 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageBuilder > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
   void *argp3 ;
   int res3 = 0 ;
-  void *argp4 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
+  void *argp4 = 0 ;
   int res4 = 0 ;
-  PyObject *swig_obj[4] ;
-  Swig::Director *director = 0;
-  bool upcall = false;
+  std::unique_ptr< bcos::bytes > rvrdeleter4 ;
+  ppc::protocol::Message::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IMessageHandler_onMessage", 4, 4, swig_obj)) SWIG_fail;
+  if ((nobjs < 4) || (nobjs > 4)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IMessageHandler_onMessage" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageBuilder *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__RouteType,  0  | 0);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IMessageHandler_onMessage" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "2"" of type '" "ppc::protocol::RouteType""'");
+    } else {
+      ppc::protocol::RouteType * temp = reinterpret_cast< ppc::protocol::RouteType * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
     }
-    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
   }
   {
     int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
     if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IMessageHandler_onMessage" "', argument " "3"" of type '" "ppc::protocol::Message::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "MessageBuilder_build" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
     }
-    if (argp3) arg3 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3);
-  }
-  {
-    res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_ppc__front__SendResponseHandler,  0  | 0);
-    if (!SWIG_IsOK(res4)) {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'"); 
-    }  
-    if (!argp4) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'");
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      arg3 = &tempshared3;
     } else {
-      ppc::front::SendResponseHandler * temp = reinterpret_cast< ppc::front::SendResponseHandler * >(argp4);
-      arg4 = *temp;
-      if (SWIG_IsNewObj(res4)) delete temp;
+      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
     }
   }
-  director = SWIG_DIRECTOR_CAST(arg1);
-  upcall = (director && (director->swig_get_self()==swig_obj[0]));
-  try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::IMessageHandler::onMessage");
+  res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_RELEASE |  0 );
+  if (!SWIG_IsOK(res4)) {
+    if (res4 == SWIG_ERROR_RELEASE_NOT_OWNED) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', cannot release ownership as memory is not owned for argument " "4"" of type '" "bcos::bytes &&""'");
     } else {
-      (arg1)->onMessage(arg2,arg3,arg4);
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
     }
-  } catch (Swig::DirectorException&) {
-    SWIG_fail;
   }
-  resultobj = SWIG_Py_Void();
+  if (!argp4) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessageBuilder_build" "', argument " "4"" of type '" "bcos::bytes &&""'"); 
+  }
+  arg4 = reinterpret_cast< bcos::bytes * >(argp4);
+  rvrdeleter4.reset(arg4);
+  result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
+  {
+    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_disown_IMessageHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_MessageBuilder_build(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[5] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageBuilder_build", 0, 4, argv))) SWIG_fail;
+  --argc;
+  if (argc == 1) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_MessageBuilder_build__SWIG_0(self, argc, argv);
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageBuilder_build__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  if (argc == 4) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_ppc__protocol__RouteType, SWIG_POINTER_NO_NULL | 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        int res = SWIG_ConvertPtr(argv[2], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+        _v = SWIG_CheckState(res);
+        if (_v) {
+          void *vptr = 0;
+          int res = SWIG_ConvertPtr(argv[3], &vptr, SWIGTYPE_p_std__vectorT_uint8_t_t, SWIG_POINTER_NO_NULL);
+          _v = SWIG_CheckState(res);
+          if (_v) {
+            return _wrap_MessageBuilder_build__SWIG_2(self, argc, argv);
+          }
+        }
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageBuilder_build'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::MessageBuilder::build()\n"
+    "    ppc::protocol::MessageBuilder::build(bcos::bytesConstRef)\n"
+    "    ppc::protocol::MessageBuilder::build(ppc::protocol::RouteType,ppc::protocol::MessageOptionalHeader::Ptr const &,bcos::bytes &&)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *MessageBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeaderBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
+  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
@@ -21715,24 +22249,20 @@ SWIGINTERN PyObject *_wrap_disown_IMessageHandler(PyObject *self, PyObject *args
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageOptionalHeaderBuilder" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  {
-    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
-    if (director) director->swig_disown();
-  }
-  
+  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21740,52 +22270,212 @@ SWIGINTERN PyObject *_wrap_disown_IMessageHandler(PyObject *self, PyObject *args
 }
 
 
-SWIGINTERN PyObject *IMessageHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *IMessageHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
+SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr result;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "2"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
+      arg2 = &tempshared2;
+    } else {
+      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
+    }
+  }
+  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
 }
 
-SWIGINTERN PyObject *_wrap_new_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+
+SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
   PyObject *resultobj = 0;
-  PyObject *arg1 = (PyObject *) 0 ;
+  ppc::protocol::MessageOptionalHeaderBuilder *arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *smartarg1 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr result;
+  
+  (void)self;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageOptionalHeaderBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeaderBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (arg1)->build();
+  {
+    std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "MessageOptionalHeaderBuilder_build", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 1) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_MessageOptionalHeaderBuilder_build__SWIG_1(self, argc, argv);
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessageOptionalHeaderBuilder_build__SWIG_0(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessageOptionalHeaderBuilder_build'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::MessageOptionalHeaderBuilder::build(ppc::protocol::MessageOptionalHeader::Ptr const &)\n"
+    "    ppc::protocol::MessageOptionalHeaderBuilder::build()\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *MessageOptionalHeaderBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_printOptionalField(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessageOptionalHeader::Ptr arg1 ;
+  void *argp1 ;
+  int res1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::front::GetPeersInfoHandler *result = 0 ;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printOptionalField" "', argument " "1"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr""'");
+    }
+    if (argp1) arg1 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1);
   }
+  result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_printMessage(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::Message::Ptr *arg1 = 0 ;
+  void *argp1 ;
+  int res1 = 0 ;
+  ppc::protocol::Message::Ptr tempshared1 ;
+  PyObject *swig_obj[1] ;
+  std::string result;
   
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    std::shared_ptr<  ppc::front::GetPeersInfoHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printMessage" "', argument " "1"" of type '" "ppc::protocol::Message::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp1) tempshared1 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1);
+      delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1);
+      arg1 = &tempshared1;
+    } else {
+      arg1 = (argp1) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1) : &tempshared1;
+    }
   }
+  result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_MessagePayload(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
@@ -21793,167 +22483,2450 @@ SWIGINTERN PyObject *_wrap_delete_GetPeersInfoHandler(PyObject *self, PyObject *
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayload" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_encode(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  bcos::bytes *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  std::shared_ptr< bcos::bytes > tempshared2 ;
+  PyObject *swig_obj[2] ;
+  int64_t result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_encode", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_encode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'");
+    }
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_encode" "', argument " "2"" of type '" "bcos::bytes &""'"); 
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared2 = *reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      delete reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2);
+      arg2 = const_cast< bcos::bytes * >(tempshared2.get());
+    } else {
+      arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
+    }
+  }
+  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_decode(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  int64_t result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_decode", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_decode" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_decode" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
+  }
+  result = (int64_t)(arg1)->decode(arg2);
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_version(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint8_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_version" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
+  resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  uint8_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  unsigned char val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setVersion", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setVersion" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_char(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setVersion" "', argument " "2"" of type '" "uint8_t""'");
+  } 
+  arg2 = static_cast< uint8_t >(val2);
+  (arg1)->setVersion(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  OutputBuffer result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_dataBuffer" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+  {
+    resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint16_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_seq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_setSeq(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  uint16_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setSeq", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setSeq" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setSeq" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setSeq(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_length(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  int64_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_length" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
+  resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_traceID(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_traceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_setTraceID(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setTraceID", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setTraceID" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayload_setTraceID" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setTraceID((std::string const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint16_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_ext" "', argument " "1"" of type '" "ppc::protocol::MessagePayload const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+  resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_setExt(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  uint16_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessagePayload_setExt", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setExt" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setExt" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  (arg1)->setExt(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_setRespPacket(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_setRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (arg1)->setRespPacket();
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayload_isRespPacket(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  bool result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_isRespPacket" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (bool)(arg1)->isRespPacket();
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *MessagePayload_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_MessagePayloadBuilder(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessagePayloadBuilder" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
+  ppc::protocol::MessagePayload::Ptr result;
+  
+  (void)self;
+  if ((nobjs < 1) || (nobjs > 1)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (arg1)->build();
+  {
+    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
+  bcos::bytesConstRef arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  ppc::protocol::MessagePayload::Ptr result;
+  
+  (void)self;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayloadBuilder_build" "', argument " "1"" of type '" "ppc::protocol::MessagePayloadBuilder *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "MessagePayloadBuilder_build" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
+  }
+  result = (arg1)->build(arg2);
+  {
+    std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "MessagePayloadBuilder_build", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 1) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      return _wrap_MessagePayloadBuilder_build__SWIG_0(self, argc, argv);
+    }
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      int res = SWIG_ConvertPtr(argv[1], 0, SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_NO_NULL | 0);
+      _v = SWIG_CheckState(res);
+      if (_v) {
+        return _wrap_MessagePayloadBuilder_build__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'MessagePayloadBuilder_build'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::MessagePayloadBuilder::build()\n"
+    "    ppc::protocol::MessagePayloadBuilder::build(bcos::bytesConstRef)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *MessagePayloadBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_IFrontClient(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFrontClient > tempshared1 ;
+  std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontClient" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFrontClient_onReceiveMessage(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFrontClient *arg1 = (ppc::front::IFrontClient *) 0 ;
+  ppc::protocol::Message::Ptr *arg2 = 0 ;
+  SwigValueWrapper< std::function< void (std::shared_ptr< bcos::Error >) > > arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFrontClient > tempshared1 ;
+  std::shared_ptr< ppc::front::IFrontClient > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  ppc::protocol::Message::Ptr tempshared2 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  PyObject *swig_obj[3] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFrontClient_onReceiveMessage", 3, 3, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontClient_onReceiveMessage" "', argument " "1"" of type '" "ppc::front::IFrontClient *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      arg1 = const_cast< ppc::front::IFrontClient * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFrontClient > * >(argp1);
+      arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFrontClient_onReceiveMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp2) tempshared2 = *reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
+      delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
+      arg2 = &tempshared2;
+    } else {
+      arg2 = (argp2) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2) : &tempshared2;
+    }
+  }
+  {
+    res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__Error_tF_t,  0  | 0);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'"); 
+    }  
+    if (!argp3) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontClient_onReceiveMessage" "', argument " "3"" of type '" "ppc::protocol::ReceiveMsgFunc""'");
+    } else {
+      ppc::protocol::ReceiveMsgFunc * temp = reinterpret_cast< ppc::protocol::ReceiveMsgFunc * >(argp3);
+      arg3 = *temp;
+      if (SWIG_IsNewObj(res3)) delete temp;
+    }
+  }
+  (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *IFrontClient_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_new_ErrorCallback(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  PyObject *arg1 = (PyObject *) 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::ErrorCallback *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  arg1 = swig_obj[0];
+  if ( arg1 != Py_None ) {
+    /* subclassed */
+    result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(arg1); 
+  } else {
+    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+    SWIG_fail;
+  }
+  
+  {
+    std::shared_ptr<  ppc::front::ErrorCallback > *smartresult = result ? new std::shared_ptr<  ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_ErrorCallback(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
+  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_ErrorCallback_onError(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
+  bcos::Error::Ptr arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
+  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  Swig::Director *director = 0;
+  bool upcall = false;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "ErrorCallback_onError", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ErrorCallback_onError" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "ErrorCallback_onError" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
+  }
+  director = SWIG_DIRECTOR_CAST(arg1);
+  upcall = (director && (director->swig_get_self()==swig_obj[0]));
+  try {
+    if (upcall) {
+      Swig::DirectorPureVirtualException::raise("ppc::front::ErrorCallback::onError");
+    } else {
+      (arg1)->onError(arg2);
+    }
+  } catch (Swig::DirectorException&) {
+    SWIG_fail;
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_disown_ErrorCallback(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::ErrorCallback *arg1 = (ppc::front::ErrorCallback *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::ErrorCallback > tempshared1 ;
+  std::shared_ptr< ppc::front::ErrorCallback > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_ErrorCallback" "', argument " "1"" of type '" "ppc::front::ErrorCallback *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::ErrorCallback > * >(argp1);
+      arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
+    if (director) director->swig_disown();
+  }
+  
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *ErrorCallback_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *ErrorCallback_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_MessageDispatcherHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  PyObject *arg1 = (PyObject *) 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::MessageDispatcherHandler *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  arg1 = swig_obj[0];
+  if ( arg1 != Py_None ) {
+    /* subclassed */
+    result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(arg1); 
+  } else {
+    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+    SWIG_fail;
+  }
+  
+  {
+    std::shared_ptr<  ppc::front::MessageDispatcherHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_MessageDispatcherHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_MessageDispatcherHandler_onMessage(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
+  ppc::protocol::Message::Ptr arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
+  Swig::Director *director = 0;
+  bool upcall = false;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "MessageDispatcherHandler_onMessage", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "MessageDispatcherHandler_onMessage" "', argument " "2"" of type '" "ppc::protocol::Message::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp2);
+  }
+  director = SWIG_DIRECTOR_CAST(arg1);
+  upcall = (director && (director->swig_get_self()==swig_obj[0]));
+  try {
+    if (upcall) {
+      Swig::DirectorPureVirtualException::raise("ppc::front::MessageDispatcherHandler::onMessage");
+    } else {
+      (arg1)->onMessage(arg2);
+    }
+  } catch (Swig::DirectorException&) {
+    SWIG_fail;
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_disown_MessageDispatcherHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::MessageDispatcherHandler *arg1 = (ppc::front::MessageDispatcherHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::MessageDispatcherHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_MessageDispatcherHandler" "', argument " "1"" of type '" "ppc::front::MessageDispatcherHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::MessageDispatcherHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
+    if (director) director->swig_disown();
+  }
+  
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *MessageDispatcherHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *MessageDispatcherHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_SendResponseHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  SwigValueWrapper< std::function< void (std::shared_ptr< bcos::bytes > &&) > > arg1 ;
+  void *argp1 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::SendResponseHandler *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_std__functionT_void_fstd__shared_ptrT_bcos__bytes_t_RRF_t,  0  | 0);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'"); 
+    }  
+    if (!argp1) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_SendResponseHandler" "', argument " "1"" of type '" "ppc::protocol::SendResponseFunction""'");
+    } else {
+      ppc::protocol::SendResponseFunction * temp = reinterpret_cast< ppc::protocol::SendResponseFunction * >(argp1);
+      arg1 = *temp;
+      if (SWIG_IsNewObj(res1)) delete temp;
+    }
+  }
+  result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_SendResponseHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SendResponseHandler" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_SendResponseHandler_sendResponse(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::SendResponseHandler *arg1 = (ppc::front::SendResponseHandler *) 0 ;
+  std::shared_ptr< bcos::bytes > *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  std::unique_ptr< std::shared_ptr< bcos::bytes > > rvrdeleter2 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "SendResponseHandler_sendResponse", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__SendResponseHandler, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SendResponseHandler_sendResponse" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1);
+  res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_RELEASE |  0 );
+  if (!SWIG_IsOK(res2)) {
+    if (res2 == SWIG_ERROR_RELEASE_NOT_OWNED) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', cannot release ownership as memory is not owned for argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'");
+    } else {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); 
+    }
+  }
+  if (!argp2) {
+    SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SendResponseHandler_sendResponse" "', argument " "2"" of type '" "std::shared_ptr< bcos::bytes > &&""'"); 
+  }
+  arg2 = reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
+  rvrdeleter2.reset(arg2);
+  (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *SendResponseHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *SendResponseHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_IMessageHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  PyObject *arg1 = (PyObject *) 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::IMessageHandler *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  arg1 = swig_obj[0];
+  if ( arg1 != Py_None ) {
+    /* subclassed */
+    result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(arg1); 
+  } else {
+    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+    SWIG_fail;
+  }
+  
+  {
+    std::shared_ptr<  ppc::front::IMessageHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_IMessageHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IMessageHandler_onMessage(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
+  bcos::Error::Ptr arg2 ;
+  ppc::protocol::Message::Ptr arg3 ;
+  SwigValueWrapper< ppc::front::SendResponseHandler > arg4 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  void *argp4 ;
+  int res4 = 0 ;
+  PyObject *swig_obj[4] ;
+  Swig::Director *director = 0;
+  bool upcall = false;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IMessageHandler_onMessage", 4, 4, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IMessageHandler_onMessage" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IMessageHandler_onMessage" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
+  }
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IMessageHandler_onMessage" "', argument " "3"" of type '" "ppc::protocol::Message::Ptr""'");
+    }
+    if (argp3) arg3 = *(reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::Message::Ptr * >(argp3);
+  }
+  {
+    res4 = SWIG_ConvertPtr(swig_obj[3], &argp4, SWIGTYPE_p_ppc__front__SendResponseHandler,  0  | 0);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'"); 
+    }  
+    if (!argp4) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IMessageHandler_onMessage" "', argument " "4"" of type '" "ppc::front::SendResponseHandler""'");
+    } else {
+      ppc::front::SendResponseHandler * temp = reinterpret_cast< ppc::front::SendResponseHandler * >(argp4);
+      arg4 = *temp;
+      if (SWIG_IsNewObj(res4)) delete temp;
+    }
+  }
+  director = SWIG_DIRECTOR_CAST(arg1);
+  upcall = (director && (director->swig_get_self()==swig_obj[0]));
+  try {
+    if (upcall) {
+      Swig::DirectorPureVirtualException::raise("ppc::front::IMessageHandler::onMessage");
+    } else {
+      (arg1)->onMessage(arg2,arg3,arg4);
+    }
+  } catch (Swig::DirectorException&) {
+    SWIG_fail;
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_disown_IMessageHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IMessageHandler *arg1 = (ppc::front::IMessageHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IMessageHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::IMessageHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_IMessageHandler" "', argument " "1"" of type '" "ppc::front::IMessageHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IMessageHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
+    if (director) director->swig_disown();
+  }
+  
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *IMessageHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *IMessageHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  PyObject *arg1 = (PyObject *) 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::GetPeersInfoHandler *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  arg1 = swig_obj[0];
+  if ( arg1 != Py_None ) {
+    /* subclassed */
+    result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(arg1); 
+  } else {
+    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+    SWIG_fail;
+  }
+  
+  {
+    std::shared_ptr<  ppc::front::GetPeersInfoHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GetPeersInfoHandler_onPeersInfo(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
+  bcos::Error::Ptr arg2 ;
+  std::string *arg3 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  int res3 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[3] ;
+  Swig::Director *director = 0;
+  bool upcall = false;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GetPeersInfoHandler_onPeersInfo", 3, 3, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
+  }
+  {
+    std::string *ptr = (std::string *)0;
+    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); 
+    }
+    arg3 = ptr;
+  }
+  director = SWIG_DIRECTOR_CAST(arg1);
+  upcall = (director && (director->swig_get_self()==swig_obj[0]));
+  try {
+    if (upcall) {
+      Swig::DirectorPureVirtualException::raise("ppc::front::GetPeersInfoHandler::onPeersInfo");
+    } else {
+      (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
+    }
+  } catch (Swig::DirectorException&) {
+    SWIG_fail;
+  }
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res3)) delete arg3;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_disown_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
+  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
+      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
+    if (director) director->swig_disown();
+  }
+  
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *GetPeersInfoHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *GetPeersInfoHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFront" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_start" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (arg1)->start();
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_stop(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_stop" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  (arg1)->stop();
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
+  ppc::front::MessageDispatcherHandler::Ptr arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  void *argp3 ;
+  int res3 = 0 ;
+  PyObject *swig_obj[3] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_register_topic_handler", 3, 3, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_topic_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_topic_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'");
+    }
+    if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
+  }
+  (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
+  ppc::front::MessageDispatcherHandler::Ptr arg3 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  void *argp3 ;
+  int res3 = 0 ;
+  PyObject *swig_obj[3] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_register_msg_handler", 3, 3, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_msg_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_msg_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'");
+    }
+    if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
+  }
+  (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_async_send_message(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  uint16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  char *arg4 = (char *) 0 ;
+  uint64_t arg5 ;
+  int arg6 ;
+  long arg7 ;
+  ppc::front::ErrorCallback::Ptr arg8 ;
+  ppc::front::IMessageHandler::Ptr arg9 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
+  int val6 ;
+  int ecode6 = 0 ;
+  long val7 ;
+  int ecode7 = 0 ;
+  void *argp8 ;
+  int res8 = 0 ;
+  void *argp9 ;
+  int res9 = 0 ;
+  PyObject *swig_obj[8] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_message", 8, 8, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_message" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_async_send_message" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_async_send_message" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      arg3 = &tempshared3;
+    } else {
+      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
+    }
+  }
+  {
+    int res; Py_ssize_t size = 0; const void *buf = 0;
+#ifndef Py_LIMITED_API
+    Py_buffer view;
+    res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO);
+#else
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated"
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size);
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#endif
+    if (res < 0) {
+      PyErr_Clear();
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_message" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'");
+    }
+#ifndef Py_LIMITED_API
+    size = view.len;
+    buf = view.buf;
+    PyBuffer_Release(&view);
+#endif
+    arg4 = (char *) buf;
+    arg5 = (uint64_t) (size / sizeof(char));
+  }
+  ecode6 = SWIG_AsVal_int(swig_obj[4], &val6);
+  if (!SWIG_IsOK(ecode6)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_async_send_message" "', argument " "6"" of type '" "int""'");
+  } 
+  arg6 = static_cast< int >(val6);
+  ecode7 = SWIG_AsVal_long(swig_obj[5], &val7);
+  if (!SWIG_IsOK(ecode7)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_message" "', argument " "7"" of type '" "long""'");
+  } 
+  arg7 = static_cast< long >(val7);
+  {
+    int newmem = 0;
+    res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t,  0 , &newmem);
+    if (!SWIG_IsOK(res8)) {
+      SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_message" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'");
+    }
+    if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8);
+  }
+  {
+    int newmem = 0;
+    res9 = SWIG_ConvertPtrAndOwn(swig_obj[7], &argp9, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t,  0 , &newmem);
+    if (!SWIG_IsOK(res9)) {
+      SWIG_exception_fail(SWIG_ArgError(res9), "in method '" "IFront_async_send_message" "', argument " "9"" of type '" "ppc::front::IMessageHandler::Ptr""'");
+    }
+    if (argp9) arg9 = *(reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9);
+  }
+  (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  char *arg2 = (char *) 0 ;
+  uint64_t arg3 ;
+  std::string *arg4 = 0 ;
+  char *arg5 = (char *) 0 ;
+  uint64_t arg6 ;
+  int arg7 ;
+  ppc::front::ErrorCallback::Ptr arg8 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  int res2 ;
+  char *buf2 = 0 ;
+  int alloc2 = 0 ;
+  unsigned long long val3 ;
+  int ecode3 = 0 ;
+  int res4 = SWIG_OLDOBJ ;
+  int val7 ;
+  int ecode7 = 0 ;
+  void *argp8 ;
+  int res8 = 0 ;
+  PyObject *swig_obj[7] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_response", 7, 7, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_response" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_async_send_response" "', argument " "2"" of type '" "char *""'");
+  }
+  arg2 = reinterpret_cast< char * >(buf2);
+  ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_async_send_response" "', argument " "3"" of type '" "uint64_t""'");
+  } 
+  arg3 = static_cast< uint64_t >(val3);
+  {
+    std::string *ptr = (std::string *)0;
+    res4 = SWIG_AsPtr_std_string(swig_obj[3], &ptr);
+    if (!SWIG_IsOK(res4)) {
+      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); 
+    }
+    arg4 = ptr;
+  }
+  {
+    int res; Py_ssize_t size = 0; const void *buf = 0;
+#ifndef Py_LIMITED_API
+    Py_buffer view;
+    res = PyObject_GetBuffer(swig_obj[4], &view, PyBUF_CONTIG_RO);
+#else
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated"
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    res = PyObject_AsReadBuffer(swig_obj[4], &buf, &size);
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#endif
+    if (res < 0) {
+      PyErr_Clear();
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_response" "', argument " "5"" of type '" "(char* payload, uint64_t payloadSize)""'");
+    }
+#ifndef Py_LIMITED_API
+    size = view.len;
+    buf = view.buf;
+    PyBuffer_Release(&view);
+#endif
+    arg5 = (char *) buf;
+    arg6 = (uint64_t) (size / sizeof(char));
+  }
+  ecode7 = SWIG_AsVal_int(swig_obj[5], &val7);
+  if (!SWIG_IsOK(ecode7)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_response" "', argument " "7"" of type '" "int""'");
+  } 
+  arg7 = static_cast< int >(val7);
+  {
+    int newmem = 0;
+    res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t,  0 , &newmem);
+    if (!SWIG_IsOK(res8)) {
+      SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_response" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'");
+    }
+    if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8);
+  }
+  (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+  resultobj = SWIG_Py_Void();
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res4)) delete arg4;
+  return resultobj;
+fail:
+  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
+  if (SWIG_IsNewObj(res4)) delete arg4;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  uint16_t arg2 ;
+  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  char *arg4 = (char *) 0 ;
+  uint64_t arg5 ;
+  int arg6 ;
+  long arg7 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  unsigned short val2 ;
+  int ecode2 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
+  int val6 ;
+  int ecode6 = 0 ;
+  long val7 ;
+  int ecode7 = 0 ;
+  PyObject *swig_obj[6] ;
+  bcos::Error::Ptr result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_push_msg", 6, 6, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_push_msg" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_push_msg" "', argument " "2"" of type '" "uint16_t""'");
+  } 
+  arg2 = static_cast< uint16_t >(val2);
+  {
+    int newmem = 0;
+    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_push_msg" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
+      arg3 = &tempshared3;
+    } else {
+      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
+    }
+  }
+  {
+    int res; Py_ssize_t size = 0; const void *buf = 0;
+#ifndef Py_LIMITED_API
+    Py_buffer view;
+    res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO);
+#else
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated"
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4996)
+#endif
+    res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size);
+#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+#endif
+    if (res < 0) {
+      PyErr_Clear();
+      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_push_msg" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'");
+    }
+#ifndef Py_LIMITED_API
+    size = view.len;
+    buf = view.buf;
+    PyBuffer_Release(&view);
+#endif
+    arg4 = (char *) buf;
+    arg5 = (uint64_t) (size / sizeof(char));
+  }
+  ecode6 = SWIG_AsVal_int(swig_obj[4], &val6);
+  if (!SWIG_IsOK(ecode6)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_push_msg" "', argument " "6"" of type '" "int""'");
+  } 
+  arg6 = static_cast< int >(val6);
+  ecode7 = SWIG_AsVal_long(swig_obj[5], &val7);
+  if (!SWIG_IsOK(ecode7)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_push_msg" "', argument " "7"" of type '" "long""'");
+  } 
+  arg7 = static_cast< long >(val7);
+  result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GetPeersInfoHandler_onPeersInfo(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_pop(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
-  bcos::Error::Ptr arg2 ;
-  std::string *arg3 = 0 ;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
+  long arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  int res3 = SWIG_OLDOBJ ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  long val3 ;
+  int ecode3 = 0 ;
   PyObject *swig_obj[3] ;
-  Swig::Director *director = 0;
-  bool upcall = false;
+  ppc::protocol::Message::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GetPeersInfoHandler_onPeersInfo", 3, 3, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_pop", 3, 3, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_pop" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_bcos__Error_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "2"" of type '" "bcos::Error::Ptr""'");
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
-    if (argp2) arg2 = *(reinterpret_cast< bcos::Error::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< bcos::Error::Ptr * >(argp2);
   }
   {
     std::string *ptr = (std::string *)0;
-    res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); 
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GetPeersInfoHandler_onPeersInfo" "', argument " "3"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    arg3 = ptr;
+    arg2 = ptr;
   }
-  director = SWIG_DIRECTOR_CAST(arg1);
-  upcall = (director && (director->swig_get_self()==swig_obj[0]));
-  try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::GetPeersInfoHandler::onPeersInfo");
-    } else {
-      (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
-    }
-  } catch (Swig::DirectorException&) {
-    SWIG_fail;
+  ecode3 = SWIG_AsVal_long(swig_obj[2], &val3);
+  if (!SWIG_IsOK(ecode3)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_pop" "', argument " "3"" of type '" "long""'");
+  } 
+  arg3 = static_cast< long >(val3);
+  result = (arg1)->pop((std::string const &)*arg2,arg3);
+  {
+    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
   }
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res3)) delete arg3;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res3)) delete arg3;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_disown_GetPeersInfoHandler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_peek(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::GetPeersInfoHandler *arg1 = (ppc::front::GetPeersInfoHandler *) 0 ;
+  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > tempshared1 ;
-  std::shared_ptr< ppc::front::GetPeersInfoHandler > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::front::IFront > tempshared1 ;
+  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  ppc::protocol::Message::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_peek", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "disown_GetPeersInfoHandler" "', argument " "1"" of type '" "ppc::front::GetPeersInfoHandler *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_peek" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::GetPeersInfoHandler > * >(argp1);
-      arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
+      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
   {
-    Swig::Director *director = SWIG_DIRECTOR_CAST(arg1);
-    if (director) director->swig_disown();
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
-  
-  resultobj = SWIG_Py_Void();
+  result = (arg1)->peek((std::string const &)*arg2);
+  {
+    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
+  }
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *GetPeersInfoHandler_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *GetPeersInfoHandler_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_asyncGetPeers(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  ppc::front::GetPeersInfoHandler::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_asyncGetPeers", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFront" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_asyncGetPeers" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -21964,7 +24937,16 @@ SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    int newmem = 0;
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t,  0 , &newmem);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_asyncGetPeers" "', argument " "2"" of type '" "ppc::front::GetPeersInfoHandler::Ptr""'");
+    }
+    if (argp2) arg2 = *(reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2);
+  }
+  (arg1)->asyncGetPeers(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21972,7 +24954,7 @@ SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_nodeInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
   void *argp1 = 0 ;
@@ -21980,6 +24962,7 @@ SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  ppc::protocol::INodeInfo::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
@@ -21988,7 +24971,7 @@ SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_start" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_nodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -21999,31 +24982,36 @@ SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->start();
-  resultobj = SWIG_Py_Void();
+  result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
+  {
+    std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_stop(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_registerTopic(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  bcos::Error::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_registerTopic", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_stop" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22034,35 +25022,49 @@ SWIGINTERN PyObject *_wrap_IFront_stop(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->stop();
-  resultobj = SWIG_Py_Void();
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  result = (arg1)->registerTopic((std::string const &)*arg2);
+  {
+    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  }
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_unRegisterTopic(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
   std::string *arg2 = 0 ;
-  ppc::front::MessageDispatcherHandler::Ptr arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
   int res2 = SWIG_OLDOBJ ;
-  void *argp3 ;
-  int res3 = 0 ;
-  PyObject *swig_obj[3] ;
+  PyObject *swig_obj[2] ;
+  bcos::Error::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_register_topic_handler", 3, 3, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterTopic", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_topic_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22077,24 +25079,18 @@ SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObjec
     std::string *ptr = (std::string *)0;
     res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_topic_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     arg2 = ptr;
   }
+  result = (arg1)->unRegisterTopic((std::string const &)*arg2);
   {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_topic_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'");
-    }
-    if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
+    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
   }
-  (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
-  resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
@@ -22103,27 +25099,24 @@ SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObjec
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_registerComponent(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
   std::string *arg2 = 0 ;
-  ppc::front::MessageDispatcherHandler::Ptr arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
   int res2 = SWIG_OLDOBJ ;
-  void *argp3 ;
-  int res3 = 0 ;
-  PyObject *swig_obj[3] ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_register_msg_handler", 3, 3, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_registerComponent", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_register_msg_handler" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22138,23 +25131,14 @@ SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject
     std::string *ptr = (std::string *)0;
     res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_register_msg_handler" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     arg2 = ptr;
   }
-  {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_register_msg_handler" "', argument " "3"" of type '" "ppc::front::MessageDispatcherHandler::Ptr""'");
-    }
-    if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
-  }
-  (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+  (arg1)->registerComponent((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -22164,43 +25148,24 @@ SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_async_send_message(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_unRegisterComponent(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  uint16_t arg2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
-  char *arg4 = (char *) 0 ;
-  uint64_t arg5 ;
-  int arg6 ;
-  long arg7 ;
-  ppc::front::ErrorCallback::Ptr arg8 ;
-  ppc::front::IMessageHandler::Ptr arg9 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  unsigned short val2 ;
-  int ecode2 = 0 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
-  int val6 ;
-  int ecode6 = 0 ;
-  long val7 ;
-  int ecode7 = 0 ;
-  void *argp8 ;
-  int res8 = 0 ;
-  void *argp9 ;
-  int res9 = 0 ;
-  PyObject *swig_obj[8] ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_message", 8, 8, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterComponent", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_message" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22211,127 +25176,45 @@ SWIGINTERN PyObject *_wrap_IFront_async_send_message(PyObject *self, PyObject *a
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_async_send_message" "', argument " "2"" of type '" "uint16_t""'");
-  } 
-  arg2 = static_cast< uint16_t >(val2);
-  {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_async_send_message" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      arg3 = &tempshared3;
-    } else {
-      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
-    }
-  }
-  {
-    int res; Py_ssize_t size = 0; const void *buf = 0;
-#ifndef Py_LIMITED_API
-    Py_buffer view;
-    res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO);
-#else
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
-    res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size);
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-#endif
-    if (res < 0) {
-      PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_message" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'");
-    }
-#ifndef Py_LIMITED_API
-    size = view.len;
-    buf = view.buf;
-    PyBuffer_Release(&view);
-#endif
-    arg4 = (char *) buf;
-    arg5 = (uint64_t) (size / sizeof(char));
-  }
-  ecode6 = SWIG_AsVal_int(swig_obj[4], &val6);
-  if (!SWIG_IsOK(ecode6)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_async_send_message" "', argument " "6"" of type '" "int""'");
-  } 
-  arg6 = static_cast< int >(val6);
-  ecode7 = SWIG_AsVal_long(swig_obj[5], &val7);
-  if (!SWIG_IsOK(ecode7)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_message" "', argument " "7"" of type '" "long""'");
-  } 
-  arg7 = static_cast< long >(val7);
   {
-    int newmem = 0;
-    res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t,  0 , &newmem);
-    if (!SWIG_IsOK(res8)) {
-      SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_message" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'");
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8);
-  }
-  {
-    int newmem = 0;
-    res9 = SWIG_ConvertPtrAndOwn(swig_obj[7], &argp9, SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t,  0 , &newmem);
-    if (!SWIG_IsOK(res9)) {
-      SWIG_exception_fail(SWIG_ArgError(res9), "in method '" "IFront_async_send_message" "', argument " "9"" of type '" "ppc::front::IMessageHandler::Ptr""'");
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (argp9) arg9 = *(reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9);
+    arg2 = ptr;
   }
-  (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+  (arg1)->unRegisterComponent((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_updateMetaInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  char *arg2 = (char *) 0 ;
-  uint64_t arg3 ;
-  std::string *arg4 = 0 ;
-  char *arg5 = (char *) 0 ;
-  uint64_t arg6 ;
-  int arg7 ;
-  ppc::front::ErrorCallback::Ptr arg8 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 ;
-  char *buf2 = 0 ;
-  int alloc2 = 0 ;
-  unsigned long long val3 ;
-  int ecode3 = 0 ;
-  int res4 = SWIG_OLDOBJ ;
-  int val7 ;
-  int ecode7 = 0 ;
-  void *argp8 ;
-  int res8 = 0 ;
-  PyObject *swig_obj[7] ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_async_send_response", 7, 7, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_updateMetaInfo", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_async_send_response" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_updateMetaInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22342,118 +25225,51 @@ SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2);
-  if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_async_send_response" "', argument " "2"" of type '" "char *""'");
-  }
-  arg2 = reinterpret_cast< char * >(buf2);
-  ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_async_send_response" "', argument " "3"" of type '" "uint64_t""'");
-  } 
-  arg3 = static_cast< uint64_t >(val3);
   {
     std::string *ptr = (std::string *)0;
-    res4 = SWIG_AsPtr_std_string(swig_obj[3], &ptr);
-    if (!SWIG_IsOK(res4)) {
-      SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); 
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_updateMetaInfo" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_async_send_response" "', argument " "4"" of type '" "std::string const &""'"); 
-    }
-    arg4 = ptr;
-  }
-  {
-    int res; Py_ssize_t size = 0; const void *buf = 0;
-#ifndef Py_LIMITED_API
-    Py_buffer view;
-    res = PyObject_GetBuffer(swig_obj[4], &view, PyBUF_CONTIG_RO);
-#else
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
-    res = PyObject_AsReadBuffer(swig_obj[4], &buf, &size);
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-#endif
-    if (res < 0) {
-      PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_async_send_response" "', argument " "5"" of type '" "(char* payload, uint64_t payloadSize)""'");
-    }
-#ifndef Py_LIMITED_API
-    size = view.len;
-    buf = view.buf;
-    PyBuffer_Release(&view);
-#endif
-    arg5 = (char *) buf;
-    arg6 = (uint64_t) (size / sizeof(char));
-  }
-  ecode7 = SWIG_AsVal_int(swig_obj[5], &val7);
-  if (!SWIG_IsOK(ecode7)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_async_send_response" "', argument " "7"" of type '" "int""'");
-  } 
-  arg7 = static_cast< int >(val7);
-  {
-    int newmem = 0;
-    res8 = SWIG_ConvertPtrAndOwn(swig_obj[6], &argp8, SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t,  0 , &newmem);
-    if (!SWIG_IsOK(res8)) {
-      SWIG_exception_fail(SWIG_ArgError(res8), "in method '" "IFront_async_send_response" "', argument " "8"" of type '" "ppc::front::ErrorCallback::Ptr""'");
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_updateMetaInfo" "', argument " "2"" of type '" "std::string const &""'"); 
     }
-    if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8);
+    arg2 = ptr;
   }
-  (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+  (arg1)->updateMetaInfo((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res4)) delete arg4;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
-  if (SWIG_IsNewObj(res4)) delete arg4;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_selectNodesByRoutePolicy(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  uint16_t arg2 ;
+  int16_t arg2 ;
   ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
-  char *arg4 = (char *) 0 ;
-  uint64_t arg5 ;
-  int arg6 ;
-  long arg7 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  unsigned short val2 ;
+  short val2 ;
   int ecode2 = 0 ;
   void *argp3 ;
   int res3 = 0 ;
   ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
-  int val6 ;
-  int ecode6 = 0 ;
-  long val7 ;
-  int ecode7 = 0 ;
-  PyObject *swig_obj[6] ;
-  bcos::Error::Ptr result;
+  PyObject *swig_obj[3] ;
+  std::vector< std::string,std::allocator< std::string > > result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_push_msg", 6, 6, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "IFront_selectNodesByRoutePolicy", 3, 3, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_push_msg" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22464,16 +25280,16 @@ SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_short(swig_obj[1], &val2);
+  ecode2 = SWIG_AsVal_short(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_push_msg" "', argument " "2"" of type '" "uint16_t""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "2"" of type '" "int16_t""'");
   } 
-  arg2 = static_cast< uint16_t >(val2);
+  arg2 = static_cast< int16_t >(val2);
   {
     int newmem = 0;
     res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
     if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_push_msg" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
@@ -22483,82 +25299,32 @@ SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
       arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
     }
   }
-  {
-    int res; Py_ssize_t size = 0; const void *buf = 0;
-#ifndef Py_LIMITED_API
-    Py_buffer view;
-    res = PyObject_GetBuffer(swig_obj[3], &view, PyBUF_CONTIG_RO);
-#else
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#elif defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
-    res = PyObject_AsReadBuffer(swig_obj[3], &buf, &size);
-#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6))
-#pragma GCC diagnostic pop
-#elif defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-#endif
-    if (res < 0) {
-      PyErr_Clear();
-      SWIG_exception_fail(SWIG_ArgError(res), "in method '" "IFront_push_msg" "', argument " "4"" of type '" "(char* payload, uint64_t payloadSize)""'");
-    }
-#ifndef Py_LIMITED_API
-    size = view.len;
-    buf = view.buf;
-    PyBuffer_Release(&view);
-#endif
-    arg4 = (char *) buf;
-    arg5 = (uint64_t) (size / sizeof(char));
-  }
-  ecode6 = SWIG_AsVal_int(swig_obj[4], &val6);
-  if (!SWIG_IsOK(ecode6)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "IFront_push_msg" "', argument " "6"" of type '" "int""'");
-  } 
-  arg6 = static_cast< int >(val6);
-  ecode7 = SWIG_AsVal_long(swig_obj[5], &val7);
-  if (!SWIG_IsOK(ecode7)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_push_msg" "', argument " "7"" of type '" "long""'");
-  } 
-  arg7 = static_cast< long >(val7);
-  result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
-  }
+  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_pop(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_IFront_getNodeDiscovery(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
-  long arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::front::IFront > tempshared1 ;
   std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  long val3 ;
-  int ecode3 = 0 ;
-  PyObject *swig_obj[3] ;
-  ppc::protocol::Message::Ptr result;
+  PyObject *swig_obj[1] ;
+  ppc::front::INodeDiscovery::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_pop", 3, 3, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_pop" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_getNodeDiscovery" "', argument " "1"" of type '" "ppc::front::IFront *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
@@ -22569,458 +25335,505 @@ SWIGINTERN PyObject *_wrap_IFront_pop(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  result = (arg1)->getNodeDiscovery();
+  {
+    std::shared_ptr<  ppc::front::INodeDiscovery > *smartresult = result ? new std::shared_ptr<  ppc::front::INodeDiscovery >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_POINTER_OWN);
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *IFront_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_IFrontBuilder(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontBuilder" "', argument " "1"" of type '" "ppc::front::IFrontBuilder *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_IFrontBuilder_buildClient(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
+  std::string arg2 ;
+  std::function< void () > arg3 ;
+  bool arg4 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp3 ;
+  int res3 = 0 ;
+  bool val4 ;
+  int ecode4 = 0 ;
+  PyObject *swig_obj[4] ;
+  ppc::front::IFrontClient::Ptr result;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "IFrontBuilder_buildClient", 4, 4, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontBuilder_buildClient" "', argument " "1"" of type '" "ppc::front::IFrontBuilder const *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1);
   {
     std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); 
+    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "IFrontBuilder_buildClient" "', argument " "2"" of type '" "std::string""'"); 
     }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_pop" "', argument " "2"" of type '" "std::string const &""'"); 
+    arg2 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  {
+    res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fF_t,  0  | 0);
+    if (!SWIG_IsOK(res3)) {
+      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'"); 
+    }  
+    if (!argp3) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'");
+    } else {
+      std::function< void () > * temp = reinterpret_cast< std::function< void () > * >(argp3);
+      arg3 = *temp;
+      if (SWIG_IsNewObj(res3)) delete temp;
     }
-    arg2 = ptr;
   }
-  ecode3 = SWIG_AsVal_long(swig_obj[2], &val3);
-  if (!SWIG_IsOK(ecode3)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_pop" "', argument " "3"" of type '" "long""'");
+  ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
+  if (!SWIG_IsOK(ecode4)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "IFrontBuilder_buildClient" "', argument " "4"" of type '" "bool""'");
   } 
-  arg3 = static_cast< long >(val3);
-  result = (arg1)->pop((std::string const &)*arg2,arg3);
+  arg4 = static_cast< bool >(val4);
+  result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
   {
-    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
+    std::shared_ptr<  ppc::front::IFrontClient > *smartresult = result ? new std::shared_ptr<  ppc::front::IFrontClient >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN);
   }
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_peek(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *IFrontBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_delete_INodeDiscovery(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::front::INodeDiscovery *arg1 = (ppc::front::INodeDiscovery *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
-  ppc::protocol::Message::Ptr result;
+  std::shared_ptr< ppc::front::INodeDiscovery > tempshared1 ;
+  std::shared_ptr< ppc::front::INodeDiscovery > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_peek", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_peek" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_INodeDiscovery" "', argument " "1"" of type '" "ppc::front::INodeDiscovery *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::INodeDiscovery > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::front::INodeDiscovery > * >(argp1);
+      arg1 = const_cast< ppc::front::INodeDiscovery * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::INodeDiscovery > * >(argp1);
+      arg1 = const_cast< ppc::front::INodeDiscovery * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_INodeDiscovery_getAliveNodeList(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::front::INodeDiscovery *arg1 = (ppc::front::INodeDiscovery *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::front::INodeDiscovery const > tempshared1 ;
+  std::shared_ptr< ppc::front::INodeDiscovery const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); 
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeDiscovery_getAliveNodeList" "', argument " "1"" of type '" "ppc::front::INodeDiscovery const *""'");
     }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_peek" "', argument " "2"" of type '" "std::string const &""'"); 
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::front::INodeDiscovery > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::front::INodeDiscovery > * >(argp1);
+      arg1 = const_cast< ppc::front::INodeDiscovery * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::front::INodeDiscovery > * >(argp1);
+      arg1 = const_cast< ppc::front::INodeDiscovery * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg2 = ptr;
   }
-  result = (arg1)->peek((std::string const &)*arg2);
-  {
-    std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
+  result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+  resultobj = swig::from(static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *INodeDiscovery_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_new_TransportBuilder(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::sdk::TransportBuilder *result = 0 ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "new_TransportBuilder", 0, 0, 0)) SWIG_fail;
+  result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_TransportBuilder(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_TransportBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
   }
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_asyncGetPeers(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_TransportBuilder_buildProTransport(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  ppc::front::GetPeersInfoHandler::Ptr arg2 ;
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  ppc::front::FrontConfig::Ptr arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
   void *argp2 ;
   int res2 = 0 ;
   PyObject *swig_obj[2] ;
+  ppc::sdk::Transport::Ptr result;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_asyncGetPeers", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_asyncGetPeers" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildProTransport", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildProTransport" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t,  0 , &newmem);
+    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
     if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_asyncGetPeers" "', argument " "2"" of type '" "ppc::front::GetPeersInfoHandler::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "TransportBuilder_buildProTransport" "', argument " "2"" of type '" "ppc::front::FrontConfig::Ptr""'");
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2);
+    if (argp2) arg2 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2);
+  }
+  result = (arg1)->buildProTransport(arg2);
+  {
+    std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_OWN);
   }
-  (arg1)->asyncGetPeers(arg2);
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_nodeInfo(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_TransportBuilder_buildConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  int arg2 ;
+  std::string arg3 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::INodeInfo::Ptr *result = 0 ;
+  int val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[3] ;
+  ppc::front::FrontConfig::Ptr result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildConfig", 3, 3, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildConfig" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
+  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "TransportBuilder_buildConfig" "', argument " "2"" of type '" "int""'");
+  } 
+  arg2 = static_cast< int >(val2);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_nodeInfo" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+    std::string *ptr = (std::string *)0;
+    int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
+    if (!SWIG_IsOK(res) || !ptr) {
+      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "TransportBuilder_buildConfig" "', argument " "3"" of type '" "std::string""'"); 
     }
+    arg3 = *ptr;
+    if (SWIG_IsNewObj(res)) delete ptr;
+  }
+  result = (arg1)->buildConfig(arg2,arg3);
+  {
+    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
   }
-  result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__INodeInfo__Ptr, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_registerTopic(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
-  bcos::Error::Ptr result;
+  PyObject *swig_obj[1] ;
+  ppc::front::FrontConfigBuilder::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_registerTopic", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerTopic" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
-  }
-  result = (arg1)->registerTopic((std::string const &)*arg2);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_frontConfigBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
   }
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
+  result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, 0 |  0 );
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_unRegisterTopic(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_TransportBuilder_initLog(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
-  bcos::Error::Ptr result;
+  std::string *arg1 = 0 ;
+  int res1 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterTopic", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterTopic" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterTopic" "', argument " "2"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
     }
-    arg2 = ptr;
-  }
-  result = (arg1)->unRegisterTopic((std::string const &)*arg2);
-  {
-    std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
+    arg1 = ptr;
   }
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
+  if (SWIG_IsNewObj(res1)) delete arg1;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_registerComponent(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *TransportBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *TransportBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_Transport(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
-  void *argp1 = 0 ;
+  ppc::front::FrontConfig::Ptr arg1 ;
+  void *argp1 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  ppc::sdk::Transport *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_registerComponent", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_registerComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_Transport" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr""'");
     }
+    if (argp1) arg1 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1));
+    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
   }
+  result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
   {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_registerComponent" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
+    std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
   }
-  (arg1)->registerComponent((std::string const &)*arg2);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_unRegisterComponent(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_Transport(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  std::string *arg2 = 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[2] ;
+  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_unRegisterComponent", 2, 2, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_unRegisterComponent" "', argument " "1"" of type '" "ppc::front::IFront *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Transport" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFront_unRegisterComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg2 = ptr;
   }
-  (arg1)->unRegisterComponent((std::string const &)*arg2);
+  (void)arg1; delete smartarg1;
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_IFront_selectNodesByRoutePolicy(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_start(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFront *arg1 = (ppc::front::IFront *) 0 ;
-  int16_t arg2 ;
-  ppc::protocol::MessageOptionalHeader::Ptr *arg3 = 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::front::IFront > tempshared1 ;
-  std::shared_ptr< ppc::front::IFront > *smartarg1 = 0 ;
-  short val2 ;
-  int ecode2 = 0 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  ppc::protocol::MessageOptionalHeader::Ptr tempshared3 ;
-  PyObject *swig_obj[3] ;
-  std::vector< std::string,std::allocator< std::string > > result;
+  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFront_selectNodesByRoutePolicy", 3, 3, swig_obj)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "1"" of type '" "ppc::front::IFront *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::front::IFront > * >(argp1);
-      arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  ecode2 = SWIG_AsVal_short(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "2"" of type '" "int16_t""'");
-  } 
-  arg2 = static_cast< int16_t >(val2);
-  {
-    int newmem = 0;
-    res3 = SWIG_ConvertPtrAndOwn(swig_obj[2], &argp3, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t,  0 , &newmem);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFront_selectNodesByRoutePolicy" "', argument " "3"" of type '" "ppc::protocol::MessageOptionalHeader::Ptr const &""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_start" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      if (argp3) tempshared3 = *reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3);
-      arg3 = &tempshared3;
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
     } else {
-      arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
-  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
+  (arg1)->start();
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *IFront_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_delete_IFrontBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_stop(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontBuilder" "', argument " "1"" of type '" "ppc::front::IFrontBuilder *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_stop" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1);
-  delete arg1;
+  (arg1)->stop();
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23028,59 +25841,38 @@ SWIGINTERN PyObject *_wrap_delete_IFrontBuilder(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_IFrontBuilder_buildClient(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_getFront(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::IFrontBuilder *arg1 = (ppc::front::IFrontBuilder *) 0 ;
-  std::string arg2 ;
-  std::function< void () > arg3 ;
-  bool arg4 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  void *argp3 ;
-  int res3 = 0 ;
-  bool val4 ;
-  int ecode4 = 0 ;
-  PyObject *swig_obj[4] ;
-  ppc::front::IFrontClient::Ptr result;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::front::IFront::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "IFrontBuilder_buildClient", 4, 4, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__front__IFrontBuilder, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "IFrontBuilder_buildClient" "', argument " "1"" of type '" "ppc::front::IFrontBuilder const *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "IFrontBuilder_buildClient" "', argument " "2"" of type '" "std::string""'"); 
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getFront" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
     }
-    arg2 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
-  }
-  {
-    res3 = SWIG_ConvertPtr(swig_obj[2], &argp3, SWIGTYPE_p_std__functionT_void_fF_t,  0  | 0);
-    if (!SWIG_IsOK(res3)) {
-      SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'"); 
-    }  
-    if (!argp3) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "IFrontBuilder_buildClient" "', argument " "3"" of type '" "std::function< void () >""'");
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
     } else {
-      std::function< void () > * temp = reinterpret_cast< std::function< void () > * >(argp3);
-      arg3 = *temp;
-      if (SWIG_IsNewObj(res3)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "IFrontBuilder_buildClient" "', argument " "4"" of type '" "bool""'");
-  } 
-  arg4 = static_cast< bool >(val4);
-  result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
+  result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
   {
-    std::shared_ptr<  ppc::front::IFrontClient > *smartresult = result ? new std::shared_ptr<  ppc::front::IFrontClient >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN);
+    std::shared_ptr<  ppc::front::IFront > *smartresult = *result ? new std::shared_ptr<  ppc::front::IFront >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -23088,81 +25880,113 @@ SWIGINTERN PyObject *_wrap_IFrontBuilder_buildClient(PyObject *self, PyObject *a
 }
 
 
-SWIGINTERN PyObject *IFrontBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__front__IFrontBuilder, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *_wrap_new_TransportBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_gateway(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::TransportBuilder *result = 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::gateway::IGateway::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "new_TransportBuilder", 0, 0, 0)) SWIG_fail;
-  result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_NEW |  0 );
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_gateway" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__gateway__IGateway__Ptr, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_TransportBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_msgPayloadBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  ppc::protocol::MessagePayloadBuilder::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_TransportBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgPayloadBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
+  {
+    std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayloadBuilder >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN);
   }
-  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
-  delete arg1;
-  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_TransportBuilder_buildProTransport(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_msgBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
-  ppc::front::FrontConfig::Ptr arg2 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  ppc::sdk::Transport::Ptr result;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::MessageBuilder::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildProTransport", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildProTransport" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
     int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "TransportBuilder_buildProTransport" "', argument " "2"" of type '" "ppc::front::FrontConfig::Ptr""'");
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2);
   }
-  result = (arg1)->buildProTransport(arg2);
+  result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
   {
-    std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_OWN);
+    std::shared_ptr<  ppc::protocol::MessageBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageBuilder >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -23170,43 +25994,38 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_buildProTransport(PyObject *self, Py
 }
 
 
-SWIGINTERN PyObject *_wrap_TransportBuilder_buildConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_msgHeaderBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
-  int arg2 ;
-  std::string arg3 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  int val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[3] ;
-  ppc::front::FrontConfig::Ptr result;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::MessageHeaderBuilder::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_buildConfig", 3, 3, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_buildConfig" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
-  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "TransportBuilder_buildConfig" "', argument " "2"" of type '" "int""'");
-  } 
-  arg2 = static_cast< int >(val2);
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    std::string *ptr = (std::string *)0;
-    int res = SWIG_AsPtr_std_string(swig_obj[2], &ptr);
-    if (!SWIG_IsOK(res) || !ptr) {
-      SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "TransportBuilder_buildConfig" "', argument " "3"" of type '" "std::string""'"); 
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgHeaderBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg3 = *ptr;
-    if (SWIG_IsNewObj(res)) delete ptr;
   }
-  result = (arg1)->buildConfig(arg2,arg3);
+  result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
   {
-    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
+    std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageHeaderBuilder >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN);
   }
   return resultobj;
 fail:
@@ -23214,165 +26033,245 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_buildConfig(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_routeInfoBuilder(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::front::FrontConfigBuilder::Ptr *result = 0 ;
+  ppc::protocol::MessageOptionalHeaderBuilder::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_frontConfigBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_routeInfoBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
+  {
+    std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN);
   }
-  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
-  result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_TransportBuilder_initLog(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_Transport_getConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::string *arg1 = 0 ;
-  int res1 = SWIG_OLDOBJ ;
+  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
+  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  ppc::front::FrontConfig::Ptr *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
-    std::string *ptr = (std::string *)0;
-    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getConfig" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
     }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
+      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
-    arg1 = ptr;
   }
-  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
-  resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res1)) delete arg1;
+  result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
+  {
+    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = *result ? new std::shared_ptr<  ppc::front::FrontConfig >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res1)) delete arg1;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *TransportBuilder_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *Transport_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *TransportBuilder_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *Transport_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   return SWIG_Python_InitShadowInstance(args);
 }
 
-SWIGINTERN PyObject *_wrap_new_Transport(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_delete_INodeInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::front::FrontConfig::Ptr arg1 ;
-  void *argp1 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::sdk::Transport *result = 0 ;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t,  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_Transport" "', argument " "1"" of type '" "ppc::front::FrontConfig::Ptr""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_INodeInfo" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
-    if (argp1) arg1 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
   }
-  result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
+  (void)arg1; delete smartarg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_INodeInfo_endPoint(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_endPoint" "', argument " "1"" of type '" "ppc::protocol::INodeInfo const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
+  result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_Transport(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_nodeID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  bcos::bytesConstRef result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_Transport" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_nodeID" "', argument " "1"" of type '" "ppc::protocol::INodeInfo const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
-  resultobj = SWIG_Py_Void();
+  result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+  resultobj = SWIG_NewPointerObj((new bcos::bytesConstRef(result)), SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_start(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_setNodeID(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  bcos::bytesConstRef arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_setNodeID", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_start" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_setNodeID" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->start();
+  {
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_bcos__bytesConstRef,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_setNodeID" "', argument " "2"" of type '" "bcos::bytesConstRef""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_setNodeID" "', argument " "2"" of type '" "bcos::bytesConstRef""'");
+    } else {
+      bcos::bytesConstRef * temp = reinterpret_cast< bcos::bytesConstRef * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
+    }
+  }
+  (arg1)->setNodeID(arg2);
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23380,322 +26279,367 @@ SWIGINTERN PyObject *_wrap_Transport_start(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_stop(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_setEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_setEndPoint", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_stop" "', argument " "1"" of type '" "ppc::sdk::Transport *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->stop();
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_setEndPoint" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_setEndPoint" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  (arg1)->setEndPoint((std::string const &)*arg2);
   resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_getFront(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_addComponent(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::front::IFront::Ptr *result = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  bool result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_addComponent", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getFront" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_addComponent" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
   {
-    std::shared_ptr<  ppc::front::IFront > *smartresult = *result ? new std::shared_ptr<  ppc::front::IFront >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_addComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
+  result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_gateway(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_eraseComponent(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::gateway::IGateway::Ptr *result = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  bool result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_eraseComponent", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_gateway" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_eraseComponent" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__gateway__IGateway__Ptr, 0 |  0 );
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_eraseComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_eraseComponent" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_msgPayloadBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_componentExist(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::MessagePayloadBuilder::Ptr *result = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
+  bool result;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_componentExist", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgPayloadBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_componentExist" "', argument " "1"" of type '" "ppc::protocol::INodeInfo const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
   {
-    std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayloadBuilder >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_componentExist" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_componentExist" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
+  result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_msgBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_copiedComponents(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::MessageBuilder::Ptr *result = 0 ;
+  std::vector< std::string,std::allocator< std::string > > result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_copiedComponents" "', argument " "1"" of type '" "ppc::protocol::INodeInfo const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
-  {
-    std::shared_ptr<  ppc::protocol::MessageBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageBuilder >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN);
-  }
+  result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+  resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_msgHeaderBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_meta(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::MessageHeaderBuilder::Ptr *result = 0 ;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_msgHeaderBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_meta" "', argument " "1"" of type '" "ppc::protocol::INodeInfo const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
-  {
-    std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageHeaderBuilder >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN);
-  }
+  result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_routeInfoBuilder(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_INodeInfo_setMeta(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
+  ppc::protocol::INodeInfo *arg1 = (ppc::protocol::INodeInfo *) 0 ;
+  std::string *arg2 = 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::MessageOptionalHeaderBuilder::Ptr *result = 0 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > tempshared1 ;
+  std::shared_ptr< ppc::protocol::INodeInfo > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "INodeInfo_setMeta", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_routeInfoBuilder" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "INodeInfo_setMeta" "', argument " "1"" of type '" "ppc::protocol::INodeInfo *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >(tempshared1.get());
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::INodeInfo > * >(argp1);
+      arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
   {
-    std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN);
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "INodeInfo_setMeta" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "INodeInfo_setMeta" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
   }
+  (arg1)->setMeta((std::string const &)*arg2);
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_Transport_getConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *INodeInfo_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *_wrap_printNodeInfo(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::sdk::Transport *arg1 = (ppc::sdk::Transport *) 0 ;
-  void *argp1 = 0 ;
+  ppc::protocol::INodeInfo::Ptr *arg1 = 0 ;
+  void *argp1 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::sdk::Transport const > tempshared1 ;
-  std::shared_ptr< ppc::sdk::Transport const > *smartarg1 = 0 ;
+  ppc::protocol::INodeInfo::Ptr tempshared1 ;
   PyObject *swig_obj[1] ;
-  ppc::front::FrontConfig::Ptr *result = 0 ;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   {
     int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t,  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Transport_getConfig" "', argument " "1"" of type '" "ppc::sdk::Transport const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printNodeInfo" "', argument " "1"" of type '" "ppc::protocol::INodeInfo::Ptr const &""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >(tempshared1.get());
+      if (argp1) tempshared1 = *reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp1);
+      delete reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp1);
+      arg1 = &tempshared1;
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::sdk::Transport > * >(argp1);
-      arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
+      arg1 = (argp1) ? reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
-  {
-    std::shared_ptr<  ppc::front::FrontConfig > *smartresult = *result ? new std::shared_ptr<  ppc::front::FrontConfig >(*result) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
-  }
+  result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *Transport_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *Transport_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
 static PyMethodDef SwigMethods[] = {
 	 { "delete_SwigPyIterator", _wrap_delete_SwigPyIterator, METH_O, NULL},
 	 { "SwigPyIterator_value", _wrap_SwigPyIterator_value, METH_O, NULL},
@@ -23731,6 +26675,14 @@ static PyMethodDef SwigMethods[] = {
 	 { "delete_SharedFront", _wrap_delete_SharedFront, METH_O, NULL},
 	 { "SharedFront_swigregister", SharedFront_swigregister, METH_O, NULL},
 	 { "SharedFront_swiginit", SharedFront_swiginit, METH_VARARGS, NULL},
+	 { "new_SharedNodeDiscovery", _wrap_new_SharedNodeDiscovery, METH_NOARGS, NULL},
+	 { "delete_SharedNodeDiscovery", _wrap_delete_SharedNodeDiscovery, METH_O, NULL},
+	 { "SharedNodeDiscovery_swigregister", SharedNodeDiscovery_swigregister, METH_O, NULL},
+	 { "SharedNodeDiscovery_swiginit", SharedNodeDiscovery_swiginit, METH_VARARGS, NULL},
+	 { "new_SharedNodeInfo", _wrap_new_SharedNodeInfo, METH_NOARGS, NULL},
+	 { "delete_SharedNodeInfo", _wrap_delete_SharedNodeInfo, METH_O, NULL},
+	 { "SharedNodeInfo_swigregister", SharedNodeInfo_swigregister, METH_O, NULL},
+	 { "SharedNodeInfo_swiginit", SharedNodeInfo_swiginit, METH_VARARGS, NULL},
 	 { "new_SharedFrontClient", _wrap_new_SharedFrontClient, METH_NOARGS, NULL},
 	 { "delete_SharedFrontClient", _wrap_delete_SharedFrontClient, METH_O, NULL},
 	 { "SharedFrontClient_swigregister", SharedFrontClient_swigregister, METH_O, NULL},
@@ -23892,6 +26844,41 @@ static PyMethodDef SwigMethods[] = {
 	 { "delete_StringVec", _wrap_delete_StringVec, METH_O, NULL},
 	 { "StringVec_swigregister", StringVec_swigregister, METH_O, NULL},
 	 { "StringVec_swiginit", StringVec_swiginit, METH_VARARGS, NULL},
+	 { "NodeInfoVec_iterator", _wrap_NodeInfoVec_iterator, METH_O, NULL},
+	 { "NodeInfoVec___nonzero__", _wrap_NodeInfoVec___nonzero__, METH_O, NULL},
+	 { "NodeInfoVec___bool__", _wrap_NodeInfoVec___bool__, METH_O, NULL},
+	 { "NodeInfoVec___len__", _wrap_NodeInfoVec___len__, METH_O, NULL},
+	 { "NodeInfoVec___getslice__", _wrap_NodeInfoVec___getslice__, METH_VARARGS, NULL},
+	 { "NodeInfoVec___setslice__", _wrap_NodeInfoVec___setslice__, METH_VARARGS, NULL},
+	 { "NodeInfoVec___delslice__", _wrap_NodeInfoVec___delslice__, METH_VARARGS, NULL},
+	 { "NodeInfoVec___delitem__", _wrap_NodeInfoVec___delitem__, METH_VARARGS, NULL},
+	 { "NodeInfoVec___getitem__", _wrap_NodeInfoVec___getitem__, METH_VARARGS, NULL},
+	 { "NodeInfoVec___setitem__", _wrap_NodeInfoVec___setitem__, METH_VARARGS, NULL},
+	 { "NodeInfoVec_pop", _wrap_NodeInfoVec_pop, METH_O, NULL},
+	 { "NodeInfoVec_append", _wrap_NodeInfoVec_append, METH_VARARGS, NULL},
+	 { "NodeInfoVec_empty", _wrap_NodeInfoVec_empty, METH_O, NULL},
+	 { "NodeInfoVec_size", _wrap_NodeInfoVec_size, METH_O, NULL},
+	 { "NodeInfoVec_swap", _wrap_NodeInfoVec_swap, METH_VARARGS, NULL},
+	 { "NodeInfoVec_begin", _wrap_NodeInfoVec_begin, METH_O, NULL},
+	 { "NodeInfoVec_end", _wrap_NodeInfoVec_end, METH_O, NULL},
+	 { "NodeInfoVec_rbegin", _wrap_NodeInfoVec_rbegin, METH_O, NULL},
+	 { "NodeInfoVec_rend", _wrap_NodeInfoVec_rend, METH_O, NULL},
+	 { "NodeInfoVec_clear", _wrap_NodeInfoVec_clear, METH_O, NULL},
+	 { "NodeInfoVec_get_allocator", _wrap_NodeInfoVec_get_allocator, METH_O, NULL},
+	 { "NodeInfoVec_pop_back", _wrap_NodeInfoVec_pop_back, METH_O, NULL},
+	 { "NodeInfoVec_erase", _wrap_NodeInfoVec_erase, METH_VARARGS, NULL},
+	 { "new_NodeInfoVec", _wrap_new_NodeInfoVec, METH_VARARGS, NULL},
+	 { "NodeInfoVec_push_back", _wrap_NodeInfoVec_push_back, METH_VARARGS, NULL},
+	 { "NodeInfoVec_front", _wrap_NodeInfoVec_front, METH_O, NULL},
+	 { "NodeInfoVec_back", _wrap_NodeInfoVec_back, METH_O, NULL},
+	 { "NodeInfoVec_assign", _wrap_NodeInfoVec_assign, METH_VARARGS, NULL},
+	 { "NodeInfoVec_resize", _wrap_NodeInfoVec_resize, METH_VARARGS, NULL},
+	 { "NodeInfoVec_insert", _wrap_NodeInfoVec_insert, METH_VARARGS, NULL},
+	 { "NodeInfoVec_reserve", _wrap_NodeInfoVec_reserve, METH_VARARGS, NULL},
+	 { "NodeInfoVec_capacity", _wrap_NodeInfoVec_capacity, METH_O, NULL},
+	 { "delete_NodeInfoVec", _wrap_delete_NodeInfoVec, METH_O, NULL},
+	 { "NodeInfoVec_swigregister", NodeInfoVec_swigregister, METH_O, NULL},
+	 { "NodeInfoVec_swiginit", NodeInfoVec_swiginit, METH_VARARGS, NULL},
 	 { "Error_buildError", _wrap_Error_buildError, METH_VARARGS, NULL},
 	 { "new_Error", _wrap_new_Error, METH_VARARGS, NULL},
 	 { "delete_Error", _wrap_delete_Error, METH_O, NULL},
@@ -23934,6 +26921,8 @@ static PyMethodDef SwigMethods[] = {
 	 { "FrontConfig_setComponents", _wrap_FrontConfig_setComponents, METH_VARARGS, NULL},
 	 { "FrontConfig_addComponent", _wrap_FrontConfig_addComponent, METH_VARARGS, NULL},
 	 { "FrontConfig_mutableComponents", _wrap_FrontConfig_mutableComponents, METH_O, NULL},
+	 { "FrontConfig_meta", _wrap_FrontConfig_meta, METH_O, NULL},
+	 { "FrontConfig_setMeta", _wrap_FrontConfig_setMeta, METH_VARARGS, NULL},
 	 { "FrontConfig_swigregister", FrontConfig_swigregister, METH_O, NULL},
 	 { "delete_FrontConfigBuilder", _wrap_delete_FrontConfigBuilder, METH_O, NULL},
 	 { "FrontConfigBuilder_build", _wrap_FrontConfigBuilder_build, METH_VARARGS, NULL},
@@ -24138,11 +27127,16 @@ static PyMethodDef SwigMethods[] = {
 		""},
 	 { "IFront_registerComponent", _wrap_IFront_registerComponent, METH_VARARGS, NULL},
 	 { "IFront_unRegisterComponent", _wrap_IFront_unRegisterComponent, METH_VARARGS, NULL},
+	 { "IFront_updateMetaInfo", _wrap_IFront_updateMetaInfo, METH_VARARGS, NULL},
 	 { "IFront_selectNodesByRoutePolicy", _wrap_IFront_selectNodesByRoutePolicy, METH_VARARGS, NULL},
+	 { "IFront_getNodeDiscovery", _wrap_IFront_getNodeDiscovery, METH_O, NULL},
 	 { "IFront_swigregister", IFront_swigregister, METH_O, NULL},
 	 { "delete_IFrontBuilder", _wrap_delete_IFrontBuilder, METH_O, NULL},
 	 { "IFrontBuilder_buildClient", _wrap_IFrontBuilder_buildClient, METH_VARARGS, NULL},
 	 { "IFrontBuilder_swigregister", IFrontBuilder_swigregister, METH_O, NULL},
+	 { "delete_INodeDiscovery", _wrap_delete_INodeDiscovery, METH_O, NULL},
+	 { "INodeDiscovery_getAliveNodeList", _wrap_INodeDiscovery_getAliveNodeList, METH_O, NULL},
+	 { "INodeDiscovery_swigregister", INodeDiscovery_swigregister, METH_O, NULL},
 	 { "new_TransportBuilder", _wrap_new_TransportBuilder, METH_NOARGS, NULL},
 	 { "delete_TransportBuilder", _wrap_delete_TransportBuilder, METH_O, NULL},
 	 { "TransportBuilder_buildProTransport", _wrap_TransportBuilder_buildProTransport, METH_VARARGS, NULL},
@@ -24164,6 +27158,19 @@ static PyMethodDef SwigMethods[] = {
 	 { "Transport_getConfig", _wrap_Transport_getConfig, METH_O, NULL},
 	 { "Transport_swigregister", Transport_swigregister, METH_O, NULL},
 	 { "Transport_swiginit", Transport_swiginit, METH_VARARGS, NULL},
+	 { "delete_INodeInfo", _wrap_delete_INodeInfo, METH_O, NULL},
+	 { "INodeInfo_endPoint", _wrap_INodeInfo_endPoint, METH_O, NULL},
+	 { "INodeInfo_nodeID", _wrap_INodeInfo_nodeID, METH_O, NULL},
+	 { "INodeInfo_setNodeID", _wrap_INodeInfo_setNodeID, METH_VARARGS, NULL},
+	 { "INodeInfo_setEndPoint", _wrap_INodeInfo_setEndPoint, METH_VARARGS, NULL},
+	 { "INodeInfo_addComponent", _wrap_INodeInfo_addComponent, METH_VARARGS, NULL},
+	 { "INodeInfo_eraseComponent", _wrap_INodeInfo_eraseComponent, METH_VARARGS, NULL},
+	 { "INodeInfo_componentExist", _wrap_INodeInfo_componentExist, METH_VARARGS, NULL},
+	 { "INodeInfo_copiedComponents", _wrap_INodeInfo_copiedComponents, METH_O, NULL},
+	 { "INodeInfo_meta", _wrap_INodeInfo_meta, METH_O, NULL},
+	 { "INodeInfo_setMeta", _wrap_INodeInfo_setMeta, METH_VARARGS, NULL},
+	 { "INodeInfo_swigregister", INodeInfo_swigregister, METH_O, NULL},
+	 { "printNodeInfo", _wrap_printNodeInfo, METH_O, NULL},
 	 { NULL, NULL, 0, NULL }
 };
 
@@ -24200,13 +27207,14 @@ static swig_type_info _swigt__p_ppc__front__IFront = {"_p_ppc__front__IFront", "
 static swig_type_info _swigt__p_ppc__front__IFrontBuilder = {"_p_ppc__front__IFrontBuilder", "ppc::front::IFrontBuilder *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__front__IFrontClient = {"_p_ppc__front__IFrontClient", "ppc::front::IFrontClient *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__front__IMessageHandler = {"_p_ppc__front__IMessageHandler", "ppc::front::IMessageHandler *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_ppc__front__INodeDiscovery = {"_p_ppc__front__INodeDiscovery", "ppc::front::INodeDiscovery *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__front__MessageDispatcherHandler = {"_p_ppc__front__MessageDispatcherHandler", "ppc::front::MessageDispatcherHandler *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__front__SendResponseHandler = {"_p_ppc__front__SendResponseHandler", "ppc::front::SendResponseHandler *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__gateway__IGateway__Ptr = {"_p_ppc__gateway__IGateway__Ptr", "ppc::gateway::IGateway::Ptr *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__EndPoint = {"_p_ppc__protocol__EndPoint", "ppc::protocol::EndPoint *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__GrpcConfig = {"_p_ppc__protocol__GrpcConfig", "ppc::protocol::GrpcConfig *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__GrpcServerConfig = {"_p_ppc__protocol__GrpcServerConfig", "ppc::protocol::GrpcServerConfig *", 0, 0, (void*)0, 0};
-static swig_type_info _swigt__p_ppc__protocol__INodeInfo__Ptr = {"_p_ppc__protocol__INodeInfo__Ptr", "ppc::protocol::INodeInfo::Ptr *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_ppc__protocol__INodeInfo = {"_p_ppc__protocol__INodeInfo", "ppc::protocol::INodeInfo *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__Message = {"_p_ppc__protocol__Message", "ppc::protocol::Message *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__MessageBuilder = {"_p_ppc__protocol__MessageBuilder", "ppc::protocol::MessageBuilder *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_ppc__protocol__MessageHeader = {"_p_ppc__protocol__MessageHeader", "ppc::protocol::MessageHeader *", 0, 0, (void*)0, 0};
@@ -24222,6 +27230,7 @@ static swig_type_info _swigt__p_short = {"_p_short", "int16_t *|int_least16_t *|
 static swig_type_info _swigt__p_signed_char = {"_p_signed_char", "int8_t *|int_fast8_t *|int_least8_t *|signed char *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_size_type = {"_p_size_type", "size_type *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__allocatorT_signed_char_t = {"_p_std__allocatorT_signed_char_t", "std::vector< signed char >::allocator_type *|std::allocator< signed char > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t = {"_p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t", "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >::allocator_type *|std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__allocatorT_std__string_t = {"_p_std__allocatorT_std__string_t", "std::vector< std::string >::allocator_type *|std::allocator< std::string > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__allocatorT_unsigned_char_t = {"_p_std__allocatorT_unsigned_char_t", "std::vector< unsigned char >::allocator_type *|std::allocator< unsigned char > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__exception = {"_p_std__exception", "std::exception *", 0, 0, (void*)0, 0};
@@ -24240,9 +27249,11 @@ static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t = {"_p_std__shared_ptrT_ppc__front__IFrontClient_t", "std::shared_ptr< ppc::front::IFrontClient > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IFront_t = {"_p_std__shared_ptrT_ppc__front__IFront_t", "std::shared_ptr< ppc::front::IFront > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t = {"_p_std__shared_ptrT_ppc__front__IMessageHandler_t", "std::shared_ptr< ppc::front::IMessageHandler > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__INodeDiscovery_t = {"_p_std__shared_ptrT_ppc__front__INodeDiscovery_t", "std::shared_ptr< ppc::front::INodeDiscovery > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t = {"_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t", "std::shared_ptr< ppc::front::MessageDispatcherHandler > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t = {"_p_std__shared_ptrT_ppc__gateway__IGateway_t", "std::shared_ptr< ppc::gateway::IGateway > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t = {"_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t", "std::shared_ptr< ppc::protocol::GrpcConfig > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__INodeInfo_t = {"_p_std__shared_ptrT_ppc__protocol__INodeInfo_t", "std::shared_ptr< ppc::protocol::INodeInfo > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t", "std::shared_ptr< ppc::protocol::MessageBuilder > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t = {"_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t", "std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t = {"_p_std__shared_ptrT_ppc__protocol__MessageHeader_t", "std::shared_ptr< ppc::protocol::MessageHeader > *", 0, 0, (void*)0, 0};
@@ -24255,6 +27266,7 @@ static swig_type_info _swigt__p_std__shared_ptrT_ppc__sdk__Transport_t = {"_p_st
 static swig_type_info _swigt__p_std__string = {"_p_std__string", "std::string *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__string_view = {"_p_std__string_view", "std::string_view *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__vectorT_int8_t_t = {"_p_std__vectorT_int8_t_t", "std::vector< int8_t > *|std::vector< signed char,std::allocator< signed char > > *|std::vector< signed char > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t = {"_p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t", "std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > *|std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string,std::allocator< std::string > > *|std::vector< std::string > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_std__vectorT_uint8_t_t = {"_p_std__vectorT_uint8_t_t", "bcos::bytes *|std::vector< uint8_t > *|std::vector< unsigned char,std::allocator< unsigned char > > *|std::vector< unsigned char > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_swig__SwigPyIterator = {"_p_swig__SwigPyIterator", "swig::SwigPyIterator *", 0, 0, (void*)0, 0};
@@ -24288,13 +27300,14 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_ppc__front__IFrontBuilder,
   &_swigt__p_ppc__front__IFrontClient,
   &_swigt__p_ppc__front__IMessageHandler,
+  &_swigt__p_ppc__front__INodeDiscovery,
   &_swigt__p_ppc__front__MessageDispatcherHandler,
   &_swigt__p_ppc__front__SendResponseHandler,
   &_swigt__p_ppc__gateway__IGateway__Ptr,
   &_swigt__p_ppc__protocol__EndPoint,
   &_swigt__p_ppc__protocol__GrpcConfig,
   &_swigt__p_ppc__protocol__GrpcServerConfig,
-  &_swigt__p_ppc__protocol__INodeInfo__Ptr,
+  &_swigt__p_ppc__protocol__INodeInfo,
   &_swigt__p_ppc__protocol__Message,
   &_swigt__p_ppc__protocol__MessageBuilder,
   &_swigt__p_ppc__protocol__MessageHeader,
@@ -24310,6 +27323,7 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_signed_char,
   &_swigt__p_size_type,
   &_swigt__p_std__allocatorT_signed_char_t,
+  &_swigt__p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t,
   &_swigt__p_std__allocatorT_std__string_t,
   &_swigt__p_std__allocatorT_unsigned_char_t,
   &_swigt__p_std__exception,
@@ -24328,9 +27342,11 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t,
   &_swigt__p_std__shared_ptrT_ppc__front__IFront_t,
   &_swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t,
+  &_swigt__p_std__shared_ptrT_ppc__front__INodeDiscovery_t,
   &_swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,
   &_swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t,
   &_swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t,
+  &_swigt__p_std__shared_ptrT_ppc__protocol__INodeInfo_t,
   &_swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t,
   &_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t,
   &_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t,
@@ -24343,6 +27359,7 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_std__string,
   &_swigt__p_std__string_view,
   &_swigt__p_std__vectorT_int8_t_t,
+  &_swigt__p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t,
   &_swigt__p_std__vectorT_std__string_t,
   &_swigt__p_std__vectorT_uint8_t_t,
   &_swigt__p_swig__SwigPyIterator,
@@ -24376,13 +27393,14 @@ static swig_cast_info _swigc__p_ppc__front__IFront[] = {  {&_swigt__p_ppc__front
 static swig_cast_info _swigc__p_ppc__front__IFrontBuilder[] = {  {&_swigt__p_ppc__front__IFrontBuilder, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__front__IFrontClient[] = {  {&_swigt__p_ppc__front__IFrontClient, 0, 0, 0},  {&_swigt__p_ppc__front__IFront, _p_ppc__front__IFrontTo_p_ppc__front__IFrontClient, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__front__IMessageHandler[] = {  {&_swigt__p_ppc__front__IMessageHandler, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_ppc__front__INodeDiscovery[] = {  {&_swigt__p_ppc__front__INodeDiscovery, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__front__MessageDispatcherHandler[] = {  {&_swigt__p_ppc__front__MessageDispatcherHandler, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__front__SendResponseHandler[] = {  {&_swigt__p_ppc__front__SendResponseHandler, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__gateway__IGateway__Ptr[] = {  {&_swigt__p_ppc__gateway__IGateway__Ptr, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__EndPoint[] = {  {&_swigt__p_ppc__protocol__EndPoint, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__GrpcConfig[] = {  {&_swigt__p_ppc__protocol__GrpcConfig, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__GrpcServerConfig[] = {  {&_swigt__p_ppc__protocol__GrpcServerConfig, 0, 0, 0},{0, 0, 0, 0}};
-static swig_cast_info _swigc__p_ppc__protocol__INodeInfo__Ptr[] = {  {&_swigt__p_ppc__protocol__INodeInfo__Ptr, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_ppc__protocol__INodeInfo[] = {  {&_swigt__p_ppc__protocol__INodeInfo, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__Message[] = {  {&_swigt__p_ppc__protocol__Message, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__MessageBuilder[] = {  {&_swigt__p_ppc__protocol__MessageBuilder, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_ppc__protocol__MessageHeader[] = {  {&_swigt__p_ppc__protocol__MessageHeader, 0, 0, 0},{0, 0, 0, 0}};
@@ -24398,6 +27416,7 @@ static swig_cast_info _swigc__p_short[] = {  {&_swigt__p_short, 0, 0, 0},{0, 0,
 static swig_cast_info _swigc__p_signed_char[] = {  {&_swigt__p_signed_char, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_size_type[] = {  {&_swigt__p_size_type, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__allocatorT_signed_char_t[] = {  {&_swigt__p_std__allocatorT_signed_char_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t[] = {  {&_swigt__p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__allocatorT_std__string_t[] = {  {&_swigt__p_std__allocatorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__allocatorT_unsigned_char_t[] = {  {&_swigt__p_std__allocatorT_unsigned_char_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__exception[] = {  {&_swigt__p_std__exception, 0, 0, 0},{0, 0, 0, 0}};
@@ -24416,9 +27435,11 @@ static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__GetPeersInfoHandler
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IFrontClient_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__front__IFrontClient_t, 0, 0, 0},  {&_swigt__p_std__shared_ptrT_ppc__front__IFront_t, _p_std__shared_ptrT_ppc__front__IFront_tTo_p_std__shared_ptrT_ppc__front__IFrontClient_t, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IFront_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__front__IFront_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__IMessageHandler_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__front__IMessageHandler_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__INodeDiscovery_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__front__INodeDiscovery_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__gateway__IGateway_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__gateway__IGateway_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__INodeInfo_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__protocol__INodeInfo_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeader_t[] = {  {&_swigt__p_std__shared_ptrT_ppc__protocol__MessageHeader_t, 0, 0, 0},{0, 0, 0, 0}};
@@ -24431,6 +27452,7 @@ static swig_cast_info _swigc__p_std__shared_ptrT_ppc__sdk__Transport_t[] = {  {&
 static swig_cast_info _swigc__p_std__string[] = {  {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__string_view[] = {  {&_swigt__p_std__string_view, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__vectorT_int8_t_t[] = {  {&_swigt__p_std__vectorT_int8_t_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t[] = {  {&_swigt__p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = {  {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_std__vectorT_uint8_t_t[] = {  {&_swigt__p_std__vectorT_uint8_t_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_swig__SwigPyIterator[] = {  {&_swigt__p_swig__SwigPyIterator, 0, 0, 0},{0, 0, 0, 0}};
@@ -24464,13 +27486,14 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_ppc__front__IFrontBuilder,
   _swigc__p_ppc__front__IFrontClient,
   _swigc__p_ppc__front__IMessageHandler,
+  _swigc__p_ppc__front__INodeDiscovery,
   _swigc__p_ppc__front__MessageDispatcherHandler,
   _swigc__p_ppc__front__SendResponseHandler,
   _swigc__p_ppc__gateway__IGateway__Ptr,
   _swigc__p_ppc__protocol__EndPoint,
   _swigc__p_ppc__protocol__GrpcConfig,
   _swigc__p_ppc__protocol__GrpcServerConfig,
-  _swigc__p_ppc__protocol__INodeInfo__Ptr,
+  _swigc__p_ppc__protocol__INodeInfo,
   _swigc__p_ppc__protocol__Message,
   _swigc__p_ppc__protocol__MessageBuilder,
   _swigc__p_ppc__protocol__MessageHeader,
@@ -24486,6 +27509,7 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_signed_char,
   _swigc__p_size_type,
   _swigc__p_std__allocatorT_signed_char_t,
+  _swigc__p_std__allocatorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t,
   _swigc__p_std__allocatorT_std__string_t,
   _swigc__p_std__allocatorT_unsigned_char_t,
   _swigc__p_std__exception,
@@ -24504,9 +27528,11 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_std__shared_ptrT_ppc__front__IFrontClient_t,
   _swigc__p_std__shared_ptrT_ppc__front__IFront_t,
   _swigc__p_std__shared_ptrT_ppc__front__IMessageHandler_t,
+  _swigc__p_std__shared_ptrT_ppc__front__INodeDiscovery_t,
   _swigc__p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t,
   _swigc__p_std__shared_ptrT_ppc__gateway__IGateway_t,
   _swigc__p_std__shared_ptrT_ppc__protocol__GrpcConfig_t,
+  _swigc__p_std__shared_ptrT_ppc__protocol__INodeInfo_t,
   _swigc__p_std__shared_ptrT_ppc__protocol__MessageBuilder_t,
   _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t,
   _swigc__p_std__shared_ptrT_ppc__protocol__MessageHeader_t,
@@ -24519,6 +27545,7 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_std__string,
   _swigc__p_std__string_view,
   _swigc__p_std__vectorT_int8_t_t,
+  _swigc__p_std__vectorT_std__shared_ptrT_ppc__protocol__INodeInfo_t_t,
   _swigc__p_std__vectorT_std__string_t,
   _swigc__p_std__vectorT_uint8_t_t,
   _swigc__p_swig__SwigPyIterator,
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index e6e082c9..c2f4ac9e 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -18,6 +18,8 @@ from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 // shared_ptr definition
 %shared_ptr(ppc::front::FrontConfig);
 %shared_ptr(ppc::front::IFront);
+%shared_ptr(ppc::front::INodeDiscovery);
+%shared_ptr(ppc::protocol::INodeInfo);
 %shared_ptr(ppc::front::IFrontClient);
 // the callbacks
 %shared_ptr(ppc::front::ErrorCallback);
@@ -50,6 +52,8 @@ from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 #include "wedpr-transport/sdk/src/Transport.h"
 #include "ppc-framework/libwrapper/Buffer.h"
 #include "ppc-framework/front/IFront.h"
+#include "ppc-framework/protocol/INodeInfo.h"
+#include "ppc-framework/front/INodeDiscovery.h"
 #include "ppc-framework/protocol/RouteType.h"
 #include "ppc-framework/front/FrontConfig.h"
 #include "ppc-framework/protocol/GrpcConfig.h"
@@ -79,11 +83,13 @@ namespace ppc::protocol{
     class EndPoint;
     class GrpcConfig;
     class RouteType;
+    class INodeInfo;
 }
 
 namespace ppc::front{
     class FrontConfig;
     class IFront;
+    class INodeDiscovery;
     class IFrontClient;
     class FrontImpl;
     class FrontBuilderImpl;
@@ -112,6 +118,8 @@ namespace bcos{
 %template(SharedGrpcConfig) std::shared_ptr;
 
 %template(SharedFront) std::shared_ptr;
+%template(SharedNodeDiscovery) std::shared_ptr;
+%template(SharedNodeInfo) std::shared_ptr;
 %template(SharedFrontClient) std::shared_ptr;
 
 %template(SharedErrorCallback) std::shared_ptr;
@@ -134,6 +142,7 @@ namespace bcos{
 %template(ubytes) std::vector;
 %template(ibytes) std::vector;
 %template(StringVec) std::vector;
+%template(NodeInfoVec) std::vector>;
 
 %include 
 %pybuffer_binary(char* data, uint64_t length)
@@ -171,7 +180,18 @@ namespace bcos{
 %ignore ppc::protocol::MessagePayload::setData;
 %ignore ppc::protocol::MessagePayload::setDataPtr;
 %ignore ppc::protocol::MessagePayload::dataPtr;
-
+%ignore ppc::front::INodeDiscovery::start;
+%ignore ppc::front::INodeDiscovery::stop;
+%ignore ppc::protocol::INodeInfo::INodeInfo;
+%ignore ppc::protocol::INodeInfo::setFront;
+%ignore ppc::protocol::INodeInfo::getFront;
+%ignore ppc::protocol::INodeInfo::components;
+%ignore ppc::protocol::INodeInfo::encode;
+%ignore ppc::protocol::INodeInfo::decode;
+%ignore ppc::protocol::INodeInfo::equal;
+%ignore ppc::protocol::INodeInfo::toJson;
+%ignore ppc::protocol::INodeInfo::setComponents;
+%ignore ppc::protocol::INodeInfoFactory;
 /*
 ///// tests  ///
 %inline {
@@ -189,6 +209,8 @@ namespace bcos{
 %include "ppc-framework/protocol/MessagePayload.h"
 
 %include "ppc-framework/front/IFront.h"
+%include "ppc-framework/front/INodeDiscovery.h"
 
 %include "wedpr-transport/sdk/src/TransportBuilder.h"
-%include "wedpr-transport/sdk/src/Transport.h"
\ No newline at end of file
+%include "wedpr-transport/sdk/src/Transport.h"
+%include "ppc-framework/protocol/INodeInfo.h"
\ No newline at end of file
diff --git a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
index 3f401309..c8853eca 100644
--- a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
+++ b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
@@ -37,10 +37,11 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keep
     m_server = std::make_shared(grpcServerConfig);
 
     FrontFactory frontFactory;
-    m_gateway =
-        std::make_shared(m_config->grpcConfig(), m_config->gatewayGrpcTarget());
-    m_front = frontFactory.build(std::make_shared(), m_msgPayloadBuilder,
-        m_routeInfoBuilder, m_gateway, m_config);
+    auto nodeInfoFactory = std::make_shared();
+    m_gateway = std::make_shared(
+        m_config->grpcConfig(), m_config->gatewayGrpcTarget(), nodeInfoFactory);
+    m_front = frontFactory.build(
+        nodeInfoFactory, m_msgPayloadBuilder, m_routeInfoBuilder, m_gateway, m_config);
 
     m_frontService = std::make_shared(m_msgBuilder, m_front);
     // register the frontService
diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py
index 3b629552..7ecf31ae 100644
--- a/python/ppc_model/common/initializer.py
+++ b/python/ppc_model/common/initializer.py
@@ -19,6 +19,8 @@
 
 
 class Initializer:
+    MODEL_SERVICE_NAME = "MODEL"
+
     def __init__(self, log_config_path, config_path, plot_lock=None):
         self.log_config_path = log_config_path
         logging.config.fileConfig(self.log_config_path)
@@ -85,9 +87,17 @@ def init_transport(self, task_manager: TaskManager,
                        send_msg_timeout_ms: int,
                        pop_msg_timeout_ms: int):
         # create the transport
-        transport = TransportLoader.build(**self.config_data)
+        transport_config = TransportLoader.build_config(**self.config_data)
+        listen_port = self.config_data["HTTP_PORT"]
+        access_entrypoint = f"{transport_config.get_self_endpoint().host()}:{listen_port}"
+        if not access_entrypoint.startswith("http://"):
+            access_entrypoint = f"http://{access_entrypoint}"
+        # register the access_entrypoint information
+        transport_config.register_service_info(
+            Initializer.MODEL_SERVICE_NAME, access_entrypoint)
+        transport = TransportLoader.load(transport_config)
         self.logger(
-            f"Create transport success, config: {transport.get_config().desc()}")
+            f"Create transport success, config: {transport.get_config().desc()}, access_entrypoint: {access_entrypoint}")
         # start the transport
         transport.start()
         self.logger().info(
diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py
index 8db479e4..f4282f40 100644
--- a/python/ppc_model/secure_model_base/secure_model_context.py
+++ b/python/ppc_model/secure_model_base/secure_model_context.py
@@ -144,18 +144,18 @@ def __init__(self,
             self.remote_psi_path = args['psi_result_path']
         if self.remote_psi_path is None:
             raise f"Must define the psi_result_path"
+        self.model_params = self.create_model_param()
+        self.reset_model_params(ModelSetting(args['model_dict']))
         # prepare the dataset and psi file
         BaseContext.load_file(storage_client=self.components.storage_client,
                               remote_path=self.remote_dataset_path,
                               local_path=self.dataset_file_path,
                               logger=self.components.logger())
-        BaseContext.load_file(storage_client=self.components.storage_client,
-                              remote_path=self.remote_psi_path,
-                              local_path=self.psi_result_path,
-                              logger=self.components.logger())
-
-        self.model_params = self.create_model_param()
-        self.reset_model_params(ModelSetting(args['model_dict']))
+        if self.model_params.use_psi:
+            BaseContext.load_file(storage_client=self.components.storage_client,
+                                  remote_path=self.remote_psi_path,
+                                  local_path=self.psi_result_path,
+                                  logger=self.components.logger())
         self.sync_file_list = {}
         if self.algorithm_type == AlgorithmType.Train.name:
             self.set_sync_file()

From 4f11c7c83e32dc3e1ecaf3673a03dcbe84a5a288 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 8 Nov 2024 18:16:37 +0800
Subject: [PATCH 083/120] fix grpc send large package failed for over package
 size limit (#82)

* fix grpc send large package failed for over package size limit

* sdk support set grpc related options
---
 cpp/ppc-framework/protocol/GrpcConfig.h       |  56 ++++
 cpp/wedpr-protocol/grpc/Common.h              |   8 +
 .../wedpr/sdk/jni/generated/GrpcConfig.java   |  26 ++
 .../jni/generated/wedpr_java_transport.java   |   4 +
 .../generated/wedpr_java_transportJNI.java    |  19 ++
 .../src/wedpr_java_transportJAVA_wrap.cxx     | 202 +++++++++++++
 .../generated/wedpr_python_transport.py       |  22 ++
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 275 ++++++++++++++++++
 8 files changed, 612 insertions(+)

diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h
index c0f2704e..4c570877 100644
--- a/cpp/ppc-framework/protocol/GrpcConfig.h
+++ b/cpp/ppc-framework/protocol/GrpcConfig.h
@@ -18,8 +18,10 @@
  * @date 2024-09-02
  */
 #pragma once
+#include "ppc-framework/Common.h"
 #include "ppc-framework/protocol/EndPoint.h"
 #include 
+#include 
 #include 
 
 namespace ppc::protocol
@@ -64,9 +66,63 @@ class GrpcConfig
 
     bool enableDnslookup() const { return m_enableDnslookup; }
 
+    uint64_t maxSendMessageSize() const { return m_maxSendMessageSize; }
+    uint64_t maxReceivedMessageSize() const { return m_maxReceivedMessageSize; }
+
+    void setMaxSendMessageSize(uint64_t maxSendMessageSize)
+    {
+        m_maxSendMessageSize = maxSendMessageSize;
+    }
+    void setMaxReceivedMessageSize(uint64_t maxReceivedMessageSize)
+    {
+        m_maxReceivedMessageSize = maxReceivedMessageSize;
+    }
+
+    /*
+    typedef enum {
+    GRPC_COMPRESS_NONE = 0,
+    GRPC_COMPRESS_DEFLATE,
+    GRPC_COMPRESS_GZIP,
+    GRPC_COMPRESS_ALGORITHMS_COUNT
+    } grpc_compression_algorithm;
+    */
+    int compressAlgorithm() const { return m_compressAlgorithm; }
+
+    void setCompressAlgorithm(int compressAlgorithm)
+    {
+        if (compressAlgorithm < 0 || compressAlgorithm > 2)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid compress algorithm, must between 0-3"));
+        }
+        m_compressAlgorithm = compressAlgorithm;
+    }
+
 protected:
     bool m_enableHealthCheck = true;
     std::string m_loadBalancePolicy = "round_robin";
     bool m_enableDnslookup = false;
+
+    // the max send message size in bytes
+    uint64_t m_maxSendMessageSize = 1024 * 1024 * 1024;
+    // the max received message size in bytes
+    uint16_t m_maxReceivedMessageSize = 1024 * 1024 * 1024;
+    int m_compressAlgorithm = 0;
 };
+
+inline std::string printGrpcConfig(ppc::protocol::GrpcConfig::Ptr const& grpcConfig)
+{
+    if (!grpcConfig)
+    {
+        return "nullptr";
+    }
+    std::ostringstream stringstream;
+    stringstream << LOG_KV("loadBalancePolicy", grpcConfig->loadBalancePolicy())
+                 << LOG_KV("enableHealthCheck", grpcConfig->enableHealthCheck())
+                 << LOG_KV("enableDnslookup", grpcConfig->enableDnslookup())
+                 << LOG_KV("maxSendMessageSize", grpcConfig->maxSendMessageSize())
+                 << LOG_KV("maxReceivedMessageSize", grpcConfig->maxReceivedMessageSize())
+                 << LOG_KV("compressAlgorithm", grpcConfig->compressAlgorithm());
+    return stringstream.str();
+}
 }  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/grpc/Common.h b/cpp/wedpr-protocol/grpc/Common.h
index b38c7e52..fe4ad928 100644
--- a/cpp/wedpr-protocol/grpc/Common.h
+++ b/cpp/wedpr-protocol/grpc/Common.h
@@ -20,10 +20,13 @@
 #pragma once
 #include "ppc-framework/Common.h"
 #include "ppc-framework/protocol/GrpcConfig.h"
+#include 
 #include 
 
 namespace ppc::protocol
 {
+#define GRPC_LOG(LEVEL) BCOS_LOG(LEVEL) << "[GRPC]"
+
 inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr const& grpcConfig)
 {
     grpc::ChannelArguments args;
@@ -47,6 +50,11 @@ inline grpc::ChannelArguments toChannelConfig(ppc::protocol::GrpcConfig::Ptr con
     {
         args.SetInt("grpc.enable_dns_srv_lookup", 1);
     }
+    args.SetMaxReceiveMessageSize(grpcConfig->maxReceivedMessageSize());
+    args.SetMaxSendMessageSize(grpcConfig->maxSendMessageSize());
+    // the compress algorithm
+    args.SetCompressionAlgorithm((grpc_compression_algorithm)(grpcConfig->compressAlgorithm()));
+    GRPC_LOG(INFO) << LOG_DESC("toChannelConfig") << printGrpcConfig(grpcConfig);
     return args;
 }
 }  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
index 35d8e79d..9668c795 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
@@ -67,4 +67,30 @@ public void setEnableDnslookup(boolean enableDnslookup) {
     public boolean enableDnslookup() {
         return wedpr_java_transportJNI.GrpcConfig_enableDnslookup(swigCPtr, this);
     }
+
+    public java.math.BigInteger maxSendMessageSize() {
+        return wedpr_java_transportJNI.GrpcConfig_maxSendMessageSize(swigCPtr, this);
+    }
+
+    public java.math.BigInteger maxReceivedMessageSize() {
+        return wedpr_java_transportJNI.GrpcConfig_maxReceivedMessageSize(swigCPtr, this);
+    }
+
+    public void setMaxSendMessageSize(java.math.BigInteger maxSendMessageSize) {
+        wedpr_java_transportJNI.GrpcConfig_setMaxSendMessageSize(
+                swigCPtr, this, maxSendMessageSize);
+    }
+
+    public void setMaxReceivedMessageSize(java.math.BigInteger maxReceivedMessageSize) {
+        wedpr_java_transportJNI.GrpcConfig_setMaxReceivedMessageSize(
+                swigCPtr, this, maxReceivedMessageSize);
+    }
+
+    public int compressAlgorithm() {
+        return wedpr_java_transportJNI.GrpcConfig_compressAlgorithm(swigCPtr, this);
+    }
+
+    public void setCompressAlgorithm(int compressAlgorithm) {
+        wedpr_java_transportJNI.GrpcConfig_setCompressAlgorithm(swigCPtr, this, compressAlgorithm);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
index 802d61d5..5143b6ce 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transport.java
@@ -13,6 +13,10 @@ public static String printFrontDesc(FrontConfig config) {
         return wedpr_java_transportJNI.printFrontDesc(FrontConfig.getCPtr(config), config);
     }
 
+    public static String printGrpcConfig(GrpcConfig grpcConfig) {
+        return wedpr_java_transportJNI.printGrpcConfig(GrpcConfig.getCPtr(grpcConfig), grpcConfig);
+    }
+
     public static String printOptionalField(MessageOptionalHeader optionalHeader) {
         return wedpr_java_transportJNI.printOptionalField(
                 MessageOptionalHeader.getCPtr(optionalHeader), optionalHeader);
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index 371a44b1..834f8348 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -404,6 +404,25 @@ public static final native void GrpcConfig_setEnableDnslookup(
 
     public static final native boolean GrpcConfig_enableDnslookup(long jarg1, GrpcConfig jarg1_);
 
+    public static final native java.math.BigInteger GrpcConfig_maxSendMessageSize(
+            long jarg1, GrpcConfig jarg1_);
+
+    public static final native java.math.BigInteger GrpcConfig_maxReceivedMessageSize(
+            long jarg1, GrpcConfig jarg1_);
+
+    public static final native void GrpcConfig_setMaxSendMessageSize(
+            long jarg1, GrpcConfig jarg1_, java.math.BigInteger jarg2);
+
+    public static final native void GrpcConfig_setMaxReceivedMessageSize(
+            long jarg1, GrpcConfig jarg1_, java.math.BigInteger jarg2);
+
+    public static final native int GrpcConfig_compressAlgorithm(long jarg1, GrpcConfig jarg1_);
+
+    public static final native void GrpcConfig_setCompressAlgorithm(
+            long jarg1, GrpcConfig jarg1_, int jarg2);
+
+    public static final native String printGrpcConfig(long jarg1, GrpcConfig jarg1_);
+
     public static final native void delete_MessageOptionalHeader(long jarg1);
 
     public static final native void MessageOptionalHeader_encode(
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index f5e146b2..2c4bf107 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -4453,6 +4453,208 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
 }
 
 
+SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jobject jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  uint64_t result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+  {
+    jbyteArray ba = jenv->NewByteArray(9);
+    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
+    jclass clazz = jenv->FindClass("java/math/BigInteger");
+    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
+    jobject bigint;
+    int i;
+    
+    bae[0] = 0;
+    for(i=1; i<9; i++ ) {
+      bae[i] = (jbyte)(result>>8*(8-i));
+    }
+    
+    jenv->ReleaseByteArrayElements(ba, bae, 0);
+    bigint = jenv->NewObject(clazz, mid, ba);
+    jenv->DeleteLocalRef(ba);
+    jresult = bigint;
+  }
+  return jresult;
+}
+
+
+SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jobject jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  uint64_t result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+  {
+    jbyteArray ba = jenv->NewByteArray(9);
+    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
+    jclass clazz = jenv->FindClass("java/math/BigInteger");
+    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
+    jobject bigint;
+    int i;
+    
+    bae[0] = 0;
+    for(i=1; i<9; i++ ) {
+      bae[i] = (jbyte)(result>>8*(8-i));
+    }
+    
+    jenv->ReleaseByteArrayElements(ba, bae, 0);
+    bigint = jenv->NewObject(clazz, mid, ba);
+    jenv->DeleteLocalRef(ba);
+    jresult = bigint;
+  }
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    jclass clazz;
+    jmethodID mid;
+    jbyteArray ba;
+    jbyte* bae;
+    jsize sz;
+    int i;
+    
+    if (!jarg2) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
+      return ;
+    }
+    clazz = jenv->GetObjectClass(jarg2);
+    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
+    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
+    bae = jenv->GetByteArrayElements(ba, 0);
+    sz = jenv->GetArrayLength(ba);
+    arg2 = 0;
+    if (sz > 0) {
+      arg2 = (uint64_t)(signed char)bae[0];
+      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
+  }
+  (arg1)->setMaxSendMessageSize(arg2);
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    jclass clazz;
+    jmethodID mid;
+    jbyteArray ba;
+    jbyte* bae;
+    jsize sz;
+    int i;
+    
+    if (!jarg2) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
+      return ;
+    }
+    clazz = jenv->GetObjectClass(jarg2);
+    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
+    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
+    bae = jenv->GetByteArrayElements(ba, 0);
+    sz = jenv->GetArrayLength(ba);
+    arg2 = 0;
+    if (sz > 0) {
+      arg2 = (uint64_t)(signed char)bae[0];
+      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
+  }
+  (arg1)->setMaxReceivedMessageSize(arg2);
+}
+
+
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1compressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  int result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+  jresult = (jint)result; 
+  return jresult;
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setCompressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  int arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  arg2 = (int)jarg2; 
+  (arg1)->setCompressAlgorithm(arg2);
+}
+
+
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_printGrpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::GrpcConfig::Ptr *arg1 = 0 ;
+  ppc::protocol::GrpcConfig::Ptr tempnull1 ;
+  std::string result;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = jarg1 ? *(ppc::protocol::GrpcConfig::Ptr **)&jarg1 : &tempnull1; 
+  result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
+  return jresult;
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageOptionalHeader(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
   std::shared_ptr< ppc::protocol::MessageOptionalHeader > *smartarg1 = 0 ;
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 2ec2e4c0..7ba209fe 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1136,11 +1136,33 @@ def setEnableDnslookup(self, enableDnslookup):
     def enableDnslookup(self):
         return _wedpr_python_transport.GrpcConfig_enableDnslookup(self)
 
+    def maxSendMessageSize(self):
+        return _wedpr_python_transport.GrpcConfig_maxSendMessageSize(self)
+
+    def maxReceivedMessageSize(self):
+        return _wedpr_python_transport.GrpcConfig_maxReceivedMessageSize(self)
+
+    def setMaxSendMessageSize(self, maxSendMessageSize):
+        return _wedpr_python_transport.GrpcConfig_setMaxSendMessageSize(self, maxSendMessageSize)
+
+    def setMaxReceivedMessageSize(self, maxReceivedMessageSize):
+        return _wedpr_python_transport.GrpcConfig_setMaxReceivedMessageSize(self, maxReceivedMessageSize)
+
+    def compressAlgorithm(self):
+        return _wedpr_python_transport.GrpcConfig_compressAlgorithm(self)
+
+    def setCompressAlgorithm(self, compressAlgorithm):
+        return _wedpr_python_transport.GrpcConfig_setCompressAlgorithm(self, compressAlgorithm)
+
 
 # Register GrpcConfig in _wedpr_python_transport:
 _wedpr_python_transport.GrpcConfig_swigregister(GrpcConfig)
 
 
+def printGrpcConfig(grpcConfig):
+    return _wedpr_python_transport.printGrpcConfig(grpcConfig)
+
+
 class MessageOptionalHeader(object):
     thisown = property(lambda x: x.this.own(), lambda x,
                        v: x.this.own(v), doc="The membership flag")
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index 8b07830c..9b71c7d9 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -18950,6 +18950,240 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *
 }
 
 
+SWIGINTERN PyObject *_wrap_GrpcConfig_maxSendMessageSize(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint64_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcConfig_maxReceivedMessageSize(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  uint64_t result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxSendMessageSize(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxSendMessageSize", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
+  (arg1)->setMaxSendMessageSize(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxReceivedMessageSize(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxReceivedMessageSize", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
+  (arg1)->setMaxReceivedMessageSize(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcConfig_compressAlgorithm(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  int result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_compressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+  resultobj = SWIG_From_int(static_cast< int >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcConfig_setCompressAlgorithm(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  int arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  int val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
+  
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setCompressAlgorithm", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "2"" of type '" "int""'");
+  } 
+  arg2 = static_cast< int >(val2);
+  (arg1)->setCompressAlgorithm(arg2);
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@@ -18961,6 +19195,40 @@ SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObjec
   return SWIG_Python_InitShadowInstance(args);
 }
 
+SWIGINTERN PyObject *_wrap_printGrpcConfig(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig::Ptr *arg1 = 0 ;
+  void *argp1 ;
+  int res1 = 0 ;
+  ppc::protocol::GrpcConfig::Ptr tempshared1 ;
+  PyObject *swig_obj[1] ;
+  std::string result;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t,  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "printGrpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig::Ptr const &""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      if (argp1) tempshared1 = *reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp1);
+      delete reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp1);
+      arg1 = &tempshared1;
+    } else {
+      arg1 = (argp1) ? reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp1) : &tempshared1;
+    }
+  }
+  result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::MessageOptionalHeader *arg1 = (ppc::protocol::MessageOptionalHeader *) 0 ;
@@ -26958,8 +27226,15 @@ static PyMethodDef SwigMethods[] = {
 	 { "GrpcConfig_setEnableHealthCheck", _wrap_GrpcConfig_setEnableHealthCheck, METH_VARARGS, NULL},
 	 { "GrpcConfig_setEnableDnslookup", _wrap_GrpcConfig_setEnableDnslookup, METH_VARARGS, NULL},
 	 { "GrpcConfig_enableDnslookup", _wrap_GrpcConfig_enableDnslookup, METH_O, NULL},
+	 { "GrpcConfig_maxSendMessageSize", _wrap_GrpcConfig_maxSendMessageSize, METH_O, NULL},
+	 { "GrpcConfig_maxReceivedMessageSize", _wrap_GrpcConfig_maxReceivedMessageSize, METH_O, NULL},
+	 { "GrpcConfig_setMaxSendMessageSize", _wrap_GrpcConfig_setMaxSendMessageSize, METH_VARARGS, NULL},
+	 { "GrpcConfig_setMaxReceivedMessageSize", _wrap_GrpcConfig_setMaxReceivedMessageSize, METH_VARARGS, NULL},
+	 { "GrpcConfig_compressAlgorithm", _wrap_GrpcConfig_compressAlgorithm, METH_O, NULL},
+	 { "GrpcConfig_setCompressAlgorithm", _wrap_GrpcConfig_setCompressAlgorithm, METH_VARARGS, NULL},
 	 { "GrpcConfig_swigregister", GrpcConfig_swigregister, METH_O, NULL},
 	 { "GrpcConfig_swiginit", GrpcConfig_swiginit, METH_VARARGS, NULL},
+	 { "printGrpcConfig", _wrap_printGrpcConfig, METH_O, NULL},
 	 { "delete_MessageOptionalHeader", _wrap_delete_MessageOptionalHeader, METH_O, NULL},
 	 { "MessageOptionalHeader_encode", _wrap_MessageOptionalHeader_encode, METH_VARARGS, NULL},
 	 { "MessageOptionalHeader_decode", _wrap_MessageOptionalHeader_decode, METH_VARARGS, NULL},

From ba4c67bd8243a506bfdbde2ca09c61fc43607f31 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 8 Nov 2024 19:09:31 +0800
Subject: [PATCH 084/120] update requirements for wedpr_ml_toolkit (#83)

* fix classfier

* update requirements for wedpr_ml_toolkit
---
 .github/workflows/cpp_full_node_workflow.yml             | 1 +
 .github/workflows/cpp_sdk_workflow.yml                   | 3 ++-
 .github/workflows/cpp_toolkit_workflow.yml               | 3 ++-
 cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py | 1 -
 python/wedpr_ml_toolkit/requirements.txt                 | 1 +
 python/wedpr_ml_toolkit/setup.py                         | 1 -
 6 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 339a6922..d855aa3b 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -4,6 +4,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "python/**"
+      - "cpp/wedpr-transport/sdk-wrapper/python/bindings/**"
       - "Changelog.md"
       - "README.md"
   pull_request:
diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml
index f4540e7e..e75cc273 100644
--- a/.github/workflows/cpp_sdk_workflow.yml
+++ b/.github/workflows/cpp_sdk_workflow.yml
@@ -4,6 +4,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "python/**"
+      - "cpp/wedpr-transport/sdk-wrapper/python/bindings/**"
       - "Changelog.md"
       - "README.md"
   pull_request:
@@ -155,4 +156,4 @@ jobs:
             mkdir -p cpp/build
             cd cpp/build
             cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
-            cmake3 --build . --parallel 3
\ No newline at end of file
+            cmake3 --build . --parallel 3
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index f8056e66..8daf3b73 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -4,6 +4,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "python/**"
+      - "cpp/wedpr-transport/sdk-wrapper/python/bindings/**"
       - "Changelog.md"
       - "README.md"
   pull_request:
@@ -164,4 +165,4 @@ jobs:
         - uses: actions/upload-artifact@v3
           with:
             name: libwedpr_python_transport.so
-            path: ./cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/libwedpr_python_transport.so
\ No newline at end of file
+            path: ./cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/libs/libwedpr_python_transport.so
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
index 23423c02..55d37361 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/setup.py
@@ -39,7 +39,6 @@ def run(self):
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
         'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: Apache License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
     ],
diff --git a/python/wedpr_ml_toolkit/requirements.txt b/python/wedpr_ml_toolkit/requirements.txt
index b6bc7e3b..2f24d54b 100644
--- a/python/wedpr_ml_toolkit/requirements.txt
+++ b/python/wedpr_ml_toolkit/requirements.txt
@@ -1,4 +1,5 @@
 hdfs
 requests~=2.31.0
 requests_toolbelt==0.9.1
+wedpr-python-gateway-sdk
 
diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
index 0ec8449b..f456a389 100644
--- a/python/wedpr_ml_toolkit/setup.py
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -35,7 +35,6 @@ def run(self):
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
         'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: Apache License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
     ],

From 0e763011fd1b2a1b8a108ad10cd140befab011cf Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 11 Nov 2024 17:59:22 +0800
Subject: [PATCH 085/120] add transport initialize to support service discovery
 for mpc (#84)

* add transport initialize to support service discovery for mpc

* optimize log

* grpc disable port reuse
---
 cpp/ppc-framework/protocol/GrpcConfig.h       |  2 +-
 cpp/tools/build_wedpr_mpc.sh                  | 38 ++++++++++++-
 .../ppc-mpc/src/MPCService.cpp                |  1 +
 .../ppc-tools/src/config/PPCConfig.h          |  7 ++-
 cpp/wedpr-main/mpc-node/CMakeLists.txt        |  2 +-
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp    | 56 ++++++++++++++++---
 cpp/wedpr-main/mpc-node/MPCInitializer.h      | 18 +++++-
 cpp/wedpr-protocol/grpc/server/GrpcServer.cpp |  2 +
 .../ppc-front/ppc-front/FrontImpl.cpp         | 18 +++---
 .../ppc-front/ppc-front/NodeDiscovery.cpp     | 10 +++-
 .../ppc-front/ppc-front/NodeDiscovery.h       |  2 +-
 cpp/wedpr-transport/sdk/src/Transport.h       |  2 +-
 12 files changed, 131 insertions(+), 27 deletions(-)

diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h
index 4c570877..6b317fc8 100644
--- a/cpp/ppc-framework/protocol/GrpcConfig.h
+++ b/cpp/ppc-framework/protocol/GrpcConfig.h
@@ -106,7 +106,7 @@ class GrpcConfig
     // the max send message size in bytes
     uint64_t m_maxSendMessageSize = 1024 * 1024 * 1024;
     // the max received message size in bytes
-    uint16_t m_maxReceivedMessageSize = 1024 * 1024 * 1024;
+    uint64_t m_maxReceivedMessageSize = 1024 * 1024 * 1024;
     int m_compressAlgorithm = 0;
 };
 
diff --git a/cpp/tools/build_wedpr_mpc.sh b/cpp/tools/build_wedpr_mpc.sh
index 2195fe7c..938b9184 100644
--- a/cpp/tools/build_wedpr_mpc.sh
+++ b/cpp/tools/build_wedpr_mpc.sh
@@ -326,6 +326,9 @@ generate_config_ini() {
     local rpc_listen_port="${3}"
     local agency_info="${4}"
     local agency_id="${5}"
+    local grpc_listen_ip="${6}"
+    local grpc_listen_port="${7}"
+    local nodeid="${8}"
 
     cat <"${output}"
 [agency]
@@ -364,6 +367,19 @@ generate_config_ini() {
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 2000
 
+[transport]
+   ; the endpoint information
+   listen_ip = ${grpc_listen_ip}
+   listen_port = ${grpc_listen_port}
+   host_ip = 
+   ; the threadPoolSize
+   thread_count = 4
+   ; the gatewayService endpoint information
+   gateway_target =  
+   ; the components
+   components =
+   nodeid=${nodeid}
+
 [cert]
     ; directory the certificates located in
     cert_path=./conf
@@ -584,6 +600,22 @@ gen_rsa_node_cert() {
     LOG_INFO "Generate ${ndpath} cert successful!"
 }
 
+# we use sm_param to generate the private key
+generate_private_key() {
+    local output_path="${1}"
+    if [ ! -d "${output_path}" ]; then
+        mkdir -p ${output_path}
+    fi
+    if [ ! -f ${sm2_params} ]; then
+        generate_sm_sm2_param ${sm2_params}
+    fi
+    ${OPENSSL_CMD} genpkey -paramfile ${sm2_params} -out ${output_path}/node.pem 2>/dev/null
+    $OPENSSL_CMD ec -in "$output_path/node.pem" -text -noout 2> /dev/null | sed -n '3,5p' | sed 's/://g' | tr "\n" " " | sed 's/ //g'  | cat > "$output_path/node.privateKey"
+    ${OPENSSL_CMD} ec -text -noout -in "${output_path}/node.pem" 2>/dev/null | sed -n '7,11p' | tr -d ": \n" | awk '{print substr($0,3);}' | cat >"$output_path"/node.nodeid
+    private_key=$(cat $output_path/node.privateKey)
+    echo ${private_key}
+}
+
 gen_sm_node_cert_with_ext() {
     local capath="$1"
     local certpath="$2"
@@ -686,7 +718,11 @@ deploy_nodes()
     # generate the config.ini
     local rpc_port=5894
     local agency_id="agency${count}"
-    generate_config_ini "${output_dir}/config.ini"  "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id}
+    local grpc_port=18100
+    # the nodeid
+    private_key=$(generate_private_key "${output_dir}/conf")
+    node_id=$(cat "${output_dir}/conf/node.nodeid")
+    generate_config_ini "${output_dir}/config.ini"  "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${listen_ip}" "${grpc_port}" "${node_id}"
     print_result
 }
 
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index d1631677..e31115f8 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -420,6 +420,7 @@ void MPCService::execCommand(const std::string cmd, int& outExitStatus, std::str
     }
     catch (const std::exception& e)
     {
+        MPC_LOG(WARNING) << LOG_DESC("[MPCService] execCommand failed") << LOG_KV("cmd", cmd);
         BOOST_THROW_EXCEPTION(
             RunMpcFailException() << errinfo_comment(
                 "invalid params:" + std::string(boost::diagnostic_information(e))));
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index 85b993bd..27fa7b69 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -248,13 +248,14 @@ class PPCConfig
         return m_frontConfig->selfEndPoint().host() + ":" + std::to_string(m_rpcConfig.listenPort);
     }
 
-private:
-    virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp,
-        std::string const& sectionName, boost::property_tree::ptree const& pt);
     // load the front config
     virtual void loadFrontConfig(bool requireTransport,
         ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder,
         boost::property_tree::ptree const& pt);
+
+private:
+    virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp,
+        std::string const& sectionName, boost::property_tree::ptree const& pt);
     // load the grpc config
     ppc::protocol::GrpcConfig::Ptr loadGrpcConfig(
         std::string const& sectionName, boost::property_tree::ptree const& pt);
diff --git a/cpp/wedpr-main/mpc-node/CMakeLists.txt b/cpp/wedpr-main/mpc-node/CMakeLists.txt
index 9b752de2..fd3942db 100644
--- a/cpp/wedpr-main/mpc-node/CMakeLists.txt
+++ b/cpp/wedpr-main/mpc-node/CMakeLists.txt
@@ -1,4 +1,4 @@
 aux_source_directory(. SRC_LIST)
 
 add_executable(${MPC_BINARY_NAME} ${SRC_LIST})
-target_link_libraries(${MPC_BINARY_NAME} PUBLIC ${MPC_TARGET} ${RPC_TARGET} ${HELPER_TARGET} TBB::tbb)
+target_link_libraries(${MPC_BINARY_NAME} PUBLIC ${MPC_TARGET} ${RPC_TARGET} ${HELPER_TARGET} ${WEDPR_TRANSPORT_SDK_TARGET} TBB::tbb)
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index 31a720bd..e1fda6f0 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -18,15 +18,23 @@
  * @date 2023-03-24
  */
 #include "MPCInitializer.h"
+#include "ppc-framework/front/FrontConfig.h"
+#include "ppc-framework/protocol/ServiceType.h"
 #include "ppc-mpc/src/MPCService.h"
 #include "ppc-tools/src/config/PPCConfig.h"
+#include "wedpr-protocol/protocol/src/ServiceConfig.h"
+#include "wedpr-transport/sdk/src/TransportBuilder.h"
 
 using namespace ppc::rpc;
 using namespace bcos;
+using namespace ppc::sdk;
 using namespace ppc::mpc;
 using namespace ppc::tools;
+using namespace ppc::front;
+using namespace ppc::protocol;
+
+MPCInitializer::MPCInitializer() : m_transportBuilder(std::make_shared()) {}
 
-/// TODO: mpc support the gateway
 void MPCInitializer::init(std::string const& _configPath)
 {
     // init the log
@@ -40,14 +48,14 @@ void MPCInitializer::init(std::string const& _configPath)
     // init the rpc
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
     // load the rpc config
-    auto ppcConfig = std::make_shared();
-    ppcConfig->loadRpcConfig(pt);
-    ppcConfig->loadMPCConfig(pt);
+    m_config = std::make_shared();
+    m_config->loadRpcConfig(pt);
+    m_config->loadMPCConfig(pt);
     // bool useMysql = pt.get("mpc.use_mysql", false);
-    auto storageConfig = ppcConfig->storageConfig();
-    auto mpcConfig = ppcConfig->mpcConfig();
-    auto rpcFactory = std::make_shared(ppcConfig->agencyID());
-    m_rpc = rpcFactory->buildRpc(ppcConfig, nullptr);
+    auto storageConfig = m_config->storageConfig();
+    auto mpcConfig = m_config->mpcConfig();
+    auto rpcFactory = std::make_shared(m_config->agencyID());
+    m_rpc = rpcFactory->buildRpc(m_config, nullptr);
     auto mpcService = std::make_shared();
     mpcService->setMPCConfig(mpcConfig);
     mpcService->setStorageConfig(storageConfig);
@@ -55,13 +63,39 @@ void MPCInitializer::init(std::string const& _configPath)
                                       std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService,
                                        std::placeholders::_1, std::placeholders::_2));
+    INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success");
+    // init the transport
+    initTransport(pt);
+}
 
+void MPCInitializer::initTransport(boost::property_tree::ptree const& property)
+{
+    INIT_LOG(INFO) << LOG_DESC("initTransport: load front config");
+    m_config->loadFrontConfig(true, m_transportBuilder->frontConfigBuilder(), property);
+    INIT_LOG(INFO) << LOG_DESC("initTransport: load front config success");
 
-    INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success");
+    // add the service meta
+    ServiceConfigBuilder serviceConfigBuilder;
+    auto entryPoint =
+        serviceConfigBuilder.buildEntryPoint(MPC_SERVICE_TYPE, m_config->accessEntrypoint());
+    auto serviceConfig = serviceConfigBuilder.buildServiceConfig();
+    serviceConfig.addEntryPoint(entryPoint);
+    auto serviceMeta = serviceConfig.encode();
+    m_config->frontConfig()->setMeta(serviceMeta);
+    INIT_LOG(INFO) << LOG_DESC("initTransport: register serviceMeta")
+                   << LOG_KV("serviceMeta", serviceMeta);
+    INIT_LOG(INFO) << LOG_DESC("initTransport: buildProTransport");
+    m_transport = m_transportBuilder->buildProTransport(m_config->frontConfig());
+    INIT_LOG(INFO) << LOG_DESC("initTransport: buildProTransport success");
 }
 
 void MPCInitializer::start()
 {
+    // start the transport
+    if (m_transport)
+    {
+        m_transport->start();
+    }
     // start the ppc mpc
     if (m_rpc)
     {
@@ -75,4 +109,8 @@ void MPCInitializer::stop()
     {
         m_rpc->stop();
     }
+    if (m_transport)
+    {
+        m_transport->stop();
+    }
 }
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.h b/cpp/wedpr-main/mpc-node/MPCInitializer.h
index 4e0da526..7606dbce 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.h
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.h
@@ -26,20 +26,36 @@ namespace ppc::rpc
 {
 class Rpc;
 }
+namespace ppc::tools
+{
+class PPCConfig;
+}
+namespace ppc::sdk
+{
+class TransportBuilder;
+class Transport;
+};  // namespace ppc::sdk
 namespace ppc::mpc
 {
 class MPCInitializer
 {
 public:
     using Ptr = std::shared_ptr;
-    MPCInitializer() {}
+    MPCInitializer();
     virtual ~MPCInitializer() { stop(); }
 
     virtual void init(std::string const& _configPath);
     virtual void start();
     virtual void stop();
 
+protected:
+    virtual void initTransport(boost::property_tree::ptree const& property);
+
 private:
+    std::shared_ptr m_config;
+    std::shared_ptr m_transportBuilder;
+    std::shared_ptr m_transport;
+
     bcos::BoostLogInitializer::Ptr m_logInitializer;
     std::shared_ptr m_rpc;
 };
diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
index ee289ee1..fef0e383 100644
--- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
@@ -39,6 +39,8 @@ void GrpcServer::start()
     }
     grpc::reflection::InitProtoReflectionServerBuilderPlugin();
     grpc::ServerBuilder builder;
+    // disable port reuse
+    builder.AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0);
     // without authentication
     builder.AddListeningPort(m_config->listenEndPoint(), grpc::InsecureServerCredentials());
     // register the service
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index cb976b08..4ff6ed09 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -56,6 +56,10 @@ void FrontImpl::start()
         return;
     }
     m_running = true;
+    if (m_nodeDiscovery)
+    {
+        m_nodeDiscovery->start();
+    }
     m_thread = std::make_shared([&] {
         bcos::pthread_setThreadName("front_io_service");
         while (m_running)
@@ -77,10 +81,7 @@ void FrontImpl::start()
         }
         FRONT_LOG(INFO) << "Front exit";
     });
-    if (m_nodeDiscovery)
-    {
-        m_nodeDiscovery->start();
-    }
+    FRONT_LOG(INFO) << LOG_DESC("start front success");
 }
 
 
@@ -97,6 +98,10 @@ void FrontImpl::stop()
         return;
     }
     m_running = false;
+    if (m_nodeDiscovery)
+    {
+        m_nodeDiscovery->stop();
+    }
     if (m_ioService)
     {
         m_ioService->stop();
@@ -113,10 +118,7 @@ void FrontImpl::stop()
             m_thread->detach();
         }
     }
-    if (m_nodeDiscovery)
-    {
-        m_nodeDiscovery->stop();
-    }
+    FRONT_LOG(INFO) << LOG_DESC("stop front success");
 }
 
 void FrontImpl::asyncSendResponse(bcos::bytesConstRef dstNode, std::string const& traceID,
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
index b1627d79..c0d8e922 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.cpp
@@ -27,7 +27,6 @@ NodeDiscovery::NodeDiscovery(ppc::gateway::IGateway::Ptr gatewayClient)
 {
     // fetch the meta information every 10s
     m_metaFetcher = std::make_shared(10 * 1000, "metaFetcher");
-    m_metaFetcher->registerTimeoutHandler([this]() { fetchMetaInfoFromGateway(); });
 }
 
 std::vector NodeDiscovery::getAliveNodeList() const
@@ -38,6 +37,15 @@ std::vector NodeDiscovery::getAliveNodeList() con
 
 void NodeDiscovery::start()
 {
+    auto self = weak_from_this();
+    m_metaFetcher->registerTimeoutHandler([self]() {
+        auto fetcher = self.lock();
+        if (!fetcher)
+        {
+            return;
+        }
+        fetcher->fetchMetaInfoFromGateway();
+    });
     if (m_metaFetcher)
     {
         m_metaFetcher->start();
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
index 067460d4..712a109e 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/NodeDiscovery.h
@@ -25,7 +25,7 @@
 
 namespace ppc::front
 {
-class NodeDiscovery : public INodeDiscovery
+class NodeDiscovery : public INodeDiscovery, public std::enable_shared_from_this
 {
 public:
     using Ptr = std::shared_ptr;
diff --git a/cpp/wedpr-transport/sdk/src/Transport.h b/cpp/wedpr-transport/sdk/src/Transport.h
index c6175c91..a3fa1680 100644
--- a/cpp/wedpr-transport/sdk/src/Transport.h
+++ b/cpp/wedpr-transport/sdk/src/Transport.h
@@ -31,7 +31,7 @@ class Transport
 public:
     using Ptr = std::shared_ptr;
     Transport(ppc::front::FrontConfig::Ptr config);
-    virtual ~Transport() = default;
+    virtual ~Transport() { stop(); }
 
     virtual void start() { m_front->start(); }
     virtual void stop() { m_front->stop(); }

From e216139ddaf320f00f93dc40e32b89839e3c548b Mon Sep 17 00:00:00 2001
From: Xinyi YAN <41045439+yanxinyi620@users.noreply.github.com>
Date: Tue, 12 Nov 2024 18:25:58 +0800
Subject: [PATCH 086/120] update data context (#86)

* Update data_context.py
---
 .../wedpr_ml_toolkit/context/data_context.py         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
index 90b6119c..36bf5d52 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
@@ -28,9 +28,9 @@ def to_psi_format(self, merge_filed, result_receiver_id_list):
         dataset_psi = []
         for dataset in self.datasets:
             if dataset.agency in result_receiver_id_list:
-                result_receiver = "true"
+                result_receiver = True
             else:
-                result_receiver = "false"
+                result_receiver = False
             dataset_psi_info = {"idFields": [merge_filed],
                                 "dataset": {"owner": dataset.dataset_owner,
                                             "ownerAgency": dataset.agency,
@@ -45,13 +45,13 @@ def to_model_formort(self, merge_filed, result_receiver_id_list):
         dataset_model = []
         for dataset in self.datasets:
             if dataset.agency in result_receiver_id_list:
-                result_receiver = "true"
+                result_receiver = True
             else:
-                result_receiver = "false"
+                result_receiver = False
             if dataset.is_label_holder:
-                label_provider = "true"
+                label_provider = True
             else:
-                label_provider = "false"
+                label_provider = False
             dataset_psi_info = {"idFields": [merge_filed],
                                 "dataset": {"owner": dataset.dataset_owner,
                                             "ownerAgency": dataset.agency,

From 0c62b3fc184bc052c5eb08a7b55f772240990bd1 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 12 Nov 2024 19:59:31 +0800
Subject: [PATCH 087/120] fix grpc bind error caused program crash && add
 exception handler for java wrapper (#85)

* fix grpc bind error caused program crash

* add exception handler for java wrapper

* fix log caused exit coredump

* add exception handler for python wrapper

* update workflow

* fix gateway exist coredump

* optimize version print
---
 .github/workflows/cpp_full_node_workflow.yml  |    6 +-
 .github/workflows/cpp_toolkit_workflow.yml    |    7 +-
 cpp/cmake/CompilerSettings.cmake              |   43 +-
 cpp/cmake/ProjectTCMalloc.cmake               |    4 +-
 cpp/ppc-framework/protocol/GrpcConfig.h       |   76 +-
 cpp/wedpr-helper/libhelper/CommandHelper.cpp  |   38 +-
 cpp/wedpr-helper/libhelper/CommandHelper.h    |    5 +-
 cpp/wedpr-initializer/Initializer.cpp         |    4 -
 cpp/wedpr-main/air-node/main.cpp              |    9 +-
 cpp/wedpr-main/cem-node/main.cpp              |    6 +-
 cpp/wedpr-main/common/NodeStarter.h           |   16 +-
 cpp/wedpr-main/gateway/CMakeLists.txt         |    2 +-
 cpp/wedpr-main/gateway/main.cpp               |    7 +-
 cpp/wedpr-main/mpc-node/CMakeLists.txt        |    2 +-
 cpp/wedpr-main/mpc-node/main.cpp              |    6 +-
 cpp/wedpr-main/pro-node/main.cpp              |    7 +-
 cpp/wedpr-protocol/grpc/server/GrpcServer.cpp |   21 +-
 .../gateway/router/GatewayRouterManager.cpp   |   10 +-
 .../ppc-gateway/ppc-gateway/p2p/Service.cpp   |   11 +
 .../ppc-gateway/ppc-gateway/p2p/Service.h     |    2 +
 .../ppc-http/demo/http_demo.cpp               |    4 +-
 cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp |    4 +-
 .../bindings/conf/wedpr_sdk_log_config.ini    |    9 +
 .../sdk/jni/generated/TransportBuilder.java   |    4 +-
 .../generated/wedpr_java_transportJNI.java    |    3 +-
 .../sdk/jni/transport/TransportConfig.java    |    7 +-
 .../sdk/jni/transport/impl/TransportImpl.java |   33 +-
 .../src/wedpr_java_transportJAVA_wrap.cxx     | 2927 +++++++++++++++--
 .../java/swig/wedpr_java_transport.i          |   11 +-
 .../generated/wedpr_python_transport.py       |    5 +-
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 2629 +++++++++++++--
 .../python/swig/wedpr_python_transport.i      |   19 +-
 .../sdk/src/ProTransportImpl.cpp              |   44 +-
 .../sdk/src/ProTransportImpl.h                |    2 +-
 cpp/wedpr-transport/sdk/src/Transport.h       |   18 +-
 .../sdk/src/TransportBuilder.cpp              |   27 +-
 .../sdk/src/TransportBuilder.h                |   26 +-
 37 files changed, 5227 insertions(+), 827 deletions(-)
 create mode 100644 cpp/wedpr-transport/sdk-wrapper/java/bindings/conf/wedpr_sdk_log_config.ini

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index d855aa3b..3158cbc9 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -66,13 +66,13 @@ jobs:
           export GCC='gcc-10'
           export CXX='g++-10'
           sudo bash -x cpp/tools/install_depends.sh -o ubuntu
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Build for macOS
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
-          mkdir -p cpp/build && cd cpp/build && cmake -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - uses: actions/upload-artifact@v4
         if: runner.os == 'macOS'      
@@ -163,7 +163,7 @@ jobs:
           rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
-          cmake3 -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+          cmake3 -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
       - name: FreeDiskSpace
         run: |
           df -lh
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index 8daf3b73..267d8c51 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -40,6 +40,11 @@ jobs:
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Prepare python
+        if: runner.os == 'Windows'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.8'
       - name: Prepare vcpkg
         if: runner.os != 'Windows'
         uses: friendlyanon/setup-vcpkg@v1
@@ -65,7 +70,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=OFF -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |
diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 9ae74f51..1d2efdc8 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -17,7 +17,6 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
     # set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "/usr/bin/time")
     # set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "/usr/bin/time")
     # Use ISO C++17 standard language.
-    set(CMAKE_CXX_FLAGS "-pthread -fPIC -fexceptions")
     # set(CMAKE_CXX_VISIBILITY_PRESET hidden)
     # Enables all the warnings about constructions that some users consider questionable,
     # and that are easy to avoid.  Also enable some extra warning flags that are not
@@ -75,11 +74,16 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
 
     option(USE_LD_GOLD "Use GNU gold linker" ON)
     if (USE_LD_GOLD)
-        execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
-        if ("${LD_VERSION}" MATCHES "GNU gold")
-            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
-            set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
-        endif ()
+        if("${LINKER}" MATCHES "gold")
+            execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=gold -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
+            if("${LD_VERSION}" MATCHES "GNU gold")
+                set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
+                set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=gold")
+            endif()
+        elseif("${LINKER}" MATCHES "mold")
+            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=mold")
+            set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=mold")
+        endif()
     endif ()
 
     # Additional GCC-specific compiler settings.
@@ -91,15 +95,32 @@ if (("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MA
         if (NOT (GCC_VERSION VERSION_GREATER ${GCC_MIN_VERSION} OR GCC_VERSION VERSION_EQUAL ${GCC_MIN_VERSION}))
             message(FATAL_ERROR "${PROJECT_NAME} requires g++ ${GCC_MIN_VERSION} or greater. Current is ${GCC_VERSION}")
         endif ()
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MARCH_TYPE}")
-        set(CMAKE_C_FLAGS "-std=c99 ${CMAKE_C_FLAGS} ${MARCH_TYPE}")
+        if(BUILD_STATIC)
+            # solve multiple definition of `__lll_lock_wait_private'
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MARCH_TYPE} -ftree-parallelize-loops=2 -flto")
+        else()
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MARCH_TYPE}")
+        endif()
+        set(CMAKE_C_FLAGS "-std=c99 -fexceptions ${CMAKE_C_FLAGS} ${MARCH_TYPE}")
 
 		# Strong stack protection was only added in GCC 4.9.
 		# Use it if we have the option to do so.
 		# See https://lwn.net/Articles/584225/
-        if (GCC_VERSION VERSION_GREATER 4.9 OR GCC_VERSION VERSION_EQUAL 4.9)
-            add_compile_options(-fstack-protector-strong)
-            add_compile_options(-fstack-protector)
+        add_compile_options(-fstack-protector-strong)
+        add_compile_options(-fstack-protector)
+
+        add_compile_options(-fPIC)
+        add_compile_options(-Wno-error=nonnull)
+        add_compile_options(-foptimize-sibling-calls)
+        add_compile_options(-Wno-stringop-overflow)
+        add_compile_options(-Wno-restrict)
+        add_compile_options(-Wno-error=format-truncation)
+
+        if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11.0)
+            add_compile_options(-Wno-stringop-overread)
+            add_compile_options(-Wno-maybe-uninitialized)
+            add_compile_options(-Wno-array-bounds)
+            add_compile_options(-Wno-aggressive-loop-optimizations)
         endif()
     # Additional Clang-specific compiler settings.
     elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
diff --git a/cpp/cmake/ProjectTCMalloc.cmake b/cpp/cmake/ProjectTCMalloc.cmake
index dcdb29c4..8abce463 100644
--- a/cpp/cmake/ProjectTCMalloc.cmake
+++ b/cpp/cmake/ProjectTCMalloc.cmake
@@ -28,13 +28,13 @@ ExternalProject_Add(gperftools
 )
 
 ExternalProject_Get_Property(gperftools SOURCE_DIR)
-add_library(TCMalloc INTERFACE IMPORTED)
+add_library(TCMalloc STATIC IMPORTED GLOBAL)
 
 set(TCMALLOC_INCLUDE_DIR ${SOURCE_DIR}/include/)
 set(TCMALLOC_LIBRARY ${SOURCE_DIR}/.libs/${TCMALLOC_LIB_NAME})
 file(MAKE_DIRECTORY ${TCMALLOC_INCLUDE_DIR})  # Must exist.
 
-set_property(TARGET TCMalloc PROPERTY INTERFACE_LINK_LIBRARIES ${TCMALLOC_LIBRARY})
+set_property(TARGET TCMalloc PROPERTY IMPORTED_LOCATION ${TCMALLOC_LIBRARY})
 set_property(TARGET TCMalloc PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${TCMALLOC_INCLUDE_DIR})
 
 add_dependencies(TCMalloc gperftools)
diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h
index 6b317fc8..c3a2655e 100644
--- a/cpp/ppc-framework/protocol/GrpcConfig.h
+++ b/cpp/ppc-framework/protocol/GrpcConfig.h
@@ -26,27 +26,6 @@
 
 namespace ppc::protocol
 {
-class GrpcServerConfig
-{
-public:
-    using Ptr = std::shared_ptr;
-    GrpcServerConfig() = default;
-    GrpcServerConfig(EndPoint endPoint, bool enableHealthCheck)
-      : m_endPoint(std::move(endPoint)), m_enableHealthCheck(enableHealthCheck)
-    {}
-    std::string listenEndPoint() const { return m_endPoint.listenEndPoint(); }
-
-    void setEndPoint(EndPoint endPoint) { m_endPoint = endPoint; }
-    void setEnableHealthCheck(bool enableHealthCheck) { m_enableHealthCheck = enableHealthCheck; }
-
-    EndPoint const& endPoint() const { return m_endPoint; }
-    EndPoint& mutableEndPoint() { return m_endPoint; }
-    bool enableHealthCheck() const { return m_enableHealthCheck; }
-
-protected:
-    ppc::protocol::EndPoint m_endPoint;
-    bool m_enableHealthCheck = true;
-};
 class GrpcConfig
 {
 public:
@@ -71,10 +50,22 @@ class GrpcConfig
 
     void setMaxSendMessageSize(uint64_t maxSendMessageSize)
     {
+        if (maxSendMessageSize > c_maxMsgSize)
+        {
+            BOOST_THROW_EXCEPTION(
+                WeDPRException() << bcos::errinfo_comment(
+                    "The MaxSendMessageSize limit is " + std::to_string(c_maxMsgSize)));
+        }
         m_maxSendMessageSize = maxSendMessageSize;
     }
     void setMaxReceivedMessageSize(uint64_t maxReceivedMessageSize)
     {
+        if (maxReceivedMessageSize > c_maxMsgSize)
+        {
+            BOOST_THROW_EXCEPTION(
+                WeDPRException() << bcos::errinfo_comment(
+                    "The MaxReceivedMessageSize limit is " + std::to_string(c_maxMsgSize)));
+        }
         m_maxReceivedMessageSize = maxReceivedMessageSize;
     }
 
@@ -102,14 +93,53 @@ class GrpcConfig
     bool m_enableHealthCheck = true;
     std::string m_loadBalancePolicy = "round_robin";
     bool m_enableDnslookup = false;
+    // Note: grpc use int to set the maxMsgSize
+    uint64_t const c_maxMsgSize = INT_MAX;
 
     // the max send message size in bytes
-    uint64_t m_maxSendMessageSize = 1024 * 1024 * 1024;
+    uint64_t m_maxSendMessageSize = c_maxMsgSize;
     // the max received message size in bytes
-    uint64_t m_maxReceivedMessageSize = 1024 * 1024 * 1024;
+    uint64_t m_maxReceivedMessageSize = c_maxMsgSize;
     int m_compressAlgorithm = 0;
 };
 
+class GrpcServerConfig : public GrpcConfig
+{
+public:
+    using Ptr = std::shared_ptr;
+    GrpcServerConfig() = default;
+    GrpcServerConfig(EndPoint endPoint, bool enableHealthCheck)
+      : m_endPoint(std::move(endPoint)), m_enableHealthCheck(enableHealthCheck)
+    {}
+    ~GrpcServerConfig() override = default;
+
+    std::string listenEndPoint() const { return m_endPoint.listenEndPoint(); }
+
+    void setEndPoint(EndPoint endPoint) { m_endPoint = endPoint; }
+    void setEnableHealthCheck(bool enableHealthCheck) { m_enableHealthCheck = enableHealthCheck; }
+
+    EndPoint const& endPoint() const { return m_endPoint; }
+    EndPoint& mutableEndPoint() { return m_endPoint; }
+    bool enableHealthCheck() const { return m_enableHealthCheck; }
+
+    uint64_t maxMsgSize() const { return m_maxMsgSize; }
+    void setMaxMsgSize(uint64_t maxMsgSize)
+    {
+        if (maxMsgSize > c_maxMsgSize)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "The maxMsgSize limit is " + std::to_string(c_maxMsgSize)));
+        }
+        m_maxMsgSize = maxMsgSize;
+    }
+
+protected:
+    ppc::protocol::EndPoint m_endPoint;
+    bool m_enableHealthCheck = true;
+    uint64_t m_maxMsgSize = c_maxMsgSize;
+};
+
+
 inline std::string printGrpcConfig(ppc::protocol::GrpcConfig::Ptr const& grpcConfig)
 {
     if (!grpcConfig)
diff --git a/cpp/wedpr-helper/libhelper/CommandHelper.cpp b/cpp/wedpr-helper/libhelper/CommandHelper.cpp
index 49c94fcd..b3b8939f 100644
--- a/cpp/wedpr-helper/libhelper/CommandHelper.cpp
+++ b/cpp/wedpr-helper/libhelper/CommandHelper.cpp
@@ -26,9 +26,9 @@
 
 using namespace ppc;
 
-void ppc::printVersion()
+void ppc::printVersion(std::string const& binaryName)
 {
-    std::cout << "PPCS-Core Version : " << PPC_PROJECT_VERSION << std::endl;
+    std::cout << binaryName << " Version : " << PPC_PROJECT_VERSION << std::endl;
     std::cout << "Build Time         : " << PPC_BUILD_TIME << std::endl;
     std::cout << "Build Type         : " << PPC_BUILD_PLATFORM << "/" << PPC_BUILD_TYPE
               << std::endl;
@@ -36,7 +36,7 @@ void ppc::printVersion()
     std::cout << "Git Commit         : " << PPC_COMMIT_HASH << std::endl;
 }
 
-CommandLineParam ppc::initCommandLine(int argc, const char* argv[])
+CommandLineParam ppc::initCommandLine(std::string const& binaryName, int argc, const char* argv[])
 {
     boost::program_options::options_description main_options("Usage of PPC");
     main_options.add_options()("help,h", "print help information")("version,v", "version of PPC")(
@@ -49,7 +49,7 @@ CommandLineParam ppc::initCommandLine(int argc, const char* argv[])
     }
     catch (...)
     {
-        printVersion();
+        printVersion(binaryName);
     }
     /// help information
     if (vm.count("help") || vm.count("h"))
@@ -60,7 +60,7 @@ CommandLineParam ppc::initCommandLine(int argc, const char* argv[])
     /// version information
     if (vm.count("version") || vm.count("v"))
     {
-        printVersion();
+        printVersion(binaryName);
         exit(0);
     }
     std::string configPath("./config.ini");
@@ -81,31 +81,3 @@ CommandLineParam ppc::initCommandLine(int argc, const char* argv[])
     }
     return ppc::CommandLineParam{configPath};
 }
-
-void ppc::initAppCommandLine(int argc, char* argv[])
-{
-    boost::program_options::options_description main_options("Usage of PPC");
-    main_options.add_options()("help,h", "print help information")("version,v", "version of PPC");
-    boost::program_options::variables_map vm;
-    try
-    {
-        boost::program_options::store(
-            boost::program_options::parse_command_line(argc, argv, main_options), vm);
-    }
-    catch (...)
-    {
-        printVersion();
-    }
-    /// help information
-    if (vm.count("help") || vm.count("h"))
-    {
-        std::cout << main_options << std::endl;
-        exit(0);
-    }
-    /// version information
-    if (vm.count("version") || vm.count("v"))
-    {
-        printVersion();
-        exit(0);
-    }
-}
diff --git a/cpp/wedpr-helper/libhelper/CommandHelper.h b/cpp/wedpr-helper/libhelper/CommandHelper.h
index 7724d0b1..73e44286 100644
--- a/cpp/wedpr-helper/libhelper/CommandHelper.h
+++ b/cpp/wedpr-helper/libhelper/CommandHelper.h
@@ -27,7 +27,6 @@ struct CommandLineParam
 {
     std::string configFilePath;
 };
-void printVersion();
-CommandLineParam initCommandLine(int argc, const char* argv[]);
-void initAppCommandLine(int argc, char* argv[]);
+void printVersion(std::string const& binaryName);
+CommandLineParam initCommandLine(std::string const& binaryName, int argc, const char* argv[]);
 }  // namespace ppc
\ No newline at end of file
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index 531018db..491f9517 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -207,10 +207,6 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     INIT_LOG(INFO) << LOG_DESC("create bs mode ecdh psi success");
 
     initMsgHandlers();
-
-    // add parallelism control for tbb
-    tbb::global_control gc(
-        tbb::global_control::max_allowed_parallelism, m_config->threadPoolSize());
 }
 
 // init the msg-handlers
diff --git a/cpp/wedpr-main/air-node/main.cpp b/cpp/wedpr-main/air-node/main.cpp
index b7cfd67f..ae54afee 100644
--- a/cpp/wedpr-main/air-node/main.cpp
+++ b/cpp/wedpr-main/air-node/main.cpp
@@ -22,6 +22,11 @@
 
 int main(int argc, const char* argv[])
 {
-   auto initializer = std::make_shared();
-    startProgram(argc, argv, "ppc-psi", initializer);
+    std::string binaryName = "ppc-psi";
+    auto initializer = std::make_shared();
+    auto ret = startProgram(argc, argv, binaryName, initializer);
+    initializer.reset();
+    std::cout << "[" << bcos::getCurrentDateTime() << "] ";
+    std::cout << "The " << binaryName << " program exit normally." << std::endl;
+    return ret;
 }
\ No newline at end of file
diff --git a/cpp/wedpr-main/cem-node/main.cpp b/cpp/wedpr-main/cem-node/main.cpp
index cfd4a724..a34f1093 100644
--- a/cpp/wedpr-main/cem-node/main.cpp
+++ b/cpp/wedpr-main/cem-node/main.cpp
@@ -53,19 +53,19 @@ int main(int argc, const char* argv[])
     auto initializer = std::make_shared();
     try
     {
-        auto param = initCommandLine(argc, argv);
+        auto param = initCommandLine("wedpr-cem", argc, argv);
         initializer->init(param.configFilePath);
         initializer->start();
     }
     catch (std::exception const& e)
     {
-        printVersion();
+        printVersion("wedpr-cem");
         std::cout << "[" << bcos::getCurrentDateTime() << "] ";
         std::cout << "start ppc-cem failed, error:" << boost::diagnostic_information(e)
                   << std::endl;
         return -1;
     }
-    printVersion();
+    printVersion("wedpr-cem");
     std::cout << "[" << bcos::getCurrentDateTime() << "] ";
     std::cout << "The ppc-cem is running..." << std::endl;
     while (!exitHandler.shouldExit())
diff --git a/cpp/wedpr-main/common/NodeStarter.h b/cpp/wedpr-main/common/NodeStarter.h
index 71dff545..fd08bd41 100644
--- a/cpp/wedpr-main/common/NodeStarter.h
+++ b/cpp/wedpr-main/common/NodeStarter.h
@@ -29,12 +29,12 @@ namespace ppc::node
 {
 template 
 int startProgram(
-    int argc, const char* argv[], std::string const& binaryName, std::shared_ptr& starter)
+    int argc, const char* argv[], std::string const& binaryName, std::shared_ptr starter)
 {
     /// set LC_ALL
     setDefaultOrCLocale();
     std::set_terminate([]() {
-        std::cerr << "terminate handler called, print stacks" << std::endl;
+        std::cout << "terminate handler called, print stacks" << std::endl;
         void* trace_elems[20];
         int trace_elem_count(backtrace(trace_elems, 20));
         char** stack_syms(backtrace_symbols(trace_elems, trace_elem_count));
@@ -43,7 +43,7 @@ int startProgram(
             std::cout << stack_syms[i] << "\n";
         }
         free(stack_syms);
-        std::cerr << "terminate handler called, print stack end" << std::endl;
+        std::cout << "terminate handler called, print stack end" << std::endl;
         abort();
     });
     // get datetime and output welcome info
@@ -55,27 +55,25 @@ int startProgram(
     // Note: the initializer must exist in the life time of the whole program
     try
     {
-        auto param = ppc::initCommandLine(argc, argv);
+        auto param = ppc::initCommandLine(binaryName, argc, argv);
         starter->init(param.configFilePath);
         starter->start();
     }
     catch (std::exception const& e)
     {
-        printVersion();
+        printVersion(binaryName);
         std::cout << "[" << bcos::getCurrentDateTime() << "] ";
         std::cout << "start " + binaryName + " failed, error:" << boost::diagnostic_information(e)
                   << std::endl;
         return -1;
     }
-    printVersion();
+    printVersion(binaryName);
     std::cout << "[" << bcos::getCurrentDateTime() << "] ";
     std::cout << "The " + binaryName + " is running..." << std::endl;
     while (!exitHandler.shouldExit())
     {
         std::this_thread::sleep_for(std::chrono::milliseconds(200));
     }
-    starter.reset();
-    std::cout << "[" << bcos::getCurrentDateTime() << "] ";
-    std::cout << "The " + binaryName + " program exit normally." << std::endl;
+    return 0;
 }
 }  // namespace ppc::node
\ No newline at end of file
diff --git a/cpp/wedpr-main/gateway/CMakeLists.txt b/cpp/wedpr-main/gateway/CMakeLists.txt
index 08a62d76..29b6af84 100644
--- a/cpp/wedpr-main/gateway/CMakeLists.txt
+++ b/cpp/wedpr-main/gateway/CMakeLists.txt
@@ -4,4 +4,4 @@ include_directories(${CMAKE_SOURCE_DIR})
 aux_source_directory(./ SRC_LIST)
 add_executable(${GATEWAY_BINARY_NAME} ${SRC_LIST})
 
-target_link_libraries(${GATEWAY_BINARY_NAME} ${GATEWAY_TARGET} ${STORAGE_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${HELPER_TARGET})
\ No newline at end of file
+target_link_libraries(${GATEWAY_BINARY_NAME} ${GATEWAY_TARGET} ${SERVICE_CLIENT_TARGET} ${SERVICE_SERVER_TARGET} ${HELPER_TARGET})
\ No newline at end of file
diff --git a/cpp/wedpr-main/gateway/main.cpp b/cpp/wedpr-main/gateway/main.cpp
index 3f309bbd..c8795457 100644
--- a/cpp/wedpr-main/gateway/main.cpp
+++ b/cpp/wedpr-main/gateway/main.cpp
@@ -23,6 +23,11 @@
 using namespace ppc::node;
 int main(int argc, const char* argv[])
 {
+    std::string binaryName = "ppc-gateway-service";
     auto initializer = std::make_shared();
-    startProgram(argc, argv, "ppc-gateway-service", initializer);
+    auto ret = startProgram(argc, argv, binaryName, initializer);
+    initializer.reset();
+    std::cout << "[" << bcos::getCurrentDateTime() << "] ";
+    std::cout << "The " << binaryName << " program exit normally." << std::endl;
+    return ret;
 }
\ No newline at end of file
diff --git a/cpp/wedpr-main/mpc-node/CMakeLists.txt b/cpp/wedpr-main/mpc-node/CMakeLists.txt
index fd3942db..2ca48eba 100644
--- a/cpp/wedpr-main/mpc-node/CMakeLists.txt
+++ b/cpp/wedpr-main/mpc-node/CMakeLists.txt
@@ -1,4 +1,4 @@
 aux_source_directory(. SRC_LIST)
 
 add_executable(${MPC_BINARY_NAME} ${SRC_LIST})
-target_link_libraries(${MPC_BINARY_NAME} PUBLIC ${MPC_TARGET} ${RPC_TARGET} ${HELPER_TARGET} ${WEDPR_TRANSPORT_SDK_TARGET} TBB::tbb)
+target_link_libraries(${MPC_BINARY_NAME} PUBLIC ${WEDPR_TRANSPORT_SDK_TARGET} ${MPC_TARGET} ${RPC_TARGET} ${HELPER_TARGET} TBB::tbb)
diff --git a/cpp/wedpr-main/mpc-node/main.cpp b/cpp/wedpr-main/mpc-node/main.cpp
index dae16bf5..1df02246 100644
--- a/cpp/wedpr-main/mpc-node/main.cpp
+++ b/cpp/wedpr-main/mpc-node/main.cpp
@@ -53,19 +53,19 @@ int main(int argc, const char* argv[])
     auto initializer = std::make_shared();
     try
     {
-        auto param = initCommandLine(argc, argv);
+        auto param = initCommandLine("wedpr-mpc", argc, argv);
         initializer->init(param.configFilePath);
         initializer->start();
     }
     catch (std::exception const& e)
     {
-        printVersion();
+        printVersion("wedpr-mpc");
         std::cout << "[" << bcos::getCurrentDateTime() << "] ";
         std::cout << "start ppc-mpc failed, error:" << boost::diagnostic_information(e)
                   << std::endl;
         return -1;
     }
-    printVersion();
+    printVersion("wedpr-mpc");
     std::cout << "[" << bcos::getCurrentDateTime() << "] ";
     std::cout << "The ppc-mpc is running..." << std::endl;
     while (!exitHandler.shouldExit())
diff --git a/cpp/wedpr-main/pro-node/main.cpp b/cpp/wedpr-main/pro-node/main.cpp
index a122784b..2e075403 100644
--- a/cpp/wedpr-main/pro-node/main.cpp
+++ b/cpp/wedpr-main/pro-node/main.cpp
@@ -23,6 +23,11 @@
 using namespace ppc::node;
 int main(int argc, const char* argv[])
 {
+    std::string binaryName = "ppc-pro-node";
     auto initializer = std::make_shared();
-    startProgram(argc, argv, "ppc-pro-node", initializer);
+    auto ret = startProgram(argc, argv, binaryName, initializer);
+    initializer.reset();
+    std::cout << "[" << bcos::getCurrentDateTime() << "] ";
+    std::cout << "The " << binaryName << " program exit normally." << std::endl;
+    return ret;
 }
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
index fef0e383..eb778d8e 100644
--- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
@@ -21,6 +21,7 @@
 #include "Common.h"
 #include "grpcpp/ext/proto_server_reflection_plugin.h"
 
+using namespace ppc;
 using namespace ppc::protocol;
 using namespace grpc;
 
@@ -43,14 +44,32 @@ void GrpcServer::start()
     builder.AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0);
     // without authentication
     builder.AddListeningPort(m_config->listenEndPoint(), grpc::InsecureServerCredentials());
+    builder.SetMaxMessageSize(m_config->maxMsgSize());
+    builder.SetMaxSendMessageSize(m_config->maxSendMessageSize());
+    builder.SetMaxReceiveMessageSize(m_config->maxReceivedMessageSize());
     // register the service
     for (auto const& service : m_bindingServices)
     {
         builder.RegisterService(service.get());
     }
     m_server = std::unique_ptr(builder.BuildAndStart());
+    if (!m_server)
+    {
+        GRPC_SERVER_LOG(INFO) << LOG_DESC(
+                                     "GrpcServer BuildAndStart failed, please check the port has "
+                                     "been occupied or not")
+                              << LOG_KV("listenEndPoint", m_config->listenEndPoint());
+        BOOST_THROW_EXCEPTION(
+            WeDPRException() << bcos::errinfo_comment("BuildAndStart grpcServer failed for bind "
+                                                      "error, please check the listenPort, current "
+                                                      "listenEndPoint: " +
+                                                      m_config->listenEndPoint()));
+    }
     GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer start success!")
-                          << LOG_KV("listenEndPoint", m_config->listenEndPoint());
+                          << LOG_KV("listenEndPoint", m_config->listenEndPoint())
+                          << LOG_KV("maxMsgSize", m_config->maxMsgSize())
+                          << LOG_KV("maxSendMessageSize", m_config->maxSendMessageSize())
+                          << LOG_KV("maxReceivedMessageSize", m_config->maxReceivedMessageSize());
 }
 
 void GrpcServer::stop()
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
index ad482cb5..c287aac9 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
@@ -64,7 +64,10 @@ void GatewayRouterManager::start()
         return;
     }
     m_running = true;
-    m_timer->start();
+    if (m_timer)
+    {
+        m_timer->start();
+    }
     ROUTER_MGR_LOG(INFO) << LOG_DESC("start GatewayRouterManager success");
 }
 
@@ -76,7 +79,10 @@ void GatewayRouterManager::stop()
         return;
     }
     m_running = false;
-    m_timer->stop();
+    if (m_timer)
+    {
+        m_timer->stop();
+    }
     ROUTER_MGR_LOG(INFO) << LOG_DESC("stop GatewayRouterManager success");
 }
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
index 080d7aa9..87eae3de 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.cpp
@@ -48,6 +48,17 @@ Service::Service(std::string const& _nodeID, RouterTableFactory::Ptr const& _rou
         boost::bind(&Service::onP2PDisconnect, this, boost::placeholders::_1));
 }
 
+void Service::stop()
+{
+    SERVICE_LOG(INFO) << LOG_DESC("stop service");
+    // stop the timerFactory
+    if (m_timerFactory)
+    {
+        m_timerFactory.reset();
+    }
+    WsService::stop();
+}
+
 void Service::onP2PConnect(WsSession::Ptr _session)
 {
     SERVICE_LOG(INFO) << LOG_DESC("Receive new p2p connection")
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
index 64477b1e..5f85c406 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/p2p/Service.h
@@ -81,6 +81,8 @@ class Service : public bcos::boostssl::ws::WsService
         m_deleteSessionHandlers.emplace_back(_handler);
     }
 
+    void stop() override;
+
 protected:
     void onRecvMessage(bcos::boostssl::MessageFace::Ptr _msg,
         bcos::boostssl::ws::WsSession::Ptr _session) override;
diff --git a/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp b/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp
index 08ad495d..072c0e7e 100644
--- a/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp
+++ b/cpp/wedpr-transport/ppc-http/demo/http_demo.cpp
@@ -48,7 +48,7 @@ int main(int argc, char const* argv[])
 
     try
     {
-        auto param = initCommandLine(argc, argv);
+        auto param = initCommandLine("http_demo", argc, argv);
         auto ppcConfig = std::make_shared();
         // not specify the certPath in air-mode
         ppcConfig->loadGatewayConfig(param.configFilePath);
@@ -67,7 +67,7 @@ int main(int argc, char const* argv[])
     }
     catch (std::exception const& e)
     {
-        ppc::printVersion();
+        ppc::printVersion("http-demo");
         std::cout << "[" << bcos::getCurrentDateTime() << "] ";
         std::cout << "start http-demo failed, error:" << boost::diagnostic_information(e)
                   << std::endl;
diff --git a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp
index 252e3ba4..9fdaf747 100644
--- a/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/demo/rpc_demo.cpp
@@ -45,7 +45,7 @@ int main(int argc, const char* argv[])
     signal(SIGINT, &ExitHandler::exitHandler);
     try
     {
-        auto param = initCommandLine(argc, argv);
+        auto param = initCommandLine("rpc_demo", argc, argv);
         auto ppcConfig = std::make_shared();
         // not specify the certPath in air-mode
         ppcConfig->loadRpcConfig(param.configFilePath);
@@ -63,7 +63,7 @@ int main(int argc, const char* argv[])
     }
     catch (std::exception const& e)
     {
-        ppc::printVersion();
+        ppc::printVersion("rpc_demo");
         std::cout << "[" << bcos::getCurrentDateTime() << "] ";
         std::cout << "start rpc-demo failed, error:" << boost::diagnostic_information(e)
                   << std::endl;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/conf/wedpr_sdk_log_config.ini b/cpp/wedpr-transport/sdk-wrapper/java/bindings/conf/wedpr_sdk_log_config.ini
new file mode 100644
index 00000000..6b5595be
--- /dev/null
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/conf/wedpr_sdk_log_config.ini
@@ -0,0 +1,9 @@
+[log]
+    enable=true
+    ; print the log to std::cout or not, default print to the log files
+    enable_console_output = false
+    log_path=./log
+    ; info debug trace
+    level=info
+    ; MB
+    max_log_file_size=200
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
index c329cf05..6ca4be2c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/TransportBuilder.java
@@ -71,7 +71,7 @@ public SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t frontConfigB
                 wedpr_java_transportJNI.TransportBuilder_frontConfigBuilder(swigCPtr, this), false);
     }
 
-    public static void initLog(String configPath) {
-        wedpr_java_transportJNI.TransportBuilder_initLog(configPath);
+    public void initLog(String configPath) {
+        wedpr_java_transportJNI.TransportBuilder_initLog(swigCPtr, this, configPath);
     }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index 834f8348..f5fedd8f 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -875,7 +875,8 @@ public static final native long TransportBuilder_buildConfig(
     public static final native long TransportBuilder_frontConfigBuilder(
             long jarg1, TransportBuilder jarg1_);
 
-    public static final native void TransportBuilder_initLog(String jarg1);
+    public static final native void TransportBuilder_initLog(
+            long jarg1, TransportBuilder jarg1_, String jarg2);
 
     public static final native long new_Transport(long jarg1, FrontConfig jarg1_);
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
index fe04a528..8449e465 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/TransportConfig.java
@@ -40,10 +40,13 @@ public class TransportConfig {
     }
 
     private static synchronized void createTransportBuilder() {
-        // TODO: make log path configuration here
-        TransportBuilder.initLog("conf/wedpr_sdk_log_config.ini");
+        if (transportBuilder != null) {
+            logger.info("transportBuilder has already been created");
+            return;
+        }
         logger.info("init transportBuilder");
         transportBuilder = new TransportBuilder();
+        transportBuilder.initLog("conf/wedpr_sdk_log_config.ini");
         logger.info("init transportBuilder success");
     }
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index 022c396d..f9ef78e3 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -44,14 +44,23 @@ public class TransportImpl implements WeDPRTransport {
     private final Transport transport;
     private final TransportConfig transportConfig;
 
-    public static WeDPRTransport build(TransportConfig transportConfig) {
-        return new TransportImpl(
-                TransportConfig.getTransportBuilder()
-                        .buildProTransport(transportConfig.getFrontConfig()),
-                transportConfig);
+    public static WeDPRTransport build(TransportConfig transportConfig) throws WeDPRSDKException {
+        try {
+            return new TransportImpl(
+                    TransportConfig.getTransportBuilder()
+                            .buildProTransport(transportConfig.getFrontConfig()),
+                    transportConfig);
+        } catch (Exception e) {
+            logger.warn(
+                    "build transport failed, transport config: {}, error: ",
+                    transportConfig.toString(),
+                    e);
+            throw e;
+        }
     }
 
-    protected TransportImpl(Transport transport, TransportConfig transportConfig) {
+    protected TransportImpl(Transport transport, TransportConfig transportConfig)
+            throws WeDPRSDKException {
         logger.info("Build Transport, config: {}", transportConfig.toString());
         this.transport = transport;
         this.transport.disOwnMemory();
@@ -60,8 +69,16 @@ protected TransportImpl(Transport transport, TransportConfig transportConfig) {
 
     @Override
     public void start() {
-        logger.info("start the transport");
-        this.transport.start();
+        try {
+            logger.info("start the transport");
+            this.transport.start();
+        } catch (Exception e) {
+            logger.warn("start the transport failed, error: ", e);
+            if (this.transport != null) {
+                this.transport.stop();
+            }
+            throw e;
+        }
     }
 
     @Override
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index 2c4bf107..fec5698c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -852,44 +852,6 @@ template  T SwigValueInit() {
 #include 		// Use the C99 official header
 
 
-SWIGINTERN void SWIG_JavaException(JNIEnv *jenv, int code, const char *msg) {
-  SWIG_JavaExceptionCodes exception_code = SWIG_JavaUnknownError;
-  switch(code) {
-  case SWIG_MemoryError:
-    exception_code = SWIG_JavaOutOfMemoryError;
-    break;
-  case SWIG_IOError:
-    exception_code = SWIG_JavaIOException;
-    break;
-  case SWIG_SystemError:
-  case SWIG_RuntimeError:
-    exception_code = SWIG_JavaRuntimeException;
-    break;
-  case SWIG_OverflowError:
-  case SWIG_IndexError:
-    exception_code = SWIG_JavaIndexOutOfBoundsException;
-    break;
-  case SWIG_DivisionByZero:
-    exception_code = SWIG_JavaArithmeticException;
-    break;
-  case SWIG_SyntaxError:
-  case SWIG_ValueError:
-  case SWIG_TypeError:
-    exception_code = SWIG_JavaIllegalArgumentException;
-    break;
-  case SWIG_UnknownError:
-  default:
-    exception_code = SWIG_JavaUnknownError;
-    break;
-  }
-  SWIG_JavaThrowException(jenv, exception_code, msg);
-}
-
-
-#include 
-#include 
-
-
 #include 
 #include 
 
@@ -1183,6 +1145,44 @@ SWIGINTERN void std_vector_Sl_std_shared_ptr_Sl_ppc_protocol_INodeInfo_Sg__Sg__d
         }
       }
 
+SWIGINTERN void SWIG_JavaException(JNIEnv *jenv, int code, const char *msg) {
+  SWIG_JavaExceptionCodes exception_code = SWIG_JavaUnknownError;
+  switch(code) {
+  case SWIG_MemoryError:
+    exception_code = SWIG_JavaOutOfMemoryError;
+    break;
+  case SWIG_IOError:
+    exception_code = SWIG_JavaIOException;
+    break;
+  case SWIG_SystemError:
+  case SWIG_RuntimeError:
+    exception_code = SWIG_JavaRuntimeException;
+    break;
+  case SWIG_OverflowError:
+  case SWIG_IndexError:
+    exception_code = SWIG_JavaIndexOutOfBoundsException;
+    break;
+  case SWIG_DivisionByZero:
+    exception_code = SWIG_JavaArithmeticException;
+    break;
+  case SWIG_SyntaxError:
+  case SWIG_ValueError:
+  case SWIG_TypeError:
+    exception_code = SWIG_JavaIllegalArgumentException;
+    break;
+  case SWIG_UnknownError:
+  default:
+    exception_code = SWIG_JavaUnknownError;
+    break;
+  }
+  SWIG_JavaThrowException(jenv, exception_code, msg);
+}
+
+
+#include 
+#include 
+
+
 struct SWIG_null_deleter {
   void operator() (void const *) const {
   }
@@ -1453,7 +1453,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >();
+  {
+    try {
+      result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< bcos::Error > **)&jresult = (result && *result) ? new std::shared_ptr< bcos::Error >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1467,7 +1475,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< bcos::Error > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1477,7 +1493,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::FrontConfig > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::FrontConfig >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1491,7 +1515,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::FrontConfig > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1501,7 +1533,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::GrpcConfig >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1515,7 +1555,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::GrpcConfig > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1525,7 +1573,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::IFront > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IFront >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1539,7 +1595,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IFront > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1549,7 +1613,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::INodeDiscovery > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::INodeDiscovery >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1563,7 +1635,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::INodeDiscovery > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1573,7 +1653,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::IFrontClient > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IFrontClient >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1587,7 +1675,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IFrontClient > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1597,7 +1693,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::ErrorCallback > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::ErrorCallback >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1611,7 +1715,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::ErrorCallback > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1621,7 +1733,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::MessageDispatcherHandler >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1635,7 +1755,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::MessageDispatcherHandler > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1645,7 +1773,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::IMessageHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::IMessageHandler >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1659,7 +1795,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::IMessageHandler > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1669,7 +1813,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::front::GetPeersInfoHandler >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1683,7 +1835,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::front::GetPeersInfoHandler > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1693,7 +1853,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::gateway::IGateway > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::gateway::IGateway >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1707,7 +1875,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::gateway::IGateway > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1717,7 +1893,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::Message > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::Message >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1731,7 +1915,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::Message > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1741,7 +1933,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeader >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1755,7 +1955,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageHeader > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1765,7 +1973,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayload >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1779,7 +1995,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessagePayload > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1789,7 +2013,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeader >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1803,7 +2035,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageOptionalHeader > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1813,7 +2053,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageBuilder >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1827,7 +2075,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageBuilder > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1837,7 +2093,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1851,7 +2115,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageHeaderBuilder > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1861,7 +2133,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1875,7 +2155,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessagePayloadBuilder > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1885,7 +2173,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1899,7 +2195,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -1909,7 +2213,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< ppc::protocol::INodeInfo > **)&jresult = (result && *result) ? new std::shared_ptr< ppc::protocol::INodeInfo >(*result) : 0;
   if (1) delete result; 
   return jresult;
@@ -1923,7 +2235,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = jarg1 ? *(std::shared_ptr< ppc::protocol::INodeInfo > **)&jarg1 : &tempnull1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -2201,7 +2521,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(std::vector< uint8_t > **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -2479,7 +2807,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(std::vector< int8_t > **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -2781,7 +3117,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(std::vector< std::string > **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3059,7 +3403,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3091,7 +3443,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   std::string arg5_str(arg5_pstr);
   arg5 = &arg5_str;
   jenv->ReleaseStringUTFChars(jarg5, arg5_pstr); 
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr<  bcos::Error > **)&jresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result)); 
   {
     jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); 
@@ -3141,7 +3501,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::Error const & reference is null");
     return 0;
   } 
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr<  bcos::Error > **)&jresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result)); 
   {
     jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); 
@@ -3189,7 +3557,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::exception const & is null");
     return 0;
   } 
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr<  bcos::Error > **)&jresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result)); 
   {
     jenv->ReleaseByteArrayElements(jarg1, (jbyte *) arg1, 0); 
@@ -3209,7 +3585,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (bcos::Error *)new bcos::Error();
+  {
+    try {
+      result = (bcos::Error *)new bcos::Error();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  bcos::Error > **)&jresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -3234,7 +3618,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   if (!arg2_pstr) return 0;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (bcos::Error *)new bcos::Error(arg1,arg2);
+  {
+    try {
+      result = (bcos::Error *)new bcos::Error(arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  bcos::Error > **)&jresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -3251,7 +3643,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  bcos::Error > **)&jarg1;
   arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3267,7 +3667,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const bcos::Error > **)&jarg1;
   arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (int64_t)((bcos::Error const *)arg1)->errorCode();
+  {
+    try {
+      result = (int64_t)((bcos::Error const *)arg1)->errorCode();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -3285,7 +3693,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const bcos::Error > **)&jarg1;
   arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
+  {
+    try {
+      result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -3303,7 +3719,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  bcos::Error > **)&jarg1;
   arg1 = (bcos::Error *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (int64_t)jarg2; 
-  (arg1)->setErrorCode(arg2);
+  {
+    try {
+      (arg1)->setErrorCode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3327,7 +3751,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setErrorMessage((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setErrorMessage((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3435,7 +3867,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (InputBuffer *)new InputBuffer();
+  {
+    try {
+      result = (InputBuffer *)new InputBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(InputBuffer **)&jresult = result; 
   return jresult;
 }
@@ -3447,7 +3887,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(InputBuffer **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3555,7 +4003,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (OutputBuffer *)new OutputBuffer();
+  {
+    try {
+      result = (OutputBuffer *)new OutputBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(OutputBuffer **)&jresult = result; 
   return jresult;
 }
@@ -3567,7 +4023,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(OutputBuffer **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3580,7 +4044,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3596,7 +4068,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
+  {
+    try {
+      result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -3614,7 +4094,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (int)jarg2; 
-  (arg1)->setThreadPoolSize(arg2);
+  {
+    try {
+      (arg1)->setThreadPoolSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3630,7 +4118,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
+  {
+    try {
+      result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -3656,7 +4152,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setNodeID((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setNodeID((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3672,7 +4176,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -3690,7 +4202,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -3713,7 +4233,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "ppc::protocol::EndPoint const & is null");
     return ;
   } 
-  (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
+  {
+    try {
+      (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3737,7 +4265,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3753,7 +4289,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
+  {
+    try {
+      result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -3774,7 +4318,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::protocol::GrpcConfig::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->setGrpcConfig(arg2);
+  {
+    try {
+      (arg1)->setGrpcConfig(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3790,7 +4342,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::GrpcConfig::Ptr **)&jresult = *result ? new ppc::protocol::GrpcConfig::Ptr(*result) : 0; 
   return jresult;
 }
@@ -3808,7 +4368,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
+  {
+    try {
+      result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::INodeInfo::Ptr **)&jresult = result ? new ppc::protocol::INodeInfo::Ptr(result) : 0; 
   return jresult;
 }
@@ -3826,7 +4394,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::vector< std::string > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
+  {
+    try {
+      result = (std::vector< std::string > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::vector< std::string > **)&jresult = result; 
   return jresult;
 }
@@ -3849,7 +4425,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "std::vector< std::string > const & is null");
     return ;
   } 
-  (arg1)->setComponents((std::vector< std::string > const &)*arg2);
+  {
+    try {
+      (arg1)->setComponents((std::vector< std::string > const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3873,7 +4457,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->addComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->addComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3889,7 +4481,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::vector< std::string > *) &(arg1)->mutableComponents();
+  {
+    try {
+      result = (std::vector< std::string > *) &(arg1)->mutableComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::vector< std::string > **)&jresult = result; 
   return jresult;
 }
@@ -3907,7 +4507,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::front::FrontConfig > **)&jarg1;
   arg1 = (ppc::front::FrontConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::front::FrontConfig const *)arg1)->meta();
+  {
+    try {
+      result = ((ppc::front::FrontConfig const *)arg1)->meta();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -3932,7 +4540,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (!arg2_pstr) return ;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setMeta(arg2);
+  {
+    try {
+      (arg1)->setMeta(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3942,7 +4558,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::front::FrontConfigBuilder **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -3955,7 +4579,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::front::FrontConfigBuilder **)&jarg1; 
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
+  {
+    try {
+      result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; 
   return jresult;
 }
@@ -3981,7 +4613,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   if (!arg3_pstr) return 0;
   (&arg3)->assign(arg3_pstr);
   jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); 
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+  {
+    try {
+      result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; 
   return jresult;
 }
@@ -3997,7 +4637,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = jarg1 ? *(ppc::front::FrontConfig::Ptr **)&jarg1 : &tempnull1; 
-  result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
+  {
+    try {
+      result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4009,7 +4657,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -4033,7 +4689,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   arg1 = &arg1_str;
   jenv->ReleaseStringUTFChars(jarg1, arg1_pstr); 
   arg2 = (uint16_t)jarg2; 
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
+  {
+    try {
+      result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -4045,7 +4709,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4058,7 +4730,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -4073,7 +4753,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -4095,7 +4783,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (!arg2_pstr) return ;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setHost(arg2);
+  {
+    try {
+      (arg1)->setHost(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4108,8 +4804,16 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setPort(arg2);
-}
+  {
+    try {
+      (arg1)->setPort(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
+}
 
 
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_EndPoint_1setListenIp(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
@@ -4129,7 +4833,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setListenIp((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setListenIp((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4142,7 +4854,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
+  {
+    try {
+      result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4157,7 +4877,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
+  {
+    try {
+      result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4172,7 +4900,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -4184,7 +4920,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
   return jresult;
 }
@@ -4207,7 +4951,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   }
   arg1 = *argp1; 
   arg2 = jarg2 ? true : false; 
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+  {
+    try {
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
   return jresult;
 }
@@ -4222,7 +4974,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+  {
+    try {
+      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4244,7 +5004,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   arg2 = *argp2; 
-  (arg1)->setEndPoint(arg2);
+  {
+    try {
+      (arg1)->setEndPoint(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4257,7 +5025,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jarg1_;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   arg2 = jarg2 ? true : false; 
-  (arg1)->setEnableHealthCheck(arg2);
+  {
+    try {
+      (arg1)->setEnableHealthCheck(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4270,7 +5046,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -4285,7 +5069,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::EndPoint **)&jresult = result; 
   return jresult;
 }
@@ -4300,7 +5092,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -4312,7 +5112,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4322,7 +5130,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -4339,7 +5155,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4355,7 +5179,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -4381,7 +5213,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4397,7 +5237,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -4415,7 +5263,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = jarg2 ? true : false; 
-  (arg1)->setEnableHealthCheck(arg2);
+  {
+    try {
+      (arg1)->setEnableHealthCheck(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4431,7 +5287,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = jarg2 ? true : false; 
-  (arg1)->setEnableDnslookup(arg2);
+  {
+    try {
+      (arg1)->setEnableDnslookup(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4447,7 +5311,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -4465,7 +5337,15 @@ SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+  {
+    try {
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jbyteArray ba = jenv->NewByteArray(9);
     jbyte* bae = jenv->GetByteArrayElements(ba, 0);
@@ -4500,7 +5380,15 @@ SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+  {
+    try {
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jbyteArray ba = jenv->NewByteArray(9);
     jbyte* bae = jenv->GetByteArrayElements(ba, 0);
@@ -4560,7 +5448,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  (arg1)->setMaxSendMessageSize(arg2);
+  {
+    try {
+      (arg1)->setMaxSendMessageSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4601,7 +5497,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  (arg1)->setMaxReceivedMessageSize(arg2);
+  {
+    try {
+      (arg1)->setMaxReceivedMessageSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4617,7 +5521,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+  {
+    try {
+      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -4635,7 +5547,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (int)jarg2; 
-  (arg1)->setCompressAlgorithm(arg2);
+  {
+    try {
+      (arg1)->setCompressAlgorithm(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4649,7 +5569,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = jarg1 ? *(ppc::protocol::GrpcConfig::Ptr **)&jarg1 : &tempnull1; 
-  result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4664,7 +5592,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4686,7 +5622,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null");
     return ;
   } 
-  ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+  {
+    try {
+      ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4737,7 +5681,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  result = (int64_t)(arg1)->decode(arg2,arg3);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -4755,7 +5707,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -4780,7 +5740,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (!arg2_pstr) return ;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setComponentType(arg2);
+  {
+    try {
+      (arg1)->setComponentType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4796,7 +5764,15 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jresult = jenv->NewByteArray((&result)->len);
     jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data);
@@ -4846,7 +5822,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  (arg1)->setSrcNodeBuffer(arg2,arg3);
+  {
+    try {
+      (arg1)->setSrcNodeBuffer(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
   {
     jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); 
   }
@@ -4866,7 +5850,15 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jresult = jenv->NewByteArray((&result)->len);
     jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data);
@@ -4916,7 +5908,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     }
     jenv->ReleaseByteArrayElements(ba, bae, 0);
   }
-  (arg1)->setDstNodeBuffer(arg2,arg3);
+  {
+    try {
+      (arg1)->setDstNodeBuffer(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
   {
     jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); 
   }
@@ -4936,7 +5936,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -4962,7 +5970,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setDstInst((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setDstInst((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -4978,7 +5994,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -5002,7 +6026,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   rvrdeleter2.reset(arg2); 
-  (arg1)->setTopic((std::string &&)*arg2);
+  {
+    try {
+      (arg1)->setTopic((std::string &&)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5026,7 +6058,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setTopic((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5042,7 +6082,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageOptionalHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -5068,7 +6116,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setSrcInst((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setSrcInst((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5081,7 +6137,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5103,7 +6167,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null");
     return ;
   } 
-  ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+  {
+    try {
+      ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5127,7 +6199,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -5145,7 +6225,15 @@ SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1t
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
+  {
+    try {
+      result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jshort)result; 
   return jresult;
 }
@@ -5163,7 +6251,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5179,7 +6275,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -5204,7 +6308,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (!arg2_pstr) return ;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setTraceID(arg2);
+  {
+    try {
+      (arg1)->setTraceID(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5220,7 +6332,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -5246,7 +6366,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setSrcGwNode((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setSrcGwNode((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5262,7 +6390,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -5288,7 +6424,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setDstGwNode((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setDstGwNode((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5304,7 +6448,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5322,7 +6474,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setPacketType(arg2);
+  {
+    try {
+      (arg1)->setPacketType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5338,7 +6498,15 @@ SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1t
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
+  {
+    try {
+      result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jshort)result; 
   return jresult;
 }
@@ -5356,7 +6524,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setTTL(arg2);
+  {
+    try {
+      (arg1)->setTTL(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5372,7 +6548,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5390,7 +6574,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5406,7 +6598,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -5427,7 +6627,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->setOptionalField(arg2);
+  {
+    try {
+      (arg1)->setOptionalField(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5443,7 +6651,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5461,7 +6677,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -5477,7 +6701,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5493,7 +6725,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::string_view **)&jresult = new std::string_view(result); 
   return jresult;
 }
@@ -5511,7 +6751,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::string_view **)&jresult = new std::string_view(result); 
   return jresult;
 }
@@ -5529,7 +6777,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5553,7 +6809,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   arg2 = *argp2; 
-  (arg1)->setRouteType(arg2);
+  {
+    try {
+      (arg1)->setRouteType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5569,7 +6833,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessageHeader > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeader *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+  {
+    try {
+      result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -5584,7 +6856,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5600,7 +6880,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::Message const *)arg1)->header();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->header();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -5621,7 +6909,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::protocol::MessageHeader::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->setHeader(arg2);
+  {
+    try {
+      (arg1)->setHeader(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5637,7 +6933,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5655,7 +6959,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5671,7 +6983,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5689,7 +7009,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setPacketType(arg2);
+  {
+    try {
+      (arg1)->setPacketType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5705,7 +7033,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -5730,7 +7066,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (!arg2_pstr) return ;
   (&arg2)->assign(arg2_pstr);
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setSeq(arg2);
+  {
+    try {
+      (arg1)->setSeq(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5746,7 +7090,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -5764,7 +7116,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5780,7 +7140,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -5796,7 +7164,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5815,7 +7191,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(std::shared_ptr< bcos::bytes > **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->setPayload(arg2);
+  {
+    try {
+      (arg1)->setPayload(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5831,7 +7215,15 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jresult = jenv->NewByteArray((&result)->len);
     jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data);
@@ -5855,7 +7247,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::protocol::MessagePayload::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->setFrontMessage(arg2);
+  {
+    try {
+      (arg1)->setFrontMessage(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5871,7 +7271,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
+  {
+    try {
+      result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessagePayload::Ptr **)&jresult = *result ? new ppc::protocol::MessagePayload::Ptr(*result) : 0; 
   return jresult;
 }
@@ -5897,7 +7305,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null");
     return 0;
   } 
-  result = (bool)(arg1)->encode(*arg2);
+  {
+    try {
+      result = (bool)(arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -5923,7 +7339,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -5941,7 +7365,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+  {
+    try {
+      result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -5959,7 +7391,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::Message > **)&jarg1;
   arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::Message const *)arg1)->payload();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->payload();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::shared_ptr< bcos::bytes > **)&jresult = result ? new std::shared_ptr< bcos::bytes >(result) : 0; 
   return jresult;
 }
@@ -5974,7 +7414,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -5998,7 +7446,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -6016,7 +7472,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -6038,7 +7502,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = jarg2 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2 : &tempnull2; 
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    try {
+      result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -6053,7 +7525,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6069,7 +7549,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; 
   return jresult;
 }
@@ -6095,7 +7583,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; 
   return jresult;
 }
@@ -6134,7 +7630,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   rvrdeleter4.reset(arg4); 
-  result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
+  {
+    try {
+      result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; 
   return jresult;
 }
@@ -6149,7 +7653,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6169,7 +7681,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = jarg2 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg2 : &tempnull2; 
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    try {
+      result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -6187,7 +7707,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessageOptionalHeaderBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageOptionalHeader::Ptr **)&jresult = result ? new ppc::protocol::MessageOptionalHeader::Ptr(result) : 0; 
   return jresult;
 }
@@ -6204,7 +7732,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jarg1_;
   argp1 = *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg1;
   if (argp1) arg1 = *argp1; 
-  result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
+  {
+    try {
+      result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -6220,7 +7756,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = jarg1 ? *(ppc::protocol::Message::Ptr **)&jarg1 : &tempnull1; 
-  result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -6232,7 +7776,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::protocol::FrontMsgExtFlag)ppc::protocol::FrontMsgExtFlag::Response;
+  {
+    try {
+      result = (ppc::protocol::FrontMsgExtFlag)ppc::protocol::FrontMsgExtFlag::Response;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -6247,7 +7799,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6271,7 +7831,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "bcos::bytes & reference is null");
     return 0;
   } 
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
+  {
+    try {
+      result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -6297,7 +7865,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -6315,7 +7891,15 @@ SWIGEXPORT jshort JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1t
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
+  {
+    try {
+      result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jshort)result; 
   return jresult;
 }
@@ -6333,7 +7917,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint8_t)jarg2; 
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6349,7 +7941,15 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   {
     jresult = jenv->NewByteArray((&result)->len);
     jenv->SetByteArrayRegion(jresult, 0, (&result)->len, (jbyte *) (&result)->data);
@@ -6370,7 +7970,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -6388,7 +7996,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setSeq(arg2);
+  {
+    try {
+      (arg1)->setSeq(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6404,7 +8020,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
+  {
+    try {
+      result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jlong)result; 
   return jresult;
 }
@@ -6422,7 +8046,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -6448,7 +8080,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setTraceID((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setTraceID((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6464,7 +8104,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -6482,7 +8130,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (uint16_t)jarg2; 
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6496,7 +8152,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6512,7 +8176,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (bool)(arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)(arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -6527,7 +8199,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayloadBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6543,7 +8223,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > **)&jarg1;
   arg1 = (ppc::protocol::MessagePayloadBuilder *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessagePayload::Ptr **)&jresult = result ? new ppc::protocol::MessagePayload::Ptr(result) : 0; 
   return jresult;
 }
@@ -6569,7 +8257,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg2 = *argp2; 
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessagePayload::Ptr **)&jresult = result ? new ppc::protocol::MessagePayload::Ptr(result) : 0; 
   return jresult;
 }
@@ -6584,7 +8280,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::INodeInfo > **)&jarg1;
   arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6600,7 +8304,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
   arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
@@ -6618,7 +8330,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
   arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(bcos::bytesConstRef **)&jresult = new bcos::bytesConstRef(result); 
   return jresult;
 }
@@ -6642,7 +8362,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   arg2 = *argp2; 
-  (arg1)->setNodeID(arg2);
+  {
+    try {
+      (arg1)->setNodeID(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6666,7 +8394,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setEndPoint((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setEndPoint((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6692,7 +8428,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -6720,7 +8464,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -6748,7 +8500,15 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jboolean)result; 
   return jresult;
 }
@@ -6766,7 +8526,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
   arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::vector< std::string > **)&jresult = new std::vector< std::string >(result); 
   return jresult;
 }
@@ -6784,7 +8552,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   
   smartarg1 = *(std::shared_ptr< const ppc::protocol::INodeInfo > **)&jarg1;
   arg1 = (ppc::protocol::INodeInfo *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -6810,7 +8586,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->setMeta((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setMeta((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6824,7 +8608,15 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   (void)jcls;
   (void)jarg1_;
   arg1 = jarg1 ? *(ppc::protocol::INodeInfo::Ptr **)&jarg1 : &tempnull1; 
-  result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
@@ -6839,7 +8631,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFrontClient > **)&jarg1;
   arg1 = (ppc::front::IFrontClient *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6865,7 +8665,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   arg3 = *argp3; 
-  (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6875,7 +8683,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(jenv);
+  {
+    try {
+      result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(jenv);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::front::ErrorCallback > **)&jresult = result ? new std::shared_ptr<  ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -6892,7 +8708,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::ErrorCallback > **)&jarg1;
   arg1 = (ppc::front::ErrorCallback *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6911,7 +8735,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::front::ErrorCallback *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(bcos::Error::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->onError(arg2);
+  {
+    try {
+      (arg1)->onError(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6943,7 +8775,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(jenv);
+  {
+    try {
+      result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(jenv);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::front::MessageDispatcherHandler > **)&jresult = result ? new std::shared_ptr<  ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -6960,7 +8800,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::MessageDispatcherHandler > **)&jarg1;
   arg1 = (ppc::front::MessageDispatcherHandler *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -6979,7 +8827,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::front::MessageDispatcherHandler *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::protocol::Message::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->onMessage(arg2);
+  {
+    try {
+      (arg1)->onMessage(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7019,7 +8875,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
     return 0;
   }
   arg1 = *argp1; 
-  result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
+  {
+    try {
+      result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::SendResponseHandler **)&jresult = result; 
   return jresult;
 }
@@ -7031,7 +8895,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::front::SendResponseHandler **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7050,7 +8922,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   rvrdeleter2.reset(arg2); 
-  (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
+  {
+    try {
+      (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7060,7 +8940,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(jenv);
+  {
+    try {
+      result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(jenv);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::front::IMessageHandler > **)&jresult = result ? new std::shared_ptr<  ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -7077,7 +8965,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IMessageHandler > **)&jarg1;
   arg1 = (ppc::front::IMessageHandler *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7110,7 +9006,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
     return ;
   }
   arg4 = *argp4; 
-  (arg1)->onMessage(arg2,arg3,arg4);
+  {
+    try {
+      (arg1)->onMessage(arg2,arg3,arg4);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7142,7 +9046,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(jenv);
+  {
+    try {
+      result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(jenv);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::front::GetPeersInfoHandler > **)&jresult = result ? new std::shared_ptr<  ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -7159,7 +9071,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::GetPeersInfoHandler > **)&jarg1;
   arg1 = (ppc::front::GetPeersInfoHandler *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7188,7 +9108,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg3_str(arg3_pstr);
   arg3 = &arg3_str;
   jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); 
-  (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
+  {
+    try {
+      (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7223,7 +9151,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7237,7 +9173,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->start();
+  {
+    try {
+      (arg1)->start();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7251,7 +9195,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->stop();
+  {
+    try {
+      (arg1)->stop();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7280,7 +9232,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
   argp3 = *(ppc::front::MessageDispatcherHandler::Ptr **)&jarg3;
   if (argp3) arg3 = *argp3; 
-  (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7309,7 +9269,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
   argp3 = *(ppc::front::MessageDispatcherHandler::Ptr **)&jarg3;
   if (argp3) arg3 = *argp3; 
-  (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7374,7 +9342,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   if (argp8) arg8 = *argp8; 
   argp9 = *(ppc::front::IMessageHandler::Ptr **)&jarg9;
   if (argp9) arg9 = *argp9; 
-  (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+  {
+    try {
+      (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
   {
     jenv->ReleaseByteArrayElements(jarg4, (jbyte *) arg4, 0); 
   }
@@ -7471,7 +9447,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg7 = (int)jarg7; 
   argp8 = *(ppc::front::ErrorCallback::Ptr **)&jarg8;
   if (argp8) arg8 = *argp8; 
-  (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+  {
+    try {
+      (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
   {
     jenv->ReleaseByteArrayElements(jarg2, (jbyte *) arg2, 0); 
   }
@@ -7536,7 +9520,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   }
   arg6 = (int)jarg6; 
   arg7 = (long)jarg7; 
-  result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
+  {
+    try {
+      result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
   {
     jenv->ReleaseByteArrayElements(jarg4, (jbyte *) arg4, 0); 
@@ -7570,7 +9562,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
   arg3 = (long)jarg3; 
-  result = (arg1)->pop((std::string const &)*arg2,arg3);
+  {
+    try {
+      result = (arg1)->pop((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; 
   return jresult;
 }
@@ -7598,7 +9598,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (arg1)->peek((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->peek((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::Message::Ptr **)&jresult = result ? new ppc::protocol::Message::Ptr(result) : 0; 
   return jresult;
 }
@@ -7619,7 +9627,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
   argp2 = *(ppc::front::GetPeersInfoHandler::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  (arg1)->asyncGetPeers(arg2);
+  {
+    try {
+      (arg1)->asyncGetPeers(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7635,7 +9651,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
+  {
+    try {
+      result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::INodeInfo::Ptr **)&jresult = *result ? new ppc::protocol::INodeInfo::Ptr(*result) : 0; 
   return jresult;
 }
@@ -7663,7 +9687,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (arg1)->registerTopic((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->registerTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
   return jresult;
 }
@@ -7691,7 +9723,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  result = (arg1)->unRegisterTopic((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->unRegisterTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(bcos::Error::Ptr **)&jresult = result ? new bcos::Error::Ptr(result) : 0; 
   return jresult;
 }
@@ -7717,7 +9757,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->registerComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->registerComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7741,7 +9789,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->unRegisterComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->unRegisterComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7765,7 +9821,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   std::string arg2_str(arg2_pstr);
   arg2 = &arg2_str;
   jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
-  (arg1)->updateMetaInfo((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->updateMetaInfo((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7787,7 +9851,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = (int16_t)jarg2; 
   arg3 = jarg3 ? *(ppc::protocol::MessageOptionalHeader::Ptr **)&jarg3 : &tempnull3; 
-  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+  {
+    try {
+      result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::vector< std::string > **)&jresult = new std::vector< std::string >(result); 
   return jresult;
 }
@@ -7805,7 +9877,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr<  ppc::front::IFront > **)&jarg1;
   arg1 = (ppc::front::IFront *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (arg1)->getNodeDiscovery();
+  {
+    try {
+      result = (arg1)->getNodeDiscovery();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::INodeDiscovery::Ptr **)&jresult = result ? new ppc::front::INodeDiscovery::Ptr(result) : 0; 
   return jresult;
 }
@@ -7817,7 +9897,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::front::IFrontBuilder **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7849,7 +9937,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   }
   arg3 = *argp3; 
   arg4 = jarg4 ? true : false; 
-  result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
+  {
+    try {
+      result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::IFrontClient::Ptr **)&jresult = result ? new ppc::front::IFrontClient::Ptr(result) : 0; 
   return jresult;
 }
@@ -7864,7 +9960,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::front::INodeDiscovery > **)&jarg1;
   arg1 = (ppc::front::INodeDiscovery *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7880,7 +9984,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::front::INodeDiscovery > **)&jarg1;
   arg1 = (ppc::front::INodeDiscovery *)(smartarg1 ? smartarg1->get() : 0); 
-  result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+  {
+    try {
+      result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > **)&jresult = new std::vector< std::shared_ptr< ppc::protocol::INodeInfo > >(result); 
   return jresult;
 }
@@ -7892,7 +10004,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::AIR;
+  {
+    try {
+      result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::AIR;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -7904,7 +10024,15 @@ SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::PRO;
+  {
+    try {
+      result = (ppc::sdk::SDKMode)ppc::sdk::SDKMode::PRO;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   jresult = (jint)result; 
   return jresult;
 }
@@ -7916,7 +10044,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   (void)jenv;
   (void)jcls;
-  result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
+  {
+    try {
+      result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::sdk::TransportBuilder **)&jresult = result; 
   return jresult;
 }
@@ -7928,7 +10064,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   (void)jenv;
   (void)jcls;
   arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; 
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -7946,7 +10090,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; 
   argp2 = *(ppc::front::FrontConfig::Ptr **)&jarg2;
   if (argp2) arg2 = *argp2; 
-  result = (arg1)->buildProTransport(arg2);
+  {
+    try {
+      result = (arg1)->buildProTransport(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::sdk::Transport::Ptr **)&jresult = result ? new ppc::sdk::Transport::Ptr(result) : 0; 
   return jresult;
 }
@@ -7972,7 +10124,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   if (!arg3_pstr) return 0;
   (&arg3)->assign(arg3_pstr);
   jenv->ReleaseStringUTFChars(jarg3, arg3_pstr); 
-  result = (arg1)->buildConfig(arg2,arg3);
+  {
+    try {
+      result = (arg1)->buildConfig(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::FrontConfig::Ptr **)&jresult = result ? new ppc::front::FrontConfig::Ptr(result) : 0; 
   return jresult;
 }
@@ -7987,27 +10147,46 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   (void)jcls;
   (void)jarg1_;
   arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; 
-  result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
+  {
+    try {
+      result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::FrontConfigBuilder::Ptr **)&jresult = result; 
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1initLog(JNIEnv *jenv, jclass jcls, jstring jarg1) {
-  std::string *arg1 = 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_TransportBuilder_1initLog(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  std::string *arg2 = 0 ;
   
   (void)jenv;
   (void)jcls;
-  if(!jarg1) {
+  (void)jarg1_;
+  arg1 = *(ppc::sdk::TransportBuilder **)&jarg1; 
+  if(!jarg2) {
     SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
     return ;
   }
-  const char *arg1_pstr = (const char *)jenv->GetStringUTFChars(jarg1, 0); 
-  if (!arg1_pstr) return ;
-  std::string arg1_str(arg1_pstr);
-  arg1 = &arg1_str;
-  jenv->ReleaseStringUTFChars(jarg1, arg1_pstr); 
-  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  {
+    try {
+      (arg1)->initLog((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -8022,7 +10201,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   (void)jarg1_;
   argp1 = *(ppc::front::FrontConfig::Ptr **)&jarg1;
   if (argp1) arg1 = *argp1; 
-  result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
+  {
+    try {
+      result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   
   *(std::shared_ptr<  ppc::sdk::Transport > **)&jresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_1) : 0;
   
@@ -8039,7 +10226,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -8053,7 +10248,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->start();
+  {
+    try {
+      (arg1)->start();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -8067,7 +10270,15 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  (arg1)->stop();
+  {
+    try {
+      (arg1)->stop();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
 }
 
 
@@ -8083,7 +10294,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
+  {
+    try {
+      result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::IFront::Ptr **)&jresult = *result ? new ppc::front::IFront::Ptr(*result) : 0; 
   return jresult;
 }
@@ -8101,7 +10320,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
+  {
+    try {
+      result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::gateway::IGateway::Ptr **)&jresult = result; 
   return jresult;
 }
@@ -8119,7 +10346,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessagePayloadBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessagePayloadBuilder::Ptr(*result) : 0; 
   return jresult;
 }
@@ -8137,7 +10372,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageBuilder::Ptr(*result) : 0; 
   return jresult;
 }
@@ -8155,7 +10398,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageHeaderBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageHeaderBuilder::Ptr(*result) : 0; 
   return jresult;
 }
@@ -8173,7 +10424,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::protocol::MessageOptionalHeaderBuilder::Ptr **)&jresult = *result ? new ppc::protocol::MessageOptionalHeaderBuilder::Ptr(*result) : 0; 
   return jresult;
 }
@@ -8191,7 +10450,15 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
   
   smartarg1 = *(std::shared_ptr< const ppc::sdk::Transport > **)&jarg1;
   arg1 = (ppc::sdk::Transport *)(smartarg1 ? smartarg1->get() : 0); 
-  result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
+  {
+    try {
+      result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return 0;
+    }
+  }
   *(ppc::front::FrontConfig::Ptr **)&jresult = *result ? new ppc::front::FrontConfig::Ptr(*result) : 0; 
   return jresult;
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index 902409c5..9e32fe66 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -2,7 +2,6 @@
 
 %include "stdint.i"
 %include "typemaps.i"
-%include 
 
 #if defined(SWIGJAVA)
 #if defined(SWIGWORDSIZE64)
@@ -255,6 +254,16 @@ WRAP(ppc::sdk::Transport)
 %ignore ppc::protocol::INodeInfo::setComponents;
 %ignore ppc::protocol::INodeInfoFactory;
 
+%include "exception.i"
+%exception {
+    try {
+        $action
+    }
+    catch (const std::exception& e) {
+        SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+        return $null;
+    }
+}
 
 /*
 ///// tests  ///
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 7ba209fe..62ab6aca 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1820,9 +1820,8 @@ def buildConfig(self, threadPoolSize, nodeID):
     def frontConfigBuilder(self):
         return _wedpr_python_transport.TransportBuilder_frontConfigBuilder(self)
 
-    @staticmethod
-    def initLog(configPath):
-        return _wedpr_python_transport.TransportBuilder_initLog(configPath)
+    def initLog(self, configPath):
+        return _wedpr_python_transport.TransportBuilder_initLog(self, configPath)
 
 
 # Register TransportBuilder in _wedpr_python_transport:
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index 9b71c7d9..abe25249 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -7311,7 +7311,14 @@ SWIGINTERN PyObject *_wrap_new_SharedBcosError(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedBcosError", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >();
+  {
+    try {
+      result = (std::shared_ptr< bcos::Error > *)new std::shared_ptr< bcos::Error >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = (result && *result) ? new std::shared_ptr<  bcos::Error >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -7348,7 +7355,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedBcosError(PyObject *self, PyObject *args
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< bcos::Error > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7373,7 +7387,14 @@ SWIGINTERN PyObject *_wrap_new_SharedFrontConfig(PyObject *self, PyObject *args)
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedFrontConfig", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::FrontConfig > *)new std::shared_ptr< ppc::front::FrontConfig >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::FrontConfig > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::FrontConfig >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
@@ -7410,7 +7431,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedFrontConfig(PyObject *self, PyObject *ar
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::FrontConfig > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7435,7 +7463,14 @@ SWIGINTERN PyObject *_wrap_new_SharedGrpcConfig(PyObject *self, PyObject *args)
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedGrpcConfig", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::GrpcConfig > *)new std::shared_ptr< ppc::protocol::GrpcConfig >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN);
@@ -7472,7 +7507,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedGrpcConfig(PyObject *self, PyObject *arg
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::GrpcConfig > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7497,7 +7539,14 @@ SWIGINTERN PyObject *_wrap_new_SharedFront(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedFront", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IFront > *)new std::shared_ptr< ppc::front::IFront >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::IFront > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::IFront >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN);
@@ -7534,7 +7583,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedFront(PyObject *self, PyObject *args) {
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IFront > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7559,7 +7615,14 @@ SWIGINTERN PyObject *_wrap_new_SharedNodeDiscovery(PyObject *self, PyObject *arg
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedNodeDiscovery", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::INodeDiscovery > *)new std::shared_ptr< ppc::front::INodeDiscovery >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::INodeDiscovery > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::INodeDiscovery >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_POINTER_OWN);
@@ -7596,7 +7659,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedNodeDiscovery(PyObject *self, PyObject *
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::INodeDiscovery > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7621,7 +7691,14 @@ SWIGINTERN PyObject *_wrap_new_SharedNodeInfo(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedNodeInfo", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::INodeInfo > *)new std::shared_ptr< ppc::protocol::INodeInfo >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
@@ -7658,7 +7735,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedNodeInfo(PyObject *self, PyObject *args)
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::INodeInfo > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7683,7 +7767,14 @@ SWIGINTERN PyObject *_wrap_new_SharedFrontClient(PyObject *self, PyObject *args)
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedFrontClient", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IFrontClient > *)new std::shared_ptr< ppc::front::IFrontClient >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::IFrontClient > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::IFrontClient >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN);
@@ -7720,7 +7811,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedFrontClient(PyObject *self, PyObject *ar
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IFrontClient > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7745,7 +7843,14 @@ SWIGINTERN PyObject *_wrap_new_SharedErrorCallback(PyObject *self, PyObject *arg
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedErrorCallback", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::ErrorCallback > *)new std::shared_ptr< ppc::front::ErrorCallback >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::ErrorCallback > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::ErrorCallback >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_OWN);
@@ -7782,7 +7887,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedErrorCallback(PyObject *self, PyObject *
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::ErrorCallback > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7807,7 +7919,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessageDispatcherHandler(PyObject *self, Py
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageDispatcherHandler", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::MessageDispatcherHandler > *)new std::shared_ptr< ppc::front::MessageDispatcherHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::MessageDispatcherHandler > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::MessageDispatcherHandler >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_OWN);
@@ -7844,7 +7963,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessageDispatcherHandler(PyObject *self,
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::MessageDispatcherHandler > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7869,7 +7995,14 @@ SWIGINTERN PyObject *_wrap_new_SharedIMessageHandler(PyObject *self, PyObject *a
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedIMessageHandler", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::IMessageHandler > *)new std::shared_ptr< ppc::front::IMessageHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::IMessageHandler > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::IMessageHandler >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_OWN);
@@ -7906,7 +8039,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedIMessageHandler(PyObject *self, PyObject
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::IMessageHandler > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7931,7 +8071,14 @@ SWIGINTERN PyObject *_wrap_new_SharedGetPeersInfoHandler(PyObject *self, PyObjec
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedGetPeersInfoHandler", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::front::GetPeersInfoHandler > *)new std::shared_ptr< ppc::front::GetPeersInfoHandler >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::GetPeersInfoHandler > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::front::GetPeersInfoHandler >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_OWN);
@@ -7968,7 +8115,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedGetPeersInfoHandler(PyObject *self, PyOb
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::front::GetPeersInfoHandler > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -7993,7 +8147,14 @@ SWIGINTERN PyObject *_wrap_new_SharedGateway(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedGateway", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::gateway::IGateway > *)new std::shared_ptr< ppc::gateway::IGateway >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::gateway::IGateway > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::gateway::IGateway >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__gateway__IGateway_t, SWIG_POINTER_OWN);
@@ -8030,7 +8191,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedGateway(PyObject *self, PyObject *args)
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::gateway::IGateway > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8055,7 +8223,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessage(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessage", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::Message > *)new std::shared_ptr< ppc::protocol::Message >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::Message >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -8092,7 +8267,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessage(PyObject *self, PyObject *args)
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::Message > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8117,7 +8299,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessageHeader(PyObject *self, PyObject *arg
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageHeader", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageHeader > *)new std::shared_ptr< ppc::protocol::MessageHeader >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessageHeader >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
@@ -8154,7 +8343,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessageHeader(PyObject *self, PyObject *
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeader > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8179,7 +8375,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessagePayload(PyObject *self, PyObject *ar
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessagePayload", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessagePayload > *)new std::shared_ptr< ppc::protocol::MessagePayload >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessagePayload >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
@@ -8216,7 +8419,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessagePayload(PyObject *self, PyObject
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayload > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8241,7 +8451,14 @@ SWIGINTERN PyObject *_wrap_new_SharedRouteInfo(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedRouteInfo", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageOptionalHeader > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeader >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -8278,7 +8495,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedRouteInfo(PyObject *self, PyObject *args
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeader > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8303,7 +8527,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessageBuilder(PyObject *self, PyObject *ar
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageBuilder", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageBuilder > *)new std::shared_ptr< ppc::protocol::MessageBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageBuilder > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessageBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN);
@@ -8340,7 +8571,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessageBuilder(PyObject *self, PyObject
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageBuilder > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8365,7 +8603,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessageHeaderBuilder(PyObject *self, PyObje
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessageHeaderBuilder", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageHeaderBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessageHeaderBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN);
@@ -8402,7 +8647,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessageHeaderBuilder(PyObject *self, PyO
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageHeaderBuilder > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8427,7 +8679,14 @@ SWIGINTERN PyObject *_wrap_new_SharedMessagePayloadBuilder(PyObject *self, PyObj
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedMessagePayloadBuilder", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *)new std::shared_ptr< ppc::protocol::MessagePayloadBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessagePayloadBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN);
@@ -8464,7 +8723,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedMessagePayloadBuilder(PyObject *self, Py
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessagePayloadBuilder > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -8489,7 +8755,14 @@ SWIGINTERN PyObject *_wrap_new_SharedRouteInfoBuilder(PyObject *self, PyObject *
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_SharedRouteInfoBuilder", 0, 0, 0)) SWIG_fail;
-  result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >();
+  {
+    try {
+      result = (std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > *)new std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder >();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = (result && *result) ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN);
@@ -8526,7 +8799,14 @@ SWIGINTERN PyObject *_wrap_delete_SharedRouteInfoBuilder(PyObject *self, PyObjec
       arg1 = (argp1) ? reinterpret_cast< std::shared_ptr< ppc::protocol::MessageOptionalHeaderBuilder > * >(argp1) : &tempshared1;
     }
   }
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -10394,7 +10674,14 @@ SWIGINTERN PyObject *_wrap_delete_ubytes(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ubytes" "', argument " "1"" of type '" "std::vector< uint8_t > *""'"); 
   }
   arg1 = reinterpret_cast< std::vector< uint8_t > * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -12262,7 +12549,14 @@ SWIGINTERN PyObject *_wrap_delete_ibytes(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ibytes" "', argument " "1"" of type '" "std::vector< int8_t > *""'"); 
   }
   arg1 = reinterpret_cast< std::vector< int8_t > * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -14160,7 +14454,14 @@ SWIGINTERN PyObject *_wrap_delete_StringVec(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_StringVec" "', argument " "1"" of type '" "std::vector< std::string > *""'"); 
   }
   arg1 = reinterpret_cast< std::vector< std::string > * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16094,7 +16395,14 @@ SWIGINTERN PyObject *_wrap_delete_NodeInfoVec(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_NodeInfoVec" "', argument " "1"" of type '" "std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > *""'"); 
   }
   arg1 = reinterpret_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo > > * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16166,7 +16474,14 @@ SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_0(PyObject *self, Py_ssize_t n
     }
     arg5 = ptr;
   }
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -16257,7 +16572,14 @@ SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_1(PyObject *self, Py_ssize_t n
       arg6 = const_cast< bcos::Error * >(reinterpret_cast< std::shared_ptr< const bcos::Error > * >(argp6)->get());
     }
   }
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(bcos::Error const &)*arg6);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -16338,7 +16660,14 @@ SWIGINTERN PyObject *_wrap_Error_buildError__SWIG_2(PyObject *self, Py_ssize_t n
     SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "Error_buildError" "', argument " "6"" of type '" "std::exception const &""'"); 
   }
   arg6 = reinterpret_cast< std::exception * >(argp6);
-  result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
+  {
+    try {
+      result = bcos::Error::buildError((char const *)arg1,(char const *)arg2,arg3,arg4,(std::string const &)*arg5,(std::exception const &)*arg6);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = new std::shared_ptr<  bcos::Error >(new bcos::Error(result));
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -16472,7 +16801,14 @@ SWIGINTERN PyObject *_wrap_new_Error__SWIG_0(PyObject *self, Py_ssize_t nobjs, P
   
   (void)self;
   if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (bcos::Error *)new bcos::Error();
+  {
+    try {
+      result = (bcos::Error *)new bcos::Error();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -16507,7 +16843,14 @@ SWIGINTERN PyObject *_wrap_new_Error__SWIG_1(PyObject *self, Py_ssize_t nobjs, P
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  result = (bcos::Error *)new bcos::Error(arg1,arg2);
+  {
+    try {
+      result = (bcos::Error *)new bcos::Error(arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -16580,7 +16923,14 @@ SWIGINTERN PyObject *_wrap_delete_Error(PyObject *self, PyObject *args) {
       arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16616,7 +16966,14 @@ SWIGINTERN PyObject *_wrap_Error_errorCode(PyObject *self, PyObject *args) {
       arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int64_t)((bcos::Error const *)arg1)->errorCode();
+  {
+    try {
+      result = (int64_t)((bcos::Error const *)arg1)->errorCode();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -16652,7 +17009,14 @@ SWIGINTERN PyObject *_wrap_Error_errorMessage(PyObject *self, PyObject *args) {
       arg1 = const_cast< bcos::Error * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
+  {
+    try {
+      result = (std::string *) &((bcos::Error const *)arg1)->errorMessage();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -16694,7 +17058,14 @@ SWIGINTERN PyObject *_wrap_Error_setErrorCode(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Error_setErrorCode" "', argument " "2"" of type '" "int64_t""'");
   } 
   arg2 = static_cast< int64_t >(val2);
-  (arg1)->setErrorCode(arg2);
+  {
+    try {
+      (arg1)->setErrorCode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -16741,7 +17112,14 @@ SWIGINTERN PyObject *_wrap_Error_setErrorMessage(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  (arg1)->setErrorMessage((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setErrorMessage((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -16876,7 +17254,14 @@ SWIGINTERN PyObject *_wrap_new_InputBuffer(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_InputBuffer", 0, 0, 0)) SWIG_fail;
-  result = (InputBuffer *)new InputBuffer();
+  {
+    try {
+      result = (InputBuffer *)new InputBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_InputBuffer, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -16899,7 +17284,14 @@ SWIGINTERN PyObject *_wrap_delete_InputBuffer(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_InputBuffer" "', argument " "1"" of type '" "InputBuffer *""'"); 
   }
   arg1 = reinterpret_cast< InputBuffer * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17032,7 +17424,14 @@ SWIGINTERN PyObject *_wrap_new_OutputBuffer(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_OutputBuffer", 0, 0, 0)) SWIG_fail;
-  result = (OutputBuffer *)new OutputBuffer();
+  {
+    try {
+      result = (OutputBuffer *)new OutputBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_OutputBuffer, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -17055,9 +17454,16 @@ SWIGINTERN PyObject *_wrap_delete_OutputBuffer(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_OutputBuffer" "', argument " "1"" of type '" "OutputBuffer *""'"); 
   }
   arg1 = reinterpret_cast< OutputBuffer * >(argp1);
-  delete arg1;
-  resultobj = SWIG_Py_Void();
-  return resultobj;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
 fail:
   return NULL;
 }
@@ -17101,7 +17507,14 @@ SWIGINTERN PyObject *_wrap_delete_FrontConfig(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17137,7 +17550,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_threadPoolSize(PyObject *self, PyObject *
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
+  {
+    try {
+      result = (int)((ppc::front::FrontConfig const *)arg1)->threadPoolSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_int(static_cast< int >(result));
   return resultobj;
 fail:
@@ -17179,7 +17599,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setThreadPoolSize(PyObject *self, PyObjec
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FrontConfig_setThreadPoolSize" "', argument " "2"" of type '" "int""'");
   } 
   arg2 = static_cast< int >(val2);
-  (arg1)->setThreadPoolSize(arg2);
+  {
+    try {
+      (arg1)->setThreadPoolSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17215,7 +17642,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_nodeID(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
+  {
+    try {
+      result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->nodeID();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -17262,7 +17696,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setNodeID(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  (arg1)->setNodeID((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setNodeID((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -17300,7 +17741,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_selfEndPoint(PyObject *self, PyObject *ar
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &((ppc::front::FrontConfig const *)arg1)->selfEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
@@ -17336,7 +17784,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_mutableSelfEndPoint(PyObject *self, PyObj
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableSelfEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
@@ -17381,7 +17836,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setSelfEndPoint(PyObject *self, PyObject
     SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "FrontConfig_setSelfEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint const &""'"); 
   }
   arg2 = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
-  (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
+  {
+    try {
+      (arg1)->setSelfEndPoint((ppc::protocol::EndPoint const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17428,7 +17890,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setGatewayGrpcTarget(PyObject *self, PyOb
     }
     arg2 = ptr;
   }
-  (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setGatewayGrpcTarget((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -17466,7 +17935,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_gatewayGrpcTarget(PyObject *self, PyObjec
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
+  {
+    try {
+      result = (std::string *) &((ppc::front::FrontConfig const *)arg1)->gatewayGrpcTarget();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -17512,7 +17988,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setGrpcConfig(PyObject *self, PyObject *a
     if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp2);
   }
-  (arg1)->setGrpcConfig(arg2);
+  {
+    try {
+      (arg1)->setGrpcConfig(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17548,7 +18031,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_grpcConfig(PyObject *self, PyObject *args
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::front::FrontConfig const *)arg1)->grpcConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN);
@@ -17587,7 +18077,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_generateNodeInfo(PyObject *self, PyObject
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
+  {
+    try {
+      result = ((ppc::front::FrontConfig const *)arg1)->generateNodeInfo();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
@@ -17626,7 +18123,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_getComponents(PyObject *self, PyObject *a
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
+  {
+    try {
+      result = (std::vector< std::string,std::allocator< std::string > > *) &((ppc::front::FrontConfig const *)arg1)->getComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(*result));
   return resultobj;
 fail:
@@ -17673,7 +18177,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setComponents(PyObject *self, PyObject *a
     }
     arg2 = ptr;
   }
-  (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2);
+  {
+    try {
+      (arg1)->setComponents((std::vector< std::string,std::allocator< std::string > > const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -17722,7 +18233,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_addComponent(PyObject *self, PyObject *ar
     }
     arg2 = ptr;
   }
-  (arg1)->addComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->addComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -17760,7 +18278,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_mutableComponents(PyObject *self, PyObjec
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents();
+  {
+    try {
+      result = (std::vector< std::string,std::allocator< std::string > > *) &(arg1)->mutableComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_std__string_t, 0 |  0 );
   return resultobj;
 fail:
@@ -17796,7 +18321,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_meta(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::FrontConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::front::FrontConfig const *)arg1)->meta();
+  {
+    try {
+      result = ((ppc::front::FrontConfig const *)arg1)->meta();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -17840,7 +18372,14 @@ SWIGINTERN PyObject *_wrap_FrontConfig_setMeta(PyObject *self, PyObject *args) {
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setMeta(arg2);
+  {
+    try {
+      (arg1)->setMeta(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17870,7 +18409,14 @@ SWIGINTERN PyObject *_wrap_delete_FrontConfigBuilder(PyObject *self, PyObject *a
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_FrontConfigBuilder" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder *""'"); 
   }
   arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -17892,7 +18438,14 @@ SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_0(PyObject *self, Py_s
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FrontConfigBuilder_build" "', argument " "1"" of type '" "ppc::front::FrontConfigBuilder const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::front::FrontConfigBuilder * >(argp1);
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
+  {
+    try {
+      result = ((ppc::front::FrontConfigBuilder const *)arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
@@ -17935,7 +18488,14 @@ SWIGINTERN PyObject *_wrap_FrontConfigBuilder_build__SWIG_1(PyObject *self, Py_s
     arg3 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+  {
+    try {
+      result = ((ppc::front::FrontConfigBuilder const *)arg1)->build(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
@@ -18025,7 +18585,14 @@ SWIGINTERN PyObject *_wrap_printFrontDesc(PyObject *self, PyObject *args) {
       arg1 = (argp1) ? reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
+  {
+    try {
+      result = ppc::front::printFrontDesc((std::shared_ptr< ppc::front::FrontConfig > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -18039,7 +18606,14 @@ SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_0(PyObject *self, Py_ssize_t nobjs
   
   (void)self;
   if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -18074,7 +18648,14 @@ SWIGINTERN PyObject *_wrap_new_EndPoint__SWIG_1(PyObject *self, Py_ssize_t nobjs
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_EndPoint" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
+  {
+    try {
+      result = (ppc::protocol::EndPoint *)new ppc::protocol::EndPoint((std::string const &)*arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NEW |  0 );
   if (SWIG_IsNewObj(res1)) delete arg1;
   return resultobj;
@@ -18134,7 +18715,14 @@ SWIGINTERN PyObject *_wrap_delete_EndPoint(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_EndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18158,7 +18746,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_host(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_host" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->host();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -18182,7 +18777,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_port(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_port" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::EndPoint const *)arg1)->port();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -18214,7 +18816,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_setHost(PyObject *self, PyObject *args) {
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setHost(arg2);
+  {
+    try {
+      (arg1)->setHost(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18244,7 +18853,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_setPort(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "EndPoint_setPort" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPort(arg2);
+  {
+    try {
+      (arg1)->setPort(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18279,7 +18895,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_setListenIp(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  (arg1)->setListenIp((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setListenIp((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -18305,7 +18928,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_entryPoint(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_entryPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
+  {
+    try {
+      result = ((ppc::protocol::EndPoint const *)arg1)->entryPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -18329,7 +18959,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_listenEndPoint(PyObject *self, PyObject *arg
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
+  {
+    try {
+      result = ((ppc::protocol::EndPoint const *)arg1)->listenEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -18353,7 +18990,14 @@ SWIGINTERN PyObject *_wrap_EndPoint_listenIp(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EndPoint_listenIp" "', argument " "1"" of type '" "ppc::protocol::EndPoint const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-  result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::EndPoint const *)arg1)->listenIp();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -18378,7 +19022,14 @@ SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize
   
   (void)self;
   if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -18416,7 +19067,14 @@ SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
-  result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+  {
+    try {
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -18475,7 +19133,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObj
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+  {
+    try {
+      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -18513,7 +19178,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  (arg1)->setEndPoint(arg2);
+  {
+    try {
+      (arg1)->setEndPoint(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18543,7 +19215,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self,
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableHealthCheck(arg2);
+  {
+    try {
+      (arg1)->setEnableHealthCheck(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18567,7 +19246,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *a
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
@@ -18591,7 +19277,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyOb
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+  {
+    try {
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
@@ -18615,7 +19308,14 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, Py
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -18638,7 +19338,14 @@ SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *arg
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
   arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18663,7 +19370,14 @@ SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) {
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail;
-  result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+  {
+    try {
+      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -18701,7 +19415,14 @@ SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18737,7 +19458,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -18784,7 +19512,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObj
     }
     arg2 = ptr;
   }
-  (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -18822,7 +19557,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -18864,7 +19606,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObj
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableHealthCheck(arg2);
+  {
+    try {
+      (arg1)->setEnableHealthCheck(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18906,7 +19655,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObjec
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
-  (arg1)->setEnableDnslookup(arg2);
+  {
+    try {
+      (arg1)->setEnableDnslookup(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -18942,7 +19698,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+  {
+    try {
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -18978,7 +19741,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_maxSendMessageSize(PyObject *self, PyObjec
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+  {
+    try {
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
@@ -19014,7 +19784,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_maxReceivedMessageSize(PyObject *self, PyO
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+  {
+    try {
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
@@ -19056,7 +19833,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxSendMessageSize(PyObject *self, PyOb
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "2"" of type '" "uint64_t""'");
   } 
   arg2 = static_cast< uint64_t >(val2);
-  (arg1)->setMaxSendMessageSize(arg2);
+  {
+    try {
+      (arg1)->setMaxSendMessageSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19098,7 +19882,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxReceivedMessageSize(PyObject *self,
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "2"" of type '" "uint64_t""'");
   } 
   arg2 = static_cast< uint64_t >(val2);
-  (arg1)->setMaxReceivedMessageSize(arg2);
+  {
+    try {
+      (arg1)->setMaxReceivedMessageSize(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19134,7 +19925,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_compressAlgorithm(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+  {
+    try {
+      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_int(static_cast< int >(result));
   return resultobj;
 fail:
@@ -19176,7 +19974,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setCompressAlgorithm(PyObject *self, PyObj
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "2"" of type '" "int""'");
   } 
   arg2 = static_cast< int >(val2);
-  (arg1)->setCompressAlgorithm(arg2);
+  {
+    try {
+      (arg1)->setCompressAlgorithm(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19221,7 +20026,14 @@ SWIGINTERN PyObject *_wrap_printGrpcConfig(PyObject *self, PyObject *args) {
       arg1 = (argp1) ? reinterpret_cast< ppc::protocol::GrpcConfig::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printGrpcConfig((std::shared_ptr< ppc::protocol::GrpcConfig > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -19256,7 +20068,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeader(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19311,7 +20130,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_encode(PyObject *self, PyObject
       arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+  {
+    try {
+      ((ppc::protocol::MessageOptionalHeader const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19370,7 +20196,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_decode(PyObject *self, PyObject
     SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "MessageOptionalHeader_decode" "', argument " "3"" of type '" "uint64_t""'");
   } 
   arg3 = static_cast< uint64_t >(val3);
-  result = (int64_t)(arg1)->decode(arg2,arg3);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -19406,7 +20239,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_componentType(PyObject *self, P
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->componentType();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -19450,7 +20290,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setComponentType(PyObject *self
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setComponentType(arg2);
+  {
+    try {
+      (arg1)->setComponentType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19486,7 +20333,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcNodeBuffer(PyObject *self, P
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcNodeBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
   }
@@ -19557,7 +20411,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcNodeBuffer(PyObject *self
     arg2 = (char *) buf;
     arg3 = (uint64_t) (size / sizeof(char));
   }
-  (arg1)->setSrcNodeBuffer(arg2,arg3);
+  {
+    try {
+      (arg1)->setSrcNodeBuffer(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19593,7 +20454,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstNodeBuffer(PyObject *self, P
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->dstNodeBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
   }
@@ -19664,7 +20532,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstNodeBuffer(PyObject *self
     arg2 = (char *) buf;
     arg3 = (uint64_t) (size / sizeof(char));
   }
-  (arg1)->setDstNodeBuffer(arg2,arg3);
+  {
+    try {
+      (arg1)->setDstNodeBuffer(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19700,7 +20575,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_dstInst(PyObject *self, PyObjec
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->dstInst();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -19747,7 +20629,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setDstInst(PyObject *self, PyOb
     }
     arg2 = ptr;
   }
-  (arg1)->setDstInst((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setDstInst((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -19785,7 +20674,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_topic(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageOptionalHeader const *)arg1)->topic();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -19835,7 +20731,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_0(PyObject *self
   }
   arg2 = reinterpret_cast< std::string * >(argp2);
   rvrdeleter2.reset(arg2);
-  (arg1)->setTopic((std::string &&)*arg2);
+  {
+    try {
+      (arg1)->setTopic((std::string &&)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -19881,7 +20784,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setTopic__SWIG_1(PyObject *self
     }
     arg2 = ptr;
   }
-  (arg1)->setTopic((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -19962,7 +20872,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_srcInst(PyObject *self, PyObjec
       arg1 = const_cast< ppc::protocol::MessageOptionalHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+  {
+    try {
+      result = ((ppc::protocol::MessageOptionalHeader const *)arg1)->srcInst();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -20009,7 +20926,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeader_setSrcInst(PyObject *self, PyOb
     }
     arg2 = ptr;
   }
-  (arg1)->setSrcInst((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setSrcInst((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -20053,7 +20977,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageHeader(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20108,7 +21039,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_encode(PyObject *self, PyObject *args)
       arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+  {
+    try {
+      ((ppc::protocol::MessageHeader const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20159,7 +21097,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_decode(PyObject *self, PyObject *args)
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -20195,7 +21140,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_version(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
+  {
+    try {
+      result = (uint8_t)((ppc::protocol::MessageHeader const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
   return resultobj;
 fail:
@@ -20237,7 +21189,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setVersion(PyObject *self, PyObject *ar
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setVersion" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20273,7 +21232,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_traceID(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->traceID();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -20317,7 +21283,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setTraceID(PyObject *self, PyObject *ar
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setTraceID(arg2);
+  {
+    try {
+      (arg1)->setTraceID(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20353,7 +21326,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_srcGwNode(PyObject *self, PyObject *arg
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->srcGwNode();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -20400,7 +21380,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setSrcGwNode(PyObject *self, PyObject *
     }
     arg2 = ptr;
   }
-  (arg1)->setSrcGwNode((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setSrcGwNode((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -20438,7 +21425,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_dstGwNode(PyObject *self, PyObject *arg
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessageHeader const *)arg1)->dstGwNode();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -20485,7 +21479,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setDstGwNode(PyObject *self, PyObject *
     }
     arg2 = ptr;
   }
-  (arg1)->setDstGwNode((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setDstGwNode((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -20523,7 +21524,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_packetType(PyObject *self, PyObject *ar
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->packetType();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -20565,7 +21573,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setPacketType(PyObject *self, PyObject
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPacketType(arg2);
+  {
+    try {
+      (arg1)->setPacketType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20601,7 +21616,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_ttl(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
+  {
+    try {
+      result = (int16_t)((ppc::protocol::MessageHeader const *)arg1)->ttl();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_short(static_cast< short >(result));
   return resultobj;
 fail:
@@ -20643,7 +21665,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setTTL(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setTTL" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setTTL(arg2);
+  {
+    try {
+      (arg1)->setTTL(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20679,7 +21708,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_ext(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -20721,7 +21757,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setExt(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessageHeader_setExt" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20757,7 +21800,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_optionalField(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->optionalField();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -20806,7 +21856,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setOptionalField(PyObject *self, PyObje
     if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2);
   }
-  (arg1)->setOptionalField(arg2);
+  {
+    try {
+      (arg1)->setOptionalField(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20842,7 +21899,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_length(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -20878,7 +21942,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_isRespPacket(PyObject *self, PyObject *
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)((ppc::protocol::MessageHeader const *)arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -20913,7 +21984,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setRespPacket(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -20949,7 +22027,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_srcP2PNodeIDView(PyObject *self, PyObje
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->srcP2PNodeIDView();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
@@ -20985,7 +22070,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_dstP2PNodeIDView(PyObject *self, PyObje
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
+  {
+    try {
+      result = ((ppc::protocol::MessageHeader const *)arg1)->dstP2PNodeIDView();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj((new std::string_view(result)), SWIGTYPE_p_std__string_view, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
@@ -21021,7 +22113,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_routeType(PyObject *self, PyObject *arg
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessageHeader const *)arg1)->routeType();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -21071,7 +22170,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_setRouteType(PyObject *self, PyObject *
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  (arg1)->setRouteType(arg2);
+  {
+    try {
+      (arg1)->setRouteType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21107,7 +22213,14 @@ SWIGINTERN PyObject *_wrap_MessageHeader_hasOptionalField(PyObject *self, PyObje
       arg1 = const_cast< ppc::protocol::MessageHeader * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+  {
+    try {
+      result = (bool)((ppc::protocol::MessageHeader const *)arg1)->hasOptionalField();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -21149,7 +22262,14 @@ SWIGINTERN PyObject *_wrap_delete_Message(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21185,7 +22305,14 @@ SWIGINTERN PyObject *_wrap_Message_header(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->header();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->header();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
@@ -21234,7 +22361,14 @@ SWIGINTERN PyObject *_wrap_Message_setHeader(PyObject *self, PyObject *args) {
     if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageHeader::Ptr * >(argp2);
   }
-  (arg1)->setHeader(arg2);
+  {
+    try {
+      (arg1)->setHeader(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21270,7 +22404,14 @@ SWIGINTERN PyObject *_wrap_Message_version(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -21312,7 +22453,14 @@ SWIGINTERN PyObject *_wrap_Message_setVersion(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setVersion" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21348,7 +22496,14 @@ SWIGINTERN PyObject *_wrap_Message_packetType(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->packetType();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -21390,7 +22545,14 @@ SWIGINTERN PyObject *_wrap_Message_setPacketType(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setPacketType" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setPacketType(arg2);
+  {
+    try {
+      (arg1)->setPacketType(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21426,7 +22588,14 @@ SWIGINTERN PyObject *_wrap_Message_seq(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::Message const *)arg1)->seq();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -21470,7 +22639,14 @@ SWIGINTERN PyObject *_wrap_Message_setSeq(PyObject *self, PyObject *args) {
     arg2 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  (arg1)->setSeq(arg2);
+  {
+    try {
+      (arg1)->setSeq(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21506,7 +22682,14 @@ SWIGINTERN PyObject *_wrap_Message_ext(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::Message const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -21548,7 +22731,14 @@ SWIGINTERN PyObject *_wrap_Message_setExt(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "Message_setExt" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21584,7 +22774,14 @@ SWIGINTERN PyObject *_wrap_Message_isRespPacket(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)((ppc::protocol::Message const *)arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -21619,7 +22816,14 @@ SWIGINTERN PyObject *_wrap_Message_setRespPacket(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21665,7 +22869,14 @@ SWIGINTERN PyObject *_wrap_Message_setPayload(PyObject *self, PyObject *args) {
     if (argp2) arg2 = *(reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
   }
-  (arg1)->setPayload(arg2);
+  {
+    try {
+      (arg1)->setPayload(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21701,7 +22912,14 @@ SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->payloadBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
   }
@@ -21749,7 +22967,14 @@ SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *arg
     if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2);
   }
-  (arg1)->setFrontMessage(arg2);
+  {
+    try {
+      (arg1)->setFrontMessage(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -21785,7 +23010,14 @@ SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
+  {
+    try {
+      result = (ppc::protocol::MessagePayload::Ptr *) &((ppc::protocol::Message const *)arg1)->frontMessage();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
@@ -21844,7 +23076,14 @@ SWIGINTERN PyObject *_wrap_Message_encode(PyObject *self, PyObject *args) {
       arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  result = (bool)(arg1)->encode(*arg2);
+  {
+    try {
+      result = (bool)(arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -21895,7 +23134,14 @@ SWIGINTERN PyObject *_wrap_Message_decode(PyObject *self, PyObject *args) {
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -21931,7 +23177,14 @@ SWIGINTERN PyObject *_wrap_Message_length(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+  {
+    try {
+      result = (uint32_t)((ppc::protocol::Message const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
   return resultobj;
 fail:
@@ -21967,7 +23220,14 @@ SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::Message const *)arg1)->payload();
+  {
+    try {
+      result = ((ppc::protocol::Message const *)arg1)->payload();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::bytes > *smartresult = result ? new std::shared_ptr<  bcos::bytes >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__bytes_t, SWIG_POINTER_OWN);
@@ -22012,7 +23272,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageHeaderBuilder(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -22062,7 +23329,14 @@ SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_0(PyObject *self, Py
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
@@ -22099,7 +23373,14 @@ SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_1(PyObject *self, Py
       arg1 = const_cast< ppc::protocol::MessageHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeader_t, SWIG_POINTER_OWN);
@@ -22154,7 +23435,14 @@ SWIGINTERN PyObject *_wrap_MessageHeaderBuilder_build__SWIG_2(PyObject *self, Py
       arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
     }
   }
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    try {
+      result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -22250,7 +23538,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageBuilder(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -22284,7 +23579,14 @@ SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_0(PyObject *self, Py_ssize
       arg1 = const_cast< ppc::protocol::MessageBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -22337,7 +23639,14 @@ SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_1(PyObject *self, Py_ssize
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -22425,7 +23734,14 @@ SWIGINTERN PyObject *_wrap_MessageBuilder_build__SWIG_2(PyObject *self, Py_ssize
   }
   arg4 = reinterpret_cast< bcos::bytes * >(argp4);
   rvrdeleter4.reset(arg4);
-  result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
+  {
+    try {
+      result = (arg1)->build(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,(bcos::bytes &&)*arg4);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -22530,7 +23846,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageOptionalHeaderBuilder(PyObject *self, P
       arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -22582,7 +23905,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_0(PyObject *
       arg2 = (argp2) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp2) : &tempshared2;
     }
   }
-  result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+  {
+    try {
+      result = (arg1)->build((ppc::protocol::MessageOptionalHeader::Ptr const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -22619,7 +23949,14 @@ SWIGINTERN PyObject *_wrap_MessageOptionalHeaderBuilder_build__SWIG_1(PyObject *
       arg1 = const_cast< ppc::protocol::MessageOptionalHeaderBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeader > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeader >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeader_t, SWIG_POINTER_OWN);
@@ -22695,7 +24032,14 @@ SWIGINTERN PyObject *_wrap_printOptionalField(PyObject *self, PyObject *args) {
     if (argp1) arg1 = *(reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp1);
   }
-  result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
+  {
+    try {
+      result = ppc::protocol::printOptionalField(SWIG_STD_MOVE(arg1));
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -22729,7 +24073,14 @@ SWIGINTERN PyObject *_wrap_printMessage(PyObject *self, PyObject *args) {
       arg1 = (argp1) ? reinterpret_cast< ppc::protocol::Message::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printMessage((std::shared_ptr< ppc::protocol::Message > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -22764,7 +24115,14 @@ SWIGINTERN PyObject *_wrap_delete_MessagePayload(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -22820,7 +24178,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_encode(PyObject *self, PyObject *args)
       arg2 = const_cast< bcos::bytes * >(reinterpret_cast< std::shared_ptr<  bcos::bytes > * >(argp2)->get());
     }
   }
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
+  {
+    try {
+      result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->encode(*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -22871,7 +24236,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_decode(PyObject *self, PyObject *args)
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (int64_t)(arg1)->decode(arg2);
+  {
+    try {
+      result = (int64_t)(arg1)->decode(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -22907,7 +24279,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_version(PyObject *self, PyObject *args
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
+  {
+    try {
+      result = (uint8_t)((ppc::protocol::MessagePayload const *)arg1)->version();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_char(static_cast< unsigned char >(result));
   return resultobj;
 fail:
@@ -22949,7 +24328,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setVersion(PyObject *self, PyObject *a
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setVersion" "', argument " "2"" of type '" "uint8_t""'");
   } 
   arg2 = static_cast< uint8_t >(val2);
-  (arg1)->setVersion(arg2);
+  {
+    try {
+      (arg1)->setVersion(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -22985,7 +24371,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_dataBuffer(PyObject *self, PyObject *a
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+  {
+    try {
+      result = ((ppc::protocol::MessagePayload const *)arg1)->dataBuffer();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     resultobj = PyBytes_FromStringAndSize((const char *)(&result)->data, (&result)->len);
   }
@@ -23023,7 +24416,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_seq(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->seq();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -23065,7 +24465,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setSeq(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setSeq" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setSeq(arg2);
+  {
+    try {
+      (arg1)->setSeq(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23101,7 +24508,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_length(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
+  {
+    try {
+      result = (int64_t)((ppc::protocol::MessagePayload const *)arg1)->length();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_long_SS_long(static_cast< long long >(result));
   return resultobj;
 fail:
@@ -23137,7 +24551,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_traceID(PyObject *self, PyObject *args
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::MessagePayload const *)arg1)->traceID();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -23184,7 +24605,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setTraceID(PyObject *self, PyObject *a
     }
     arg2 = ptr;
   }
-  (arg1)->setTraceID((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setTraceID((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -23222,7 +24650,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_ext(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+  {
+    try {
+      result = (uint16_t)((ppc::protocol::MessagePayload const *)arg1)->ext();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_unsigned_SS_short(static_cast< unsigned short >(result));
   return resultobj;
 fail:
@@ -23264,7 +24699,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setExt(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "MessagePayload_setExt" "', argument " "2"" of type '" "uint16_t""'");
   } 
   arg2 = static_cast< uint16_t >(val2);
-  (arg1)->setExt(arg2);
+  {
+    try {
+      (arg1)->setExt(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23299,7 +24741,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_setRespPacket(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->setRespPacket();
+  {
+    try {
+      (arg1)->setRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23335,7 +24784,14 @@ SWIGINTERN PyObject *_wrap_MessagePayload_isRespPacket(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (bool)(arg1)->isRespPacket();
+  {
+    try {
+      result = (bool)(arg1)->isRespPacket();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
@@ -23377,7 +24833,14 @@ SWIGINTERN PyObject *_wrap_delete_MessagePayloadBuilder(PyObject *self, PyObject
       arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23411,7 +24874,14 @@ SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_0(PyObject *self, P
       arg1 = const_cast< ppc::protocol::MessagePayloadBuilder * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->build();
+  {
+    try {
+      result = (arg1)->build();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
@@ -23464,7 +24934,14 @@ SWIGINTERN PyObject *_wrap_MessagePayloadBuilder_build__SWIG_1(PyObject *self, P
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  result = (arg1)->build(arg2);
+  {
+    try {
+      result = (arg1)->build(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayload > *smartresult = result ? new std::shared_ptr<  ppc::protocol::MessagePayload >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, SWIG_POINTER_OWN);
@@ -23547,7 +25024,14 @@ SWIGINTERN PyObject *_wrap_delete_IFrontClient(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFrontClient * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23615,7 +25099,14 @@ SWIGINTERN PyObject *_wrap_IFrontClient_onReceiveMessage(PyObject *self, PyObjec
       if (SWIG_IsNewObj(res3)) delete temp;
     }
   }
-  (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->onReceiveMessage((ppc::protocol::Message::Ptr const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23640,14 +25131,21 @@ SWIGINTERN PyObject *_wrap_new_ErrorCallback(PyObject *self, PyObject *args) {
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  {
+    try {
+      if ( arg1 != Py_None ) {
+        /* subclassed */
+        result = (ppc::front::ErrorCallback *)new SwigDirector_ErrorCallback(arg1); 
+      } else {
+        SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+        SWIG_fail;
+      }
+      
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
   }
-  
   {
     std::shared_ptr<  ppc::front::ErrorCallback > *smartresult = result ? new std::shared_ptr<  ppc::front::ErrorCallback >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__ErrorCallback_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -23685,7 +25183,14 @@ SWIGINTERN PyObject *_wrap_delete_ErrorCallback(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::front::ErrorCallback * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23736,10 +25241,17 @@ SWIGINTERN PyObject *_wrap_ErrorCallback_onError(PyObject *self, PyObject *args)
   director = SWIG_DIRECTOR_CAST(arg1);
   upcall = (director && (director->swig_get_self()==swig_obj[0]));
   try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::ErrorCallback::onError");
-    } else {
-      (arg1)->onError(arg2);
+    {
+      try {
+        if (upcall) {
+          Swig::DirectorPureVirtualException::raise("ppc::front::ErrorCallback::onError");
+        } else {
+          (arg1)->onError(arg2);
+        }
+      }
+      catch (const std::exception& e) {
+        SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+      }
     }
   } catch (Swig::DirectorException&) {
     SWIG_fail;
@@ -23811,14 +25323,21 @@ SWIGINTERN PyObject *_wrap_new_MessageDispatcherHandler(PyObject *self, PyObject
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  {
+    try {
+      if ( arg1 != Py_None ) {
+        /* subclassed */
+        result = (ppc::front::MessageDispatcherHandler *)new SwigDirector_MessageDispatcherHandler(arg1); 
+      } else {
+        SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+        SWIG_fail;
+      }
+      
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
   }
-  
   {
     std::shared_ptr<  ppc::front::MessageDispatcherHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::MessageDispatcherHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__MessageDispatcherHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -23856,7 +25375,14 @@ SWIGINTERN PyObject *_wrap_delete_MessageDispatcherHandler(PyObject *self, PyObj
       arg1 = const_cast< ppc::front::MessageDispatcherHandler * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -23907,10 +25433,17 @@ SWIGINTERN PyObject *_wrap_MessageDispatcherHandler_onMessage(PyObject *self, Py
   director = SWIG_DIRECTOR_CAST(arg1);
   upcall = (director && (director->swig_get_self()==swig_obj[0]));
   try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::MessageDispatcherHandler::onMessage");
-    } else {
-      (arg1)->onMessage(arg2);
+    {
+      try {
+        if (upcall) {
+          Swig::DirectorPureVirtualException::raise("ppc::front::MessageDispatcherHandler::onMessage");
+        } else {
+          (arg1)->onMessage(arg2);
+        }
+      }
+      catch (const std::exception& e) {
+        SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+      }
     }
   } catch (Swig::DirectorException&) {
     SWIG_fail;
@@ -23996,7 +25529,14 @@ SWIGINTERN PyObject *_wrap_new_SendResponseHandler(PyObject *self, PyObject *arg
       if (SWIG_IsNewObj(res1)) delete temp;
     }
   }
-  result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
+  {
+    try {
+      result = (ppc::front::SendResponseHandler *)new ppc::front::SendResponseHandler(arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__front__SendResponseHandler, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -24019,7 +25559,14 @@ SWIGINTERN PyObject *_wrap_delete_SendResponseHandler(PyObject *self, PyObject *
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SendResponseHandler" "', argument " "1"" of type '" "ppc::front::SendResponseHandler *""'"); 
   }
   arg1 = reinterpret_cast< ppc::front::SendResponseHandler * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24058,7 +25605,14 @@ SWIGINTERN PyObject *_wrap_SendResponseHandler_sendResponse(PyObject *self, PyOb
   }
   arg2 = reinterpret_cast< std::shared_ptr< bcos::bytes > * >(argp2);
   rvrdeleter2.reset(arg2);
-  (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
+  {
+    try {
+      (arg1)->sendResponse((std::shared_ptr< bcos::bytes > &&)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24087,14 +25641,21 @@ SWIGINTERN PyObject *_wrap_new_IMessageHandler(PyObject *self, PyObject *args) {
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  {
+    try {
+      if ( arg1 != Py_None ) {
+        /* subclassed */
+        result = (ppc::front::IMessageHandler *)new SwigDirector_IMessageHandler(arg1); 
+      } else {
+        SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+        SWIG_fail;
+      }
+      
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
   }
-  
   {
     std::shared_ptr<  ppc::front::IMessageHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::IMessageHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IMessageHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -24132,7 +25693,14 @@ SWIGINTERN PyObject *_wrap_delete_IMessageHandler(PyObject *self, PyObject *args
       arg1 = const_cast< ppc::front::IMessageHandler * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24211,10 +25779,17 @@ SWIGINTERN PyObject *_wrap_IMessageHandler_onMessage(PyObject *self, PyObject *a
   director = SWIG_DIRECTOR_CAST(arg1);
   upcall = (director && (director->swig_get_self()==swig_obj[0]));
   try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::IMessageHandler::onMessage");
-    } else {
-      (arg1)->onMessage(arg2,arg3,arg4);
+    {
+      try {
+        if (upcall) {
+          Swig::DirectorPureVirtualException::raise("ppc::front::IMessageHandler::onMessage");
+        } else {
+          (arg1)->onMessage(arg2,arg3,arg4);
+        }
+      }
+      catch (const std::exception& e) {
+        SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+      }
     }
   } catch (Swig::DirectorException&) {
     SWIG_fail;
@@ -24286,14 +25861,21 @@ SWIGINTERN PyObject *_wrap_new_GetPeersInfoHandler(PyObject *self, PyObject *arg
   if (!args) SWIG_fail;
   swig_obj[0] = args;
   arg1 = swig_obj[0];
-  if ( arg1 != Py_None ) {
-    /* subclassed */
-    result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(arg1); 
-  } else {
-    SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
-    SWIG_fail;
+  {
+    try {
+      if ( arg1 != Py_None ) {
+        /* subclassed */
+        result = (ppc::front::GetPeersInfoHandler *)new SwigDirector_GetPeersInfoHandler(arg1); 
+      } else {
+        SWIG_SetErrorMsg(PyExc_RuntimeError,"accessing abstract class or protected constructor"); 
+        SWIG_fail;
+      }
+      
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
   }
-  
   {
     std::shared_ptr<  ppc::front::GetPeersInfoHandler > *smartresult = result ? new std::shared_ptr<  ppc::front::GetPeersInfoHandler >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__GetPeersInfoHandler_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -24331,7 +25913,14 @@ SWIGINTERN PyObject *_wrap_delete_GetPeersInfoHandler(PyObject *self, PyObject *
       arg1 = const_cast< ppc::front::GetPeersInfoHandler * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24395,10 +25984,17 @@ SWIGINTERN PyObject *_wrap_GetPeersInfoHandler_onPeersInfo(PyObject *self, PyObj
   director = SWIG_DIRECTOR_CAST(arg1);
   upcall = (director && (director->swig_get_self()==swig_obj[0]));
   try {
-    if (upcall) {
-      Swig::DirectorPureVirtualException::raise("ppc::front::GetPeersInfoHandler::onPeersInfo");
-    } else {
-      (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
+    {
+      try {
+        if (upcall) {
+          Swig::DirectorPureVirtualException::raise("ppc::front::GetPeersInfoHandler::onPeersInfo");
+        } else {
+          (arg1)->onPeersInfo(arg2,(std::string const &)*arg3);
+        }
+      }
+      catch (const std::exception& e) {
+        SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+      }
     }
   } catch (Swig::DirectorException&) {
     SWIG_fail;
@@ -24489,7 +26085,14 @@ SWIGINTERN PyObject *_wrap_delete_IFront(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24524,7 +26127,14 @@ SWIGINTERN PyObject *_wrap_IFront_start(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->start();
+  {
+    try {
+      (arg1)->start();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24559,7 +26169,14 @@ SWIGINTERN PyObject *_wrap_IFront_stop(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->stop();
+  {
+    try {
+      (arg1)->stop();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24618,7 +26235,14 @@ SWIGINTERN PyObject *_wrap_IFront_register_topic_handler(PyObject *self, PyObjec
     if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
   }
-  (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->register_topic_handler((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -24679,7 +26303,14 @@ SWIGINTERN PyObject *_wrap_IFront_register_msg_handler(PyObject *self, PyObject
     if (argp3) arg3 = *(reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::MessageDispatcherHandler::Ptr * >(argp3);
   }
-  (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+  {
+    try {
+      (arg1)->register_msg_handler((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -24816,7 +26447,14 @@ SWIGINTERN PyObject *_wrap_IFront_async_send_message(PyObject *self, PyObject *a
     if (argp9) arg9 = *(reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::IMessageHandler::Ptr * >(argp9);
   }
-  (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+  {
+    try {
+      (arg1)->async_send_message(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -24935,7 +26573,14 @@ SWIGINTERN PyObject *_wrap_IFront_async_send_response(PyObject *self, PyObject *
     if (argp8) arg8 = *(reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::ErrorCallback::Ptr * >(argp8);
   }
-  (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+  {
+    try {
+      (arg1)->async_send_response(arg2,arg3,(std::string const &)*arg4,arg5,arg6,arg7,arg8);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
   if (SWIG_IsNewObj(res4)) delete arg4;
@@ -25051,7 +26696,14 @@ SWIGINTERN PyObject *_wrap_IFront_push_msg(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "IFront_push_msg" "', argument " "7"" of type '" "long""'");
   } 
   arg7 = static_cast< long >(val7);
-  result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
+  {
+    try {
+      result = (arg1)->push_msg(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3,arg4,arg5,arg6,arg7);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -25110,7 +26762,14 @@ SWIGINTERN PyObject *_wrap_IFront_pop(PyObject *self, PyObject *args) {
     SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "IFront_pop" "', argument " "3"" of type '" "long""'");
   } 
   arg3 = static_cast< long >(val3);
-  result = (arg1)->pop((std::string const &)*arg2,arg3);
+  {
+    try {
+      result = (arg1)->pop((std::string const &)*arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -25163,7 +26822,14 @@ SWIGINTERN PyObject *_wrap_IFront_peek(PyObject *self, PyObject *args) {
     }
     arg2 = ptr;
   }
-  result = (arg1)->peek((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->peek((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::Message > *smartresult = result ? new std::shared_ptr<  ppc::protocol::Message >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, SWIG_POINTER_OWN);
@@ -25214,7 +26880,14 @@ SWIGINTERN PyObject *_wrap_IFront_asyncGetPeers(PyObject *self, PyObject *args)
     if (argp2) arg2 = *(reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::GetPeersInfoHandler::Ptr * >(argp2);
   }
-  (arg1)->asyncGetPeers(arg2);
+  {
+    try {
+      (arg1)->asyncGetPeers(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -25250,7 +26923,14 @@ SWIGINTERN PyObject *_wrap_IFront_nodeInfo(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
+  {
+    try {
+      result = (ppc::protocol::INodeInfo::Ptr *) &(arg1)->nodeInfo();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::INodeInfo > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::INodeInfo >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__INodeInfo_t, SWIG_POINTER_OWN);
@@ -25301,7 +26981,14 @@ SWIGINTERN PyObject *_wrap_IFront_registerTopic(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  result = (arg1)->registerTopic((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->registerTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -25354,7 +27041,14 @@ SWIGINTERN PyObject *_wrap_IFront_unRegisterTopic(PyObject *self, PyObject *args
     }
     arg2 = ptr;
   }
-  result = (arg1)->unRegisterTopic((std::string const &)*arg2);
+  {
+    try {
+      result = (arg1)->unRegisterTopic((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  bcos::Error > *smartresult = result ? new std::shared_ptr<  bcos::Error >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_bcos__Error_t, SWIG_POINTER_OWN);
@@ -25406,7 +27100,14 @@ SWIGINTERN PyObject *_wrap_IFront_registerComponent(PyObject *self, PyObject *ar
     }
     arg2 = ptr;
   }
-  (arg1)->registerComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->registerComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -25455,7 +27156,14 @@ SWIGINTERN PyObject *_wrap_IFront_unRegisterComponent(PyObject *self, PyObject *
     }
     arg2 = ptr;
   }
-  (arg1)->unRegisterComponent((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->unRegisterComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -25504,7 +27212,14 @@ SWIGINTERN PyObject *_wrap_IFront_updateMetaInfo(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  (arg1)->updateMetaInfo((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->updateMetaInfo((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -25567,7 +27282,14 @@ SWIGINTERN PyObject *_wrap_IFront_selectNodesByRoutePolicy(PyObject *self, PyObj
       arg3 = (argp3) ? reinterpret_cast< ppc::protocol::MessageOptionalHeader::Ptr * >(argp3) : &tempshared3;
     }
   }
-  result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+  {
+    try {
+      result = (arg1)->selectNodesByRoutePolicy(arg2,(ppc::protocol::MessageOptionalHeader::Ptr const &)*arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
   return resultobj;
 fail:
@@ -25603,7 +27325,14 @@ SWIGINTERN PyObject *_wrap_IFront_getNodeDiscovery(PyObject *self, PyObject *arg
       arg1 = const_cast< ppc::front::IFront * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (arg1)->getNodeDiscovery();
+  {
+    try {
+      result = (arg1)->getNodeDiscovery();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::INodeDiscovery > *smartresult = result ? new std::shared_ptr<  ppc::front::INodeDiscovery >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__INodeDiscovery_t, SWIG_POINTER_OWN);
@@ -25636,7 +27365,14 @@ SWIGINTERN PyObject *_wrap_delete_IFrontBuilder(PyObject *self, PyObject *args)
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_IFrontBuilder" "', argument " "1"" of type '" "ppc::front::IFrontBuilder *""'"); 
   }
   arg1 = reinterpret_cast< ppc::front::IFrontBuilder * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -25693,7 +27429,14 @@ SWIGINTERN PyObject *_wrap_IFrontBuilder_buildClient(PyObject *self, PyObject *a
     SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "IFrontBuilder_buildClient" "', argument " "4"" of type '" "bool""'");
   } 
   arg4 = static_cast< bool >(val4);
-  result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
+  {
+    try {
+      result = ((ppc::front::IFrontBuilder const *)arg1)->buildClient(arg2,arg3,arg4);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::IFrontClient > *smartresult = result ? new std::shared_ptr<  ppc::front::IFrontClient >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFrontClient_t, SWIG_POINTER_OWN);
@@ -25738,7 +27481,14 @@ SWIGINTERN PyObject *_wrap_delete_INodeDiscovery(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::front::INodeDiscovery * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -25774,7 +27524,14 @@ SWIGINTERN PyObject *_wrap_INodeDiscovery_getAliveNodeList(PyObject *self, PyObj
       arg1 = const_cast< ppc::front::INodeDiscovery * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+  {
+    try {
+      result = ((ppc::front::INodeDiscovery const *)arg1)->getAliveNodeList();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = swig::from(static_cast< std::vector< std::shared_ptr< ppc::protocol::INodeInfo >,std::allocator< std::shared_ptr< ppc::protocol::INodeInfo > > > >(result));
   return resultobj;
 fail:
@@ -25795,7 +27552,14 @@ SWIGINTERN PyObject *_wrap_new_TransportBuilder(PyObject *self, PyObject *args)
   
   (void)self;
   if (!SWIG_Python_UnpackTuple(args, "new_TransportBuilder", 0, 0, 0)) SWIG_fail;
-  result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
+  {
+    try {
+      result = (ppc::sdk::TransportBuilder *)new ppc::sdk::TransportBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__sdk__TransportBuilder, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
@@ -25818,7 +27582,14 @@ SWIGINTERN PyObject *_wrap_delete_TransportBuilder(PyObject *self, PyObject *arg
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_TransportBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
   }
   arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
-  delete arg1;
+  {
+    try {
+      delete arg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -25853,7 +27624,14 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_buildProTransport(PyObject *self, Py
     if (argp2) arg2 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp2);
   }
-  result = (arg1)->buildProTransport(arg2);
+  {
+    try {
+      result = (arg1)->buildProTransport(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_OWN);
@@ -25897,7 +27675,14 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_buildConfig(PyObject *self, PyObject
     arg3 = *ptr;
     if (SWIG_IsNewObj(res)) delete ptr;
   }
-  result = (arg1)->buildConfig(arg2,arg3);
+  {
+    try {
+      result = (arg1)->buildConfig(arg2,arg3);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::FrontConfig > *smartresult = result ? new std::shared_ptr<  ppc::front::FrontConfig >(result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
@@ -25924,7 +27709,14 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, P
     SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_frontConfigBuilder" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
   }
   arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
-  result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
+  {
+    try {
+      result = (ppc::front::FrontConfigBuilder::Ptr *) &(arg1)->frontConfigBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfigBuilder_t, 0 |  0 );
   return resultobj;
 fail:
@@ -25934,30 +27726,44 @@ SWIGINTERN PyObject *_wrap_TransportBuilder_frontConfigBuilder(PyObject *self, P
 
 SWIGINTERN PyObject *_wrap_TransportBuilder_initLog(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  std::string *arg1 = 0 ;
-  int res1 = SWIG_OLDOBJ ;
-  PyObject *swig_obj[1] ;
+  ppc::sdk::TransportBuilder *arg1 = (ppc::sdk::TransportBuilder *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "TransportBuilder_initLog", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__sdk__TransportBuilder, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "ppc::sdk::TransportBuilder *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::sdk::TransportBuilder * >(argp1);
   {
     std::string *ptr = (std::string *)0;
-    res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "TransportBuilder_initLog" "', argument " "2"" of type '" "std::string const &""'"); 
     }
     if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "TransportBuilder_initLog" "', argument " "1"" of type '" "std::string const &""'"); 
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "TransportBuilder_initLog" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  {
+    try {
+      (arg1)->initLog((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
-    arg1 = ptr;
   }
-  ppc::sdk::TransportBuilder::initLog((std::string const &)*arg1);
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res1)) delete arg1;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res1)) delete arg1;
+  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
@@ -25993,7 +27799,14 @@ SWIGINTERN PyObject *_wrap_new_Transport(PyObject *self, PyObject *args) {
     if (argp1) arg1 = *(reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1));
     if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::front::FrontConfig::Ptr * >(argp1);
   }
-  result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
+  {
+    try {
+      result = (ppc::sdk::Transport *)new ppc::sdk::Transport(arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::sdk::Transport > *smartresult = result ? new std::shared_ptr<  ppc::sdk::Transport >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__sdk__Transport_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
@@ -26031,7 +27844,14 @@ SWIGINTERN PyObject *_wrap_delete_Transport(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -26066,7 +27886,14 @@ SWIGINTERN PyObject *_wrap_Transport_start(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->start();
+  {
+    try {
+      (arg1)->start();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -26101,7 +27928,14 @@ SWIGINTERN PyObject *_wrap_Transport_stop(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (arg1)->stop();
+  {
+    try {
+      (arg1)->stop();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -26137,7 +27971,14 @@ SWIGINTERN PyObject *_wrap_Transport_getFront(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
+  {
+    try {
+      result = (ppc::front::IFront::Ptr *) &((ppc::sdk::Transport const *)arg1)->getFront();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::IFront > *smartresult = *result ? new std::shared_ptr<  ppc::front::IFront >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__IFront_t, SWIG_POINTER_OWN);
@@ -26176,7 +28017,14 @@ SWIGINTERN PyObject *_wrap_Transport_gateway(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
+  {
+    try {
+      result = (ppc::gateway::IGateway::Ptr *) &((ppc::sdk::Transport const *)arg1)->gateway();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__gateway__IGateway__Ptr, 0 |  0 );
   return resultobj;
 fail:
@@ -26212,7 +28060,14 @@ SWIGINTERN PyObject *_wrap_Transport_msgPayloadBuilder(PyObject *self, PyObject
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessagePayloadBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgPayloadBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessagePayloadBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessagePayloadBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayloadBuilder_t, SWIG_POINTER_OWN);
@@ -26251,7 +28106,14 @@ SWIGINTERN PyObject *_wrap_Transport_msgBuilder(PyObject *self, PyObject *args)
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageBuilder_t, SWIG_POINTER_OWN);
@@ -26290,7 +28152,14 @@ SWIGINTERN PyObject *_wrap_Transport_msgHeaderBuilder(PyObject *self, PyObject *
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->msgHeaderBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageHeaderBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageHeaderBuilder_t, SWIG_POINTER_OWN);
@@ -26329,7 +28198,14 @@ SWIGINTERN PyObject *_wrap_Transport_routeInfoBuilder(PyObject *self, PyObject *
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
+  {
+    try {
+      result = (ppc::protocol::MessageOptionalHeaderBuilder::Ptr *) &((ppc::sdk::Transport const *)arg1)->routeInfoBuilder();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::MessageOptionalHeaderBuilder >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessageOptionalHeaderBuilder_t, SWIG_POINTER_OWN);
@@ -26368,7 +28244,14 @@ SWIGINTERN PyObject *_wrap_Transport_getConfig(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::sdk::Transport * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
+  {
+    try {
+      result = (ppc::front::FrontConfig::Ptr *) &((ppc::sdk::Transport const *)arg1)->getConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   {
     std::shared_ptr<  ppc::front::FrontConfig > *smartresult = *result ? new std::shared_ptr<  ppc::front::FrontConfig >(*result) : 0;
     resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__front__FrontConfig_t, SWIG_POINTER_OWN);
@@ -26417,7 +28300,14 @@ SWIGINTERN PyObject *_wrap_delete_INodeInfo(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  (void)arg1; delete smartarg1;
+  {
+    try {
+      (void)arg1; delete smartarg1;
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -26453,7 +28343,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_endPoint(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+  {
+    try {
+      result = (std::string *) &((ppc::protocol::INodeInfo const *)arg1)->endPoint();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
@@ -26489,7 +28386,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_nodeID(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->nodeID();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_NewPointerObj((new bcos::bytesConstRef(result)), SWIGTYPE_p_bcos__bytesConstRef, SWIG_POINTER_OWN |  0 );
   return resultobj;
 fail:
@@ -26539,7 +28443,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_setNodeID(PyObject *self, PyObject *args) {
       if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
-  (arg1)->setNodeID(arg2);
+  {
+    try {
+      (arg1)->setNodeID(arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
@@ -26586,7 +28497,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_setEndPoint(PyObject *self, PyObject *args)
     }
     arg2 = ptr;
   }
-  (arg1)->setEndPoint((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setEndPoint((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -26636,7 +28554,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_addComponent(PyObject *self, PyObject *args
     }
     arg2 = ptr;
   }
-  result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)(arg1)->addComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -26686,7 +28611,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_eraseComponent(PyObject *self, PyObject *ar
     }
     arg2 = ptr;
   }
-  result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)(arg1)->eraseComponent((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -26736,7 +28668,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_componentExist(PyObject *self, PyObject *ar
     }
     arg2 = ptr;
   }
-  result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+  {
+    try {
+      result = (bool)((ppc::protocol::INodeInfo const *)arg1)->componentExist((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_bool(static_cast< bool >(result));
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -26774,7 +28713,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_copiedComponents(PyObject *self, PyObject *
       arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->copiedComponents();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result));
   return resultobj;
 fail:
@@ -26810,7 +28756,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_meta(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::INodeInfo * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+  {
+    try {
+      result = ((ppc::protocol::INodeInfo const *)arg1)->meta();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -26857,7 +28810,14 @@ SWIGINTERN PyObject *_wrap_INodeInfo_setMeta(PyObject *self, PyObject *args) {
     }
     arg2 = ptr;
   }
-  (arg1)->setMeta((std::string const &)*arg2);
+  {
+    try {
+      (arg1)->setMeta((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_Py_Void();
   if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
@@ -26900,7 +28860,14 @@ SWIGINTERN PyObject *_wrap_printNodeInfo(PyObject *self, PyObject *args) {
       arg1 = (argp1) ? reinterpret_cast< ppc::protocol::INodeInfo::Ptr * >(argp1) : &tempshared1;
     }
   }
-  result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+  {
+    try {
+      result = ppc::protocol::printNodeInfo((std::shared_ptr< ppc::protocol::INodeInfo > const &)*arg1);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
   resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
@@ -27417,7 +29384,7 @@ static PyMethodDef SwigMethods[] = {
 	 { "TransportBuilder_buildProTransport", _wrap_TransportBuilder_buildProTransport, METH_VARARGS, NULL},
 	 { "TransportBuilder_buildConfig", _wrap_TransportBuilder_buildConfig, METH_VARARGS, NULL},
 	 { "TransportBuilder_frontConfigBuilder", _wrap_TransportBuilder_frontConfigBuilder, METH_O, NULL},
-	 { "TransportBuilder_initLog", _wrap_TransportBuilder_initLog, METH_O, NULL},
+	 { "TransportBuilder_initLog", _wrap_TransportBuilder_initLog, METH_VARARGS, NULL},
 	 { "TransportBuilder_swigregister", TransportBuilder_swigregister, METH_O, NULL},
 	 { "TransportBuilder_swiginit", TransportBuilder_swiginit, METH_VARARGS, NULL},
 	 { "new_Transport", _wrap_new_Transport, METH_O, NULL},
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index c2f4ac9e..59c6ba39 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -8,11 +8,11 @@ from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 %module(moduleimport=MODULEIMPORT) wedpr_python_transport
 %module(directors="1") wedpr_python_transport
 
-%include 
-%include 
-%include 
-%include 
-%include 
+%include "stdint.i"
+%include "cpointer.i"
+%include "std_vector.i"
+%include "std_string.i"
+%include "std_shared_ptr.i"
 %include "typemaps.i"
 
 // shared_ptr definition
@@ -198,6 +198,15 @@ namespace bcos{
 }
 ///// tests  ///
 */
+%include "exception.i"
+%exception {
+    try {
+        $action
+    }
+    catch (const std::exception& e) {
+        SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+}
 
 // define the interface should been exposed
 %include "bcos-utilities/Error.h"
diff --git a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
index c8853eca..dbc5bb5c 100644
--- a/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
+++ b/cpp/wedpr-transport/sdk/src/ProTransportImpl.cpp
@@ -48,6 +48,13 @@ ProTransportImpl::ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keep
     m_server->registerService(m_frontService);
 }
 
+ProTransportImpl::~ProTransportImpl()
+{
+    TRANSPORT_LOG(INFO) << LOG_DESC("stop pro transport");
+    stop();
+    TRANSPORT_LOG(INFO) << LOG_DESC("stop pro transport success");
+}
+
 void ProTransportImpl::start()
 {
     m_timer = std::make_shared(m_keepAlivePeriodMs, "frontKeepAlive");
@@ -60,18 +67,39 @@ void ProTransportImpl::start()
         }
         transport->keepAlive();
     });
-    m_timer->start();
-    m_server->start();
-    // Note: the server is inited after start
-    m_frontService->setHealthCheckService(m_server->server()->GetHealthCheckService());
-    m_front->start();
+    if (m_timer)
+    {
+        m_timer->start();
+    }
+    if (m_server)
+    {
+        m_server->start();
+    }
+    if (m_frontService)
+    {
+        // Note: the server is inited after start
+        m_frontService->setHealthCheckService(m_server->server()->GetHealthCheckService());
+    }
+    if (m_front)
+    {
+        m_front->start();
+    }
 }
 
 void ProTransportImpl::stop()
 {
-    m_timer->stop();
-    m_server->stop();
-    m_front->stop();
+    if (m_timer)
+    {
+        m_timer->stop();
+    }
+    if (m_server)
+    {
+        m_server->stop();
+    }
+    if (m_front)
+    {
+        m_front->stop();
+    }
 }
 
 void ProTransportImpl::keepAlive()
diff --git a/cpp/wedpr-transport/sdk/src/ProTransportImpl.h b/cpp/wedpr-transport/sdk/src/ProTransportImpl.h
index cb5d4873..663fd1aa 100644
--- a/cpp/wedpr-transport/sdk/src/ProTransportImpl.h
+++ b/cpp/wedpr-transport/sdk/src/ProTransportImpl.h
@@ -36,7 +36,7 @@ class ProTransportImpl : public Transport, public std::enable_shared_from_this
;
     ProTransportImpl(ppc::front::FrontConfig::Ptr config, int keepAlivePeriodMs = 3000);
-
+    ~ProTransportImpl();
     void start() override;
     void stop() override;
 
diff --git a/cpp/wedpr-transport/sdk/src/Transport.h b/cpp/wedpr-transport/sdk/src/Transport.h
index a3fa1680..bfcc4414 100644
--- a/cpp/wedpr-transport/sdk/src/Transport.h
+++ b/cpp/wedpr-transport/sdk/src/Transport.h
@@ -33,8 +33,22 @@ class Transport
     Transport(ppc::front::FrontConfig::Ptr config);
     virtual ~Transport() { stop(); }
 
-    virtual void start() { m_front->start(); }
-    virtual void stop() { m_front->stop(); }
+    virtual void start()
+    {
+        if (!m_front)
+        {
+            return;
+        }
+        m_front->start();
+    }
+    virtual void stop()
+    {
+        if (!m_front)
+        {
+            return;
+        }
+        m_front->stop();
+    }
 
     virtual ppc::front::IFront::Ptr const& getFront() const { return m_front; }
 
diff --git a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
index c6ebcf32..f8f326f1 100644
--- a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
+++ b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
@@ -29,9 +29,6 @@
 using namespace ppc::sdk;
 using namespace ppc::front;
 
-bcos::BoostLogInitializer::Ptr TransportBuilder::g_logInitializer;
-std::once_flag TransportBuilder::g_flag;
-
 TransportBuilder::TransportBuilder()
 {
     m_frontConfigBuilder = std::make_shared(
@@ -60,3 +57,27 @@ ppc::front::FrontConfig::Ptr TransportBuilder::buildConfig(int threadPoolSize, s
 {
     return m_frontConfigBuilder->build(threadPoolSize, nodeID);
 }
+
+
+void TransportBuilder::initLog(const std::string& configPath)
+{
+    bcos::Guard l(x_logInited);
+    if (m_logInited)
+    {
+        return;
+    }
+    m_logInited = true;
+    // init the log
+    boost::property_tree::ptree pt;
+    try
+    {
+        boost::property_tree::read_ini(configPath, pt);
+    }
+    catch (std::exception const& e)
+    {
+        // disable the log when the configPath not exists
+        pt.put("log.enable", false);
+    }
+    m_logInitializer = std::make_shared();
+    m_logInitializer->initLog(pt, bcos::FileLogger, "gateway_sdk_log");
+}
\ No newline at end of file
diff --git a/cpp/wedpr-transport/sdk/src/TransportBuilder.h b/cpp/wedpr-transport/sdk/src/TransportBuilder.h
index e579abc8..51727fa9 100644
--- a/cpp/wedpr-transport/sdk/src/TransportBuilder.h
+++ b/cpp/wedpr-transport/sdk/src/TransportBuilder.h
@@ -51,30 +51,12 @@ class TransportBuilder
     ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder() { return m_frontConfigBuilder; }
 
 
-    static void initLog(const std::string& configPath)
-    {
-        std::call_once(g_flag, [configPath]() {
-            // init the log
-            boost::property_tree::ptree pt;
-            try
-            {
-                boost::property_tree::read_ini(configPath, pt);
-            }
-            catch (std::exception const& e)
-            {
-                // disable the log when the configPath not exists
-                pt.put("log.enable", false);
-            }
-
-            g_logInitializer = std::make_shared();
-            g_logInitializer->initLog(pt, bcos::FileLogger, "gateway_sdk_log");
-        });
-    }
+    void initLog(const std::string& configPath);
 
 private:
     ppc::front::FrontConfigBuilder::Ptr m_frontConfigBuilder;
-
-    static std::once_flag g_flag;
-    static bcos::BoostLogInitializer::Ptr g_logInitializer;
+    bcos::BoostLogInitializer::Ptr m_logInitializer;
+    bool m_logInited = false;
+    bcos::Mutex x_logInited;
 };
 }  // namespace ppc::sdk
\ No newline at end of file

From 154096843e9f7bf6ff3bf1b4fca4a725d00ee312 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 13 Nov 2024 20:28:09 +0800
Subject: [PATCH 088/120] fix ci (#87)

* add tests for grpc

* fix ci

* use fisco registry vcpkg
---
 .github/workflows/cpp_full_node_workflow.yml  |  9 ++++
 cpp/cmake/python.cmake                        |  3 +-
 cpp/test-utils/TaskMock.h                     |  4 +-
 cpp/tools/install_depends.sh                  | 13 ++++++
 cpp/vcpkg-configuration.json                  |  3 +-
 cpp/vcpkg.json                                |  3 +-
 .../tests/bs-ecdh-psi/TestBsEcdhCache.cpp     |  1 +
 .../tests/cm2020-psi/FakeCM2020PSIFactory.h   |  2 +-
 .../tests/cm2020-psi/TestCM2020Impl.cpp       |  2 +
 .../tests/labeled-psi/FakeLabeledPSIFactory.h |  2 +-
 .../tests/labeled-psi/TestLabeledPSIImpl.cpp  |  2 +-
 .../tests/labeled-psi/TestSenderDB.cpp        |  2 +-
 .../tests/ra2018-psi/TestEcdhPSIImpl.cpp      |  1 +
 .../tests/ra2018-psi/TestRA2018Impl.cpp       |  1 +
 .../tests/ra2018-psi/TestRA2018Message.cpp    |  1 +
 .../ppc-psi/tests/ra2018-psi/mock/Common.h    |  2 +
 cpp/wedpr-protocol/grpc/CMakeLists.txt        |  5 +++
 cpp/wedpr-protocol/grpc/tests/CMakeLists.txt  | 10 +++++
 cpp/wedpr-protocol/grpc/tests/TestGrpc.cpp    | 41 +++++++++++++++++++
 cpp/wedpr-protocol/grpc/tests/main.cpp        |  2 +
 .../sdk/src/TransportBuilder.cpp              |  2 +
 21 files changed, 101 insertions(+), 10 deletions(-)
 create mode 100644 cpp/wedpr-protocol/grpc/tests/CMakeLists.txt
 create mode 100644 cpp/wedpr-protocol/grpc/tests/TestGrpc.cpp
 create mode 100644 cpp/wedpr-protocol/grpc/tests/main.cpp

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 3158cbc9..fee8d67a 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -74,6 +74,12 @@ jobs:
           bash -x cpp/tools/install_depends.sh -o macos
           mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gperftools-build-err.log
+          path: /home/runner/work/WeDPR-Component/WeDPR-Component/cpp/deps/src/gperftools-stamp/gperftools-build-err.log
       - uses: actions/upload-artifact@v4
         if: runner.os == 'macOS'      
         with:
@@ -171,6 +177,9 @@ jobs:
           du -sch /usr/local/share/vcpkg/buildtrees/grpc/src/*
           rm -rf /usr/local/share/vcpkg/buildtrees/grpc/src
           rm -rf /usr/local/share/vcpkg/buildtrees/grpc/*-dbg
+          rm -rf /usr/local/share/vcpkg/buildtrees/openssl/src
+          rm -rf /usr/local/share/vcpkg/buildtrees/openssl/*-dbg
+          rm -rf /usr/local/share/vcpkg/buildtrees/*/*-dbg
           df -lh
       - name: Compile
         run: |   
diff --git a/cpp/cmake/python.cmake b/cpp/cmake/python.cmake
index c726136e..cbd54a70 100644
--- a/cpp/cmake/python.cmake
+++ b/cpp/cmake/python.cmake
@@ -9,7 +9,8 @@ message("Python_Development_FOUND:${Python_Development_FOUND}")
 message("Python_LIBRARIES:${Python_LIBRARIES}")
 message("Python_INCLUDE_DIRS:${Python_INCLUDE_DIRS}")
 
-list(APPEND CMAKE_SWIG_FLAGS "-DPY3")
+list(APPEND CMAKE_SWIG_FLAGS "-D${Python_LIBRARIES}")
+message("## CMAKE_SWIG_FLAGS: ${CMAKE_SWIG_FLAGS}")
 
 function(search_python_module)
   set(options NO_VERSION)
diff --git a/cpp/test-utils/TaskMock.h b/cpp/test-utils/TaskMock.h
index 1956946a..7056478c 100644
--- a/cpp/test-utils/TaskMock.h
+++ b/cpp/test-utils/TaskMock.h
@@ -19,10 +19,8 @@
  */
 #pragma once
 #include "ppc-framework/protocol/Task.h"
-#include "ppc-io/src/FileLineReader.h"
 #include 
 
-using namespace ppc::io;
 using namespace ppc::protocol;
 namespace ppc::test
 {
@@ -91,4 +89,4 @@ inline void checkTask(Task::Ptr _task1, Task::Ptr _task2)
         checkParty(it.second, _task2->getPartyByID(it.first));
     }
 }
-}  // namespace ppc::test
\ No newline at end of file
+}  // namespace ppc::test
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index d7a83ca2..3825a31b 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -111,11 +111,24 @@ install_ubuntu_depends()
     LOG_INFO "install depends for ubuntu success ..."
 }
 
+install_iconv_depend()
+{
+    LOG_INFO "download and install libiconv..."
+    wget --no-check-certificate https://mirrors.tuna.tsinghua.edu.cn/gnu/libiconv/libiconv-1.16.tar.gz && tar -xvf libiconv-1.16.tar.gz
+    
+    cd libiconv-1.16 && ./configure --with-pic && make -j4 && make install
+    cd .. && rm -rf libiconv-1.16*
+    df -lh
+    LOG_INFO "download and install libiconv success..."
+}
+
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
     brew install autoconf nasm lcov
+    brew link libiconv
     install_gsasl_depend "macos"
+    install_iconv_depend
     LOG_INFO "install depends for macos success ..."
 }
 
diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json
index ade9fdc7..0a2e904f 100644
--- a/cpp/vcpkg-configuration.json
+++ b/cpp/vcpkg-configuration.json
@@ -9,7 +9,8 @@
         "bcos-utilities",
         "bcos-boostssl",
         "tarscpp",
-        "libhdfs3"
+        "libhdfs3",
+        "tbb"
       ]
     }
   ]
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index 69a47350..6b2f1f75 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -118,7 +118,8 @@
         },
         "tarscpp",
         "tbb",
-        "libxml2"
+        "libxml2",
+        "libiconv"
       ]
     },
     "sdk": {
diff --git a/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp b/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp
index 05eb185d..c7049f71 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/bs-ecdh-psi/TestBsEcdhCache.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "ppc-io/src/DataResourceLoaderImpl.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h"
 #include "ppc-psi/src/bs-ecdh-psi/core/BsEcdhCache.h"
 #include "test-utils/FakeFront.h"
diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h
index 6bf27d64..66c2e975 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/FakeCM2020PSIFactory.h
@@ -18,11 +18,11 @@
  * @date 2022-12-19
  */
 #pragma once
-
 #include "ppc-crypto-core/src/hash/BLAKE2bHash.h"
 #include "ppc-crypto/src/ecc/Ed25519EccCrypto.h"
 #include "ppc-framework/crypto/CryptoBox.h"
 #include "ppc-io/src/DataResourceLoaderImpl.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-tools/src/config/PPCConfig.h"
 #include "test-utils/FakeFront.h"
 #include 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
index bee77f70..c4fdf8eb 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "FakeCM2020PSIFactory.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-psi/src/cm2020-psi/CM2020PSIImpl.h"
 #include "protocol/src/JsonTaskImpl.h"
 #include "test-utils/FakeFront.h"
@@ -29,6 +30,7 @@
 #include 
 
 using namespace ppc::psi;
+using namespace ppc::io;
 using namespace bcos;
 using namespace bcos::test;
 using namespace ppc::crypto;
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
index ab863356..e369b6c1 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
@@ -23,10 +23,10 @@
 #include "ppc-crypto/src/ecc/Ed25519EccCrypto.h"
 #include "ppc-framework/crypto/CryptoBox.h"
 #include "ppc-io/src/DataResourceLoaderImpl.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-tools/src/config/PPCConfig.h"
 #include "test-utils/FakeFront.h"
 #include 
-
 using namespace bcos;
 using namespace ppc::protocol;
 using namespace ppc::psi;
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
index d7a75168..f341b6f2 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "FakeLabeledPSIFactory.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-psi/src/labeled-psi/LabeledPSIImpl.h"
 #include "ppc-psi/src/labeled-psi/protocol/LabeledPSIResult.h"
 #include "protocol/src/JsonTaskImpl.h"
@@ -27,7 +28,6 @@
 #include 
 #include 
 #include 
-
 using namespace ppc::psi;
 using namespace bcos;
 using namespace bcos::test;
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp
index 2bf6c8ea..0a79fad4 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestSenderDB.cpp
@@ -22,13 +22,13 @@
 #include "ppc-crypto-core/src/hash/Sha256Hash.h"
 #include "ppc-crypto/src/ecc/Ed25519EccCrypto.h"
 #include "ppc-crypto/src/oprf/EcdhOprf.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-psi/src/labeled-psi/core/LabeledPSIParams.h"
 #include "ppc-psi/src/labeled-psi/core/SenderDB.h"
 #include "protocol/src/JsonTaskImpl.h"
 #include "test-utils/TaskMock.h"
 #include 
 #include 
-
 using namespace ppc::psi;
 using namespace bcos;
 using namespace bcos::test;
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
index a100a4cd..856eabbc 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
@@ -20,6 +20,7 @@
 #include "mock/Common.h"
 #include "mock/EcdhPSIFixture.h"
 // Note: it's better not to depends on the task-impl
+#include "ppc-io/src/FileLineReader.h"
 #include "protocol/src/JsonTaskImpl.h"
 #include "test-utils/TaskMock.h"
 #include 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
index 074766ea..261cd401 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
@@ -20,6 +20,7 @@
 #include "mock/Common.h"
 #include "mock/RA2018PSIFixture.h"
 // Note: it's better not to depends on the task-impl
+#include "ppc-io/src/FileLineReader.h"
 #include "protocol/src/JsonTaskImpl.h"
 #include "test-utils/TaskMock.h"
 #include 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp
index 59f2b576..074068a9 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Message.cpp
@@ -19,6 +19,7 @@
  */
 #include "mock/RA2018MessageFixture.h"
 #include "ppc-crypto-core/src/hash/Sha256Hash.h"
+#include "ppc-io/src/FileLineReader.h"
 #include "ppc-psi/src/ra2018-psi/Common.h"
 #include 
 #include 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h
index ed8e3b8f..80459986 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/Common.h
@@ -18,12 +18,14 @@
  * @date 2022-12-29
  */
 #pragma once
+#include "ppc-io/src/FileLineReader.h"
 #include "protocol/src/JsonTaskImpl.h"
 #include "test-utils/TaskMock.h"
 #include 
 
 using namespace bcos;
 using namespace ppc::protocol;
+using namespace ppc::io;
 
 namespace ppc::test
 {
diff --git a/cpp/wedpr-protocol/grpc/CMakeLists.txt b/cpp/wedpr-protocol/grpc/CMakeLists.txt
index b72e308b..ad8a8d16 100644
--- a/cpp/wedpr-protocol/grpc/CMakeLists.txt
+++ b/cpp/wedpr-protocol/grpc/CMakeLists.txt
@@ -27,3 +27,8 @@ target_link_libraries(${SERVICE_PB_TARGET} PUBLIC ${PB_PROTOCOL_TARGET} gRPC::gr
 
 add_subdirectory(client)
 add_subdirectory(server)
+if (TESTS)
+    enable_testing()
+    set(CTEST_OUTPUT_ON_FAILURE TRUE)
+    add_subdirectory(tests)
+endif ()
diff --git a/cpp/wedpr-protocol/grpc/tests/CMakeLists.txt b/cpp/wedpr-protocol/grpc/tests/CMakeLists.txt
new file mode 100644
index 00000000..321dff57
--- /dev/null
+++ b/cpp/wedpr-protocol/grpc/tests/CMakeLists.txt
@@ -0,0 +1,10 @@
+file(GLOB_RECURSE SOURCES "*.cpp" "*.h")
+
+# cmake settings
+set(TEST_BINARY_NAME test-grpc-protocol)
+
+add_executable(${TEST_BINARY_NAME} ${SOURCES})
+target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
+
+target_link_libraries(${TEST_BINARY_NAME} ${SERVICE_PB_TARGET} ${BCOS_UTILITIES_TARGET} ${BOOST_UNIT_TEST})
+add_test(NAME test-grpc-protocol WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
diff --git a/cpp/wedpr-protocol/grpc/tests/TestGrpc.cpp b/cpp/wedpr-protocol/grpc/tests/TestGrpc.cpp
new file mode 100644
index 00000000..6764816f
--- /dev/null
+++ b/cpp/wedpr-protocol/grpc/tests/TestGrpc.cpp
@@ -0,0 +1,41 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file TestGrpc.cpp
+ * @author: yujiechen
+ * @date 2022-11-12
+ */
+
+#include "wedpr-protocol/grpc/Common.h"
+#include 
+#include 
+
+using namespace ppc::protocol;
+using namespace bcos::test;
+using namespace grpc;
+
+BOOST_FIXTURE_TEST_SUITE(GrpcTest, TestPromptFixture)
+
+BOOST_AUTO_TEST_CASE(testGrpc)
+{
+    grpc::ChannelArguments args;
+    uint64_t msgSize = INT_MAX;
+    int convertedMsgSize = msgSize;
+    std::cout << "compare with -1, result: " << (convertedMsgSize >= -1) << std::endl;
+    args.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX);
+    std::cout << "##### " << GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH << ", " << INT_MAX << std::endl;
+}
+
+BOOST_AUTO_TEST_SUITE_END()
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/grpc/tests/main.cpp b/cpp/wedpr-protocol/grpc/tests/main.cpp
new file mode 100644
index 00000000..2c06f9b5
--- /dev/null
+++ b/cpp/wedpr-protocol/grpc/tests/main.cpp
@@ -0,0 +1,2 @@
+#define BOOST_TEST_MAIN
+#include 
\ No newline at end of file
diff --git a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
index f8f326f1..69d60d31 100644
--- a/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
+++ b/cpp/wedpr-transport/sdk/src/TransportBuilder.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "TransportBuilder.h"
+#include "Common.h"
 #include "ProTransportImpl.h"
 #include "Transport.h"
 #include "TransportImpl.h"
@@ -80,4 +81,5 @@ void TransportBuilder::initLog(const std::string& configPath)
     }
     m_logInitializer = std::make_shared();
     m_logInitializer->initLog(pt, bcos::FileLogger, "gateway_sdk_log");
+    TRANSPORT_LOG(INFO) << LOG_DESC("init log success") << LOG_KV("configPath", configPath);
 }
\ No newline at end of file

From 34f237dc4dabf9e999bca8ba0f292d769a05d86c Mon Sep 17 00:00:00 2001
From: octopus <912554887@qq.com>
Date: Fri, 8 Nov 2024 17:20:14 +0800
Subject: [PATCH 089/120] path update

---
 cpp/wedpr-computing/ppc-mpc/src/Common.h      |   6 +-
 .../ppc-mpc/src/MPCService.cpp                | 170 ++++++++++++------
 cpp/wedpr-computing/ppc-mpc/src/MPCService.h  |   4 +-
 .../ppc-mpc/tests/TestMPCService.cpp          |   4 +-
 4 files changed, 122 insertions(+), 62 deletions(-)

diff --git a/cpp/wedpr-computing/ppc-mpc/src/Common.h b/cpp/wedpr-computing/ppc-mpc/src/Common.h
index 574c2656..ec4b8bc2 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/Common.h
+++ b/cpp/wedpr-computing/ppc-mpc/src/Common.h
@@ -39,8 +39,9 @@ struct JobInfo
     int selfIndex;
     bool isMalicious;
     int bitLength;
-    std::string inputFileName;
-    std::string outputFileName;
+    std::string mpcFilePath;
+    std::string inputFilePath;
+    std::string outputFilePath;
     std::string gatewayEngineEndpoint;
 };
 
@@ -51,6 +52,7 @@ const std::string MPC_RELATIVE_PATH = "/Programs/Source/";
 const std::string MPC_ALGORITHM_FILE_SUFFIX = ".mpc";
 const std::string MPC_ALGORITHM_COMPILER = "compile.py";
 const std::string MPC_PREPARE_FILE = "mpc_prepare.csv";
+const std::string MPC_RESULT_FILE = "mpc_result.csv";
 
 enum MpcBinaryType
 {
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index e31115f8..e7ff40ef 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -19,6 +19,7 @@
  */
 
 #include "MPCService.h"
+#include "Common.h"
 #include "ppc-framework/io/DataResourceLoader.h"
 #include "ppc-io/src/DataResourceLoaderImpl.h"
 #include "ppc-io/src/FileLineReader.h"
@@ -44,95 +45,128 @@ using namespace ppc::tools;
 using namespace ppc::storage;
 using namespace ppc::rpc;
 
+void MPCService::removeAllFiles(const std::vector &files)
+{
+    for (const auto &file : files)
+    {
+        if (file.empty())
+        {
+            continue;
+        }
+        try {
+            if (boost::filesystem::exists(file))
+            {
+                boost::filesystem::remove_all(file);
+
+                MPC_LOG(INFO) << LOG_DESC("[MPCService][removeAllFiles]")
+                      << LOG_KV("file", file);
+            }
+        } catch (...) {
+            MPC_LOG(INFO) << LOG_DESC("[MPCService][removeAllFiles]")
+                      << LOG_DESC("remove file exception")
+                      << LOG_KV("file", file);
+        }
+    }
+}
 
 void MPCService::doRun(Json::Value const& request, Json::Value& response)
 {
     auto startT = utcSteadyTime();
+
+    std::string localPathPrefix;
+    std::string mpcFileLocalPath;
     try
     {  // 0 get jobInfo and make command
         auto jobInfo = paramsToJobInfo(request);
+
+        std::string jobId = jobInfo.jobId;
+        int participantCount = jobInfo.participantCount;
+        int selfIndex = jobInfo.selfIndex;
+
         std::string mpcCmd;
         makeCommand(mpcCmd, jobInfo);
 
-        std::string hdfsPathPrefix =
-            m_mpcConfig.datasetHDFSPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR;
-        std::string localPathPrefix =
+        localPathPrefix =
             m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR;
 
-        // 1 download algorithm file
-        std::string algorithmFileHdfsPath =
-            hdfsPathPrefix + jobInfo.jobId + MPC_ALGORITHM_FILE_SUFFIX;
+        // 1 download mpc algorithm file
+        std::string mpcFileHdfsPath = jobInfo.mpcFilePath;
         std::string mpcRootPath = m_mpcConfig.mpcRootPathNoGateway;
         if (jobInfo.mpcNodeUseGateway)
         {
             mpcRootPath = m_mpcConfig.mpcRootPath;
         }
-        std::string algorithmFileLocalPath =
+        
+        mpcFileLocalPath =
             mpcRootPath + MPC_RELATIVE_PATH + jobInfo.jobId + MPC_ALGORITHM_FILE_SUFFIX;
-        if (!boost::filesystem::exists(algorithmFileLocalPath))
-        {
-            auto lineReader1 =
-                initialize_lineReader(jobInfo, algorithmFileHdfsPath, DataResourceType::HDFS);
-            auto lineWriter1 =
-                initialize_lineWriter(jobInfo, algorithmFileLocalPath, DataResourceType::FILE);
-            readAndSaveFile(lineReader1, lineWriter1);
-        }
 
-        // 2 download dataset file
-        std::string datasetFileHdfsPath = hdfsPathPrefix + jobInfo.inputFileName;
-        std::string datasetFileLocalPath = localPathPrefix + jobInfo.inputFileName;
-        if (!boost::filesystem::exists(datasetFileLocalPath))
-        {
-            auto lineReader2 =
-                initialize_lineReader(jobInfo, datasetFileHdfsPath, DataResourceType::HDFS);
-            auto lineWriter2 =
-                initialize_lineWriter(jobInfo, datasetFileLocalPath, DataResourceType::FILE);
-            readAndSaveFile(lineReader2, lineWriter2);
-        }
+        auto mpcFileReader =
+                initialize_lineReader(jobInfo, mpcFileHdfsPath, DataResourceType::HDFS);
+        auto mpcFileWriter =
+                initialize_lineWriter(jobInfo, mpcFileLocalPath, DataResourceType::FILE);
+            readAndSaveFile(mpcFileHdfsPath, mpcFileLocalPath, mpcFileReader, mpcFileWriter);
+
+
+        // 2 download mpc prepare file
+        std::string mpcPrepareFileHdfsPath = jobInfo.inputFilePath;
+        
+        // std::string inputFilePath =
+        // m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR + MPC_PREPARE_FILE;
+
+        std::string mpcPrepareFileLocalPath = localPathPrefix + MPC_PREPARE_FILE + "-P" + std::to_string(selfIndex) + "-0";
+        auto datasetFileReader =
+                initialize_lineReader(jobInfo, mpcPrepareFileHdfsPath, DataResourceType::HDFS);
+            auto datasetFileWriter =
+                initialize_lineWriter(jobInfo, mpcPrepareFileLocalPath, DataResourceType::FILE);
+            readAndSaveFile(mpcPrepareFileHdfsPath, mpcPrepareFileLocalPath, datasetFileReader, datasetFileWriter);
 
         // 3 run mpc job
         int outExitStatus = MPC_SUCCESS;
         std::string outResult;
         execCommand(mpcCmd, outExitStatus, outResult);
+
+        if (outExitStatus != MPC_SUCCESS)
+        {
+            removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
+            MPC_LOG(ERROR) << LOG_DESC("[MPCService][doRun]") 
+                << "run mpc job failed"
+                << LOG_KV("jobId", jobId)
+                << LOG_KV("outExitStatus", outExitStatus)
+                << LOG_KV("outResult", outResult);
+            BOOST_THROW_EXCEPTION(RunMpcFailException() << errinfo_comment(outResult));
+        }
+        
         std::string message = "run mpc job successfully";
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) << LOG_DESC(message);
+        // MPC_LOG(DEBUG) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) << LOG_KV("outResult", outResult);
+        response["code"] = MPC_SUCCESS;
+        response["message"] = "success";
 
         // 4 upload result file
-        std::string resultFileHdfsPath = hdfsPathPrefix + jobInfo.outputFileName;
-        std::string resultFileLocalPath = localPathPrefix + jobInfo.outputFileName;
+        std::string resultFileHdfsPath =  jobInfo.outputFilePath;
+        std::string resultFileLocalPath = localPathPrefix + MPC_RESULT_FILE;
         writeStringToFile(outResult, resultFileLocalPath);
 
-        auto lineReader3 =
+        auto resultFileReader =
             initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE);
-        auto lineWriter3 =
+        auto resultFileWriter =
             initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS);
-        readAndSaveFile(lineReader3, lineWriter3);
+        readAndSaveFile(resultFileLocalPath, resultFileHdfsPath, resultFileReader, resultFileWriter);
 
-        if (outExitStatus != MPC_SUCCESS)
-        {
-            message = "run mpc job failed";
-            MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC(message);
-            BOOST_THROW_EXCEPTION(RunMpcFailException() << errinfo_comment(message));
-        }
-        else
-        {
-            MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC(message);
-            response["code"] = MPC_SUCCESS;
-            response["message"] = "success";
-        }
-        if (boost::filesystem::exists(localPathPrefix))
-        {
-            boost::filesystem::remove_all(localPathPrefix);
-        }
+        removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
     }
     catch (const std::exception& e)
     {
+        removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
+
         const std::string diagnostic_information = std::string(boost::diagnostic_information(e));
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC("run mpc job failed:")
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC("run mpc job failed")
                       << LOG_DESC(diagnostic_information);
         response["code"] = MPC_FAILED;
         response["message"] = diagnostic_information;
     }
-    MPC_LOG(INFO) << LOG_DESC("run mpc") << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+
+    MPC_LOG(INFO) << LOG_DESC("run mpc") << LOG_KV("request",  request.toStyledString())<< LOG_KV("timecost(ms)", utcSteadyTime() - startT);
 }
 
 void MPCService::runMpcRpc(Json::Value const& request, RespFunc func)
@@ -200,7 +234,7 @@ void MPCService::writeStringToFile(const std::string& content, const std::string
     file << buffer.str();
 }
 
-void MPCService::readAndSaveFile(LineReader::Ptr lineReader, LineWriter::Ptr lineWriter)
+void MPCService::readAndSaveFile(const std::string &readerFilePath, const std::string &writerFilePath, LineReader::Ptr lineReader, LineWriter::Ptr lineWriter)
 {
     uint64_t lineSize = 0;
     int64_t readPerBatchLines = m_mpcConfig.readPerBatchLines;
@@ -216,7 +250,10 @@ void MPCService::readAndSaveFile(LineReader::Ptr lineReader, LineWriter::Ptr lin
         lineWriter->writeLine(dataBatch, DataSchema::String, "\n");
     }
     lineWriter->close();
-    MPC_LOG(INFO) << LOG_DESC("save file ok") << LOG_KV("file lines", lineSize);
+    MPC_LOG(INFO) << LOG_DESC("save file ok") 
+        << LOG_KV("readerFilePath", readerFilePath)
+        << LOG_KV("writerFilePath", writerFilePath)
+        << LOG_KV("file lines", lineSize);
 }
 
 LineReader::Ptr MPCService::initialize_lineReader(
@@ -282,8 +319,9 @@ JobInfo MPCService::paramsToJobInfo(const Json::Value& params)
         jobInfo.selfIndex = params["selfIndex"].asInt();
         jobInfo.isMalicious = params["isMalicious"].asBool();
         jobInfo.bitLength = params["bitLength"].asInt();
-        jobInfo.inputFileName = params["inputFileName"].asString();
-        jobInfo.outputFileName = params["outputFileName"].asString();
+        jobInfo.mpcFilePath = params["mpcFilePath"].asString();
+        jobInfo.inputFilePath = params["inputFilePath"].asString();
+        jobInfo.outputFilePath = params["outputFilePath"].asString();
         jobInfo.gatewayEngineEndpoint = params["gatewayEngineEndpoint"].asString();
         return jobInfo;
     }
@@ -297,23 +335,36 @@ JobInfo MPCService::paramsToJobInfo(const Json::Value& params)
 
 void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo)
 {
+    std::string jobId = jobInfo.jobId;
     std::string mpcRootPath = m_mpcConfig.mpcRootPath;
     if (jobInfo.mpcNodeUseGateway)
     {
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] use gateway to connect node");
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] use gateway to connect node") 
+            << LOG_KV("jobId", jobId);
     }
     else
     {
         mpcRootPath = m_mpcConfig.mpcRootPathNoGateway;
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] direct connect node");
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] direct connect node")
+            << LOG_KV("jobId", jobId);
     }
     int r = chdir(mpcRootPath.c_str());
     if (r == 0)
     {
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] change dir ok");
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand] change dir ok")
+            << LOG_KV("jobId", jobId);
+    }
+    else
+    {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][makeCommand] change dir fail") 
+            << LOG_KV("mpcRootPath", mpcRootPath)
+            << LOG_KV("ret", r)
+            << LOG_KV("jobId", jobId);
+            ;
     }
     std::string compileFilePath = mpcRootPath + PATH_SEPARATOR + MPC_ALGORITHM_COMPILER;
     int participantCount = jobInfo.participantCount;
+    int selfIndex = jobInfo.selfIndex;
     bool isMalicious = jobInfo.isMalicious;
     std::string mpcBinFileName;
     std::string compileOption;
@@ -321,6 +372,11 @@ void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo)
 
     if (!boost::filesystem::exists(compileFilePath))
     {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService] compile file not exist") 
+            << LOG_KV("compileFilePath", compileFilePath)
+            << LOG_KV("jobId", jobId);
+            ;
+
         BOOST_THROW_EXCEPTION(MpcCompilerNotExistException()
                               << errinfo_comment("compile file not exist:" + compileFilePath));
     }
@@ -359,7 +415,7 @@ void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo)
     {
         cmd += "-N " + std::to_string(jobInfo.participantCount) + " ";
     }
-    MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand]") << LOG_KV("mpcCmd", cmd);
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][makeCommand]") << LOG_KV("jobId", jobId) << LOG_KV("mpcCmd", cmd);
 }
 
 void MPCService::getMpcProtocol(const int participantCount, const bool isMalicious,
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
index 7539a67d..a9f601c6 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
@@ -50,12 +50,14 @@ class MPCService
     void execCommand(const std::string cmd, int& outExitStatus, std::string& outResult);
 
     void writeStringToFile(const std::string& content, const std::string& filePath);
-    void readAndSaveFile(ppc::io::LineReader::Ptr lineReader, ppc::io::LineWriter::Ptr lineWriter);
+    void readAndSaveFile(const std::string &readerFilePath, const std::string &writerFilePath,ppc::io::LineReader::Ptr lineReader, ppc::io::LineWriter::Ptr lineWriter);
     ppc::io::LineReader::Ptr initialize_lineReader(const JobInfo& jobInfo,
         const std::string& readerFilePath, ppc::protocol::DataResourceType type);
     ppc::io::LineWriter::Ptr initialize_lineWriter(const JobInfo& jobInfo,
         const std::string& writerFilePath, ppc::protocol::DataResourceType type);
 
+    void removeAllFiles(const std::vector &files);
+
 private:
     ppc::tools::MPCConfig m_mpcConfig;
     ppc::tools::StorageConfig m_storageConfig;
diff --git a/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp b/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp
index 75e427cc..5b523bbc 100644
--- a/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/tests/TestMPCService.cpp
@@ -84,8 +84,8 @@ BOOST_AUTO_TEST_CASE(testMPCService)
     jobInfo.selfIndex = 0;
     jobInfo.isMalicious = false;
     jobInfo.bitLength = 128;
-    jobInfo.inputFileName = "mpc_prepare.csv";
-    jobInfo.outputFileName = "mpc_output.txt";
+    jobInfo.inputFilePath = "mpc_prepare.csv";
+    jobInfo.outputFilePath = "mpc_output.txt";
     jobInfo.gatewayEngineEndpoint = "127.0.0.1:6789";
 
     mpcService->makeCommand(cmd, jobInfo);

From ee0c5d70de99686ca6a69d433af84cf777432e7c Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 18 Nov 2024 15:08:56 +0800
Subject: [PATCH 090/120] refactor multiple psi (#88)

* add tests for grpc

* fix ci

* use fisco registry vcpkg

* refactor multiple psi

* fix read data by size bug

* optimize memory for ecdh-mul-psi

* optimize dataResourceLoader

* add storage option check

* optimize multi-psi perf when exist multi-partners

* optimize memory use

* update generated sdk

* use uint64_t to record refInfo

* send intersection by batch
---
 cpp/ppc-framework/io/DataBatch.h              |   13 +
 cpp/ppc-framework/io/LineReader.h             |    1 +
 .../protocol/{ServiceType.h => Constant.h}    |    3 +-
 cpp/ppc-framework/protocol/DataResource.h     |    5 +-
 cpp/ppc-framework/protocol/GrpcConfig.h       |   42 +-
 cpp/ppc-framework/protocol/Message.h          |   41 +-
 cpp/ppc-framework/protocol/MessagePayload.h   |    6 +
 cpp/ppc-framework/protocol/PPCMessageFace.h   |    2 +
 cpp/ppc-framework/protocol/Protocol.h         |   50 +-
 cpp/tools/install_depends.sh                  |    3 +-
 cpp/wedpr-computing/ppc-psi/src/PSIConfig.h   |   10 +
 .../ppc-psi/src/ecdh-multi-psi/Common.h       |    4 +
 .../src/ecdh-multi-psi/EcdhMultiCache.cpp     |  483 ++++++++
 .../src/ecdh-multi-psi/EcdhMultiCache.h       |  393 ++++---
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp   |  100 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.h     |   18 +-
 .../core/EcdhMultiPSICalculator.cpp           |  375 ++----
 .../core/EcdhMultiPSICalculator.h             |   40 +-
 .../core/EcdhMultiPSIMaster.cpp               |  446 ++-----
 .../ecdh-multi-psi/core/EcdhMultiPSIMaster.h  |   89 +-
 .../core/EcdhMultiPSIPartner.cpp              |  185 ++-
 .../ecdh-multi-psi/core/EcdhMultiPSIPartner.h |   44 +-
 .../ppc-psi/src/psi-framework/CMakeLists.txt  |    2 +-
 .../src/psi-framework/PSIFramework.cpp        |    9 +
 .../ppc-psi/src/psi-framework/TaskState.h     |   34 +-
 .../interfaces/PSIMessageInterface.h          |    7 +-
 .../src/psi-framework/protocol/PSI.tars       |    2 +-
 .../src/psi-framework/protocol/PSIMessage.cpp |   10 +
 .../src/psi-framework/protocol/PSIMessage.h   |   39 +-
 .../ppc-crypto/src/CMakeLists.txt             |    2 +-
 .../ppc-tools/src/config/PPCConfig.cpp        |    2 +-
 .../ppc-tools/src/config/PPCConfig.h          |    3 +-
 cpp/wedpr-initializer/Initializer.cpp         |    2 +-
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp    |    2 +-
 cpp/wedpr-protocol/grpc/server/GrpcServer.cpp |   14 +-
 cpp/wedpr-protocol/protocol/src/PPCMessage.h  |   12 +-
 .../ppc-io/src/BaseFileLineReader.h           |    4 +-
 cpp/wedpr-storage/ppc-io/src/CMakeLists.txt   |    2 +-
 .../ppc-io/src/DataResourceLoaderImpl.cpp     |   74 +-
 .../ppc-io/src/DataResourceLoaderImpl.h       |   20 +-
 cpp/wedpr-storage/ppc-io/src/FileLineReader.h |    2 +-
 cpp/wedpr-storage/ppc-io/src/HDFSReader.h     |    3 +-
 .../ppc-io/src/SQLResultReader.h              |    1 +
 .../ppc-storage/src/CMakeLists.txt            |    2 +-
 .../ppc-storage/src/FileStorageFactoryImpl.h  |    1 +
 .../ppc-storage/src/SQLStorageFactoryImpl.h   |    1 +
 .../ppc-front/ppc-front/Front.cpp             |   46 +-
 .../ppc-front/ppc-front/FrontImpl.cpp         |    4 +-
 .../wedpr/sdk/jni/generated/GrpcConfig.java   |   16 +-
 .../sdk/jni/generated/GrpcServerConfig.java   |   13 +-
 .../wedpr/sdk/jni/generated/Message.java      |    9 +-
 .../sdk/jni/generated/MessagePayload.java     |    4 +
 .../generated/wedpr_java_transportJNI.java    |   65 +-
 .../src/wedpr_java_transportJAVA_wrap.cxx     |  701 ++++++-----
 .../java/swig/wedpr_java_transport.i          |    3 +
 .../generated/wedpr_python_transport.py       |   84 +-
 .../src/wedpr_python_transportPYTHON_wrap.cxx | 1041 +++++++++--------
 .../python/swig/wedpr_python_transport.i      |    5 +
 58 files changed, 2567 insertions(+), 2027 deletions(-)
 rename cpp/ppc-framework/protocol/{ServiceType.h => Constant.h} (91%)
 create mode 100644 cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp

diff --git a/cpp/ppc-framework/io/DataBatch.h b/cpp/ppc-framework/io/DataBatch.h
index b0da8d2e..a8c394d1 100644
--- a/cpp/ppc-framework/io/DataBatch.h
+++ b/cpp/ppc-framework/io/DataBatch.h
@@ -20,6 +20,7 @@
 #pragma once
 #include "../Common.h"
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -201,6 +202,18 @@ class DataBatch
 
     std::shared_ptr>& mutableData() { return m_data; }
 
+    void release()
+    {
+        if (!m_data)
+        {
+            return;
+        }
+        m_data->clear();
+        m_data.reset();
+        // free after release
+        MallocExtension::instance()->ReleaseFreeMemory();
+    }
+
 private:
     std::shared_ptr> m_data;
     uint64_t m_capacityBytes = 0;
diff --git a/cpp/ppc-framework/io/LineReader.h b/cpp/ppc-framework/io/LineReader.h
index a1031a06..5672490e 100644
--- a/cpp/ppc-framework/io/LineReader.h
+++ b/cpp/ppc-framework/io/LineReader.h
@@ -38,6 +38,7 @@ class LineReader
     virtual uint64_t columnSize() const = 0;
     virtual ppc::protocol::DataResourceType type() const = 0;
     virtual void clean() {}
+    virtual bool readFinished() const = 0;
 };
 
 class LineReaderFactory
diff --git a/cpp/ppc-framework/protocol/ServiceType.h b/cpp/ppc-framework/protocol/Constant.h
similarity index 91%
rename from cpp/ppc-framework/protocol/ServiceType.h
rename to cpp/ppc-framework/protocol/Constant.h
index 4222064a..fd9b8306 100644
--- a/cpp/ppc-framework/protocol/ServiceType.h
+++ b/cpp/ppc-framework/protocol/Constant.h
@@ -13,7 +13,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  *
- * @file ServiceType.h
+ * @file Constant.h
  * @author: yujiechen
  * @date 2024-11-07
  */
@@ -26,4 +26,5 @@ namespace ppc::protocol
 {
 const static std::string PSI_SERVICE_TYPE = "PSI";
 const static std::string MPC_SERVICE_TYPE = "MPC";
+const static size_t LARGE_MSG_THRESHOLD = 30 * 1024 * 1024;
 }  // namespace ppc::protocol
diff --git a/cpp/ppc-framework/protocol/DataResource.h b/cpp/ppc-framework/protocol/DataResource.h
index 3a095050..012314c9 100644
--- a/cpp/ppc-framework/protocol/DataResource.h
+++ b/cpp/ppc-framework/protocol/DataResource.h
@@ -125,10 +125,7 @@ inline std::string printDataResourceInfo(DataResource::ConstPtr _dataResource)
     {
         stringstream << LOG_KV("dataType", _dataResource->desc()->type())
                      << LOG_KV("command", _dataResource->desc()->accessCommand())
-                     << LOG_KV("path", _dataResource->desc()->path())
-                     << LOG_KV("bizSeqNo", _dataResource->desc()->bizSeqNo())
-                     << LOG_KV("fileID", _dataResource->desc()->fileID())
-                     << LOG_KV("fileMd5", _dataResource->desc()->fileMd5());
+                     << LOG_KV("path", _dataResource->desc()->path());
     }
     if (_dataResource->outputDesc())
     {
diff --git a/cpp/ppc-framework/protocol/GrpcConfig.h b/cpp/ppc-framework/protocol/GrpcConfig.h
index c3a2655e..7a5b2f33 100644
--- a/cpp/ppc-framework/protocol/GrpcConfig.h
+++ b/cpp/ppc-framework/protocol/GrpcConfig.h
@@ -89,10 +89,22 @@ class GrpcConfig
         m_compressAlgorithm = compressAlgorithm;
     }
 
+    uint64_t maxMsgSize() const { return m_maxMsgSize; }
+    void setMaxMsgSize(uint64_t maxMsgSize)
+    {
+        if (maxMsgSize > c_maxMsgSize)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "The maxMsgSize limit is " + std::to_string(c_maxMsgSize)));
+        }
+        m_maxMsgSize = maxMsgSize;
+    }
+
 protected:
     bool m_enableHealthCheck = true;
     std::string m_loadBalancePolicy = "round_robin";
     bool m_enableDnslookup = false;
+
     // Note: grpc use int to set the maxMsgSize
     uint64_t const c_maxMsgSize = INT_MAX;
 
@@ -100,18 +112,22 @@ class GrpcConfig
     uint64_t m_maxSendMessageSize = c_maxMsgSize;
     // the max received message size in bytes
     uint64_t m_maxReceivedMessageSize = c_maxMsgSize;
+    // the max msg size
+    uint64_t m_maxMsgSize = c_maxMsgSize;
     int m_compressAlgorithm = 0;
 };
 
-class GrpcServerConfig : public GrpcConfig
+class GrpcServerConfig
 {
 public:
     using Ptr = std::shared_ptr;
-    GrpcServerConfig() = default;
-    GrpcServerConfig(EndPoint endPoint, bool enableHealthCheck)
-      : m_endPoint(std::move(endPoint)), m_enableHealthCheck(enableHealthCheck)
-    {}
-    ~GrpcServerConfig() override = default;
+    GrpcServerConfig() { m_grpcConfig = std::make_shared(); }
+    GrpcServerConfig(EndPoint endPoint, bool enableHealthCheck) : GrpcServerConfig()
+    {
+        m_endPoint = std::move(endPoint);
+        m_enableHealthCheck = enableHealthCheck;
+    }
+    virtual ~GrpcServerConfig() = default;
 
     std::string listenEndPoint() const { return m_endPoint.listenEndPoint(); }
 
@@ -122,21 +138,13 @@ class GrpcServerConfig : public GrpcConfig
     EndPoint& mutableEndPoint() { return m_endPoint; }
     bool enableHealthCheck() const { return m_enableHealthCheck; }
 
-    uint64_t maxMsgSize() const { return m_maxMsgSize; }
-    void setMaxMsgSize(uint64_t maxMsgSize)
-    {
-        if (maxMsgSize > c_maxMsgSize)
-        {
-            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
-                                      "The maxMsgSize limit is " + std::to_string(c_maxMsgSize)));
-        }
-        m_maxMsgSize = maxMsgSize;
-    }
+    GrpcConfig::Ptr const& grpcConfig() const { return m_grpcConfig; }
 
 protected:
     ppc::protocol::EndPoint m_endPoint;
     bool m_enableHealthCheck = true;
-    uint64_t m_maxMsgSize = c_maxMsgSize;
+    // the grpc config
+    GrpcConfig::Ptr m_grpcConfig;
 };
 
 
diff --git a/cpp/ppc-framework/protocol/Message.h b/cpp/ppc-framework/protocol/Message.h
index 0317b875..205f1ce3 100644
--- a/cpp/ppc-framework/protocol/Message.h
+++ b/cpp/ppc-framework/protocol/Message.h
@@ -205,7 +205,14 @@ class Message
 
     bool isRespPacket() const { return m_header->isRespPacket(); }
     void setRespPacket() { m_header->setRespPacket(); }
-    void setPayload(std::shared_ptr _payload) { m_payload = std::move(_payload); }
+    void setPayload(std::shared_ptr _payload)
+    {
+        m_payload = std::move(_payload);
+        if (m_payload)
+        {
+            m_payloadLen = m_payload->size();
+        }
+    }
     // for swig wrapper
     OutputBuffer payloadBuffer() const
     {
@@ -216,9 +223,18 @@ class Message
         return OutputBuffer{(unsigned char*)m_payload->data(), m_payload->size()};
     }
 
-    void setFrontMessage(MessagePayload::Ptr frontMessage)
+    void setFrontMessage(MessagePayload::Ptr frontMessage, bool releasePayload = false)
     {
         m_frontMessage = std::move(frontMessage);
+        if (!releasePayload)
+        {
+            return;
+        }
+        if (m_payload)
+        {
+            m_payload->clear();
+            bcos::bytes().swap(*m_payload);
+        }
     }
 
     MessagePayload::Ptr const& frontMessage() const { return m_frontMessage; }
@@ -227,17 +243,30 @@ class Message
     virtual bool encode(bcos::bytes& _buffer) = 0;
     virtual int64_t decode(bcos::bytesConstRef _buffer) = 0;
 
-    virtual uint32_t length() const
-    {
-        return m_header->length() + (m_payload ? m_payload->size() : 0);
-    }
+    virtual uint32_t length() const { return m_header->length() + m_payloadLen; }
 
     virtual std::shared_ptr payload() const { return m_payload; }
 
+    void releasePayload()
+    {
+        if (m_payload)
+        {
+            m_payload->clear();
+            bcos::bytes().swap(*m_payload);
+        }
+        if (m_frontMessage)
+        {
+            m_frontMessage->releasePayload();
+        }
+    }
+
 protected:
     MessageHeader::Ptr m_header;
     // Note: allocate here in case of wsService nullptr access caused coredump
     std::shared_ptr m_payload = std::make_shared();
+    uint64_t m_payloadLen = 0;
+    ;
+
     MessagePayload::Ptr m_frontMessage = nullptr;
 };
 
diff --git a/cpp/ppc-framework/protocol/MessagePayload.h b/cpp/ppc-framework/protocol/MessagePayload.h
index 8d2194d9..68ba0739 100644
--- a/cpp/ppc-framework/protocol/MessagePayload.h
+++ b/cpp/ppc-framework/protocol/MessagePayload.h
@@ -77,6 +77,12 @@ class MessagePayload
 
     virtual bool isRespPacket() { return m_ext &= (uint16_t)FrontMsgExtFlag::Response; }
 
+    void releasePayload()
+    {
+        m_data.clear();
+        bcos::bytes().swap(m_data);
+    }
+
 protected:
     // the front payload version, used to support compatibility
     // Note: must init here to 0, otherwise, it will be unexpected value in some other platform
diff --git a/cpp/ppc-framework/protocol/PPCMessageFace.h b/cpp/ppc-framework/protocol/PPCMessageFace.h
index 04fc9a76..263750da 100644
--- a/cpp/ppc-framework/protocol/PPCMessageFace.h
+++ b/cpp/ppc-framework/protocol/PPCMessageFace.h
@@ -75,6 +75,8 @@ class PPCMessageFace
     virtual bool response() const = 0;
     // set the message to be response
     virtual void setResponse() = 0;
+
+    virtual void releasePayload() {}
 };
 
 class PPCMessageFaceFactory
diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h
index 37909ef0..f16ec21a 100644
--- a/cpp/ppc-framework/protocol/Protocol.h
+++ b/cpp/ppc-framework/protocol/Protocol.h
@@ -23,7 +23,7 @@
 #define WIN32_LEAN_AND_MEAN
 #include 
 #endif
-
+#include "ppc-framework/Common.h"
 #include 
 #include 
 #include 
@@ -402,6 +402,35 @@ struct SQLConnectionOption
     std::string user;
     std::string password;
     std::string database;
+    void check() const
+    {
+        if (host.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid SQL option: Must set the host!"));
+        }
+        if (user.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid SQL option: Must set the user!"));
+        }
+        if (password.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid SQL option: Must set the password!"));
+        }
+        if (database.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid SQL option: Must set the database!"));
+        }
+        if (port == 0 || port > 65535)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid SQL Option, Must set valid port!"));
+        }
+    }
+
     inline std::string desc() const
     {
         std::stringstream oss;
@@ -423,6 +452,25 @@ struct FileStorageConnectionOption
     bool replaceDataNodeOnFailure = false;
     // the default connection-timeout for the hdfs is 1000ms
     uint16_t connectionTimeout = 1000;
+
+    void check() const
+    {
+        if (nameNode.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid HDFS Option, Must set the nameNode!"));
+        }
+        if (userName.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid HDFS Option, Must set the userName!"));
+        }
+        if (nameNodePort == 0 || nameNodePort > 65535)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid HDFS Option, Must set valid namenodeport!"));
+        }
+    }
     inline std::string desc() const
     {
         std::stringstream oss;
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index 3825a31b..5852e25f 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -125,8 +125,7 @@ install_iconv_depend()
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
-    brew install autoconf nasm lcov
-    brew link libiconv
+    brew install autoconf nasm lcov iconv
     install_gsasl_depend "macos"
     install_iconv_depend
     LOG_INFO "install depends for macos success ..."
diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
index 1b81b545..af8cdf64 100644
--- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
@@ -23,8 +23,10 @@
 #include "ppc-framework/Helper.h"
 #include "ppc-framework/front/FrontInterface.h"
 #include "ppc-framework/io/DataResourceLoader.h"
+#include "ppc-framework/protocol/Constant.h"
 #include "ppc-framework/protocol/Protocol.h"
 #include "psi-framework/interfaces/PSIMessageInterface.h"
+#include 
 #include 
 #include 
 
@@ -95,6 +97,14 @@ class PSIConfig
                 }
             },
             _responseCallback);
+        // release the large buffer if no-need to use
+        if (ppcMsg->data() && ppcMsg->data()->size() > ppc::protocol::LARGE_MSG_THRESHOLD)
+        {
+            PSI_LOG(INFO) << LOG_DESC("sendMsg: Release large buffer since the message")
+                          << LOG_KV("size", ppcMsg->data()->size());
+            ppcMsg->releasePayload();
+            MallocExtension::instance()->ReleaseFreeMemory();
+        }
     }
 
     void asyncSendResponse(bcos::bytes const& fromNode, std::string const& _taskID,
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h
index 957579c6..1654f33b 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/Common.h
@@ -5,6 +5,10 @@
 namespace ppc::psi
 {
 #define ECDH_MULTI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-MULTI-PSI")
+#define ECDH_MASTER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-MULTI-PSI-MASTER")
+#define ECDH_PARTNER_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-MULTI-PSI-PARTNER")
+#define ECDH_CAL_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("ECDH-MULTI-PSI-CALCULATOR")
+
 DERIVE_PPC_EXCEPTION(ECDHMULTIException);
 enum class EcdhMultiPSIMessageType : uint8_t
 {
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
new file mode 100644
index 00000000..02bde0e2
--- /dev/null
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
@@ -0,0 +1,483 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file EcdhCache.h
+ * @author: zachma
+ * @date 2023-5-28
+ */
+#include "EcdhMultiCache.h"
+#include 
+#include 
+
+using namespace ppc::psi;
+using namespace ppc::protocol;
+using namespace bcos;
+
+void MasterCache::addCalculatorCipher(std::string _peerId, std::vector&& _cipherData,
+    std::vector const& dataIndex, uint32_t seq, uint32_t dataBatchCount)
+{
+    auto peerIndex = getPeerIndex(_peerId);
+    if (peerIndex == -1)
+    {
+        ECDH_MULTI_LOG(WARNING) << LOG_DESC("Invalid calculator") << LOG_KV("peer", _peerId);
+        return;
+    }
+    bcos::Guard l(m_mutex);
+    m_calculatorCipherSeqs.insert(seq);
+    if (dataBatchCount)
+    {
+        m_calculatorDataBatchCount = dataBatchCount;
+    }
+    uint64_t i = 0;
+    for (auto&& it : _cipherData)
+    {
+        updateMasterDataRef(peerIndex, std::move(it), dataIndex[i]);
+        i++;
+    }
+    // try to merge the
+    if (m_calculatorDataBatchCount > 0 &&
+        m_calculatorCipherSeqs.size() == m_calculatorDataBatchCount)
+    {
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("The master receive all cipher data from the calculator")
+                             << LOG_KV("seq", seq) << LOG_KV("calculator", _peerId)
+                             << LOG_KV("masterData", m_masterDataRef.size()) << printCacheState();
+        m_finishedPartners.insert(_peerId);
+        // try to merge
+        mergeMasterCipher(_peerId, peerIndex);
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC(
+                                "addCalculatorCipher: master receive cipher data from calculator")
+                         << LOG_KV("calculator", _peerId) << printCacheState() << LOG_KV("seq", seq)
+                         << LOG_KV("receivedSize", _cipherData.size())
+                         << LOG_KV("masterData", m_masterDataRef.size())
+                         << LOG_KV("dataBatchCount", m_calculatorDataBatchCount);
+    // release the cipherData
+    _cipherData.clear();
+    std::vector().swap(_cipherData);
+    MallocExtension::instance()->ReleaseFreeMemory();
+}
+
+void MasterCache::updateMasterDataRef(
+    unsigned short _peerIndex, bcos::bytes&& data, int32_t dataIndex)
+{
+    // not merged case
+    if (!m_peerMerged)
+    {
+        // new data case
+        if (!m_masterDataRef.count(data))
+        {
+            MasterCipherRef ref;
+            ref.refInfo |= (1 << _peerIndex);
+            ref.updateDataIndex(dataIndex);
+            m_masterDataRef.insert(std::make_pair(std::move(data), ref));
+            return;
+        }
+        // existed data case
+        m_masterDataRef[data].refInfo |= (1 << _peerIndex);
+        m_masterDataRef[data].updateDataIndex(dataIndex);
+        return;
+    }
+
+    // merged case, only record the intersection case, increase the refCount
+    if (m_masterDataRef.count(data))
+    {
+        m_masterDataRef[data].refCount += 1;
+        m_masterDataRef[data].updateDataIndex(dataIndex);
+    }
+}
+
+
+void MasterCache::addPartnerCipher(std::string _peerId, std::vector&& _cipherData,
+    uint32_t seq, uint32_t parternerDataCount)
+{
+    auto peerIndex = getPeerIndex(_peerId);
+    if (peerIndex == -1)
+    {
+        ECDH_MULTI_LOG(WARNING) << LOG_DESC("Invalid peerId") << LOG_KV("peer", _peerId);
+        return;
+    }
+    bcos::Guard lock(m_mutex);
+    // record the data-ref-count
+    for (auto&& data : _cipherData)
+    {
+        updateMasterDataRef(peerIndex, std::move(data), -1);
+    }
+    m_partnerCipherSeqs[_peerId].insert(seq);
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("addPartnerCipher") << LOG_KV("partner", _peerId)
+                         << LOG_KV("seq", seq)
+                         << LOG_KV("receivedBatch", m_partnerCipherSeqs.at(_peerId).size())
+                         << LOG_KV("cipherDataSize", _cipherData.size())
+                         << LOG_KV("masterDataSize", m_masterDataRef.size())
+                         << LOG_KV("parternerDataCount", parternerDataCount) << printCacheState();
+    _cipherData.clear();
+    std::vector().swap(_cipherData);
+    MallocExtension::instance()->ReleaseFreeMemory();
+    if (parternerDataCount > 0)
+    {
+        m_parternerDataCount.insert(std::make_pair(_peerId, parternerDataCount));
+    }
+    if (!m_parternerDataCount.count(_peerId))
+    {
+        return;
+    }
+    auto expectedCount = m_parternerDataCount.at(_peerId);
+    if (m_partnerCipherSeqs[_peerId].size() == expectedCount)
+    {
+        m_finishedPartners.insert(_peerId);
+        // merge when find the send-finished peer
+        mergeMasterCipher(_peerId, peerIndex);
+    }
+}
+
+void MasterCache::mergeMasterCipher(std::string const& peerId, unsigned short peerIndex)
+{
+    if (m_peerMerged)
+    {
+        return;
+    }
+    // no need to merge when partnerCount is 1
+    if (m_peerCount == 1)
+    {
+        return;
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("Receive whole data from peer, mergeMasterCipher")
+                         << LOG_KV("distinct-masterDataSize-before-merge", m_masterDataRef.size())
+                         << LOG_KV("finishedPeer", peerId) << LOG_KV("partnerCount", m_peerCount);
+    auto startT = utcSteadyTime();
+    for (auto it = m_masterDataRef.begin(); it != m_masterDataRef.end();)
+    {
+        // not has intersect-element with the finished peer
+        if (!(it->second.refInfo & (1 << peerIndex)))
+        {
+            it = m_masterDataRef.erase(it);
+            continue;
+        }
+        // reset the refCount
+        it->second.resetRefCount();
+        it++;
+    }
+    m_peerMerged = true;
+    // release the free memory after merged
+    MallocExtension::instance()->ReleaseFreeMemory();
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("mergeMasterCipher finished")
+                         << LOG_KV("distinct-masterDataSize-after-merge", m_masterDataRef.size())
+                         << LOG_KV("finishedPeer", peerId) << LOG_KV("peerIndex", peerIndex)
+                         << LOG_KV("timecost", (utcSteadyTime() - startT));
+}
+
+// get the cipher-data intersection: h(x)^a && h(Y)^a
+bool MasterCache::tryToIntersection()
+{
+    if (!shouldIntersection())
+    {
+        return false;
+    }
+    m_cacheState = CacheState::IntersectionProgressing;
+
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToIntersection ") << printCacheState()
+                         << LOG_KV("* masterData", m_masterDataRef.size());
+    auto startT = utcSteadyTime();
+    // iterator the masterDataRef to obtain intersection
+    for (auto&& it : m_masterDataRef)
+    {
+        if (!m_masterDataRef.count(it.first))
+        {
+            continue;
+        }
+        if (m_masterDataRef.at(it.first).refCount != m_peerCount)
+        {
+            continue;
+        }
+        if (m_masterDataRef.at(it.first).dataIndex == -1)
+        {
+            continue;
+        }
+        // intersection case
+        m_intersecCipher.emplace_back(std::move(it.first));
+        m_intersecCipherIndex.emplace_back(it.second.dataIndex);
+    }
+    releaseCache();
+    m_cacheState = CacheState::Intersectioned;
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToIntersection success") << printCacheState()
+                         << LOG_KV("* intersectionSize", m_intersecCipher.size())
+                         << LOG_KV("* timecost", (utcSteadyTime() - startT));
+    return true;
+}
+
+void MasterCache::encryptAndSendIntersection(uint64_t dataBatchIdx, bcos::bytes const& randomKey,
+    std::map const& calculators)
+{
+    auto startT = utcSteadyTime();
+    auto message = m_config->psiMsgFactory()->createPSIMessage(
+        uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR));
+    message->setFrom(m_taskState->task()->selfParty()->id());
+    size_t startIndex = dataBatchIdx * m_config->dataBatchSize();
+    size_t endIndex = std::min(startIndex + m_config->dataBatchSize(), m_intersecCipher.size());
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("encryptAndSendIntersection")
+                         << LOG_KV("cipherCount", m_intersecCipher.size())
+                         << LOG_KV("dataBatchIdx", dataBatchIdx) << LOG_KV("start", startIndex)
+                         << LOG_KV("end", endIndex) << printCacheState();
+    message->resizeData(endIndex - startIndex);
+    if (endIndex >= m_intersecCipher.size())
+    {
+        message->setDataBatchCount(dataBatchIdx + 1);
+    }
+    tbb::parallel_for(tbb::blocked_range(startIndex, endIndex), [&](auto const& range) {
+        for (auto i = range.begin(); i < range.end(); i++)
+        {
+            auto cipherValue = m_config->eccCrypto()->ecMultiply(m_intersecCipher[i], randomKey);
+            message->setDataPair(
+                i - startIndex, m_intersecCipherIndex[i - startIndex], cipherValue);
+        }
+    });
+    auto taskID = m_taskState->task()->id();
+    auto self = weak_from_this();
+    for (auto const& calcultor : calculators)
+    {
+        m_config->generateAndSendPPCMessage(
+            calcultor.first, taskID, message,
+            [self](bcos::Error::Ptr&& _error) {
+                if (!_error)
+                {
+                    return;
+                }
+            },
+            dataBatchIdx);
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("encryptAndSendIntersection success")
+                         << LOG_KV("cipherCount", m_intersecCipher.size())
+                         << LOG_KV("dataBatchIdx", dataBatchIdx)
+                         << LOG_KV("timecost", utcSteadyTime() - startT) << printCacheState();
+}
+
+void MasterCache::encryptIntersection(
+    bcos::bytes const& randomKey, std::map const& calculators)
+{
+    auto dataBatchCount =
+        std::max((uint64_t)(m_intersecCipher.size() + m_config->dataBatchSize() - 1) /
+                     (uint64_t)m_config->dataBatchSize(),
+            (uint64_t)1);
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("encryptIntersection")
+                         << LOG_KV("cipherCount", m_intersecCipher.size())
+                         << LOG_KV("dataBatchCount", dataBatchCount) << printCacheState();
+    auto startT = utcSteadyTime();
+    for (uint64_t i = 0; i < dataBatchCount; i++)
+    {
+        encryptAndSendIntersection(i, randomKey, calculators);
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("encryptIntersection success")
+                         << LOG_KV("timecost", (utcSteadyTime() - startT))
+                         << LOG_KV("cipherCount", m_intersecCipher.size()) << printCacheState();
+    // Note: release the m_intersecCipher, make share it not been used after released
+    releaseIntersection();
+}
+
+bcos::bytes CalculatorCache::getPlainDataByIndex(uint64_t index)
+{
+    uint64_t startIndex = 0;
+    uint64_t endIndex = 0;
+    for (auto const& it : m_plainData)
+    {
+        endIndex += it->size();
+        if (index >= startIndex && index < endIndex)
+        {
+            return it->getBytes((index - startIndex));
+        }
+        startIndex += it->size();
+    }
+    return bcos::bytes();
+}
+
+bool CalculatorCache::tryToFinalize()
+{
+    if (!shouldFinalize())
+    {
+        return false;
+    }
+    auto startT = utcSteadyTime();
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize: compute intersection")
+                         << LOG_KV("* cipherRef", m_cipherRef.size()) << printCacheState();
+    m_cacheState = CacheState::Finalizing;
+    // find the intersection
+    for (auto const& it : m_cipherRef)
+    {
+        if (it.second.refCount < 2)
+        {
+            continue;
+        }
+        if (it.second.plainDataIndex >= 0)
+        {
+            m_intersectionResult.emplace_back(getPlainDataByIndex(it.second.plainDataIndex));
+        }
+    }
+    m_cacheState = CacheState::Finalized;
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize:  compute intersection success")
+                         << printCacheState() << LOG_KV("* cipherRef", m_cipherRef.size())
+                         << LOG_KV("* intersectionSize", m_intersectionResult.size())
+                         << LOG_KV("* timecost", (utcSteadyTime() - startT));
+
+    releaseDataAfterFinalize();
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize: syncIntersections") << printCacheState();
+    m_cacheState = CacheState::Syncing;
+    syncIntersections();
+    m_cacheState = CacheState::Synced;
+
+    m_cacheState = CacheState::StoreProgressing;
+    m_taskState->storePSIResult(m_config->dataResourceLoader(), m_intersectionResult);
+    m_cacheState = CacheState::Stored;
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize: syncIntersections and store success")
+                         << printCacheState();
+    return true;
+}
+
+void CalculatorCache::syncIntersections()
+{
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("*** syncIntersections **") << printCacheState();
+    auto peers = m_taskState->task()->getAllPeerParties();
+    auto taskID = m_taskState->task()->id();
+    // notify task result
+    if (!m_syncResult)
+    {
+        auto message = m_config->psiMsgFactory()->createPSIMessage(
+            uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL));
+        message->setFrom(m_taskState->task()->selfParty()->id());
+        message->setVersion(-1);
+        for (auto& peer : peers)
+        {
+            m_config->generateAndSendPPCMessage(
+                peer.first, taskID, message,
+                [taskID, peer](bcos::Error::Ptr&& _error) {
+                    if (_error && _error->errorCode() != 0)
+                    {
+                        ECDH_MULTI_LOG(WARNING)
+                            << LOG_DESC("sync task result to peer failed")
+                            << LOG_KV("peer", peer.first) << LOG_KV("taskID", taskID)
+                            << LOG_KV("code", _error->errorCode())
+                            << LOG_KV("msg", _error->errorMessage());
+                        return;
+                    }
+                },
+                0);
+        }
+        return;
+    }
+    // sync intersectionResult to all peers
+    auto message = m_config->psiMsgFactory()->createPSIMessage(
+        uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL));
+    message->setData(m_intersectionResult);
+    message->setFrom(m_taskState->task()->selfParty()->id());
+    message->setVersion(0);
+    for (auto& peer : peers)
+    {
+        m_config->generateAndSendPPCMessage(
+            peer.first, taskID, message,
+            [taskID, peer](bcos::Error::Ptr&& _error) {
+                if (_error && _error->errorCode() != 0)
+                {
+                    ECDH_MULTI_LOG(WARNING)
+                        << LOG_DESC("sync psi result to peer failed") << LOG_KV("peer", peer.first)
+                        << LOG_KV("taskID", taskID) << LOG_KV("code", _error->errorCode())
+                        << LOG_KV("msg", _error->errorMessage());
+                    return;
+                }
+            },
+            0);
+    }
+}
+
+void CalculatorCache::updateCipherRef(bcos::bytes&& data, int32_t index)
+{
+    // case that receive at least one completed data, only record the intersection data
+    if (m_receiveAllMasterCipher || m_receiveAllIntersection)
+    {
+        if (!m_cipherRef.count(data))
+        {
+            return;
+        }
+    }
+    // new data case
+    if (!m_cipherRef.count(data))
+    {
+        CipherRefDetail cipherRef;
+        cipherRef.refCount = 1;
+        cipherRef.updatePlainIndex(index);
+        m_cipherRef.insert(std::make_pair(std::move(data), std::move(cipherRef)));
+        return;
+    }
+    // existed data case
+    m_cipherRef[data].refCount += 1;
+    m_cipherRef[data].updatePlainIndex(index);
+}
+
+
+bool CalculatorCache::appendMasterCipher(
+    std::vector&& _cipherData, uint32_t seq, uint32_t dataBatchSize)
+{
+    bcos::Guard lock(m_mutex);
+    m_receivedMasterCipher.insert(seq);
+    for (auto&& it : _cipherData)
+    {
+        updateCipherRef(std::move(it), -1);
+    }
+    if (m_masterDataBatchSize == 0 && dataBatchSize > 0)
+    {
+        m_masterDataBatchSize = dataBatchSize;
+    }
+    if (!m_receiveAllMasterCipher && m_receivedMasterCipher.size() == m_masterDataBatchSize)
+    {
+        m_receiveAllMasterCipher = true;
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("appendMasterCipher") << LOG_KV("dataSize", _cipherData.size())
+                         << LOG_KV("cipherRefSize", m_cipherRef.size()) << printCacheState();
+    // release the cipherData
+    _cipherData.clear();
+    std::vector().swap(_cipherData);
+    MallocExtension::instance()->ReleaseFreeMemory();
+    return m_receiveAllMasterCipher;
+}
+
+void CalculatorCache::addIntersectionCipher(std::vector&& _cipherData,
+    std::vector const& dataIndex, uint32_t seq, uint64_t dataBatchCount)
+{
+    auto startT = utcSteadyTime();
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("addIntersectionCipher") << LOG_KV("seq", seq)
+                         << LOG_KV("dataSize", _cipherData.size())
+                         << LOG_KV("cipherRefSize", m_cipherRef.size()) << printCacheState()
+                         << LOG_KV("dataBatchCount", dataBatchCount);
+    bcos::Guard lock(m_mutex);
+    uint64_t i = 0;
+    for (auto&& it : _cipherData)
+    {
+        updateCipherRef(std::move(it), dataIndex[i]);
+        i++;
+    }
+    m_receivedIntersections.insert(seq);
+    if (dataBatchCount > 0)
+    {
+        m_intersectionBatchCount = dataBatchCount;
+    }
+    if (m_intersectionBatchCount > 0 && m_receivedIntersections.size() >= m_intersectionBatchCount)
+    {
+        m_receiveAllIntersection = true;
+    }
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("addIntersectionCipher finshed")
+                         << LOG_KV("timecost", utcSteadyTime() - startT) << LOG_KV("seq", seq)
+                         << LOG_KV("receiveAll", m_receiveAllIntersection)
+                         << LOG_KV("cipherRefSize", m_cipherRef.size()) << printCacheState();
+    // release the cipherData
+    _cipherData.clear();
+    std::vector().swap(_cipherData);
+    MallocExtension::instance()->ReleaseFreeMemory();
+}
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
index 0dd01f07..f3a9e416 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
@@ -19,229 +19,300 @@
  */
 #pragma once
 #include "Common.h"
+#include "EcdhMultiPSIConfig.h"
+#include "ppc-psi/src/Common.h"
 #include "ppc-psi/src/psi-framework/TaskState.h"
 #include 
 #include 
+#include 
 
 namespace ppc::psi
 {
-/// the master data-cache
-class MasterCipherDataCache
+struct MasterCipherRef
 {
-public:
-    using Ptr = std::shared_ptr;
-    MasterCipherDataCache() {}
-    virtual ~MasterCipherDataCache()
-    {
-        m_masterFinalIntersectionCipherData.clear();
-        m_masterTaskPeersFinishedList.clear();
-        m_masterCipherDataFromCalculator.clear();
-        m_masterCipherDataFromPartner.clear();
-        m_CipherDataFromCalculatorSubSeq.clear();
-        m_CipherDataFromPartnerSubSeq.clear();
-        std::map().swap(m_masterFinalIntersectionCipherData);
-        std::set().swap(m_masterTaskPeersFinishedList);
-        std::map().swap(m_masterCipherDataFromCalculator);
-        std::map>().swap(m_masterCipherDataFromPartner);
-        std::set().swap(m_CipherDataFromCalculatorSubSeq);
-        std::map>().swap(m_CipherDataFromPartnerSubSeq);
-        MallocExtension::instance()->ReleaseFreeMemory();
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("the master cipher datacache destroyed ")
-                             << LOG_KV("taskID", m_taskID);
-    }
+    // support at least 63 peers
+    uint64_t refInfo = 0;
+    unsigned short refCount = 0;
+    int32_t dataIndex = -1;
 
-    void appendMasterCipherDataFromCalculator(std::string _peerId,
-        std::map&& _cipherData, uint32_t seq, uint32_t needSendTimes)
+    void resetRefCount()
     {
-        try
+        auto ref = refInfo;
+        do
         {
-            bcos::WriteGuard lock(x_peerTasks);
-            m_masterCipherDataFromCalculator.insert(_cipherData.begin(), _cipherData.end());
-            m_CipherDataFromCalculatorSubSeq.insert(seq);
-            ECDH_MULTI_LOG(INFO) << LOG_KV(
-                "Part1-C:Master Receive H(X)*A Size: ", m_CipherDataFromCalculatorSubSeq.size());
-            if (m_CipherDataFromCalculatorSubSeq.size() == needSendTimes)
+            if (ref & 0x1)
             {
-                m_masterTaskPeersFinishedList.insert(_peerId);
+                refCount++;
             }
-        }
-        catch (std::exception& e)
-        {
-            ECDH_MULTI_LOG(ERROR) << LOG_DESC("appendMasterCipherDataFromCalculator Exception:")
-                                  << boost::diagnostic_information(e);
-        }
+            ref >>= 1;
+        } while (ref > 0);
     }
-
-    void appendMasterCipherDataFromPartner(std::string _peerId,
-        std::vector&& _cipherData, uint32_t seq, uint32_t needSendTimes)
+    void updateDataIndex(int32_t index)
     {
-        try
+        if (index == -1)
         {
-            bcos::WriteGuard lock(x_peerTasks);
-            m_masterCipherDataFromPartner[_peerId].insert(
-                m_masterCipherDataFromPartner[_peerId].end(), _cipherData.begin(),
-                _cipherData.end());
-
-            m_CipherDataFromPartnerSubSeq[_peerId].insert(seq);
-            ECDH_MULTI_LOG(INFO) << LOG_KV(
-                "Part1-P:Master Receive H(Y)*A Size: ", m_CipherDataFromPartnerSubSeq.size());
-            if (m_CipherDataFromPartnerSubSeq[_peerId].size() == needSendTimes)
-            {
-                m_masterTaskPeersFinishedList.insert(_peerId);
-            }
+            return;
         }
-        catch (std::exception& e)
+        dataIndex = index;
+    }
+};
+/// the master data-cache
+class MasterCache : public std::enable_shared_from_this
+{
+public:
+    using Ptr = std::shared_ptr;
+    MasterCache(TaskState::Ptr const& taskState, EcdhMultiPSIConfig::Ptr const& config)
+      : m_taskState(taskState),
+        m_config(config),
+        m_peerCount(m_taskState->task()->getAllPeerParties().size())
+    {
+        for (auto const& it : m_taskState->task()->getAllPeerParties())
         {
-            ECDH_MULTI_LOG(ERROR) << LOG_DESC("appendMasterCipherDataFromPartner Exception:")
-                                  << boost::diagnostic_information(e);
+            m_peers.emplace_back(it.first);
         }
     }
 
-    std::set const& masterTaskPeersFinishedList()
+    virtual ~MasterCache()
     {
-        return m_masterTaskPeersFinishedList;
+        releaseCache();
+        releaseIntersection();
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("the master cipher datacache destroyed ")
+                             << LOG_KV("taskID", m_taskState->task()->id());
     }
 
-    std::map const& masterFinalIntersectionCipherData()
+    void addCalculatorCipher(std::string _peerId, std::vector&& _cipherData,
+        std::vector const& dataIndex, uint32_t seq, uint32_t dataBatchCount);
+
+    void addPartnerCipher(std::string _peerId, std::vector&& _cipherData, uint32_t seq,
+        uint32_t parternerDataCount);
+
+    bool tryToIntersection();
+
+    std::string printCacheState()
     {
-        return m_masterFinalIntersectionCipherData;
+        std::ostringstream stringstream;
+        stringstream << LOG_KV("taskID", m_taskState->task()->id())
+                     << LOG_KV("CacheState", m_cacheState);
+        return stringstream.str();
     }
 
-    void tryToGetCipherDataIntersection()
+    void encryptIntersection(bcos::bytes const& randomKey,
+        std::map const& calculators);
+
+private:
+    void encryptAndSendIntersection(uint64_t dataBatchIdx, bcos::bytes const& randomKey,
+        std::map const& calculators);
+
+    bool shouldIntersection()
     {
-        // partner intersection _masterCipherDataIntersectionFromPartner
-        std::vector _masterCipherDataIntersectionFromPartnerVector;
-        std::map cnt;
-        uint32_t n = m_masterCipherDataFromPartner.size();
-        for (auto& data : m_masterCipherDataFromPartner)
+        // only evaluating state should intersection
+        if (m_cacheState != CacheState::Evaluating)
         {
-            std::set _temp(data.second.begin(), data.second.end());
-            for (auto& v : _temp)
+            return false;
+        }
+        auto allPeerParties = m_taskState->task()->getAllPeerParties();
+        if (allPeerParties.size() == m_finishedPartners.size())
+        {
+            for (auto const& it : allPeerParties)
             {
-                cnt[v]++;
-                if (cnt[v] == n)
+                if (!m_finishedPartners.contains(it.first))
                 {
-                    _masterCipherDataIntersectionFromPartnerVector.push_back(v);
+                    return false;
                 }
             }
+            return true;
         }
+        return false;
+    }
 
-        // calculator intersection partners m_masterFinalIntersectionCipherData
-        m_masterFinalIntersectionCipherData.clear();
-        std::map temp_out;
-        for (auto& data : m_masterCipherDataFromCalculator)
-        {
-            // temp_out.insert(std::make_pair(data.second, data.first));
-            temp_out.emplace(std::make_pair(data.second, data.first));
-        }
+    void releaseIntersection()
+    {
+        m_intersecCipher.clear();
+        m_intersecCipherIndex.clear();
 
-        for (const auto& data : _masterCipherDataIntersectionFromPartnerVector)
+        // release the intersection information
+        std::vector().swap(m_intersecCipher);
+        std::vector().swap(m_intersecCipherIndex);
+
+        MallocExtension::instance()->ReleaseFreeMemory();
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("releaseIntersection")
+                             << LOG_KV("taskID", m_taskState->task()->id());
+    }
+
+    void releaseCache()
+    {
+        m_masterDataRef.clear();
+
+        // release the parterner cipher
+        std::map().swap(m_masterDataRef);
+        MallocExtension::instance()->ReleaseFreeMemory();
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("releaseCache")
+                             << LOG_KV("taskID", m_taskState->task()->id());
+    }
+
+    void mergeMasterCipher(std::string const& peerId, unsigned short peerIndex);
+    void updateMasterDataRef(unsigned short _peerIndex, bcos::bytes&& data, int32_t dataIndex);
+
+    signed short getPeerIndex(std::string const& peer)
+    {
+        for (unsigned short i = 0; i < m_peers.size(); i++)
         {
-            auto it = temp_out.find(data);
-            if (it != temp_out.end())
+            if (m_peers[i] == peer)
             {
-                // m_masterFinalIntersectionCipherData.insert(std::make_pair(it->second,
-                // it->first));
-                m_masterFinalIntersectionCipherData.emplace(std::make_pair(it->second, it->first));
+                return i;
             }
         }
+        return -1;
     }
 
 private:
-    std::string m_taskID;
-
-    // store the cipher-data of the master
-    std::map m_masterFinalIntersectionCipherData;
-    std::set m_masterTaskPeersFinishedList;
-    std::map m_masterCipherDataFromCalculator;
-    std::map> m_masterCipherDataFromPartner;
-    // std::vector m_masterCipherDataIntersectionFromPartner;
-    std::set m_CipherDataFromCalculatorSubSeq;
-    std::map> m_CipherDataFromPartnerSubSeq;
-    bcos::SharedMutex x_peerTasks;
+    TaskState::Ptr m_taskState;
+    EcdhMultiPSIConfig::Ptr m_config;
+    unsigned short m_peerCount;
+    std::vector m_peers;
+
+    CacheState m_cacheState = CacheState::Evaluating;
+
+    // the intersection cipher data of the master
+    std::vector m_intersecCipher;
+    std::vector m_intersecCipherIndex;
+
+    std::set m_calculatorCipherSeqs;
+    uint32_t m_calculatorDataBatchCount = 0;
+
+    //  data => refered peers
+    std::map m_masterDataRef;
+
+    // partnerId=>received partner seqs
+    std::map> m_partnerCipherSeqs;
+    // peerId==>dataCount
+    std::map m_parternerDataCount;
+    std::set m_finishedPartners;
+
+    bool m_peerMerged = false;
+
+    bcos::Mutex m_mutex;
 };
 
-class CalculatorCipherDataCache
+// the cipher ref count
+// data ==> {ref count, plainData}
+struct CipherRefDetail
+{
+    unsigned short refCount = 0;
+    int32_t plainDataIndex = -1;
+
+    void updatePlainIndex(int32_t index)
+    {
+        if (index == -1)
+        {
+            return;
+        }
+        plainDataIndex = index;
+    }
+};
+
+class CalculatorCache : public std::enable_shared_from_this
 {
 public:
-    using Ptr = std::shared_ptr;
-    CalculatorCipherDataCache() {}
-    virtual ~CalculatorCipherDataCache()
+    using Ptr = std::shared_ptr;
+    CalculatorCache(
+        TaskState::Ptr const& taskState, bool syncResult, EcdhMultiPSIConfig::Ptr const& config)
+      : m_taskState(taskState), m_syncResult(syncResult), m_config(config)
+    {}
+    virtual ~CalculatorCache()
     {
-        m_CipherDataFromCalculatorSubSeq.clear();
-        m_calculatorIntersectionSubSeq.clear();
-        m_calculatorCipherDataSubSeq.clear();
-        m_calculatorCipherData.clear();
-        m_calculatorIntersectionCipherDataMap.clear();
-        m_calculatorIntersectionCipherDataFinalMap.clear();
-        std::set().swap(m_CipherDataFromCalculatorSubSeq);
-        std::set().swap(m_calculatorIntersectionSubSeq);
-        std::set().swap(m_calculatorCipherDataSubSeq);
-        std::vector().swap(m_calculatorCipherData);
-        std::map().swap(m_calculatorIntersectionCipherDataMap);
-        std::map().swap(m_calculatorIntersectionCipherDataFinalMap);
+        releaseDataAfterFinalize();
+
+        m_intersectionResult.clear();
+        std::vector().swap(m_intersectionResult);
         MallocExtension::instance()->ReleaseFreeMemory();
         ECDH_MULTI_LOG(INFO) << LOG_DESC("the calculator cipher datacache destroyed")
-                             << LOG_KV("taskID", m_taskID);
+                             << LOG_KV("taskID", m_taskState->task()->id());
     }
 
-    void tryToGetCipherDataIntersection()
-    {
-        m_calculatorIntersectionCipherDataFinalMap.clear();
-        std::map temp_out;
-        for (auto& data : m_calculatorIntersectionCipherDataMap)
-        {
-            // temp_out.insert(std::make_pair(data.second, data.first));
-            temp_out.emplace(std::make_pair(data.second, data.first));
-        }
+    bool tryToFinalize();
 
-        for (const auto& data : m_calculatorCipherData)
-        {
-            auto it = temp_out.find(data);
-            if (it != temp_out.end())
-            {
-                m_calculatorIntersectionCipherDataFinalMap.emplace(
-                    std::make_pair(it->second, it->first));
-            }
-        }
+    bool appendMasterCipher(
+        std::vector&& _cipherData, uint32_t seq, uint32_t dataBatchSize);
+
+    void addIntersectionCipher(std::vector&& _cipherData,
+        std::vector const& dataIndex, uint32_t seq, uint64_t dataBatchCount);
+
+    void appendPlainData(ppc::io::DataBatch::Ptr const& data)
+    {
+        bcos::WriteGuard l(x_plainData);
+        m_plainData.emplace_back(data);
     }
 
-    bool setCalculatorCipherData(
-        std::vector&& _cipherData, uint32_t seq, uint32_t needSendTimes)
+    std::string printCacheState()
     {
-        bcos::WriteGuard lock(x_setCalculatorCipherData);
-        m_calculatorCipherData.insert(
-            m_calculatorCipherData.end(), _cipherData.begin(), _cipherData.end());
-        m_calculatorCipherDataSubSeq.insert(seq);
-        return m_calculatorCipherDataSubSeq.size() == needSendTimes;
+        std::ostringstream stringstream;
+        stringstream << LOG_KV("taskID", m_taskState->task()->id())
+                     << LOG_KV("CacheState", m_cacheState)
+                     << LOG_KV("intersectionSize", m_intersectionResult.size());
+        return stringstream.str();
     }
 
-    bool setCalculatorIntersectionCipherDataMap(
-        std::map&& _cipherData, uint32_t seq, uint32_t needSendTimes)
+private:
+    bcos::bytes getPlainDataByIndex(uint64_t index);
+    bool shouldFinalize()
     {
-        bcos::WriteGuard lock(x_setCalculatorIntersectionCipherData);
-        m_calculatorIntersectionCipherDataMap.insert(_cipherData.begin(), _cipherData.end());
-        m_calculatorIntersectionSubSeq.insert(seq);
-        return m_calculatorIntersectionSubSeq.size() == needSendTimes;
+        // only can finalize in Evaluating state
+        if (m_cacheState != CacheState::Evaluating)
+        {
+            return false;
+        }
+        if (!m_receiveAllIntersection)
+        {
+            return false;
+        }
+        if (m_receivedMasterCipher.size() == 0)
+        {
+            return false;
+        }
+        return m_receivedMasterCipher.size() == m_masterDataBatchSize;
     }
 
-    std::map const& calculatorIntersectionCipherDataFinalMap()
+    void syncIntersections();
+
+    void releaseDataAfterFinalize()
     {
-        return m_calculatorIntersectionCipherDataFinalMap;
+        for (auto const& it : m_plainData)
+        {
+            it->release();
+        }
+        m_cipherRef.clear();
+        std::map().swap(m_cipherRef);
+        MallocExtension::instance()->ReleaseFreeMemory();
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("releaseDataAfterFinalize")
+                             << LOG_KV("taskID", m_taskState->task()->id());
     }
 
+    void updateCipherRef(bcos::bytes&& data, int32_t index);
+
 private:
-    std::string m_taskID;
-
-    // store the cipher-data of the calculator
-    std::set m_CipherDataFromCalculatorSubSeq;
-    std::set m_calculatorIntersectionSubSeq;
-    std::set m_calculatorCipherDataSubSeq;
-    std::vector m_calculatorCipherData;
-    std::map m_calculatorIntersectionCipherDataMap;
-    std::map m_calculatorIntersectionCipherDataFinalMap;
-
-    mutable boost::shared_mutex x_setCalculatorCipherData;
-    mutable boost::shared_mutex x_setCalculatorIntersectionCipherData;
+    TaskState::Ptr m_taskState;
+    bool m_syncResult;
+    EcdhMultiPSIConfig::Ptr m_config;
+    CacheState m_cacheState = CacheState::Evaluating;
+
+    std::vector m_plainData;
+    bcos::SharedMutex x_plainData;
+
+    std::map m_cipherRef;
+
+    // the seqs of the data received from master
+    std::set m_receivedMasterCipher;
+    uint32_t m_masterDataBatchSize = 0;
+    bool m_receiveAllMasterCipher = false;
+    mutable bcos::Mutex m_mutex;
+
+    // the intersection cipher received from master
+    std::set m_receivedIntersections;
+    bool m_receiveAllIntersection = false;
+    uint32_t m_intersectionBatchCount = 0;
+
+    // the final result
+    std::vector m_intersectionResult;
 };
-}  // namespace ppc::psi
\ No newline at end of file
+}  // namespace ppc::psi
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
index 1f7ec3e1..f47a405f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
@@ -1,5 +1,7 @@
 #include "EcdhMultiPSIImpl.h"
 #include "Common.h"
+#include "ppc-framework/protocol/Constant.h"
+#include 
 
 using namespace ppc::psi;
 using namespace ppc::protocol;
@@ -46,40 +48,39 @@ void EcdhMultiPSIImpl::handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg)
             case (uint32_t)EcdhMultiPSIMessageType::GENERATE_RANDOM_TO_PARTNER:
             {
                 // calculator -> partner (A)
-                psi->onComputeAndEncryptSet(_msg);
+                psi->onReceiveRandomA(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR:
             {
                 // calculator -> master H(X)*A
-                psi->onHandlerIntersectEncryptSetFromCalculator(_msg);
+                psi->onReceiveCalCipher(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER:
             {
                 // patner -> master H(Y)*A
-                psi->onHandlerIntersectEncryptSetFromPartner(_msg);
+                psi->onReceiveCipherFromPartner(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR:
             {
-                psi->onHandlerIntersectEncryptSetToCalculator(_msg);
+                psi->onReceiveIntersecCipher(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR:
             {
-                psi->onHandlerEncryptSetToCalculator(_msg);
+                psi->onReceiveMasterCipher(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::
                 RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER:
             {
-                psi->onHandlerEncryptIntersectionSetFromCalculatorToMaster(_msg);
                 break;
             }
             case (uint32_t)EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL:
             {
-                psi->onHandlerSyncFinalResultToAllPeer(_msg);
+                psi->onReceivePSIResult(_msg);
                 break;
             }
             default:
@@ -103,9 +104,11 @@ void EcdhMultiPSIImpl::handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg)
 void EcdhMultiPSIImpl::asyncRunTask(
     ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished)
 {
+    auto taskID = _task->id();
     auto taskState =
         m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), false, m_config);
-    taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() {
+    auto self = weak_from_this();
+    taskState->registerNotifyPeerFinishHandler([self, _task]() {
         auto psi = self.lock();
         if (!psi)
         {
@@ -113,13 +116,36 @@ void EcdhMultiPSIImpl::asyncRunTask(
         }
         psi->noticePeerToFinish(_task);
     });
+    taskState->registerFinalizeHandler([self, taskID]() {
+        auto psi = self.lock();
+        if (!psi)
+        {
+            return;
+        }
+        // erase the taskInfo from the gateway
+        psi->m_config->front()->eraseTaskInfo(taskID);
+        psi->removeCalculator(taskID);
+        psi->removeMaster(taskID);
+        psi->removePartner(taskID);
+        psi->removePendingTask(taskID);
+    });
     addPendingTask(taskState);
-
+    // over the peer limit
+    if (_task->getAllPeerParties().size() > c_max_peer_size)
+    {
+        auto error = std::make_shared(
+            -1, "at most support " + std::to_string(c_max_peer_size) + " peers, over the limit!");
+        ECDH_MULTI_LOG(WARNING) << LOG_DESC("asyncRunTask failed")
+                                << LOG_KV("msg", error->errorMessage());
+        onSelfError(_task->id(), error, true);
+    }
     try
     {
         auto dataResource = _task->selfParty()->dataResource();
         auto reader = loadReader(_task->id(), dataResource, DataSchema::Bytes);
-        taskState->setReader(reader, -1);
+        auto sqlReader = (reader->type() == ppc::protocol::DataResourceType::MySQL);
+        auto nextParam = sqlReader ? 0 : m_config->dataBatchSize();
+        taskState->setReader(reader, nextParam);
         auto role = _task->selfParty()->partyIndex();
         auto receivers = _task->getReceiverLists();
         ECDH_MULTI_LOG(INFO) << LOG_DESC("Start a asyncRunTask ") << LOG_KV("taskID", _task->id())
@@ -221,7 +247,7 @@ void EcdhMultiPSIImpl::checkFinishedTask()
         for (auto it = m_pendingTasks.begin(); it != m_pendingTasks.end();)
         {
             auto task = it->second;
-            if (task->finished())
+            if (task->taskDone())
             {
                 finishedTask.insert(it->first);
             }
@@ -239,6 +265,7 @@ void EcdhMultiPSIImpl::checkFinishedTask()
 
 void EcdhMultiPSIImpl::onReceivedErrorNotification(const std::string& _taskID)
 {
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("onReceivedErrorNotification") << LOG_KV("taskID", _taskID);
     // finish the task while the peer is failed
     auto taskState = findPendingTask(_taskID);
     if (taskState)
@@ -296,13 +323,21 @@ void EcdhMultiPSIImpl::executeWorker()
         psiMsg->setUUID(pop_msg->uuid());
         ECDH_MULTI_LOG(TRACE) << LOG_DESC("onReceiveMessage") << printPSIMessage(psiMsg)
                               << LOG_KV("uuid", psiMsg->uuid());
+        // release the large payload immediately
+        if (payLoad && payLoad->size() >= ppc::protocol::LARGE_MSG_THRESHOLD)
+        {
+            ECDH_MULTI_LOG(INFO) << LOG_DESC("Release large message payload")
+                                 << LOG_KV("size", payLoad->size());
+            pop_msg->releasePayload();
+            MallocExtension::instance()->ReleaseFreeMemory();
+        }
         handlerPSIReceiveMessage(psiMsg);
         return;
     }
     waitSignal();
 }
 
-void EcdhMultiPSIImpl::onComputeAndEncryptSet(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceiveRandomA(PSIMessageInterface::Ptr _msg)
 {
     auto partner = findPartner(_msg->taskID());
     if (partner)
@@ -310,71 +345,70 @@ void EcdhMultiPSIImpl::onComputeAndEncryptSet(PSIMessageInterface::Ptr _msg)
         if (_msg->takeData().size() == 1)
         {
             auto msgData = _msg->getData(0);
-            partner->oncomputeAndEncryptSet(
+            partner->onReceiveRandomA(
                 std::make_shared(bcos::bytes(msgData.begin(), msgData.end())));
         }
     }
 }
 
-void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceiveCalCipher(PSIMessageInterface::Ptr _msg)
 {
     auto master = findMaster(_msg->taskID());
     if (master)
     {
-        master->onHandlerIntersectEncryptSetFromCalculator(_msg);
+        master->onReceiveCalCipher(_msg);
     }
 }
 
-void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceiveCipherFromPartner(PSIMessageInterface::Ptr _msg)
 {
     auto master = findMaster(_msg->taskID());
     if (master)
     {
-        master->onHandlerIntersectEncryptSetFromPartner(_msg);
+        master->onReceiveCipherFromPartner(_msg);
     }
 }
 
-void EcdhMultiPSIImpl::onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceiveIntersecCipher(PSIMessageInterface::Ptr _msg)
 {
     auto calculator = findCalculator(_msg->taskID());
     if (calculator)
     {
-        calculator->onHandlerIntersectEncryptSetToCalculator(_msg);
+        calculator->onReceiveIntersecCipher(_msg);
     }
 }
 
-void EcdhMultiPSIImpl::onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceiveMasterCipher(PSIMessageInterface::Ptr _msg)
 {
     auto calculator = findCalculator(_msg->taskID());
     if (calculator)
     {
-        calculator->onHandlerEncryptSetToCalculator(_msg);
+        calculator->onReceiveMasterCipher(_msg);
     }
 }
 
-void EcdhMultiPSIImpl::onHandlerEncryptIntersectionSetFromCalculatorToMaster(
-    PSIMessageInterface::Ptr _msg)
-{
-    auto master = findMaster(_msg->taskID());
-    if (master)
-    {
-        master->onHandlerEncryptIntersectionSetFromCalculatorToMaster(_msg);
-    }
-}
 
-void EcdhMultiPSIImpl::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIImpl::onReceivePSIResult(PSIMessageInterface::Ptr _msg)
 {
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("onReceivePSIResult") << printPSIMessage(_msg);
+    auto startT = utcSteadyTime();
     auto master = findMaster(_msg->taskID());
     if (master)
     {
-        master->onHandlerSyncFinalResultToAllPeer(_msg);
+        master->onReceivePSIResult(_msg);
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("Master onReceivePSIResult finished")
+                             << printPSIMessage(_msg)
+                             << LOG_KV("timecost", (utcSteadyTime() - startT));
         return;
     }
 
     auto partner = findPartner(_msg->taskID());
     if (partner)
     {
-        partner->onHandlerSyncFinalResultToAllPeer(_msg);
+        partner->onReceivePSIResult(_msg);
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner onReceivePSIResult finished")
+                             << printPSIMessage(_msg)
+                             << LOG_KV("timecost", (utcSteadyTime() - startT));
         return;
     }
 }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
index b2d8e968..df4e8034 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
@@ -48,15 +48,13 @@ class EcdhMultiPSIImpl : public std::enable_shared_from_this,
 
 protected:
     bool m_enableOutputExists = false;
-    virtual void onComputeAndEncryptSet(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveRandomA(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveCalCipher(PSIMessageInterface::Ptr _msg);
     virtual void handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerEncryptIntersectionSetFromCalculatorToMaster(
-        PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveIntersecCipher(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveMasterCipher(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveCipherFromPartner(PSIMessageInterface::Ptr _msg);
+    virtual void onReceivePSIResult(PSIMessageInterface::Ptr _msg);
 
     EcdhMultiPSICalculator::Ptr findCalculator(const std::string& _taskID)
     {
@@ -149,6 +147,10 @@ class EcdhMultiPSIImpl : public std::enable_shared_from_this,
     void wakeupWorker() { m_signal.notify_all(); }
 
     const int c_popWaitMs = 5;
+
+    // at most support 63 peers
+    const int c_max_peer_size = 63;
+
     EcdhMultiPSIConfig::Ptr m_config;
     TaskState::Ptr m_taskMultiState;
     EcdhMultiPSIMsgQueuePtr m_msgQueue;
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
index 3427edde..c63fe38e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
@@ -17,14 +17,14 @@ EcdhMultiPSICalculator::EcdhMultiPSICalculator(
     auto task = m_taskState->task();
     auto receivers = task->getReceiverLists();
     m_taskID = task->id();
-    m_calculatorCipherDataCache = std::make_shared();
     m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
                                                     m_config->selfParty()) != receivers.end());
+    m_calculatorCache = std::make_shared(m_taskState, m_syncResult, m_config);
 }
 
 void EcdhMultiPSICalculator::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task)
 {
-    InitAsyncTask(_task);
+    initTask(_task);
     auto randA = generateRandomA(_task->id());
     m_randomA = randA;
     m_config->threadPool()->enqueue([self = weak_from_this(), _task, randA]() {
@@ -33,32 +33,32 @@ void EcdhMultiPSICalculator::asyncStartRunTask(ppc::protocol::Task::ConstPtr _ta
         {
             return;
         }
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator asyncStartRunTask as calculator");
-        calculator->computeAndEncryptSet(_task->id(), randA);
+        ECDH_CAL_LOG(INFO) << LOG_DESC("Calculator asyncStartRunTask as calculator");
+        calculator->blindData(_task->id(), randA);
     });
 }
 
-// PART1: Calculator -> Partners (A)
+// PART1: Calculator -> Partners (randomValue)
 bcos::bytes EcdhMultiPSICalculator::generateRandomA(std::string _taskID)
 {
-    ECDH_MULTI_LOG(INFO) << LOG_KV("PART1: Calculator Start New Task :", _taskID);
-    auto A = m_config->eccCrypto()->generateRandomScalar();
+    ECDH_CAL_LOG(INFO) << LOG_KV("PART1: Calculator Start New Task :", _taskID);
+    auto randomValue = m_config->eccCrypto()->generateRandomScalar();
     // send to all partners
     auto message = m_config->psiMsgFactory()->createPSIMessage(
         uint32_t(EcdhMultiPSIMessageType::GENERATE_RANDOM_TO_PARTNER));
-    message->setData(std::vector{A});
-    message->setDataBatchCount(A.size());
+    message->setData(std::vector{randomValue});
+    message->setDataBatchCount(randomValue.size());
     message->setFrom(m_taskState->task()->selfParty()->id());
-    for (auto& partner : m_partnerParties)
+    for (auto const& partner : m_partnerParties)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_KV("PART1: Calculator generateRandomA to ", partner.first)
-                             << LOG_KV(" Random A ", *toHexString(A));
+        ECDH_CAL_LOG(INFO) << LOG_KV("PART1: Calculator generateRandomA to ", partner.first)
+                           << LOG_KV(" Random: ", *toHexString(randomValue));
         m_config->generateAndSendPPCMessage(
             partner.first, _taskID, message,
             [self = weak_from_this(), partner](bcos::Error::Ptr&& _error) {
                 if (!_error)
                 {
-                    ECDH_MULTI_LOG(INFO)
+                    ECDH_CAL_LOG(INFO)
                         << LOG_KV("PART1: Calculator generateRandomA success to ", partner.first);
                     return;
                 }
@@ -70,84 +70,90 @@ bcos::bytes EcdhMultiPSICalculator::generateRandomA(std::string _taskID)
             },
             0);
     }
-    return A;
+    return randomValue;
 }
 
 // PART2: Calculator -> Master H(X)*A
-void EcdhMultiPSICalculator::computeAndEncryptSet(std::string _taskID, bcos::bytes _randA)
+void EcdhMultiPSICalculator::blindData(std::string _taskID, bcos::bytes _randA)
 {
-    ECDH_MULTI_LOG(INFO) << LOG_KV(
-        "PART2:Calculator send to Master Use RandomA: ", *toHexString(_randA));
-    auto startT = utcSteadyTime();
-    m_originInputs = m_taskState->loadAllData();
-    if (!m_originInputs || m_originInputs->size() == 0)
-    {
-        BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty"));
-    }
-    auto inputSize = m_originInputs->size();
-    auto batchSize = m_config->dataBatchSize();
-    auto needSendTimes = 0;
-
-    // send counts
-    if (inputSize % batchSize == 0)
-    {
-        needSendTimes = inputSize / batchSize;
-    }
-    else
-    {
-        needSendTimes = inputSize / batchSize + 1;
-    }
-    ECDH_MULTI_LOG(INFO) << LOG_KV("PART2:Calculator Send Data Batch Times: ", needSendTimes);
     try
     {
-        auto hash = m_config->hash();
-        uint32_t readStart = 0, readCount = 0;
-        tbb::concurrent_map encryptedHashMap;
-        tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) {
-            for (auto i = range.begin(); i < range.end(); i++)
+        ECDH_CAL_LOG(INFO) << LOG_DESC("blindData") << LOG_KV("task", _taskID);
+        auto reader = m_taskState->reader();
+        uint64_t dataOffset = 0;
+        do
+        {
+            if (m_taskState->loadFinished())
             {
-                auto data = m_originInputs->getBytes(i);
-                auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size()));
-                auto point = m_config->eccCrypto()->hashToCurve(hashData);
-                auto hashSet = m_config->eccCrypto()->ecMultiply(point, _randA);
-                // encryptedHashMap.insert(std::make_pair(i, hashSet));
-                encryptedHashMap.emplace(std::make_pair(i, hashSet));
+                break;
             }
-        });
-
-        while (readCount < inputSize)
-        {
-            if (inputSize < batchSize)
+            DataBatch::Ptr dataBatch = nullptr;
+            uint32_t seq = 0;
             {
-                readCount = batchSize;
+                bcos::Guard l(m_mutex);
+                // Note: next is not thread-safe
+                dataBatch =
+                    m_taskState->reader()->next(m_taskState->readerParam(), DataSchema::Bytes);
+                if (!dataBatch)
+                {
+                    ECDH_CAL_LOG(INFO) << LOG_DESC("blindData return for all data loaded")
+                                       << LOG_KV("task", _taskID);
+                    m_taskState->setFinished(true);
+                    break;
+                }
+                // allocate seq
+                seq = m_taskState->allocateSeq();
+                if (m_taskState->sqlReader())
+                {
+                    m_taskState->setFinished(true);
+                }
             }
-            else if (readCount + batchSize < inputSize)
+            m_calculatorCache->appendPlainData(dataBatch);
+            // encrypt
+            auto startT = utcSteadyTime();
+            std::vector encryptedData(dataBatch->size());
+            tbb::parallel_for(
+                tbb::blocked_range(0U, dataBatch->size()), [&](auto const& range) {
+                    for (auto i = range.begin(); i < range.end(); i++)
+                    {
+                        auto const& data = dataBatch->get(i);
+                        auto hashData =
+                            m_config->hash()->hash(bcos::bytesConstRef(data.data(), data.size()));
+                        auto point = m_config->eccCrypto()->hashToCurve(hashData);
+                        encryptedData[i] = m_config->eccCrypto()->ecMultiply(point, _randA);
+                    }
+                });
+            ECDH_CAL_LOG(INFO) << LOG_DESC("blindData encrypt success")
+                               << LOG_KV("dataSize", encryptedData.size())
+                               << LOG_KV("task", m_taskState->task()->id()) << LOG_KV("seq", seq)
+                               << LOG_KV("timecost", (utcSteadyTime() - startT));
+            ECDH_CAL_LOG(INFO) << LOG_DESC("blindData: send cipher to the master")
+                               << LOG_KV("masterSize", m_masterParties.size())
+                               << LOG_KV("dataSize", encryptedData.size())
+                               << LOG_KV("task", m_taskState->task()->id());
+            auto message = m_config->psiMsgFactory()->createPSIMessage(
+                uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR));
+            message->constructData(encryptedData, dataOffset);
+            // release the encryptedData
+            std::vector().swap(encryptedData);
+            message->setFrom(m_taskState->task()->selfParty()->id());
+            if (reader->readFinished())
             {
-                readCount += batchSize;
+                message->setDataBatchCount(m_taskState->sendedDataBatchSize());
             }
             else
             {
-                readCount = inputSize;
+                // 0 means not finished
+                message->setDataBatchCount(0);
             }
-            std::map cHashMap;
-            ConcurrentSTLToCommon(encryptedHashMap, readStart, readCount, cHashMap);
-            ECDH_MULTI_LOG(INFO) << LOG_KV(
-                "PART2:Calculator compute the H(X)*A encryptedHashMap success size: ",
-                cHashMap.size());
-            for (auto& master : m_masterParties)
+            // send cipher
+            for (auto const& master : m_masterParties)
             {
-                auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t(
-                    EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_CALCULATOR));
-                message->setDataMap(std::move(cHashMap));
-                message->setFrom(m_taskState->task()->selfParty()->id());
-                message->setDataBatchCount(needSendTimes);
                 m_config->generateAndSendPPCMessage(
                     master.first, _taskID, message,
                     [self = weak_from_this(), master](bcos::Error::Ptr&& _error) {
                         if (!_error)
                         {
-                            ECDH_MULTI_LOG(INFO) << LOG_KV(
-                                "PART2:Send the EncryptSet success to Master: ", master.first);
                             return;
                         }
                         auto psi = self.lock();
@@ -156,21 +162,21 @@ void EcdhMultiPSICalculator::computeAndEncryptSet(std::string _taskID, bcos::byt
                             return;
                         }
                     },
-                    readCount);
+                    seq);
+                dataOffset += dataBatch->size();
             }
-            readStart += batchSize;
-        }
+        } while (!m_taskState->sqlReader());
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in computeAndEncryptSet:")
-                             << boost::diagnostic_information(e);
+        ECDH_CAL_LOG(WARNING) << LOG_DESC("Exception in blindData:")
+                              << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
 
-void EcdhMultiPSICalculator::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
+void EcdhMultiPSICalculator::initTask(ppc::protocol::Task::ConstPtr _task)
 {
     // Init all Roles from all Peers
     auto peerParties = _task->getAllPeerParties();
@@ -194,32 +200,25 @@ void EcdhMultiPSICalculator::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
 }
 
 // Part3: Calculator store Intersection_XY^b <- Master (response)
-void EcdhMultiPSICalculator::onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSICalculator::onReceiveIntersecCipher(PSIMessageInterface::Ptr _msg)
 {
-    auto encryptedMap = _msg->takeDataMap();
-    ECDH_MULTI_LOG(INFO) << LOG_KV(
-        "Part3: Calculator Receive Intersection_XY^b onHandlerIntersectEncryptSetToCalculator "
-        "Received Dataset size: ",
-        encryptedMap.size());
+    auto cipherData = _msg->takeData();
+    auto const& dataIndex = _msg->dataIndex();
+    ECDH_CAL_LOG(INFO) << LOG_DESC("onReceiveIntersecCipher") << printPSIMessage(_msg)
+                       << LOG_KV("dataSize", cipherData.size());
     try
     {
-        auto seq = _msg->seq();
-        auto needSendTimes = _msg->dataBatchCount();
-        bool finished = m_calculatorCipherDataCache->setCalculatorIntersectionCipherDataMap(
-            std::move(encryptedMap), seq, needSendTimes);
-        if (finished == false)
+        m_calculatorCache->addIntersectionCipher(
+            std::move(cipherData), dataIndex, _msg->seq(), _msg->dataBatchCount());
+        auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t(
+            EcdhMultiPSIMessageType::RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER));
+        message->setFrom(m_taskState->task()->selfParty()->id());
+        // try to finalize
+        auto ret = m_calculatorCache->tryToFinalize();
+        for (auto const& master : m_masterParties)
         {
-            return;
-        }
-        for (auto& master : m_masterParties)
-        {
-            ECDH_MULTI_LOG(INFO) << LOG_KV(
-                "Part3: onHandlerIntersectEncryptSetToCalculator Send the Response to Master: ",
-                master.first);
-            auto message =
-                m_config->psiMsgFactory()->createPSIMessage(uint32_t(EcdhMultiPSIMessageType::
-                        RETURN_ENCRYPTED_INTERSECTION_SET_FROM_CALCULATOR_TO_MASTER));
-            message->setFrom(m_taskState->task()->selfParty()->id());
+            ECDH_CAL_LOG(INFO) << LOG_DESC("onReceiveIntersecCipher: send response to the master")
+                               << LOG_KV("master", master.first) << printPSIMessage(_msg);
             m_config->generateAndSendPPCMessage(
                 master.first, m_taskID, message,
                 [self = weak_from_this()](bcos::Error::Ptr&& _error) {
@@ -235,195 +234,65 @@ void EcdhMultiPSICalculator::onHandlerIntersectEncryptSetToCalculator(PSIMessage
                 },
                 0);
         }
+        if (!ret)
+        {
+            return;
+        }
+        // return the rpc
+        m_taskState->setFinished(true);
+        m_taskState->onTaskFinished();
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetToCalculator:")
-                             << boost::diagnostic_information(e);
+        ECDH_CAL_LOG(WARNING) << LOG_DESC("Exception in onReceiveIntersecCipher:")
+                              << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
 // Part4 : Intersection_XY^b ∩ H(Z)^b^a
-void EcdhMultiPSICalculator::onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSICalculator::onReceiveMasterCipher(PSIMessageInterface::Ptr _msg)
 {
-    auto encryptedSet = _msg->takeData();
-    auto inputSize = encryptedSet.size();
-    ECDH_MULTI_LOG(INFO) << LOG_KV(
-        " Part4: Master onHandlerEncryptSetToCalculator Received H(Z)*B Dataset size: ", inputSize);
-    std::vector encryptedHashSet;
-    encryptedHashSet.reserve(inputSize);
-    encryptedHashSet.resize(inputSize);
     try
     {
-        tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) {
+        auto cipher = _msg->takeData();
+        ECDH_CAL_LOG(INFO) << LOG_DESC("onReceiveMasterCipher") << printPSIMessage(_msg)
+                           << LOG_KV("cipher", cipher.size());
+        std::vector encryptedCipher(cipher.size());
+        tbb::parallel_for(tbb::blocked_range(0U, cipher.size()), [&](auto const& range) {
             for (auto i = range.begin(); i < range.end(); i++)
             {
-                auto data = encryptedSet.at(i);
-                if (data.data())
-                {
-                    auto hashSet = m_config->eccCrypto()->ecMultiply(data, m_randomA);
-                    encryptedHashSet[i] = hashSet;
-                }
+                encryptedCipher[i] = m_config->eccCrypto()->ecMultiply(cipher.at(i), m_randomA);
             }
         });
+        // release the cipher
+        std::vector().swap(cipher);
 
-        auto needTimes = _msg->dataBatchCount();
         auto seq = _msg->seq();
-        bool finished = m_calculatorCipherDataCache->setCalculatorCipherData(
-            std::move(encryptedHashSet), seq, needTimes);
+        bool finished = m_calculatorCache->appendMasterCipher(
+            std::move(encryptedCipher), seq, _msg->dataBatchCount());
         if (finished == false)
         {
             return;
         }
-        m_calculatorCipherDataCache->tryToGetCipherDataIntersection();
-        auto cipherDataResult =
-            m_calculatorCipherDataCache->calculatorIntersectionCipherDataFinalMap();
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Part4: Master onHandlerEncryptSetToCalculator "
-            "Intersection_XY^b ∩ H(Z)^b^a Dataset "
-            "size: ",
-            cipherDataResult.size());
-
-        if (!m_originInputs)
-        {
-            m_originInputs = m_taskState->loadAllData();
-            if (!m_originInputs || m_originInputs->size() == 0)
-            {
-                BOOST_THROW_EXCEPTION(
-                    ECDHMULTIException() << bcos::errinfo_comment("data is empty"));
-            }
-        }
-
-        m_finalResults.clear();
-        if (cipherDataResult.size() > 0)
+        auto ret = m_calculatorCache->tryToFinalize();
+        if (!ret)
         {
-            for (auto& _res : cipherDataResult)
-            {
-                m_finalResults.push_back(m_originInputs->getBytes(_res.first));
-            }
+            return;
         }
-
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Part4: onHandlerEncryptSetToCalculator Store "
-            "Intersection_XY^b ∩ H(Z)^b^a Success "
-            "Dataset size: ",
-            cipherDataResult.size());
-        // sync Result or status to All peers
-        syncResultToAllPeers();
-        m_taskState->storePSIResult(m_config->dataResourceLoader(), m_finalResults);
-
         // return the rpc
         m_taskState->setFinished(true);
         m_taskState->onTaskFinished();
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerEncryptSetToCalculator:")
-                             << boost::diagnostic_information(e);
+        ECDH_CAL_LOG(WARNING) << LOG_DESC("Exception in onReceiveMasterCipher:")
+                              << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
 
-void EcdhMultiPSICalculator::syncResultToAllPeers()
-{
-    auto all_peers = m_taskState->task()->getAllPeerParties();
-    ECDH_MULTI_LOG(INFO) << LOG_KV("Calculator: Calculator isSyncedResult:", m_syncResult);
-    if (!m_syncResult)
-    {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator:No Need To SyncResultToAllPeers");
-        for (auto& _peer : all_peers)
-        {
-            auto message = m_config->psiMsgFactory()->createPSIMessage(
-                uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL));
-            message->setFrom(m_taskState->task()->selfParty()->id());
-            message->setVersion(-1);
-            m_config->generateAndSendPPCMessage(
-                _peer.first, m_taskID, message,
-                [self = weak_from_this(), _peer](bcos::Error::Ptr&& _error) {
-                    if (!_error)
-                    {
-                        ECDH_MULTI_LOG(INFO)
-                            << LOG_KV("Calculator:Calculator sync no Need Sync To Peer Success:",
-                                   _peer.first);
-                        return;
-                    }
-                    auto psi = self.lock();
-                    if (!psi)
-                    {
-                        return;
-                    }
-                },
-                0);
-        }
-        return;
-    }
-    auto batchSize = m_config->dataBatchSize();
-    auto inputSize = m_finalResults.size();
-    auto needSendTimes = 0;
-    // send counts
-    if (inputSize % batchSize == 0)
-    {
-        needSendTimes = inputSize / batchSize;
-    }
-    else
-    {
-        needSendTimes = inputSize / batchSize + 1;
-    }
-
-    ECDH_MULTI_LOG(INFO) << LOG_KV(
-        "Final:Calculator Send Final Intersection Batch Times: ", needSendTimes);
-    uint32_t readStart = 0, readCount = 0;
-    while (readCount < inputSize)
-    {
-        if (inputSize < batchSize)
-        {
-            readCount = batchSize;
-        }
-        else if (readCount + batchSize < inputSize)
-        {
-            readCount += batchSize;
-        }
-        else
-        {
-            readCount = inputSize;
-        }
-
-        std::vector chunk_final_result;
-        splitVector(m_finalResults, readStart, readCount, chunk_final_result);
-
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Final:Calculator Send Final Intersection Chunk Size: ", chunk_final_result.size());
-        for (auto& _peer : all_peers)
-        {
-            auto message = m_config->psiMsgFactory()->createPSIMessage(
-                uint32_t(EcdhMultiPSIMessageType::SYNC_FINAL_RESULT_TO_ALL));
-            message->setData(chunk_final_result);
-            message->setFrom(m_taskState->task()->selfParty()->id());
-            message->setVersion(0);
-            message->setDataBatchCount(needSendTimes);
-            m_config->generateAndSendPPCMessage(
-                _peer.first, m_taskID, message,
-                [self = weak_from_this(), _peer](bcos::Error::Ptr&& _error) {
-                    if (!_error)
-                    {
-                        ECDH_MULTI_LOG(INFO)
-                            << LOG_KV("Final:Calculator sync result To Peer Success:", _peer.first);
-                        return;
-                    }
-                    auto psi = self.lock();
-                    if (!psi)
-                    {
-                        return;
-                    }
-                },
-                readCount);
-        }
-        readStart += batchSize;
-    }
-}
-
 void EcdhMultiPSICalculator::onTaskError(std::string&& _error)
 {
     auto result = std::make_shared(m_taskState->task()->id());
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h
index d024b335..40250a17 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.h
@@ -15,25 +15,20 @@ class EcdhMultiPSICalculator : public std::enable_shared_from_thissetData(std::vector());
-        }
         MallocExtension::instance()->ReleaseFreeMemory();
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("the calculator destroyed") << LOG_KV("taskID", m_taskID);
+        ECDH_CAL_LOG(INFO) << LOG_DESC("the calculator destroyed") << LOG_KV("taskID", m_taskID);
     }
 
     virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task);
-    virtual void onHandlerIntersectEncryptSetToCalculator(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerEncryptSetToCalculator(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveIntersecCipher(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveMasterCipher(PSIMessageInterface::Ptr _msg);
 
     const std::string& taskID() const { return m_taskID; }
 
 protected:
     virtual bcos::bytes generateRandomA(std::string _taskID);
-    virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task);
-    virtual void computeAndEncryptSet(std::string _taskID, bcos::bytes _randA);
-    virtual void syncResultToAllPeers();
+    virtual void initTask(ppc::protocol::Task::ConstPtr _task);
+    virtual void blindData(std::string _taskID, bcos::bytes _randA);
     virtual void onTaskError(std::string&& _error);
     void ConcurrentSTLToCommon(
         tbb::concurrent_map _cMap, std::map& result)
@@ -65,30 +60,9 @@ class EcdhMultiPSICalculator : public std::enable_shared_from_this& _vectors, uint32_t _start, uint32_t _end,
-        std::vector& _outVecs)
-    {
-        uint32_t index = 0;
-        for (auto vec : _vectors)
-        {
-            if (index < _start)
-            {
-                index++;
-                continue;
-            }
-            else if (index >= _end)
-            {
-                break;
-            }
-            _outVecs.push_back(vec);
-            index++;
-        }
-    };
-
 
 private:
     bool m_syncResult{false};
-    ppc::io::DataBatch::Ptr m_originInputs;
     EcdhMultiPSIConfig::Ptr m_config;
     bcos::bytes m_randomA;
     std::string m_taskID;
@@ -97,6 +71,8 @@ class EcdhMultiPSICalculator : public std::enable_shared_from_this m_partnerParties;
     std::map m_masterParties;
     std::vector m_finalResults;
-    CalculatorCipherDataCache::Ptr m_calculatorCipherDataCache;
+    CalculatorCache::Ptr m_calculatorCache;
+
+    mutable bcos::Mutex m_mutex;
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index 877e1516..8da588d4 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -17,21 +17,30 @@ EcdhMultiPSIMaster::EcdhMultiPSIMaster(EcdhMultiPSIConfig::Ptr _config, TaskStat
     auto task = m_taskState->task();
     auto receivers = task->getReceiverLists();
     m_taskID = task->id();
-    m_masterCipherDataCache = std::make_shared();
-    m_final_counts[m_taskID] = 0;
+    m_masterCache = std::make_shared(m_taskState, m_config);
     m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
                                                     m_config->selfParty()) != receivers.end());
 }
 
 void EcdhMultiPSIMaster::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task)
 {
-    InitAsyncTask(_task);
-    ECDH_MULTI_LOG(INFO) << LOG_DESC("Master asyncStartRunTask") << printTaskInfo(_task);
+    ECDH_MASTER_LOG(INFO) << LOG_DESC("Master asyncStartRunTask") << printTaskInfo(_task);
+    initTask(_task);
     auto B = m_config->eccCrypto()->generateRandomScalar();
     m_randomB = std::make_shared(B);
+    auto self = weak_from_this();
+    m_config->threadPool()->enqueue([self]() {
+        auto master = self.lock();
+        if (!master)
+        {
+            return;
+        }
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("Master blindData") << LOG_KV("task", master->m_taskID);
+        master->blindData();
+    });
 }
 
-void EcdhMultiPSIMaster::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
+void EcdhMultiPSIMaster::initTask(ppc::protocol::Task::ConstPtr _task)
 {
     // Init all Roles from all Peers
     auto peerParties = _task->getAllPeerParties();
@@ -55,405 +64,168 @@ void EcdhMultiPSIMaster::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
 }
 
 // Part1-C,2-C: Master -> Calculator [H(X)*A ∩ H(Y)*A]*B
-void EcdhMultiPSIMaster::onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIMaster::onReceiveCalCipher(PSIMessageInterface::Ptr _msg)
 {
     try
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Part1-C: Master Receive H(X)*A ")
-                             << LOG_KV(" PSIMessageFace Size: ", _msg->takeDataMap().size())
-                             << LOG_KV(" PSIMessageFace SEQ: ", _msg->seq())
-                             << LOG_KV(" PSIMessageFace dataBatchCount: ", _msg->dataBatchCount());
-        WriteGuard lock(x_appendMasterCipherDataFromCalculator);
-        m_masterCipherDataCache->appendMasterCipherDataFromCalculator(
-            _msg->from(), _msg->takeDataMap(), _msg->seq(), _msg->dataBatchCount());
-        if (loadCipherDataFinished())
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("onReceiveCalCipher") << printPSIMessage(_msg);
+        auto&& data = _msg->takeData();
+        auto const& dataIndex = _msg->dataIndex();
+        m_masterCache->addCalculatorCipher(
+            _msg->from(), std::move(data), dataIndex, _msg->seq(), _msg->dataBatchCount());
+        auto ret = m_masterCache->tryToIntersection();
+        if (!ret)
         {
-            m_masterCipherDataCache->tryToGetCipherDataIntersection();
-            auto cipherMaps = m_masterCipherDataCache->masterFinalIntersectionCipherData();
-            ECDH_MULTI_LOG(INFO) << LOG_DESC("Part2-C:Master [H(X)*A ∩ H(Y)*A]")
-                                 << LOG_KV(
-                                        " Received Cipher Set Success Size: ", cipherMaps.size());
-            tbb::concurrent_map encryptedInterHashMap;
-            tbb::parallel_for_each(cipherMaps.begin(), cipherMaps.end(), [&](auto const& _pair) {
-                auto index = _pair.first;
-                auto value = _pair.second;
-                if (value.data())
-                {
-                    auto hashSet = m_config->eccCrypto()->ecMultiply(value, *m_randomB);
-                    // encryptedInterHashMap.insert(std::make_pair(index, hashSet));
-                    encryptedInterHashMap.emplace(std::make_pair(index, hashSet));
-                }
-            });
-
-            auto inputSize = encryptedInterHashMap.size();
-            auto batchSize = m_config->dataBatchSize();
-            auto needSendTimes = 0;
-            // send counts
-            if (inputSize % batchSize == 0)
-            {
-                needSendTimes = inputSize / batchSize;
-            }
-            else
-            {
-                needSendTimes = inputSize / batchSize + 1;
-            }
-            uint32_t readStart = 0, readCount = 0;
-            while (readStart < inputSize)
-            {
-                if (inputSize < batchSize)
-                {
-                    readCount = batchSize;
-                }
-                else if (readCount + batchSize < inputSize)
-                {
-                    readCount += batchSize;
-                }
-                else
-                {
-                    readCount = inputSize;
-                }
-                std::map cHashMap;
-                ConcurrentSTLToCommon(encryptedInterHashMap, readStart, readCount, cHashMap);
-
-                for (auto& calcultor : m_calculatorParties)
-                {
-                    ECDH_MULTI_LOG(INFO)
-                        << LOG_KV(
-                               "Part2-C:Master Send the [H(X)*A ∩ H(Y)*A]*B success to "
-                               "Calculator: ",
-                               calcultor.first)
-                        << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", cHashMap.size());
-                    auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t(
-                        EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR));
-                    message->setDataMap(std::move(cHashMap));
-                    message->setFrom(m_taskState->task()->selfParty()->id());
-                    message->setDataBatchCount(needSendTimes);
-                    m_config->generateAndSendPPCMessage(
-                        calcultor.first, m_taskID, message,
-                        [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                            if (!_error)
-                            {
-                                return;
-                            }
-                            auto psi = self.lock();
-                            if (!psi)
-                            {
-                                return;
-                            }
-                        },
-                        readCount);
-                }
-                readStart += batchSize;
-            }
+            return;
         }
+        m_masterCache->encryptIntersection(*m_randomB, m_calculatorParties);
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetFromCalculator:")
-                             << boost::diagnostic_information(e);
+        ECDH_MASTER_LOG(WARNING) << LOG_DESC("Exception in onReceiveCalCipher:")
+                                 << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
 // Part1-P,2-P: Partner -> Master [H(X)*A ∩ H(Y)*A]*B
-void EcdhMultiPSIMaster::onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIMaster::onReceiveCipherFromPartner(PSIMessageInterface::Ptr _msg)
 {
     try
     {
-        WriteGuard lock(x_appendMasterCipherDataFromPartner);
-        m_masterCipherDataCache->appendMasterCipherDataFromPartner(
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("onReceiveCipherFromPartner") << printPSIMessage(_msg);
+        m_masterCache->addPartnerCipher(
             _msg->from(), _msg->takeData(), _msg->seq(), _msg->dataBatchCount());
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Part1-P: Master Receive H(Y)*A")
-                             << LOG_KV(" PSIMessageFace Size: ", _msg->takeData().size())
-                             << LOG_KV(" PSIMessageFace SEQ: ", _msg->seq())
-                             << LOG_KV(" PSIMessageFace dataBatchCount: ", _msg->dataBatchCount());
-
-        if (loadCipherDataFinished())
+        auto ret = m_masterCache->tryToIntersection();
+        if (!ret)
         {
-            m_masterCipherDataCache->tryToGetCipherDataIntersection();
-            auto cipherMaps = m_masterCipherDataCache->masterFinalIntersectionCipherData();
-            ECDH_MULTI_LOG(INFO) << LOG_DESC("Part2-P:Master Receive [H(X)*A ∩ H(Y)*A]")
-                                 << LOG_KV(" Receive [H(X)*A ∩ H(Y)*A] Success Size: ",
-                                        cipherMaps.size());
-            tbb::concurrent_map encryptedInterHashMap;
-
-            tbb::parallel_for_each(cipherMaps.begin(), cipherMaps.end(), [&](auto const& _pair) {
-                auto index = _pair.first;
-                auto value = _pair.second;
-                if (value.data())
-                {
-                    auto hashSet = m_config->eccCrypto()->ecMultiply(value, *m_randomB);
-                    // encryptedInterHashMap.insert(std::make_pair(index, hashSet));
-                    encryptedInterHashMap.emplace(std::make_pair(index, hashSet));
-                }
-            });
-
-            auto inputSize = encryptedInterHashMap.size();
-            auto batchSize = m_config->dataBatchSize();
-            auto needSendTimes = 0;
-            // send counts
-            if (inputSize == 0)
-            {
-                needSendTimes = 1;
-                for (auto& calcultor : m_calculatorParties)
-                {
-                    std::map cHashMap;
-                    ECDH_MULTI_LOG(INFO)
-                        << LOG_KV(
-                               "Part2-P: Master Send the [H(X)*A ∩ H(Y)*A]*B success to "
-                               "Calculator: ",
-                               calcultor.first)
-                        << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", 0);
-                    auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t(
-                        EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR));
-                    message->setDataMap(std::move(cHashMap));
-                    message->setFrom(m_taskState->task()->selfParty()->id());
-                    message->setDataBatchCount(needSendTimes);
-                    m_config->generateAndSendPPCMessage(
-                        calcultor.first, m_taskID, message,
-                        [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                            if (!_error)
-                            {
-                                return;
-                            }
-                            auto psi = self.lock();
-                            if (!psi)
-                            {
-                                return;
-                            }
-                        },
-                        0);
-                }
-                return;
-            }
-            else if (inputSize % batchSize == 0)
-            {
-                needSendTimes = inputSize / batchSize;
-            }
-            else
-            {
-                needSendTimes = inputSize / batchSize + 1;
-            }
-            uint32_t readStart = 0, readCount = 0;
-            while (readCount < inputSize)
-            {
-                if (inputSize < batchSize)
-                {
-                    readCount = batchSize;
-                }
-                else if (readCount + batchSize < inputSize)
-                {
-                    readCount += batchSize;
-                }
-                else
-                {
-                    readCount = inputSize;
-                }
-                std::map cHashMap;
-                ConcurrentSTLToCommon(encryptedInterHashMap, readStart, readCount, cHashMap);
-
-                for (auto& calcultor : m_calculatorParties)
-                {
-                    ECDH_MULTI_LOG(INFO)
-                        << LOG_KV(
-                               "Part2-P: Master Send the [H(X)*A ∩ H(Y)*A]*B success to "
-                               "Calculator: ",
-                               calcultor.first)
-                        << LOG_KV("EncryptSet [H(X)*A ∩ H(Y)*A]*B Size: ", cHashMap.size());
-                    auto message = m_config->psiMsgFactory()->createPSIMessage(uint32_t(
-                        EcdhMultiPSIMessageType::SEND_ENCRYPTED_INTERSECTION_SET_TO_CALCULATOR));
-                    message->setDataMap(std::move(cHashMap));
-                    message->setFrom(m_taskState->task()->selfParty()->id());
-                    message->setDataBatchCount(needSendTimes);
-                    m_config->generateAndSendPPCMessage(
-                        calcultor.first, m_taskID, message,
-                        [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                            if (!_error)
-                            {
-                                return;
-                            }
-                            auto psi = self.lock();
-                            if (!psi)
-                            {
-                                return;
-                            }
-                        },
-                        readCount);
-                }
-                readStart += batchSize;
-            }
+            return;
         }
+        m_masterCache->encryptIntersection(*m_randomB, m_calculatorParties);
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in onHandlerIntersectEncryptSetFromPartner:")
-                             << boost::diagnostic_information(e);
+        ECDH_MASTER_LOG(WARNING) << LOG_DESC("Exception in onReceiveCipherFromPartner:")
+                                 << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
 // Part3: Master -> Calculator H(Z)*B
-void EcdhMultiPSIMaster::onHandlerEncryptIntersectionSetFromCalculatorToMaster(
-    PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIMaster::blindData()
 {
     try
     {
-        m_originInputs = m_taskState->loadAllData();
-        if (!m_originInputs || m_originInputs->size() == 0)
-        {
-            BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty"));
-        }
-        auto inputSize = m_originInputs->size();
-        auto batchSize = m_config->dataBatchSize();
-        auto needSendTimes = 0;
-        // send counts
-        if (inputSize % batchSize == 0)
-        {
-            needSendTimes = inputSize / batchSize;
-        }
-        else
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("blindData") << LOG_KV("taskID", m_taskID);
+        auto startT = utcSteadyTime();
+        auto reader = m_taskState->reader();
+        do
         {
-            needSendTimes = inputSize / batchSize + 1;
-        }
-        ECDH_MULTI_LOG(INFO) << LOG_KV("Part3: Master computeAndEncryptSet RandomB: ",
-                                    m_randomB->data())
-                             << LOG_KV(" Load All Data Size: ", inputSize);
-        auto hash = m_config->hash();
-        uint32_t readStart = 0, readCount = 0;
-        std::vector encryptedHashSet;
-        encryptedHashSet.reserve(inputSize);
-        encryptedHashSet.resize(inputSize);
-        tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) {
-            for (auto i = range.begin(); i < range.end(); i++)
+            if (m_taskState->loadFinished())
             {
-                auto data = m_originInputs->getBytes(i);
-                auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size()));
-                auto point = m_config->eccCrypto()->hashToCurve(hashData);
-                auto hashSet = m_config->eccCrypto()->ecMultiply(point, *m_randomB);
-                encryptedHashSet[i] = hashSet;
+                break;
             }
-        });
-        while (readCount < inputSize)
-        {
-            if (inputSize < batchSize)
+            DataBatch::Ptr dataBatch = nullptr;
+            uint32_t seq = 0;
             {
-                readCount = batchSize;
+                bcos::Guard l(m_mutex);
+                // Note: next is not thread-safe
+                dataBatch =
+                    m_taskState->reader()->next(m_taskState->readerParam(), DataSchema::Bytes);
+                if (!dataBatch)
+                {
+                    ECDH_CAL_LOG(INFO) << LOG_DESC("blindData return for all data loaded")
+                                       << LOG_KV("task", m_taskState->task()->id());
+                    m_taskState->setFinished(true);
+                    break;
+                }
+                // allocate seq
+                seq = m_taskState->allocateSeq();
+                if (m_taskState->sqlReader())
+                {
+                    m_taskState->setFinished(true);
+                }
             }
-            else if (readCount + batchSize < inputSize)
+            auto startT = utcSteadyTime();
+            ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data")
+                                  << LOG_KV("dataSize", dataBatch->size());
+            std::vector cipher(dataBatch->size());
+            tbb::parallel_for(
+                tbb::blocked_range(0U, dataBatch->size()), [&](auto const& range) {
+                    for (auto i = range.begin(); i < range.end(); i++)
+                    {
+                        auto const& data = dataBatch->get(i);
+                        auto hashData =
+                            m_config->hash()->hash(bcos::bytesConstRef(data.data(), data.size()));
+                        auto point = m_config->eccCrypto()->hashToCurve(hashData);
+                        cipher[i] = m_config->eccCrypto()->ecMultiply(point, *m_randomB);
+                    }
+                });
+            // can release databatch after encrypted
+            dataBatch->release();
+            ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data success")
+                                  << LOG_KV("dataSize", cipher.size()) << LOG_KV("task", m_taskID)
+                                  << LOG_KV("seq", seq)
+                                  << LOG_KV("timecost", (utcSteadyTime() - startT));
+
+            ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: send encrypted data to all calculator")
+                                  << LOG_KV("task", m_taskID)
+                                  << LOG_KV("calculators", m_calculatorParties.size());
+            auto message = m_config->psiMsgFactory()->createPSIMessage(
+                uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR));
+            message->setData(cipher);
+            if (reader->readFinished())
             {
-                readCount += batchSize;
+                message->setDataBatchCount(m_taskState->sendedDataBatchSize());
             }
             else
             {
-                readCount = inputSize;
+                // 0 means not finished
+                message->setDataBatchCount(0);
             }
-
-            std::vector encryptedHashSetSplit;
-            splitVector(encryptedHashSet, readStart, readCount, encryptedHashSetSplit);
-
-            ECDH_MULTI_LOG(INFO)
-                << LOG_KV("Part3: Master compute the H(Z)*B encryptedHashSet success size: ",
-                       encryptedHashSet.size())
-                << LOG_KV("Part3: Master compute the H(Z)*B encryptedHashSetSplit success size: ",
-                       encryptedHashSetSplit.size());
-
-            for (auto& calcultor : m_calculatorParties)
+            message->setFrom(m_taskState->task()->selfParty()->id());
+            for (auto const& calcultor : m_calculatorParties)
             {
-                ECDH_MULTI_LOG(INFO)
-                    << LOG_KV(
-                           " Send the EncryptSet H(Z)*B success to Calculator: ", calcultor.first)
-                    << LOG_KV("EncryptSet Sample: ", encryptedHashSet[0].data());
-                auto message = m_config->psiMsgFactory()->createPSIMessage(
-                    uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR));
-                message->setData(encryptedHashSetSplit);
-                message->setDataBatchCount(needSendTimes);
-                message->setFrom(m_taskState->task()->selfParty()->id());
+                // TODO: handle the send failed case
                 m_config->generateAndSendPPCMessage(
                     calcultor.first, m_taskID, message,
-                    [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                        if (!_error)
-                        {
-                            return;
-                        }
-                        auto psi = self.lock();
-                        if (!psi)
+                    [](bcos::Error::Ptr&& _error) {
+                        if (_error)
                         {
                             return;
                         }
                     },
-                    readCount);
+                    seq);
             }
-            readStart += batchSize;
-        }
+        } while (!m_taskState->sqlReader());
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO)
-            << LOG_DESC("Exception in onHandlerEncryptIntersectionSetFromCalculatorToMaster:")
-            << boost::diagnostic_information(e);
+        ECDH_MASTER_LOG(WARNING) << LOG_DESC("blindData exception")
+                                 << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
-void EcdhMultiPSIMaster::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIMaster::onReceivePSIResult(PSIMessageInterface::Ptr _msg)
 {
-    ECDH_MULTI_LOG(INFO)
-        << LOG_KV("Final: Master Get SyncFinalResultToAllPeer From Calculator : ",
-               _msg->takeData().size())
-        << LOG_KV("Final: Master Get SyncFinalResultToAllPeer From Calculator count: ",
-               m_final_counts[m_taskID])
-        << LOG_KV("Final: Master isSyncedResult:", m_syncResult);
-
+    ECDH_MASTER_LOG(INFO) << LOG_DESC("onReceivePSIResult") << printPSIMessage(_msg);
     if (m_syncResult)
     {
-        auto needTimes = _msg->dataBatchCount();
-        auto res = _msg->takeData();
-
-        trimVector(res);
-        bcos::WriteGuard l(x_final_count);
-        m_final_vectors.insert(m_final_vectors.end(), res.begin(), res.end());
-        m_final_counts[m_taskID]++;
-        if (m_final_counts[m_taskID] < needTimes)
-        {
-            return;
-        }
-
-        m_taskState->storePSIResult(m_config->dataResourceLoader(), m_final_vectors);
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Final: Master onHandlerSyncFinalResultToAllPeer Store Intersection_XY^b ∩ H(Z)^b^a "
-            "Success"
-            "Dataset size: ",
-            m_final_vectors.size());
+        m_taskState->storePSIResult(m_config->dataResourceLoader(), _msg->takeData());
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("onReceivePSIResult: store psi result success")
+                              << printPSIMessage(_msg);
     }
     else
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Master:No Need To ReceiveResultFromCalculator");
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("Master:No Need To store the psi result")
+                              << printPSIMessage(_msg);
     }
 
     m_taskState->setFinished(true);
     m_taskState->onTaskFinished();
 }
 
-bool EcdhMultiPSIMaster::loadCipherDataFinished()
-{
-    auto allPeerParties = m_taskState->task()->getAllPeerParties();
-    auto finishedPeers = m_masterCipherDataCache->masterTaskPeersFinishedList();
-    if (allPeerParties.size() == finishedPeers.size())
-    {
-        for (auto& _peer : allPeerParties)
-        {
-            if (!finishedPeers.contains(_peer.first))
-            {
-                return false;
-            }
-        }
-        return true;
-    }
-    else
-    {
-        return false;
-    }
-}
 
 void EcdhMultiPSIMaster::onTaskError(std::string&& _error)
 {
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h
index 30b57041..2e684cca 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.h
@@ -13,91 +13,23 @@ class EcdhMultiPSIMaster : public std::enable_shared_from_thissetData(std::vector());
-        }
-        std::vector().swap(m_final_vectors);
         MallocExtension::instance()->ReleaseFreeMemory();
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("the master destroyed") << LOG_KV("taskID", m_taskID);
+        ECDH_MASTER_LOG(INFO) << LOG_DESC("the master destroyed") << LOG_KV("taskID", m_taskID);
     }
     virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task);
-    virtual void onHandlerIntersectEncryptSetFromCalculator(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerIntersectEncryptSetFromPartner(PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerEncryptIntersectionSetFromCalculatorToMaster(
-        PSIMessageInterface::Ptr _msg);
-    virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveCalCipher(PSIMessageInterface::Ptr _msg);
+    virtual void onReceiveCipherFromPartner(PSIMessageInterface::Ptr _msg);
+    virtual void blindData();
+    virtual void onReceivePSIResult(PSIMessageInterface::Ptr _msg);
 
     const std::string& taskID() const { return m_taskID; }
 
 protected:
-    virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task);
-    virtual bool loadCipherDataFinished();
+    virtual void initTask(ppc::protocol::Task::ConstPtr _task);
     virtual void onTaskError(std::string&& _error);
-    void ConcurrentSTLToCommon(
-        tbb::concurrent_map _cMap, std::map& result)
-    {
-        ConcurrentSTLToCommon(_cMap, 0, _cMap.size(), result);
-    };
-
-    void ConcurrentSTLToCommon(tbb::concurrent_map _cMap,
-        uint32_t _startIndex, uint32_t _endIndex, std::map& result)
-    {
-        std::mutex mutex;
-        tbb::concurrent_map::const_iterator iter;
-        uint32_t index = 0;
-        for (iter = _cMap.begin(); iter != _cMap.end(); iter++)
-        {
-            std::lock_guard lock(mutex);
-            if (index < _startIndex)
-            {
-                index++;
-                continue;
-            }
-            else if (index >= _endIndex)
-            {
-                break;
-            }
-            // result.insert(std::make_pair(iter->first, iter->second));
-            result.emplace(std::make_pair(iter->first, iter->second));
-            index++;
-        }
-    };
-
-    virtual void trimVector(std::vector& _vectors)
-    {
-        for (auto it = _vectors.begin(); it != _vectors.end();)
-        {
-            if (!it->data())
-                it = _vectors.erase(it);
-            else
-                ++it;
-        }
-    }
-
-    virtual void splitVector(std::vector& _vectors, uint32_t _start, uint32_t _end,
-        std::vector& _outVecs)
-    {
-        uint32_t index = 0;
-        for (auto vec : _vectors)
-        {
-            if (index < _start)
-            {
-                index++;
-                continue;
-            }
-            else if (index >= _end)
-            {
-                break;
-            }
-            _outVecs.push_back(vec);
-            index++;
-        }
-    };
 
 private:
     bool m_syncResult{false};
-    ppc::io::DataBatch::Ptr m_originInputs;
     std::map m_calculatorParties;
     std::map m_partnerParties;
     std::map m_masterParties;
@@ -105,13 +37,8 @@ class EcdhMultiPSIMaster : public std::enable_shared_from_this m_final_vectors;
-    std::map m_final_counts;
-    mutable bcos::SharedMutex x_final_count;
+    mutable bcos::Mutex m_mutex;
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
index 8c4845f8..cab0d0c1 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
@@ -15,12 +15,11 @@ EcdhMultiPSIPartner::EcdhMultiPSIPartner(EcdhMultiPSIConfig::Ptr _config, TaskSt
     auto task = m_taskState->task();
     auto receivers = task->getReceiverLists();
     m_taskID = task->id();
-    m_final_counts[m_taskID] = 0;
     m_syncResult = (task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
                                                     m_config->selfParty()) != receivers.end());
 }
 
-void EcdhMultiPSIPartner::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
+void EcdhMultiPSIPartner::initTask(ppc::protocol::Task::ConstPtr _task)
 {
     // Init all Roles from all Peers
     auto peerParties = _task->getAllPeerParties();
@@ -44,90 +43,82 @@ void EcdhMultiPSIPartner::InitAsyncTask(ppc::protocol::Task::ConstPtr _task)
 }
 
 // PART1: Partner -> Master H(Y)*A
-void EcdhMultiPSIPartner::oncomputeAndEncryptSet(bcos::bytesPointer _randA)
+void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
 {
     try
     {
-        ECDH_MULTI_LOG(INFO) << LOG_KV("Part1:Partner Receive RandomA: ", _randA->data());
-        auto originInputs = m_taskState->loadAllData();
-        if (!originInputs || originInputs->size() == 0)
+        ECDH_PARTNER_LOG(INFO) << LOG_DESC("onReceiveRandomA and blindData")
+                               << printTaskInfo(m_taskState->task());
+        auto reader = m_taskState->reader();
+        uint64_t dataOffset = 0;
+        do
         {
-            BOOST_THROW_EXCEPTION(ECDHMULTIException() << bcos::errinfo_comment("data is empty"));
-        }
-
-        auto inputSize = originInputs->size();
-        auto batchSize = m_config->dataBatchSize();
-        auto needSendTimes = 0;
-
-        // send counts
-        if (inputSize % batchSize == 0)
-        {
-            needSendTimes = inputSize / batchSize;
-        }
-        else
-        {
-            needSendTimes = inputSize / batchSize + 1;
-        }
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Part1: Partner load the resource success size: ", inputSize);
-        auto hash = m_config->hash();
-        std::vector encryptedHashSet;
-        encryptedHashSet.reserve(inputSize);
-        encryptedHashSet.resize(inputSize);
-        tbb::parallel_for(tbb::blocked_range(0U, inputSize), [&](auto const& range) {
-            for (auto i = range.begin(); i < range.end(); i++)
+            if (m_taskState->loadFinished())
             {
-                auto data = originInputs->getBytes(i);
-                auto hashData = hash->hash(bcos::bytesConstRef(data.data(), data.size()));
-                auto point = m_config->eccCrypto()->hashToCurve(hashData);
-                auto hashSet = m_config->eccCrypto()->ecMultiply(point, *_randA);
-                encryptedHashSet[i] = hashSet;
+                break;
             }
-        });
-        uint32_t readStart = 0, readCount = 0;
-
-        while (readCount < inputSize)
-        {
-            if (inputSize < batchSize)
+            DataBatch::Ptr dataBatch = nullptr;
+            uint32_t seq = 0;
             {
-                readCount = batchSize;
+                bcos::Guard l(m_mutex);
+                // Note: next is not thread-safe
+                dataBatch =
+                    m_taskState->reader()->next(m_taskState->readerParam(), DataSchema::Bytes);
+                if (!dataBatch)
+                {
+                    ECDH_PARTNER_LOG(INFO)
+                        << LOG_DESC("blindData: encode partner cipher return for all data loaded")
+                        << LOG_KV("task", m_taskID);
+                    m_taskState->setFinished(true);
+                    break;
+                }
+                // allocate seq
+                seq = m_taskState->allocateSeq();
+                if (m_taskState->sqlReader())
+                {
+                    m_taskState->setFinished(true);
+                }
             }
-            else if (readCount + batchSize < inputSize)
+            ECDH_PARTNER_LOG(INFO) << LOG_DESC("blindData: encode parterner cipher")
+                                   << LOG_KV("size", dataBatch->size()) << LOG_KV("seq", seq)
+                                   << LOG_KV("task", m_taskState->task()->id());
+            auto startT = utcSteadyTime();
+            std::vector cipherData(dataBatch->size());
+            tbb::parallel_for(
+                tbb::blocked_range(0U, dataBatch->size()), [&](auto const& range) {
+                    for (auto i = range.begin(); i < range.end(); i++)
+                    {
+                        auto const& data = dataBatch->get(i);
+                        auto hashData =
+                            m_config->hash()->hash(bcos::bytesConstRef(data.data(), data.size()));
+                        auto point = m_config->eccCrypto()->hashToCurve(hashData);
+                        cipherData[i] = m_config->eccCrypto()->ecMultiply(point, *_randA);
+                    }
+                });
+            ECDH_PARTNER_LOG(INFO) << LOG_DESC("blindData: encode parterner cipher success")
+                                   << LOG_KV("size", dataBatch->size())
+                                   << LOG_KV("timecost", (utcSteadyTime() - startT))
+                                   << printTaskInfo(m_taskState->task());
+
+            ECDH_PARTNER_LOG(INFO)
+                << LOG_DESC("blindData: send cipher data to master")
+                << LOG_KV("size", dataBatch->size()) << printTaskInfo(m_taskState->task());
+            auto message = m_config->psiMsgFactory()->createPSIMessage(
+                uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER));
+            message->setData(std::move(cipherData));
+            message->setFrom(m_taskState->task()->selfParty()->id());
+            if (reader->readFinished())
             {
-                readCount += batchSize;
+                message->setDataBatchCount(m_taskState->sendedDataBatchSize());
             }
             else
             {
-                readCount = inputSize;
+                // 0 means not finished
+                message->setDataBatchCount(0);
             }
-
-            std::vector encryptedHashSetSplit;
-            splitVector(encryptedHashSet, readStart, readCount, encryptedHashSetSplit);
-
-            ECDH_MULTI_LOG(INFO)
-                << LOG_KV("Part1: Partner compute the EncryptSet success encryptedHashSet size: ",
-                       encryptedHashSet.size())
-                << LOG_KV(
-                       "Part1: Partner compute the EncryptSet success encryptedHashSetSplit "
-                       "readStart: ",
-                       readStart)
-                << LOG_KV(
-                       "Part1: Partner compute the EncryptSet success encryptedHashSetSplit "
-                       "readCount: ",
-                       readCount)
-                << LOG_KV(
-                       "Part1: Partner compute the EncryptSet success encryptedHashSetSplit "
-                       "size: ",
-                       encryptedHashSetSplit.size());
-
             // generate and send encryptedHashSet
-            for (auto& master : m_masterParties)
+            for (auto const& master : m_masterParties)
             {
-                auto message = m_config->psiMsgFactory()->createPSIMessage(
-                    uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER));
-                message->setData(encryptedHashSetSplit);
-                message->setFrom(m_taskState->task()->selfParty()->id());
-                message->setDataBatchCount(needSendTimes);
                 m_config->generateAndSendPPCMessage(
                     master.first, m_taskState->task()->id(), message,
                     [self = weak_from_this()](bcos::Error::Ptr&& _error) {
@@ -141,52 +132,36 @@ void EcdhMultiPSIPartner::oncomputeAndEncryptSet(bcos::bytesPointer _randA)
                             return;
                         }
                     },
-                    readCount);
+                    seq);
             }
-            readStart += batchSize;
-        }
+            ECDH_PARTNER_LOG(INFO) << LOG_DESC("blindData: send cipher data to master success")
+                                   << LOG_KV("size", dataBatch->size())
+                                   << printTaskInfo(m_taskState->task()) << LOG_KV("seq", seq);
+            dataBatch->release();
+        } while (!m_taskState->sqlReader());
     }
     catch (std::exception& e)
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Exception in oncomputeAndEncryptSet:")
-                             << boost::diagnostic_information(e);
+        ECDH_PARTNER_LOG(WARNING) << LOG_DESC("onReceiveRandomA exception")
+                                  << boost::diagnostic_information(e);
         onTaskError(boost::diagnostic_information(e));
     }
 }
 
-void EcdhMultiPSIPartner::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg)
+void EcdhMultiPSIPartner::onReceivePSIResult(PSIMessageInterface::Ptr _msg)
 {
+    ECDH_PARTNER_LOG(INFO) << LOG_DESC("onReceivePSIResult") << printPSIMessage(_msg);
     if (m_syncResult)
     {
-        ECDH_MULTI_LOG(INFO)
-            << LOG_KV("Final: Partner Get SyncFinalResultToAllPeer From Calculator : ",
-                   _msg->takeData().size())
-            << LOG_KV("Final: Partner Get SyncFinalResultToAllPeer From Calculator count: ",
-                   m_final_counts[m_taskID])
-            << LOG_KV("Final: Parter isSyncedResult:", m_syncResult);
-        auto needTimes = _msg->dataBatchCount();
-        auto res = _msg->takeData();
-        trimVector(res);
-        bcos::WriteGuard l(x_final_count);
-        m_final_vectors.insert(m_final_vectors.end(), res.begin(), res.end());
-        m_final_counts[m_taskID]++;
-        if (m_final_counts[m_taskID] < needTimes)
-        {
-            return;
-        }
-
-        m_taskState->storePSIResult(m_config->dataResourceLoader(), m_final_vectors);
-        ECDH_MULTI_LOG(INFO) << LOG_KV(
-            "Final: Partner onHandlerSyncFinalResultToAllPeer Store Intersection_XY^b ∩ H(Z)^b^a "
-            "Success"
-            "Dataset size: ",
-            m_final_vectors.size());
+        m_taskState->storePSIResult(m_config->dataResourceLoader(), _msg->takeData());
+        ECDH_PARTNER_LOG(INFO) << LOG_DESC("onReceivePSIResult: store psi result success")
+                               << printPSIMessage(_msg);
     }
     else
     {
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner:No Need To ReceiveResultFromCalculator");
+        ECDH_PARTNER_LOG(INFO) << LOG_DESC("Master:No Need To store the psi result")
+                               << printPSIMessage(_msg);
     }
-
     m_taskState->setFinished(true);
     m_taskState->onTaskFinished();
 }
@@ -194,9 +169,9 @@ void EcdhMultiPSIPartner::onHandlerSyncFinalResultToAllPeer(PSIMessageInterface:
 
 void EcdhMultiPSIPartner::asyncStartRunTask(ppc::protocol::Task::ConstPtr _task)
 {
-    InitAsyncTask(_task);
-    ECDH_MULTI_LOG(INFO) << LOG_DESC("Partner asyncStartRunTask as partner")
-                         << printTaskInfo(_task);
+    initTask(_task);
+    ECDH_PARTNER_LOG(INFO) << LOG_DESC("Partner asyncStartRunTask as partner")
+                           << printTaskInfo(_task);
 }
 
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h
index f3e5f4a8..b84fef03 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.h
@@ -14,52 +14,20 @@ class EcdhMultiPSIPartner : public std::enable_shared_from_this().swap(m_final_vectors);
         MallocExtension::instance()->ReleaseFreeMemory();
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("the partner destroyed") << LOG_KV("taskID", m_taskID);
+        ECDH_PARTNER_LOG(INFO) << LOG_DESC("the partner destroyed") << LOG_KV("taskID", m_taskID);
     }
 
-    virtual void oncomputeAndEncryptSet(bcos::bytesPointer _randA);
+    virtual void onReceiveRandomA(bcos::bytesPointer _randA);
     virtual void asyncStartRunTask(ppc::protocol::Task::ConstPtr _task);
-    virtual void onHandlerSyncFinalResultToAllPeer(PSIMessageInterface::Ptr _msg);
+    virtual void onReceivePSIResult(PSIMessageInterface::Ptr _msg);
 
     const std::string& taskID() const { return m_taskID; }
 
 protected:
-    virtual void InitAsyncTask(ppc::protocol::Task::ConstPtr _task);
+    virtual void initTask(ppc::protocol::Task::ConstPtr _task);
     virtual void onTaskError(std::string&& _error);
 
-    virtual void trimVector(std::vector& _vectors)
-    {
-        for (auto it = _vectors.begin(); it != _vectors.end();)
-        {
-            if (!it->data())
-                it = _vectors.erase(it);
-            else
-                ++it;
-        }
-    }
-
-    virtual void splitVector(std::vector& _vectors, uint32_t _start, uint32_t _end,
-        std::vector& _outVecs)
-    {
-        uint32_t index = 0;
-        for (auto vec : _vectors)
-        {
-            if (index < _start)
-            {
-                index++;
-                continue;
-            }
-            else if (index >= _end)
-            {
-                break;
-            }
-            _outVecs.push_back(vec);
-            index++;
-        }
-    };
-
 private:
     bool m_syncResult{false};
     EcdhMultiPSIConfig::Ptr m_config;
@@ -69,8 +37,6 @@ class EcdhMultiPSIPartner : public std::enable_shared_from_this m_partnerParties;
     std::map m_masterParties;
 
-    mutable bcos::SharedMutex x_final_count;
-    std::vector m_final_vectors;
-    std::map m_final_counts;
+    mutable bcos::Mutex m_mutex;
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
index 63d30086..173b326f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
@@ -26,4 +26,4 @@ file(GLOB_RECURSE SRCS *.cpp)
 add_library(${PSI_FRAMEWORK_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${PSI_FRAMEWORK_TARGET} PUBLIC
     $)
-target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET})
+target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET} TCMalloc)
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp
index 6010656e..c7dbc3e1 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/PSIFramework.cpp
@@ -19,6 +19,7 @@
  */
 #include "PSIFramework.h"
 #include "../Common.h"
+#include "ppc-framework/protocol/Constant.h"
 #include "ppc-framework/protocol/GlobalConfig.h"
 
 using namespace ppc::task;
@@ -154,6 +155,14 @@ void PSIFramework::onReceiveMessage(PPCMessageFace::Ptr _msg)
         m_msgQueue->push(psiMsg);
         PSI_FRAMEWORK_LOG(TRACE) << LOG_DESC("onReceiveMessage") << printPSIMessage(psiMsg)
                                  << LOG_KV("uuid", _msg->uuid());
+        // release the large payload immediately
+        if (payLoad && payLoad->size() >= ppc::protocol::LARGE_MSG_THRESHOLD)
+        {
+            PSI_FRAMEWORK_LOG(INFO)
+                << LOG_DESC("Release large message payload") << LOG_KV("size", payLoad->size());
+            _msg->releasePayload();
+            MallocExtension::instance()->ReleaseFreeMemory();
+        }
         // notify to handle the message
         m_signalled.notify_all();
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
index f6184cde..1de20188 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
@@ -48,7 +48,10 @@ class TaskState : public std::enable_shared_from_this
         m_taskStartTime = bcos::utcSteadyTime();
     }
 
-    virtual ~TaskState() {}
+    virtual ~TaskState()
+    {
+        PSI_LOG(INFO) << LOG_DESC("** TaskState Destructor") << LOG_KV("task", m_task->id());
+    }
 
     ppc::task::TaskResponseCallback const& callback() { return m_callback; }
     ppc::task::TaskResponseCallback takeCallback() { return std::move(m_callback); }
@@ -128,6 +131,7 @@ class TaskState : public std::enable_shared_from_this
 
     int32_t allocateSeq()
     {
+        bcos::RecursiveGuard l(m_mutex);
         m_currentSeq.store(m_currentSeq.load() + 1);
         {
             bcos::WriteGuard l(x_seqList);
@@ -202,10 +206,11 @@ class TaskState : public std::enable_shared_from_this
         {
             return;
         }
-        PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("task", m_task->id())
+        PSI_LOG(INFO) << LOG_DESC("* onTaskFinished") << LOG_KV("task", m_task->id())
                       << LOG_KV("success", m_successCount) << LOG_KV("failed", m_failedCount)
                       << LOG_KV("loadFinished", m_finished.load())
-                      << LOG_KV("callback", m_callback ? "withCallback" : "emptyCallback");
+                      << LOG_KV("callback", m_callback ? "withCallback" : "emptyCallback")
+                      << LOG_KV("taskTimecost", taskPendingTime());
         auto result = std::make_shared(m_task->id());
         try
         {
@@ -240,7 +245,8 @@ class TaskState : public std::enable_shared_from_this
         }
         catch (std::exception const& e)
         {
-            PSI_LOG(WARNING) << LOG_DESC("onTaskFinished exception")
+            PSI_LOG(WARNING) << LOG_DESC("* onTaskFinished exception")
+                             << LOG_KV("taskTimeCost", taskPendingTime())
                              << LOG_KV("msg", boost::diagnostic_information(e));
             auto error = std::make_shared(-1, boost::diagnostic_information(e));
             result->setError(std::move(error));
@@ -361,6 +367,10 @@ class TaskState : public std::enable_shared_from_this
         dataBatch->setData(_data);
         m_writer->writeLine(dataBatch, ppc::io::DataSchema::Bytes);
         m_writer->flush();
+        PSI_LOG(INFO) << LOG_DESC("**** storePSIResult success ****")
+                      << LOG_KV("* task", m_task->id())
+                      << LOG_KV("* IntersectionCount", _data.size())
+                      << LOG_KV("* TaskTimecost", taskPendingTime());
     }
 
     std::function takeFinalizeHandler() { return std::move(m_finalizeHandler); }
@@ -377,6 +387,20 @@ class TaskState : public std::enable_shared_from_this
         {
             return;
         }
+        // should been called even when exception
+        if (m_finalizeHandler)
+        {
+            try
+            {
+                m_finalizeHandler();
+            }
+            catch (std::exception const& e)
+            {
+                PSI_LOG(WARNING) << LOG_DESC("finalize task exception")
+                                 << LOG_KV("taskID", m_task->id())
+                                 << LOG_KV("msg", boost::diagnostic_information(e));
+            }
+        }
         auto taskResult = std::make_shared(m_task->id());
         auto msg = "Task " + m_task->id() + " exception, error : " + _errorMsg;
         auto error = std::make_shared(-1, msg);
@@ -411,6 +435,8 @@ class TaskState : public std::enable_shared_from_this
 
     uint64_t taskPendingTime() { return (bcos::utcSteadyTime() - m_taskStartTime); }
 
+    uint32_t sendedDataBatchSize() const { return m_seqList.size(); }
+
 protected:
     ppc::protocol::Task::ConstPtr m_task;
     ppc::task::TaskResponseCallback m_callback;
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h
index 3632c74f..1f779927 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/interfaces/PSIMessageInterface.h
@@ -44,19 +44,22 @@ class PSIMessageInterface
     virtual int32_t version() const = 0;
     // get all the data
     virtual std::vector takeData() = 0;
-    virtual void setDataMap(std::map&& _map) = 0;
-    virtual std::map takeDataMap() = 0;
+    virtual void constructData(std::vector const& data, uint64_t startIndex) = 0;
+    virtual std::vector const& dataIndex() const = 0;
     // Note: can't call this after takeData
     virtual uint64_t dataSize() const = 0;
     // get the data item
     // Note: must ensure the object will not released before access the function
     virtual bcos::bytesConstRef getData(uint64_t _index) = 0;
+    virtual uint64_t getDataCount() const = 0;
 
     virtual void setPartyID(std::string const& _partyID) = 0;
     virtual void setResourceID(std::string const& _resourceID) = 0;
     virtual void setVersion(int32_t _version) = 0;
     virtual void setData(std::vector const& _data) = 0;
     virtual void setData(std::vector&& _data) = 0;
+    virtual void setDataPair(uint64_t pos, uint64_t dataIndex, bcos::bytes const& data) = 0;
+    virtual void resizeData(uint64_t size) = 0;
     // copy [_offset, _offset + _len) into the data-field
     virtual void setData(
         std::vector const& _data, uint64_t _offset, uint64_t _len) = 0;
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars
index 13645e10..a1f459d5 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSI.tars
@@ -13,7 +13,7 @@ module ppctars{
         3 optional string partyID;
         4 optional string resourceID;
         5 optional vector> data;
-        6 optional map> dataMap;
+        6 optional vector dataIndex;
         7 optional int errorCode;
         8 optional string errorMessage;
         9 optional vector taskList;
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp
index 376a9f91..336bafdf 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.cpp
@@ -19,10 +19,20 @@
  */
 #include "PSIMessage.h"
 #include "wedpr-protocol/tars/Common.h"
+#include 
 
 using namespace ppc::psi;
 using namespace bcos;
 
+// destructor the psi message
+PSIMessage::~PSIMessage()
+{
+    m_inner()->data.clear();
+    m_inner()->dataIndex.clear();
+    std::vector>().swap(m_inner()->data);
+    std::vector().swap(m_inner()->dataIndex);
+    MallocExtension::instance()->ReleaseFreeMemory();
+}
 // encode the PSIMessage
 bytesPointer PSIMessage::encode() const
 {
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h
index 8cdeff6d..ed34d6c9 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/protocol/PSIMessage.h
@@ -36,7 +36,7 @@ class PSIMessage : virtual public PSIMessageInterface
     explicit PSIMessage(bcos::bytesConstRef _data) : PSIMessage() { decode(_data); }
     explicit PSIMessage(std::function _inner) : m_inner(std::move(_inner))
     {}
-    ~PSIMessage() {}
+    ~PSIMessage();
 
     // the packet-type
     uint32_t packetType() const override { return m_inner()->packetType; }
@@ -62,18 +62,18 @@ class PSIMessage : virtual public PSIMessageInterface
         return result;
     }
 
-    std::map takeDataMap() override
+    std::vector const& dataIndex() const override { return m_inner()->dataIndex; }
+
+    void resizeData(uint64_t size) override
     {
-        std::map result;
-        for (auto& val : m_inner()->dataMap)
-        {
-            if (val.second.data())
-            {
-                result[val.first].assign(
-                    m_inner()->dataMap[val.first].begin(), m_inner()->dataMap[val.first].end());
-            }
-        }
-        return result;
+        m_inner()->data.resize(size);
+        m_inner()->dataIndex.resize(size);
+    }
+
+    void setDataPair(uint64_t pos, uint64_t dataIndex, bcos::bytes const& data) override
+    {
+        m_inner()->dataIndex[pos] = dataIndex;
+        m_inner()->data[pos].assign(data.begin(), data.end());
     }
 
     // Note: must ensure the object will not released before access the function
@@ -82,6 +82,8 @@ class PSIMessage : virtual public PSIMessageInterface
         auto const& dataItem = m_inner()->data.at(_index);
         return bcos::bytesConstRef((const bcos::byte*)(dataItem.data()), dataItem.size());
     }
+
+    uint64_t getDataCount() const override { return m_inner()->data.size(); }
     void setPartyID(std::string const& _partyID) override { m_inner()->partyID = _partyID; }
     void setResourceID(std::string const& _resourceID) override
     {
@@ -98,7 +100,7 @@ class PSIMessage : virtual public PSIMessageInterface
     {
         m_inner()->data.clear();
         uint64_t endPos = std::min(_offset + _len, (uint64_t)_data.size());
-        m_inner()->data.resize((endPos - _offset + 1));
+        m_inner()->data.resize((endPos - _offset));
         uint64_t j = 0;
         for (uint64_t i = _offset; i < endPos; i++)
         {
@@ -118,13 +120,16 @@ class PSIMessage : virtual public PSIMessageInterface
         }
     }
 
-    void setDataMap(std::map&& _map) override
+    void constructData(std::vector const& data, uint64_t startIndex) override
     {
-        m_inner()->dataMap.clear();
-        for (auto& val : _map)
+        m_inner()->dataIndex.clear();
+        int64_t i = startIndex;
+        for (auto const& it : data)
         {
-            m_inner()->dataMap[val.first].assign(val.second.begin(), val.second.end());
+            m_inner()->dataIndex.emplace_back(i);
+            i++;
         }
+        setData(data);
     }
 
     void appendData(bcos::bytes const& _dataItem) override
diff --git a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
index 11a8de42..45f4b2c7 100644
--- a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
@@ -6,7 +6,7 @@ find_package(OpenSSL REQUIRED)
 message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
 message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
-target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB})
+target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB} TCMalloc)
 
 if (ENABLE_IPP_CRYPTO)
     find_package(ipp-crypto REQUIRED)
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index fb2aad73..963dc7fe 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -559,7 +559,7 @@ void PPCConfig::loadHDFSConfig(boost::property_tree::ptree const& _pt)
     // the name node port
     option->nameNodePort = _pt.get("hdfs_storage.name_node_port", 8020);
     // the user
-    option->userName = _pt.get("hdfs_storage.user", "root");
+    option->userName = _pt.get("hdfs_storage.user", "");
     // checkNonEmptyField("hdfs_storage.user", option->userName);
     // the token
     option->token = _pt.get("hdfs_storage.token", "");
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index 27fa7b69..e8a0497f 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -70,7 +70,8 @@ struct GatewayConfig
     // the boostssl limit is 32MBytes
     constexpr static uint64_t DefaultMaxAllowedMsgSize = 100;
     constexpr static uint64_t MinMsgSize = 10 * 1024 * 1024;
-    constexpr static uint64_t MaxMsgSize = 1024 * 1024 * 1024;
+    // set the max msg size to 2GB
+    constexpr static uint64_t MaxMsgSize = uint64_t(1 << 31);
     constexpr static int MinUnreachableDistance = 2;
 
     NetworkConfig networkConfig;
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index 491f9517..9c1c0a78 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -32,7 +32,7 @@
 //TODO: optimize here to implement EcdhConnPSI
 #include "ppc-psi/src/ecdh-conn-psi/EcdhConnPSIFactory.h"
 #endif
-#include "ppc-framework/protocol/ServiceType.h"
+#include "ppc-framework/protocol/Constant.h"
 #include "ppc-front/Front.h"
 #include "ppc-front/FrontConfigImpl.h"
 #include "ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h"
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index e1fda6f0..08e03224 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -19,7 +19,7 @@
  */
 #include "MPCInitializer.h"
 #include "ppc-framework/front/FrontConfig.h"
-#include "ppc-framework/protocol/ServiceType.h"
+#include "ppc-framework/protocol/Constant.h"
 #include "ppc-mpc/src/MPCService.h"
 #include "ppc-tools/src/config/PPCConfig.h"
 #include "wedpr-protocol/protocol/src/ServiceConfig.h"
diff --git a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
index eb778d8e..6013a920 100644
--- a/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
+++ b/cpp/wedpr-protocol/grpc/server/GrpcServer.cpp
@@ -44,9 +44,9 @@ void GrpcServer::start()
     builder.AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0);
     // without authentication
     builder.AddListeningPort(m_config->listenEndPoint(), grpc::InsecureServerCredentials());
-    builder.SetMaxMessageSize(m_config->maxMsgSize());
-    builder.SetMaxSendMessageSize(m_config->maxSendMessageSize());
-    builder.SetMaxReceiveMessageSize(m_config->maxReceivedMessageSize());
+    builder.SetMaxMessageSize(m_config->grpcConfig()->maxMsgSize());
+    builder.SetMaxSendMessageSize(m_config->grpcConfig()->maxSendMessageSize());
+    builder.SetMaxReceiveMessageSize(m_config->grpcConfig()->maxReceivedMessageSize());
     // register the service
     for (auto const& service : m_bindingServices)
     {
@@ -67,9 +67,11 @@ void GrpcServer::start()
     }
     GRPC_SERVER_LOG(INFO) << LOG_DESC("GrpcServer start success!")
                           << LOG_KV("listenEndPoint", m_config->listenEndPoint())
-                          << LOG_KV("maxMsgSize", m_config->maxMsgSize())
-                          << LOG_KV("maxSendMessageSize", m_config->maxSendMessageSize())
-                          << LOG_KV("maxReceivedMessageSize", m_config->maxReceivedMessageSize());
+                          << LOG_KV("maxMsgSize", m_config->grpcConfig()->maxMsgSize())
+                          << LOG_KV(
+                                 "maxSendMessageSize", m_config->grpcConfig()->maxSendMessageSize())
+                          << LOG_KV("maxReceivedMessageSize",
+                                 m_config->grpcConfig()->maxReceivedMessageSize());
 }
 
 void GrpcServer::stop()
diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.h b/cpp/wedpr-protocol/protocol/src/PPCMessage.h
index 4ce25faf..c2ad3d29 100644
--- a/cpp/wedpr-protocol/protocol/src/PPCMessage.h
+++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.h
@@ -43,7 +43,7 @@ class PPCMessage : public PPCMessageFace
 
     using Ptr = std::shared_ptr;
     PPCMessage() { m_data = std::make_shared(); }
-    ~PPCMessage() override = default;
+    ~PPCMessage() override { releasePayload(); }
 
     uint8_t version() const override { return m_version; }
     void setVersion(uint8_t _version) override { m_version = _version; }
@@ -87,6 +87,16 @@ class PPCMessage : public PPCMessageFace
 
     bcos::bytes const& senderNode() const override { return m_senderNode; }
 
+    void releasePayload() override
+    {
+        if (!m_data)
+        {
+            return;
+        }
+        m_data->clear();
+        bcos::bytes().swap(*m_data);
+    }
+
 protected:
     std::string encodeMap(const std::map& _map);
     std::map decodeMap(const std::string& _encval);
diff --git a/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h b/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h
index 001d6812..45a54854 100644
--- a/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h
+++ b/cpp/wedpr-storage/ppc-io/src/BaseFileLineReader.h
@@ -47,7 +47,7 @@ class BaseFileLineReader : public LineReader
 
 protected:
     virtual bool allocateCurrentBlock() = 0;
-    virtual bool readFinished() const = 0;
+
 
     template 
     int64_t readLineFromBlock(DataBatch::Ptr _dataBatch, int64_t _size)
@@ -96,7 +96,7 @@ class BaseFileLineReader : public LineReader
         while (expectedSize > 0 && (allocateCurrentBlock()))
         {
             auto readLineNum = readLineFromBlock(dataBatch, expectedSize);
-            expectedSize = _size - readLineNum;
+            expectedSize = expectedSize - readLineNum;
         }
         IO_LOG(INFO) << LOG_DESC("readLineData") << LOG_KV("size", _size)
                      << LOG_KV("readedSize", dataBatch->size());
diff --git a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
index d26ddab5..a730f0de 100644
--- a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
@@ -1,4 +1,4 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${IO_TARGET} ${SRCS})
 # Note: the DataBatch depends on tbb
-target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB})
\ No newline at end of file
+target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} TCMalloc ${HDFS_LIB} ${CPU_FEATURES_LIB})
\ No newline at end of file
diff --git a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
index 279984c3..00d9cfdf 100644
--- a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
+++ b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
@@ -32,6 +32,55 @@ using namespace ppc::storage;
 using namespace ppc::protocol;
 using namespace bcos;
 
+
+DataResourceLoaderImpl::DataResourceLoaderImpl(
+    ppc::protocol::SQLConnectionOption::Ptr const& _sqlConnectionOpt,
+    ppc::protocol::FileStorageConnectionOption::Ptr const& _fileStorageConnectionOpt,
+    ppc::protocol::RemoteStorageConnectionOption::Ptr const& _remoteStorageConnectionOpt,
+    ppc::storage::SQLStorageFactory::Ptr const& _sqlStorageFactory,
+    ppc::storage::FileStorageFactory::Ptr const& _fileStorageFactory,
+    ppc::storage::RemoteStorageFactory::Ptr const& _remoteStorageFactory)
+  : m_sqlConnectionOpt(_sqlConnectionOpt),
+    m_fileStorageConnectionOpt(_fileStorageConnectionOpt),
+    m_remoteStorageConnectionOpt(_remoteStorageConnectionOpt),
+    m_sqlStorageFactory(_sqlStorageFactory),
+    m_fileStorageFactory(_fileStorageFactory),
+    m_remoteStorageFactory(_remoteStorageFactory)
+{}
+
+void DataResourceLoaderImpl::lazyLoadHdfsStorage()
+{
+    if (!m_fileStorageConnectionOpt)
+    {
+        return;
+    }
+    bcos::Guard l(x_hdfsStorage);
+    if (m_hdfsStorage)
+    {
+        return;
+    }
+    m_hdfsStorage =
+        m_fileStorageFactory->createFileStorage(DataResourceType::HDFS, m_fileStorageConnectionOpt);
+    IO_LOG(INFO) << LOG_DESC("lazyLoadHdfsStorage") << m_fileStorageConnectionOpt->desc();
+    return;
+}
+
+void DataResourceLoaderImpl::lazyLoadSqlStorage()
+{
+    if (!m_sqlConnectionOpt)
+    {
+        return;
+    }
+    bcos::Guard l(x_sqlStorage);
+    if (m_sqlStorage)
+    {
+        return;
+    }
+    m_sqlStorage =
+        m_sqlStorageFactory->createSQLStorage(DataResourceType::MySQL, m_sqlConnectionOpt);
+    IO_LOG(INFO) << LOG_DESC("lazyLoadSqlStorage") << m_sqlConnectionOpt->desc();
+}
+
 LineReader::Ptr DataResourceLoaderImpl::loadReader(DataResourceDesc::ConstPtr _desc,
     DataSchema _schema, bool _parseByColumn, FileStorage::Ptr const& _fileStorage)
 {
@@ -76,8 +125,8 @@ LineReader::Ptr DataResourceLoaderImpl::loadSQLResource(
     }
     else if (m_sqlConnectionOpt)
     {
-        storage = m_sqlStorageFactory->createSQLStorage(
-            (ppc::protocol::DataResourceType)_desc->type(), m_sqlConnectionOpt);
+        lazyLoadSqlStorage();
+        storage = m_sqlStorage;
     }
     else
     {
@@ -106,8 +155,8 @@ LineReader::Ptr DataResourceLoaderImpl::loadHDFSResource(
         }
         else if (m_fileStorageConnectionOpt)
         {
-            storage = m_fileStorageFactory->createFileStorage(
-                (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt);
+            lazyLoadHdfsStorage();
+            storage = m_hdfsStorage;
         }
         else
         {
@@ -144,8 +193,8 @@ void DataResourceLoaderImpl::deleteResource(
             }
             else if (m_fileStorageConnectionOpt)
             {
-                storage = m_fileStorageFactory->createFileStorage(
-                    (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt);
+                lazyLoadHdfsStorage();
+                storage = m_hdfsStorage;
             }
             else
             {
@@ -194,8 +243,8 @@ void DataResourceLoaderImpl::renameResource(ppc::protocol::DataResourceDesc::Con
             }
             else if (m_fileStorageConnectionOpt)
             {
-                storage = m_fileStorageFactory->createFileStorage(
-                    (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt);
+                lazyLoadHdfsStorage();
+                storage = m_hdfsStorage;
             }
             else
             {
@@ -253,8 +302,8 @@ void DataResourceLoaderImpl::checkResourceExists(
             }
             else if (m_fileStorageConnectionOpt)
             {
-                storage = m_fileStorageFactory->createFileStorage(
-                    (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt);
+                lazyLoadHdfsStorage();
+                storage = m_hdfsStorage;
             }
             else
             {
@@ -307,9 +356,8 @@ LineWriter::Ptr DataResourceLoaderImpl::loadWriter(
             }
             else if (m_fileStorageConnectionOpt)
             {
-                // create the hdfsStorage
-                storage = m_fileStorageFactory->createFileStorage(
-                    (ppc::protocol::DataResourceType)_desc->type(), m_fileStorageConnectionOpt);
+                lazyLoadHdfsStorage();
+                storage = m_hdfsStorage;
             }
             else
             {
diff --git a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
index a1c850fa..dc54f056 100644
--- a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
+++ b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
@@ -35,14 +35,7 @@ class DataResourceLoaderImpl : public DataResourceLoader
         ppc::protocol::RemoteStorageConnectionOption::Ptr const& _remoteStorageConnectionOpt,
         ppc::storage::SQLStorageFactory::Ptr const& _sqlStorageFactory,
         ppc::storage::FileStorageFactory::Ptr const& _fileStorageFactory,
-        ppc::storage::RemoteStorageFactory::Ptr const& _remoteStorageFactory)
-      : m_sqlConnectionOpt(_sqlConnectionOpt),
-        m_fileStorageConnectionOpt(_fileStorageConnectionOpt),
-        m_remoteStorageConnectionOpt(_remoteStorageConnectionOpt),
-        m_sqlStorageFactory(_sqlStorageFactory),
-        m_fileStorageFactory(_fileStorageFactory),
-        m_remoteStorageFactory(_remoteStorageFactory)
-    {}
+        ppc::storage::RemoteStorageFactory::Ptr const& _remoteStorageFactory);
 
     DataResourceLoaderImpl() = default;
     ~DataResourceLoaderImpl() override = default;
@@ -70,9 +63,20 @@ class DataResourceLoaderImpl : public DataResourceLoader
     virtual LineReader::Ptr loadHDFSResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
         ppc::storage::FileStorage::Ptr const& _fileStorage);
 
+    void lazyLoadHdfsStorage();
+    void lazyLoadSqlStorage();
+
 private:
+    // the sql storage
     ppc::protocol::SQLConnectionOption::Ptr m_sqlConnectionOpt;
+    ppc::storage::SQLStorage::Ptr m_sqlStorage;
+    bcos::Mutex x_sqlStorage;
+
+    // the hdfs storage
     ppc::protocol::FileStorageConnectionOption::Ptr m_fileStorageConnectionOpt;
+    ppc::storage::FileStorage::Ptr m_hdfsStorage;
+    bcos::Mutex x_hdfsStorage;
+
     ppc::protocol::RemoteStorageConnectionOption::Ptr m_remoteStorageConnectionOpt;
 
     ppc::storage::SQLStorageFactory::Ptr m_sqlStorageFactory;
diff --git a/cpp/wedpr-storage/ppc-io/src/FileLineReader.h b/cpp/wedpr-storage/ppc-io/src/FileLineReader.h
index 612a6654..fca4b47a 100644
--- a/cpp/wedpr-storage/ppc-io/src/FileLineReader.h
+++ b/cpp/wedpr-storage/ppc-io/src/FileLineReader.h
@@ -38,10 +38,10 @@ class FileLineReader : public BaseFileLineReader
     {
         return ppc::protocol::DataResourceType::FILE;
     }
+    bool readFinished() const override { return (m_offset == m_length); }
 
 protected:
     bool allocateCurrentBlock() override;
-    bool readFinished() const override { return (m_offset == m_length); }
 
 protected:
     std::string m_filePath;
diff --git a/cpp/wedpr-storage/ppc-io/src/HDFSReader.h b/cpp/wedpr-storage/ppc-io/src/HDFSReader.h
index c8169d97..31aeab30 100644
--- a/cpp/wedpr-storage/ppc-io/src/HDFSReader.h
+++ b/cpp/wedpr-storage/ppc-io/src/HDFSReader.h
@@ -41,9 +41,10 @@ class HDFSReader : public BaseFileLineReader
         return ppc::protocol::DataResourceType::HDFS;
     }
 
+    bool readFinished() const override { return (m_offset == m_length); }
+
 protected:
     bool allocateCurrentBlock() override;
-    bool readFinished() const override { return (m_offset == m_length); }
     virtual bool tryToOpenNewFile();
 
     inline bool loadDataAndCreateBufferParser()
diff --git a/cpp/wedpr-storage/ppc-io/src/SQLResultReader.h b/cpp/wedpr-storage/ppc-io/src/SQLResultReader.h
index d2d9439d..99b92843 100644
--- a/cpp/wedpr-storage/ppc-io/src/SQLResultReader.h
+++ b/cpp/wedpr-storage/ppc-io/src/SQLResultReader.h
@@ -64,6 +64,7 @@ class SQLResultReader : public LineReader
     {
         throw std::runtime_error("SQLResultReader: unimplemented readBytes!");
     }
+    bool readFinished() const override { return true; }
 
 private:
     ppc::storage::QueryResult::Ptr m_result;
diff --git a/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
index e51b2278..ce85047c 100644
--- a/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
@@ -4,4 +4,4 @@ add_library(${STORAGE_TARGET} ${SRCS})
 find_package(redis++ REQUIRED)
 find_package(unofficial-mysql-connector-cpp REQUIRED)
 
-target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
+target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static TCMalloc unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
diff --git a/cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h b/cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h
index 5fcd1074..57e16265 100644
--- a/cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h
+++ b/cpp/wedpr-storage/ppc-storage/src/FileStorageFactoryImpl.h
@@ -36,6 +36,7 @@ class FileStorageFactoryImpl : public FileStorageFactory
         {
         case ppc::protocol::DataResourceType::HDFS:
         {
+            _option->check();
             return std::make_shared(_option);
         }
         default:
diff --git a/cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h b/cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h
index 40d26cb5..1c1695df 100644
--- a/cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h
+++ b/cpp/wedpr-storage/ppc-storage/src/SQLStorageFactoryImpl.h
@@ -39,6 +39,7 @@ class SQLStorageFactoryImpl : public SQLStorageFactory
         {
         case ppc::protocol::DataResourceType::MySQL:
         {
+            _option->check();
             return std::make_shared(_option);
         }
         default:
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
index 73cd811d..4f1fedb1 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
@@ -19,6 +19,7 @@
  */
 #include "Front.h"
 #include "FrontImpl.h"
+#include "ppc-framework/protocol/Constant.h"
 #include "ppc-utilities/Utilities.h"
 
 using namespace ppc;
@@ -172,28 +173,37 @@ void Front::registerMessageHandler(uint8_t _taskType, uint8_t _algorithmType,
 {
     uint16_t type = ((uint16_t)_taskType << 8) | _algorithmType;
     auto self = weak_from_this();
-    m_front->registerMessageHandler(
-        std::to_string(type), [self, type, _handler](ppc::protocol::Message::Ptr msg) {
-            auto front = self.lock();
-            if (!front)
+    m_front->registerMessageHandler(std::to_string(type), [self, type, _handler](
+                                                              ppc::protocol::Message::Ptr msg) {
+        auto front = self.lock();
+        if (!front)
+        {
+            return;
+        }
+        try
+        {
+            if (msg == nullptr)
             {
+                _handler(nullptr);
                 return;
             }
-            try
+            _handler(front->m_messageFactory->decodePPCMessage(msg));
+            auto length = msg->length();
+            // release the payload of the large packet after the msg decoded
+            if (length >= ppc::protocol::LARGE_MSG_THRESHOLD)
             {
-                if (msg == nullptr)
-                {
-                    _handler(nullptr);
-                    return;
-                }
-                _handler(front->m_messageFactory->decodePPCMessage(msg));
+                FRONT_LOG(INFO) << LOG_DESC("RecvMsg: Release large payload for node after used")
+                                << LOG_KV("msgSize", length);
+                // release the payload
+                msg->releasePayload();
             }
-            catch (std::exception const& e)
-            {
-                FRONT_LOG(WARNING) << LOG_DESC("Call handler for component failed")
-                                   << LOG_KV("componentType", type)
-                                   << LOG_KV("error", boost::diagnostic_information(e));
-            }
-        });
+        }
+        catch (std::exception const& e)
+        {
+            FRONT_LOG(WARNING) << LOG_DESC("Call handler for component failed")
+                               << LOG_KV("componentType", type)
+                               << LOG_KV("error", boost::diagnostic_information(e));
+        }
+    });
     m_front->registerComponent(std::to_string(type));
 }
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
index 4ff6ed09..0441d1fa 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/FrontImpl.cpp
@@ -265,7 +265,9 @@ void FrontImpl::onReceiveMessage(Message::Ptr const& msg, ReceiveMsgFunc callbac
         }
         FRONT_LOG(TRACE) << LOG_BADGE("onReceiveMessage") << LOG_KV("msg", printMessage(msg));
         auto frontMessage = m_messageFactory->build(bcos::ref(*(msg->payload())));
-        msg->setFrontMessage(frontMessage);
+        // release the payload buffer since it useless now
+        msg->setFrontMessage(frontMessage, true);
+
         // the response packet, dispatcher by callback
         if (frontMessage->isRespPacket())
         {
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
index 9668c795..5860ec62 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcConfig.java
@@ -25,16 +25,12 @@ protected void swigSetCMemOwn(boolean own) {
         swigCMemOwn = own;
     }
 
-    @SuppressWarnings({"deprecation", "removal"})
-    protected void finalize() {
-        delete();
-    }
-
     public synchronized void delete() {
         if (swigCPtr != 0) {
             if (swigCMemOwn) {
                 swigCMemOwn = false;
-                wedpr_java_transportJNI.delete_GrpcConfig(swigCPtr);
+                throw new UnsupportedOperationException(
+                        "C++ destructor does not have public access");
             }
             swigCPtr = 0;
         }
@@ -93,4 +89,12 @@ public int compressAlgorithm() {
     public void setCompressAlgorithm(int compressAlgorithm) {
         wedpr_java_transportJNI.GrpcConfig_setCompressAlgorithm(swigCPtr, this, compressAlgorithm);
     }
+
+    public java.math.BigInteger maxMsgSize() {
+        return wedpr_java_transportJNI.GrpcConfig_maxMsgSize(swigCPtr, this);
+    }
+
+    public void setMaxMsgSize(java.math.BigInteger maxMsgSize) {
+        wedpr_java_transportJNI.GrpcConfig_setMaxMsgSize(swigCPtr, this, maxMsgSize);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java
index 179abdc3..3f138170 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/GrpcServerConfig.java
@@ -33,16 +33,12 @@ protected static long swigRelease(GrpcServerConfig obj) {
         return ptr;
     }
 
-    @SuppressWarnings({"deprecation", "removal"})
-    protected void finalize() {
-        delete();
-    }
-
     public synchronized void delete() {
         if (swigCPtr != 0) {
             if (swigCMemOwn) {
                 swigCMemOwn = false;
-                wedpr_java_transportJNI.delete_GrpcServerConfig(swigCPtr);
+                throw new UnsupportedOperationException(
+                        "C++ destructor does not have public access");
             }
             swigCPtr = 0;
         }
@@ -86,4 +82,9 @@ public EndPoint mutableEndPoint() {
     public boolean enableHealthCheck() {
         return wedpr_java_transportJNI.GrpcServerConfig_enableHealthCheck(swigCPtr, this);
     }
+
+    public GrpcConfig grpcConfig() {
+        long cPtr = wedpr_java_transportJNI.GrpcServerConfig_grpcConfig(swigCPtr, this);
+        return (cPtr == 0) ? null : new GrpcConfig(cPtr, true);
+    }
 }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
index dc50506c..ab64b268 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/Message.java
@@ -100,11 +100,6 @@ public byte[] payloadBuffer() {
         return wedpr_java_transportJNI.Message_payloadBuffer(swigCPtr, this);
     }
 
-    public void setFrontMessage(MessagePayload frontMessage) {
-        wedpr_java_transportJNI.Message_setFrontMessage(
-                swigCPtr, this, MessagePayload.getCPtr(frontMessage), frontMessage);
-    }
-
     public MessagePayload frontMessage() {
         long cPtr = wedpr_java_transportJNI.Message_frontMessage(swigCPtr, this);
         return (cPtr == 0) ? null : new MessagePayload(cPtr, true);
@@ -129,6 +124,10 @@ public ubytes payload() {
         return (cPtr == 0) ? null : new ubytes(cPtr, true);
     }
 
+    public void releasePayload() {
+        wedpr_java_transportJNI.Message_releasePayload(swigCPtr, this);
+    }
+
     public void disOwnMemory() {
         swigSetCMemOwn(false);
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
index 3998cdba..c83bc33d 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/MessagePayload.java
@@ -98,6 +98,10 @@ public boolean isRespPacket() {
         return wedpr_java_transportJNI.MessagePayload_isRespPacket(swigCPtr, this);
     }
 
+    public void releasePayload() {
+        wedpr_java_transportJNI.MessagePayload_releasePayload(swigCPtr, this);
+    }
+
     public void disOwnMemory() {
         swigSetCMemOwn(false);
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
index f5fedd8f..6899432c 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/generated/wedpr_java_transportJNI.java
@@ -361,34 +361,8 @@ public static final native long FrontConfigBuilder_build__SWIG_1(
 
     public static final native String EndPoint_listenIp(long jarg1, EndPoint jarg1_);
 
-    public static final native long new_GrpcServerConfig__SWIG_0();
-
-    public static final native long new_GrpcServerConfig__SWIG_1(
-            long jarg1, EndPoint jarg1_, boolean jarg2);
-
-    public static final native String GrpcServerConfig_listenEndPoint(
-            long jarg1, GrpcServerConfig jarg1_);
-
-    public static final native void GrpcServerConfig_setEndPoint(
-            long jarg1, GrpcServerConfig jarg1_, long jarg2, EndPoint jarg2_);
-
-    public static final native void GrpcServerConfig_setEnableHealthCheck(
-            long jarg1, GrpcServerConfig jarg1_, boolean jarg2);
-
-    public static final native long GrpcServerConfig_endPoint(long jarg1, GrpcServerConfig jarg1_);
-
-    public static final native long GrpcServerConfig_mutableEndPoint(
-            long jarg1, GrpcServerConfig jarg1_);
-
-    public static final native boolean GrpcServerConfig_enableHealthCheck(
-            long jarg1, GrpcServerConfig jarg1_);
-
-    public static final native void delete_GrpcServerConfig(long jarg1);
-
     public static final native long new_GrpcConfig();
 
-    public static final native void delete_GrpcConfig(long jarg1);
-
     public static final native String GrpcConfig_loadBalancePolicy(long jarg1, GrpcConfig jarg1_);
 
     public static final native void GrpcConfig_setLoadBalancePolicy(
@@ -421,6 +395,37 @@ public static final native void GrpcConfig_setMaxReceivedMessageSize(
     public static final native void GrpcConfig_setCompressAlgorithm(
             long jarg1, GrpcConfig jarg1_, int jarg2);
 
+    public static final native java.math.BigInteger GrpcConfig_maxMsgSize(
+            long jarg1, GrpcConfig jarg1_);
+
+    public static final native void GrpcConfig_setMaxMsgSize(
+            long jarg1, GrpcConfig jarg1_, java.math.BigInteger jarg2);
+
+    public static final native long new_GrpcServerConfig__SWIG_0();
+
+    public static final native long new_GrpcServerConfig__SWIG_1(
+            long jarg1, EndPoint jarg1_, boolean jarg2);
+
+    public static final native String GrpcServerConfig_listenEndPoint(
+            long jarg1, GrpcServerConfig jarg1_);
+
+    public static final native void GrpcServerConfig_setEndPoint(
+            long jarg1, GrpcServerConfig jarg1_, long jarg2, EndPoint jarg2_);
+
+    public static final native void GrpcServerConfig_setEnableHealthCheck(
+            long jarg1, GrpcServerConfig jarg1_, boolean jarg2);
+
+    public static final native long GrpcServerConfig_endPoint(long jarg1, GrpcServerConfig jarg1_);
+
+    public static final native long GrpcServerConfig_mutableEndPoint(
+            long jarg1, GrpcServerConfig jarg1_);
+
+    public static final native boolean GrpcServerConfig_enableHealthCheck(
+            long jarg1, GrpcServerConfig jarg1_);
+
+    public static final native long GrpcServerConfig_grpcConfig(
+            long jarg1, GrpcServerConfig jarg1_);
+
     public static final native String printGrpcConfig(long jarg1, GrpcConfig jarg1_);
 
     public static final native void delete_MessageOptionalHeader(long jarg1);
@@ -570,9 +575,6 @@ public static final native void Message_setPayload(
 
     public static final native byte[] Message_payloadBuffer(long jarg1, Message jarg1_);
 
-    public static final native void Message_setFrontMessage(
-            long jarg1, Message jarg1_, long jarg2, MessagePayload jarg2_);
-
     public static final native long Message_frontMessage(long jarg1, Message jarg1_);
 
     public static final native boolean Message_encode(
@@ -584,6 +586,8 @@ public static final native boolean Message_encode(
 
     public static final native long Message_payload(long jarg1, Message jarg1_);
 
+    public static final native void Message_releasePayload(long jarg1, Message jarg1_);
+
     public static final native void delete_MessageHeaderBuilder(long jarg1);
 
     public static final native long MessageHeaderBuilder_build__SWIG_0(
@@ -665,6 +669,9 @@ public static final native void MessagePayload_setExt(
     public static final native boolean MessagePayload_isRespPacket(
             long jarg1, MessagePayload jarg1_);
 
+    public static final native void MessagePayload_releasePayload(
+            long jarg1, MessagePayload jarg1_);
+
     public static final native void delete_MessagePayloadBuilder(long jarg1);
 
     public static final native long MessagePayloadBuilder_build__SWIG_0(
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
index fec5698c..f839dee8 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/java/src/wedpr_java_transportJAVA_wrap.cxx
@@ -4914,116 +4914,123 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_10(JNIEnv *jenv, jclass jcls) {
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcConfig(JNIEnv *jenv, jclass jcls) {
   jlong jresult = 0 ;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
+  ppc::protocol::GrpcConfig *result = 0 ;
   
   (void)jenv;
   (void)jcls;
   {
     try {
-      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
+  
+  *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_1) : 0;
+  
   return jresult;
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
-  jlong jresult = 0 ;
-  ppc::protocol::EndPoint arg1 ;
-  bool arg2 ;
-  ppc::protocol::EndPoint *argp1 ;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1loadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  std::string *result = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  argp1 = *(ppc::protocol::EndPoint **)&jarg1; 
-  if (!argp1) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint");
-    return 0;
-  }
-  arg1 = *argp1; 
-  arg2 = jarg2 ? true : false; 
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
+  jresult = jenv->NewStringUTF(result->c_str()); 
   return jresult;
 }
 
 
-SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1listenEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jstring jresult = 0 ;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  std::string result;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setLoadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::string *arg2 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  if(!jarg2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
+    return ;
+  }
+  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
+  if (!arg2_pstr) return ;
+  std::string arg2_str(arg2_pstr);
+  arg2 = &arg2_str;
+  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
   {
     try {
-      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return 0;
+      return ;
     }
   }
-  jresult = jenv->NewStringUTF((&result)->c_str()); 
-  return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  ppc::protocol::EndPoint arg2 ;
-  ppc::protocol::EndPoint *argp2 ;
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jboolean jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  bool result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  (void)jarg2_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
-  argp2 = *(ppc::protocol::EndPoint **)&jarg2; 
-  if (!argp2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint");
-    return ;
-  }
-  arg2 = *argp2; 
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      (arg1)->setEndPoint(arg2);
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return ;
+      return 0;
     }
   }
+  jresult = (jboolean)result; 
+  return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   bool arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   arg2 = jarg2 ? true : false; 
   {
     try {
@@ -5037,127 +5044,231 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1endPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  ppc::protocol::EndPoint *result = 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  bool arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  arg2 = jarg2 ? true : false; 
   {
     try {
-      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
+      (arg1)->setEnableDnslookup(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return 0;
+      return ;
     }
   }
-  *(ppc::protocol::EndPoint **)&jresult = result; 
-  return jresult;
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1mutableEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jlong jresult = 0 ;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  ppc::protocol::EndPoint *result = 0 ;
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jboolean jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  bool result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  *(ppc::protocol::EndPoint **)&jresult = result; 
+  jresult = (jboolean)result; 
   return jresult;
 }
 
 
-SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jboolean jresult = 0 ;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  bool result;
+SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jobject jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  uint64_t result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  jresult = (jboolean)result; 
+  {
+    jbyteArray ba = jenv->NewByteArray(9);
+    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
+    jclass clazz = jenv->FindClass("java/math/BigInteger");
+    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
+    jobject bigint;
+    int i;
+    
+    bae[0] = 0;
+    for(i=1; i<9; i++ ) {
+      bae[i] = (jbyte)(result>>8*(8-i));
+    }
+    
+    jenv->ReleaseByteArrayElements(ba, bae, 0);
+    bigint = jenv->NewObject(clazz, mid, ba);
+    jenv->DeleteLocalRef(ba);
+    jresult = bigint;
+  }
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1GrpcServerConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) {
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jobject jresult = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  uint64_t result;
   
   (void)jenv;
   (void)jcls;
-  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      delete arg1;
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return ;
+      return 0;
     }
   }
+  {
+    jbyteArray ba = jenv->NewByteArray(9);
+    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
+    jclass clazz = jenv->FindClass("java/math/BigInteger");
+    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
+    jobject bigint;
+    int i;
+    
+    bae[0] = 0;
+    for(i=1; i<9; i++ ) {
+      bae[i] = (jbyte)(result>>8*(8-i));
+    }
+    
+    jenv->ReleaseByteArrayElements(ba, bae, 0);
+    bigint = jenv->NewObject(clazz, mid, ba);
+    jenv->DeleteLocalRef(ba);
+    jresult = bigint;
+  }
+  return jresult;
 }
 
 
-SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcConfig(JNIEnv *jenv, jclass jcls) {
-  jlong jresult = 0 ;
-  ppc::protocol::GrpcConfig *result = 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
   (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
+  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    jclass clazz;
+    jmethodID mid;
+    jbyteArray ba;
+    jbyte* bae;
+    jsize sz;
+    int i;
+    
+    if (!jarg2) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
+      return ;
+    }
+    clazz = jenv->GetObjectClass(jarg2);
+    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
+    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
+    bae = jenv->GetByteArrayElements(ba, 0);
+    sz = jenv->GetArrayLength(ba);
+    arg2 = 0;
+    if (sz > 0) {
+      arg2 = (uint64_t)(signed char)bae[0];
+      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
+  }
   {
     try {
-      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+      (arg1)->setMaxSendMessageSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return 0;
+      return ;
     }
   }
-  
-  *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_1) : 0;
-  
-  return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1GrpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1) {
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
   (void)jcls;
+  (void)jarg1_;
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    jclass clazz;
+    jmethodID mid;
+    jbyteArray ba;
+    jbyte* bae;
+    jsize sz;
+    int i;
+    
+    if (!jarg2) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
+      return ;
+    }
+    clazz = jenv->GetObjectClass(jarg2);
+    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
+    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
+    bae = jenv->GetByteArrayElements(ba, 0);
+    sz = jenv->GetArrayLength(ba);
+    arg2 = 0;
+    if (sz > 0) {
+      arg2 = (uint64_t)(signed char)bae[0];
+      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
+  }
   {
     try {
-      (void)arg1; delete smartarg1;
+      (arg1)->setMaxReceivedMessageSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
@@ -5167,11 +5278,11 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1loadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jstring jresult = 0 ;
+SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1compressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jint jresult = 0 ;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  std::string *result = 0 ;
+  int result;
   
   (void)jenv;
   (void)jcls;
@@ -5181,21 +5292,21 @@ SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  jresult = jenv->NewStringUTF(result->c_str()); 
+  jresult = (jint)result; 
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setLoadBalancePolicy(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2) {
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setCompressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  int arg2 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
@@ -5204,18 +5315,10 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  if(!jarg2) {
-    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "null string");
-    return ;
-  }
-  const char *arg2_pstr = (const char *)jenv->GetStringUTFChars(jarg2, 0); 
-  if (!arg2_pstr) return ;
-  std::string arg2_str(arg2_pstr);
-  arg2 = &arg2_str;
-  jenv->ReleaseStringUTFChars(jarg2, arg2_pstr); 
+  arg2 = (int)jarg2; 
   {
     try {
-      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+      (arg1)->setCompressAlgorithm(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
@@ -5225,11 +5328,11 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jboolean jresult = 0 ;
+SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxMsgSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jobject jresult = 0 ;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  bool result;
+  uint64_t result;
   
   (void)jenv;
   (void)jcls;
@@ -5239,21 +5342,38 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxMsgSize();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  jresult = (jboolean)result; 
+  {
+    jbyteArray ba = jenv->NewByteArray(9);
+    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
+    jclass clazz = jenv->FindClass("java/math/BigInteger");
+    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
+    jobject bigint;
+    int i;
+    
+    bae[0] = 0;
+    for(i=1; i<9; i++ ) {
+      bae[i] = (jbyte)(result>>8*(8-i));
+    }
+    
+    jenv->ReleaseByteArrayElements(ba, bae, 0);
+    bigint = jenv->NewObject(clazz, mid, ba);
+    jenv->DeleteLocalRef(ba);
+    jresult = bigint;
+  }
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxMsgSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  bool arg2 ;
+  uint64_t arg2 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   
   (void)jenv;
@@ -5262,34 +5382,35 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
   
   smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
   arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = jarg2 ? true : false; 
   {
-    try {
-      (arg1)->setEnableHealthCheck(arg2);
-    }
-    catch (const std::exception& e) {
-      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+    jclass clazz;
+    jmethodID mid;
+    jbyteArray ba;
+    jbyte* bae;
+    jsize sz;
+    int i;
+    
+    if (!jarg2) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
       return ;
     }
+    clazz = jenv->GetObjectClass(jarg2);
+    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
+    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
+    bae = jenv->GetByteArrayElements(ba, 0);
+    sz = jenv->GetArrayLength(ba);
+    arg2 = 0;
+    if (sz > 0) {
+      arg2 = (uint64_t)(signed char)bae[0];
+      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
   }
-}
-
-
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setEnableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  bool arg2 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = jarg2 ? true : false; 
   {
     try {
-      (arg1)->setEnableDnslookup(arg2);
+      (arg1)->setMaxMsgSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
@@ -5299,158 +5420,120 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1enableDnslookup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jboolean jresult = 0 ;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  bool result;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_10(JNIEnv *jenv, jclass jcls) {
+  jlong jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
   
   (void)jenv;
   (void)jcls;
-  (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
   {
     try {
-      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  jresult = (jboolean)result; 
+  *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
   return jresult;
 }
 
 
-SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jobject jresult = 0 ;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  uint64_t result;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_new_1GrpcServerConfig_1_1SWIG_11(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
+  jlong jresult = 0 ;
+  ppc::protocol::EndPoint arg1 ;
+  bool arg2 ;
+  ppc::protocol::EndPoint *argp1 ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  argp1 = *(ppc::protocol::EndPoint **)&jarg1; 
+  if (!argp1) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint");
+    return 0;
+  }
+  arg1 = *argp1; 
+  arg2 = jarg2 ? true : false; 
   {
     try {
-      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return 0;
-    }
-  }
-  {
-    jbyteArray ba = jenv->NewByteArray(9);
-    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
-    jclass clazz = jenv->FindClass("java/math/BigInteger");
-    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
-    jobject bigint;
-    int i;
-    
-    bae[0] = 0;
-    for(i=1; i<9; i++ ) {
-      bae[i] = (jbyte)(result>>8*(8-i));
+      return 0;
     }
-    
-    jenv->ReleaseByteArrayElements(ba, bae, 0);
-    bigint = jenv->NewObject(clazz, mid, ba);
-    jenv->DeleteLocalRef(ba);
-    jresult = bigint;
   }
+  *(ppc::protocol::GrpcServerConfig **)&jresult = result; 
   return jresult;
 }
 
 
-SWIGEXPORT jobject JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1maxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jobject jresult = 0 ;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  uint64_t result;
+SWIGEXPORT jstring JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1listenEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jstring jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  std::string result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   {
     try {
-      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  {
-    jbyteArray ba = jenv->NewByteArray(9);
-    jbyte* bae = jenv->GetByteArrayElements(ba, 0);
-    jclass clazz = jenv->FindClass("java/math/BigInteger");
-    jmethodID mid = jenv->GetMethodID(clazz, "", "([B)V");
-    jobject bigint;
-    int i;
-    
-    bae[0] = 0;
-    for(i=1; i<9; i++ ) {
-      bae[i] = (jbyte)(result>>8*(8-i));
-    }
-    
-    jenv->ReleaseByteArrayElements(ba, bae, 0);
-    bigint = jenv->NewObject(clazz, mid, ba);
-    jenv->DeleteLocalRef(ba);
-    jresult = bigint;
-  }
+  jresult = jenv->NewStringUTF((&result)->c_str()); 
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxSendMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  uint64_t arg2 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::EndPoint arg2 ;
+  ppc::protocol::EndPoint *argp2 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  (void)jarg2_;
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  argp2 = *(ppc::protocol::EndPoint **)&jarg2; 
+  if (!argp2) {
+    SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "Attempt to dereference null ppc::protocol::EndPoint");
+    return ;
+  }
+  arg2 = *argp2; 
   {
-    jclass clazz;
-    jmethodID mid;
-    jbyteArray ba;
-    jbyte* bae;
-    jsize sz;
-    int i;
-    
-    if (!jarg2) {
-      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
-      return ;
+    try {
+      (arg1)->setEndPoint(arg2);
     }
-    clazz = jenv->GetObjectClass(jarg2);
-    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
-    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
-    bae = jenv->GetByteArrayElements(ba, 0);
-    sz = jenv->GetArrayLength(ba);
-    arg2 = 0;
-    if (sz > 0) {
-      arg2 = (uint64_t)(signed char)bae[0];
-      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
   }
+}
+
+
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1setEnableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jboolean jarg2) {
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  bool arg2 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
+  arg2 = jarg2 ? true : false; 
   {
     try {
-      (arg1)->setMaxSendMessageSize(arg2);
+      (arg1)->setEnableHealthCheck(arg2);
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
@@ -5460,102 +5543,95 @@ SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tra
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setMaxReceivedMessageSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jobject jarg2) {
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  uint64_t arg2 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1endPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   {
-    jclass clazz;
-    jmethodID mid;
-    jbyteArray ba;
-    jbyte* bae;
-    jsize sz;
-    int i;
-    
-    if (!jarg2) {
-      SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, "BigInteger null");
-      return ;
+    try {
+      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
     }
-    clazz = jenv->GetObjectClass(jarg2);
-    mid = jenv->GetMethodID(clazz, "toByteArray", "()[B");
-    ba = (jbyteArray)jenv->CallObjectMethod(jarg2, mid);
-    bae = jenv->GetByteArrayElements(ba, 0);
-    sz = jenv->GetArrayLength(ba);
-    arg2 = 0;
-    if (sz > 0) {
-      arg2 = (uint64_t)(signed char)bae[0];
-      for(i=1; iReleaseByteArrayElements(ba, bae, 0);
   }
+  *(ppc::protocol::EndPoint **)&jresult = result; 
+  return jresult;
+}
+
+
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1mutableEndPoint(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::EndPoint *result = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   {
     try {
-      (arg1)->setMaxReceivedMessageSize(arg2);
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return ;
+      return 0;
     }
   }
+  *(ppc::protocol::EndPoint **)&jresult = result; 
+  return jresult;
 }
 
 
-SWIGEXPORT jint JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1compressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
-  jint jresult = 0 ;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  int result;
+SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1enableHealthCheck(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jboolean jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  bool result;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr< const ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   {
     try {
-      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
       return 0;
     }
   }
-  jresult = (jint)result; 
+  jresult = (jboolean)result; 
   return jresult;
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcConfig_1setCompressAlgorithm(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jint jarg2) {
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  int arg2 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_GrpcServerConfig_1grpcConfig(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  jlong jresult = 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig::Ptr *result = 0 ;
   
   (void)jenv;
   (void)jcls;
   (void)jarg1_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::GrpcConfig > **)&jarg1;
-  arg1 = (ppc::protocol::GrpcConfig *)(smartarg1 ? smartarg1->get() : 0); 
-  arg2 = (int)jarg2; 
+  arg1 = *(ppc::protocol::GrpcServerConfig **)&jarg1; 
   {
     try {
-      (arg1)->setCompressAlgorithm(arg2);
+      result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::protocol::GrpcServerConfig const *)arg1)->grpcConfig();
     }
     catch (const std::exception& e) {
       SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return ;
+      return 0;
     }
   }
+  *(ppc::protocol::GrpcConfig::Ptr **)&jresult = *result ? new ppc::protocol::GrpcConfig::Ptr(*result) : 0; 
+  return jresult;
 }
 
 
@@ -7232,33 +7308,6 @@ SWIGEXPORT jbyteArray JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1jav
 }
 
 
-SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1setFrontMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jobject jarg2_) {
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  ppc::protocol::MessagePayload::Ptr arg2 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  ppc::protocol::MessagePayload::Ptr *argp2 ;
-  
-  (void)jenv;
-  (void)jcls;
-  (void)jarg1_;
-  (void)jarg2_;
-  
-  smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
-  arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
-  argp2 = *(ppc::protocol::MessagePayload::Ptr **)&jarg2;
-  if (argp2) arg2 = *argp2; 
-  {
-    try {
-      (arg1)->setFrontMessage(arg2);
-    }
-    catch (const std::exception& e) {
-      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
-      return ;
-    }
-  }
-}
-
-
 SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1frontMessage(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
   jlong jresult = 0 ;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
@@ -7405,6 +7454,28 @@ SWIGEXPORT jlong JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1tr
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_Message_1releasePayload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::Message > **)&jarg1;
+  arg1 = (ppc::protocol::Message *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    try {
+      (arg1)->releasePayload();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessageHeaderBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::protocol::MessageHeaderBuilder *arg1 = (ppc::protocol::MessageHeaderBuilder *) 0 ;
   std::shared_ptr< ppc::protocol::MessageHeaderBuilder > *smartarg1 = 0 ;
@@ -8190,6 +8261,28 @@ SWIGEXPORT jboolean JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_
 }
 
 
+SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_MessagePayload_1releasePayload(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_) {
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  
+  (void)jenv;
+  (void)jcls;
+  (void)jarg1_;
+  
+  smartarg1 = *(std::shared_ptr<  ppc::protocol::MessagePayload > **)&jarg1;
+  arg1 = (ppc::protocol::MessagePayload *)(smartarg1 ? smartarg1->get() : 0); 
+  {
+    try {
+      (arg1)->releasePayload();
+    }
+    catch (const std::exception& e) {
+      SWIG_JavaThrowException(jenv, SWIG_JavaRuntimeException, std::string(boost::diagnostic_information(e)).c_str());
+      return ;
+    }
+  }
+}
+
+
 SWIGEXPORT void JNICALL Java_com_webank_wedpr_sdk_jni_generated_wedpr_1java_1transportJNI_delete_1MessagePayloadBuilder(JNIEnv *jenv, jclass jcls, jlong jarg1) {
   ppc::protocol::MessagePayloadBuilder *arg1 = (ppc::protocol::MessagePayloadBuilder *) 0 ;
   std::shared_ptr< ppc::protocol::MessagePayloadBuilder > *smartarg1 = 0 ;
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
index 9e32fe66..879134e6 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/java/swig/wedpr_java_transport.i
@@ -253,6 +253,9 @@ WRAP(ppc::sdk::Transport)
 %ignore ppc::protocol::INodeInfo::toJson;
 %ignore ppc::protocol::INodeInfo::setComponents;
 %ignore ppc::protocol::INodeInfoFactory;
+%ignore ppc::protocol::Message::setFrontMessage;
+%ignore ppc::protocol::GrpcConfig::~GrpcConfig;
+%ignore ppc::protocol::GrpcServerConfig::~GrpcServerConfig;
 
 %include "exception.i"
 %exception {
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
index 62ab6aca..17190df1 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/generated/wedpr_python_transport.py
@@ -1075,39 +1075,6 @@ def listenIp(self):
 _wedpr_python_transport.EndPoint_swigregister(EndPoint)
 
 
-class GrpcServerConfig(object):
-    thisown = property(lambda x: x.this.own(), lambda x,
-                       v: x.this.own(v), doc="The membership flag")
-    __repr__ = _swig_repr
-
-    def __init__(self, *args):
-        _wedpr_python_transport.GrpcServerConfig_swiginit(
-            self, _wedpr_python_transport.new_GrpcServerConfig(*args))
-
-    def listenEndPoint(self):
-        return _wedpr_python_transport.GrpcServerConfig_listenEndPoint(self)
-
-    def setEndPoint(self, endPoint):
-        return _wedpr_python_transport.GrpcServerConfig_setEndPoint(self, endPoint)
-
-    def setEnableHealthCheck(self, enableHealthCheck):
-        return _wedpr_python_transport.GrpcServerConfig_setEnableHealthCheck(self, enableHealthCheck)
-
-    def endPoint(self):
-        return _wedpr_python_transport.GrpcServerConfig_endPoint(self)
-
-    def mutableEndPoint(self):
-        return _wedpr_python_transport.GrpcServerConfig_mutableEndPoint(self)
-
-    def enableHealthCheck(self):
-        return _wedpr_python_transport.GrpcServerConfig_enableHealthCheck(self)
-    __swig_destroy__ = _wedpr_python_transport.delete_GrpcServerConfig
-
-
-# Register GrpcServerConfig in _wedpr_python_transport:
-_wedpr_python_transport.GrpcServerConfig_swigregister(GrpcServerConfig)
-
-
 class GrpcConfig(object):
     thisown = property(lambda x: x.this.own(), lambda x,
                        v: x.this.own(v), doc="The membership flag")
@@ -1116,7 +1083,6 @@ class GrpcConfig(object):
     def __init__(self):
         _wedpr_python_transport.GrpcConfig_swiginit(
             self, _wedpr_python_transport.new_GrpcConfig())
-    __swig_destroy__ = _wedpr_python_transport.delete_GrpcConfig
 
     def loadBalancePolicy(self):
         return _wedpr_python_transport.GrpcConfig_loadBalancePolicy(self)
@@ -1154,11 +1120,52 @@ def compressAlgorithm(self):
     def setCompressAlgorithm(self, compressAlgorithm):
         return _wedpr_python_transport.GrpcConfig_setCompressAlgorithm(self, compressAlgorithm)
 
+    def maxMsgSize(self):
+        return _wedpr_python_transport.GrpcConfig_maxMsgSize(self)
+
+    def setMaxMsgSize(self, maxMsgSize):
+        return _wedpr_python_transport.GrpcConfig_setMaxMsgSize(self, maxMsgSize)
+
 
 # Register GrpcConfig in _wedpr_python_transport:
 _wedpr_python_transport.GrpcConfig_swigregister(GrpcConfig)
 
 
+class GrpcServerConfig(object):
+    thisown = property(lambda x: x.this.own(), lambda x,
+                       v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self, *args):
+        _wedpr_python_transport.GrpcServerConfig_swiginit(
+            self, _wedpr_python_transport.new_GrpcServerConfig(*args))
+
+    def listenEndPoint(self):
+        return _wedpr_python_transport.GrpcServerConfig_listenEndPoint(self)
+
+    def setEndPoint(self, endPoint):
+        return _wedpr_python_transport.GrpcServerConfig_setEndPoint(self, endPoint)
+
+    def setEnableHealthCheck(self, enableHealthCheck):
+        return _wedpr_python_transport.GrpcServerConfig_setEnableHealthCheck(self, enableHealthCheck)
+
+    def endPoint(self):
+        return _wedpr_python_transport.GrpcServerConfig_endPoint(self)
+
+    def mutableEndPoint(self):
+        return _wedpr_python_transport.GrpcServerConfig_mutableEndPoint(self)
+
+    def enableHealthCheck(self):
+        return _wedpr_python_transport.GrpcServerConfig_enableHealthCheck(self)
+
+    def grpcConfig(self):
+        return _wedpr_python_transport.GrpcServerConfig_grpcConfig(self)
+
+
+# Register GrpcServerConfig in _wedpr_python_transport:
+_wedpr_python_transport.GrpcServerConfig_swigregister(GrpcServerConfig)
+
+
 def printGrpcConfig(grpcConfig):
     return _wedpr_python_transport.printGrpcConfig(grpcConfig)
 
@@ -1365,9 +1372,6 @@ def setPayload(self, _payload):
     def payloadBuffer(self):
         return _wedpr_python_transport.Message_payloadBuffer(self)
 
-    def setFrontMessage(self, frontMessage):
-        return _wedpr_python_transport.Message_setFrontMessage(self, frontMessage)
-
     def frontMessage(self):
         return _wedpr_python_transport.Message_frontMessage(self)
 
@@ -1383,6 +1387,9 @@ def length(self):
     def payload(self):
         return _wedpr_python_transport.Message_payload(self)
 
+    def releasePayload(self):
+        return _wedpr_python_transport.Message_releasePayload(self)
+
 
 # Register Message in _wedpr_python_transport:
 _wedpr_python_transport.Message_swigregister(Message)
@@ -1502,6 +1509,9 @@ def setRespPacket(self):
     def isRespPacket(self):
         return _wedpr_python_transport.MessagePayload_isRespPacket(self)
 
+    def releasePayload(self):
+        return _wedpr_python_transport.MessagePayload_releasePayload(self)
+
 
 # Register MessagePayload in _wedpr_python_transport:
 _wedpr_python_transport.MessagePayload_swigregister(MessagePayload)
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
index abe25249..f5b2674f 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
+++ b/cpp/wedpr-transport/sdk-wrapper/python/src/wedpr_python_transportPYTHON_wrap.cxx
@@ -19016,171 +19016,209 @@ SWIGINTERN PyObject *EndPoint_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject
   return SWIG_Python_InitShadowInstance(args);
 }
 
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
+SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
+  ppc::protocol::GrpcConfig *result = 0 ;
   
   (void)self;
-  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail;
   {
     try {
-      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
+  {
+    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::EndPoint arg1 ;
-  bool arg2 ;
-  void *argp1 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  void *argp1 = 0 ;
   int res1 = 0 ;
-  bool val2 ;
-  int ecode2 = 0 ;
-  ppc::protocol::GrpcServerConfig *result = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  std::string *result = 0 ;
   
   (void)self;
-  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
   {
-    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); 
-    }  
-    if (!argp1) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_loadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
-      arg1 = *temp;
-      if (SWIG_IsNewObj(res1)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'");
-  } 
-  arg2 = static_cast< bool >(val2);
   {
     try {
-      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
+      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_new_GrpcServerConfig(PyObject *self, PyObject *args) {
-  Py_ssize_t argc;
-  PyObject *argv[3] = {
-    0
-  };
+SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject *swig_obj[2] ;
   
-  if (!(argc = SWIG_Python_UnpackTuple(args, "new_GrpcServerConfig", 0, 2, argv))) SWIG_fail;
-  --argc;
-  if (argc == 0) {
-    return _wrap_new_GrpcServerConfig__SWIG_0(self, argc, argv);
+  (void)self;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setLoadBalancePolicy", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  if (argc == 2) {
-    int _v = 0;
-    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NO_NULL | 0);
-    _v = SWIG_CheckState(res);
-    if (_v) {
-      {
-        int res = SWIG_AsVal_bool(argv[1], NULL);
-        _v = SWIG_CheckState(res);
-      }
-      if (_v) {
-        return _wrap_new_GrpcServerConfig__SWIG_1(self, argc, argv);
-      }
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
     }
+    arg2 = ptr;
   }
-  
+  {
+    try {
+      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
 fail:
-  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_GrpcServerConfig'.\n"
-    "  Possible C/C++ prototypes are:\n"
-    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig()\n"
-    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig(ppc::protocol::EndPoint,bool)\n");
-  return 0;
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  ppc::protocol::EndPoint arg2 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  bool val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEndPoint", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
   {
-    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); 
-    }  
-    if (!argp2) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'");
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
     } else {
-      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
-      arg2 = *temp;
-      if (SWIG_IsNewObj(res2)) delete temp;
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+  } 
+  arg2 = static_cast< bool >(val2);
   {
     try {
-      (arg1)->setEndPoint(arg2);
+      (arg1)->setEnableHealthCheck(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
@@ -19193,31 +19231,43 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   bool val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableDnslookup", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
   {
     try {
-      (arg1)->setEnableHealthCheck(arg2);
+      (arg1)->setEnableDnslookup(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
@@ -19230,181 +19280,203 @@ SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self,
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
-    try {
-      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
-    }
-    catch (const std::exception& e) {
-      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  PyObject *swig_obj[1] ;
-  ppc::protocol::EndPoint *result = 0 ;
-  
-  (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
-  }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
+      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_maxSendMessageSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  uint64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_GrpcServerConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_maxReceivedMessageSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
+  uint64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_DISOWN |  0 );
-  if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
   }
-  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      delete arg1;
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_Py_Void();
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *GrpcServerConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  PyObject *obj;
-  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_NewClientData(obj));
-  return SWIG_Py_Void();
-}
-
-SWIGINTERN PyObject *GrpcServerConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
-  return SWIG_Python_InitShadowInstance(args);
-}
-
-SWIGINTERN PyObject *_wrap_new_GrpcConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxSendMessageSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *result = 0 ;
+  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
+  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "new_GrpcConfig", 0, 0, 0)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxSendMessageSize", 2, 2, swig_obj)) SWIG_fail;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
+      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
   {
     try {
-      result = (ppc::protocol::GrpcConfig *)new ppc::protocol::GrpcConfig();
+      (arg1)->setMaxSendMessageSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  {
-    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(result SWIG_NO_NULL_DELETER_SWIG_POINTER_NEW) : 0;
-    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_NEW | SWIG_POINTER_OWN);
-  }
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxReceivedMessageSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  uint64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxReceivedMessageSize", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_GrpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
@@ -19415,9 +19487,14 @@ SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
   {
     try {
-      (void)arg1; delete smartarg1;
+      (arg1)->setMaxReceivedMessageSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
@@ -19430,7 +19507,7 @@ SWIGINTERN PyObject *_wrap_delete_GrpcConfig(PyObject *self, PyObject *args) {
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_compressAlgorithm(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
@@ -19438,7 +19515,7 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject
   std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
   std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  std::string *result = 0 ;
+  int result;
   
   (void)self;
   if (!args) SWIG_fail;
@@ -19447,7 +19524,7 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_loadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_compressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
@@ -19460,37 +19537,38 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_loadBalancePolicy(PyObject *self, PyObject
   }
   {
     try {
-      result = (std::string *) &((ppc::protocol::GrpcConfig const *)arg1)->loadBalancePolicy();
+      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  resultobj = SWIG_From_int(static_cast< int >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setCompressAlgorithm(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  std::string *arg2 = 0 ;
+  int arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  int res2 = SWIG_OLDOBJ ;
+  int val2 ;
+  int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setLoadBalancePolicy", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setCompressAlgorithm", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
@@ -19501,35 +19579,27 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setLoadBalancePolicy(PyObject *self, PyObj
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  {
-    std::string *ptr = (std::string *)0;
-    res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    if (!ptr) {
-      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcConfig_setLoadBalancePolicy" "', argument " "2"" of type '" "std::string const &""'"); 
-    }
-    arg2 = ptr;
-  }
+  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "2"" of type '" "int""'");
+  } 
+  arg2 = static_cast< int >(val2);
   {
     try {
-      (arg1)->setLoadBalancePolicy((std::string const &)*arg2);
+      (arg1)->setCompressAlgorithm(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
   resultobj = SWIG_Py_Void();
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return resultobj;
 fail:
-  if (SWIG_IsNewObj(res2)) delete arg2;
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_maxMsgSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
   void *argp1 = 0 ;
@@ -19537,7 +19607,7 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject
   std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
   std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  uint64_t result;
   
   (void)self;
   if (!args) SWIG_fail;
@@ -19546,7 +19616,7 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxMsgSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
@@ -19559,38 +19629,38 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_enableHealthCheck(PyObject *self, PyObject
   }
   {
     try {
-      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableHealthCheck();
+      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxMsgSize();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxMsgSize(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  bool arg2 ;
+  uint64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
   std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  bool val2 ;
+  unsigned long long val2 ;
   int ecode2 = 0 ;
   PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxMsgSize", 2, 2, swig_obj)) SWIG_fail;
   {
     int newmem = 0;
     res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxMsgSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
     }
     if (newmem & SWIG_CAST_NEW_MEMORY) {
       tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
@@ -19601,14 +19671,14 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObj
       arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
     }
   }
-  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxMsgSize" "', argument " "2"" of type '" "uint64_t""'");
   } 
-  arg2 = static_cast< bool >(val2);
+  arg2 = static_cast< uint64_t >(val2);
   {
     try {
-      (arg1)->setEnableHealthCheck(arg2);
+      (arg1)->setMaxMsgSize(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
@@ -19621,382 +19691,366 @@ SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableHealthCheck(PyObject *self, PyObj
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setEnableDnslookup(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
+SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  return SWIG_Python_InitShadowInstance(args);
+}
+
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
+  
+  (void)self;
+  if ((nobjs < 0) || (nobjs > 0)) SWIG_fail;
+  {
+    try {
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) {
+  PyObject *resultobj = 0;
+  ppc::protocol::EndPoint arg1 ;
   bool arg2 ;
-  void *argp1 = 0 ;
+  void *argp1 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
   bool val2 ;
   int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  ppc::protocol::GrpcServerConfig *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setEnableDnslookup", 2, 2, swig_obj)) SWIG_fail;
+  if ((nobjs < 2) || (nobjs > 2)) SWIG_fail;
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
+    res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
     if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'"); 
+    }  
+    if (!argp1) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "new_GrpcServerConfig" "', argument " "1"" of type '" "ppc::protocol::EndPoint""'");
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp1);
+      arg1 = *temp;
+      if (SWIG_IsNewObj(res1)) delete temp;
     }
   }
   ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setEnableDnslookup" "', argument " "2"" of type '" "bool""'");
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_GrpcServerConfig" "', argument " "2"" of type '" "bool""'");
   } 
   arg2 = static_cast< bool >(val2);
   {
     try {
-      (arg1)->setEnableDnslookup(arg2);
+      result = (ppc::protocol::GrpcServerConfig *)new ppc::protocol::GrpcServerConfig(arg1,arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_Py_Void();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_POINTER_NEW |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_enableDnslookup(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_new_GrpcServerConfig(PyObject *self, PyObject *args) {
+  Py_ssize_t argc;
+  PyObject *argv[3] = {
+    0
+  };
+  
+  if (!(argc = SWIG_Python_UnpackTuple(args, "new_GrpcServerConfig", 0, 2, argv))) SWIG_fail;
+  --argc;
+  if (argc == 0) {
+    return _wrap_new_GrpcServerConfig__SWIG_0(self, argc, argv);
+  }
+  if (argc == 2) {
+    int _v = 0;
+    int res = SWIG_ConvertPtr(argv[0], 0, SWIGTYPE_p_ppc__protocol__EndPoint, SWIG_POINTER_NO_NULL | 0);
+    _v = SWIG_CheckState(res);
+    if (_v) {
+      {
+        int res = SWIG_AsVal_bool(argv[1], NULL);
+        _v = SWIG_CheckState(res);
+      }
+      if (_v) {
+        return _wrap_new_GrpcServerConfig__SWIG_1(self, argc, argv);
+      }
+    }
+  }
+  
+fail:
+  SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_GrpcServerConfig'.\n"
+    "  Possible C/C++ prototypes are:\n"
+    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig()\n"
+    "    ppc::protocol::GrpcServerConfig::GrpcServerConfig(ppc::protocol::EndPoint,bool)\n");
+  return 0;
+}
+
+
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_listenEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  bool result;
+  std::string result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_enableDnslookup" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_listenEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      result = (bool)((ppc::protocol::GrpcConfig const *)arg1)->enableDnslookup();
+      result = ((ppc::protocol::GrpcServerConfig const *)arg1)->listenEndPoint();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_bool(static_cast< bool >(result));
+  resultobj = SWIG_From_std_string(static_cast< std::string >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_maxSendMessageSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  ppc::protocol::EndPoint arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  uint64_t result;
+  void *argp2 ;
+  int res2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
+  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEndPoint", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
+  }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
+    res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_ppc__protocol__EndPoint,  0  | 0);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'"); 
+    }  
+    if (!argp2) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "GrpcServerConfig_setEndPoint" "', argument " "2"" of type '" "ppc::protocol::EndPoint""'");
     } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
+      ppc::protocol::EndPoint * temp = reinterpret_cast< ppc::protocol::EndPoint * >(argp2);
+      arg2 = *temp;
+      if (SWIG_IsNewObj(res2)) delete temp;
     }
   }
   {
     try {
-      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxSendMessageSize();
+      (arg1)->setEndPoint(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_maxReceivedMessageSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_setEnableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
+  bool arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
-  PyObject *swig_obj[1] ;
-  uint64_t result;
+  bool val2 ;
+  int ecode2 = 0 ;
+  PyObject *swig_obj[2] ;
   
   (void)self;
-  if (!args) SWIG_fail;
-  swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_maxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!SWIG_Python_UnpackTuple(args, "GrpcServerConfig_setEnableHealthCheck", 2, 2, swig_obj)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
+  ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcServerConfig_setEnableHealthCheck" "', argument " "2"" of type '" "bool""'");
+  } 
+  arg2 = static_cast< bool >(val2);
   {
     try {
-      result = (uint64_t)((ppc::protocol::GrpcConfig const *)arg1)->maxReceivedMessageSize();
+      (arg1)->setEnableHealthCheck(arg2);
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  resultobj = SWIG_Py_Void();
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxSendMessageSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_endPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  uint64_t arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  unsigned long long val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxSendMessageSize", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_endPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxSendMessageSize" "', argument " "2"" of type '" "uint64_t""'");
-  } 
-  arg2 = static_cast< uint64_t >(val2);
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      (arg1)->setMaxSendMessageSize(arg2);
+      result = (ppc::protocol::EndPoint *) &((ppc::protocol::GrpcServerConfig const *)arg1)->endPoint();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_Py_Void();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setMaxReceivedMessageSize(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_mutableEndPoint(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  uint64_t arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  unsigned long long val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::EndPoint *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setMaxReceivedMessageSize", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_mutableEndPoint" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig *""'"); 
   }
-  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setMaxReceivedMessageSize" "', argument " "2"" of type '" "uint64_t""'");
-  } 
-  arg2 = static_cast< uint64_t >(val2);
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      (arg1)->setMaxReceivedMessageSize(arg2);
+      result = (ppc::protocol::EndPoint *) &(arg1)->mutableEndPoint();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_Py_Void();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ppc__protocol__EndPoint, 0 |  0 );
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_compressAlgorithm(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_enableHealthCheck(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig const > *smartarg1 = 0 ;
   PyObject *swig_obj[1] ;
-  int result;
+  bool result;
   
   (void)self;
   if (!args) SWIG_fail;
   swig_obj[0] = args;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_compressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig const *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr< const ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_enableHealthCheck" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      result = (int)((ppc::protocol::GrpcConfig const *)arg1)->compressAlgorithm();
+      result = (bool)((ppc::protocol::GrpcServerConfig const *)arg1)->enableHealthCheck();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_From_int(static_cast< int >(result));
+  resultobj = SWIG_From_bool(static_cast< bool >(result));
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *_wrap_GrpcConfig_setCompressAlgorithm(PyObject *self, PyObject *args) {
+SWIGINTERN PyObject *_wrap_GrpcServerConfig_grpcConfig(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
-  ppc::protocol::GrpcConfig *arg1 = (ppc::protocol::GrpcConfig *) 0 ;
-  int arg2 ;
+  ppc::protocol::GrpcServerConfig *arg1 = (ppc::protocol::GrpcServerConfig *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > tempshared1 ;
-  std::shared_ptr< ppc::protocol::GrpcConfig > *smartarg1 = 0 ;
-  int val2 ;
-  int ecode2 = 0 ;
-  PyObject *swig_obj[2] ;
+  PyObject *swig_obj[1] ;
+  ppc::protocol::GrpcConfig::Ptr *result = 0 ;
   
   (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "GrpcConfig_setCompressAlgorithm", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "1"" of type '" "ppc::protocol::GrpcConfig *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::GrpcConfig > * >(argp1);
-      arg1 = const_cast< ppc::protocol::GrpcConfig * >((smartarg1 ? smartarg1->get() : 0));
-    }
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ppc__protocol__GrpcServerConfig, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "GrpcServerConfig_grpcConfig" "', argument " "1"" of type '" "ppc::protocol::GrpcServerConfig const *""'"); 
   }
-  ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
-  if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "GrpcConfig_setCompressAlgorithm" "', argument " "2"" of type '" "int""'");
-  } 
-  arg2 = static_cast< int >(val2);
+  arg1 = reinterpret_cast< ppc::protocol::GrpcServerConfig * >(argp1);
   {
     try {
-      (arg1)->setCompressAlgorithm(arg2);
+      result = (ppc::protocol::GrpcConfig::Ptr *) &((ppc::protocol::GrpcServerConfig const *)arg1)->grpcConfig();
     }
     catch (const std::exception& e) {
       SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
     }
   }
-  resultobj = SWIG_Py_Void();
+  {
+    std::shared_ptr<  ppc::protocol::GrpcConfig > *smartresult = *result ? new std::shared_ptr<  ppc::protocol::GrpcConfig >(*result) : 0;
+    resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_POINTER_OWN);
+  }
   return resultobj;
 fail:
   return NULL;
 }
 
 
-SWIGINTERN PyObject *GrpcConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *GrpcServerConfig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
-  SWIG_TypeNewClientData(SWIGTYPE_p_std__shared_ptrT_ppc__protocol__GrpcConfig_t, SWIG_NewClientData(obj));
+  SWIG_TypeNewClientData(SWIGTYPE_p_ppc__protocol__GrpcServerConfig, SWIG_NewClientData(obj));
   return SWIG_Py_Void();
 }
 
-SWIGINTERN PyObject *GrpcConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *GrpcServerConfig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   return SWIG_Python_InitShadowInstance(args);
 }
 
@@ -22929,59 +22983,6 @@ SWIGINTERN PyObject *_wrap_Message_payloadBuffer(PyObject *self, PyObject *args)
 }
 
 
-SWIGINTERN PyObject *_wrap_Message_setFrontMessage(PyObject *self, PyObject *args) {
-  PyObject *resultobj = 0;
-  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
-  ppc::protocol::MessagePayload::Ptr arg2 ;
-  void *argp1 = 0 ;
-  int res1 = 0 ;
-  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
-  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
-  void *argp2 ;
-  int res2 = 0 ;
-  PyObject *swig_obj[2] ;
-  
-  (void)self;
-  if (!SWIG_Python_UnpackTuple(args, "Message_setFrontMessage", 2, 2, swig_obj)) SWIG_fail;
-  {
-    int newmem = 0;
-    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
-    if (!SWIG_IsOK(res1)) {
-      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_setFrontMessage" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
-    }
-    if (newmem & SWIG_CAST_NEW_MEMORY) {
-      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
-    } else {
-      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
-      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
-    }
-  }
-  {
-    int newmem = 0;
-    res2 = SWIG_ConvertPtrAndOwn(swig_obj[1], &argp2, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t,  0 , &newmem);
-    if (!SWIG_IsOK(res2)) {
-      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Message_setFrontMessage" "', argument " "2"" of type '" "ppc::protocol::MessagePayload::Ptr""'");
-    }
-    if (argp2) arg2 = *(reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2));
-    if (newmem & SWIG_CAST_NEW_MEMORY) delete reinterpret_cast< ppc::protocol::MessagePayload::Ptr * >(argp2);
-  }
-  {
-    try {
-      (arg1)->setFrontMessage(arg2);
-    }
-    catch (const std::exception& e) {
-      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
-    }
-  }
-  resultobj = SWIG_Py_Void();
-  return resultobj;
-fail:
-  return NULL;
-}
-
-
 SWIGINTERN PyObject *_wrap_Message_frontMessage(PyObject *self, PyObject *args) {
   PyObject *resultobj = 0;
   ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
@@ -23238,6 +23239,48 @@ SWIGINTERN PyObject *_wrap_Message_payload(PyObject *self, PyObject *args) {
 }
 
 
+SWIGINTERN PyObject *_wrap_Message_releasePayload(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::Message *arg1 = (ppc::protocol::Message *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::Message > tempshared1 ;
+  std::shared_ptr< ppc::protocol::Message > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__Message_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Message_releasePayload" "', argument " "1"" of type '" "ppc::protocol::Message *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::Message > * >(argp1);
+      arg1 = const_cast< ppc::protocol::Message * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    try {
+      (arg1)->releasePayload();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *Message_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@@ -24799,6 +24842,48 @@ SWIGINTERN PyObject *_wrap_MessagePayload_isRespPacket(PyObject *self, PyObject
 }
 
 
+SWIGINTERN PyObject *_wrap_MessagePayload_releasePayload(PyObject *self, PyObject *args) {
+  PyObject *resultobj = 0;
+  ppc::protocol::MessagePayload *arg1 = (ppc::protocol::MessagePayload *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > tempshared1 ;
+  std::shared_ptr< ppc::protocol::MessagePayload > *smartarg1 = 0 ;
+  PyObject *swig_obj[1] ;
+  
+  (void)self;
+  if (!args) SWIG_fail;
+  swig_obj[0] = args;
+  {
+    int newmem = 0;
+    res1 = SWIG_ConvertPtrAndOwn(swig_obj[0], &argp1, SWIGTYPE_p_std__shared_ptrT_ppc__protocol__MessagePayload_t, 0 |  0 , &newmem);
+    if (!SWIG_IsOK(res1)) {
+      SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "MessagePayload_releasePayload" "', argument " "1"" of type '" "ppc::protocol::MessagePayload *""'");
+    }
+    if (newmem & SWIG_CAST_NEW_MEMORY) {
+      tempshared1 = *reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      delete reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >(tempshared1.get());
+    } else {
+      smartarg1 = reinterpret_cast< std::shared_ptr<  ppc::protocol::MessagePayload > * >(argp1);
+      arg1 = const_cast< ppc::protocol::MessagePayload * >((smartarg1 ? smartarg1->get() : 0));
+    }
+  }
+  {
+    try {
+      (arg1)->releasePayload();
+    }
+    catch (const std::exception& e) {
+      SWIG_exception(SWIG_RuntimeError, std::string(boost::diagnostic_information(e)).c_str());
+    }
+  }
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *MessagePayload_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
@@ -29175,18 +29260,7 @@ static PyMethodDef SwigMethods[] = {
 	 { "EndPoint_listenIp", _wrap_EndPoint_listenIp, METH_O, NULL},
 	 { "EndPoint_swigregister", EndPoint_swigregister, METH_O, NULL},
 	 { "EndPoint_swiginit", EndPoint_swiginit, METH_VARARGS, NULL},
-	 { "new_GrpcServerConfig", _wrap_new_GrpcServerConfig, METH_VARARGS, NULL},
-	 { "GrpcServerConfig_listenEndPoint", _wrap_GrpcServerConfig_listenEndPoint, METH_O, NULL},
-	 { "GrpcServerConfig_setEndPoint", _wrap_GrpcServerConfig_setEndPoint, METH_VARARGS, NULL},
-	 { "GrpcServerConfig_setEnableHealthCheck", _wrap_GrpcServerConfig_setEnableHealthCheck, METH_VARARGS, NULL},
-	 { "GrpcServerConfig_endPoint", _wrap_GrpcServerConfig_endPoint, METH_O, NULL},
-	 { "GrpcServerConfig_mutableEndPoint", _wrap_GrpcServerConfig_mutableEndPoint, METH_O, NULL},
-	 { "GrpcServerConfig_enableHealthCheck", _wrap_GrpcServerConfig_enableHealthCheck, METH_O, NULL},
-	 { "delete_GrpcServerConfig", _wrap_delete_GrpcServerConfig, METH_O, NULL},
-	 { "GrpcServerConfig_swigregister", GrpcServerConfig_swigregister, METH_O, NULL},
-	 { "GrpcServerConfig_swiginit", GrpcServerConfig_swiginit, METH_VARARGS, NULL},
 	 { "new_GrpcConfig", _wrap_new_GrpcConfig, METH_NOARGS, NULL},
-	 { "delete_GrpcConfig", _wrap_delete_GrpcConfig, METH_O, NULL},
 	 { "GrpcConfig_loadBalancePolicy", _wrap_GrpcConfig_loadBalancePolicy, METH_O, NULL},
 	 { "GrpcConfig_setLoadBalancePolicy", _wrap_GrpcConfig_setLoadBalancePolicy, METH_VARARGS, NULL},
 	 { "GrpcConfig_enableHealthCheck", _wrap_GrpcConfig_enableHealthCheck, METH_O, NULL},
@@ -29199,8 +29273,20 @@ static PyMethodDef SwigMethods[] = {
 	 { "GrpcConfig_setMaxReceivedMessageSize", _wrap_GrpcConfig_setMaxReceivedMessageSize, METH_VARARGS, NULL},
 	 { "GrpcConfig_compressAlgorithm", _wrap_GrpcConfig_compressAlgorithm, METH_O, NULL},
 	 { "GrpcConfig_setCompressAlgorithm", _wrap_GrpcConfig_setCompressAlgorithm, METH_VARARGS, NULL},
+	 { "GrpcConfig_maxMsgSize", _wrap_GrpcConfig_maxMsgSize, METH_O, NULL},
+	 { "GrpcConfig_setMaxMsgSize", _wrap_GrpcConfig_setMaxMsgSize, METH_VARARGS, NULL},
 	 { "GrpcConfig_swigregister", GrpcConfig_swigregister, METH_O, NULL},
 	 { "GrpcConfig_swiginit", GrpcConfig_swiginit, METH_VARARGS, NULL},
+	 { "new_GrpcServerConfig", _wrap_new_GrpcServerConfig, METH_VARARGS, NULL},
+	 { "GrpcServerConfig_listenEndPoint", _wrap_GrpcServerConfig_listenEndPoint, METH_O, NULL},
+	 { "GrpcServerConfig_setEndPoint", _wrap_GrpcServerConfig_setEndPoint, METH_VARARGS, NULL},
+	 { "GrpcServerConfig_setEnableHealthCheck", _wrap_GrpcServerConfig_setEnableHealthCheck, METH_VARARGS, NULL},
+	 { "GrpcServerConfig_endPoint", _wrap_GrpcServerConfig_endPoint, METH_O, NULL},
+	 { "GrpcServerConfig_mutableEndPoint", _wrap_GrpcServerConfig_mutableEndPoint, METH_O, NULL},
+	 { "GrpcServerConfig_enableHealthCheck", _wrap_GrpcServerConfig_enableHealthCheck, METH_O, NULL},
+	 { "GrpcServerConfig_grpcConfig", _wrap_GrpcServerConfig_grpcConfig, METH_O, NULL},
+	 { "GrpcServerConfig_swigregister", GrpcServerConfig_swigregister, METH_O, NULL},
+	 { "GrpcServerConfig_swiginit", GrpcServerConfig_swiginit, METH_VARARGS, NULL},
 	 { "printGrpcConfig", _wrap_printGrpcConfig, METH_O, NULL},
 	 { "delete_MessageOptionalHeader", _wrap_delete_MessageOptionalHeader, METH_O, NULL},
 	 { "MessageOptionalHeader_encode", _wrap_MessageOptionalHeader_encode, METH_VARARGS, NULL},
@@ -29261,12 +29347,12 @@ static PyMethodDef SwigMethods[] = {
 	 { "Message_setRespPacket", _wrap_Message_setRespPacket, METH_O, NULL},
 	 { "Message_setPayload", _wrap_Message_setPayload, METH_VARARGS, NULL},
 	 { "Message_payloadBuffer", _wrap_Message_payloadBuffer, METH_O, NULL},
-	 { "Message_setFrontMessage", _wrap_Message_setFrontMessage, METH_VARARGS, NULL},
 	 { "Message_frontMessage", _wrap_Message_frontMessage, METH_O, NULL},
 	 { "Message_encode", _wrap_Message_encode, METH_VARARGS, NULL},
 	 { "Message_decode", _wrap_Message_decode, METH_VARARGS, NULL},
 	 { "Message_length", _wrap_Message_length, METH_O, NULL},
 	 { "Message_payload", _wrap_Message_payload, METH_O, NULL},
+	 { "Message_releasePayload", _wrap_Message_releasePayload, METH_O, NULL},
 	 { "Message_swigregister", Message_swigregister, METH_O, NULL},
 	 { "delete_MessageHeaderBuilder", _wrap_delete_MessageHeaderBuilder, METH_O, NULL},
 	 { "MessageHeaderBuilder_build", _wrap_MessageHeaderBuilder_build, METH_VARARGS, NULL},
@@ -29294,6 +29380,7 @@ static PyMethodDef SwigMethods[] = {
 	 { "MessagePayload_setExt", _wrap_MessagePayload_setExt, METH_VARARGS, NULL},
 	 { "MessagePayload_setRespPacket", _wrap_MessagePayload_setRespPacket, METH_O, NULL},
 	 { "MessagePayload_isRespPacket", _wrap_MessagePayload_isRespPacket, METH_O, NULL},
+	 { "MessagePayload_releasePayload", _wrap_MessagePayload_releasePayload, METH_O, NULL},
 	 { "MessagePayload_swigregister", MessagePayload_swigregister, METH_O, NULL},
 	 { "delete_MessagePayloadBuilder", _wrap_delete_MessagePayloadBuilder, METH_O, NULL},
 	 { "MessagePayloadBuilder_build", _wrap_MessagePayloadBuilder_build, METH_VARARGS, NULL},
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
index 59c6ba39..83be7ca0 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
+++ b/cpp/wedpr-transport/sdk-wrapper/python/swig/wedpr_python_transport.i
@@ -41,6 +41,7 @@ from wedpr_python_gateway_sdk.libs import _wedpr_python_transport
 %shared_ptr(ppc::protocol::GrpcConfig);
 %shared_ptr(ppc::sdk::Transport);
 
+
 %{
 #define SWIG_FILE_WITH_INIT
 #include 
@@ -192,6 +193,10 @@ namespace bcos{
 %ignore ppc::protocol::INodeInfo::toJson;
 %ignore ppc::protocol::INodeInfo::setComponents;
 %ignore ppc::protocol::INodeInfoFactory;
+%ignore ppc::protocol::Message::setFrontMessage;
+
+%ignore ppc::protocol::GrpcConfig::~GrpcConfig;
+%ignore ppc::protocol::GrpcServerConfig::~GrpcServerConfig;
 /*
 ///// tests  ///
 %inline {

From 1f887ea0092e0f3e8306afff65042dcee00418ac Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 18 Nov 2024 16:34:06 +0800
Subject: [PATCH 091/120] optimize log (#91)

---
 .../ppc-pir/src/CMakeLists.txt                |   2 +-
 .../src/ecdh-multi-psi/EcdhMultiCache.cpp     |   5 +-
 .../src/ecdh-multi-psi/EcdhMultiCache.h       |   6 +-
 .../core/EcdhMultiPSICalculator.cpp           |   6 +-
 .../core/EcdhMultiPSIMaster.cpp               |   5 +-
 .../core/EcdhMultiPSIPartner.cpp              |  12 +-
 .../ppc-psi/src/psi-framework/TaskGuarder.cpp | 273 ++++++++++++++
 .../ppc-psi/src/psi-framework/TaskGuarder.h   | 244 +------------
 .../ppc-psi/src/psi-framework/TaskState.cpp   | 336 ++++++++++++++++++
 .../ppc-psi/src/psi-framework/TaskState.h     | 315 +---------------
 10 files changed, 648 insertions(+), 556 deletions(-)
 create mode 100644 cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
 create mode 100644 cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp

diff --git a/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
index 5f73920e..03844ecb 100644
--- a/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
@@ -28,4 +28,4 @@ add_library(${PIR_TARGET} ${SOURCES} ${OUT_TARS_H_LIST})
 target_include_directories(${PIR_TARGET} PUBLIC
         $)
 
-target_link_libraries(${PIR_TARGET} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
+target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
index 02bde0e2..abf466d9 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
@@ -328,7 +328,7 @@ bool CalculatorCache::tryToFinalize()
                          << LOG_KV("* intersectionSize", m_intersectionResult.size())
                          << LOG_KV("* timecost", (utcSteadyTime() - startT));
 
-    releaseDataAfterFinalize();
+    releaseCipherCache();
     ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize: syncIntersections") << printCacheState();
     m_cacheState = CacheState::Syncing;
     syncIntersections();
@@ -472,8 +472,9 @@ void CalculatorCache::addIntersectionCipher(std::vector&& _cipherDa
     {
         m_receiveAllIntersection = true;
     }
-    ECDH_MULTI_LOG(INFO) << LOG_DESC("addIntersectionCipher finshed")
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("addIntersectionCipher success")
                          << LOG_KV("timecost", utcSteadyTime() - startT) << LOG_KV("seq", seq)
+                         << LOG_KV("dataSize", _cipherData.size())
                          << LOG_KV("receiveAll", m_receiveAllIntersection)
                          << LOG_KV("cipherRefSize", m_cipherRef.size()) << printCacheState();
     // release the cipherData
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
index f3a9e416..a127630f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
@@ -221,7 +221,7 @@ class CalculatorCache : public std::enable_shared_from_this
     {}
     virtual ~CalculatorCache()
     {
-        releaseDataAfterFinalize();
+        releaseCipherCache();
 
         m_intersectionResult.clear();
         std::vector().swap(m_intersectionResult);
@@ -275,7 +275,7 @@ class CalculatorCache : public std::enable_shared_from_this
 
     void syncIntersections();
 
-    void releaseDataAfterFinalize()
+    void releaseCipherCache()
     {
         for (auto const& it : m_plainData)
         {
@@ -284,7 +284,7 @@ class CalculatorCache : public std::enable_shared_from_this
         m_cipherRef.clear();
         std::map().swap(m_cipherRef);
         MallocExtension::instance()->ReleaseFreeMemory();
-        ECDH_MULTI_LOG(INFO) << LOG_DESC("releaseDataAfterFinalize")
+        ECDH_MULTI_LOG(INFO) << LOG_DESC("releaseCipherCache")
                              << LOG_KV("taskID", m_taskState->task()->id());
     }
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
index c63fe38e..107b0e29 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
@@ -123,12 +123,12 @@ void EcdhMultiPSICalculator::blindData(std::string _taskID, bcos::bytes _randA)
                         encryptedData[i] = m_config->eccCrypto()->ecMultiply(point, _randA);
                     }
                 });
-            ECDH_CAL_LOG(INFO) << LOG_DESC("blindData encrypt success")
+            ECDH_CAL_LOG(INFO) << LOG_DESC("blindData encrypt success") << LOG_KV("seq", seq)
                                << LOG_KV("dataSize", encryptedData.size())
-                               << LOG_KV("task", m_taskState->task()->id()) << LOG_KV("seq", seq)
+                               << LOG_KV("task", m_taskState->task()->id())
                                << LOG_KV("timecost", (utcSteadyTime() - startT));
             ECDH_CAL_LOG(INFO) << LOG_DESC("blindData: send cipher to the master")
-                               << LOG_KV("masterSize", m_masterParties.size())
+                               << LOG_KV("seq", seq) << LOG_KV("masterSize", m_masterParties.size())
                                << LOG_KV("dataSize", encryptedData.size())
                                << LOG_KV("task", m_taskState->task()->id());
             auto message = m_config->psiMsgFactory()->createPSIMessage(
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index 8da588d4..f21680f8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -147,7 +147,8 @@ void EcdhMultiPSIMaster::blindData()
                 }
             }
             auto startT = utcSteadyTime();
-            ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data")
+            ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data") << LOG_KV("seq", seq)
+                                  << LOG_KV("task", m_taskState->task()->id())
                                   << LOG_KV("dataSize", dataBatch->size());
             std::vector cipher(dataBatch->size());
             tbb::parallel_for(
@@ -169,7 +170,7 @@ void EcdhMultiPSIMaster::blindData()
                                   << LOG_KV("timecost", (utcSteadyTime() - startT));
 
             ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: send encrypted data to all calculator")
-                                  << LOG_KV("task", m_taskID)
+                                  << LOG_KV("seq", seq) << LOG_KV("task", m_taskID)
                                   << LOG_KV("calculators", m_calculatorParties.size());
             auto message = m_config->psiMsgFactory()->createPSIMessage(
                 uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_CALCULATOR));
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
index cab0d0c1..e52f626c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
@@ -95,14 +95,14 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
                         cipherData[i] = m_config->eccCrypto()->ecMultiply(point, *_randA);
                     }
                 });
-            ECDH_PARTNER_LOG(INFO) << LOG_DESC("blindData: encode parterner cipher success")
-                                   << LOG_KV("size", dataBatch->size())
-                                   << LOG_KV("timecost", (utcSteadyTime() - startT))
-                                   << printTaskInfo(m_taskState->task());
+            ECDH_PARTNER_LOG(INFO)
+                << LOG_DESC("blindData: encode parterner cipher success") << LOG_KV("seq", seq)
+                << LOG_KV("task", m_taskState->task()->id()) << LOG_KV("size", dataBatch->size())
+                << LOG_KV("timecost", (utcSteadyTime() - startT));
 
             ECDH_PARTNER_LOG(INFO)
-                << LOG_DESC("blindData: send cipher data to master")
-                << LOG_KV("size", dataBatch->size()) << printTaskInfo(m_taskState->task());
+                << LOG_DESC("blindData: send cipher data to master") << LOG_KV("seq", seq)
+                << LOG_KV("size", dataBatch->size()) << LOG_KV("task", m_taskState->task()->id());
             auto message = m_config->psiMsgFactory()->createPSIMessage(
                 uint32_t(EcdhMultiPSIMessageType::SEND_ENCRYPTED_SET_TO_MASTER_FROM_PARTNER));
             message->setData(std::move(cipherData));
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
new file mode 100644
index 00000000..07cf6732
--- /dev/null
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
@@ -0,0 +1,273 @@
+/**
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file TaskGuarder.cpp
+ * @author: shawnhe
+ * @date 2022-01-07
+ */
+#include "TaskGuarder.h"
+
+using namespace ppc::psi;
+using namespace ppc::protocol;
+using namespace ppc::io;
+
+
+bcos::Error::Ptr TaskGuarder::checkTask(const Task::ConstPtr& _task, uint16_t _partiesCount,
+    bool _enforceSelfInput, bool _enforceSelfOutput, bool _enforcePeerResource,
+    bool _enforceSelfResource)
+{
+    {
+        // check task id
+        bcos::ReadGuard l(x_pendingTasks);
+        if (m_pendingTasks.contains(_task->id()))
+        {
+            return std::make_shared(
+                (int)PSIRetCode::DuplicatedTask, "task already exists");
+        }
+    }
+
+    // check self party
+    auto const& selfParty = _task->selfParty();
+    if (!selfParty)
+    {
+        return std::make_shared(
+            (int)PSIRetCode::TaskParamsError, "must specify self party info");
+    }
+    if (selfParty->partyIndex() == (uint16_t)protocol::PartyType::Client)
+    {
+        _enforceSelfInput = true;
+        _enforceSelfOutput = true;
+    }
+
+    // check self data resource
+    if (_enforceSelfResource)
+    {
+        auto const& dataResource = selfParty->dataResource();
+        if (!dataResource)
+        {
+            return std::make_shared(
+                (int)PSIRetCode::TaskParamsError, "no data resource specified for self party");
+        }
+        auto originData = dataResource->rawData();
+        if (_enforceSelfInput && originData.empty() && !dataResource->desc())
+        {
+            return std::make_shared(
+                (int)PSIRetCode::TaskParamsError, "no input resource specified for self party");
+        }
+        if (_enforceSelfOutput && originData.empty() && !dataResource->outputDesc())
+        {
+            return std::make_shared(
+                (int)PSIRetCode::TaskParamsError, "no output resource specified for self party");
+        }
+    }
+
+    // check party index
+    std::set indexSet;
+    auto index = selfParty->partyIndex();
+    if (_partiesCount > 1 && index >= _partiesCount)
+    {
+        return std::make_shared(
+            (int)PSIRetCode::TaskParamsError, "invalid partyIndex: " + std::to_string(index));
+    }
+    indexSet.insert(index);
+
+    // check the peer size
+    auto const& peerParties = _task->getAllPeerParties();
+    if (peerParties.size() != uint(_partiesCount - 1))
+    {
+        std::string errorMessage = "expected parties count: " + std::to_string(_partiesCount) +
+                                   ", current is " + std::to_string(peerParties.size());
+        return std::make_shared((int)PSIRetCode::TaskParamsError, errorMessage);
+    }
+
+    // check the party index of peers
+    for (auto& it : peerParties)
+    {
+        index = it.second->partyIndex();
+        if (_partiesCount > 1 && index >= _partiesCount)
+        {
+            return std::make_shared(
+                (int)PSIRetCode::TaskParamsError, "invalid partyIndex: " + std::to_string(index));
+        }
+        if (indexSet.contains(index))
+        {
+            return std::make_shared(
+                (int)PSIRetCode::TaskParamsError, "repeated party index: " + std::to_string(index));
+        }
+        indexSet.insert(index);
+    }
+
+    // check the data resource of peer
+    if (!_enforcePeerResource)
+    {
+        return nullptr;
+    }
+    for (auto& it : peerParties)
+    {
+        if (!it.second->dataResource())
+        {
+            return std::make_shared(
+                (int)PSIRetCode::NotSpecifyPeerDataResource, "must specify the peer data resource");
+        }
+    }
+    return nullptr;
+}
+
+
+void TaskGuarder::noticePeerToFinish(const Task::ConstPtr& _task)
+{
+    auto const& peerParties = _task->getAllPeerParties();
+    if (peerParties.empty())
+    {
+        return;
+    }
+
+    for (const auto& peer : peerParties)
+    {
+        noticePeerToFinish(_task->id(), peer.first);
+    }
+}
+
+void TaskGuarder::noticePeerToFinish(const std::string& _taskID, const std::string& _peer)
+{
+    PSI_LOG(INFO) << LOG_DESC("noticePeerToFinish") << LOG_KV("task", _taskID)
+                  << LOG_KV("peer", _peer);
+    try
+    {
+        if (_peer.empty())
+        {
+            return;
+        }
+
+        auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
+            uint8_t(m_type), _taskID, std::make_shared());
+        message->setMessageType(uint8_t(CommonMessageType::ErrorNotification));
+        m_config->front()->asyncSendMessage(
+            _peer, message, m_config->networkTimeout(), [](const bcos::Error::Ptr&) {}, nullptr);
+    }
+    catch (std::exception& e)
+    {
+        PSI_LOG(ERROR) << LOG_DESC("noticePeerToFinish")
+                       << LOG_KV("exception", boost::diagnostic_information(e));
+    }
+}
+
+void TaskGuarder::checkPeerActivity()
+{
+    bcos::ReadGuard l(x_pendingTasks);
+    for (auto const& it : m_pendingTasks)
+    {
+        if (it.second->onlySelfRun())
+        {
+            // no need to check peer
+            continue;
+        }
+
+        auto task = it.second->task();
+        auto const& peerParties = task->getAllPeerParties();
+        if (peerParties.empty())
+        {
+            continue;
+        }
+
+        for (const auto& peer : peerParties)
+        {
+            auto message =
+                m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
+                    uint8_t(m_type), task->id(), std::make_shared());
+            message->setMessageType(uint8_t(CommonMessageType::PingPeer));
+            m_config->front()->asyncSendMessage(
+                peer.first, message, m_config->networkTimeout(),
+                [this, task, peerID = peer.first](bcos::Error::Ptr&& _error) {
+                    if (!_error || _error->errorCode() == 0)
+                    {
+                        return;
+                    }
+                    onSelfError(task->id(),
+                        std::make_shared(
+                            (int)PSIRetCode::PeerNodeDown, "peer node is down, id: " + peerID),
+                        false);
+                },
+                nullptr);
+        }
+    }
+    m_pingTimer->restart();
+}
+
+
+LineReader::Ptr TaskGuarder::loadReader(std::string const& _taskID,
+    protocol::DataResource::ConstPtr const& _dataResource, DataSchema _dataSchema,
+    uint32_t _columnSize)
+{
+    if (!_dataResource->rawData().empty())
+    {
+        return nullptr;
+    }
+    auto reader =
+        m_config->dataResourceLoader()->loadReader(_dataResource->desc(), _dataSchema, true);
+
+    if (reader->columnSize() == 0 ||
+        (reader->type() == DataResourceType::MySQL && reader->columnSize() != _columnSize))
+    {
+        auto errorMsg = "load data for task " + _taskID + "failed, expect " +
+                        std::to_string(_columnSize) + " column, current column size is " +
+                        std::to_string(reader->columnSize());
+        BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::LoadDataFailed, errorMsg));
+    }
+    return reader;
+}
+
+LineWriter::Ptr TaskGuarder::loadWriter(std::string const& _taskID,
+    DataResource::ConstPtr const& _dataResource, bool _enableOutputExists)
+{
+    if (!_dataResource->rawData().empty())
+    {
+        return nullptr;
+    }
+    if (!_enableOutputExists)
+    {
+        // Note: if the output-resource already exists, will throw exception
+        m_config->dataResourceLoader()->checkResourceExists(_dataResource->outputDesc());
+    }
+    return m_config->dataResourceLoader()->loadWriter(_dataResource->outputDesc(), true);
+}
+
+TaskState::Ptr TaskGuarder::findPendingTask(const std::string& _taskID)
+{
+    bcos::ReadGuard l(x_pendingTasks);
+    auto it = m_pendingTasks.find(_taskID);
+    if (it == m_pendingTasks.end())
+    {
+        return nullptr;
+    }
+    return it->second;
+}
+
+void TaskGuarder::addPendingTask(TaskState::Ptr _taskState)
+{
+    bcos::WriteGuard l(x_pendingTasks);
+    auto id = _taskState->task()->id();
+    m_pendingTasks[id] = std::move(_taskState);
+}
+
+void TaskGuarder::removePendingTask(const std::string& _taskID)
+{
+    bcos::WriteGuard l(x_pendingTasks);
+    auto it = m_pendingTasks.find(_taskID);
+    if (it != m_pendingTasks.end())
+    {
+        m_pendingTasks.erase(it);
+    }
+}
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
index c7da3a68..c64c0604 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
@@ -63,104 +63,7 @@ class TaskGuarder
 
     bcos::Error::Ptr checkTask(const ppc::protocol::Task::ConstPtr& _task, uint16_t _partiesCount,
         bool _enforceSelfInput, bool _enforceSelfOutput, bool _enforcePeerResource,
-        bool _enforceSelfResource = true)
-    {
-        {
-            // check task id
-            bcos::ReadGuard l(x_pendingTasks);
-            if (m_pendingTasks.contains(_task->id()))
-            {
-                return std::make_shared(
-                    (int)PSIRetCode::DuplicatedTask, "task already exists");
-            }
-        }
-
-        // check self party
-        auto const& selfParty = _task->selfParty();
-        if (!selfParty)
-        {
-            return std::make_shared(
-                (int)PSIRetCode::TaskParamsError, "must specify self party info");
-        }
-        if (selfParty->partyIndex() == (uint16_t)protocol::PartyType::Client)
-        {
-            _enforceSelfInput = true;
-            _enforceSelfOutput = true;
-        }
-
-        // check self data resource
-        if (_enforceSelfResource)
-        {
-            auto const& dataResource = selfParty->dataResource();
-            if (!dataResource)
-            {
-                return std::make_shared(
-                    (int)PSIRetCode::TaskParamsError, "no data resource specified for self party");
-            }
-            auto originData = dataResource->rawData();
-            if (_enforceSelfInput && originData.empty() && !dataResource->desc())
-            {
-                return std::make_shared(
-                    (int)PSIRetCode::TaskParamsError, "no input resource specified for self party");
-            }
-            if (_enforceSelfOutput && originData.empty() && !dataResource->outputDesc())
-            {
-                return std::make_shared((int)PSIRetCode::TaskParamsError,
-                    "no output resource specified for self party");
-            }
-        }
-
-        // check party index
-        std::set indexSet;
-        auto index = selfParty->partyIndex();
-        if (_partiesCount > 1 && index >= _partiesCount)
-        {
-            return std::make_shared(
-                (int)PSIRetCode::TaskParamsError, "invalid partyIndex: " + std::to_string(index));
-        }
-        indexSet.insert(index);
-
-        // check the peer size
-        auto const& peerParties = _task->getAllPeerParties();
-        if (peerParties.size() != uint(_partiesCount - 1))
-        {
-            std::string errorMessage = "expected parties count: " + std::to_string(_partiesCount) +
-                                       ", current is " + std::to_string(peerParties.size());
-            return std::make_shared((int)PSIRetCode::TaskParamsError, errorMessage);
-        }
-
-        // check the party index of peers
-        for (auto& it : peerParties)
-        {
-            index = it.second->partyIndex();
-            if (_partiesCount > 1 && index >= _partiesCount)
-            {
-                return std::make_shared((int)PSIRetCode::TaskParamsError,
-                    "invalid partyIndex: " + std::to_string(index));
-            }
-            if (indexSet.contains(index))
-            {
-                return std::make_shared((int)PSIRetCode::TaskParamsError,
-                    "repeated party index: " + std::to_string(index));
-            }
-            indexSet.insert(index);
-        }
-
-        // check the data resource of peer
-        if (!_enforcePeerResource)
-        {
-            return nullptr;
-        }
-        for (auto& it : peerParties)
-        {
-            if (!it.second->dataResource())
-            {
-                return std::make_shared((int)PSIRetCode::NotSpecifyPeerDataResource,
-                    "must specify the peer data resource");
-            }
-        }
-        return nullptr;
-    }
+        bool _enforceSelfResource = true);
 
     // this only work for two-party task
     static std::string getPeerID(ppc::protocol::Task::ConstPtr _task)
@@ -173,152 +76,23 @@ class TaskGuarder
         return peerParties.begin()->second->id();
     }
 
-    void noticePeerToFinish(const ppc::protocol::Task::ConstPtr& _task)
-    {
-        auto const& peerParties = _task->getAllPeerParties();
-        if (peerParties.empty())
-        {
-            return;
-        }
+    void noticePeerToFinish(const ppc::protocol::Task::ConstPtr& _task);
+    void noticePeerToFinish(const std::string& _taskID, const std::string& _peer);
 
-        for (const auto& peer : peerParties)
-        {
-            noticePeerToFinish(_task->id(), peer.first);
-        }
-    }
-
-    void noticePeerToFinish(const std::string& _taskID, const std::string& _peer)
-    {
-        PSI_LOG(INFO) << LOG_DESC("noticePeerToFinish") << LOG_KV("task", _taskID)
-                      << LOG_KV("peer", _peer);
-        try
-        {
-            if (_peer.empty())
-            {
-                return;
-            }
-
-            auto message =
-                m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-                    uint8_t(m_type), _taskID, std::make_shared());
-            message->setMessageType(uint8_t(CommonMessageType::ErrorNotification));
-            m_config->front()->asyncSendMessage(
-                _peer, message, m_config->networkTimeout(), [](const bcos::Error::Ptr&) {},
-                nullptr);
-        }
-        catch (std::exception& e)
-        {
-            PSI_LOG(ERROR) << LOG_DESC("noticePeerToFinish")
-                           << LOG_KV("exception", boost::diagnostic_information(e));
-        }
-    }
-
-    void checkPeerActivity()
-    {
-        bcos::ReadGuard l(x_pendingTasks);
-        for (auto const& it : m_pendingTasks)
-        {
-            if (it.second->onlySelfRun())
-            {
-                // no need to check peer
-                continue;
-            }
-
-            auto task = it.second->task();
-            auto const& peerParties = task->getAllPeerParties();
-            if (peerParties.empty())
-            {
-                continue;
-            }
-
-            for (const auto& peer : peerParties)
-            {
-                auto message =
-                    m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(protocol::TaskType::PSI),
-                        uint8_t(m_type), task->id(), std::make_shared());
-                message->setMessageType(uint8_t(CommonMessageType::PingPeer));
-                m_config->front()->asyncSendMessage(
-                    peer.first, message, m_config->networkTimeout(),
-                    [this, task, peerID = peer.first](bcos::Error::Ptr&& _error) {
-                        if (!_error || _error->errorCode() == 0)
-                        {
-                            return;
-                        }
-                        onSelfError(task->id(),
-                            std::make_shared(
-                                (int)PSIRetCode::PeerNodeDown, "peer node is down, id: " + peerID),
-                            false);
-                    },
-                    nullptr);
-            }
-        }
-        m_pingTimer->restart();
-    }
+    void checkPeerActivity();
 
     io::LineReader::Ptr loadReader(std::string const& _taskID,
         protocol::DataResource::ConstPtr const& _dataResource, io::DataSchema _dataSchema,
-        uint32_t _columnSize = 1)
-    {
-        if (!_dataResource->rawData().empty())
-        {
-            return nullptr;
-        }
-        auto reader =
-            m_config->dataResourceLoader()->loadReader(_dataResource->desc(), _dataSchema, true);
-
-        if (reader->columnSize() == 0 || (reader->type() == protocol::DataResourceType::MySQL &&
-                                             reader->columnSize() != _columnSize))
-        {
-            auto errorMsg = "load data for task " + _taskID + "failed, expect " +
-                            std::to_string(_columnSize) + " column, current column size is " +
-                            std::to_string(reader->columnSize());
-            BOOST_THROW_EXCEPTION(BCOS_ERROR((int)PSIRetCode::LoadDataFailed, errorMsg));
-        }
-        return reader;
-    }
+        uint32_t _columnSize = 1);
 
     io::LineWriter::Ptr loadWriter(std::string const& _taskID,
-        protocol::DataResource::ConstPtr const& _dataResource, bool _enableOutputExists)
-    {
-        if (!_dataResource->rawData().empty())
-        {
-            return nullptr;
-        }
-        if (!_enableOutputExists)
-        {
-            // Note: if the output-resource already exists, will throw exception
-            m_config->dataResourceLoader()->checkResourceExists(_dataResource->outputDesc());
-        }
-        return m_config->dataResourceLoader()->loadWriter(_dataResource->outputDesc(), true);
-    }
+        protocol::DataResource::ConstPtr const& _dataResource, bool _enableOutputExists);
 
-    TaskState::Ptr findPendingTask(const std::string& _taskID)
-    {
-        bcos::ReadGuard l(x_pendingTasks);
-        auto it = m_pendingTasks.find(_taskID);
-        if (it == m_pendingTasks.end())
-        {
-            return nullptr;
-        }
-        return it->second;
-    }
+    TaskState::Ptr findPendingTask(const std::string& _taskID);
 
-    void addPendingTask(TaskState::Ptr _taskState)
-    {
-        bcos::WriteGuard l(x_pendingTasks);
-        auto id = _taskState->task()->id();
-        m_pendingTasks[id] = std::move(_taskState);
-    }
+    void addPendingTask(TaskState::Ptr _taskState);
 
-    void removePendingTask(const std::string& _taskID)
-    {
-        bcos::WriteGuard l(x_pendingTasks);
-        auto it = m_pendingTasks.find(_taskID);
-        if (it != m_pendingTasks.end())
-        {
-            m_pendingTasks.erase(it);
-        }
-    }
+    void removePendingTask(const std::string& _taskID);
 
 protected:
     PSIConfig::Ptr m_config;
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
new file mode 100644
index 00000000..9d479303
--- /dev/null
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
@@ -0,0 +1,336 @@
+/**
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file TaskState.cpp
+ * @author: yujiechen
+ * @date 2022-11-11
+ */
+#include "TaskState.h"
+using namespace ppc::psi;
+using namespace ppc::io;
+using namespace ppc::protocol;
+
+
+DataBatch::Ptr TaskState::loadAllData()
+{
+    DataBatch::Ptr results;
+    auto originData = m_task->selfParty()->dataResource()->rawData();
+    if (!originData.empty())
+    {
+        results = std::make_shared();
+        results->setDataSchema(DataSchema::Bytes);
+
+        std::vector data;
+        for (auto& line : originData[0])
+        {
+            data.emplace_back(bcos::bytes(line.begin(), line.end()));
+        }
+        results->setData(std::move(data));
+    }
+    else
+    {
+        if (m_reader)
+        {
+            int64_t nextParam = m_sqlReader ? 0 : -1;
+            results = m_reader->next(nextParam, io::DataSchema::Bytes);
+        }
+    }
+    if (!results)
+    {
+        results = std::make_shared();
+    }
+    return results;
+}
+
+void TaskState::writeLines(const DataBatch::Ptr& _data, DataSchema _schema)
+{
+    if (m_writer)
+    {
+        m_writer->writeLine(_data, _schema);
+        m_writer->flush();
+        m_writer->close();
+        m_writer->upload();
+        m_uploaded = true;
+    }
+}
+
+void TaskState::writeBytes(bcos::bytesConstRef _data)
+{
+    if (m_writer)
+    {
+        m_writer->writeBytes(_data);
+        m_writer->flush();
+        m_writer->close();
+        m_writer->upload();
+        m_uploaded = true;
+    }
+}
+
+int32_t TaskState::allocateSeq()
+{
+    bcos::RecursiveGuard l(m_mutex);
+    m_currentSeq.store(m_currentSeq.load() + 1);
+    {
+        bcos::WriteGuard l(x_seqList);
+        m_seqList.insert(m_currentSeq.load());
+    }
+    return m_currentSeq;
+}
+
+void TaskState::eraseFinishedTaskSeq(uint32_t _seq, bool _success)
+{
+    {
+        bcos::UpgradableGuard l(x_seqList);
+        auto it = m_seqList.find(_seq);
+        if (it == m_seqList.end())
+        {
+            return;
+        }
+        bcos::UpgradeGuard ul(l);
+        m_seqList.erase(it);
+        if (_success)
+        {
+            m_successCount++;
+        }
+        else
+        {
+            m_failedCount++;
+        }
+    }
+    try
+    {
+        // trigger the callback when the sub-task finished
+        // Note: the subTaskHandler may go wrong
+        if (m_onSubTaskFinished)
+        {
+            m_onSubTaskFinished();
+        }
+        PSI_LOG(INFO) << LOG_DESC("eraseFinishedTaskSeq") << LOG_KV("task", m_task->id())
+                      << LOG_KV("seq", _seq) << LOG_KV("success", _success)
+                      << LOG_KV("seqs", m_seqList.size()) << LOG_KV("successCount", m_successCount)
+                      << LOG_KV("failedCount", m_failedCount);
+    }
+    catch (std::exception const& e)
+    {
+        PSI_LOG(WARNING) << LOG_DESC(
+                                "eraseFinishedTaskSeq error for calls the sub-task-finalize "
+                                "handler exception")
+                         << LOG_KV("msg", boost::diagnostic_information(e));
+        onTaskException(boost::diagnostic_information(e));
+    }
+}
+
+
+// trigger the callback to response
+void TaskState::onTaskFinished()
+{
+    // avoid repeated calls
+    if (m_taskDone.exchange(true))
+    {
+        return;
+    }
+    PSI_LOG(INFO) << LOG_DESC("* onTaskFinished") << LOG_KV("task", m_task->id())
+                  << LOG_KV("success", m_successCount) << LOG_KV("failed", m_failedCount)
+                  << LOG_KV("loadFinished", m_finished.load())
+                  << LOG_KV("callback", m_callback ? "withCallback" : "emptyCallback")
+                  << LOG_KV("taskTimecost", taskPendingTime());
+    auto result = std::make_shared(m_task->id());
+    try
+    {
+        // upload the psi-result
+        if (m_writer && !m_uploaded)
+        {
+            m_writer->upload();
+            m_uploaded = true;
+        }
+        if (m_finalizeHandler)
+        {
+            m_finalizeHandler();
+        }
+        if (m_failedCount > 0)
+        {
+            auto error = std::make_shared(
+                -1, "task " + m_task->id() + " failed for " +
+                        boost::lexical_cast(m_failedCount) + " error!");
+            result->setError(std::move(error));
+            result->setStatus(toString(TaskStatus::FAILED));
+        }
+        else
+        {
+            result->setStatus(toString(TaskStatus::COMPLETED));
+        }
+
+        // clear file
+        if (m_reader)
+        {
+            m_reader->clean();
+        }
+    }
+    catch (std::exception const& e)
+    {
+        PSI_LOG(WARNING) << LOG_DESC("* onTaskFinished exception")
+                         << LOG_KV("taskTimeCost", taskPendingTime())
+                         << LOG_KV("msg", boost::diagnostic_information(e));
+        auto error = std::make_shared(-1, boost::diagnostic_information(e));
+        result->setError(std::move(error));
+        result->setStatus(toString(TaskStatus::FAILED));
+    }
+    if (m_callback)
+    {
+        m_callback(std::move(result));
+    }
+}
+
+void TaskState::onTaskFinished(TaskResult::Ptr _result, bool _noticePeer)
+{
+    // avoid repeated calls
+    if (m_taskDone.exchange(true))
+    {
+        return;
+    }
+    PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("task", m_task->id())
+                  << LOG_KV("success", m_successCount) << LOG_KV("onlySelfRun", m_onlySelfRun)
+                  << LOG_KV("finished", m_finished.load()) << LOG_KV("noticePeer", _noticePeer);
+    if (!_result)
+    {
+        _result = std::make_shared(m_task->id());
+    }
+    try
+    {
+        _result->setStatus(toString(TaskStatus::COMPLETED));
+        if (_result->error() && _result->error()->errorCode() != 0)
+        {
+            _result->setStatus(toString(TaskStatus::FAILED));
+        }
+        // Note: we consider that the task success even if the handler exception
+        if (_noticePeer && !m_onlySelfRun && _result->error() && _result->error()->errorCode() &&
+            m_notifyPeerFinishHandler)
+        {
+            m_notifyPeerFinishHandler();
+        }
+
+        if (m_finalizeHandler)
+        {
+            m_finalizeHandler();
+        }
+
+        m_finished.exchange(true);
+
+        // clear file
+        if (m_reader)
+        {
+            m_reader->clean();
+        }
+    }
+    catch (std::exception const& e)
+    {
+        PSI_LOG(WARNING) << LOG_DESC("onTaskFinished exception")
+                         << LOG_KV("msg", boost::diagnostic_information(e));
+        auto error = std::make_shared(-1, boost::diagnostic_information(e));
+        _result->setError(std::move(error));
+        _result->setStatus(toString(TaskStatus::FAILED));
+    }
+    if (m_callback)
+    {
+        m_callback(std::move(_result));
+    }
+}
+
+void TaskState::onPeerNotifyFinish()
+{
+    PSI_LOG(WARNING) << LOG_BADGE("onReceivePeerError") << LOG_KV("taskID", m_task->id());
+    auto tesult = std::make_shared(task()->id());
+    tesult->setError(std::make_shared(
+        (int)PSIRetCode::PeerNotifyFinish, "job participant sent an error"));
+    onTaskFinished(std::move(tesult), false);
+}
+
+// Note: must store the result serially
+void TaskState::storePSIResult(
+    DataResourceLoader::Ptr const& _resourceLoader, std::vector const& _data)
+{
+    bcos::RecursiveGuard l(m_mutex);
+    // try to generate-default-output-desc to make sure the server output exists even if not
+    // specified
+    tryToGenerateDefaultOutputDesc();
+    auto dataResource = m_task->selfParty()->dataResource();
+    // load the writer
+    if (!m_writer)
+    {
+        m_writer = _resourceLoader->loadWriter(dataResource->outputDesc());
+    }
+    auto dataBatch = std::make_shared();
+    dataBatch->setData(_data);
+    m_writer->writeLine(dataBatch, DataSchema::Bytes);
+    m_writer->flush();
+    PSI_LOG(INFO) << LOG_DESC("**** storePSIResult success ****") << LOG_KV("* task", m_task->id())
+                  << LOG_KV("* IntersectionCount", _data.size())
+                  << LOG_KV("* TaskTimecost", taskPendingTime());
+}
+
+void TaskState::onTaskException(std::string const& _errorMsg)
+{
+    // set the task finished
+    setFinished(true);
+    {
+        bcos::WriteGuard l(x_seqList);
+        m_seqList.clear();
+    }
+    if (!m_callback)
+    {
+        return;
+    }
+    // should been called even when exception
+    if (m_finalizeHandler)
+    {
+        try
+        {
+            m_finalizeHandler();
+        }
+        catch (std::exception const& e)
+        {
+            PSI_LOG(WARNING) << LOG_DESC("finalize task exception")
+                             << LOG_KV("taskID", m_task->id())
+                             << LOG_KV("msg", boost::diagnostic_information(e));
+        }
+    }
+    auto taskResult = std::make_shared(m_task->id());
+    auto msg = "Task " + m_task->id() + " exception, error : " + _errorMsg;
+    auto error = std::make_shared(-1, msg);
+    taskResult->setError(std::move(error));
+    m_callback(std::move(taskResult));
+    PSI_LOG(WARNING) << LOG_DESC(msg);
+}
+
+void TaskState::tryToGenerateDefaultOutputDesc()
+{
+    auto dataResource = m_task->selfParty()->mutableDataResource();
+    if (!dataResource)
+    {
+        dataResource = std::make_shared();
+        m_task->mutableSelfParty()->setDataResource(dataResource);
+    }
+    if (dataResource->outputDesc())
+    {
+        return;
+    }
+    auto outputDesc = std::make_shared();
+    auto dstPath = c_resultPath + "/" + m_task->id() + ".result";
+    outputDesc->setPath(dstPath);
+    outputDesc->setType((uint16_t)(DataResourceType::FILE));
+    dataResource->setOutputDesc(outputDesc);
+    PSI_LOG(INFO) << LOG_DESC("GenerateDefaultOutputDesc for the output-desc not specified")
+                  << LOG_KV("task", m_task->id()) << LOG_KV("path", dstPath);
+}
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
index 1de20188..0fe0e16f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
@@ -74,115 +74,13 @@ class TaskState : public std::enable_shared_from_this
     int64_t readerParam() const { return m_readerParam; }
     bool sqlReader() const { return m_sqlReader; }
 
-    io::DataBatch::Ptr loadAllData()
-    {
-        io::DataBatch::Ptr results;
-        auto originData = m_task->selfParty()->dataResource()->rawData();
-        if (!originData.empty())
-        {
-            results = std::make_shared();
-            results->setDataSchema(ppc::io::DataSchema::Bytes);
-
-            std::vector data;
-            for (auto& line : originData[0])
-            {
-                data.emplace_back(bcos::bytes(line.begin(), line.end()));
-            }
-            results->setData(std::move(data));
-        }
-        else
-        {
-            if (m_reader)
-            {
-                int64_t nextParam = m_sqlReader ? 0 : -1;
-                results = m_reader->next(nextParam, io::DataSchema::Bytes);
-            }
-        }
-        if (!results)
-        {
-            results = std::make_shared();
-        }
-        return results;
-    }
-
-    void writeLines(const io::DataBatch::Ptr& _data, io::DataSchema _schema)
-    {
-        if (m_writer)
-        {
-            m_writer->writeLine(_data, _schema);
-            m_writer->flush();
-            m_writer->close();
-            m_writer->upload();
-            m_uploaded = true;
-        }
-    }
-
-    void writeBytes(bcos::bytesConstRef _data)
-    {
-        if (m_writer)
-        {
-            m_writer->writeBytes(_data);
-            m_writer->flush();
-            m_writer->close();
-            m_writer->upload();
-            m_uploaded = true;
-        }
-    }
+    io::DataBatch::Ptr loadAllData();
+    void writeLines(const io::DataBatch::Ptr& _data, io::DataSchema _schema);
+    void writeBytes(bcos::bytesConstRef _data);
 
-    int32_t allocateSeq()
-    {
-        bcos::RecursiveGuard l(m_mutex);
-        m_currentSeq.store(m_currentSeq.load() + 1);
-        {
-            bcos::WriteGuard l(x_seqList);
-            m_seqList.insert(m_currentSeq.load());
-        }
-        return m_currentSeq;
-    }
+    int32_t allocateSeq();
 
-    void eraseFinishedTaskSeq(uint32_t _seq, bool _success)
-    {
-        {
-            bcos::UpgradableGuard l(x_seqList);
-            auto it = m_seqList.find(_seq);
-            if (it == m_seqList.end())
-            {
-                return;
-            }
-            bcos::UpgradeGuard ul(l);
-            m_seqList.erase(it);
-            if (_success)
-            {
-                m_successCount++;
-            }
-            else
-            {
-                m_failedCount++;
-            }
-        }
-        try
-        {
-            // trigger the callback when the sub-task finished
-            // Note: the subTaskHandler may go wrong
-            if (m_onSubTaskFinished)
-            {
-                m_onSubTaskFinished();
-            }
-            PSI_LOG(INFO) << LOG_DESC("eraseFinishedTaskSeq") << LOG_KV("task", m_task->id())
-                          << LOG_KV("seq", _seq) << LOG_KV("success", _success)
-                          << LOG_KV("seqs", m_seqList.size())
-                          << LOG_KV("successCount", m_successCount)
-                          << LOG_KV("failedCount", m_failedCount);
-        }
-        catch (std::exception const& e)
-        {
-            PSI_LOG(WARNING) << LOG_DESC(
-                                    "eraseFinishedTaskSeq error for calls the sub-task-finalize "
-                                    "handler exception")
-                             << LOG_KV("msg", boost::diagnostic_information(e));
-            onTaskException(boost::diagnostic_information(e));
-        }
-    }
+    void eraseFinishedTaskSeq(uint32_t _seq, bool _success);
 
     virtual void setFinished(bool _finished) { m_finished.store(_finished); }
 
@@ -199,129 +97,11 @@ class TaskState : public std::enable_shared_from_this
     }
 
     // trigger the callback to response
-    virtual void onTaskFinished()
-    {
-        // avoid repeated calls
-        if (m_taskDone.exchange(true))
-        {
-            return;
-        }
-        PSI_LOG(INFO) << LOG_DESC("* onTaskFinished") << LOG_KV("task", m_task->id())
-                      << LOG_KV("success", m_successCount) << LOG_KV("failed", m_failedCount)
-                      << LOG_KV("loadFinished", m_finished.load())
-                      << LOG_KV("callback", m_callback ? "withCallback" : "emptyCallback")
-                      << LOG_KV("taskTimecost", taskPendingTime());
-        auto result = std::make_shared(m_task->id());
-        try
-        {
-            // upload the psi-result
-            if (m_writer && !m_uploaded)
-            {
-                m_writer->upload();
-                m_uploaded = true;
-            }
-            if (m_finalizeHandler)
-            {
-                m_finalizeHandler();
-            }
-            if (m_failedCount > 0)
-            {
-                auto error = std::make_shared(
-                    -1, "task " + m_task->id() + " failed for " +
-                            boost::lexical_cast(m_failedCount) + " error!");
-                result->setError(std::move(error));
-                result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
-            }
-            else
-            {
-                result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::COMPLETED));
-            }
-
-            // clear file
-            if (m_reader)
-            {
-                m_reader->clean();
-            }
-        }
-        catch (std::exception const& e)
-        {
-            PSI_LOG(WARNING) << LOG_DESC("* onTaskFinished exception")
-                             << LOG_KV("taskTimeCost", taskPendingTime())
-                             << LOG_KV("msg", boost::diagnostic_information(e));
-            auto error = std::make_shared(-1, boost::diagnostic_information(e));
-            result->setError(std::move(error));
-            result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
-        }
-        if (m_callback)
-        {
-            m_callback(std::move(result));
-        }
-    }
-
-    virtual void onTaskFinished(ppc::protocol::TaskResult::Ptr _result, bool _noticePeer)
-    {
-        // avoid repeated calls
-        if (m_taskDone.exchange(true))
-        {
-            return;
-        }
-        PSI_LOG(INFO) << LOG_DESC("onTaskFinished") << LOG_KV("task", m_task->id())
-                      << LOG_KV("success", m_successCount) << LOG_KV("onlySelfRun", m_onlySelfRun)
-                      << LOG_KV("finished", m_finished.load()) << LOG_KV("noticePeer", _noticePeer);
-        if (!_result)
-        {
-            _result = std::make_shared(m_task->id());
-        }
-        try
-        {
-            _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::COMPLETED));
-            if (_result->error() && _result->error()->errorCode() != 0)
-            {
-                _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
-            }
-            // Note: we consider that the task success even if the handler exception
-            if (_noticePeer && !m_onlySelfRun && _result->error() &&
-                _result->error()->errorCode() && m_notifyPeerFinishHandler)
-            {
-                m_notifyPeerFinishHandler();
-            }
-
-            if (m_finalizeHandler)
-            {
-                m_finalizeHandler();
-            }
-
-            m_finished.exchange(true);
-
-            // clear file
-            if (m_reader)
-            {
-                m_reader->clean();
-            }
-        }
-        catch (std::exception const& e)
-        {
-            PSI_LOG(WARNING) << LOG_DESC("onTaskFinished exception")
-                             << LOG_KV("msg", boost::diagnostic_information(e));
-            auto error = std::make_shared(-1, boost::diagnostic_information(e));
-            _result->setError(std::move(error));
-            _result->setStatus(ppc::protocol::toString(ppc::protocol::TaskStatus::FAILED));
-        }
-        if (m_callback)
-        {
-            m_callback(std::move(_result));
-        }
-    }
+    virtual void onTaskFinished();
 
-    virtual void onPeerNotifyFinish()
-    {
-        PSI_LOG(WARNING) << LOG_BADGE("onReceivePeerError") << LOG_KV("taskID", m_task->id());
-        auto tesult = std::make_shared(task()->id());
-        tesult->setError(std::make_shared(
-            (int)PSIRetCode::PeerNotifyFinish, "job participant sent an error"));
-        onTaskFinished(std::move(tesult), false);
-    }
+    virtual void onTaskFinished(ppc::protocol::TaskResult::Ptr _result, bool _noticePeer);
 
+    virtual void onPeerNotifyFinish();
     void setWorker(std::function const& _worker) { m_worker = _worker; }
     void executeWork()
     {
@@ -351,88 +131,15 @@ class TaskState : public std::enable_shared_from_this
 
     // Note: must store the result serially
     void storePSIResult(ppc::io::DataResourceLoader::Ptr const& _resourceLoader,
-        std::vector const& _data)
-    {
-        bcos::RecursiveGuard l(m_mutex);
-        // try to generate-default-output-desc to make sure the server output exists even if not
-        // specified
-        tryToGenerateDefaultOutputDesc();
-        auto dataResource = m_task->selfParty()->dataResource();
-        // load the writer
-        if (!m_writer)
-        {
-            m_writer = _resourceLoader->loadWriter(dataResource->outputDesc());
-        }
-        auto dataBatch = std::make_shared();
-        dataBatch->setData(_data);
-        m_writer->writeLine(dataBatch, ppc::io::DataSchema::Bytes);
-        m_writer->flush();
-        PSI_LOG(INFO) << LOG_DESC("**** storePSIResult success ****")
-                      << LOG_KV("* task", m_task->id())
-                      << LOG_KV("* IntersectionCount", _data.size())
-                      << LOG_KV("* TaskTimecost", taskPendingTime());
-    }
+        std::vector const& _data);
 
     std::function takeFinalizeHandler() { return std::move(m_finalizeHandler); }
 
-    void onTaskException(std::string const& _errorMsg)
-    {
-        // set the task finished
-        setFinished(true);
-        {
-            bcos::WriteGuard l(x_seqList);
-            m_seqList.clear();
-        }
-        if (!m_callback)
-        {
-            return;
-        }
-        // should been called even when exception
-        if (m_finalizeHandler)
-        {
-            try
-            {
-                m_finalizeHandler();
-            }
-            catch (std::exception const& e)
-            {
-                PSI_LOG(WARNING) << LOG_DESC("finalize task exception")
-                                 << LOG_KV("taskID", m_task->id())
-                                 << LOG_KV("msg", boost::diagnostic_information(e));
-            }
-        }
-        auto taskResult = std::make_shared(m_task->id());
-        auto msg = "Task " + m_task->id() + " exception, error : " + _errorMsg;
-        auto error = std::make_shared(-1, msg);
-        taskResult->setError(std::move(error));
-        m_callback(std::move(taskResult));
-        PSI_LOG(WARNING) << LOG_DESC(msg);
-    }
-
+    void onTaskException(std::string const& _errorMsg);
     bool loadFinished() const { return m_finished.load(); }
 
     // generate default output-desc for given task
-    void tryToGenerateDefaultOutputDesc()
-    {
-        auto dataResource = m_task->selfParty()->mutableDataResource();
-        if (!dataResource)
-        {
-            dataResource = std::make_shared();
-            m_task->mutableSelfParty()->setDataResource(dataResource);
-        }
-        if (dataResource->outputDesc())
-        {
-            return;
-        }
-        auto outputDesc = std::make_shared();
-        auto dstPath = c_resultPath + "/" + m_task->id() + ".result";
-        outputDesc->setPath(dstPath);
-        outputDesc->setType((uint16_t)(ppc::protocol::DataResourceType::FILE));
-        dataResource->setOutputDesc(outputDesc);
-        PSI_LOG(INFO) << LOG_DESC("GenerateDefaultOutputDesc for the output-desc not specified")
-                      << LOG_KV("task", m_task->id()) << LOG_KV("path", dstPath);
-    }
-
+    void tryToGenerateDefaultOutputDesc();
     uint64_t taskPendingTime() { return (bcos::utcSteadyTime() - m_taskStartTime); }
 
     uint32_t sendedDataBatchSize() const { return m_seqList.size(); }

From c4687a4ae96815966ca77bd85446bf39a7031fd6 Mon Sep 17 00:00:00 2001
From: octopus <912554887@qq.com>
Date: Fri, 15 Nov 2024 16:20:35 +0800
Subject: [PATCH 092/120] support asynrun rpc method

---
 cpp/wedpr-computing/ppc-mpc/src/Common.h      |  17 +
 .../ppc-mpc/src/MPCService.cpp                | 404 +++++++++++++-----
 cpp/wedpr-computing/ppc-mpc/src/MPCService.h  |  42 +-
 .../ppc-tools/src/config/MPCConfig.h          |   2 +-
 .../ppc-tools/src/config/PPCConfig.cpp        |   7 +-
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp    |  13 +
 .../ppc-io/src/FileSystemDeleter.h            |  37 ++
 7 files changed, 419 insertions(+), 103 deletions(-)
 create mode 100644 cpp/wedpr-storage/ppc-io/src/FileSystemDeleter.h

diff --git a/cpp/wedpr-computing/ppc-mpc/src/Common.h b/cpp/wedpr-computing/ppc-mpc/src/Common.h
index ec4b8bc2..43ff4df8 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/Common.h
+++ b/cpp/wedpr-computing/ppc-mpc/src/Common.h
@@ -45,8 +45,25 @@ struct JobInfo
     std::string gatewayEngineEndpoint;
 };
 
+// job status
+struct JobStatus
+{
+    std::string jobId;
+    std::string status;
+    std::string message;
+    int64_t startTimeMs;
+    int64_t timeCostMs;
+};
+
+const std::string MPC_JOB_RUNNNING = "RUNNING";
+const std::string MPC_JOB_COMPLETED = "COMPLETED";
+const std::string MPC_JOB_FAILED = "FAILED";
+const std::string MPC_JOB_KILLED = "KILLED";
+
 const int MPC_SUCCESS = 0;
+const int MPC_DUPLICATED = 1;
 const int MPC_FAILED = -1;
+
 const std::string PATH_SEPARATOR = "/";
 const std::string MPC_RELATIVE_PATH = "/Programs/Source/";
 const std::string MPC_ALGORITHM_FILE_SUFFIX = ".mpc";
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index e7ff40ef..6b3ce337 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -24,8 +24,10 @@
 #include "ppc-io/src/DataResourceLoaderImpl.h"
 #include "ppc-io/src/FileLineReader.h"
 #include "ppc-io/src/FileLineWriter.h"
+#include "ppc-io/src/FileSystemDeleter.h"
 #include "ppc-storage/src/FileStorageFactoryImpl.h"
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -45,134 +47,339 @@ using namespace ppc::tools;
 using namespace ppc::storage;
 using namespace ppc::rpc;
 
-void MPCService::removeAllFiles(const std::vector &files)
+bool MPCService::addJobIfNotRunning(const JobInfo& jobInfo)
 {
-    for (const auto &file : files)
+    std::lock_guard l(x_job2Status);
+    auto it = m_job2Status.find(jobInfo.jobId);
+    if (it != m_job2Status.end())
     {
-        if (file.empty())
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][addJob]") 
+                << "job already exists"
+                << LOG_KV("jobId", jobInfo.jobId)
+                << LOG_KV("code", it->second.code)
+                << LOG_KV("message", it->second.message);
+
+        if (it->second.status == MPC_JOB_RUNNNING) 
         {
-            continue;
+            // job is running
+            return false;
         }
-        try {
-            if (boost::filesystem::exists(file))
-            {
-                boost::filesystem::remove_all(file);
+       
+        m_job2Status.erase(it);
+    }
 
-                MPC_LOG(INFO) << LOG_DESC("[MPCService][removeAllFiles]")
-                      << LOG_KV("file", file);
-            }
-        } catch (...) {
-            MPC_LOG(INFO) << LOG_DESC("[MPCService][removeAllFiles]")
-                      << LOG_DESC("remove file exception")
-                      << LOG_KV("file", file);
-        }
+    JobStatus jobStatus;
+    jobStatus.jobId = jobInfo.jobId;
+    jobStatus.status = MPC_JOB_RUNNNING;
+    jobStatus.message = "job is running";
+    jobStatus.startTimeMs = utcSteadyTime();
+    jobStatus.timeCostMs = 0;
+
+    m_job2Status[jobInfo.jobId] = jobStatus;
+    m_runningJobs.insert(jobInfo.jobId);
+
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][addJob]") 
+                << "job added successfully"
+                << LOG_KV("jobId", jobInfo.jobId)
+                << LOG_KV("runningJobsCount", m_runningJobs.size())
+                ;
+    return true;
+}
+
+bool MPCService::queryJobStatus(const std::string &jobId, JobStatus &jobStatus)
+{
+    std::lock_guard l(x_job2Status);
+    auto it = m_job2Status.find(jobId);
+    if (it == m_job2Status.end())
+    {
+        MPC_LOG(DEBUG) << LOG_DESC("[MPCService][queryJobStatus]") 
+                << "job does not exist"
+                << LOG_KV("jobId", jobId);
+        return false;
     }
+
+    jobStatus = it->second;
+
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][queryJobStatus]") 
+                << "find job"
+                << LOG_KV("jobId", jobId)
+                << LOG_KV("code", jobStatus.code)
+                << LOG_KV("message", jobStatus.message)
+                << LOG_KV("timeCostMs", jobStatus.timeCostMs);
+    return true;
 }
 
-void MPCService::doRun(Json::Value const& request, Json::Value& response)
+void MPCService::onFinish(const std::string &jobId, const std::string &msg)
 {
-    auto startT = utcSteadyTime();
+    std::lock_guard l(x_job2Status);
+    m_runningJobs.erase(jobId);
+    auto it = m_job2Status.find(jobId);
+    if (it == m_job2Status.end())
+    {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][onFinish]") 
+                << "job does not exist"
+                << LOG_KV("jobId", jobId);
+        return;
+    }
 
-    std::string localPathPrefix;
-    std::string mpcFileLocalPath;
-    try
-    {  // 0 get jobInfo and make command
-        auto jobInfo = paramsToJobInfo(request);
+    JobStatus &jobStatus = it->second;
+    jobStatus.status = MPC_JOB_COMPLETED;
+    jobStatus.message = msg;
+    jobStatus.timeCostMs = utcSteadyTime() - jobStatus.startTimeMs;
 
-        std::string jobId = jobInfo.jobId;
-        int participantCount = jobInfo.participantCount;
-        int selfIndex = jobInfo.selfIndex;
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][onFinish]") 
+                << "job finished"
+                << LOG_KV("jobId", jobId)
+                << LOG_KV("timeCostMs", jobStatus.timeCostMs)
+                ;
+}
 
-        std::string mpcCmd;
-        makeCommand(mpcCmd, jobInfo);
+void MPCService::onFailed(const std::string &jobId, const std::string &msg)
+{
+    std::lock_guard l(x_job2Status);
+    m_runningJobs.erase(jobId);
+    auto it = m_job2Status.find(jobId);
+    if (it == m_job2Status.end())
+    {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][onFailed]") 
+                << "job does not exist"
+                << LOG_KV("jobId", jobId);
+        return;
+    }
 
-        localPathPrefix =
-            m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR;
+    JobStatus &jobStatus = it->second;
+    jobStatus.status = MPC_JOB_FAILED;
+    jobStatus.message = msg;
+    jobStatus.timeCostMs = utcSteadyTime() - jobStatus.startTimeMs;
 
-        // 1 download mpc algorithm file
-        std::string mpcFileHdfsPath = jobInfo.mpcFilePath;
-        std::string mpcRootPath = m_mpcConfig.mpcRootPathNoGateway;
-        if (jobInfo.mpcNodeUseGateway)
-        {
-            mpcRootPath = m_mpcConfig.mpcRootPath;
-        }
-        
-        mpcFileLocalPath =
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][onFailed]") 
+                << LOG_KV("jobId", jobId)
+                << LOG_KV("msg", msg) 
+                << LOG_KV("timeCostMs", jobStatus.timeCostMs);
+}
+
+
+void MPCService::onKill(const std::string &jobId, const std::string &msg)
+{
+    std::lock_guard l(x_job2Status);
+    m_runningJobs.erase(jobId);
+    auto it = m_job2Status.find(jobId);
+    if (it == m_job2Status.end())
+    {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][onKill]") 
+                << "job does not exist"
+                << LOG_KV("jobId", jobId);
+        return;
+    }
+
+    JobStatus &jobStatus = it->second;
+    jobStatus.status = MPC_JOB_KILLED;
+    jobStatus.message = msg;
+    jobStatus.timeCostMs = utcSteadyTime() - jobStatus.startTimeMs;
+
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][onKill]") 
+                << "job is killed"
+                << LOG_KV("jobId", jobId)
+                << LOG_KV("timeCostMs", jobStatus.timeCostMs)
+                ;
+}
+
+void MPCService::doRun(const JobInfo& jobInfo) 
+{
+    auto startT = utcSteadyTime();
+    
+    std::string mpcFileHdfsPath = jobInfo.mpcFilePath;
+    std::string mpcRootPath = m_mpcConfig.mpcRootPathNoGateway;
+    if (jobInfo.mpcNodeUseGateway)
+    {
+        mpcRootPath = m_mpcConfig.mpcRootPath;
+    }
+    std::string localPathPrefix =
+            m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR;
+    std::string  mpcFileLocalPath =
             mpcRootPath + MPC_RELATIVE_PATH + jobInfo.jobId + MPC_ALGORITHM_FILE_SUFFIX;
 
-        auto mpcFileReader =
-                initialize_lineReader(jobInfo, mpcFileHdfsPath, DataResourceType::HDFS);
-        auto mpcFileWriter =
-                initialize_lineWriter(jobInfo, mpcFileLocalPath, DataResourceType::FILE);
-            readAndSaveFile(mpcFileHdfsPath, mpcFileLocalPath, mpcFileReader, mpcFileWriter);
+    FileSystemDeleter fileSystemDeleter({localPathPrefix, mpcFileLocalPath});
+
+    // 0 get jobInfo and make command
+    std::string jobId = jobInfo.jobId;
+    int participantCount = jobInfo.participantCount;
+    int selfIndex = jobInfo.selfIndex;
 
+    std::string mpcCmd;
+    makeCommand(mpcCmd, jobInfo);
+
+    // 1 download mpc algorithm file
+    auto mpcFileReader =
+            initialize_lineReader(jobInfo, mpcFileHdfsPath, DataResourceType::HDFS);
+    auto mpcFileWriter =
+            initialize_lineWriter(jobInfo, mpcFileLocalPath, DataResourceType::FILE);
+    readAndSaveFile(mpcFileHdfsPath, mpcFileLocalPath, mpcFileReader, mpcFileWriter);
+
+    // 2 download mpc prepare file
+    std::string mpcPrepareFileHdfsPath = jobInfo.inputFilePath;
+    
+    std::string mpcPrepareFileLocalPath = localPathPrefix + MPC_PREPARE_FILE + "-P" + std::to_string(selfIndex) + "-0";
+    auto datasetFileReader =
+            initialize_lineReader(jobInfo, mpcPrepareFileHdfsPath, DataResourceType::HDFS);
+    auto datasetFileWriter =
+            initialize_lineWriter(jobInfo, mpcPrepareFileLocalPath, DataResourceType::FILE);
+    readAndSaveFile(mpcPrepareFileHdfsPath, mpcPrepareFileLocalPath, datasetFileReader, datasetFileWriter);
+
+    // 3 run mpc job
+    int outExitStatus = MPC_SUCCESS;
+    std::string outResult;
+    execCommand(mpcCmd, outExitStatus, outResult);
+
+    if (outExitStatus != MPC_SUCCESS)
+    {
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][doRun]") 
+            << "run mpc job failed"
+            << LOG_KV("jobId", jobId)
+            << LOG_KV("outExitStatus", outExitStatus)
+            << LOG_KV("outResult", outResult);
+        BOOST_THROW_EXCEPTION(RunMpcFailException() << errinfo_comment(outResult));
+    }
 
-        // 2 download mpc prepare file
-        std::string mpcPrepareFileHdfsPath = jobInfo.inputFilePath;
-        
-        // std::string inputFilePath =
-        // m_mpcConfig.jobPath + PATH_SEPARATOR + jobInfo.jobId + PATH_SEPARATOR + MPC_PREPARE_FILE;
+    MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) 
+        << LOG_DESC("run mpc job successfully");
 
-        std::string mpcPrepareFileLocalPath = localPathPrefix + MPC_PREPARE_FILE + "-P" + std::to_string(selfIndex) + "-0";
-        auto datasetFileReader =
-                initialize_lineReader(jobInfo, mpcPrepareFileHdfsPath, DataResourceType::HDFS);
-            auto datasetFileWriter =
-                initialize_lineWriter(jobInfo, mpcPrepareFileLocalPath, DataResourceType::FILE);
-            readAndSaveFile(mpcPrepareFileHdfsPath, mpcPrepareFileLocalPath, datasetFileReader, datasetFileWriter);
+    // 4 upload result file
+    std::string resultFileHdfsPath =  jobInfo.outputFilePath;
+    std::string resultFileLocalPath = localPathPrefix + MPC_RESULT_FILE;
+    writeStringToFile(outResult, resultFileLocalPath);
 
-        // 3 run mpc job
-        int outExitStatus = MPC_SUCCESS;
-        std::string outResult;
-        execCommand(mpcCmd, outExitStatus, outResult);
+    auto resultFileReader =
+        initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE);
+    auto resultFileWriter =
+        initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS);
+    readAndSaveFile(resultFileLocalPath, resultFileHdfsPath, resultFileReader, resultFileWriter);
 
-        if (outExitStatus != MPC_SUCCESS)
-        {
-            removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
-            MPC_LOG(ERROR) << LOG_DESC("[MPCService][doRun]") 
-                << "run mpc job failed"
-                << LOG_KV("jobId", jobId)
-                << LOG_KV("outExitStatus", outExitStatus)
-                << LOG_KV("outResult", outResult);
-            BOOST_THROW_EXCEPTION(RunMpcFailException() << errinfo_comment(outResult));
-        }
-        
-        std::string message = "run mpc job successfully";
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) << LOG_DESC(message);
-        // MPC_LOG(DEBUG) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) << LOG_KV("outResult", outResult);
-        response["code"] = MPC_SUCCESS;
-        response["message"] = "success";
+    MPC_LOG(INFO) << LOG_DESC("do run mpc") << LOG_KV("jodId", jobInfo.jobId)<< LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+}
 
-        // 4 upload result file
-        std::string resultFileHdfsPath =  jobInfo.outputFilePath;
-        std::string resultFileLocalPath = localPathPrefix + MPC_RESULT_FILE;
-        writeStringToFile(outResult, resultFileLocalPath);
+void MPCService::queryMpcRpc(Json::Value const& request, RespFunc func)
+{
+    auto jobId = request["jobId"].asString();
 
-        auto resultFileReader =
-            initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE);
-        auto resultFileWriter =
-            initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS);
-        readAndSaveFile(resultFileLocalPath, resultFileHdfsPath, resultFileReader, resultFileWriter);
+    JobStatus jobStatus;
+    auto result = queryJobStatus(jobId, jobStatus);
 
-        removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
+    Json::Value response;
+    if (result)
+    {
+        response["code"] = MPC_SUCCESS;
+        response["status"] = jobStatus.status;
+        response["message"] = jobStatus.message;
+        response["timeCostMs"] = jobStatus.timeCostMs;
+    } 
+    else 
+    {
+        response["code"] = MPC_FAILED;
+        response["status"] = "";
+        response["message"] = "job does not exist";
+        response["timeCostMs"] = -1;
+    }
+
+    func(nullptr, std::move(response));
+}
+
+void MPCService::runMpcRpcByJobInfo(const JobInfo& jobInfo)
+{
+    try
+    {
+        doRun(jobInfo);
+        onFinish(jobInfo.jobId, "run mpc job successfully");
     }
     catch (const std::exception& e)
     {
-        removeAllFiles(std::vector{localPathPrefix, mpcFileLocalPath});
+        onFailed(jobInfo.jobId, boost::diagnostic_information(e));
+    }
+}
+
+void MPCService::runMpcRpc(Json::Value const& request, RespFunc func)
+{
+    Json::Value response;
+    
+    try
+    {
+        auto jobInfo = paramsToJobInfo(request);
+        auto r = addJobIfNotRunning(jobInfo);
+        if (r)
+        {  
+            doRun(jobInfo);
+            onFinish(jobInfo.jobId, "run mpc job successfully");
 
+            response["code"] = MPC_SUCCESS;
+            response["message"] = "success";
+        }
+        else 
+        {
+            response["code"] = MPC_DUPLICATED;
+            response["message"] = "duplicated submit job";
+        }
+    }
+    catch (const std::exception& e)
+    {
         const std::string diagnostic_information = std::string(boost::diagnostic_information(e));
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_DESC("run mpc job failed")
-                      << LOG_DESC(diagnostic_information);
+
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][runMpcRpc]") << LOG_DESC("run mpc submit job failed")
+                      << LOG_DESC(diagnostic_information)
+                      << LOG_KV("request", request.toStyledString());
+
         response["code"] = MPC_FAILED;
         response["message"] = diagnostic_information;
     }
 
-    MPC_LOG(INFO) << LOG_DESC("run mpc") << LOG_KV("request",  request.toStyledString())<< LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+    func(nullptr, std::move(response));
 }
 
-void MPCService::runMpcRpc(Json::Value const& request, RespFunc func)
+void MPCService::asyncRunMpcRpc(Json::Value const& request, RespFunc func)
 {
     Json::Value response;
-    doRun(request, response);
+
+    try
+    {
+        auto jobInfo = paramsToJobInfo(request);
+        auto r = addJobIfNotRunning(jobInfo);
+
+        if (r)
+        {  
+            response["code"] = MPC_SUCCESS;
+            response["message"] = "success";
+        }
+        else 
+        {
+            response["code"] = MPC_DUPLICATED;
+            response["message"] = "duplicated submit job";
+        }
+
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc submit job successfully")
+                      << LOG_KV("request", request.toStyledString())
+                      << LOG_KV("response", response.toStyledString());
+
+        // async run mpc job
+        m_threadPool->enqueue([self = weak_from_this(), jobInfo]() {
+            auto service = self.lock();
+            if (!service)
+            {
+                return;
+            }
+            service->runMpcRpcByJobInfo(jobInfo);
+        });
+    }
+    catch (const std::exception& e)
+    {
+        const std::string diagnostic_information = std::string(boost::diagnostic_information(e));
+
+        MPC_LOG(ERROR) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc submit job failed")
+                      << LOG_DESC(diagnostic_information)
+                      << LOG_KV("request", request.toStyledString());
+
+        response["code"] = MPC_FAILED;
+        response["message"] = diagnostic_information;
+    }
+
     func(nullptr, std::move(response));
 }
 
@@ -186,7 +393,7 @@ void MPCService::doKill(Json::Value const& request, Json::Value& response)
 
         std::string killCmd = "ps -ef |grep mpc | grep " + jobId +
                               " | grep -v grep | awk '{print $2}' | xargs kill -9";
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("killCmd", killCmd);
+        MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("jobId", jobId) << LOG_KV("killCmd", killCmd);
 
         int outExitStatus = 0;
         std::string outResult;
@@ -195,14 +402,17 @@ void MPCService::doKill(Json::Value const& request, Json::Value& response)
         if (outExitStatus == 0)
         {
             message = "Kill mpc job successfully";
-            MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_DESC(message);
+            onKill(jobId, message);
+            MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("jobId", jobId) << LOG_DESC(message);
             response["code"] = MPC_SUCCESS;
             response["message"] = "success";
         }
         else
         {
             message = "Kill mpc job failed";
-            MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_DESC(message);
+            MPC_LOG(INFO) << LOG_DESC("[MPCService][doKill]") << LOG_KV("jobId", jobId) << LOG_DESC(message)
+                << LOG_KV("outExitStatus", outExitStatus)
+                << LOG_KV("outResult", outResult);
             response["code"] = MPC_FAILED;
             response["message"] = message;
         }
@@ -215,7 +425,9 @@ void MPCService::doKill(Json::Value const& request, Json::Value& response)
         response["code"] = MPC_FAILED;
         response["message"] = diagnostic_information;
     }
-    MPC_LOG(INFO) << LOG_DESC("kill mpc job") << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
+    MPC_LOG(INFO) << LOG_DESC("kill mpc job") 
+        << LOG_KV("request", response.toStyledString())
+        << LOG_KV("timecost(ms)", utcSteadyTime() - startT);
 }
 
 void MPCService::killMpcRpc(Json::Value const& request, RespFunc func)
@@ -360,7 +572,6 @@ void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo)
             << LOG_KV("mpcRootPath", mpcRootPath)
             << LOG_KV("ret", r)
             << LOG_KV("jobId", jobId);
-            ;
     }
     std::string compileFilePath = mpcRootPath + PATH_SEPARATOR + MPC_ALGORITHM_COMPILER;
     int participantCount = jobInfo.participantCount;
@@ -375,7 +586,6 @@ void MPCService::makeCommand(std::string& cmd, const JobInfo& jobInfo)
         MPC_LOG(ERROR) << LOG_DESC("[MPCService] compile file not exist") 
             << LOG_KV("compileFilePath", compileFilePath)
             << LOG_KV("jobId", jobId);
-            ;
 
         BOOST_THROW_EXCEPTION(MpcCompilerNotExistException()
                               << errinfo_comment("compile file not exist:" + compileFilePath));
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
index a9f601c6..ebc89d45 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
@@ -25,19 +25,35 @@
 #include "ppc-tools/src/config/MPCConfig.h"
 #include "ppc-tools/src/config/StorageConfig.h"
 #include 
+#include 
+#include 
+#include 
+#include 
 #include 
+#include 
 
 namespace ppc::mpc
 {
-class MPCService
+class MPCService: std::enable_shared_from_this
 {
 public:
     using Ptr = std::shared_ptr;
     MPCService() = default;
-    virtual ~MPCService() = default;
+    virtual ~MPCService() 
+    {
+        if (m_threadPool)
+        {
+            m_threadPool->stop();
+        }
+    }
 
     void runMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func);
     void killMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func);
+    void asyncRunMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func);
+    void queryMpcRpc(Json::Value const& request, ppc::rpc::RespFunc func);
+
+    void runMpcRpcByJobInfo(const JobInfo& jobInfo);
+
     void setMPCConfig(ppc::tools::MPCConfig const& mpcConfig);
     void setStorageConfig(ppc::tools::StorageConfig const& storageConfig);
 
@@ -45,8 +61,14 @@ class MPCService
     void makeCommand(std::string& cmd, const JobInfo& jobInfo);
     void getMpcProtocol(const int participantCount, const bool isMalicious,
         std::string& mpcBinFileName, std::string& compileOption);
-    void doRun(Json::Value const& request, Json::Value& response);
+
+    void doRun(const JobInfo& jobInfo);
     void doKill(Json::Value const& request, Json::Value& response);
+
+    void onFinish(const std::string &jobId, const std::string &msg);
+    void onFailed(const std::string &jobId, const std::string &msg);
+    void onKill(const std::string &jobId, const std::string &msg);
+
     void execCommand(const std::string cmd, int& outExitStatus, std::string& outResult);
 
     void writeStringToFile(const std::string& content, const std::string& filePath);
@@ -56,10 +78,22 @@ class MPCService
     ppc::io::LineWriter::Ptr initialize_lineWriter(const JobInfo& jobInfo,
         const std::string& writerFilePath, ppc::protocol::DataResourceType type);
 
-    void removeAllFiles(const std::vector &files);
+    void setThreadPool(std::shared_ptr threadPool)
+    {
+        m_threadPool = threadPool;
+    }
+
+    bool addJobIfNotRunning(const JobInfo& jobInfo);
+
+    bool queryJobStatus(const std::string &jobId, JobStatus &jobStatus);
 
 private:
+    std::mutex x_job2Status;
+    std::unordered_map m_job2Status;
+    std::unordered_set m_runningJobs;
     ppc::tools::MPCConfig m_mpcConfig;
     ppc::tools::StorageConfig m_storageConfig;
+
+    std::shared_ptr m_threadPool;
 };
 }  // namespace ppc::mpc
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
index 2e342582..955c3924 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
@@ -25,10 +25,10 @@ namespace ppc::tools
 {
 struct MPCConfig
 {
-    std::string datasetHDFSPath;
     std::string jobPath;
     std::string mpcRootPath;
     std::string mpcRootPathNoGateway;
     uint64_t readPerBatchLines;
+    int threadPoolSize;
 };
 }  // namespace ppc::tools
\ No newline at end of file
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index fb2aad73..31c92642 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -613,10 +613,15 @@ void PPCConfig::loadCEMConfig(boost::property_tree::ptree const& _pt)
 
 void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt)
 {
-    m_mpcConfig.datasetHDFSPath = _pt.get("mpc.dataset_hdfs_path", "./");
     m_mpcConfig.jobPath = _pt.get("mpc.job_path", "./");
     m_mpcConfig.mpcRootPath = _pt.get("mpc.mpc_root_path", "./");
     m_mpcConfig.mpcRootPathNoGateway = _pt.get("mpc.mpc_root_path_no_gateway", "./");
     m_mpcConfig.readPerBatchLines = _pt.get("mpc.read_per_batch_lines", 100000);
+    m_mpcConfig.threadPoolSize = _pt.get("mpc.async_thread_pool_size", -1);
+    if (m_mpcConfig.threadPoolSize < 0)
+    {
+        m_mpcConfig.threadPoolSize = std::thread::hardware_concurrency();
+    }
+
     loadHDFSConfig(_pt);
 }
\ No newline at end of file
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index e1fda6f0..ab7964e2 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -56,13 +56,26 @@ void MPCInitializer::init(std::string const& _configPath)
     auto mpcConfig = m_config->mpcConfig();
     auto rpcFactory = std::make_shared(m_config->agencyID());
     m_rpc = rpcFactory->buildRpc(m_config, nullptr);
+
+    int threadPoolSize = mpcConfig.threadPoolSize;
+    auto threadPool = std::make_shared("mpc-pool", threadPoolSize);
+
+    INIT_LOG(INFO) << LOG_DESC("init the mpc threadpool")
+        << LOG_KV("threadPoolSize", threadPoolSize);
+
     auto mpcService = std::make_shared();
     mpcService->setMPCConfig(mpcConfig);
     mpcService->setStorageConfig(storageConfig);
+    mpcService->setThreadPool(threadPool);
+
     m_rpc->registerHandler("run", std::bind(&MPCService::runMpcRpc, mpcService,
                                       std::placeholders::_1, std::placeholders::_2));
+    m_rpc->registerHandler("asyncRun", std::bind(&MPCService::asyncRunMpcRpc, mpcService,
+                                      std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService,
                                        std::placeholders::_1, std::placeholders::_2));
+    m_rpc->registerHandler("query", std::bind(&MPCService::queryMpcRpc, mpcService,
+                                       std::placeholders::_1, std::placeholders::_2));
     INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success");
     // init the transport
     initTransport(pt);
diff --git a/cpp/wedpr-storage/ppc-io/src/FileSystemDeleter.h b/cpp/wedpr-storage/ppc-io/src/FileSystemDeleter.h
new file mode 100644
index 00000000..dff414d1
--- /dev/null
+++ b/cpp/wedpr-storage/ppc-io/src/FileSystemDeleter.h
@@ -0,0 +1,37 @@
+#pragma once
+#include 
+#include 
+#include 
+#include 
+
+class FileSystemDeleter
+{  
+public:
+    FileSystemDeleter(std::vector filePaths): m_filePaths(filePaths) {}
+    ~FileSystemDeleter() 
+    {
+        removeAllFiles(m_filePaths);
+    }
+
+public:
+    void removeAllFiles(const std::vector &files) 
+    {
+        for (const auto &file : files)
+        {
+            try 
+            {
+                if (!file.empty() && boost::filesystem::exists(file))
+                {
+                    boost::filesystem::remove_all(file);
+                    BCOS_LOG(INFO) << LOG_DESC("remove file") << LOG_KV("file", file);
+                }
+            } catch (...) 
+            {
+                // do nothing
+            }
+        }
+    }
+
+private:
+    std::vector m_filePaths;
+};
\ No newline at end of file

From 427d4efa7ec77dbbf334faa82fbdf5a9e01c9232 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 19 Nov 2024 00:16:39 +0800
Subject: [PATCH 093/120] add exception handle when send message failed for
 ecdh-multi-psi (#92)

* add exception handle when send message failed for ecdh-multi-psi

* python transport sdk support initLog
---
 .../ppc-pir/tests/CMakeLists.txt              |  4 +-
 .../src/cm2020-psi/core/CM2020PSIReceiver.cpp | 11 +++--
 .../src/cm2020-psi/core/CM2020PSISender.cpp   |  6 +++
 .../src/ecdh-multi-psi/EcdhMultiCache.cpp     | 29 +++++++++--
 .../core/EcdhMultiPSICalculator.cpp           | 48 +++++++++++++------
 .../core/EcdhMultiPSIMaster.cpp               | 18 +++++--
 .../core/EcdhMultiPSIPartner.cpp              | 12 ++++-
 .../ppc-psi/src/psi-framework/TaskState.cpp   | 13 +++++
 cpp/wedpr-protocol/protocol/src/PPCMessage.h  |  3 +-
 .../ppc-front/ppc-front/Front.cpp             |  2 +
 .../transport/impl/transport_loader.py        |  1 +
 python/ppc_model/conf/logging.conf            |  4 +-
 .../ppc_model/conf/wedpr_sdk_log_config.ini   | 17 +++++++
 python/ppc_model/ppc_model_app.py             |  2 +-
 python/ppc_model/task/task_manager.py         |  8 ++--
 python/ppc_model/tools/start.sh               |  9 +++-
 16 files changed, 146 insertions(+), 41 deletions(-)
 create mode 100644 python/ppc_model/conf/wedpr_sdk_log_config.ini

diff --git a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
index 8af1b5f9..dc64b608 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
@@ -5,8 +5,6 @@ set(TEST_BINARY_NAME test-ppc-pir)
 
 add_executable(${TEST_BINARY_NAME} ${SOURCES})
 target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
-# target_link_libraries(${PIR_TARGET} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
 
-# target_link_libraries(${TEST_BINARY_NAME} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
-target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${RPC_TARGET} ${CRYPTO_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
+target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${CRYPTO_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
 add_test(NAME test-pir WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
index 2525882f..faf4abf0 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSIReceiver.cpp
@@ -170,7 +170,8 @@ void CM2020PSIReceiver::prepareInputs()
         m_originLocations.resize(m_rInputSize);
         m_oprfOutputs.reserve(m_rInputSize);
         m_oprfOutputs.resize(m_rInputSize);
-
+        CM2020_PSI_LOG(INFO) << LOG_BADGE("prepareInputs success") << LOG_KV("taskID", m_taskID)
+                             << LOG_KV("inputSize", m_rInputSize);
         syncInputsSize();
     }
     catch (const std::exception& e)
@@ -187,7 +188,7 @@ void CM2020PSIReceiver::syncInputsSize()
     encodeUnsignedNum(data, m_rInputSize);
     CM2020_PSI_LOG(TRACE) << LOG_BADGE("syncInputsSize") << LOG_KV("taskID", m_taskID)
                           << LOG_KV("data", *bcos::toHexString(*data))
-                          << LOG_KV("m_rInputSize", m_rInputSize);
+                          << LOG_KV("inputSize", m_rInputSize);
     auto message = m_config->ppcMsgFactory()->buildPPCMessage(
         uint8_t(TaskType::PSI), uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, data);
     message->setMessageType(uint8_t(CM2020PSIMessageType::RECEIVER_SIZE));
@@ -242,7 +243,8 @@ void CM2020PSIReceiver::onBatchPointBReceived(PPCMessageFace::Ptr _message)
     {
         return;
     }
-    CM2020_PSI_LOG(INFO) << LOG_BADGE("handleBatchPointB") << LOG_KV("taskID", m_taskID);
+    CM2020_PSI_LOG(INFO) << LOG_BADGE("handleBatchPointB") << LOG_KV("taskID", m_taskID)
+                         << LOG_KV("payloadSize", _message->data()->size());
 
     try
     {
@@ -261,7 +263,8 @@ void CM2020PSIReceiver::onSenderSizeReceived(front::PPCMessageFace::Ptr _message
     {
         return;
     }
-    CM2020_PSI_LOG(INFO) << LOG_BADGE("onSenderSizeReceived") << LOG_KV("taskID", m_taskID);
+    CM2020_PSI_LOG(INFO) << LOG_BADGE("onSenderSizeReceived") << LOG_KV("taskID", m_taskID)
+                         << LOG_KV("payloadSize", _message->data()->size());
     try
     {
         decodeUnsignedNum(m_sInputSize, _message->data());
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
index c7f9a947..f8a4b93b 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/core/CM2020PSISender.cpp
@@ -209,6 +209,8 @@ void CM2020PSISender::onReceiverSizeReceived(front::PPCMessageFace::Ptr _message
     {
         decodeUnsignedNum(m_rInputSize, _message->data());
         m_progress->mark("RECEIVE_SIZE");
+        CM2020_PSI_LOG(INFO) << LOG_BADGE("onReceiverSizeReceived") << LOG_KV("taskID", m_taskID)
+                             << LOG_KV("inputSize", m_rInputSize);
     }
     catch (const std::exception& e)
     {
@@ -238,6 +240,10 @@ void CM2020PSISender::onPointAReceived(front::PPCMessageFace::Ptr _message)
 
         auto retPair = m_ot->receiverGeneratePointsB(m_otChoices, pointA);
 
+        CM2020_PSI_LOG(INFO) << LOG_BADGE("onPointAReceived and send pointB")
+                             << LOG_KV("taskID", m_taskID)
+                             << LOG_KV("dataSize", retPair.first->size())
+                             << LOG_KV("otNumber", m_otChoices->size());
         // send batch point B to ot sender
         auto message = m_config->ppcMsgFactory()->buildPPCMessage(uint8_t(TaskType::PSI),
             uint8_t(TaskAlgorithmType::CM_PSI_2PC), m_taskID, retPair.first);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
index abf466d9..5e7a7168 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
@@ -244,15 +244,26 @@ void MasterCache::encryptAndSendIntersection(uint64_t dataBatchIdx, bcos::bytes
     });
     auto taskID = m_taskState->task()->id();
     auto self = weak_from_this();
-    for (auto const& calcultor : calculators)
+    for (auto const& calculator : calculators)
     {
         m_config->generateAndSendPPCMessage(
-            calcultor.first, taskID, message,
-            [self](bcos::Error::Ptr&& _error) {
-                if (!_error)
+            calculator.first, taskID, message,
+            [self, calculator](bcos::Error::Ptr&& _error) {
+                if (!_error || _error->errorCode() == 0)
                 {
                     return;
                 }
+                auto cache = self.lock();
+                if (!cache)
+                {
+                    return;
+                }
+                ECDH_MULTI_LOG(WARNING)
+                    << LOG_DESC("encryptAndSendIntersection: send message to calcultor failed")
+                    << LOG_KV("task", cache->m_taskState->task()->id())
+                    << LOG_KV("calculator", calculator.first) << LOG_KV("code", _error->errorCode())
+                    << LOG_KV("msg", _error->errorMessage());
+                cache->m_taskState->onTaskException(_error->errorMessage());
             },
             dataBatchIdx);
     }
@@ -317,10 +328,16 @@ bool CalculatorCache::tryToFinalize()
         {
             continue;
         }
-        if (it.second.plainDataIndex >= 0)
+        if (it.second.plainDataIndex > 0)
         {
             m_intersectionResult.emplace_back(getPlainDataByIndex(it.second.plainDataIndex));
         }
+        // means the header field, swap with the first element
+        if (it.second.plainDataIndex == 0)
+        {
+            m_intersectionResult.emplace_back(m_intersectionResult[0]);
+            m_intersectionResult[0] = getPlainDataByIndex(it.second.plainDataIndex);
+        }
     }
     m_cacheState = CacheState::Finalized;
     ECDH_MULTI_LOG(INFO) << LOG_DESC("* tryToFinalize:  compute intersection success")
@@ -356,6 +373,7 @@ void CalculatorCache::syncIntersections()
         message->setVersion(-1);
         for (auto& peer : peers)
         {
+            // Note: sync task failed will not change the task status
             m_config->generateAndSendPPCMessage(
                 peer.first, taskID, message,
                 [taskID, peer](bcos::Error::Ptr&& _error) {
@@ -381,6 +399,7 @@ void CalculatorCache::syncIntersections()
     message->setVersion(0);
     for (auto& peer : peers)
     {
+        // Note: sync task failed will not change the task status
         m_config->generateAndSendPPCMessage(
             peer.first, taskID, message,
             [taskID, peer](bcos::Error::Ptr&& _error) {
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
index 107b0e29..c9e8b483 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
@@ -49,17 +49,20 @@ bcos::bytes EcdhMultiPSICalculator::generateRandomA(std::string _taskID)
     message->setData(std::vector{randomValue});
     message->setDataBatchCount(randomValue.size());
     message->setFrom(m_taskState->task()->selfParty()->id());
+    auto self = weak_from_this();
     for (auto const& partner : m_partnerParties)
     {
-        ECDH_CAL_LOG(INFO) << LOG_KV("PART1: Calculator generateRandomA to ", partner.first)
+        ECDH_CAL_LOG(INFO) << LOG_DESC("send generateRandomA to partner")
+                           << LOG_KV("partner", partner.first)
                            << LOG_KV(" Random: ", *toHexString(randomValue));
         m_config->generateAndSendPPCMessage(
             partner.first, _taskID, message,
-            [self = weak_from_this(), partner](bcos::Error::Ptr&& _error) {
-                if (!_error)
+            [self, _taskID, partner](bcos::Error::Ptr&& _error) {
+                if (!_error || _error->errorCode() == 0)
                 {
                     ECDH_CAL_LOG(INFO)
-                        << LOG_KV("PART1: Calculator generateRandomA success to ", partner.first);
+                        << LOG_DESC("send generated randomA to partner success")
+                        << LOG_KV("task", _taskID) << LOG_KV("partner", partner.first);
                     return;
                 }
                 auto psi = self.lock();
@@ -67,6 +70,15 @@ bcos::bytes EcdhMultiPSICalculator::generateRandomA(std::string _taskID)
                 {
                     return;
                 }
+                if (!psi->m_taskState)
+                {
+                    return;
+                }
+                ECDH_CAL_LOG(WARNING)
+                    << LOG_DESC("send generated randomA to partner failed")
+                    << LOG_KV("task", _taskID) << LOG_KV("partner", partner.first)
+                    << LOG_KV("code", _error->errorCode()) << LOG_KV("msg", _error->errorMessage());
+                psi->m_taskState->onTaskException(_error->errorMessage());
             },
             0);
     }
@@ -147,12 +159,13 @@ void EcdhMultiPSICalculator::blindData(std::string _taskID, bcos::bytes _randA)
                 message->setDataBatchCount(0);
             }
             // send cipher
+            auto self = weak_from_this();
             for (auto const& master : m_masterParties)
             {
                 m_config->generateAndSendPPCMessage(
                     master.first, _taskID, message,
-                    [self = weak_from_this(), master](bcos::Error::Ptr&& _error) {
-                        if (!_error)
+                    [self, _taskID, master](bcos::Error::Ptr&& _error) {
+                        if (!_error || _error->errorCode() == 0)
                         {
                             return;
                         }
@@ -161,6 +174,12 @@ void EcdhMultiPSICalculator::blindData(std::string _taskID, bcos::bytes _randA)
                         {
                             return;
                         }
+                        ECDH_CAL_LOG(WARNING)
+                            << LOG_DESC("send blindedData to master failed")
+                            << LOG_KV("task", _taskID) << LOG_KV("master", master.first)
+                            << LOG_KV("code", _error->errorCode())
+                            << LOG_KV("msg", _error->errorMessage());
+                        psi->m_taskState->onTaskException(_error->errorMessage());
                     },
                     seq);
                 dataOffset += dataBatch->size();
@@ -215,22 +234,23 @@ void EcdhMultiPSICalculator::onReceiveIntersecCipher(PSIMessageInterface::Ptr _m
         message->setFrom(m_taskState->task()->selfParty()->id());
         // try to finalize
         auto ret = m_calculatorCache->tryToFinalize();
+        auto taskID = m_taskID;
         for (auto const& master : m_masterParties)
         {
             ECDH_CAL_LOG(INFO) << LOG_DESC("onReceiveIntersecCipher: send response to the master")
                                << LOG_KV("master", master.first) << printPSIMessage(_msg);
+            // no any bad influences when send response failed
             m_config->generateAndSendPPCMessage(
-                master.first, m_taskID, message,
-                [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                    if (!_error)
-                    {
-                        return;
-                    }
-                    auto psi = self.lock();
-                    if (!psi)
+                master.first, taskID, message,
+                [taskID](bcos::Error::Ptr&& _error) {
+                    if (!_error || _error->errorCode() == 0)
                     {
                         return;
                     }
+                    ECDH_CAL_LOG(WARNING)
+                        << LOG_DESC("onReceiveIntersecCipher: send response to the master failed")
+                        << LOG_KV("task", taskID) << LOG_KV("code", _error->errorCode())
+                        << LOG_KV("msg", _error->errorMessage());
                 },
                 0);
         }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index f21680f8..ee4769d8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -185,16 +185,28 @@ void EcdhMultiPSIMaster::blindData()
                 message->setDataBatchCount(0);
             }
             message->setFrom(m_taskState->task()->selfParty()->id());
+            auto self = weak_from_this();
             for (auto const& calcultor : m_calculatorParties)
             {
-                // TODO: handle the send failed case
                 m_config->generateAndSendPPCMessage(
                     calcultor.first, m_taskID, message,
-                    [](bcos::Error::Ptr&& _error) {
-                        if (_error)
+                    [self, seq, calcultor](bcos::Error::Ptr&& _error) {
+                        if (!_error || _error->errorCode() == 0)
                         {
                             return;
                         }
+                        auto psi = self.lock();
+                        if (!psi)
+                        {
+                            return;
+                        }
+                        ECDH_MASTER_LOG(WARNING)
+                            << LOG_DESC("send blindedData to calculator failed")
+                            << LOG_KV("calculator", calcultor.first) << LOG_KV("seq", seq)
+                            << LOG_KV("task", psi->m_taskState->task()->id())
+                            << LOG_KV("code", _error->errorCode())
+                            << LOG_KV("msg", _error->errorMessage());
+                        psi->m_taskState->onTaskException(_error->errorMessage());
                     },
                     seq);
             }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
index e52f626c..63463024 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
@@ -117,12 +117,13 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
                 message->setDataBatchCount(0);
             }
             // generate and send encryptedHashSet
+            auto self = weak_from_this();
             for (auto const& master : m_masterParties)
             {
                 m_config->generateAndSendPPCMessage(
                     master.first, m_taskState->task()->id(), message,
-                    [self = weak_from_this()](bcos::Error::Ptr&& _error) {
-                        if (!_error)
+                    [self, master, seq](bcos::Error::Ptr&& _error) {
+                        if (!_error || _error->errorCode() == 0)
                         {
                             return;
                         }
@@ -131,6 +132,13 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
                         {
                             return;
                         }
+                        ECDH_PARTNER_LOG(WARNING)
+                            << LOG_DESC("send blinded data to master failed") << LOG_KV("seq", seq)
+                            << LOG_KV("master", master.first)
+                            << LOG_KV("task", psi->m_taskState->task()->id())
+                            << LOG_KV("code", _error->errorCode())
+                            << LOG_KV("msg", _error->errorMessage());
+                        psi->m_taskState->onTaskException(_error->errorMessage());
                     },
                     seq);
             }
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
index 9d479303..1e0c91b8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
@@ -306,6 +306,19 @@ void TaskState::onTaskException(std::string const& _errorMsg)
                              << LOG_KV("msg", boost::diagnostic_information(e));
         }
     }
+    if (m_notifyPeerFinishHandler)
+    {
+        try
+        {
+            m_notifyPeerFinishHandler();
+        }
+        catch (std::exception const& e)
+        {
+            PSI_LOG(WARNING) << LOG_DESC("notifyPeerFinish exception")
+                             << LOG_KV("taskID", m_task->id())
+                             << LOG_KV("msg", boost::diagnostic_information(e));
+        }
+    }
     auto taskResult = std::make_shared(m_task->id());
     auto msg = "Task " + m_task->id() + " exception, error : " + _errorMsg;
     auto error = std::make_shared(-1, msg);
diff --git a/cpp/wedpr-protocol/protocol/src/PPCMessage.h b/cpp/wedpr-protocol/protocol/src/PPCMessage.h
index c2ad3d29..c6cc6978 100644
--- a/cpp/wedpr-protocol/protocol/src/PPCMessage.h
+++ b/cpp/wedpr-protocol/protocol/src/PPCMessage.h
@@ -43,7 +43,8 @@ class PPCMessage : public PPCMessageFace
 
     using Ptr = std::shared_ptr;
     PPCMessage() { m_data = std::make_shared(); }
-    ~PPCMessage() override { releasePayload(); }
+    // Note: the payload passed in by the upper layer cannot be released at will
+    ~PPCMessage() override = default;
 
     uint8_t version() const override { return m_version; }
     void setVersion(uint8_t _version) override { m_version = _version; }
diff --git a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
index 4f1fedb1..ed974de0 100644
--- a/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
+++ b/cpp/wedpr-transport/ppc-front/ppc-front/Front.cpp
@@ -139,6 +139,8 @@ void Front::asyncSendMessage(const std::string& _agencyID, front::PPCMessageFace
     m_front->asyncSendMessage((uint16_t)RouteType::ROUTE_THROUGH_TOPIC, routeInfo,
         bcos::bytesConstRef((bcos::byte*)data.data(), data.size()), _message->seq(), _timeout,
         _callback, msgCallback);
+    // release the data
+    bcos::bytes().swap(data);
 }
 
 // send response when receiving message from given agencyID
diff --git a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
index 81f756a3..cb602232 100644
--- a/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
+++ b/cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py
@@ -47,6 +47,7 @@ def __repr__(self):
 
 class TransportLoader:
     transport_builder = TransportBuilder()
+    transport_builder.initLog("conf/wedpr_sdk_log_config.ini")
 
     @staticmethod
     def load(transport_config: TransportConfig) -> Transport:
diff --git a/python/ppc_model/conf/logging.conf b/python/ppc_model/conf/logging.conf
index f78ab644..e600039c 100644
--- a/python/ppc_model/conf/logging.conf
+++ b/python/ppc_model/conf/logging.conf
@@ -3,7 +3,7 @@ keys=root,wsgi
 
 [logger_root]
 level=INFO
-handlers=consoleHandler,fileHandler
+handlers=fileHandler
 
 [logger_wsgi]
 level = INFO
@@ -12,7 +12,7 @@ qualname = wsgi
 propagate = 0
 
 [handlers]
-keys=fileHandler,consoleHandler,accessHandler
+keys=fileHandler,accessHandler
 
 [handler_accessHandler]
 class=handlers.TimedRotatingFileHandler
diff --git a/python/ppc_model/conf/wedpr_sdk_log_config.ini b/python/ppc_model/conf/wedpr_sdk_log_config.ini
new file mode 100644
index 00000000..448b4833
--- /dev/null
+++ b/python/ppc_model/conf/wedpr_sdk_log_config.ini
@@ -0,0 +1,17 @@
+[log]
+enable = true
+log_path = ./log
+; info debug trace
+level = info
+; MB
+max_log_file_size = 200
+; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message
+;format = %Severity%|ppcs-gateway|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message%
+format = %Severity%|%TimeStamp%|%Message%
+enable_rotate_by_hour = true
+; if archive_path is empty, the archive function will be disabled
+; archive_path = ./log/
+;compress_archive_file = true
+; ; 0 = no limit, in MB
+; max_archive_size = 0
+; min_free_space = 0
diff --git a/python/ppc_model/ppc_model_app.py b/python/ppc_model/ppc_model_app.py
index c2780600..036301c3 100644
--- a/python/ppc_model/ppc_model_app.py
+++ b/python/ppc_model/ppc_model_app.py
@@ -68,7 +68,7 @@ def register_task_handler():
                         TransLogger(app, setup_console_handler=False), numthreads=2)
 
     protocol = 'http'
-    message = f"Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']} successfully"
+    message = f"* Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']} successfully"
     print(message)
     components.logger().info(message)
     server.start()
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index 3ded3756..d39205ba 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -244,8 +244,8 @@ def _terminate_timeout_tasks(self):
         with self._rw_lock.gen_rlock():
             for task_id, value in self._tasks.items():
                 alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
-                if alive_time >= self._task_timeout_s and value[0] == TaskStatus.RUNNING.value:
+                              value.start_time).total_seconds()
+                if alive_time >= self._task_timeout_s and value.task_status == TaskStatus.RUNNING.value:
                     tasks_to_kill.append(task_id)
 
         for task_id in tasks_to_kill:
@@ -257,9 +257,9 @@ def _cleanup_finished_tasks(self):
         with self._rw_lock.gen_rlock():
             for task_id, value in self._tasks.items():
                 alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
+                              value.start_time).total_seconds()
                 if alive_time >= self._task_timeout_s + 3600:
-                    tasks_to_cleanup.append((task_id, value[3]))
+                    tasks_to_cleanup.append((task_id, value.job_id))
         with self._rw_lock.gen_wlock():
             for task_id, job_id in tasks_to_cleanup:
                 if task_id in self._tasks:
diff --git a/python/ppc_model/tools/start.sh b/python/ppc_model/tools/start.sh
index 2eb4834f..ca412013 100644
--- a/python/ppc_model/tools/start.sh
+++ b/python/ppc_model/tools/start.sh
@@ -11,6 +11,10 @@ LOG_INFO() {
 }
 binary_path=${SHELL_FOLDER}/ppc_model_app.py
 cd ${SHELL_FOLDER}
+log_dir=logs
+if [ ! -f ${log_dir} ];then
+   mkdir -p logs
+fi
 node=$(basename ${SHELL_FOLDER})
 node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
 
@@ -18,7 +22,8 @@ if [ ! -z ${node_pid} ];then
     echo " ${node} is running, pid is $node_pid."
     exit 0
 else
-    nohup python ${binary_path} > start.out 2>&1 &
+    # -u means Force the stdout and stderr streams to be unbuffered
+    nohup python -u ${binary_path} > start.out 2>&1 &
     sleep 1.5
 fi
 try_times=4
@@ -34,4 +39,4 @@ do
     sleep 0.5
     ((i=i+1))
 done
-echo -e "\033[31m  Exceed waiting time. Please try again to start ${node} \033[0m"
\ No newline at end of file
+echo -e "\033[31m  Exceed waiting time. Please try again to start ${node} \033[0m"

From 43e84d30c3b2b78143c1a371f40bd88cb888fb11 Mon Sep 17 00:00:00 2001
From: octopus <912554887@qq.com>
Date: Tue, 19 Nov 2024 10:26:02 +0800
Subject: [PATCH 094/120] fix compile

---
 cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index 6b3ce337..b7138cb4 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -56,7 +56,7 @@ bool MPCService::addJobIfNotRunning(const JobInfo& jobInfo)
         MPC_LOG(INFO) << LOG_DESC("[MPCService][addJob]") 
                 << "job already exists"
                 << LOG_KV("jobId", jobInfo.jobId)
-                << LOG_KV("code", it->second.code)
+                << LOG_KV("status", it->second.status)
                 << LOG_KV("message", it->second.message);
 
         if (it->second.status == MPC_JOB_RUNNNING) 
@@ -103,7 +103,7 @@ bool MPCService::queryJobStatus(const std::string &jobId, JobStatus &jobStatus)
     MPC_LOG(INFO) << LOG_DESC("[MPCService][queryJobStatus]") 
                 << "find job"
                 << LOG_KV("jobId", jobId)
-                << LOG_KV("code", jobStatus.code)
+                << LOG_KV("status", jobStatus.status)
                 << LOG_KV("message", jobStatus.message)
                 << LOG_KV("timeCostMs", jobStatus.timeCostMs);
     return true;

From 85a94195288e3ca4e1584f857bb87fb4dedaa55d Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 19 Nov 2024 17:01:01 +0800
Subject: [PATCH 095/120] update dependecies (#94)

---
 cpp/tools/ppc-builder/requirements.txt        |  1 -
 python/ppc_common/ppc_utils/utils.py          | 52 -------------------
 python/requirements.txt                       |  4 +-
 .../wedpr_ml_toolkit/common/utils/utils.py    | 10 ----
 4 files changed, 1 insertion(+), 66 deletions(-)

diff --git a/cpp/tools/ppc-builder/requirements.txt b/cpp/tools/ppc-builder/requirements.txt
index aee2aa42..3d7ce2f7 100644
--- a/cpp/tools/ppc-builder/requirements.txt
+++ b/cpp/tools/ppc-builder/requirements.txt
@@ -1,4 +1,3 @@
 configparser
-requests
 toml
 uuid
diff --git a/python/ppc_common/ppc_utils/utils.py b/python/ppc_common/ppc_utils/utils.py
index 6450a4a4..5c38ee20 100644
--- a/python/ppc_common/ppc_utils/utils.py
+++ b/python/ppc_common/ppc_utils/utils.py
@@ -11,7 +11,6 @@
 from enum import Enum, unique
 
 import jwt
-from ecdsa import SigningKey, SECP256k1, VerifyingKey
 from gmssl import func, sm2, sm3
 from google.protobuf.descriptor import FieldDescriptor
 from jsoncomment import JsonComment
@@ -326,57 +325,6 @@ def make_job_event_message(job_id, job_priority, initiator_agency_id, receiver_a
     return message.encode('utf-8')
 
 
-def sign_with_secp256k1(message, private_key_str):
-    if isinstance(private_key_str, str):
-        private_key = SigningKey.from_string(
-            bytes().fromhex(private_key_str), curve=SECP256k1)
-    else:
-        private_key = SigningKey.from_string(private_key_str, curve=SECP256k1)
-    signature_bytes = private_key.sign(
-        make_hash(message, CryptoType.ECDSA, HashType.BYTES))
-    return str(encode(signature_bytes), 'utf-8')
-
-
-def verify_with_secp256k1(message, signature_str, public_key_str):
-    if isinstance(public_key_str, str):
-        verify_key = VerifyingKey.from_string(
-            bytes().fromhex(public_key_str), curve=SECP256k1)
-    else:
-        verify_key = VerifyingKey.from_string(
-            decode(public_key_str), curve=SECP256k1)
-    return verify_key.verify(decode(signature_str), make_hash(message, CryptoType.ECDSA, HashType.BYTES))
-
-
-def sign_with_sm2(message, private_key_str):
-    sm2_crypt = sm2.CryptSM2(private_key_str, "")
-    random_hex_str = func.random_hex(sm2_crypt.para_len)
-    message_hash = make_hash(message, CryptoType.GM).encode(encoding='utf-8')
-    signature_str = sm2_crypt.sign(message_hash, random_hex_str)
-    return signature_str
-
-
-def verify_with_sm2(message, signature_str, public_key_str):
-    sm2_crypt = sm2.CryptSM2("", public_key_str)
-    message_hash = make_hash(message, CryptoType.GM).encode(encoding='utf-8')
-    return sm2_crypt.verify(signature_str, message_hash)
-
-
-def make_signature(message, private_key_str, crypto_type):
-    if crypto_type == CryptoType.ECDSA:
-        return sign_with_secp256k1(message, private_key_str)
-
-    if crypto_type == CryptoType.GM:
-        return sign_with_sm2(message, private_key_str)
-
-
-def verify_signature(message, signature, public_key_str, crypto_type):
-    if crypto_type == CryptoType.ECDSA:
-        return verify_with_secp256k1(message, signature, public_key_str)
-
-    if crypto_type == CryptoType.GM:
-        return verify_with_sm2(message, signature, public_key_str)
-
-
 def exec_bash_command(cmd):
     """replace commands.get_status_output
 
diff --git a/python/requirements.txt b/python/requirements.txt
index c72f5f73..6fb0a6b7 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -27,9 +27,8 @@ pycryptodome==3.9.9
 pyjwt
 pyyaml
 mysqlclient==2.1.0
-waitress==2.1.2
+waitress==3.0.1
 sqlparse~=0.4.1
-ecdsa==0.19.0
 toolz~=0.11.1
 tenacity==7.0.0
 coincurve~=13.0.0
@@ -47,7 +46,6 @@ requests-toolbelt==0.9.1
 hdfs
 scikit-learn~=0.24.2
 gmpy2
-openpyxl
 networkx
 pydot
 snowland-smx
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
index 303c8dae..84556a46 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
@@ -5,7 +5,6 @@
 import os
 import random
 from wedpr_ml_toolkit.common.utils.constant import Constant
-from urllib.parse import urlencode, urlparse, parse_qs, quote
 
 
 class IdPrefixEnum(Enum):
@@ -22,15 +21,6 @@ def generate_nonce(nonce_len):
     return ''.join(str(random.choice(Constant.NUMERIC_ARRAY)) for _ in range(nonce_len))
 
 
-def add_params_to_url(url, params):
-    parsed_url = urlparse(url)
-    query_params = parse_qs(parsed_url.query)
-    for key, value in params.items():
-        query_params[key] = value
-    new_query = urlencode(query_params, doseq=True)
-    return parsed_url._replace(query=new_query).geturl()
-
-
 def file_exists(_file):
     if os.path.exists(_file) and os.path.isfile(_file):
         return True

From 75b56b730903fcfd09b1debf6904f235c6589725 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 19 Nov 2024 20:52:53 +0800
Subject: [PATCH 096/120] fix select nullptr front when dispatcherMessage
 caused coredump (#95)

* fix waitress version

* fix select nullptr front when dispatcherMessage caused coredump

* fix model can't send message when some gateway down
---
 cpp/ppc-framework/protocol/INodeInfo.h        |  2 +-
 cpp/wedpr-protocol/grpc/client/GrpcClient.cpp | 10 ++++++++
 .../protobuf/src/NodeInfoImpl.h               |  9 +++++++-
 .../gateway/router/GatewayNodeInfoImpl.cpp    | 23 +++++++++++++++----
 .../gateway/router/GatewayRouterManager.cpp   | 10 ++++----
 .../gateway/router/LocalRouter.cpp            |  9 ++++++--
 python/requirements.txt                       |  2 +-
 7 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/cpp/ppc-framework/protocol/INodeInfo.h b/cpp/ppc-framework/protocol/INodeInfo.h
index 68b0f862..ae1b61f7 100644
--- a/cpp/ppc-framework/protocol/INodeInfo.h
+++ b/cpp/ppc-framework/protocol/INodeInfo.h
@@ -57,7 +57,7 @@ class INodeInfo
     virtual void decode(bcos::bytesConstRef data) = 0;
 
     virtual void setFront(std::shared_ptr&& front) = 0;
-    virtual std::shared_ptr const& getFront() const = 0;
+    virtual std::shared_ptr getFront() const = 0;
 
     virtual bool equal(INodeInfo::Ptr const& info)
     {
diff --git a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
index 9eb3429c..682d2f4f 100644
--- a/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
+++ b/cpp/wedpr-protocol/grpc/client/GrpcClient.cpp
@@ -75,6 +75,7 @@ bcos::Error::Ptr GrpcClient::broadCast(
     std::function callback)
 {
     auto result = std::make_shared(0, "");
+    int successCount = 0;
     for (auto const& channel : m_broadcastChannels)
     {
         try
@@ -91,6 +92,10 @@ bcos::Error::Ptr GrpcClient::broadCast(
                 result->setErrorCode(error->errorCode());
                 result->setErrorMessage(result->errorMessage() + error->errorMessage() + "; ");
             }
+            else
+            {
+                successCount++;
+            }
         }
         catch (std::exception const& e)
         {
@@ -99,5 +104,10 @@ bcos::Error::Ptr GrpcClient::broadCast(
                 << LOG_KV("error", boost::diagnostic_information(e));
         }
     }
+    // at least one success
+    if (successCount > 0)
+    {
+        return std::make_shared(0, "success");
+    }
     return result;
 }
\ No newline at end of file
diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
index c7fc72b0..2d4cf90e 100644
--- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
+++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.h
@@ -117,9 +117,14 @@ class NodeInfoImpl : public INodeInfo
 
     void setFront(std::shared_ptr&& front) override
     {
+        bcos::WriteGuard l(x_front);
         m_front = std::move(front);
     }
-    std::shared_ptr const& getFront() const override { return m_front; }
+    std::shared_ptr getFront() const override
+    {
+        bcos::ReadGuard l(x_front);
+        return m_front;
+    }
 
     void toJson(Json::Value& jsonObject) const override;
 
@@ -142,6 +147,8 @@ class NodeInfoImpl : public INodeInfo
 
 private:
     std::shared_ptr m_front;
+    mutable bcos::SharedMutex x_front;
+
     std::set m_components;
     mutable bcos::SharedMutex x_components;
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
index 3b3bb35f..c96edbe5 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
@@ -155,9 +155,10 @@ std::vector> GatewayNodeInfoImpl::choo
     bcos::ReadGuard l(x_nodeList);
     for (auto const& it : m_nodeList)
     {
-        if (it.second->components().count(component))
+        auto front = it.second->getFront();
+        if (front && it.second->components().count(component))
         {
-            result.emplace_back(it.second->getFront());
+            result.emplace_back(front);
         }
         if (!result.empty() && !selectAll)
         {
@@ -175,7 +176,11 @@ std::vector> GatewayNodeInfoImpl::choo
     bcos::ReadGuard l(x_nodeList);
     for (auto const& it : m_nodeList)
     {
-        result.emplace_back(it.second->getFront());
+        auto front = it.second->getFront();
+        if (front)
+        {
+            result.emplace_back(front);
+        }
         if (!result.empty() && !selectAll)
         {
             break;
@@ -194,7 +199,11 @@ std::vector> GatewayNodeInfoImpl::choo
         bcos::ReadGuard l(x_nodeList);
         for (auto const& it : m_nodeList)
         {
-            result.emplace_back(it.second->getFront());
+            auto front = it.second->getFront();
+            if (front)
+            {
+                result.emplace_back(front);
+            }
         }
         return result;
     }
@@ -210,7 +219,11 @@ std::vector> GatewayNodeInfoImpl::choo
         // ignore the fromNode
         if (selectedNode != nullptr && selectedNode->nodeID().toBytes() != fromNode)
         {
-            result.emplace_back(selectedNode->getFront());
+            auto front = selectedNode->getFront();
+            if (front)
+            {
+                result.emplace_back(front);
+            }
         }
         if (!result.empty() && !selectAll)
         {
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
index c287aac9..c464eb2e 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
@@ -103,9 +103,8 @@ void GatewayRouterManager::onReceiveNodeSeqMessage(MessageFace::Ptr msg, WsSessi
     auto statusSeq =
         boost::asio::detail::socket_ops::network_to_host_long(*((uint32_t*)msg->payload()->data()));
     auto p2pMessage = std::dynamic_pointer_cast(msg);
-    auto const& from = (p2pMessage->header()->srcGwNode().size() > 0) ?
-                           p2pMessage->header()->srcGwNode() :
-                           session->nodeId();
+    auto from = (p2pMessage->header()->srcGwNode().size() > 0) ? p2pMessage->header()->srcGwNode() :
+                                                                 session->nodeId();
     auto statusSeqChanged = statusChanged(from, statusSeq);
     if (!statusSeqChanged)
     {
@@ -151,9 +150,8 @@ void GatewayRouterManager::onReceiveRequestNodeStatusMsg(
     MessageFace::Ptr msg, WsSession::Ptr session)
 {
     auto p2pMessage = std::dynamic_pointer_cast(msg);
-    auto const& from = (!p2pMessage->header()->srcGwNode().empty()) ?
-                           p2pMessage->header()->srcGwNode() :
-                           session->nodeId();
+    auto from = (!p2pMessage->header()->srcGwNode().empty()) ? p2pMessage->header()->srcGwNode() :
+                                                               session->nodeId();
 
     auto nodeStatusData = m_localRouter->generateNodeStatus();
     if (!nodeStatusData)
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
index cb6c6a4c..5ada018c 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/LocalRouter.cpp
@@ -98,6 +98,10 @@ bool LocalRouter::dispatcherMessage(
         int i = 0;
         for (auto const& front : frontList)
         {
+            if (!front)
+            {
+                continue;
+            }
             if (i == 0)
             {
                 front->onReceiveMessage(msg->msg(), callback);
@@ -153,9 +157,10 @@ std::vector LocalRouter::chooseReceiver(
     case (uint16_t)RouteType::ROUTE_THROUGH_NODEID:
     {
         auto gatewayInfo = m_routerInfo->nodeInfo(msg->header()->optionalField()->dstNode());
-        if (gatewayInfo != nullptr)
+        auto front = gatewayInfo ? gatewayInfo->getFront() : nullptr;
+        if (gatewayInfo != nullptr && front)
         {
-            receivers.emplace_back(gatewayInfo->getFront());
+            receivers.emplace_back(front);
         }
         return receivers;
     }
diff --git a/python/requirements.txt b/python/requirements.txt
index 6fb0a6b7..4476487c 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -27,7 +27,7 @@ pycryptodome==3.9.9
 pyjwt
 pyyaml
 mysqlclient==2.1.0
-waitress==3.0.1
+waitress==3.0.0
 sqlparse~=0.4.1
 toolz~=0.11.1
 tenacity==7.0.0

From 61646682b95e8793720165c202f5c4b3bddca135 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 20 Nov 2024 09:47:28 +0800
Subject: [PATCH 097/120] fix thread-safe problem (#96)

---
 .../ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp         | 3 +++
 .../ppc-gateway/gateway/router/GatewayRouterManager.cpp        | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
index c96edbe5..c23a695b 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayNodeInfoImpl.cpp
@@ -99,6 +99,7 @@ bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info)
         // update the meta
         if (meta != existedNodeInfo->meta())
         {
+            bcos::WriteGuard l(x_nodeList);
             existedNodeInfo->setMeta(meta);
             GATEWAY_LOG(INFO) << LOG_DESC("tryAddNodeInfo, update the meta, updated nodeInfo")
                               << printNodeInfo(existedNodeInfo);
@@ -107,6 +108,7 @@ bool GatewayNodeInfoImpl::tryAddNodeInfo(INodeInfo::Ptr const& info)
         auto components = info->copiedComponents();
         if (components != existedNodeInfo->copiedComponents())
         {
+            bcos::WriteGuard l(x_nodeList);
             existedNodeInfo->setComponents(
                 std::set(components.begin(), components.end()));
             GATEWAY_LOG(INFO) << LOG_DESC("tryAddNodeInfo, update the components, updated nodeInfo")
@@ -260,6 +262,7 @@ void GatewayNodeInfoImpl::unRegisterTopic(bcos::bytes const& nodeID, std::string
 
 void GatewayNodeInfoImpl::encode(bcos::bytes& data) const
 {
+    bcos::ReadGuard l(x_nodeList);
     encodePBObject(data, m_rawGatewayInfo);
 }
 
diff --git a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
index c464eb2e..19f80e8e 100644
--- a/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
+++ b/cpp/wedpr-transport/ppc-gateway/ppc-gateway/gateway/router/GatewayRouterManager.cpp
@@ -163,7 +163,7 @@ void GatewayRouterManager::onReceiveRequestNodeStatusMsg(
     }
     ROUTER_MGR_LOG(TRACE) << LOG_DESC(
                                  "onReceiveRequestNodeStatusMsg: response the latest nodeStatus")
-                          << LOG_KV("node", printP2PIDElegantly(from));
+                          << LOG_KV("to", printP2PIDElegantly(from));
     m_service->asyncSendMessageByP2PNodeID(
         (uint16_t)GatewayPacketType::ResponseNodeStatus, from, nodeStatusData);
 }

From acbdd04cd6270c8d461e990c8cf896ceef7754e5 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 20 Nov 2024 21:39:31 +0800
Subject: [PATCH 098/120] air node gateway not required to load transport
 config (#97)

* fix pir link

* air node gateway not required to load transport config

* optimize link

* optmize link
---
 cpp/ppc-framework/io/DataBatch.h              |  3 ---
 .../ppc-pir/src/CMakeLists.txt                |  2 +-
 .../ppc-pir/tests/CMakeLists.txt              |  2 +-
 .../ppc-psi/src/bs-ecdh-psi/CMakeLists.txt    |  2 +-
 .../ppc-psi/src/cm2020-psi/CMakeLists.txt     |  2 +-
 .../ppc-psi/src/ecdh-conn-psi/CMakeLists.txt  |  2 +-
 .../ppc-psi/src/ecdh-multi-psi/CMakeLists.txt |  2 +-
 .../src/ecdh-multi-psi/EcdhMultiCache.h       |  2 ++
 .../core/EcdhMultiPSIMaster.cpp               |  2 ++
 .../core/EcdhMultiPSIPartner.cpp              |  2 ++
 .../ppc-psi/src/ecdh-psi/CMakeLists.txt       |  2 +-
 .../ppc-psi/src/labeled-psi/CMakeLists.txt    |  2 +-
 .../ppc-psi/src/psi-framework/CMakeLists.txt  |  2 +-
 .../ppc-psi/src/ra2018-psi/CMakeLists.txt     |  2 +-
 .../ppc-psi/tests/CMakeLists.txt              |  4 ++--
 .../ppc-crypto/src/CMakeLists.txt             |  4 ++--
 cpp/wedpr-helper/libhelper/CMakeLists.txt     |  2 +-
 cpp/wedpr-helper/ppc-tools/CMakeLists.txt     |  2 +-
 .../ppc-tools/src/config/PPCConfig.cpp        | 20 ++++++++++++-------
 .../ppc-tools/src/config/PPCConfig.h          |  6 +++---
 .../air-node/AirNodeInitializer.cpp           |  2 +-
 cpp/wedpr-main/air-node/CMakeLists.txt        |  2 +-
 cpp/wedpr-main/gateway/GatewayInitializer.cpp |  2 +-
 cpp/wedpr-main/pro-node/CMakeLists.txt        |  2 +-
 cpp/wedpr-storage/ppc-io/src/CMakeLists.txt   |  2 +-
 .../ppc-storage/src/CMakeLists.txt            |  2 +-
 26 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/cpp/ppc-framework/io/DataBatch.h b/cpp/ppc-framework/io/DataBatch.h
index a8c394d1..310fd1c0 100644
--- a/cpp/ppc-framework/io/DataBatch.h
+++ b/cpp/ppc-framework/io/DataBatch.h
@@ -20,7 +20,6 @@
 #pragma once
 #include "../Common.h"
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -210,8 +209,6 @@ class DataBatch
         }
         m_data->clear();
         m_data.reset();
-        // free after release
-        MallocExtension::instance()->ReleaseFreeMemory();
     }
 
 private:
diff --git a/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
index 03844ecb..d1b6a28a 100644
--- a/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/src/CMakeLists.txt
@@ -28,4 +28,4 @@ add_library(${PIR_TARGET} ${SOURCES} ${OUT_TARS_H_LIST})
 target_include_directories(${PIR_TARGET} PUBLIC
         $)
 
-target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${IO_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} TBB::tbb TCMalloc)
+target_link_libraries(${PIR_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET})
diff --git a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
index dc64b608..ba2cabe0 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-pir/tests/CMakeLists.txt
@@ -6,5 +6,5 @@ set(TEST_BINARY_NAME test-ppc-pir)
 add_executable(${TEST_BINARY_NAME} ${SOURCES})
 target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
 
-target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${CRYPTO_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${PIR_TARGET} ${PROTOCOL_TARGET} ${BOOST_UNIT_TEST})
+target_link_libraries(${TEST_BINARY_NAME} PUBLIC ${PIR_TARGET} ${BOOST_UNIT_TEST})
 add_test(NAME test-pir WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt
index ed1d5866..25163dcd 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/CMakeLists.txt
@@ -1,4 +1,4 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${BS_ECDH_PSI_TARGET} ${SRCS})
 
-target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb TCMalloc)
+target_link_libraries(${BS_ECDH_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} wedpr_ecc TBB::tbb)
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt
index 58810e52..74fdf391 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CMakeLists.txt
@@ -28,4 +28,4 @@ add_library(${CM2020_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${CM2020_PSI_TARGET} PUBLIC
         $)
 
-target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
+target_link_libraries(${CM2020_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt
index 9915c66d..2affec1b 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/CMakeLists.txt
@@ -28,4 +28,4 @@ add_library(${ECDH_CONN_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${ECDH_CONN_PSI_TARGET} PUBLIC
         $)
 
-target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb TCMalloc)
\ No newline at end of file
+target_link_libraries(${ECDH_CONN_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} ${TOOLS_TARGET} ${PROTOBUF_TARGET} TBB::tbb)
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt
index a8d547c2..a1097acd 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/CMakeLists.txt
@@ -28,4 +28,4 @@ add_library(${ECDH_MULTI_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${ECDH_MULTI_PSI_TARGET} PUBLIC
         $)
 
-target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${FRONT_TARGET} ${BCOS_UTILITIES_TARGET} ${TARS_PROTOCOL_TARGET} ${CRYPTO_TARGET} TBB::tbb TCMalloc)
\ No newline at end of file
+target_link_libraries(${ECDH_MULTI_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${CRYPTO_TARGET} TBB::tbb)
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
index a127630f..a4fc1f60 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.h
@@ -280,6 +280,8 @@ class CalculatorCache : public std::enable_shared_from_this
         for (auto const& it : m_plainData)
         {
             it->release();
+            // free after release
+            MallocExtension::instance()->ReleaseFreeMemory();
         }
         m_cipherRef.clear();
         std::map().swap(m_cipherRef);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index ee4769d8..97295541 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -164,6 +164,8 @@ void EcdhMultiPSIMaster::blindData()
                 });
             // can release databatch after encrypted
             dataBatch->release();
+            // free after release
+            MallocExtension::instance()->ReleaseFreeMemory();
             ECDH_MASTER_LOG(INFO) << LOG_DESC("blindData: encrypt data success")
                                   << LOG_KV("dataSize", cipher.size()) << LOG_KV("task", m_taskID)
                                   << LOG_KV("seq", seq)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
index 63463024..c51617fc 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
@@ -146,6 +146,8 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
                                    << LOG_KV("size", dataBatch->size())
                                    << printTaskInfo(m_taskState->task()) << LOG_KV("seq", seq);
             dataBatch->release();
+            // free after release
+            MallocExtension::instance()->ReleaseFreeMemory();
         } while (!m_taskState->sqlReader());
     }
     catch (std::exception& e)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt
index 261ee37e..e3876908 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/CMakeLists.txt
@@ -1,3 +1,3 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${ECDH_2PC_PSI_TARGET} ${SRCS})
-target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET} TCMalloc)
\ No newline at end of file
+target_link_libraries(${ECDH_2PC_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt
index a0a3c07e..5c029ce3 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/CMakeLists.txt
@@ -27,4 +27,4 @@ add_library(${LABELED_PSI_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${LABELED_PSI_TARGET} PUBLIC
         $)
 
-target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb TCMalloc)
+target_link_libraries(${LABELED_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} apsi SEAL::seal Kuku::kuku TBB::tbb)
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
index 173b326f..74d893ce 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/CMakeLists.txt
@@ -26,4 +26,4 @@ file(GLOB_RECURSE SRCS *.cpp)
 add_library(${PSI_FRAMEWORK_TARGET} ${SRCS} ${OUT_TARS_H_LIST})
 target_include_directories(${PSI_FRAMEWORK_TARGET} PUBLIC
     $)
-target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} ${TARS_PROTOCOL_TARGET} TCMalloc)
+target_link_libraries(${PSI_FRAMEWORK_TARGET} PUBLIC TCMalloc ${FRONT_TARGET} ${PROTOCOL_TARGET} ${TARS_PROTOCOL_TARGET} ${IO_TARGET})
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt
index d38c319b..0d00d3ca 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/CMakeLists.txt
@@ -1,3 +1,3 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${RA2018_PSI_TARGET} ${SRCS})
-target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TARS_PROTOCOL_TARGET} ${TOOLS_TARGET})
\ No newline at end of file
+target_link_libraries(${RA2018_PSI_TARGET} PUBLIC ${PSI_FRAMEWORK_TARGET} ${TOOLS_TARGET})
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt b/cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt
index 26b00337..327c5bc0 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt
+++ b/cpp/wedpr-computing/ppc-psi/tests/CMakeLists.txt
@@ -6,6 +6,6 @@ set(TEST_BINARY_NAME test-ppc-psi)
 add_executable(${TEST_BINARY_NAME} ${SOURCES})
 target_include_directories(${TEST_BINARY_NAME} PRIVATE .)
 
-target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
-target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${PROTOCOL_TARGET} ${IO_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
+target_link_libraries(${TEST_BINARY_NAME} ${BS_ECDH_PSI_TARGET} ${ECDH_CONN_PSI_TARGET} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
+target_link_libraries(${TEST_BINARY_NAME} ${RA2018_PSI_TARGET} ${LABELED_PSI_TARGET} ${CM2020_PSI_TARGET} ${ECDH_2PC_PSI_TARGET} ${LABELED_PSI_TARGET} ${CRYPTO_TARGET} ${BOOST_UNIT_TEST})
 add_test(NAME test-ppc-psi WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} COMMAND ${TEST_BINARY_NAME})
\ No newline at end of file
diff --git a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
index 45f4b2c7..a11c824f 100644
--- a/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
+++ b/cpp/wedpr-crypto/ppc-crypto/src/CMakeLists.txt
@@ -6,9 +6,9 @@ find_package(OpenSSL REQUIRED)
 message(STATUS "OPENSSL_INCLUDE_DIR: ${OPENSSL_INCLUDE_DIR}")
 message(STATUS "OPENSSL_LIBRARIES: ${OPENSSL_LIBRARIES}")
 
-target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB} TCMalloc)
+target_link_libraries(${CRYPTO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${CRYPTO_CORE_TARGET} OpenSSL::Crypto unofficial-sodium::sodium TBB::tbb ${CPU_FEATURES_LIB})
 
 if (ENABLE_IPP_CRYPTO)
     find_package(ipp-crypto REQUIRED)
     target_link_libraries(${CRYPTO_TARGET} PUBLIC ipp-crypto::crypto_mb)
-endif ()
\ No newline at end of file
+endif ()
diff --git a/cpp/wedpr-helper/libhelper/CMakeLists.txt b/cpp/wedpr-helper/libhelper/CMakeLists.txt
index 77e19f19..43b4a8e1 100644
--- a/cpp/wedpr-helper/libhelper/CMakeLists.txt
+++ b/cpp/wedpr-helper/libhelper/CMakeLists.txt
@@ -5,4 +5,4 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_library(${HELPER_TARGET} ${SRCS})
 find_package(Boost COMPONENTS filesystem program_options)
-target_link_libraries(${HELPER_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} Boost::filesystem Boost::program_options)
\ No newline at end of file
+target_link_libraries(${HELPER_TARGET} PUBLIC Boost::filesystem Boost::program_options)
\ No newline at end of file
diff --git a/cpp/wedpr-helper/ppc-tools/CMakeLists.txt b/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
index 26cb4e2f..5e8810c2 100644
--- a/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
+++ b/cpp/wedpr-helper/ppc-tools/CMakeLists.txt
@@ -3,7 +3,7 @@ aux_source_directory(src/config SRCS)
 
 add_library(${TOOLS_TARGET} ${SRCS})
 
-target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})
+target_link_libraries(${TOOLS_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BOOST_SERIALIZATION} ${CPU_FEATURES_LIB})
 
 if (TESTS)
     enable_testing()
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index 5d832da1..d19b6b37 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -30,10 +30,11 @@ using namespace ppc::protocol;
 using namespace ppc::storage;
 using namespace bcos;
 
-void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
+void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport)
 {
     // load the network config
-    PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config");
+    PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load the network config")
+                        << LOG_KV("requireTransport", requireTransport);
     // gateway default enable-ssl
     loadNetworkConfig(
         m_gatewayConfig.networkConfig, _pt, "gateway", NetworkConfig::DefaultRpcListenPort, false);
@@ -73,11 +74,16 @@ void PPCConfig::loadGatewayConfig(boost::property_tree::ptree const& _pt)
             InvalidConfig() << errinfo_comment(
                 "The value of gateway.seq_sync_period_ms must no little than 3s"));
     }
-    // load the grpcConfig
-    m_grpcConfig = loadGrpcConfig("transport", _pt);
-    // load the GrpcServerConfig
-    loadEndpointConfig(
-        m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
+    // only need load grpc config when requireTransport
+    if (requireTransport)
+    {
+        PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig: load grpc config");
+        // load the grpcConfig
+        m_grpcConfig = loadGrpcConfig("transport", _pt);
+        // load the GrpcServerConfig
+        loadEndpointConfig(
+            m_gatewayConfig.grpcServerConfig->mutableEndPoint(), false, "transport", _pt);
+    }
     // the agencyID
     m_agencyID = _pt.get("agency.id", "");
     if (m_agencyID.empty())
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index e8a0497f..89866d48 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -153,12 +153,12 @@ class PPCConfig
         loadRpcConfig(iniConfig);
     }
 
-    void loadGatewayConfig(std::string const& _configPath)
+    void loadGatewayConfig(std::string const& _configPath, bool requireTransport)
     {
         PPCConfig_LOG(INFO) << LOG_DESC("loadGatewayConfig") << LOG_KV("path", _configPath);
         boost::property_tree::ptree iniConfig;
         boost::property_tree::read_ini(_configPath, iniConfig);
-        loadGatewayConfig(iniConfig);
+        loadGatewayConfig(iniConfig, requireTransport);
     }
 
     void loadFrontConfig(bool requireTransport,
@@ -180,7 +180,7 @@ class PPCConfig
         loadNetworkConfig(m_rpcConfig, _pt, "rpc", NetworkConfig::DefaultRpcListenPort, true);
     }
 
-    virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt);
+    virtual void loadGatewayConfig(boost::property_tree::ptree const& _pt, bool requireTransport);
 
 
     NetworkConfig const& rpcConfig() const { return m_rpcConfig; }
diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
index 4c4ad2f5..38c6709d 100644
--- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
+++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
@@ -79,7 +79,7 @@ void AirNodeInitializer::initGateway(std::string const& _configPath)
     INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
     // not specify the certPath in air-mode
     auto config = m_nodeInitializer->config();
-    config->loadGatewayConfig(_configPath);
+    config->loadGatewayConfig(_configPath, false);
 
     auto threadPool = std::make_shared(
         "gateway", config->gatewayConfig().networkConfig.threadPoolSize);
diff --git a/cpp/wedpr-main/air-node/CMakeLists.txt b/cpp/wedpr-main/air-node/CMakeLists.txt
index 53e7bfb9..263c69f8 100644
--- a/cpp/wedpr-main/air-node/CMakeLists.txt
+++ b/cpp/wedpr-main/air-node/CMakeLists.txt
@@ -1,4 +1,4 @@
 aux_source_directory(. SRC_LIST)
 
 add_executable(${AIR_BINARY_NAME} ${SRC_LIST})
-target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
\ No newline at end of file
+target_link_libraries(${AIR_BINARY_NAME} PUBLIC ${GATEWAY_TARGET} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
\ No newline at end of file
diff --git a/cpp/wedpr-main/gateway/GatewayInitializer.cpp b/cpp/wedpr-main/gateway/GatewayInitializer.cpp
index c5610fda..8a2950bc 100644
--- a/cpp/wedpr-main/gateway/GatewayInitializer.cpp
+++ b/cpp/wedpr-main/gateway/GatewayInitializer.cpp
@@ -46,7 +46,7 @@ void GatewayInitializer::init(std::string const& _configPath)
     INIT_LOG(INFO) << LOG_DESC("initGateway: ") << _configPath;
     auto config = std::make_shared();
 
-    config->loadGatewayConfig(_configPath);
+    config->loadGatewayConfig(_configPath, true);
     auto threadPool = std::make_shared(
         "gateway", config->gatewayConfig().networkConfig.threadPoolSize);
 
diff --git a/cpp/wedpr-main/pro-node/CMakeLists.txt b/cpp/wedpr-main/pro-node/CMakeLists.txt
index f83f6c10..b4745a32 100644
--- a/cpp/wedpr-main/pro-node/CMakeLists.txt
+++ b/cpp/wedpr-main/pro-node/CMakeLists.txt
@@ -3,4 +3,4 @@ include_directories(${CMAKE_SOURCE_DIR})
 aux_source_directory(./ SRC_LIST)
 add_executable(${PRO_BINARY_NAME} ${SRC_LIST})
 
-target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET} TCMalloc)
+target_link_libraries(${PRO_BINARY_NAME} ${RPC_TARGET} ${INIT_LIB} ${HELPER_TARGET})
diff --git a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
index a730f0de..7f4be366 100644
--- a/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-io/src/CMakeLists.txt
@@ -1,4 +1,4 @@
 file(GLOB_RECURSE SRCS *.cpp)
 add_library(${IO_TARGET} ${SRCS})
 # Note: the DataBatch depends on tbb
-target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${BCOS_BOOSTSSL_TARGET} TCMalloc ${HDFS_LIB} ${CPU_FEATURES_LIB})
\ No newline at end of file
+target_link_libraries(${IO_TARGET} PUBLIC ${BCOS_UTILITIES_TARGET} ${HDFS_LIB} ${CPU_FEATURES_LIB})
\ No newline at end of file
diff --git a/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt b/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
index ce85047c..5e21c8da 100644
--- a/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
+++ b/cpp/wedpr-storage/ppc-storage/src/CMakeLists.txt
@@ -4,4 +4,4 @@ add_library(${STORAGE_TARGET} ${SRCS})
 find_package(redis++ REQUIRED)
 find_package(unofficial-mysql-connector-cpp REQUIRED)
 
-target_link_libraries(${STORAGE_TARGET} PUBLIC ${BCOS_BOOSTSSL_TARGET} redis++::redis++_static TCMalloc unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})
+target_link_libraries(${STORAGE_TARGET} PUBLIC redis++::redis++_static unofficial::mysql-connector-cpp::connector resolv ${HDFS_LIB})

From 77c138f6b91dc40259edef6e78b837efb04e9670 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 22 Nov 2024 16:30:44 +0800
Subject: [PATCH 099/120] fix wedpr_ml_toolkit (#98)

* fix wedpr_ml_toolkit

* upgrade wedpr_ml_toolkit

* remove pyproject.toml
---
 python/wedpr_ml_toolkit/pyproject.toml        | 87 -------------------
 python/wedpr_ml_toolkit/requirements.txt      |  4 +-
 python/wedpr_ml_toolkit/setup.py              |  2 +-
 .../wedpr_ml_toolkit/_version.py              |  8 --
 .../wedpr_ml_toolkit/common/utils/constant.py |  2 +-
 5 files changed, 4 insertions(+), 99 deletions(-)
 delete mode 100644 python/wedpr_ml_toolkit/pyproject.toml
 delete mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py

diff --git a/python/wedpr_ml_toolkit/pyproject.toml b/python/wedpr_ml_toolkit/pyproject.toml
deleted file mode 100644
index dacbc3ad..00000000
--- a/python/wedpr_ml_toolkit/pyproject.toml
+++ /dev/null
@@ -1,87 +0,0 @@
-# autoflake is used for autoformatting Python code
-#
-# ref: https://github.com/PyCQA/autoflake#readme
-#
-[tool.autoflake]
-ignore-init-module-imports = true
-remove-all-unused-imports = true
-remove-duplicate-keys = true
-#remove-unused-variables = true
-
-
-# isort is used for autoformatting Python code
-#
-# ref: https://pycqa.github.io/isort/
-#
-[tool.isort]
-profile = "black"
-
-
-# black is used for autoformatting Python code
-#
-# ref: https://black.readthedocs.io/en/stable/
-#
-[tool.black]
-skip-string-normalization = true
-# target-version should be all supported versions, see
-# https://github.com/psf/black/issues/751#issuecomment-473066811
-target_version = [
-    "py38",
-    "py39",
-    "py310",
-    "py311",
-]
-
-
-# pytest is used for running Python based tests
-#
-# ref: https://docs.pytest.org/en/stable/
-#
-[tool.pytest.ini_options]
-addopts = "--verbose --color=yes --durations=10 --cov=wedpr_ml_toolkit"
-asyncio_mode = "auto"
-testpaths = [
-    "test"
-]
-
-
-# pytest-cov / coverage is used to measure code coverage of tests
-#
-# ref: https://coverage.readthedocs.io/en/stable/config.html
-#
-[tool.coverage.run]
-omit = [
-  "test/**",
-]
-
-
-# tbump is used to simplify and standardize the release process when updating
-# the version, making a git commit and tag, and pushing changes.
-#
-# ref: https://github.com/your-tools/tbump#readme
-#
-[tool.tbump]
-github_url = "https://github.com/WeBankBlockchain/WeDPR-Component"
-
-[tool.tbump.version]
-current = "1.0.0.dev"
-regex = '''
-    (?P\d+)
-    \.
-    (?P\d+)
-    \.
-    (?P\d+)
-    (?P
((a|b|rc)\d+)|)
-    \.?
-    (?P(?<=\.)dev\d*|)
-'''
-
-[tool.tbump.git]
-message_template = "Bump to {new_version}"
-tag_template = "{new_version}"
-
-[[tool.tbump.file]]
-src = "setup.py"
-
-[[tool.tbump.file]]
-src = "wedpr_ml_toolkit/_version.py"
diff --git a/python/wedpr_ml_toolkit/requirements.txt b/python/wedpr_ml_toolkit/requirements.txt
index 2f24d54b..a9a92a16 100644
--- a/python/wedpr_ml_toolkit/requirements.txt
+++ b/python/wedpr_ml_toolkit/requirements.txt
@@ -1,5 +1,5 @@
-hdfs
+setuptools>=40.0.0
+hdfs>=2.7.2
 requests~=2.31.0
 requests_toolbelt==0.9.1
-wedpr-python-gateway-sdk
 
diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
index f456a389..ff256ff0 100644
--- a/python/wedpr_ml_toolkit/setup.py
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr_ml_toolkit',
     packages=find_packages(),
-    version="1.0.0.dev",
+    version="1.0.0.dev-20241125",
     description="wedpr-ml-toolkit: The ML toolkit for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py
deleted file mode 100644
index ec0d7829..00000000
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/_version.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# __version__ should be updated using tbump, based on configuration in
-# pyproject.toml, according to instructions in RELEASE.md.
-#
-__version__ = "1.0.0.dev"
-
-# version_info looks like (1, 2, 3, "dev") if __version__ is 1.2.3.dev
-version_info = tuple(int(p) if p.isdigit()
-                     else p for p in __version__.split("."))
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
index b323d962..2f3ded88 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
@@ -3,7 +3,7 @@
 class Constant:
     NUMERIC_ARRAY = [i for i in range(10)]
     HTTP_STATUS_OK = 200
-    WEDPR_API_PREFIX = '/api/wedpr/v3/'
+    WEDPR_API_PREFIX = 'api/wedpr/v3/'
     DEFAULT_SUBMIT_JOB_URI = f'{WEDPR_API_PREFIX}project/submitJob'
     DEFAULT_QUERY_JOB_STATUS_URL = f'{WEDPR_API_PREFIX}project/queryJobByCondition'
     DEFAULT_QUERY_JOB_DETAIL_URL = f'{WEDPR_API_PREFIX}scheduler/queryJobDetail'

From 02d5d902bc3b84e48614ee0291a26ba660b0df1d Mon Sep 17 00:00:00 2001
From: octopus <912554887@qq.com>
Date: Fri, 22 Nov 2024 16:45:32 +0800
Subject: [PATCH 100/120] fix bug

---
 .../ppc-mpc/src/MPCService.cpp                | 29 +++++++++++--------
 cpp/wedpr-computing/ppc-mpc/src/MPCService.h  |  2 +-
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index b7138cb4..1bce96ee 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -347,26 +347,31 @@ void MPCService::asyncRunMpcRpc(Json::Value const& request, RespFunc func)
         {  
             response["code"] = MPC_SUCCESS;
             response["message"] = "success";
+
+            MPC_LOG(INFO) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc submit job successfully")
+                      << LOG_KV("request", request.toStyledString())
+                      << LOG_KV("response", response.toStyledString());
+
+            // async run mpc job
+            m_threadPool->enqueue([self = weak_from_this(), jobInfo]() {
+                auto service = self.lock();
+                if (!service)
+                {
+                    MPC_LOG(ERROR) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc service is null");
+                    return;
+                }
+                service->runMpcRpcByJobInfo(jobInfo);
+            });
         }
         else 
         {
             response["code"] = MPC_DUPLICATED;
             response["message"] = "duplicated submit job";
-        }
 
-        MPC_LOG(INFO) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc submit job successfully")
+            MPC_LOG(INFO) << LOG_DESC("[MPCService][asyncRunMpcRpc]") << LOG_DESC("async run mpc duplicated submit job")
                       << LOG_KV("request", request.toStyledString())
                       << LOG_KV("response", response.toStyledString());
-
-        // async run mpc job
-        m_threadPool->enqueue([self = weak_from_this(), jobInfo]() {
-            auto service = self.lock();
-            if (!service)
-            {
-                return;
-            }
-            service->runMpcRpcByJobInfo(jobInfo);
-        });
+        }
     }
     catch (const std::exception& e)
     {
diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
index ebc89d45..a91b0cfa 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.h
@@ -34,7 +34,7 @@
 
 namespace ppc::mpc
 {
-class MPCService: std::enable_shared_from_this
+class MPCService: public std::enable_shared_from_this
 {
 public:
     using Ptr = std::shared_ptr;

From 89b8a6bb4df6e41842b3771819c5800b6a187765 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 25 Nov 2024 15:59:05 +0800
Subject: [PATCH 101/120] move checkHostResource to psi module for notifing
 peers (#100)

* fix mysql gone

* add try-except for model-controller

* upgrade setuptools to 70.0.0

* change minNeedMemoryGB to 1

* clear the generated psi output file if run-failed

* move checkHostResource to psi module for notifing peers

* register-component in case of scheduler model-nodes in other wedpr-envs

* add components information to the serviceMeta

* query status from the cache firstly
---
 cpp/ppc-framework/io/DataResourceLoader.h     |   6 +
 cpp/tools/build_ppc.sh                        |   4 +-
 cpp/tools/build_wedpr_cem.sh                  |   2 -
 cpp/tools/ppc-builder/src/tpl/config.ini.node |   4 +-
 cpp/wedpr-computing/ppc-psi/src/Common.h      |  15 +++
 cpp/wedpr-computing/ppc-psi/src/PSIConfig.h   |  19 ++-
 .../src/bs-ecdh-psi/BsEcdhPSIFactory.h        |   7 +-
 .../ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp |   3 +-
 .../ppc-psi/src/cm2020-psi/CM2020PSIConfig.h  |   4 +-
 .../ppc-psi/src/cm2020-psi/CM2020PSIFactory.h |   4 +-
 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp  |   2 +
 .../src/ecdh-conn-psi/EcdhConnPSIConfig.h     |   3 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIConfig.h   |   6 +-
 .../ecdh-multi-psi/EcdhMultiPSIFactory.cpp    |   4 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIFactory.h  |   3 +-
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp   |  73 +++++-----
 .../ppc-psi/src/ecdh-psi/EcdhPSIConfig.h      |   5 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp   |   4 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIFactory.h     |   3 +-
 .../src/labeled-psi/LabeledPSIConfig.h        |   4 +-
 .../src/labeled-psi/LabeledPSIFactory.cpp     |   9 +-
 .../src/labeled-psi/LabeledPSIFactory.h       |   2 +-
 .../src/labeled-psi/LabeledPSIImpl.cpp        | 118 ++++++++--------
 .../ppc-psi/src/psi-framework/TaskState.cpp   |  20 +++
 .../ppc-psi/src/psi-framework/TaskState.h     |   3 +
 .../ppc-psi/src/ra2018-psi/RA2018PSIConfig.h  |   5 +-
 .../src/ra2018-psi/RA2018PSIFactory.cpp       |   7 +-
 .../ppc-psi/src/ra2018-psi/RA2018PSIFactory.h |   3 +-
 .../tests/ra2018-psi/mock/EcdhPSIFixture.h    |   5 +-
 .../tests/ra2018-psi/mock/RA2018PSIFixture.h  |   8 +-
 .../ppc-tools/src/common/MemInfo.h            |   2 +-
 .../ppc-tools/src/config/NetworkConfig.h      |   1 -
 .../ppc-tools/src/config/PPCConfig.cpp        |  13 +-
 .../ppc-tools/src/config/PPCConfig.h          |   3 +
 cpp/wedpr-initializer/Initializer.cpp         |  12 +-
 .../ppc-io/src/DataResourceLoaderImpl.h       |   5 +-
 cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp       |  18 ---
 cpp/wedpr-transport/ppc-rpc/src/Rpc.h         |   6 -
 .../ppc-rpc/src/RpcFactory.cpp                |   1 -
 .../sdk/jni/transport/impl/TransportImpl.java |  15 ++-
 .../sdk/jni/transport/model/ServiceMeta.java  |   8 ++
 .../ppc_common/deps_services/mysql_storage.py |   9 +-
 .../deps_services/sql_storage_api.py          |   3 +-
 python/ppc_model/common/initializer.py        |   2 +
 python/ppc_model/conf/application-sample.yml  |   1 +
 .../network/http/model_controller.py          | 126 +++++++++++-------
 .../secure_lgbm/vertical/active_party.py      |   2 -
 .../secure_lgbm/vertical/passive_party.py     |   2 -
 python/ppc_model/task/task_manager.py         |   8 +-
 python/wedpr_ml_toolkit/requirements.txt      |   2 +-
 50 files changed, 358 insertions(+), 236 deletions(-)

diff --git a/cpp/ppc-framework/io/DataResourceLoader.h b/cpp/ppc-framework/io/DataResourceLoader.h
index 9f3db868..25a36492 100644
--- a/cpp/ppc-framework/io/DataResourceLoader.h
+++ b/cpp/ppc-framework/io/DataResourceLoader.h
@@ -50,5 +50,11 @@ class DataResourceLoader
 
     virtual void checkResourceExists(ppc::protocol::DataResourceDesc::ConstPtr _desc,
         ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0;
+
+    virtual void deleteResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
+        ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0;
+
+    virtual void renameResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
+        std::string const& _new_path, ppc::storage::FileStorage::Ptr const& _storage = nullptr) = 0;
 };
 }  // namespace ppc::io
\ No newline at end of file
diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh
index c7ec3f7a..401169ad 100644
--- a/cpp/tools/build_ppc.sh
+++ b/cpp/tools/build_ppc.sh
@@ -411,6 +411,8 @@ generate_node_config_ini() {
     ; data_location = data
     ; task_timeout_minutes = 180
     ; thread_count = 8
+    ; GB, reject task if available memory is less than min_needed_memory
+    ; min_needed_memory = 1
 
 [transport]
    ; the endpoint information
@@ -458,8 +460,6 @@ generate_node_config_ini() {
     ; ssl connection switch, if disable the ssl connection, default: false
     ;disable_ssl = true
     ;disable_db = true
-    ; GB, reject task if available memory is less than min_needed_memory
-    ; min_needed_memory = 5
 
 [cert]
     ; directory the certificates located in
diff --git a/cpp/tools/build_wedpr_cem.sh b/cpp/tools/build_wedpr_cem.sh
index 0a2a6ed2..c972e559 100644
--- a/cpp/tools/build_wedpr_cem.sh
+++ b/cpp/tools/build_wedpr_cem.sh
@@ -351,8 +351,6 @@ generate_config_ini() {
     sm_ssl=${sm_mode}
     ; ssl connection switch, if disable the ssl connection, default: false
     ;disable_ssl = true
-    ; GB, reject task if available memory is less than min_needed_memory
-    ; min_needed_memory = 5
 
 [hdfs_storage]
     ; the hdfs configuration
diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node
index d47d7883..2f0b7ab6 100644
--- a/cpp/tools/ppc-builder/src/tpl/config.ini.node
+++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node
@@ -9,6 +9,8 @@
     ; data_location = data
     ; task_timeout_minutes = 180
     ; thread_count = 8
+    ; GB, reject task if available memory is less than min_needed_memory
+    ; min_needed_memory = 1
 
 [crypto]
     sm_crypto = false
@@ -23,8 +25,6 @@
     ; ssl connection switch, if disable the ssl connection, default: true
     disable_ssl = true
     ; disable_db = true
-    ; GB, reject task if available memory is less than min_needed_memory
-    ; min_needed_memory = 5
 
 [cert]
     ; directory the certificates located in
diff --git a/cpp/wedpr-computing/ppc-psi/src/Common.h b/cpp/wedpr-computing/ppc-psi/src/Common.h
index 392e63d2..4b95a8f5 100644
--- a/cpp/wedpr-computing/ppc-psi/src/Common.h
+++ b/cpp/wedpr-computing/ppc-psi/src/Common.h
@@ -18,6 +18,8 @@
  * @date 2022-12-26
  */
 #pragma once
+#include "ppc-framework/Common.h"
+#include "ppc-tools/src/common/MemInfo.h"
 #include 
 #include 
 #define PSI_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("PSI")
@@ -149,4 +151,17 @@ inline std::ostream& operator<<(std::ostream& _out, CacheState const& _state)
     }
     return _out;
 }
+
+inline void checkHostResource(uint64_t minNeededMemoryGB)
+{
+#ifdef __linux__
+    if (!ppc::tools::hasAvailableMem(minNeededMemoryGB))
+    {
+        BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                  "Lack of memory, please try again later. minNeededMemoryGB: " +
+                                  std::to_string(minNeededMemoryGB)));
+    }
+#endif
+}
+
 }  // namespace ppc::psi
diff --git a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
index af8cdf64..9bfd1650 100644
--- a/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/PSIConfig.h
@@ -37,26 +37,31 @@ class PSIConfig
 public:
     using Ptr = std::shared_ptr;
     PSIConfig(ppc::protocol::TaskAlgorithmType _algorithmType,
-        ppc::io::DataResourceLoader::Ptr _dataResourceLoader)
-      : m_algorithmType(_algorithmType), m_dataResourceLoader(std::move(_dataResourceLoader))
+        ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t minNeededMemoryGB = 1)
+      : m_algorithmType(_algorithmType),
+        m_dataResourceLoader(std::move(_dataResourceLoader)),
+        m_minNeededMemoryGB(minNeededMemoryGB)
     {}
 
     PSIConfig(ppc::protocol::TaskAlgorithmType _algorithmType, const std::string& _selfParty,
         ppc::front::FrontInterface::Ptr _front,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory,
-        ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes)
+        ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
+        uint32_t minNeededMemoryGB = 1)
       : m_algorithmType(_algorithmType),
         m_selfParty(_selfParty),
         m_front(std::move(_front)),
         m_ppcMsgFactory(std::move(_ppcMsgFactory)),
         m_dataResourceLoader(std::move(_dataResourceLoader)),
         m_networkTimeout(_holdingMessageMinutes * 60 * 1000),
-        m_taskExpireTime(m_networkTimeout)
+        m_taskExpireTime(m_networkTimeout),
+        m_minNeededMemoryGB(minNeededMemoryGB)
     {
         PSI_LOG(INFO) << LOG_DESC("create PSIConfig") << LOG_KV("algorithmType", m_algorithmType)
                       << LOG_KV("holdingMessageMinutes", _holdingMessageMinutes)
                       << LOG_KV("networkTimeout", m_networkTimeout)
-                      << LOG_KV("taskExpireTime", m_taskExpireTime);
+                      << LOG_KV("taskExpireTime", m_taskExpireTime)
+                      << LOG_KV("minNeededMemoryGB", m_minNeededMemoryGB);
     }
 
     virtual ~PSIConfig() = default;
@@ -128,6 +133,8 @@ class PSIConfig
 
     std::vector agencyList() const { return m_front->agencies(); }
 
+    uint32_t minNeededMemoryGB() const { return m_minNeededMemoryGB; }
+
 protected:
     ppc::front::PPCMessageFace::Ptr generatePPCMsg(
         std::string const& _taskID, PSIMessageInterface::Ptr const& _msg, uint32_t _seq)
@@ -161,5 +168,7 @@ class PSIConfig
 
     // the task-expire time
     int m_taskExpireTime = 10000;
+
+    uint32_t m_minNeededMemoryGB = 1;
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
index 272fbd1d..cd3adf3c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIFactory.h
@@ -34,10 +34,11 @@ class BsEcdhPSIFactory
     virtual ~BsEcdhPSIFactory() = default;
 
     virtual BsEcdhPSIImpl::Ptr buildBsEcdhPSI(bcos::ThreadPool::Ptr _threadPool,
-        ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _timeoutMinutes)
+        ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _timeoutMinutes,
+        uint32_t minNeededMemoryGB = 1)
     {
-        auto config = std::make_shared(
-            ppc::protocol::TaskAlgorithmType::BS_ECDH_PSI, std::move(_dataResourceLoader));
+        auto config = std::make_shared(ppc::protocol::TaskAlgorithmType::BS_ECDH_PSI,
+            std::move(_dataResourceLoader), minNeededMemoryGB);
         return std::make_shared(
             std::move(config), std::move(_threadPool), _timeoutMinutes);
     }
diff --git a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp
index 1985ddcb..64cccd0a 100644
--- a/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/bs-ecdh-psi/BsEcdhPSIImpl.cpp
@@ -147,7 +147,8 @@ void BsEcdhPSIImpl::asyncRunTask(
         // init task state
         auto taskState = std::make_shared(
             _request->taskID, TaskStatus::PENDING, m_timeoutMinutes);
-
+        // check the memory
+        checkHostResource(m_config->minNeededMemoryGB());
         // create ecdh cache
         auto cache = std::make_shared(
             _request->taskID, m_threadPool, m_taskGuarder, _request->dataResource,
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
index bd89fb11..204dd5ab 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIConfig.h
@@ -41,11 +41,11 @@ class CM2020PSIConfig : public PSIConfig
     CM2020PSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front,
         ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
-        uint16_t _parallelism = 3,
+        uint32_t minNeededMemoryGB = 1, uint16_t _parallelism = 3,
         const front::PPCMessageFactory::Ptr& _msgFactory =
             std::make_shared())
       : PSIConfig(ppc::protocol::TaskAlgorithmType::CM_PSI_2PC, _selfPartyID, std::move(_front),
-            _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes),
+            _msgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes, minNeededMemoryGB),
         m_cryptoBox(std::move(_cryptoBox)),
         m_threadPool(std::move(_threadPool)),
         m_parallelism(_parallelism)
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h
index 9a047f7b..a2f07c63 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIFactory.h
@@ -34,11 +34,11 @@ class CM2020PSIFactory
     virtual CM2020PSIImpl::Ptr buildCM2020PSI(std::string const& _selfParty,
         ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox,
         bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader,
-        int _holdingMessageMinutes, uint16_t _parallelism)
+        int _holdingMessageMinutes, uint16_t _parallelism, uint32_t minNeededMemoryGB = 1)
     {
         auto config = std::make_shared(_selfParty, std::move(_front),
             std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader),
-            _holdingMessageMinutes, _parallelism);
+            _holdingMessageMinutes, minNeededMemoryGB, _parallelism);
         return std::make_shared(config);
     }
 };
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
index e0b62a7e..53d8f08e 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
@@ -155,6 +155,8 @@ void CM2020PSIImpl::asyncRunTask()
 
         psi->noticePeerToFinish(task);
     });
+    // check the memory
+    checkHostResource(m_config->minNeededMemoryGB());
     addPendingTask(taskState);
 
     try
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
index 147dbab5..75dbe36d 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIConfig.h
@@ -42,7 +42,8 @@ class EcdhConnPSIConfig : public PSIConfig
         ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader,
         bcos::ThreadPool::Ptr _threadPool)
       : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_CONN, _ppcConfig->agencyID(), _front,
-            _ppcMsgFactory, _dataResourceLoader, _ppcConfig->holdingMessageMinutes()),
+            _ppcMsgFactory, _dataResourceLoader, _ppcConfig->holdingMessageMinutes(),
+            _ppcConfig->minNeededMemoryGB()),
         m_msgFactory(_psiMsgFactory),
         m_threadPool(_threadPool),
         m_ecdhCryptoFactory(_ecdhCryptoFactory),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
index d24a8f2c..e4d3bb5f 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIConfig.h
@@ -17,11 +17,13 @@ class EcdhMultiPSIConfig : public PSIConfig
     EcdhMultiPSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front,
         ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, uint32_t _dataBatchSize,
-        int _holdingMessageMinutes, EcdhMultiPSIMessageFactory::Ptr const& _psiMsgFactory,
+        int _holdingMessageMinutes, uint32_t minNeededMemoryGB,
+        EcdhMultiPSIMessageFactory::Ptr const& _psiMsgFactory,
         const front::PPCMessageFactory::Ptr& _ppcMsgFactory =
             std::make_shared())
       : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_MULTI, _selfPartyID, std::move(_front),
-            _ppcMsgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes),
+            _ppcMsgFactory, std::move(_dataResourceLoader), _holdingMessageMinutes,
+            minNeededMemoryGB),
         m_threadPool(std::move(_threadPool)),
         m_cryptoBox(std::move(_cryptoBox)),
         m_psiMsgFactory(std::move(_psiMsgFactory)),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp
index d78de97c..8f4329b3 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.cpp
@@ -28,7 +28,7 @@ using namespace ppc::crypto;
 
 EcdhMultiPSIImpl::Ptr EcdhMultiPSIFactory::createEcdhMultiPSI(PPCConfig::Ptr const& _ppcConfig,
     FrontInterface::Ptr _front, CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
-    DataResourceLoader::Ptr _dataResourceLoader)
+    DataResourceLoader::Ptr _dataResourceLoader, uint32_t minNeededMemoryGB)
 {
     auto psiMsgFactory = std::make_shared();
     auto const& ecdhParam = _ppcConfig->ecdhMultiPSIConfig();
@@ -36,6 +36,6 @@ EcdhMultiPSIImpl::Ptr EcdhMultiPSIFactory::createEcdhMultiPSI(PPCConfig::Ptr con
     int _holdingMessageMinutes = _ppcConfig->holdingMessageMinutes();
     auto config = std::make_shared(_selfParty, std::move(_front),
         std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader),
-        ecdhParam.dataBatchSize, _holdingMessageMinutes, psiMsgFactory);
+        ecdhParam.dataBatchSize, _holdingMessageMinutes, minNeededMemoryGB, psiMsgFactory);
     return std::make_shared(config);
 }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h
index a02b0ac8..eaa7e505 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIFactory.h
@@ -37,6 +37,7 @@ class EcdhMultiPSIFactory
     virtual EcdhMultiPSIImpl::Ptr createEcdhMultiPSI(
         std::shared_ptr const& _ppcConfig,
         ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox,
-        bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader);
+        bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader,
+        uint32_t minNeededMemoryGB = 1);
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
index f47a405f..94d7ec08 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
@@ -104,43 +104,46 @@ void EcdhMultiPSIImpl::handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg)
 void EcdhMultiPSIImpl::asyncRunTask(
     ppc::protocol::Task::ConstPtr _task, ppc::task::TaskResponseCallback&& _onTaskFinished)
 {
-    auto taskID = _task->id();
-    auto taskState =
-        m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), false, m_config);
-    auto self = weak_from_this();
-    taskState->registerNotifyPeerFinishHandler([self, _task]() {
-        auto psi = self.lock();
-        if (!psi)
-        {
-            return;
-        }
-        psi->noticePeerToFinish(_task);
-    });
-    taskState->registerFinalizeHandler([self, taskID]() {
-        auto psi = self.lock();
-        if (!psi)
-        {
-            return;
-        }
-        // erase the taskInfo from the gateway
-        psi->m_config->front()->eraseTaskInfo(taskID);
-        psi->removeCalculator(taskID);
-        psi->removeMaster(taskID);
-        psi->removePartner(taskID);
-        psi->removePendingTask(taskID);
-    });
-    addPendingTask(taskState);
-    // over the peer limit
-    if (_task->getAllPeerParties().size() > c_max_peer_size)
-    {
-        auto error = std::make_shared(
-            -1, "at most support " + std::to_string(c_max_peer_size) + " peers, over the limit!");
-        ECDH_MULTI_LOG(WARNING) << LOG_DESC("asyncRunTask failed")
-                                << LOG_KV("msg", error->errorMessage());
-        onSelfError(_task->id(), error, true);
-    }
     try
     {
+        auto taskID = _task->id();
+        auto taskState =
+            m_taskStateFactory->createTaskState(_task, std::move(_onTaskFinished), false, m_config);
+        auto self = weak_from_this();
+        taskState->registerNotifyPeerFinishHandler([self, _task]() {
+            auto psi = self.lock();
+            if (!psi)
+            {
+                return;
+            }
+            psi->noticePeerToFinish(_task);
+        });
+        taskState->registerFinalizeHandler([self, taskID]() {
+            auto psi = self.lock();
+            if (!psi)
+            {
+                return;
+            }
+            // erase the taskInfo from the gateway
+            psi->m_config->front()->eraseTaskInfo(taskID);
+            psi->removeCalculator(taskID);
+            psi->removeMaster(taskID);
+            psi->removePartner(taskID);
+            psi->removePendingTask(taskID);
+        });
+        // check the memory
+        checkHostResource(m_config->minNeededMemoryGB());
+        addPendingTask(taskState);
+        // over the peer limit
+        if (_task->getAllPeerParties().size() > c_max_peer_size)
+        {
+            auto error = std::make_shared(-1,
+                "at most support " + std::to_string(c_max_peer_size) + " peers, over the limit!");
+            ECDH_MULTI_LOG(WARNING)
+                << LOG_DESC("asyncRunTask failed") << LOG_KV("msg", error->errorMessage());
+            onSelfError(_task->id(), error, true);
+        }
+
         auto dataResource = _task->selfParty()->dataResource();
         auto reader = loadReader(_task->id(), dataResource, DataSchema::Bytes);
         auto sqlReader = (reader->type() == ppc::protocol::DataResourceType::MySQL);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
index d2810d16..f7ec036d 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIConfig.h
@@ -36,10 +36,11 @@ class EcdhPSIConfig : public PSIConfig
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory,
         PSIMessageFactory::Ptr const& _msgFactory,
         ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader, uint32_t _dataBatchSize,
-        int _holdingMessageMinutes, bcos::ThreadPool::Ptr const& _threadPool = nullptr,
+        int _holdingMessageMinutes, uint32_t minNeededMemoryGB = 1,
+        bcos::ThreadPool::Ptr const& _threadPool = nullptr,
         int _threadPoolSize = std::thread::hardware_concurrency())
       : PSIConfig(ppc::protocol::TaskAlgorithmType::ECDH_PSI_2PC, _selfParty, _front,
-            _ppcMsgFactory, _dataResourceLoader, _holdingMessageMinutes),
+            _ppcMsgFactory, _dataResourceLoader, _holdingMessageMinutes, minNeededMemoryGB),
         m_msgFactory(_msgFactory),
         m_threadPool(_threadPool),
         m_ecdhCryptoFactory(_ecdhCryptoFactory),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp
index 16d3aa94..670c8347 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.cpp
@@ -31,12 +31,12 @@ using namespace ppc::io;
 EcdhPSIImpl::Ptr EcdhPSIFactory::createEcdhPSI(PPCConfig::Ptr const& _ppcConfig,
     ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory, FrontInterface::Ptr _front,
     PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool,
-    DataResourceLoader::Ptr const& _dataResourceLoader)
+    DataResourceLoader::Ptr const& _dataResourceLoader, uint32_t minNeededMemoryGB)
 {
     auto psiMsgFactory = std::make_shared();
     auto const& ecdhParam = _ppcConfig->ecdhPSIConfig();
     auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory,
         _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize,
-        _ppcConfig->holdingMessageMinutes(), _threadPool);
+        _ppcConfig->holdingMessageMinutes(), minNeededMemoryGB, _threadPool);
     return std::make_shared(config);
 }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h
index 2f0c8a4f..08775356 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIFactory.h
@@ -37,6 +37,7 @@ class EcdhPSIFactory
         ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory,
         ppc::front::FrontInterface::Ptr _front,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool,
-        ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader);
+        ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader,
+        uint32_t minNeededMemoryGB = 1);
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
index 8fe48aab..2456fd11 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIConfig.h
@@ -40,9 +40,11 @@ class LabeledPSIConfig : public PSIConfig
     LabeledPSIConfig(std::string const& _selfPartyID, ppc::front::FrontInterface::Ptr _front,
         ppc::crypto::CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
+        uint32_t minNeededMemoryGB = 1,
         front::PPCMessageFactory::Ptr _msgFactory = std::make_shared())
       : PSIConfig(ppc::protocol::TaskAlgorithmType::LABELED_PSI_2PC, _selfPartyID,
-            std::move(_front), _msgFactory, _dataResourceLoader, _holdingMessageMinutes),
+            std::move(_front), _msgFactory, _dataResourceLoader, _holdingMessageMinutes,
+            minNeededMemoryGB),
         m_cryptoBox(std::move(_cryptoBox)),
         m_threadPool(std::move(_threadPool)),
         m_messageFactory(_msgFactory)
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp
index 3e6558b2..30f41f81 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.cpp
@@ -26,10 +26,11 @@ using namespace ppc::front;
 
 LabeledPSIImpl::Ptr LabeledPSIFactory::buildLabeledPSI(std::string const& _selfParty,
     FrontInterface::Ptr _front, CryptoBox::Ptr _cryptoBox, bcos::ThreadPool::Ptr _threadPool,
-    DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes)
+    DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
+    uint32_t minNeededMemoryGB)
 {
-    auto config =
-        std::make_shared(_selfParty, std::move(_front), std::move(_cryptoBox),
-            std::move(_threadPool), std::move(_dataResourceLoader), _holdingMessageMinutes);
+    auto config = std::make_shared(_selfParty, std::move(_front),
+        std::move(_cryptoBox), std::move(_threadPool), std::move(_dataResourceLoader),
+        _holdingMessageMinutes, minNeededMemoryGB);
     return std::make_shared(config);
 }
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h
index 36bfeb08..407dfa45 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIFactory.h
@@ -34,6 +34,6 @@ class LabeledPSIFactory
     virtual LabeledPSIImpl::Ptr buildLabeledPSI(std::string const& _selfParty,
         ppc::front::FrontInterface::Ptr _front, ppc::crypto::CryptoBox::Ptr _cryptoBox,
         bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader,
-        int _holdingMessageMinutes);
+        int _holdingMessageMinutes, uint32_t minNeededMemoryGB = 1);
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
index 0ac28d7e..e56ead44 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.cpp
@@ -82,69 +82,83 @@ void LabeledPSIImpl::asyncRunTask(
         }
     };
 
-
-    // there is always a self-help party
-    auto role = _task->selfParty()->partyIndex();
-    if (role == uint16_t(PartyType::Client))
+    try
     {
-        auto error = checkTask(_task, 2, true, true, false);
-        if (error)
+        // there is always a self-help party
+        auto role = _task->selfParty()->partyIndex();
+        if (role == uint16_t(PartyType::Client))
         {
-            LABELED_PSI_LOG(WARNING) << LOG_DESC("failed to check task, " + error->errorMessage())
-                                     << printTaskInfo(_task);
-            // notice peer to cancel task
-            noticePeerToFinish(_task);
-
-            auto result = std::make_shared(_task->id());
-            result->setError(std::move(error));
-            onTaskFinished(std::move(result));
-            return;
-        }
-
-        // add pending task
-        auto taskState = m_taskStateFactory->createTaskState(_task, std::move(onTaskFinished));
-        taskState->setPeerID(getPeerID(_task));
-        taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() {
-            auto psi = self.lock();
-            if (!psi)
+            auto error = checkTask(_task, 2, true, true, false);
+            if (error)
             {
+                LABELED_PSI_LOG(WARNING)
+                    << LOG_DESC("failed to check task, " + error->errorMessage())
+                    << printTaskInfo(_task);
+                // notice peer to cancel task
+                noticePeerToFinish(_task);
+
+                auto result = std::make_shared(_task->id());
+                result->setError(std::move(error));
+                onTaskFinished(std::move(result));
                 return;
             }
 
-            psi->noticePeerToFinish(_task);
-        });
-        addPendingTask(taskState);
-
-        auto oprfClient = std::make_shared(
-            sizeof(apsi::Item::value_type) + sizeof(apsi::LabelKey), m_config->hash(),
-            m_config->eccCrypto());
-        auto receiver = std::make_shared(m_config, taskState, oprfClient);
-        receiver->asyncRunTask();
-        addReceiver(receiver);
+            // add pending task
+            auto taskState = m_taskStateFactory->createTaskState(_task, std::move(onTaskFinished));
+            taskState->setPeerID(getPeerID(_task));
+            taskState->registerNotifyPeerFinishHandler([self = weak_from_this(), _task]() {
+                auto psi = self.lock();
+                if (!psi)
+                {
+                    return;
+                }
 
-        // notify the taskInfo to the front
-        error = m_config->front()->notifyTaskInfo(_task->id());
-        if (error && error->errorCode())
+                psi->noticePeerToFinish(_task);
+            });
+            // check the memory
+            checkHostResource(m_config->minNeededMemoryGB());
+            addPendingTask(taskState);
+
+            auto oprfClient = std::make_shared(
+                sizeof(apsi::Item::value_type) + sizeof(apsi::LabelKey), m_config->hash(),
+                m_config->eccCrypto());
+            auto receiver = std::make_shared(m_config, taskState, oprfClient);
+            receiver->asyncRunTask();
+            addReceiver(receiver);
+
+            // notify the taskInfo to the front
+            error = m_config->front()->notifyTaskInfo(_task->id());
+            if (error && error->errorCode())
+            {
+                onSelfError(_task->id(), error, true);
+            }
+        }
+        else if (role == uint16_t(PartyType::Server))
         {
-            onSelfError(_task->id(), error, true);
+            asyncRunSenderTask(_task, std::move(onTaskFinished));
+        }
+        else
+        {
+            LABELED_PSI_LOG(WARNING) << LOG_DESC("undefined task role") << unsigned(role);
+            // notice peer to cancel task
+            noticePeerToFinish(_task);
+
+            auto result = std::make_shared(_task->id());
+            result->setError(std::make_shared(
+                (int)LabeledPSIRetCode::UNDEFINED_TASK_ROLE, "undefined task role"));
+            onTaskFinished(std::move(result));
+            noticePeerToFinish(_task);
+            return;
         }
     }
-    else if (role == uint16_t(PartyType::Server))
-    {
-        asyncRunSenderTask(_task, std::move(onTaskFinished));
-    }
-    else
+    catch (std::exception const& e)
     {
-        LABELED_PSI_LOG(WARNING) << LOG_DESC("undefined task role") << unsigned(role);
-        // notice peer to cancel task
-        noticePeerToFinish(_task);
-
-        auto result = std::make_shared(_task->id());
-        result->setError(std::make_shared(
-            (int)LabeledPSIRetCode::UNDEFINED_TASK_ROLE, "undefined task role"));
-        onTaskFinished(std::move(result));
-        noticePeerToFinish(_task);
-        return;
+        LABELED_PSI_LOG(WARNING) << LOG_DESC("asyncRunTask exception")
+                                 << LOG_KV("task", printTaskInfo(_task))
+                                 << LOG_KV("error", boost::diagnostic_information(e));
+        auto error = std::make_shared(
+            -1, "asyncRunTask failed for " + boost::diagnostic_information(e));
+        onSelfError(_task->id(), error, true);
     }
 }
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
index 1e0c91b8..2dbf0b07 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
@@ -166,6 +166,8 @@ void TaskState::onTaskFinished()
                         boost::lexical_cast(m_failedCount) + " error!");
             result->setError(std::move(error));
             result->setStatus(toString(TaskStatus::FAILED));
+            // remove the generated output files if run failed
+            removeGeneratedOutputFile();
         }
         else
         {
@@ -193,6 +195,22 @@ void TaskState::onTaskFinished()
     }
 }
 
+void TaskState::removeGeneratedOutputFile()
+{
+    if (!m_task || !m_task->selfParty() || !m_task->selfParty()->dataResource())
+    {
+        return;
+    }
+    auto outputDataResource = m_task->selfParty()->dataResource();
+    if (!outputDataResource->desc())
+    {
+        return;
+    }
+    PSI_LOG(INFO) << LOG_DESC("removeGeneratedFilesForFailed")
+                  << LOG_KV("task", printTaskInfo(m_task));
+    m_config->dataResourceLoader()->deleteResource(outputDataResource->desc());
+}
+
 void TaskState::onTaskFinished(TaskResult::Ptr _result, bool _noticePeer)
 {
     // avoid repeated calls
@@ -213,6 +231,8 @@ void TaskState::onTaskFinished(TaskResult::Ptr _result, bool _noticePeer)
         if (_result->error() && _result->error()->errorCode() != 0)
         {
             _result->setStatus(toString(TaskStatus::FAILED));
+            // remove the generated output files if run failed
+            removeGeneratedOutputFile();
         }
         // Note: we consider that the task success even if the handler exception
         if (_noticePeer && !m_onlySelfRun && _result->error() && _result->error()->errorCode() &&
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
index 0fe0e16f..a06d1ab8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.h
@@ -144,6 +144,9 @@ class TaskState : public std::enable_shared_from_this
 
     uint32_t sendedDataBatchSize() const { return m_seqList.size(); }
 
+private:
+    void removeGeneratedOutputFile();
+
 protected:
     ppc::protocol::Task::ConstPtr m_task;
     ppc::task::TaskResponseCallback m_callback;
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
index 9b4e2d5f..4f79010b 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIConfig.h
@@ -39,11 +39,12 @@ class RA2018PSIConfig : public PSIConfig
         ppc::tools::CuckoofilterOption::Ptr _cuckooFilterOption, bcos::ThreadPool::Ptr _threadPool,
         ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage,
         ppc::io::DataResourceLoader::Ptr _dataResourceLoader, int _holdingMessageMinutes,
-        std::string const& _dbName = "ra2018", uint64_t _cuckooFilterCacheSize = 255 * 1024 * 1024,
+        uint32_t minNeededMemoryGB = 1, std::string const& _dbName = "ra2018",
+        uint64_t _cuckooFilterCacheSize = 255 * 1024 * 1024,
         uint64_t _ra2018CacheCapacity = 1024 * 1024 * 1024, int _dataBatchSize = -1,
         int _threadPoolSize = std::thread::hardware_concurrency())
       : PSIConfig(ppc::protocol::TaskAlgorithmType::RA_PSI_2PC, _selfParty, _front, _ppcMsgFactory,
-            _dataResourceLoader, _holdingMessageMinutes),
+            _dataResourceLoader, _holdingMessageMinutes, minNeededMemoryGB),
         m_oprf(std::move(_oprf)),
         m_binHashImpl(std::move(_binHashImpl)),
         m_ra2018MsgFactory(std::make_shared()),
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp
index 5a589914..75615833 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.cpp
@@ -32,14 +32,15 @@ RA2018PSIImpl::Ptr RA2018PSIFactory::createRA2018PSI(std::string const& _selfPar
     RA2018OprfInterface::Ptr const& _oprf, Hash::Ptr _binHashImpl,
     PPCMessageFaceFactory::Ptr _ppcMsgFactory, SQLStorage::Ptr _storage,
     FileStorage::Ptr _fileStorage, bcos::ThreadPool::Ptr _threadPool,
-    DataResourceLoader::Ptr _dataResourceLoader)
+    DataResourceLoader::Ptr _dataResourceLoader, uint32_t minNeededMemoryGB)
 {
     auto const& ra2018Config = _config->ra2018PSIConfig();
 
     auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl,
         _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage,
-        _dataResourceLoader, _config->holdingMessageMinutes(), ra2018Config.dbName,
-        ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize, ra2018Config.dataBatchSize);
+        _dataResourceLoader, _config->holdingMessageMinutes(), minNeededMemoryGB,
+        ra2018Config.dbName, ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize,
+        ra2018Config.dataBatchSize);
 
     RA2018PSIStorage::Ptr psiStorage = nullptr;
     if (!_config->disableRA2018())
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h
index ed107105..ab6d2513 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIFactory.h
@@ -39,6 +39,7 @@ class RA2018PSIFactory
         ppc::crypto::RA2018OprfInterface::Ptr const& _oprf, ppc::crypto::Hash::Ptr _binHashImpl,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory,
         ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage,
-        bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader);
+        bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader,
+        uint32_t minNeededMemoryGB = 1);
 };
 }  // namespace ppc::psi
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
index 2bbbd065..37209c9a 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
@@ -78,13 +78,14 @@ class FakeEcdhPSIFactory : public EcdhPSIFactory
         ppc::crypto::ECDHCryptoFactory::Ptr const& _ecdhCryptoFactory,
         ppc::front::FrontInterface::Ptr _front,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory, bcos::ThreadPool::Ptr _threadPool,
-        ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader) override
+        ppc::io::DataResourceLoader::Ptr const& _dataResourceLoader,
+        uint32_t minNeededMemoryGB = 1) override
     {
         auto psiMsgFactory = std::make_shared();
         auto const& ecdhParam = _ppcConfig->ecdhPSIConfig();
         auto config = std::make_shared(_ppcConfig->agencyID(), _ecdhCryptoFactory,
             _front, _ppcMsgFactory, psiMsgFactory, _dataResourceLoader, ecdhParam.dataBatchSize,
-            10000, _threadPool);
+            10000, minNeededMemoryGB, _threadPool);
         // enforce the taskExpireTime to 3000ms
         config->setTaskExpireTime(3000);
         return std::make_shared(config);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
index d5a9d1bc..c0bb0d34 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
@@ -83,14 +83,14 @@ class FakeRA2018PSIFactory : public RA2018PSIFactory
         ppc::crypto::RA2018OprfInterface::Ptr const& _oprf, ppc::crypto::Hash::Ptr _binHashImpl,
         ppc::front::PPCMessageFaceFactory::Ptr _ppcMsgFactory,
         ppc::storage::SQLStorage::Ptr _storage, ppc::storage::FileStorage::Ptr _fileStorage,
-        bcos::ThreadPool::Ptr _threadPool,
-        ppc::io::DataResourceLoader::Ptr _dataResourceLoader) override
+        bcos::ThreadPool::Ptr _threadPool, ppc::io::DataResourceLoader::Ptr _dataResourceLoader,
+        uint32_t minNeededMemoryGB = 1) override
     {
         auto const& ra2018Config = _config->ra2018PSIConfig();
         auto config = std::make_shared(_selfParty, _front, _oprf, _binHashImpl,
             _ppcMsgFactory, ra2018Config.cuckooFilterOption, _threadPool, _storage, _fileStorage,
-            _dataResourceLoader, 10000, ra2018Config.dbName, ra2018Config.cuckooFilterCacheSize,
-            ra2018Config.cacheSize, ra2018Config.dataBatchSize);
+            _dataResourceLoader, 10000, minNeededMemoryGB, ra2018Config.dbName,
+            ra2018Config.cuckooFilterCacheSize, ra2018Config.cacheSize, ra2018Config.dataBatchSize);
         // enforce the taskExpireTime to 3000ms
         config->setTaskExpireTime(3000);
         // use the FakeRA2018PSIStorage
diff --git a/cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h b/cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h
index 9c3e5108..f9cb1f25 100644
--- a/cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h
+++ b/cpp/wedpr-helper/ppc-tools/src/common/MemInfo.h
@@ -28,7 +28,7 @@
 
 namespace ppc::tools
 {
-bool hasAvailableMem(uint32_t _minNeedMemoryGB)
+inline bool hasAvailableMem(uint32_t _minNeedMemoryGB)
 {
     std::ifstream meminfo("/proc/meminfo");
     if (!meminfo.is_open())
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h
index a070cc12..c0e513fb 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/NetworkConfig.h
@@ -31,7 +31,6 @@ struct NetworkConfig
     uint32_t threadPoolSize;
     std::string token;
     bool disableSsl;
-    uint32_t minNeededMemoryGB;
 
     bool enableSM;
     // the gateway protocol
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index d19b6b37..27234303 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -254,14 +254,12 @@ void PPCConfig::loadNetworkConfig(NetworkConfig& _config, boost::property_tree::
     {
         _config.url = _pt.get(_sectionName + ".url", "/api/v1/interconn/invoke");
     }
-    _config.minNeededMemoryGB = _pt.get(_sectionName + ".min_needed_memory", 5);
     PPCConfig_LOG(INFO) << LOG_BADGE("loadNetworkConfig") << LOG_KV("section", _sectionName)
                         << LOG_KV("listenIp", _config.listenIp)
                         << LOG_KV("listenPort", _config.listenPort)
                         << LOG_KV("threadCount", _config.threadPoolSize)
                         << LOG_KV("disableSsl", _config.disableSsl)
-                        << LOG_KV("protocol", _config.protocol) << LOG_KV("url", _config.url)
-                        << LOG_KV("minNeededMemory", _config.minNeededMemoryGB);
+                        << LOG_KV("protocol", _config.protocol) << LOG_KV("url", _config.url);
     // no need to load the certificate when disable-ssl
     if (_config.disableSsl)
     {
@@ -530,11 +528,18 @@ void PPCConfig::loadCommonNodeConfig(boost::property_tree::ptree const& _pt)
     m_threadPoolSize = _pt.get(
         "agency.thread_count", static_cast(std::thread::hardware_concurrency() * 0.75));
 
+    m_minNeededMemoryGB = _pt.get("agency.min_needed_memory", 1);
+    if (m_minNeededMemoryGB == 0)
+    {
+        BOOST_THROW_EXCEPTION(InvalidConfig() << bcos::errinfo_comment(
+                                  "Invalid agency.min_needed_memory, must larger than 0"));
+    }
     PPCConfig_LOG(INFO) << LOG_DESC("loadCommonNodeConfig success")
                         << LOG_KV("agencyID", m_agencyID) << LOG_KV("dataLocation", m_dataLocation)
                         << LOG_KV("smCrypto", m_smCrypto)
                         << LOG_KV("taskTimeoutMinutes", m_taskTimeoutMinutes)
-                        << LOG_KV("threadPoolSize", m_threadPoolSize);
+                        << LOG_KV("threadPoolSize", m_threadPoolSize)
+                        << LOG_KV("minNeededMemoryGB", m_minNeededMemoryGB);
 }
 
 void PPCConfig::loadHDFSConfig(boost::property_tree::ptree const& _pt)
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index 89866d48..6ac74457 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -254,6 +254,8 @@ class PPCConfig
         ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder,
         boost::property_tree::ptree const& pt);
 
+    uint32_t minNeededMemoryGB() const { return m_minNeededMemoryGB; }
+
 private:
     virtual void loadEndpointConfig(ppc::protocol::EndPoint& endPoint, bool requireHostIp,
         std::string const& sectionName, boost::property_tree::ptree const& pt);
@@ -307,6 +309,7 @@ class PPCConfig
 
     uint16_t m_seqSyncPeriod = 5000;
 
+    uint32_t m_minNeededMemoryGB;
     // the front config
     // TODO: parse the frontConfig
     ppc::front::FrontConfig::Ptr m_frontConfig;
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index 9c1c0a78..384fd44a 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -149,7 +149,8 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     auto labeledPSIFactory = std::make_shared();
     m_labeledPSI = labeledPSIFactory->buildLabeledPSI(m_config->agencyID(), m_ppcFront, cryptoBox,
         std::make_shared("t_labeled-psi", m_config->threadPoolSize()),
-        m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes());
+        m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes(),
+        m_config->minNeededMemoryGB());
     INIT_LOG(INFO) << LOG_DESC("init the labeled-psi success");
 
     // init the cm2020-psi
@@ -158,7 +159,7 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     m_cm2020PSI = cm2020PSIFactory->buildCM2020PSI(m_config->agencyID(), m_ppcFront, cryptoBox,
         std::make_shared("t_cm2020-psi", m_config->threadPoolSize()),
         m_protocolInitializer->dataResourceLoader(), m_config->holdingMessageMinutes(),
-        m_config->cm2020PSIConfig().parallelism);
+        m_config->cm2020PSIConfig().parallelism, m_config->minNeededMemoryGB());
     INIT_LOG(INFO) << LOG_DESC("init the cm2020-psi success");
 
     // init the ecdh-psi
@@ -167,7 +168,7 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     auto ecdhCryptoFactory = std::make_shared(m_config->privateKey());
     m_ecdhPSI = ecdhPSIFactory->createEcdhPSI(m_config, ecdhCryptoFactory, m_ppcFront,
         m_protocolInitializer->ppcMsgFactory(), nullptr,
-        m_protocolInitializer->dataResourceLoader());
+        m_protocolInitializer->dataResourceLoader(), m_config->minNeededMemoryGB());
     INIT_LOG(INFO) << LOG_DESC("create ecdh-psi success");
 
 #if 0
@@ -186,7 +187,7 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     auto ecdhMultiPSIFactory = std::make_shared();
     m_ecdhMultiPSI = ecdhMultiPSIFactory->createEcdhMultiPSI(m_config, m_ppcFront, cryptoBox,
         std::make_shared("t_ecdh-multi-psi", std::thread::hardware_concurrency()),
-        m_protocolInitializer->dataResourceLoader());
+        m_protocolInitializer->dataResourceLoader(), m_config->minNeededMemoryGB());
     INIT_LOG(INFO) << LOG_DESC("create ecdh-multi-psi success");
 
     // init the ot-pir
@@ -203,7 +204,8 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     auto bsEcdhPSIFactory = std::make_shared();
     m_bsEcdhPSI = bsEcdhPSIFactory->buildBsEcdhPSI(
         std::make_shared("t_bs-ecdh-psi", m_config->threadPoolSize()),
-        m_protocolInitializer->dataResourceLoader(), m_config->taskTimeoutMinutes());
+        m_protocolInitializer->dataResourceLoader(), m_config->taskTimeoutMinutes(),
+        m_config->minNeededMemoryGB());
     INIT_LOG(INFO) << LOG_DESC("create bs mode ecdh psi success");
 
     initMsgHandlers();
diff --git a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
index dc54f056..4287f758 100644
--- a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
+++ b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.h
@@ -51,10 +51,11 @@ class DataResourceLoaderImpl : public DataResourceLoader
         ppc::storage::FileStorage::Ptr const& _storage = nullptr) override;
 
     void deleteResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
-        ppc::storage::FileStorage::Ptr const& _storage = nullptr);
+        ppc::storage::FileStorage::Ptr const& _storage = nullptr) override;
 
     void renameResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
-        std::string const& _new_path, ppc::storage::FileStorage::Ptr const& _storage = nullptr);
+        std::string const& _new_path,
+        ppc::storage::FileStorage::Ptr const& _storage = nullptr) override;
 
 protected:
     virtual LineReader::Ptr loadSQLResource(ppc::protocol::DataResourceDesc::ConstPtr _desc,
diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
index a33e4acc..ea9dce49 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.cpp
@@ -22,7 +22,6 @@
 #include "JsonResponse.h"
 #include "ppc-framework/protocol/Protocol.h"
 #include "ppc-framework/rpc/RpcTypeDef.h"
-#include "ppc-tools/src/common/MemInfo.h"
 #include "ppc-tools/src/config/ParamChecker.h"
 #include 
 
@@ -180,8 +179,6 @@ void Rpc::runTask(Json::Value const& _req, RespFunc _respFunc)
         BOOST_THROW_EXCEPTION(
             BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set"));
     }
-    checkHostResource();
-
     auto task = m_taskFactory->createTask(_req);
     auto taskHandler = getTaskHandler(task->type(), task->algorithm());
     // not find the handler
@@ -237,8 +234,6 @@ void Rpc::asyncRunTask(Json::Value const& _req, RespFunc _respFunc)
         BOOST_THROW_EXCEPTION(
             BCOS_ERROR((int64_t)RpcError::StorageNotSet, "storage for rpc not set"));
     }
-    checkHostResource();
-
     auto task = m_taskFactory->createTask(_req);
     auto taskHandler = getTaskHandler(task->type(), task->algorithm());
     // not find the handler
@@ -312,8 +307,6 @@ void Rpc::asyncRunBsModeTask(Json::Value const& _req, RespFunc _respFunc)
         BOOST_THROW_EXCEPTION(
             BCOS_ERROR((int64_t)RpcError::BsModePsiNotSet, "bs psi for rpc not set"));
     }
-    checkHostResource();
-
     auto request = std::make_shared(_req, m_prePath);
     m_bsEcdhPSI->asyncRunTask(request, [&](psi::BsEcdhResult::Ptr&& _result) {
         _respFunc(_result->error(), _result->serializeToJson());
@@ -423,14 +416,3 @@ void Rpc::updateBsModeTaskStatus(Json::Value const& _req, RespFunc _respFunc)
     auto result = m_bsEcdhPSI->updateTaskStatus(request);
     _respFunc(result->error(), result->serializeToJson());
 }
-
-void Rpc::checkHostResource()
-{
-#ifdef __linux__
-    if (!hasAvailableMem(m_minNeededMemoryGB))
-    {
-        BOOST_THROW_EXCEPTION(
-            BCOS_ERROR((int64_t)RpcError::LackOfMemory, "Lack of memory, please try again later."));
-    }
-#endif
-}
diff --git a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h
index fe19b420..be1da870 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/Rpc.h
+++ b/cpp/wedpr-transport/ppc-rpc/src/Rpc.h
@@ -100,8 +100,6 @@ class Rpc : public RpcInterface
 
     void setBsEcdhPSI(psi::BsEcdhPSIInterface::Ptr _psi) { m_bsEcdhPSI = std::move(_psi); }
 
-    void setMinNeededMemory(uint32_t _minNeededMemory) { m_minNeededMemoryGB = _minNeededMemory; }
-
 protected:
     virtual void onRPCRequest(bcos::boostssl::http::HttpRequest&& _request,
         std::function _responseHandler);
@@ -135,8 +133,6 @@ class Rpc : public RpcInterface
 
     virtual void getPeers(Json::Value const& _req, RespFunc _respFunc);
 
-    void checkHostResource();
-
 private:
     std::string m_prePath;
     std::shared_ptr m_wsService;
@@ -157,7 +153,5 @@ class Rpc : public RpcInterface
     mutable bcos::SharedMutex x_taskHandlers;
 
     psi::BsEcdhPSIInterface::Ptr m_bsEcdhPSI;
-
-    uint32_t m_minNeededMemoryGB;
 };
 }  // namespace ppc::rpc
diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp
index 92c1ed22..95953dd0 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/RpcFactory.cpp
@@ -41,7 +41,6 @@ Rpc::Ptr RpcFactory::buildRpc(
 
     auto rpc = std::make_shared(
         wsService, gateway, m_selfPartyID, _config->rpcConfig().token, _config->dataLocation());
-    rpc->setMinNeededMemory(_config->rpcConfig().minNeededMemoryGB);
     return rpc;
 }
 
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
index f9ef78e3..a3986303 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/impl/TransportImpl.java
@@ -471,7 +471,10 @@ public List selectNodeListByPolicy(
     }
 
     private void parseServiceMeta(
-            List entryPointInfos, String serviceName, String meta) {
+            List entryPointInfos,
+            String serviceName,
+            String meta,
+            StringVec componentsVec) {
         try {
             if (StringUtils.isBlank(meta)) {
                 return;
@@ -484,6 +487,10 @@ private void parseServiceMeta(
             for (ServiceMeta.EntryPointMeta entryPointMeta : serviceMeta.getServiceInfos()) {
                 if (entryPointMeta.getServiceName().equalsIgnoreCase(serviceName)) {
                     entryPointInfos.add(entryPointMeta);
+                    List components = new ArrayList<>();
+                    for (int i = 0; i < componentsVec.size(); i++) {
+                        entryPointMeta.getComponents().add(componentsVec.get(i));
+                    }
                 }
             }
         } catch (Exception e) {
@@ -496,7 +503,11 @@ public List getAliveEntryPoints(String serviceName)
         NodeInfoVec nodeInfoList = this.transport.getFront().getNodeDiscovery().getAliveNodeList();
         List result = new ArrayList<>();
         for (int i = 0; i < nodeInfoList.size(); i++) {
-            parseServiceMeta(result, serviceName, nodeInfoList.get(i).meta());
+            parseServiceMeta(
+                    result,
+                    serviceName,
+                    nodeInfoList.get(i).meta(),
+                    nodeInfoList.get(i).copiedComponents());
         }
         return result;
     }
diff --git a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java
index c99702a3..9b5c3fcd 100644
--- a/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java
+++ b/cpp/wedpr-transport/sdk-wrapper/java/bindings/src/main/java/com/webank/wedpr/sdk/jni/transport/model/ServiceMeta.java
@@ -37,6 +37,7 @@ public class ServiceMeta {
     public static class EntryPointMeta {
         private String serviceName;
         private String entryPoint;
+        private List components = new ArrayList<>();
 
         public EntryPointMeta(String serviceName, String entryPoint) {
             this.serviceName = serviceName;
@@ -52,6 +53,13 @@ public String getUrl(String uriPath) {
             }
             return Common.getUrl(entryPoint + Constant.URI_SPLITER + uriPath);
         }
+
+        public void setComponents(List components) {
+            if (components == null) {
+                return;
+            }
+            this.components = components;
+        }
     }
     // Note: here can't been changed
     private List serviceInfos = new ArrayList<>();
diff --git a/python/ppc_common/deps_services/mysql_storage.py b/python/ppc_common/deps_services/mysql_storage.py
index ada66a53..d8c45f53 100644
--- a/python/ppc_common/deps_services/mysql_storage.py
+++ b/python/ppc_common/deps_services/mysql_storage.py
@@ -16,20 +16,19 @@ def __init__(self, storage_config):
             connect_args = {'schema': storage_config.db_name}
         self._mysql_engine = create_engine(self._engine_url, pool_recycle=self._storage_config.pool_recycle,
                                            pool_size=self._storage_config.pool_size, max_overflow=self._storage_config.max_overflow,
-                                           pool_timeout=self._storage_config.pool_timeout, connect_args=connect_args)
+                                           pool_timeout=self._storage_config.pool_timeout,
+                                           pool_pre_ping=self._storage_config.pool_pre_ping,
+                                           connect_args=connect_args)
         self._session_factory = sessionmaker(bind=self._mysql_engine)
-        # Note: scoped_session is threadLocal
-        self._session = scoped_session(self._session_factory)
 
     @contextmanager
     def _get_session(self):
-        session = self._session()
+        session = self._session_factory()
         try:
             yield session
             session.commit()
         except Exception:
             session.rollback()
-            self._session.remove()
             raise
         finally:
             session.close()
diff --git a/python/ppc_common/deps_services/sql_storage_api.py b/python/ppc_common/deps_services/sql_storage_api.py
index 7f240f05..a8eba6c7 100644
--- a/python/ppc_common/deps_services/sql_storage_api.py
+++ b/python/ppc_common/deps_services/sql_storage_api.py
@@ -7,7 +7,7 @@ class SQLStorageConfig(ABC):
     """
     configuration for sql storage
     """
-    DEFAULT_RECYCLE = 7200
+    DEFAULT_RECYCLE = 3600
     DEFAULT_POOL_SIZE = 16
     DEFAULT_MAX_OVERFLOW = 8
     DEFAULT_POOL_TIMEOUT = 30
@@ -24,6 +24,7 @@ def __init__(self, url, pool_recycle=None, pool_size=None, max_overflow=None, po
         self.db_type = db_type
         self.engine_url = url
         self.db_name = db_name
+        self.pool_pre_ping = True
 
 
 class SQLStorageAPI:
diff --git a/python/ppc_model/common/initializer.py b/python/ppc_model/common/initializer.py
index 7ecf31ae..310ca778 100644
--- a/python/ppc_model/common/initializer.py
+++ b/python/ppc_model/common/initializer.py
@@ -98,6 +98,8 @@ def init_transport(self, task_manager: TaskManager,
         transport = TransportLoader.load(transport_config)
         self.logger(
             f"Create transport success, config: {transport.get_config().desc()}, access_entrypoint: {access_entrypoint}")
+        # the configuration used to distinguish different wedpr-privacy-zone
+        transport.register_component(self.config_data['WEDPR_ZONE'])
         # start the transport
         transport.start()
         self.logger().info(
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index b2762275..1bc97e73 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -1,6 +1,7 @@
 HOST: "0.0.0.0"
 HTTP_PORT: 43471
 
+WEDPR_ZONE: 'wedpr1'
 AGENCY_ID: 'WeBank'
 
 PUBLIC_KEY_LENGTH: 2048
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index 0c565c0d..4e1e235e 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -28,64 +28,92 @@ def post(self, model_id):
         """
         Run a specific task by task_id.
         """
-        args = request.get_json()
-        task_id = model_id
-        components.logger().info(
-            f"run task request, task_id: {task_id}")
-        task_type = args['task_type']
-        components.task_manager.run_task(
-            task_id, ModelTask(task_type), (args,))
-        return utils.BASE_RESPONSE
+        try:
+            args = request.get_json()
+            task_id = model_id
+            components.logger().info(
+                f"run task request, task_id: {task_id}")
+            task_type = args['task_type']
+            components.task_manager.run_task(
+                task_id, ModelTask(task_type), (args,))
+            return utils.BASE_RESPONSE
+        except Exception as e:
+            response = {}
+            response.update({'errorCode': -1})
+            response.update(
+                {'message': f'submit task {model_id} failed for {e}'})
+            return response
 
     @api.response(200, 'Task status retrieved successfully.', response_task_status)
     def get(self, model_id):
-        """
-        Get the status of a specific task by task_id.
-        """
-        response = utils.BASE_RESPONSE
-        task_id = model_id
-        status, traffic_volume, exec_result = components.task_manager.status(
-            task_id)
-        response['data'] = {
-            'status': status,
-            'traffic_volume': traffic_volume,
-            'exec_result': exec_result,
-        }
-        return response
+        try:
+            """
+            Get the status of a specific task by task_id.
+            """
+            response = utils.BASE_RESPONSE
+            task_id = model_id
+            status, traffic_volume, exec_result = components.task_manager.status(
+                task_id)
+            response['data'] = {
+                'status': status,
+                'traffic_volume': traffic_volume,
+                'exec_result': exec_result,
+            }
+            return response
+        except Exception as e:
+            response = {}
+            response.update({'errorCode': -1})
+            response.update(
+                {'message': f'query task status for {model_id} failed for {e}'})
+            return response
 
     @api.response(200, 'Task killed successfully.', response_base)
     def delete(self, model_id):
-        """
-        Kill a specific task by job_id.
-        """
-        job_id = model_id
-        components.logger().info(f"kill request, job_id: {job_id}")
-        components.task_manager.kill_task(job_id)
-        return utils.BASE_RESPONSE
+        try:
+            """
+            Kill a specific task by job_id.
+            """
+            job_id = model_id
+            components.logger().info(f"kill request, job_id: {job_id}")
+            components.task_manager.kill_task(job_id)
+            return utils.BASE_RESPONSE
+        except Exception as e:
+            response = {}
+            response.update({'errorCode': -1})
+            response.update(
+                {'message': f'kill task {model_id} failed for {e}'})
+            return response
 
 
 @ns_get_job_result.route('/')
 class ModelResultCollection(Resource):
     @api.response(201, 'Get task result successfully.', response_base)
     def post(self, task_id):
-        """
-        Get the result related to the task_id
-        """
-        start_t = time.time()
-        args = request.get_json()
-        components.logger().info(
-            f"get task result, task_id: {task_id}, args: {args}")
-        user_name = args['user']
-        task_type = args['jobType']
-        only_fetch_log = {'True': True, 'False': False}.get(
-            args['onlyFetchLog'])
-        components.logger().info(
-            f"get_job_direct_result_response, job: {task_id}")
-        task_result_request = TaskResultRequest(
-            task_id, task_type, only_fetch_log)
-        job_result_handler = TaskResultHandler(
-            task_result_request=task_result_request, components=components)
-        response = job_result_handler.get_response()
-        components.logger().info(
-            f"get_job_direct_result_response success, user: {user_name}, job: {task_id}, timecost: {time.time() - start_t}s")
-        return response
+        try:
+            """
+            Get the result related to the task_id
+            """
+            start_t = time.time()
+            args = request.get_json()
+            components.logger().info(
+                f"get task result, task_id: {task_id}, args: {args}")
+            user_name = args['user']
+            task_type = args['jobType']
+            only_fetch_log = {'True': True, 'False': False}.get(
+                args['onlyFetchLog'])
+            components.logger().info(
+                f"get_job_direct_result_response, job: {task_id}")
+            task_result_request = TaskResultRequest(
+                task_id, task_type, only_fetch_log)
+            job_result_handler = TaskResultHandler(
+                task_result_request=task_result_request, components=components)
+            response = job_result_handler.get_response()
+            components.logger().info(
+                f"get_job_direct_result_response success, user: {user_name}, job: {task_id}, timecost: {time.time() - start_t}s")
+            return response
+        except Exception as e:
+            response = {}
+            response.update({'errorCode': -1})
+            response.update(
+                {'message': f'query task log for {model_id} failed for {e}'})
+            return response
diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py
index ff6eff58..22663562 100644
--- a/python/ppc_model/secure_lgbm/vertical/active_party.py
+++ b/python/ppc_model/secure_lgbm/vertical/active_party.py
@@ -211,8 +211,6 @@ def _build_tree(self, feature_select, instance, glist, hlist, depth=0, weight=0)
             return weight
 
         self._leaf_id += 1
-        print('tree', self._tree_id, 'leaf', self._leaf_id, 'instance', len(instance),
-              'glist', len(glist), 'hlist', len(hlist))
         if self.params.colsample_bylevel > 0 and self.params.colsample_bylevel < 1:
             feature_select_level = sorted(np.random.choice(
                 feature_select, size=int(len(feature_select) * self.params.colsample_bylevel), replace=False))
diff --git a/python/ppc_model/secure_lgbm/vertical/passive_party.py b/python/ppc_model/secure_lgbm/vertical/passive_party.py
index 325f241b..0a3d773f 100644
--- a/python/ppc_model/secure_lgbm/vertical/passive_party.py
+++ b/python/ppc_model/secure_lgbm/vertical/passive_party.py
@@ -125,8 +125,6 @@ def _build_tree(self, instance, ghlist, depth=0, weight=0):
             return weight
 
         self._leaf_id += 1
-        print('tree', self._tree_id, 'leaf', self._leaf_id, 'instance', len(instance),
-              'ghlist', len(ghlist))
         best_split_info = self._find_best_split(instance, ghlist)
 
         if best_split_info.best_gain > 0 and best_split_info.best_gain > self.params.min_split_gain:
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index d39205ba..bba6c80e 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -83,7 +83,7 @@ def on_task_finished(self, task_result: TaskResult):
         worker_recorder = self.query_task(task_result.task_id)
         if worker_recorder is None:
             self.logger.warn(
-                f"TaskPersistent error, the task not found! task_result: {task_result}")
+                f"TaskPersistent error, the task {task_result.task_id} not found! task_result: {task_result}")
             return
         # update the task result
         worker_recorder.status = task_result.task_status
@@ -188,6 +188,12 @@ def status(self, task_id: str) -> [str, float, str]:
         """
         返回: 任务状态, 通讯量(MB), 执行耗时(s)
         """
+        # hit the cache: first query the memory
+        with self._rw_lock.gen_rlock():
+            if task_id in self._tasks.keys():
+                task_result = self._tasks.get(task_id)
+                return task_result.task_status, 0, task_result.exec_result
+        # miss the cache: query from the db
         result = self.task_persistent.query_task(task_id)
         if result is None:
             return TaskStatus.NotFound.value, 0.0, None
diff --git a/python/wedpr_ml_toolkit/requirements.txt b/python/wedpr_ml_toolkit/requirements.txt
index a9a92a16..39045603 100644
--- a/python/wedpr_ml_toolkit/requirements.txt
+++ b/python/wedpr_ml_toolkit/requirements.txt
@@ -1,4 +1,4 @@
-setuptools>=40.0.0
+setuptools>=70.0.0
 hdfs>=2.7.2
 requests~=2.31.0
 requests_toolbelt==0.9.1

From 114e7900d870b1da073791271e11b8789e11db2d Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 25 Nov 2024 20:09:33 +0800
Subject: [PATCH 102/120] fix cm2020 check hostResource (#101)

* fix cm2020 check hostResource

* complement log

* psi support retry

* unregister front taskInfo
---
 cpp/ppc-framework/protocol/DataResource.h     |  4 +++
 cpp/ppc-framework/protocol/PartyResource.h    |  4 +++
 cpp/ppc-framework/protocol/Task.h             | 14 ++++++++
 cpp/ppc-framework/rpc/RpcStatusInterface.h    |  2 --
 cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp | 10 +++---
 cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h   |  4 +--
 .../ppc-pir/tests/FakeOtPIRFactory.h          |  4 +--
 .../ppc-pir/tests/TestBaseOT.cpp              |  2 ++
 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp  | 18 +++++-----
 .../ppc-psi/src/cm2020-psi/CM2020PSIImpl.h    |  6 +---
 .../src/ecdh-conn-psi/EcdhConnPSIImpl.cpp     |  6 ++--
 .../src/ecdh-conn-psi/EcdhConnPSIImpl.h       |  5 +--
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp   | 16 ++++-----
 .../src/ecdh-multi-psi/EcdhMultiPSIImpl.h     |  3 +-
 .../core/EcdhMultiPSICalculator.cpp           |  2 +-
 .../core/EcdhMultiPSIMaster.cpp               |  2 +-
 .../core/EcdhMultiPSIPartner.cpp              |  2 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp      |  2 +-
 .../ppc-psi/src/ecdh-psi/EcdhPSIImpl.h        |  3 --
 .../src/labeled-psi/LabeledPSIImpl.cpp        | 11 +++---
 .../ppc-psi/src/labeled-psi/LabeledPSIImpl.h  |  5 +--
 .../ppc-psi/src/psi-framework/TaskGuarder.cpp |  2 ++
 .../ppc-psi/src/psi-framework/TaskGuarder.h   |  2 +-
 .../ppc-psi/src/psi-framework/TaskState.cpp   | 14 +++++---
 .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp  |  2 +-
 .../ppc-psi/src/ra2018-psi/RA2018PSIImpl.h    |  3 --
 .../tests/cm2020-psi/FakeCM2020PSIFactory.h   |  4 +--
 .../tests/cm2020-psi/TestCM2020Impl.cpp       |  2 ++
 .../tests/labeled-psi/FakeLabeledPSIFactory.h |  4 +--
 .../tests/labeled-psi/TestLabeledPSIImpl.cpp  |  4 +++
 .../tests/ra2018-psi/TestEcdhPSIImpl.cpp      |  4 +++
 .../tests/ra2018-psi/TestRA2018Impl.cpp       |  3 ++
 .../tests/ra2018-psi/mock/EcdhPSIFixture.h    |  1 -
 .../tests/ra2018-psi/mock/RA2018PSIFixture.h  |  1 -
 .../air-node/AirNodeInitializer.cpp           |  3 +-
 .../pro-node/ProNodeInitializer.cpp           |  3 +-
 .../protocol/src/JsonTaskImpl.cpp             |  5 +++
 .../ppc-io/src/DataResourceLoaderImpl.cpp     |  4 +--
 cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp | 34 +++++++++++--------
 cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h   | 10 ++++--
 40 files changed, 132 insertions(+), 98 deletions(-)

diff --git a/cpp/ppc-framework/protocol/DataResource.h b/cpp/ppc-framework/protocol/DataResource.h
index 012314c9..b69a5f65 100644
--- a/cpp/ppc-framework/protocol/DataResource.h
+++ b/cpp/ppc-framework/protocol/DataResource.h
@@ -119,6 +119,10 @@ class DataResource
 
 inline std::string printDataResourceInfo(DataResource::ConstPtr _dataResource)
 {
+    if (!_dataResource)
+    {
+        return "empty";
+    }
     std::ostringstream stringstream;
     stringstream << LOG_KV("dataResource", _dataResource->resourceID());
     if (_dataResource->desc())
diff --git a/cpp/ppc-framework/protocol/PartyResource.h b/cpp/ppc-framework/protocol/PartyResource.h
index 98bb4984..0c0e66f7 100644
--- a/cpp/ppc-framework/protocol/PartyResource.h
+++ b/cpp/ppc-framework/protocol/PartyResource.h
@@ -67,6 +67,10 @@ using ConstParties = std::vector;
 
 inline std::string printPartyInfo(PartyResource::ConstPtr _party)
 {
+    if (!_party)
+    {
+        return "empty";
+    }
     std::ostringstream stringstream;
     stringstream << LOG_KV("partyId", _party->id()) << LOG_KV("partyIndex", _party->partyIndex())
                  << LOG_KV("desc", _party->desc());
diff --git a/cpp/ppc-framework/protocol/Task.h b/cpp/ppc-framework/protocol/Task.h
index 1c9afccf..705fa670 100644
--- a/cpp/ppc-framework/protocol/Task.h
+++ b/cpp/ppc-framework/protocol/Task.h
@@ -143,6 +143,15 @@ class Task
     // decode the task
     virtual void decode(std::string_view _taskData) = 0;
     virtual std::string encode() const = 0;
+
+    virtual bool enableOutputExists() const { return m_enableOutputExists; }
+    virtual void setEnableOutputExists(bool enableOutputExists)
+    {
+        m_enableOutputExists = enableOutputExists;
+    }
+
+protected:
+    bool m_enableOutputExists = false;
 };
 
 class TaskFactory
@@ -160,9 +169,14 @@ class TaskFactory
 inline std::string printTaskInfo(Task::ConstPtr _task)
 {
     std::ostringstream stringstream;
+    if (!_task)
+    {
+        return "empty";
+    }
     stringstream << LOG_KV("id", _task->id())
                  << LOG_KV("type", (ppc::protocol::TaskType)_task->type())
                  << LOG_KV("algorithm", (ppc::protocol::TaskAlgorithmType)_task->algorithm())
+                 << LOG_KV("enableOutputExists", _task->enableOutputExists())
                  << LOG_KV("taskPtr", _task);
     if (_task->selfParty())
     {
diff --git a/cpp/ppc-framework/rpc/RpcStatusInterface.h b/cpp/ppc-framework/rpc/RpcStatusInterface.h
index 5a5b18d2..db971436 100644
--- a/cpp/ppc-framework/rpc/RpcStatusInterface.h
+++ b/cpp/ppc-framework/rpc/RpcStatusInterface.h
@@ -44,7 +44,5 @@ class RpcStatusInterface
     virtual bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) = 0;
     virtual bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) = 0;
     virtual protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) = 0;
-    virtual bcos::Error::Ptr deleteGateway(const std::string& _agencyID) = 0;
-    virtual std::vector listGateway() = 0;
 };
 }  // namespace ppc::rpc
\ No newline at end of file
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
index e344408e..a37fd892 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.cpp
@@ -149,10 +149,11 @@ void OtPIRImpl::onReceiveMessage(ppc::front::PPCMessageFace::Ptr _msg)
     }
 }
 
-void OtPIRImpl::onReceivedErrorNotification(const std::string& _taskID)
+void OtPIRImpl::onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message)
 {
+    PIR_LOG(WARNING) << LOG_DESC("onReceivedErrorNotification") << printPPCMsg(_message);
     // finish the task while the peer is failed
-    auto taskState = findPendingTask(_taskID);
+    auto taskState = findPendingTask(_message->taskID());
     if (taskState)
     {
         taskState->onPeerNotifyFinish();
@@ -242,7 +243,7 @@ void OtPIRImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr& _me
             {
             case int(CommonMessageType::ErrorNotification):
             {
-                pir->onReceivedErrorNotification(_message->taskID());
+                pir->onReceivedErrorNotification(_message);
                 break;
             }
             case int(CommonMessageType::PingPeer):
@@ -444,7 +445,8 @@ void OtPIRImpl::asyncRunTask()
                            << LOG_KV("requestAgencyDataset", taskMessage.requestAgencyDataset)
                            << LOG_KV("prefixLength", taskMessage.prefixLength)
                            << LOG_KV("searchId", taskMessage.searchId);
-            auto writer = loadWriter(task->id(), dataResource, m_enableOutputExists);
+            auto writer =
+                loadWriter(task->id(), dataResource, m_taskState->task()->enableOutputExists());
             m_taskState->setWriter(writer);
             runSenderGenerateCipher(taskMessage);
         }
diff --git a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
index 56aeac7d..d48773a2 100644
--- a/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
+++ b/cpp/wedpr-computing/ppc-pir/src/OtPIRImpl.h
@@ -59,7 +59,7 @@ class OtPIRImpl : public std::enable_shared_from_this,
     // register to the front to get the message related to ot-pir
     void onReceiveMessage(ppc::front::PPCMessageFace::Ptr _message) override;
 
-    void onReceivedErrorNotification(const std::string& _taskID) override;
+    void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message) override;
     void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override;
 
@@ -150,8 +150,6 @@ class OtPIRImpl : public std::enable_shared_from_this,
             m_senders.erase(it);
         }
     }
-    // allow the output-path exists, for ut
-    bool m_enableOutputExists = false;
     // 为true时启动时会从配置中加载文件作为匹配源
     bool m_enableMemoryFile = false;
     ppc::protocol::DataResource m_resource;
diff --git a/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h b/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h
index 23ee2179..30e6ce60 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h
+++ b/cpp/wedpr-computing/ppc-pir/tests/FakeOtPIRFactory.h
@@ -46,9 +46,7 @@ class FakeOtPIRImpl : public OtPIRImpl
     using Ptr = std::shared_ptr;
     FakeOtPIRImpl(OtPIRConfig::Ptr const& _config, unsigned _idleTimeMs = 0)
       : OtPIRImpl(_config, _idleTimeMs)
-    {
-        m_enableOutputExists = true;
-    }
+    {}
     ~FakeOtPIRImpl() override = default;
 };
 
diff --git a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
index 621a4255..615e309b 100644
--- a/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
+++ b/cpp/wedpr-computing/ppc-pir/tests/TestBaseOT.cpp
@@ -194,6 +194,7 @@ void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, b
     auto senderPIRTask = std::make_shared(senderAgencyName);
     senderPIRTask->setId(_taskID);
     senderPIRTask->setParam(_params);
+    senderPIRTask->setEnableOutputExists(true);
     senderPIRTask->setSelf(_senderParty);
     senderPIRTask->addParty(_receiverParty);
     senderPIRTask->setSyncResultToPeer(_syncResults);
@@ -203,6 +204,7 @@ void testOTPIRImplFunc(const std::string& _taskID, const std::string& _params, b
     auto receiverPIRTask = std::make_shared(receiverAgencyName);
     receiverPIRTask->setId(_taskID);
     receiverPIRTask->setParam(_params);
+    receiverPIRTask->setEnableOutputExists(true);
     receiverPIRTask->setSelf(_receiverParty);
     receiverPIRTask->addParty(_senderParty);
     receiverPIRTask->setSyncResultToPeer(_syncResults);
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
index 53d8f08e..fa56d7a0 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.cpp
@@ -152,15 +152,14 @@ void CM2020PSIImpl::asyncRunTask()
         {
             return;
         }
-
+        CM2020_PSI_LOG(INFO) << LOG_DESC("noticePeerToFinish") << printTaskInfo(task);
         psi->noticePeerToFinish(task);
     });
-    // check the memory
-    checkHostResource(m_config->minNeededMemoryGB());
-    addPendingTask(taskState);
-
     try
     {
+        addPendingTask(taskState);
+        // check the memory
+        checkHostResource(m_config->minNeededMemoryGB());
         // prepare reader and writer
         auto dataResource = task->selfParty()->dataResource();
         auto reader = loadReader(task->id(), dataResource, DataSchema::Bytes);
@@ -169,7 +168,7 @@ void CM2020PSIImpl::asyncRunTask()
         auto role = task->selfParty()->partyIndex();
         if (role == uint16_t(PartyType::Client) || task->syncResultToPeer())
         {
-            auto writer = loadWriter(task->id(), dataResource, m_enableOutputExists);
+            auto writer = loadWriter(task->id(), dataResource, task->enableOutputExists());
             taskState->setWriter(writer);
         }
 
@@ -319,10 +318,11 @@ void CM2020PSIImpl::stop()
     CM2020_PSI_LOG(INFO) << LOG_DESC("CM2020-PSI stopped");
 }
 
-void CM2020PSIImpl::onReceivedErrorNotification(const std::string& _taskID)
+void CM2020PSIImpl::onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message)
 {
+    CM2020_PSI_LOG(INFO) << LOG_DESC("onReceivedErrorNotification") << printPPCMsg(_message);
     // finish the task while the peer is failed
-    auto taskState = findPendingTask(_taskID);
+    auto taskState = findPendingTask(_message->taskID());
     if (taskState)
     {
         taskState->onPeerNotifyFinish();
@@ -410,7 +410,7 @@ void CM2020PSIImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr&
             {
             case int(CommonMessageType::ErrorNotification):
             {
-                psi->onReceivedErrorNotification(_message->taskID());
+                psi->onReceivedErrorNotification(_message);
                 break;
             }
             case int(CommonMessageType::PingPeer):
diff --git a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h
index 003fea3c..01c6d0ce 100644
--- a/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/cm2020-psi/CM2020PSIImpl.h
@@ -62,7 +62,7 @@ class CM2020PSIImpl : public bcos::Worker,
 
     void start() override;
     void stop() override;
-    void onReceivedErrorNotification(const std::string& _taskID) override;
+    void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message) override;
     void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override;
 
@@ -148,10 +148,6 @@ class CM2020PSIImpl : public bcos::Worker,
         }
     }
 
-protected:
-    // allow the output-path exists, for ut
-    bool m_enableOutputExists = false;
-
 private:
     void waitSignal()
     {
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
index 6e989d85..6e7a3717 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.cpp
@@ -55,7 +55,7 @@ void EcdhConnPSIImpl::asyncRunTask(
     if (role == uint16_t(PartyType::Client))
     {
         ECDH_CONN_LOG(INFO) << LOG_DESC("Client do the Task") << LOG_KV("taskID", _task->id());
-        auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists);
+        auto writer = loadWriter(_task->id(), dataResource, _task->enableOutputExists());
         ecdhTaskState->setWriter(writer);
         auto client = std::make_shared(m_config, ecdhTaskState);
         addClient(client);
@@ -66,7 +66,7 @@ void EcdhConnPSIImpl::asyncRunTask(
         ECDH_CONN_LOG(INFO) << LOG_DESC("Server do the Task") << LOG_KV("taskID", _task->id());
         if (_task->syncResultToPeer())
         {
-            auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists);
+            auto writer = loadWriter(_task->id(), dataResource, _task->enableOutputExists());
             ecdhTaskState->setWriter(writer);
         }
         auto server = std::make_shared(m_config, ecdhTaskState);
@@ -131,7 +131,7 @@ void EcdhConnPSIImpl::stop()
     }
 }
 
-void EcdhConnPSIImpl::onReceivedErrorNotification(const std::string& _taskID) {}
+void EcdhConnPSIImpl::onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const&) {}
 
 void EcdhConnPSIImpl::onSelfError(
     const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer)
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h
index 53d7cd4a..bb34a6a8 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-conn-psi/EcdhConnPSIImpl.h
@@ -61,14 +61,11 @@ class EcdhConnPSIImpl : public std::enable_shared_from_this,
     void start() override;
     void stop() override;
 
-    void onReceivedErrorNotification(const std::string& _taskID) override;
+    void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message) override;
     void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override;
     void executeWorker() override;
 
-protected:
-    bool m_enableOutputExists = false;
-
     void handlerPSIReceiveMessage(PSIConnMessage::Ptr _msg);
     void onHandShakeRequestHandler(const std::string& _taskId, const bcos::bytes& _msg);
     void onHandShakeResponseHandler(const std::string& _taskId, const bcos::bytes& _msg);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
index 94d7ec08..bb6d0d6d 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.cpp
@@ -131,9 +131,9 @@ void EcdhMultiPSIImpl::asyncRunTask(
             psi->removePartner(taskID);
             psi->removePendingTask(taskID);
         });
+        addPendingTask(taskState);
         // check the memory
         checkHostResource(m_config->minNeededMemoryGB());
-        addPendingTask(taskState);
         // over the peer limit
         if (_task->getAllPeerParties().size() > c_max_peer_size)
         {
@@ -155,7 +155,7 @@ void EcdhMultiPSIImpl::asyncRunTask(
                              << LOG_KV("roleId", role);
         if (role == uint16_t(PartiesType::Calculator))
         {
-            auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists);
+            auto writer = loadWriter(_task->id(), dataResource, _task->enableOutputExists());
             taskState->setWriter(writer);
             ECDH_MULTI_LOG(INFO) << LOG_DESC("Calculator do the Task")
                                  << LOG_KV("taskID", _task->id());
@@ -170,7 +170,7 @@ void EcdhMultiPSIImpl::asyncRunTask(
             if (_task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
                                                  m_config->selfParty()) != receivers.end())
             {
-                auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists);
+                auto writer = loadWriter(_task->id(), dataResource, _task->enableOutputExists());
                 taskState->setWriter(writer);
             }
             auto partner = std::make_shared(m_config, taskState);
@@ -183,7 +183,7 @@ void EcdhMultiPSIImpl::asyncRunTask(
             if (_task->syncResultToPeer() && std::find(receivers.begin(), receivers.end(),
                                                  m_config->selfParty()) != receivers.end())
             {
-                auto writer = loadWriter(_task->id(), dataResource, m_enableOutputExists);
+                auto writer = loadWriter(_task->id(), dataResource, _task->enableOutputExists());
                 taskState->setWriter(writer);
             }
             auto master = std::make_shared(m_config, taskState);
@@ -266,11 +266,11 @@ void EcdhMultiPSIImpl::checkFinishedTask()
     }
 }
 
-void EcdhMultiPSIImpl::onReceivedErrorNotification(const std::string& _taskID)
+void EcdhMultiPSIImpl::onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message)
 {
-    ECDH_MULTI_LOG(INFO) << LOG_DESC("onReceivedErrorNotification") << LOG_KV("taskID", _taskID);
+    ECDH_MULTI_LOG(INFO) << LOG_DESC("onReceivedErrorNotification") << printPPCMsg(_message);
     // finish the task while the peer is failed
-    auto taskState = findPendingTask(_taskID);
+    auto taskState = findPendingTask(_message->taskID());
     if (taskState)
     {
         taskState->onPeerNotifyFinish();
@@ -308,7 +308,7 @@ void EcdhMultiPSIImpl::executeWorker()
         auto pop_msg = _msg.second;
         if (pop_msg->messageType() == uint8_t(CommonMessageType::ErrorNotification))
         {
-            onReceivedErrorNotification(pop_msg->taskID());
+            onReceivedErrorNotification(pop_msg);
             return;
         }
         else if (pop_msg->messageType() == uint8_t(CommonMessageType::PingPeer))
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
index df4e8034..3be24c7b 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiPSIImpl.h
@@ -40,14 +40,13 @@ class EcdhMultiPSIImpl : public std::enable_shared_from_this,
     void stop() override;
 
     void checkFinishedTask();
-    void onReceivedErrorNotification(const std::string& _taskID) override;
+    void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message) override;
     void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override;
     void executeWorker() override;
 
 
 protected:
-    bool m_enableOutputExists = false;
     virtual void onReceiveRandomA(PSIMessageInterface::Ptr _msg);
     virtual void onReceiveCalCipher(PSIMessageInterface::Ptr _msg);
     virtual void handlerPSIReceiveMessage(PSIMessageInterface::Ptr _msg);
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
index c9e8b483..47d65bce 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSICalculator.cpp
@@ -95,7 +95,7 @@ void EcdhMultiPSICalculator::blindData(std::string _taskID, bcos::bytes _randA)
         uint64_t dataOffset = 0;
         do
         {
-            if (m_taskState->loadFinished())
+            if (m_taskState->loadFinished() || m_taskState->taskDone())
             {
                 break;
             }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
index 97295541..52632e4a 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIMaster.cpp
@@ -121,7 +121,7 @@ void EcdhMultiPSIMaster::blindData()
         auto reader = m_taskState->reader();
         do
         {
-            if (m_taskState->loadFinished())
+            if (m_taskState->loadFinished() || m_taskState->taskDone())
             {
                 break;
             }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
index c51617fc..67775c74 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/core/EcdhMultiPSIPartner.cpp
@@ -53,7 +53,7 @@ void EcdhMultiPSIPartner::onReceiveRandomA(bcos::bytesPointer _randA)
         uint64_t dataOffset = 0;
         do
         {
-            if (m_taskState->loadFinished())
+            if (m_taskState->loadFinished() || m_taskState->taskDone())
             {
                 break;
             }
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
index 694dfe86..3aa192bc 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.cpp
@@ -207,7 +207,7 @@ bool EcdhPSIImpl::initTaskState(TaskState::Ptr const& _taskState)
         if (!server)
         {
             m_cache->insertServerCipherCache(task->id(), _taskState);
-            if (!m_enableOutputExists)
+            if (!_taskState->task()->enableOutputExists())
             {
                 // Note: if the output-resource already exists, will throw exception
                 m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc());
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h
index 107280bf..52887156 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-psi/EcdhPSIImpl.h
@@ -85,8 +85,5 @@ class EcdhPSIImpl : public PSIFramework, public std::enable_shared_from_thisnoticePeerToFinish(_task);
             });
+            addPendingTask(taskState);
             // check the memory
             checkHostResource(m_config->minNeededMemoryGB());
-            addPendingTask(taskState);
 
             auto oprfClient = std::make_shared(
                 sizeof(apsi::Item::value_type) + sizeof(apsi::LabelKey), m_config->hash(),
@@ -220,10 +220,11 @@ void LabeledPSIImpl::stop()
     LABELED_PSI_LOG(INFO) << LOG_DESC("LabeledPSI stopped");
 }
 
-void LabeledPSIImpl::onReceivedErrorNotification(const std::string& _taskID)
+void LabeledPSIImpl::onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message)
 {
+    LABELED_PSI_LOG(WARNING) << LOG_DESC("onReceivedErrorNotification") << printPPCMsg(_message);
     // finish the task while the peer is failed
-    auto taskState = findPendingTask(_taskID);
+    auto taskState = findPendingTask(_message->taskID());
     if (taskState)
     {
         taskState->onPeerNotifyFinish();
@@ -553,7 +554,7 @@ void LabeledPSIImpl::saveSenderCache(const ppc::protocol::Task::ConstPtr& _task)
         auto dataResource = _task->selfParty()->dataResource();
 
         LineWriter::Ptr writer;
-        if (!m_enableOutputExists)
+        if (!_task->enableOutputExists())
         {
             // Note: if the output-resource already exists, will throw exception
             m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc());
@@ -690,7 +691,7 @@ void LabeledPSIImpl::handleReceivedMessage(const ppc::front::PPCMessageFace::Ptr
             {
             case int(CommonMessageType::ErrorNotification):
             {
-                psi->onReceivedErrorNotification(_message->taskID());
+                psi->onReceivedErrorNotification(_message);
                 break;
             }
             case int(CommonMessageType::PingPeer):
diff --git a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h
index 9cd06c4e..0d213fa2 100644
--- a/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/labeled-psi/LabeledPSIImpl.h
@@ -62,7 +62,7 @@ class LabeledPSIImpl : public bcos::Worker,
     void start() override;
     void stop() override;
 
-    void onReceivedErrorNotification(const std::string& _taskID) override;
+    void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const& _message) override;
     void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer) override;
 
@@ -116,9 +116,6 @@ class LabeledPSIImpl : public bcos::Worker,
         }
     }
 
-protected:
-    // allow the output-path exists, for ut
-    bool m_enableOutputExists = false;
 
 private:
     void waitSignal()
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
index 07cf6732..208602b9 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.cpp
@@ -199,6 +199,8 @@ void TaskGuarder::checkPeerActivity()
                         std::make_shared(
                             (int)PSIRetCode::PeerNodeDown, "peer node is down, id: " + peerID),
                         false);
+                    PSI_LOG(INFO) << LOG_DESC("checkPeerActivity: peer node-down")
+                                  << LOG_KV("peer", peerID) << LOG_KV("task", task->id());
                 },
                 nullptr);
         }
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
index c64c0604..8587d23c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskGuarder.h
@@ -44,7 +44,7 @@ class TaskGuarder
     }
     virtual ~TaskGuarder() = default;
 
-    virtual void onReceivedErrorNotification(const std::string& _taskID){};
+    virtual void onReceivedErrorNotification(ppc::front::PPCMessageFace::Ptr const&){};
     virtual void onSelfError(
         const std::string& _taskID, bcos::Error::Ptr _error, bool _noticePeer){};
 
diff --git a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
index 2dbf0b07..1ea77322 100644
--- a/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/psi-framework/TaskState.cpp
@@ -201,13 +201,16 @@ void TaskState::removeGeneratedOutputFile()
     {
         return;
     }
+    if (!m_writer)
+    {
+        return;
+    }
     auto outputDataResource = m_task->selfParty()->dataResource();
     if (!outputDataResource->desc())
     {
         return;
     }
-    PSI_LOG(INFO) << LOG_DESC("removeGeneratedFilesForFailed")
-                  << LOG_KV("task", printTaskInfo(m_task));
+    PSI_LOG(INFO) << LOG_DESC("removeGeneratedOutputFile") << LOG_KV("task", printTaskInfo(m_task));
     m_config->dataResourceLoader()->deleteResource(outputDataResource->desc());
 }
 
@@ -271,10 +274,11 @@ void TaskState::onTaskFinished(TaskResult::Ptr _result, bool _noticePeer)
 void TaskState::onPeerNotifyFinish()
 {
     PSI_LOG(WARNING) << LOG_BADGE("onReceivePeerError") << LOG_KV("taskID", m_task->id());
-    auto tesult = std::make_shared(task()->id());
-    tesult->setError(std::make_shared(
+    auto result = std::make_shared(task()->id());
+    result->setError(std::make_shared(
         (int)PSIRetCode::PeerNotifyFinish, "job participant sent an error"));
-    onTaskFinished(std::move(tesult), false);
+    onTaskFinished(std::move(result), false);
+    removeGeneratedOutputFile();
 }
 
 // Note: must store the result serially
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp
index 7b98051b..38f19ddb 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.cpp
@@ -646,7 +646,7 @@ void RA2018PSIImpl::runClientPSI(TaskState::Ptr const& _taskState)
     {
         return;
     }
-    if (!m_enableOutputExists)
+    if (!_taskState->task()->enableOutputExists())
     {
         // Note: if the output-resource already exists, will throw exception
         m_config->dataResourceLoader()->checkResourceExists(dataResource->outputDesc());
diff --git a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h
index d47c7253..c3cf9a3c 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h
+++ b/cpp/wedpr-computing/ppc-psi/src/ra2018-psi/RA2018PSIImpl.h
@@ -179,8 +179,5 @@ class RA2018PSIImpl : public PSIFramework, public std::enable_shared_from_this;
     FakeCM2020PSIImpl(CM2020PSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0)
       : CM2020PSIImpl(_config, _idleTimeMs)
-    {
-        m_enableOutputExists = true;
-    }
+    {}
     ~FakeCM2020PSIImpl() override = default;
 };
 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
index c4fdf8eb..6014a1df 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/cm2020-psi/TestCM2020Impl.cpp
@@ -153,6 +153,7 @@ void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _param
     senderPSITask->setId(_taskID);
     senderPSITask->setParam(_params);
     senderPSITask->setSelf(_senderParty);
+    senderPSITask->setEnableOutputExists(true);
     senderPSITask->addParty(_receiverParty);
     senderPSITask->setSyncResultToPeer(_syncResults);
     senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::CM_PSI_2PC);
@@ -161,6 +162,7 @@ void testCM2020PSIImplFunc(const std::string& _taskID, const std::string& _param
     receiverPSITask->setId(_taskID);
     receiverPSITask->setParam(_params);
     receiverPSITask->setSelf(_receiverParty);
+    receiverPSITask->setEnableOutputExists(true);
     receiverPSITask->addParty(_senderParty);
     receiverPSITask->setSyncResultToPeer(_syncResults);
     receiverPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::CM_PSI_2PC);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
index e369b6c1..350d861a 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/FakeLabeledPSIFactory.h
@@ -43,9 +43,7 @@ class FakeLabeledPSIImpl : public LabeledPSIImpl
     using Ptr = std::shared_ptr;
     FakeLabeledPSIImpl(LabeledPSIConfig::Ptr const& _config, unsigned _idleTimeMs = 0)
       : LabeledPSIImpl(_config, _idleTimeMs)
-    {
-        m_enableOutputExists = true;
-    }
+    {}
     ~FakeLabeledPSIImpl() override = default;
 };
 
diff --git a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
index f341b6f2..62f6fd2e 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/labeled-psi/TestLabeledPSIImpl.cpp
@@ -59,6 +59,7 @@ void runSetup(LabeledPSIImpl::Ptr _sender)
 
     auto senderPSITask = std::make_shared(senderAgencyName);
     senderPSITask->setId("0x00000000");
+    senderPSITask->setEnableOutputExists(true);
     senderPSITask->setParam(R"(["setup_sender_db","32"])");
     senderPSITask->setSelf(senderParty);
     senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
@@ -95,6 +96,7 @@ void saveCache(LabeledPSIImpl::Ptr _sender)
 
     auto senderPSITask = std::make_shared(senderAgencyName);
     senderPSITask->setId("0x00000012");
+    senderPSITask->setEnableOutputExists(true);
     senderPSITask->setParam(R"(["save_sender_cache"])");
     senderPSITask->setSelf(senderParty);
     senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
@@ -234,6 +236,7 @@ void testLabeledPSIImplFunc(const std::string& _taskID, const std::string& _para
     // trigger the psi task
     auto senderPSITask = std::make_shared(senderAgencyName);
     senderPSITask->setId(_taskID);
+    senderPSITask->setEnableOutputExists(true);
     senderPSITask->setParam(_params);
     senderPSITask->setSelf(_senderParty);
     senderPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
@@ -241,6 +244,7 @@ void testLabeledPSIImplFunc(const std::string& _taskID, const std::string& _para
 
     auto receiverPSITask = std::make_shared(receiverAgencyName);
     receiverPSITask->setId(_taskID);
+    receiverPSITask->setEnableOutputExists(true);
     receiverPSITask->setSelf(_receiverParty);
     receiverPSITask->setAlgorithm((uint8_t)TaskAlgorithmType::LABELED_PSI_2PC);
     receiverPSITask->addParty(_senderParty);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
index 856eabbc..c0083254 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestEcdhPSIImpl.cpp
@@ -87,6 +87,8 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS
     auto clientPSITask = std::make_shared(clientAgencyName);
     std::string taskID = "runPSI";
     clientPSITask->setId(taskID);
+    clientPSITask->setEnableOutputExists(true);
+    clientPSITask->setEnableOutputExists(true);
     clientPSITask->setType((int8_t)TaskType::PSI);
     clientPSITask->setAlgorithm((int8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     clientPSITask->setSelf(clientParty);
@@ -103,6 +105,8 @@ void testEcdhImplFunc(int64_t _dataBatchSize, std::string const& _serverPSIDataS
     {
         serverPSITask->setId(taskID);
     }
+    serverPSITask->setEnableOutputExists(true);
+    serverPSITask->setEnableOutputExists(true);
     serverPSITask->setType((int8_t)TaskType::PSI);
     serverPSITask->setAlgorithm((int8_t)TaskAlgorithmType::ECDH_PSI_2PC);
     serverPSITask->addParty(clientParty);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
index 261cd401..2e6c06fa 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/TestRA2018Impl.cpp
@@ -105,6 +105,7 @@ void testRA2018PSIImplFunc(int _dataBatchSize, CuckoofilterOption::Ptr option,
     auto offlineFullEvaluateTask = std::make_shared(serverAgencyName);
     offlineFullEvaluateTask->setId("offlineFullEvaluate");
     offlineFullEvaluateTask->setSelf(serverParty);
+    offlineFullEvaluateTask->setEnableOutputExists(true);
     // insert operation
     std::string param = "[\"data_preprocessing\", 0]";
     offlineFullEvaluateTask->setParam(param);
@@ -127,6 +128,7 @@ void testRA2018PSIImplFunc(int _dataBatchSize, CuckoofilterOption::Ptr option,
     auto clientPSITask = std::make_shared(clientAgencyName);
     std::string taskID = "runPSI";
     clientPSITask->setId(taskID);
+    clientPSITask->setEnableOutputExists(true);
     clientPSITask->setSelf(clientParty);
     clientPSITask->addParty(serverParty);
     param = "[\"ra2018_psi\"]";
@@ -135,6 +137,7 @@ void testRA2018PSIImplFunc(int _dataBatchSize, CuckoofilterOption::Ptr option,
     // the server task
     auto serverPSITask = std::make_shared(serverAgencyName);
     serverPSITask->setSelf(serverParty);
+    serverPSITask->setEnableOutputExists(true);
     serverPSITask->setId(taskID);
     serverPSITask->setParam(param);
     serverPSITask->addParty(clientParty);
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
index 37209c9a..4404c8b3 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/EcdhPSIFixture.h
@@ -49,7 +49,6 @@ class FakeEcdhPSIImpl : public EcdhPSIImpl
     {
         // set the m_started flag to be true
         m_started = true;
-        m_enableOutputExists = true;
         m_taskSyncTimer->registerTimeoutHandler([this]() { syncTaskInfo(); });
         m_taskSyncTimer->start();
     }
diff --git a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
index c0bb0d34..83eaffbe 100644
--- a/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
+++ b/cpp/wedpr-computing/ppc-psi/tests/ra2018-psi/mock/RA2018PSIFixture.h
@@ -51,7 +51,6 @@ class FakeRA2018Impl : public RA2018PSIImpl
     {
         // set the m_started flag to be true
         m_started = true;
-        m_enableOutputExists = true;
         m_taskSyncTimer->registerTimeoutHandler([this]() { syncTaskInfo(); });
         m_taskSyncTimer->start();
     }
diff --git a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
index 38c6709d..a9636629 100644
--- a/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
+++ b/cpp/wedpr-main/air-node/AirNodeInitializer.cpp
@@ -62,7 +62,8 @@ void AirNodeInitializer::init(std::string const& _configPath)
 
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
     // init RpcStatusInterface
-    RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared();
+    RpcStatusInterface::Ptr rpcStatusInterface =
+        std::make_shared(m_nodeInitializer->ppcFront());
 
 
     auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID());
diff --git a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
index 4d54658e..f92750c8 100644
--- a/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
+++ b/cpp/wedpr-main/pro-node/ProNodeInitializer.cpp
@@ -53,7 +53,8 @@ void ProNodeInitializer::init(std::string const& _configPath)
 
     INIT_LOG(INFO) << LOG_DESC("init the rpc");
     // init RpcStatusInterface
-    RpcStatusInterface::Ptr rpcStatusInterface = std::make_shared();
+    RpcStatusInterface::Ptr rpcStatusInterface =
+        std::make_shared(m_nodeInitializer->ppcFront());
 
 
     auto rpcFactory = std::make_shared(m_nodeInitializer->config()->agencyID());
diff --git a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
index 6e7af353..a0052b37 100644
--- a/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
+++ b/cpp/wedpr-protocol/protocol/src/JsonTaskImpl.cpp
@@ -54,6 +54,10 @@ void JsonTaskImpl::decodeJsonValue(Json::Value const& root)
     {
         BOOST_THROW_EXCEPTION(InvalidParameter() << errinfo_comment("Must specify the taskType"));
     }
+    if (root.isMember("enableOutputExists"))
+    {
+        m_enableOutputExists = root["enableOutputExists"].asBool();
+    }
     checkNull(root["type"], "taskType");
     m_type = root["type"].asUInt();
     // the taskAlgorithm
@@ -309,6 +313,7 @@ std::string JsonTaskImpl::encode() const
     // sync-result or not
     taskInfo["syncResult"] = m_syncResultToPeer;
     taskInfo["lowBandwidth"] = m_lowBandwidth;
+    taskInfo["enableOutputExists"] = m_enableOutputExists;
 
     Json::Value receiverList;
     for (auto const& it : m_receiverLists)
diff --git a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
index 00d9cfdf..6d951bf4 100644
--- a/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
+++ b/cpp/wedpr-storage/ppc-io/src/DataResourceLoaderImpl.cpp
@@ -213,7 +213,7 @@ void DataResourceLoaderImpl::deleteResource(
     {
         BOOST_THROW_EXCEPTION(
             UnSupportedDataResource() << errinfo_comment(
-                "checkResourceExists: Only support File/HDFS now, passed in resource type: " +
+                "deleteResource: Only support File/HDFS now, passed in resource type: " +
                 std::to_string(_desc->type())));
     }
     }
@@ -264,7 +264,7 @@ void DataResourceLoaderImpl::renameResource(ppc::protocol::DataResourceDesc::Con
     {
         BOOST_THROW_EXCEPTION(
             UnSupportedDataResource() << errinfo_comment(
-                "checkResourceExists: Only support File/HDFS now, passed in resource type: " +
+                "renameResource: Only support File/HDFS now, passed in resource type: " +
                 std::to_string(_desc->type())));
     }
     }
diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
index 86074c47..31f923d1 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
@@ -72,9 +72,26 @@ void RpcMemory::cleanTask()
 bcos::Error::Ptr RpcMemory::insertTask(protocol::Task::Ptr _task)
 {
     WriteGuard l(x_tasks);
-    if (m_tasks.find(_task->id()) != m_tasks.end())
+    auto it = m_tasks.find(_task->id());
+    if (it != m_tasks.end())
     {
-        return std::make_shared(PPCRetCode::WRITE_RPC_STATUS_ERROR, "task exists");
+        auto taskResult = it->second.second;
+        // the task already exists case
+        if (!taskResult || taskResult->status() == toString(TaskStatus::RUNNING))
+        {
+            return std::make_shared(PPCRetCode::WRITE_RPC_STATUS_ERROR, "task exists");
+        }
+        if (taskResult)
+        {
+            RPC_STATUS_LOG(INFO) << LOG_DESC("find the existed not running-task")
+                                 << LOG_KV("task", _task->id())
+                                 << LOG_KV("status", taskResult->status());
+            if (taskResult->status() != toString(TaskState::COMPLETED))
+            {
+                // erase the task_id
+                m_front->eraseTaskInfo(_task->id());
+            }
+        }
     }
     auto taskResult = std::make_shared(_task->id());
     taskResult->setStatus(toString(TaskStatus::RUNNING));
@@ -107,15 +124,4 @@ TaskResult::Ptr RpcMemory::getTaskStatus(const std::string& _taskID)
     }
 
     return m_tasks[_taskID].second;
-}
-
-
-bcos::Error::Ptr RpcMemory::deleteGateway(const std::string& _agencyID)
-{
-    return nullptr;
-}
-
-std::vector RpcMemory::listGateway()
-{
-    return {};
-}
+}
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h
index 78a447c5..bb2b0a3c 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h
+++ b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.h
@@ -20,6 +20,7 @@
  */
 
 #pragma once
+#include "ppc-framework/front/FrontInterface.h"
 #include "ppc-framework/rpc/RpcStatusInterface.h"
 #include 
 #include 
@@ -31,7 +32,10 @@ class RpcMemory : public RpcStatusInterface
 public:
     using Ptr = std::shared_ptr;
 
-    RpcMemory() : m_taskCleaner(std::make_shared(60 * 60 * 1000, "taskCleaner")) {}
+    RpcMemory(ppc::front::FrontInterface::Ptr front)
+      : m_front(std::move(front)),
+        m_taskCleaner(std::make_shared(60 * 60 * 1000, "taskCleaner"))
+    {}
     ~RpcMemory() override = default;
 
     void start() override;
@@ -40,13 +44,13 @@ class RpcMemory : public RpcStatusInterface
     bcos::Error::Ptr insertTask(protocol::Task::Ptr _task) override;
     bcos::Error::Ptr updateTaskStatus(protocol::TaskResult::Ptr _taskResult) override;
     protocol::TaskResult::Ptr getTaskStatus(const std::string& _taskID) override;
-    bcos::Error::Ptr deleteGateway(const std::string& _agencyID) override;
-    std::vector listGateway() override;
 
 protected:
     void cleanTask();
 
 private:
+    ppc::front::FrontInterface::Ptr m_front;
+
     mutable bcos::SharedMutex x_tasks;
     std::unordered_map> m_tasks;
     std::shared_ptr m_taskCleaner;

From 07a1d94f9358e658a46cedc5adbec04f15655cd6 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 26 Nov 2024 12:12:57 +0800
Subject: [PATCH 103/120] Simplify the implementation of DatasetToolkit (#102)

* Simplify the implementation of DatasetToolkit

* use ModelSetting class to replace model_setting dict
---
 .../wedpr_ml_toolkit/test/test_dataset.ipynb  | 209 ------------
 .../wedpr_ml_toolkit/test/test_ml_toolkit.py  |  58 ++--
 python/wedpr_ml_toolkit/test/test_psi.ipynb   | 142 --------
 .../wedpr_ml_toolkit/test/test_xgboost.ipynb  | 311 ------------------
 .../common/utils/base_object.py               |  11 +
 .../wedpr_ml_toolkit/common/utils/constant.py |   1 +
 .../wedpr_ml_toolkit/common/utils/utils.py    |  16 +
 .../config/wedpr_ml_config.py                 |   6 +
 .../config/wedpr_model_setting.py             |  79 -----
 .../wedpr_ml_toolkit/context/data_context.py  |  44 +--
 .../context/dataset_context.py                |  55 ++++
 .../wedpr_ml_toolkit/context/job_context.py   | 141 ++++----
 .../wedpr_ml_toolkit/context/model_setting.py | 131 ++++++++
 .../wedpr_ml_toolkit/toolkit/__init__.py      |   0
 .../toolkit/dataset_toolkit.py                |  67 ----
 .../transport/wedpr_remote_dataset_client.py  |  86 +++++
 .../transport/wedpr_remote_job_client.py      |  15 +-
 .../wedpr_ml_toolkit/wedpr_ml_toolkit.py      |  28 +-
 18 files changed, 462 insertions(+), 938 deletions(-)
 delete mode 100644 python/wedpr_ml_toolkit/test/test_dataset.ipynb
 delete mode 100644 python/wedpr_ml_toolkit/test/test_psi.ipynb
 delete mode 100644 python/wedpr_ml_toolkit/test/test_xgboost.ipynb
 delete mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py
 create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
 create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/model_setting.py
 delete mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py
 delete mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
 create mode 100644 python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py

diff --git a/python/wedpr_ml_toolkit/test/test_dataset.ipynb b/python/wedpr_ml_toolkit/test/test_dataset.ipynb
deleted file mode 100644
index e6fcef07..00000000
--- a/python/wedpr_ml_toolkit/test/test_dataset.ipynb
+++ /dev/null
@@ -1,209 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "['/usr/lib/python3/dist-packages/wedpr_ml_toolkit/', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python\\\\wedpr_ml_toolkit', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python', 'd:\\\\github\\\\wedpr3.0\\\\WeDPR-Component\\\\python', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\python38.zip', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\DLLs', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib', 'c:\\\\Users\\\\yanxi\\\\anaconda3', '', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\win32', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\win32\\\\lib', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\Pythonwin', 'c:\\\\Users\\\\yanxi\\\\anaconda3\\\\lib\\\\site-packages\\\\IPython\\\\extensions', 'C:\\\\Users\\\\yanxi\\\\.ipython']\n"
-     ]
-    }
-   ],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
-    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
-    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 读取配置文件\n",
-    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
-    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "http://139.159.202.235:50070 /user/ppc/milestone2/sgd/test_user SGD\n",
-      "/user/ppc/milestone2/sgd/test_user\\d-101\n",
-      "   id  y        x1        x2        x3        x4        x5        x6  \\\n",
-      "0   0  1  0.954183  0.652034  0.704070  0.180889  0.025025  0.511596   \n",
-      "1   1  1  0.302088  0.462222  0.435542  0.029966  0.931294  0.848483   \n",
-      "2   2  1  0.468104  0.430161  0.239322  0.588153  0.470668  0.225856   \n",
-      "3   3  0  0.152269  0.811666  0.834451  0.354288  0.635447  0.062092   \n",
-      "4   4  0  0.841470  0.800512  0.451507  0.118651  0.748845  0.557916   \n",
-      "\n",
-      "         x7        x8        x9       x10  \n",
-      "0  0.529848  0.759689  0.159081  0.556419  \n",
-      "1  0.962787  0.224096  0.464418  0.208487  \n",
-      "2  0.564879  0.730366  0.394245  0.299081  \n",
-      "3  0.424057  0.202234  0.577448  0.636958  \n",
-      "4  0.030906  0.514350  0.340864  0.123303  \n"
-     ]
-    }
-   ],
-   "source": [
-    "# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path\n",
-    "# 1. pd.Dataframe\n",
-    "df = pd.DataFrame({\n",
-    "    'id': np.arange(0, 100),  # id列，顺序整数\n",
-    "    'y': np.random.randint(0, 2, size=100),\n",
-    "    # x1到x10列，随机数\n",
-    "    **{f'x{i}': np.random.rand(100) for i in range(1, 11)}\n",
-    "})\n",
-    "\n",
-    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
-    "                          dataset_owner='flyhuang1',\n",
-    "                          agency=wedpr_config.user_config.agency_name,\n",
-    "                          values=df,\n",
-    "                          is_label_holder=True)\n",
-    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
-    "dataset1.storage_client = None  # 本地测试时跳过hdfs上传/下载过程\n",
-    "dataset1.save_values(path='d-101')\n",
-    "print(dataset1.dataset_path)\n",
-    "print(dataset1.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "http://139.159.202.235:50070 /user/ppc/milestone2/sgd/test_user WeBank\n",
-      "/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\n",
-      "/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\n",
-      "   id        z1        z2        z3        z4        z5        z6        z7  \\\n",
-      "0   0  0.597205  0.942475  0.886443  0.560584  0.254432  0.370152  0.076031   \n",
-      "1   1  0.778616  0.607374  0.616211  0.602282  0.385989  0.816963  0.756814   \n",
-      "2   2  0.999795  0.596794  0.240741  0.241070  0.857676  0.342412  0.066459   \n",
-      "3   3  0.968410  0.895163  0.636140  0.978791  0.237098  0.095272  0.938806   \n",
-      "4   4  0.921513  0.454901  0.004514  0.769216  0.627185  0.676253  0.184952   \n",
-      "\n",
-      "         z8        z9       z10  \n",
-      "0  0.587627  0.851390  0.864929  \n",
-      "1  0.661537  0.865674  0.050091  \n",
-      "2  0.473916  0.080120  0.477873  \n",
-      "3  0.452399  0.953515  0.405465  \n",
-      "4  0.877475  0.316322  0.139290  \n"
-     ]
-    }
-   ],
-   "source": [
-    "# 2. hdfs_path\n",
-    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
-    "                          dataset_owner='flyhuang',\n",
-    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
-    "                          agency=\"WeBank\")\n",
-    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
-    "print(dataset2.dataset_path)\n",
-    "dataset2.storage_client = None  # 本地测试时跳过hdfs上传/下载过程\n",
-    "\n",
-    "# 提供本地测试数据\n",
-    "if dataset2.storage_client is None:\n",
-    "    # 支持更新dataset的values数据\n",
-    "    df2 = pd.DataFrame({\n",
-    "        'id': np.arange(0, 100),  # id列，顺序整数\n",
-    "        **{f'z{i}': np.random.rand(100) for i in range(1, 11)}  # x1到x10列，随机数\n",
-    "    })\n",
-    "    dataset2.update_values(values=df2)\n",
-    "    dataset2.save_values()\n",
-    "    print(dataset2.dataset_path)\n",
-    "    print(dataset2.values.head())\n",
-    "\n",
-    "# 对于己方数据集支持load_values，其他方数据集无需load_values，可直接使用\n",
-    "if dataset2.storage_client is not None:\n",
-    "    # 仅支持load本机构hdfs的数据集\n",
-    "    dataset2.load_values(header=0)\n",
-    "    print(dataset2.dataset_path)\n",
-    "    print(dataset2.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/user/ppc/milestone2/sgd/test_user\\d-101\n",
-      "   id  y        x1        x2        x3        x4        x5        x6  \\\n",
-      "0   0  1  0.954183  0.652034  0.704070  0.180889  0.025025  0.511596   \n",
-      "1   1  1  0.302088  0.462222  0.435542  0.029966  0.931294  0.848483   \n",
-      "2   2  1  0.468104  0.430161  0.239322  0.588153  0.470668  0.225856   \n",
-      "3   3  0  0.152269  0.811666  0.834451  0.354288  0.635447  0.062092   \n",
-      "4   4  0  0.841470  0.800512  0.451507  0.118651  0.748845  0.557916   \n",
-      "\n",
-      "         x7        x8        x9       x10  \n",
-      "0  0.529848  0.759689  0.159081  0.556419  \n",
-      "1  0.962787  0.224096  0.464418  0.208487  \n",
-      "2  0.564879  0.730366  0.394245  0.299081  \n",
-      "3  0.424057  0.202234  0.577448  0.636958  \n",
-      "4  0.030906  0.514350  0.340864  0.123303  \n"
-     ]
-    }
-   ],
-   "source": [
-    "# 更新数据集\n",
-    "if dataset1.storage_client is not None:\n",
-    "    dataset1.update_values(\n",
-    "        path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485')\n",
-    "    dataset1.load_values(header=0)\n",
-    "print(dataset1.dataset_path)\n",
-    "print(dataset1.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
index 3d076a80..6e87fa49 100644
--- a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
@@ -5,10 +5,10 @@
 from sklearn import metrics
 from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder
 from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit
-from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.context.job_context import JobType
-from wedpr_ml_toolkit.config.wedpr_model_setting import PreprocessingModelSetting
+from wedpr_ml_toolkit.context.model_setting import PreprocessingSetting
 
 
 class WeDPRMlToolkitTestWrapper:
@@ -25,17 +25,17 @@ def test_submit_job(self):
             # x1到x10列，随机数
             **{f'x{i}': np.random.rand(100) for i in range(1, 11)}
         })
-
-        dataset1 = DatasetToolkit(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
+        dataset1 = DatasetContext(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
+                                  dataset_client=self.wedpr_ml_toolkit.get_dataset_client(),
                                   storage_workspace=self.wedpr_config.user_config.get_workspace_path(),
-                                  agency=self.wedpr_config.user_config.agency_name,
-                                  values=df,
+                                  dataset_id="d-9743660607744005",
                                   is_label_holder=True)
-        dataset1.save_values(path='d-101')
+        dataset1.save_values(df, path='d-101')
 
         # hdfs_path
-        dataset2 = DatasetToolkit(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
-                                  dataset_path="d-9606695119693829", agency="WeBank")
+        dataset2 = DatasetContext(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
+                                  dataset_client=self.wedpr_ml_toolkit.get_dataset_client(),
+                                  dataset_id="d-9743674298214405")
 
         dataset2.storage_client = None
         # dataset2.load_values()
@@ -46,33 +46,42 @@ def test_submit_job(self):
                 # x1到x10列，随机数
                 **{f'z{i}': np.random.rand(100) for i in range(1, 11)}
             })
-            dataset2.update_values(values=df2)
+            dataset2.save_values(values=df2)
         if dataset1.storage_client is not None:
-            dataset1.update_values(
-                path='/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485')
-            dataset1.load_values()
+            # save values to dataset1
+            dataset1.save_values(df)
+            (values, columns, shape) = dataset1.load_values()
+            print(f"### values: {values}")
 
         # 构建 dataset context
         dataset = DataContext(dataset1, dataset2)
 
         # init the job context
-        project_name = "1"
-
+        project_id = "9737304249804806"
+        print("* build psi job context")
         psi_job_context = self.wedpr_ml_toolkit.build_job_context(
-            JobType.PSI, project_name, dataset, None, "id")
+            JobType.PSI, project_id, dataset, None, "id")
         print(psi_job_context.participant_id_list,
               psi_job_context.result_receiver_id_list)
         # 执行psi任务
+        print("* submit psi job")
         psi_job_id = psi_job_context.submit()
+        print(f"* submit psi job success, job_id: {psi_job_id}")
         psi_result = psi_job_context.fetch_job_result(psi_job_id, True)
-
+        print(
+            f"* fetch_job_result for psi job {psi_job_id} success, result: {psi_result}")
         # 初始化
-        preprocessing_data = DataContext(dataset1)
+        print(f"* build pre-processing data-context")
+        preprocessing_data = DataContext(dataset1, dataset2)
         preprocessing_job_context = self.wedpr_ml_toolkit.build_job_context(
-            JobType.PREPROCESSING, project_name, preprocessing_data, PreprocessingModelSetting())
+            JobType.PREPROCESSING, project_id, preprocessing_data, PreprocessingSetting())
         # 执行预处理任务
-        fe_job_id = preprocessing_job_context.submit(dataset)
+        print(f"* submit pre-processing job")
+        fe_job_id = preprocessing_job_context.submit()
+        print(f"* submit pre-processing job success, job_id: {fe_job_id}")
         fe_result = preprocessing_job_context.fetch_job_result(fe_job_id, True)
+        print(
+            f"* fetch pre-processing job result success, job_id: {fe_job_id}, result: {fe_result}")
         print(preprocessing_job_context.participant_id_list,
               preprocessing_job_context.result_receiver_id_list)
 
@@ -89,14 +98,7 @@ def test_query_job(self, job_id: str, block_until_finish):
 class TestMlToolkit(unittest.TestCase):
     def test_query_jobs(self):
         wrapper = WeDPRMlToolkitTestWrapper("config.properties")
-        # the success job case
-        success_job_id = "9630202187032582"
-        wrapper.test_query_job(success_job_id, False)
-        # wrapper.test_query_job(success_job_id, True)
-        # the fail job case
-        # failed_job_id = "9630156365047814"
-        # wrapper.test_query_job(success_job_id, False)
-        # wrapper.test_query_job(success_job_id, True)
+        job_id = wrapper.test_submit_job()
 
 
 if __name__ == '__main__':
diff --git a/python/wedpr_ml_toolkit/test/test_psi.ipynb b/python/wedpr_ml_toolkit/test/test_psi.ipynb
deleted file mode 100644
index 435b3586..00000000
--- a/python/wedpr_ml_toolkit/test/test_psi.ipynb
+++ /dev/null
@@ -1,142 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
-    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
-    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit\n",
-    "from wedpr_ml_toolkit.context.data_context import DataContext\n",
-    "from wedpr_ml_toolkit.context.job_context import JobType"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 读取配置文件\n",
-    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
-    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 注册 dataset1\n",
-    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
-    "                          dataset_owner='flyhuang1',\n",
-    "                          agency=wedpr_config.user_config.agency_name,\n",
-    "                          dataset_id = 'd-9606704699156485',\n",
-    "                          dataset_path=\"/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485\",\n",
-    "                          is_label_holder=True)\n",
-    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
-    "dataset1.load_values(header=0)\n",
-    "print(dataset1.dataset_path)\n",
-    "print(dataset1.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 注册 dataset2\n",
-    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
-    "                          dataset_owner='flyhuang',\n",
-    "                          dataset_id = 'd-9606695119693829',\n",
-    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
-    "                          agency=\"WeBank\")\n",
-    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
-    "dataset2.load_values(header=0)\n",
-    "print(dataset2.dataset_path)\n",
-    "print(dataset2.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 构建 dataset context\n",
-    "dataset = DataContext(dataset1, dataset2)\n",
-    "print(dataset.datasets)\n",
-    "\n",
-    "# init the job context\n",
-    "project_id = \"9606702107011078\"\n",
-    "\n",
-    "# 构造psi任务配置\n",
-    "psi_job_context = wedpr_ml_toolkit.build_job_context(\n",
-    "    JobType.PSI, project_id, dataset, None, \"id\")\n",
-    "print(psi_job_context.participant_id_list, psi_job_context.result_receiver_id_list)\n",
-    "print(psi_job_context.project_id)\n",
-    "\n",
-    "psi_job_param = psi_job_context.build()\n",
-    "print(psi_job_param.taskParties)\n",
-    "print(psi_job_param.datasetList)\n",
-    "print(psi_job_param.job)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 执行psi任务\n",
-    "# psi_job_id = '9670241574201350'  # 测试时跳过创建新任务过程\n",
-    "psi_job_id = psi_job_context.submit()\n",
-    "print(psi_job_id)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 获取psi任务结果\n",
-    "# psi_job_id = '9670241574201350'  # 测试时跳过创建新任务过程\n",
-    "print(psi_job_id)\n",
-    "psi_result = psi_job_context.parse_result(psi_job_id, True)\n",
-    "psi_result.load_values()\n",
-    "print(psi_result.values.shape)\n",
-    "print(psi_result.values.head())"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/python/wedpr_ml_toolkit/test/test_xgboost.ipynb b/python/wedpr_ml_toolkit/test/test_xgboost.ipynb
deleted file mode 100644
index 84082084..00000000
--- a/python/wedpr_ml_toolkit/test/test_xgboost.ipynb
+++ /dev/null
@@ -1,311 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder\n",
-    "from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit\n",
-    "from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit\n",
-    "from wedpr_ml_toolkit.context.data_context import DataContext\n",
-    "from wedpr_ml_toolkit.context.job_context import JobType"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 读取配置文件\n",
-    "wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')\n",
-    "wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 注册 dataset1\n",
-    "dataset1 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),\n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(),\n",
-    "                          dataset_owner='flyhuang1',\n",
-    "                          agency=wedpr_config.user_config.agency_name,\n",
-    "                          dataset_id = 'd-9606704699156485',\n",
-    "                          dataset_path=\"/user/ppc/milestone2/sgd/flyhuang1/d-9606704699156485\",\n",
-    "                          is_label_holder=True)\n",
-    "print(dataset1.storage_client.storage_client.endpoint, dataset1.storage_workspace, dataset1.agency)\n",
-    "dataset1.load_values(header=0)\n",
-    "print(dataset1.dataset_path)\n",
-    "print(dataset1.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 注册 dataset2\n",
-    "dataset2 = DatasetToolkit(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), \n",
-    "                          storage_workspace=wedpr_config.user_config.get_workspace_path(), \n",
-    "                          dataset_owner='flyhuang',\n",
-    "                          dataset_id = 'd-9606695119693829',\n",
-    "                          dataset_path=\"/user/ppc/milestone2/webank/flyhuang/d-9606695119693829\", \n",
-    "                          agency=\"WeBank\")\n",
-    "print(dataset2.storage_client.storage_client.endpoint, dataset2.storage_workspace, dataset2.agency)\n",
-    "dataset2.load_values(header=0)\n",
-    "print(dataset2.dataset_path)\n",
-    "print(dataset2.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 构建 dataset context\n",
-    "dataset = DataContext(dataset1, dataset2)\n",
-    "print(dataset.datasets)\n",
-    "\n",
-    "# init the job context\n",
-    "project_id = \"9606702107011078\"\n",
-    "\n",
-    "# 构造xgb任务配置\n",
-    "model_setting = {'use_psi': 0, 'fillna': 0, 'na_select': 1, 'filloutlier': 0, 'normalized': 0, 'standardized': 0, 'categorical': '', 'psi_select_col': '', 'psi_select_base': '', 'psi_select_thresh': 0.3, 'psi_select_bins': 4, 'corr_select': 0, 'use_iv': 0, 'group_num': 4, 'iv_thresh': 0.1, 'use_goss': 0, 'test_dataset_percentage': 0.3, 'learning_rate': 0.1, 'num_trees': 6, 'max_depth': 3, 'max_bin': 4, 'silent': 0, 'subsample': 1, 'colsample_bytree': 1, 'colsample_bylevel': 1, 'reg_alpha': 0, 'reg_lambda': 1, 'gamma': 0, 'min_child_weight': 0, 'min_child_samples': 10, 'seed': 2024, 'early_stopping_rounds': 0, 'eval_metric': 'auc', 'verbose_eval': 1, 'eval_set_column': '', 'train_set_value': '', 'eval_set_value': '', 'train_features': ''}\n",
-    "\n",
-    "xgb_job_context = wedpr_ml_toolkit.build_job_context(\n",
-    "    JobType.XGB_TRAINING, project_id, dataset, model_setting, \"id\")\n",
-    "print(xgb_job_context.participant_id_list, xgb_job_context.result_receiver_id_list)\n",
-    "print(xgb_job_context.project_id)\n",
-    "\n",
-    "xgb_job_param = xgb_job_context.build()\n",
-    "print(xgb_job_param.taskParties)\n",
-    "print(xgb_job_param.datasetList)\n",
-    "print(xgb_job_param.job)\n",
-    "# import json\n",
-    "# print(json.dumps(xgb_job_param.__dict__))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 执行xgb任务\n",
-    "# xgb_job_id = '9707983191943174'  # 测试时跳过创建新任务过程\n",
-    "xgb_job_id = xgb_job_context.submit()\n",
-    "print(xgb_job_id)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 获取xgb任务结果\n",
-    "# xgb_job_id = '9707983191943174'  # 测试时跳过创建新任务过程\n",
-    "print(xgb_job_id)\n",
-    "xgb_result = xgb_job_context.parse_result(xgb_job_id, True)\n",
-    "xgb_result.train_result.load_values(header = 0)\n",
-    "xgb_result.test_result.load_values(header = 0)\n",
-    "print(xgb_result.train_result.values.head())\n",
-    "print(xgb_result.test_result.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 明文处理预测结果\n",
-    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "data = xgb_result.test_result.values\n",
-    "\n",
-    "# 提取真实标签和预测概率\n",
-    "y_true = data['class_label']\n",
-    "y_pred_proba = data['class_pred']\n",
-    "y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5\n",
-    "\n",
-    "# 计算评估指标\n",
-    "accuracy = accuracy_score(y_true, y_pred)\n",
-    "precision = precision_score(y_true, y_pred)\n",
-    "recall = recall_score(y_true, y_pred)\n",
-    "f1 = f1_score(y_true, y_pred)\n",
-    "auc = roc_auc_score(y_true, y_pred_proba)\n",
-    "\n",
-    "print(f\"Accuracy: {accuracy:.2f}\")\n",
-    "print(f\"Precision: {precision:.2f}\")\n",
-    "print(f\"Recall: {recall:.2f}\")\n",
-    "print(f\"F1 Score: {f1:.2f}\")\n",
-    "print(f\"AUC: {auc:.2f}\")\n",
-    "\n",
-    "# ROC 曲线\n",
-    "fpr, tpr, _ = roc_curve(y_true, y_pred_proba)\n",
-    "plt.figure(figsize=(12, 5))\n",
-    "\n",
-    "# ROC 曲线\n",
-    "plt.subplot(1, 2, 1)\n",
-    "plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')\n",
-    "plt.plot([0, 1], [0, 1], 'k--')\n",
-    "plt.xlabel('False Positive Rate')\n",
-    "plt.ylabel('True Positive Rate')\n",
-    "plt.title('ROC Curve')\n",
-    "plt.legend()\n",
-    "\n",
-    "# 精确率-召回率曲线\n",
-    "precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)\n",
-    "plt.subplot(1, 2, 2)\n",
-    "plt.plot(recall_vals, precision_vals)\n",
-    "plt.xlabel('Recall')\n",
-    "plt.ylabel('Precision')\n",
-    "plt.title('Precision-Recall Curve')\n",
-    "\n",
-    "plt.tight_layout()\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 构造xgb预测任务配置\n",
-    "model_setting = {'use_psi': 0, 'use_iv': 0}\n",
-    "\n",
-    "xgb_job_context = wedpr_ml_toolkit.build_job_context(\n",
-    "    JobType.XGB_PREDICTING, project_id, dataset, model_setting, \"id\", xgb_result.model)\n",
-    "print(xgb_job_context.participant_id_list, xgb_job_context.result_receiver_id_list)\n",
-    "print(xgb_job_context.project_id)\n",
-    "\n",
-    "xgb_job_param = xgb_job_context.build()\n",
-    "print(xgb_job_param.taskParties)\n",
-    "print(xgb_job_param.datasetList)\n",
-    "# import json\n",
-    "# print(json.dumps(xgb_job_param.__dict__))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 执行xgb预测任务\n",
-    "# xgb_job_id = '9708824062994438'  # 测试时跳过创建新任务过程\n",
-    "xgb_job_id = xgb_job_context.submit()\n",
-    "print(xgb_job_id)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 获取xgb预测任务结果\n",
-    "# xgb_job_id = '9708824062994438'  # 测试时跳过创建新任务过程\n",
-    "print(xgb_job_id)\n",
-    "xgb_result = xgb_job_context.parse_result(xgb_job_id, True)\n",
-    "xgb_result.test_result.load_values(header = 0)\n",
-    "print(xgb_result.test_result.values.head())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 明文处理预测结果\n",
-    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "data = xgb_result.test_result.values\n",
-    "\n",
-    "# 提取真实标签和预测概率\n",
-    "y_true = data['class_label']\n",
-    "y_pred_proba = data['class_pred']\n",
-    "y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5\n",
-    "\n",
-    "# 计算评估指标\n",
-    "accuracy = accuracy_score(y_true, y_pred)\n",
-    "precision = precision_score(y_true, y_pred)\n",
-    "recall = recall_score(y_true, y_pred)\n",
-    "f1 = f1_score(y_true, y_pred)\n",
-    "auc = roc_auc_score(y_true, y_pred_proba)\n",
-    "\n",
-    "print(f\"Accuracy: {accuracy:.2f}\")\n",
-    "print(f\"Precision: {precision:.2f}\")\n",
-    "print(f\"Recall: {recall:.2f}\")\n",
-    "print(f\"F1 Score: {f1:.2f}\")\n",
-    "print(f\"AUC: {auc:.2f}\")\n",
-    "\n",
-    "# ROC 曲线\n",
-    "fpr, tpr, _ = roc_curve(y_true, y_pred_proba)\n",
-    "plt.figure(figsize=(12, 5))\n",
-    "\n",
-    "# ROC 曲线\n",
-    "plt.subplot(1, 2, 1)\n",
-    "plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')\n",
-    "plt.plot([0, 1], [0, 1], 'k--')\n",
-    "plt.xlabel('False Positive Rate')\n",
-    "plt.ylabel('True Positive Rate')\n",
-    "plt.title('ROC Curve')\n",
-    "plt.legend()\n",
-    "\n",
-    "# 精确率-召回率曲线\n",
-    "precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)\n",
-    "plt.subplot(1, 2, 2)\n",
-    "plt.plot(recall_vals, precision_vals)\n",
-    "plt.xlabel('Recall')\n",
-    "plt.ylabel('Precision')\n",
-    "plt.title('Precision-Recall Curve')\n",
-    "\n",
-    "plt.tight_layout()\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py
index 03b9ef9b..9eda6a3e 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/base_object.py
@@ -26,3 +26,14 @@ def execute_with_retry(self, request_func, retry_times, retry_wait_seconds, *arg
                     time.sleep(retry_wait_seconds)
                 else:
                     raise e
+
+
+class WeDPRResponse(BaseObject):
+    def __init__(self, **params: Any):
+        self.code = None
+        self.msg = None
+        self.data = None
+        self.set_params(**params)
+
+    def success(self):
+        return self.code is not None and self.code == 0
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
index 2f3ded88..b9e227d2 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
@@ -7,6 +7,7 @@ class Constant:
     DEFAULT_SUBMIT_JOB_URI = f'{WEDPR_API_PREFIX}project/submitJob'
     DEFAULT_QUERY_JOB_STATUS_URL = f'{WEDPR_API_PREFIX}project/queryJobByCondition'
     DEFAULT_QUERY_JOB_DETAIL_URL = f'{WEDPR_API_PREFIX}scheduler/queryJobDetail'
+    DEFAULT_QUERY_DATASET_URL = f'{WEDPR_API_PREFIX}dataset/queryDataset'
     PSI_RESULT_FILE = "psi_result.csv"
 
     FEATURE_BIN_FILE = "feature_bin.json"
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
index 84556a46..0468971e 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
@@ -34,3 +34,19 @@ def delete_file(path):
         shutil.rmtree(path)
     else:
         raise (Exception(' path not exisited ! path => %s', path))
+
+
+def get_config_value(key, default_value, config_value, required):
+    if required and config_value is None:
+        raise Exception(f"Invalid config for '{key}' for not set!")
+    if config_value is None:
+        return default_value
+    value = config_value
+    if type(config_value) is dict:
+        if key in config_value:
+            value = config_value[key]
+        else:
+            value = default_value
+    if value is None:
+        return default_value
+    return value
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
index 7edd83b9..10b77ed2 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
@@ -31,6 +31,11 @@ def __init__(self, polling_interval_s: int = 5, max_retries: int = 2, retry_dela
         self.query_job_detail_uri = query_job_detail_uri
 
 
+class DatasetConfig(BaseObject):
+    def __init__(self, query_dataset_uri=Constant.DEFAULT_QUERY_DATASET_URL):
+        self.query_dataset_uri = query_dataset_uri
+
+
 class StorageConfig(BaseObject):
     def __init__(self, storage_endpoint: str = None):
         self.storage_endpoint = storage_endpoint
@@ -70,6 +75,7 @@ def __init__(self, config_dict):
         self.http_config.set_params(**config_dict)
         self.agency_config = AgencyConfig()
         self.agency_config.set_params(**config_dict)
+        self.dataset_config = DatasetConfig()
 
 
 class WeDPRMlConfigBuilder:
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py
deleted file mode 100644
index aa108ac5..00000000
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_model_setting.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# -*- coding: utf-8 -*-
-
-class PreprocessingModelSetting:
-    def __init__(self):
-        self.use_psi = False
-        self.fillna = False
-        self.na_select = 1.0
-        self.filloutlier = False
-        self.normalized = False
-        self.standardized = False
-        self.categorical = ''
-        self.psi_select_col = ''
-        self.psi_select_base = ''
-        self.psi_select_base = 0.3
-        self.psi_select_bins = 4
-        self.corr_select = 0
-        self.use_goss = False
-
-
-class FeatureEngineeringEngineModelSetting:
-    def __init__(self):
-        self.use_iv = False
-        self.group_num = 4
-        self.iv_thresh = 0.1
-
-
-class CommmonSecureModelSetting:
-    def __init__(self):
-        self.learning_rate = 0.1
-        self.eval_set_column = ''
-        self.train_set_value = ''
-        self.eval_set_value = ''
-        self.verbose_eval = 1
-        self.silent = False
-        self.train_features = ''
-        self.random_state = None
-        self.n_jobs = 0
-
-
-class SecureLGBMModelSetting(CommmonSecureModelSetting):
-    def __init__(self):
-        super().__init__()
-        self.test_size = 0.3
-        self.num_trees = 6
-        self.max_depth = 3
-        self.max_bin = 4
-        self.subsample = 1.0
-        self.colsample_bytree = 1
-        self.colsample_bylevel = 1
-        self.reg_alpha = 0
-        self.reg_lambda = 1.0
-        self.gamma = 0.0
-        self.min_child_weight = 0.0
-        self.min_child_samples = 10
-        self.seed = 2024
-        self.early_stopping_rounds = 5
-        self.eval_metric = "auc"
-        self.threads = 8
-        self.one_hot = 0
-
-
-class SecureLRModelSetting(CommmonSecureModelSetting):
-    def __init__(self):
-        super().__init__()
-        self.feature_rate = 1.0
-        self.batch_size = 16
-        self.epochs = 3
-
-
-class ModelSetting(PreprocessingModelSetting, FeatureEngineeringEngineModelSetting, SecureLGBMModelSetting, SecureLRModelSetting):
-    def __init__(self):
-        # init PreprocessingSetting
-        super().__init__()
-        # init FeatureEngineeringEngineSetting
-        super(FeatureEngineeringEngineModelSetting, self).__init__(model_dict)
-        # init SecureLGBMSetting
-        super(SecureLGBMModelSetting, self).__init__(model_dict)
-        # init SecureLRSetting
-        super(SecureLRModelSetting, self).__init__(model_dict)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
index 36bf5d52..3e4ee46b 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
@@ -1,6 +1,7 @@
 import os
 
 from wedpr_ml_toolkit.common import utils
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 
 
 class DataContext:
@@ -10,15 +11,16 @@ def __init__(self, *datasets):
 
         self._check_datasets()
 
-    def _save_dataset(self, dataset):
-        if dataset.dataset_path is None:
+    def _save_dataset(self, dataset: DatasetContext):
+        file_path = dataset.dataset_meta.file_path
+        if file_path is None:
             dataset.dataset_id = utils.make_id(
                 utils.IdPrefixEnum.DATASET.value)
-            dataset.dataset_path = os.path.join(
+            file_path = os.path.join(
                 dataset.storage_workspace, dataset.dataset_id)
             if dataset.storage_client is not None:
                 dataset.storage_client.upload(
-                    dataset.values, dataset.dataset_path)
+                    dataset.values, file_path)
 
     def _check_datasets(self):
         for dataset in self.datasets:
@@ -27,24 +29,30 @@ def _check_datasets(self):
     def to_psi_format(self, merge_filed, result_receiver_id_list):
         dataset_psi = []
         for dataset in self.datasets:
-            if dataset.agency in result_receiver_id_list:
+            if dataset.dataset_meta.ownerAgencyName in result_receiver_id_list:
                 result_receiver = True
             else:
                 result_receiver = False
-            dataset_psi_info = {"idFields": [merge_filed],
-                                "dataset": {"owner": dataset.dataset_owner,
-                                            "ownerAgency": dataset.agency,
-                                            "path": dataset.dataset_path,
-                                            "storageTypeStr": "HDFS",
-                                            "datasetID": dataset.dataset_id},
-                                "receiveResult": result_receiver}
+            dataset_psi_info = self.__generate_dataset_info__(
+                merge_filed, result_receiver, None, dataset)
             dataset_psi.append(dataset_psi_info)
         return dataset_psi
 
+    def __generate_dataset_info__(self, id_field: str, receive_result: bool, label_provider: bool, dataset: DatasetContext):
+        return {"idFields": [id_field],
+                "dataset": {"owner": dataset.dataset_meta.ownerUserName,
+                            "ownerAgency": dataset.dataset_meta.ownerAgencyName,
+                            "path": dataset.dataset_meta.file_path,
+                            "storageTypeStr": "HDFS",
+                            "datasetID": dataset.dataset_id},
+                "receiveResult": receive_result,
+                "labelProvider": label_provider
+                }
+
     def to_model_formort(self, merge_filed, result_receiver_id_list):
         dataset_model = []
         for dataset in self.datasets:
-            if dataset.agency in result_receiver_id_list:
+            if dataset.dataset_meta.ownerAgencyName in result_receiver_id_list:
                 result_receiver = True
             else:
                 result_receiver = False
@@ -52,13 +60,7 @@ def to_model_formort(self, merge_filed, result_receiver_id_list):
                 label_provider = True
             else:
                 label_provider = False
-            dataset_psi_info = {"idFields": [merge_filed],
-                                "dataset": {"owner": dataset.dataset_owner,
-                                            "ownerAgency": dataset.agency,
-                                            "path": dataset.dataset_path,
-                                            "storageTypeStr": "HDFS",
-                                            "datasetID": dataset.dataset_id},
-                                "labelProvider": label_provider,
-                                "receiveResult": result_receiver}
+            dataset_psi_info = self.__generate_dataset_info__(
+                merge_filed, result_receiver, label_provider, dataset)
             dataset_model.append(dataset_psi_info)
         return dataset_model
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
new file mode 100644
index 00000000..14243fa1
--- /dev/null
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
@@ -0,0 +1,55 @@
+import os
+import pandas as pd
+from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import WeDPRDatasetClient
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+
+
+class DatasetContext:
+
+    def __init__(self,
+                 storage_entrypoint: StorageEntryPoint,
+                 storage_workspace=None,
+                 dataset_client: WeDPRDatasetClient = None,
+                 dataset_meta=None,
+                 dataset_id=None,
+                 is_label_holder=False):
+        self.storage_client = storage_entrypoint
+        self.dataset_client = dataset_client
+        self.dataset_id = dataset_id
+        params = {}
+        self.dataset_meta = dataset_meta
+        # fetch the dataset information
+        if self.dataset_meta is None and self.dataset_id is not None and self.dataset_client is not None:
+            self.dataset_meta = self.dataset_client.query_dataset(
+                self.dataset_id)
+        self.is_label_holder = is_label_holder
+        # the storage workspace
+        self.storage_workspace = storage_workspace
+
+    def load_values(self, header=None):
+        # 加载hdfs的数据集
+        if self.storage_client is not None:
+            values = self.storage_client.download(
+                self.dataset_meta.file_path, header=header)
+            if values is None:
+                return values, None, None
+            return values, values.columns, values.shape
+
+    def save_values(self, values: pd.DataFrame = None, path=None):
+        target_path = self.dataset_meta.file_path
+        # 保存数据到hdfs目录
+        if path is not None:
+            target_path = path
+        # add the storage_workspace
+        if self.storage_workspace is not None and \
+                not target_path.startswith(self.storage_workspace):
+            target_path = os.path.join(
+                self.storage_workspace, target_path)
+        if self.storage_client is not None:
+            self.storage_client.upload(values, target_path)
+
+    def update_path(self, path: str = None):
+        # 将数据集存入hdfs相同路径，替换旧数据集
+        if path is not None:
+            self.dataset_meta.file_path = path
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
index 7bcfb109..81321ca5 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
 
-from wedpr_ml_toolkit.toolkit.dataset_toolkit import DatasetToolkit
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobParam
@@ -10,6 +10,8 @@
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobType, ModelType
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import ModelResult
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+from wedpr_ml_toolkit.context.model_setting import ModelSetting
 
 
 class JobContext:
@@ -46,10 +48,10 @@ def __init_participant__(self):
         participant_id_list = []
         dataset_id_list = []
         for dataset in self.dataset.datasets:
-            participant_id_list.append(dataset.agency)
+            participant_id_list.append(dataset.dataset_meta.ownerAgencyName)
             dataset_id_list.append(dataset.dataset_id)
-            self.task_parties.append({'userName': dataset.dataset_owner,
-                                      'agency': dataset.agency})
+            self.task_parties.append({'userName': dataset.dataset_meta.ownerUserName,
+                                      'agency': dataset.dataset_meta.ownerAgencyName})
         self.participant_id_list = participant_id_list
         self.dataset_id_list = dataset_id_list
 
@@ -58,7 +60,7 @@ def __init_label_information__(self):
         label_columns = None
         for dataset in self.dataset.datasets:
             if dataset.is_label_holder:
-                label_holder_agency = dataset.agency
+                label_holder_agency = dataset.dataset_meta.ownerAgencyName
                 label_columns = 'y'
         self.label_holder_agency = label_holder_agency
         self.label_columns = label_columns
@@ -92,12 +94,14 @@ def fetch_job_result(self, job_id, block_until_success):
         # ...
 
         # query_job_detail
-        result_detail = self.remote_job_client.query_job_detail(job_id, block_until_success)
+        result_detail = self.remote_job_client.query_job_detail(
+            job_id, block_until_success)
         return result_detail
 
 
 class PSIJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.merge_field = merge_field
 
@@ -117,16 +121,19 @@ def build(self) -> JobParam:
     def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
 
-        psi_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                storage_workspace=None, 
-                                dataset_owner=self.storage_entry_point.user_config.user,
-                                dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
+        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user, agency=self.create_agency,
+                                   file_path=result_detail.resultFileInfo['path'])
+        psi_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                    storage_workspace=None,
+                                    dataset_meta=dataset_meta)
 
         return psi_result
 
 
 class PreprocessingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting: ModelSetting,
+                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -138,7 +145,7 @@ def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
@@ -146,16 +153,13 @@ def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
 
         pre_result = result_detail
-        # pre_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-        #                         storage_workspace=None, 
-        #                         dataset_owner=self.storage_entry_point.user_config.user,
-        #                         dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
-
         return pre_result
 
 
 class FeatureEngineeringJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting: ModelSetting,
+                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -167,7 +171,7 @@ def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
@@ -175,16 +179,13 @@ def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
 
         fe_result = result_detail
-        # fe_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-        #                         storage_workspace=None, 
-        #                         dataset_owner=self.storage_entry_point.user_config.user,
-        #                         dataset_path=result_detail.resultFileInfo['path'], agency=self.create_agency)
-
         return fe_result
 
 
 class SecureLGBMTrainingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting: ModelSetting,
+                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -198,28 +199,33 @@ def build(self) -> JobParam:
         # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
         #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
     def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
-        # result_detail.modelResultDetail['ModelResult']
-        train_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
-        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
-
-        xgb_result = ModelResult(job_id, train_result, test_result, result_detail.model, ModelType.XGB_MODEL_SETTING.name)
+        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                   file_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
+        train_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                      storage_workspace=None,
+                                      dataset_meta=dataset_meta)
+
+        test_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                        file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                     storage_workspace=None,
+                                     dataset_meta=test_dataset_meta)
+
+        xgb_result = ModelResult(job_id, train_result, test_result,
+                                 result_detail.model, ModelType.XGB_MODEL_SETTING.name)
         return xgb_result
 
 
 class SecureLGBMPredictJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, predict_algorithm, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting: ModelSetting, predict_algorithm,
+                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -234,23 +240,27 @@ def build(self) -> JobParam:
         # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
         #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting, 'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict(),
+             'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
     def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
-        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                   file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'],
+                                   agency=self.create_agency)
+        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                     dataset_meta=dataset_meta)
 
         xgb_result = ModelResult(job_id, test_result=test_result)
         return xgb_result
 
 
 class SecureLRTrainingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting:  ModelSetting,
+                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -262,28 +272,32 @@ def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
     def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
-        # result_detail.modelResultDetail['ModelResult']
-        train_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
-        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
-
-        lr_result = ModelResult(job_id, train_result, test_result, result_detail.model, ModelType.LR_MODEL_SETTING.name)
+        train_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                         file_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
+        train_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                      dataset_meta=train_dataset_meta)
+
+        test_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                        file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                     storage_workspace=None,
+                                     dataset_meta=test_dataset_meta)
+
+        lr_result = ModelResult(job_id, train_result, test_result,
+                                result_detail.model, ModelType.LR_MODEL_SETTING.name)
         return lr_result
 
 
 class SecureLRPredictJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, model_setting, predict_algorithm, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+                 project_id: str, model_setting: ModelSetting, predict_algorithm, dataset: DataContext = None,
+                 my_agency=None, merge_field: str = 'id'):
         super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
         self.model_setting = model_setting
         self.merge_field = merge_field
@@ -296,16 +310,17 @@ def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
         job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting, 'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
+            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict(),
+             'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
     def parse_result(self, job_id, block_until_success):
         result_detail = self.fetch_job_result(job_id, block_until_success)
-        test_result = DatasetToolkit(storage_entrypoint=self.storage_entry_point, 
-                                    storage_workspace=None, 
-                                    dataset_owner=self.storage_entry_point.user_config.user,
-                                    dataset_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
+                                   file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
+        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
+                                     dataset_meta=dataset_meta)
 
         lr_result = ModelResult(job_id, test_result=test_result)
         return lr_result
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/model_setting.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/model_setting.py
new file mode 100644
index 00000000..dec57e21
--- /dev/null
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/model_setting.py
@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+from wedpr_ml_toolkit.common.utils import utils
+from wedpr_ml_toolkit.common.utils.base_object import BaseObject
+
+
+class PreprocessingSetting(BaseObject):
+    def __init__(self, model_dict=None):
+        self.use_psi = utils.get_config_value(
+            "use_psi", False, model_dict, False)
+        self.fillna = utils.get_config_value(
+            "fillna", False, model_dict, False)
+        self.na_select = float(utils.get_config_value(
+            "na_select", 1.0, model_dict, False))
+        self.filloutlier = utils.get_config_value(
+            "filloutlier", False, model_dict, False)
+        self.normalized = utils.get_config_value(
+            "normalized", False, model_dict, False)
+        self.standardized = utils.get_config_value(
+            "standardized", False, model_dict, False)
+        self.categorical = utils.get_config_value(
+            "categorical", '', model_dict, False)
+        self.psi_select_col = utils.get_config_value(
+            "psi_select_col", '', model_dict, False)
+        self.psi_select_base = utils.get_config_value(
+            "psi_select_base", '', model_dict, False)
+        self.psi_select_base = float(utils.get_config_value(
+            "psi_select_thresh", 0.3, model_dict, False))
+        self.psi_select_bins = int(utils.get_config_value(
+            "psi_select_bins", 4, model_dict, False))
+        self.corr_select = float(utils.get_config_value(
+            "corr_select", 0, model_dict, False))
+        self.use_goss = utils.get_config_value(
+            "use_goss", False, model_dict, False)
+
+
+class FeatureEngineeringEngineSetting(BaseObject):
+    def __init__(self, model_dict=None):
+        self.use_iv = utils.get_config_value(
+            "use_iv", False, model_dict, False)
+        self.group_num = int(utils.get_config_value(
+            "group_num", 4, model_dict, False))
+        self.iv_thresh = float(utils.get_config_value(
+            "iv_thresh", 0.1, model_dict, False))
+
+
+class CommonModelSetting(BaseObject):
+    def __init__(self, model_dict=None):
+        self.learning_rate = float(utils.get_config_value(
+            "learning_rate", 0.1, model_dict, False))
+
+        self.eval_set_column = utils.get_config_value(
+            "eval_set_column", "", model_dict, False)
+        self.train_set_value = utils.get_config_value(
+            "train_set_value", "", model_dict, False)
+        self.eval_set_value = utils.get_config_value(
+            "eval_set_value", "", model_dict, False)
+        self.verbose_eval = int(utils.get_config_value(
+            "verbose_eval", 1, model_dict, False))
+        self.silent = utils.get_config_value(
+            "silent", False, model_dict, False)
+        self.train_features = utils.get_config_value(
+            "train_features", "", model_dict, False)
+        random_state_str = utils.get_config_value(
+            "random_state", "", model_dict, False)
+        if len(random_state_str) > 0:
+            self.random_state = int(random_state_str)
+        self.n_jobs = int(utils.get_config_value(
+            "n_jobs", 0, model_dict, False))
+
+
+class SecureLGBMSetting(CommonModelSetting, BaseObject):
+    def __init__(self, model_dict=None):
+        super().__init__(model_dict)
+        self.test_size = float(utils.get_config_value(
+            "test_dataset_percentage", 0.3, model_dict, False))
+        self.num_trees = int(utils.get_config_value(
+            "num_trees", 6, model_dict, False))
+        self.max_depth = int(utils.get_config_value(
+            "max_depth", 3, model_dict, False))
+        self.max_bin = int(utils.get_config_value(
+            "max_bin", 4, model_dict, False))
+
+        self.subsample = float(utils.get_config_value(
+            "subsample", 1, model_dict, False))
+        self.colsample_bytree = float(utils.get_config_value(
+            "colsample_bytree", 1, model_dict, False))
+        self.colsample_bylevel = float(utils.get_config_value(
+            "colsample_bylevel", 1, model_dict, False))
+        self.reg_alpha = float(utils.get_config_value(
+            "reg_alpha", 0, model_dict, False))
+        self.reg_lambda = float(utils.get_config_value(
+            "reg_lambda", 1, model_dict, False))
+        self.gamma = float(utils.get_config_value(
+            "gamma", 0, model_dict, False))
+        self.min_child_weight = float(utils.get_config_value(
+            "min_child_weight", 0.0, model_dict, False))
+        self.min_child_samples = int(utils.get_config_value(
+            "min_child_samples", 10, model_dict, False))
+        self.seed = int(utils.get_config_value(
+            "seed", 2024, model_dict, False))
+        self.early_stopping_rounds = int(utils.get_config_value(
+            "early_stopping_rounds", 5, model_dict, False))
+        self.eval_metric = utils.get_config_value(
+            "eval_metric", "auc", model_dict, False)
+        self.threads = int(utils.get_config_value(
+            "threads", 8, model_dict, False))
+        self.one_hot = utils.get_config_value(
+            "one_hot", 0, model_dict, False)
+
+
+class SecureLRSetting(CommonModelSetting, BaseObject):
+    def __init__(self, model_dict=None):
+        super().__init__(model_dict)
+        self.feature_rate = float(utils.get_config_value(
+            "feature_rate", 1.0, model_dict, False))
+        self.batch_size = int(utils.get_config_value(
+            "batch_size", 16, model_dict, False))
+        self.epochs = int(utils.get_config_value(
+            "epochs", 3, model_dict, False))
+
+
+class ModelSetting(PreprocessingSetting, FeatureEngineeringEngineSetting, SecureLGBMSetting, SecureLRSetting, BaseObject):
+    def __init__(self, model_dict=None):
+        # init PreprocessingSetting
+        super().__init__(model_dict)
+        # init FeatureEngineeringEngineSetting
+        FeatureEngineeringEngineSetting.__init__(self, model_dict)
+        # init SecureLGBMSetting
+        SecureLGBMSetting.__init__(self, model_dict)
+        # init SecureLRSetting
+        SecureLRSetting.__init__(self, model_dict)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
deleted file mode 100644
index cf9f3327..00000000
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/toolkit/dataset_toolkit.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import os
-import pandas as pd
-from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
-
-
-class DatasetToolkit:
-
-    def __init__(self,
-                 storage_entrypoint: StorageEntryPoint,
-                 storage_workspace,
-                 dataset_id=None,
-                 dataset_path=None,
-                 dataset_owner=None,
-                 agency=None,
-                 values=None,
-                 is_label_holder=False):
-        self.dataset_id = dataset_id
-        self.dataset_path = dataset_path
-        self.dataset_owner = dataset_owner
-        self.agency = agency
-        self.values = values
-        self.is_label_holder = is_label_holder
-        self.columns = None
-        self.shape = None
-
-        self.storage_client = storage_entrypoint
-        self.storage_workspace = storage_workspace
-
-        if self.values is not None:
-            self.columns = self.values.columns
-            self.shape = self.values.shape
-
-    def load_values(self, header = None):
-        # 加载hdfs的数据集
-        if self.storage_client is not None:
-            self.values = self.storage_client.download(self.dataset_path, header=header)
-            self.columns = self.values.columns
-            self.shape = self.values.shape
-
-    def save_values(self, path=None):
-        # 保存数据到hdfs目录
-        if path is not None:
-            self.dataset_path = path
-        if self.storage_workspace is not None and \
-            not self.dataset_path.startswith(self.storage_workspace):
-            self.dataset_path = os.path.join(
-                self.storage_workspace, self.dataset_path)
-        if self.storage_client is not None:
-            self.storage_client.upload(self.values, self.dataset_path)
-
-    def update_values(self, values: pd.DataFrame = None, path: str = None):
-        # 将数据集存入hdfs相同路径，替换旧数据集
-        if values is not None:
-            self.values = values
-            self.columns = self.values.columns
-            self.shape = self.values.shape
-        if path is not None:
-            self.dataset_path = path
-        if values is not None and self.storage_client is not None:
-            self.storage_client.upload(self.values, self.dataset_path)
-
-    def update_path(self, path: str = None):
-        # 将数据集存入hdfs相同路径，替换旧数据集
-        if path is not None:
-            self.dataset_path = path
-        if self.values is not None:
-            self.values = None
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
new file mode 100644
index 00000000..1e69833b
--- /dev/null
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+from wedpr_ml_toolkit.common.utils.base_object import BaseObject
+from wedpr_ml_toolkit.common.utils.base_object import WeDPRResponse
+from wedpr_ml_toolkit.transport.wedpr_entrypoint import WeDPREntryPoint
+from wedpr_ml_toolkit.config.wedpr_ml_config import DatasetConfig
+from wedpr_ml_toolkit.config.wedpr_ml_config import HttpConfig
+from wedpr_ml_toolkit.config.wedpr_ml_config import AuthConfig
+from typing import Any
+import json
+
+
+class DatasetMeta(BaseObject):
+    def __init__(self, dataset_id: str = None,
+                 dataset_label: str = None,
+                 dataset_title: str = None,
+                 dataset_desc: str = None,
+                 dataset_fields: str = None,
+                 dataset_hash: str = None,
+                 dataset_size: int = -1,
+                 dataset_record_count: int = -1,
+                 dataset_column_count: int = -1,
+                 dataset_storage_type: str = None,
+                 dataset_storage_path: str = None,
+                 datasource_type: str = None,
+                 datasource_meta: str = None,
+                 agency: str = None,
+                 user: str = None,
+                 status: str = None,
+                 status_desc: str = None,
+                 file_path: str = None,
+                 **params: Any):
+        self.datasetId = dataset_id
+        self.datasetLabel = dataset_label
+        self.datasetTitle = dataset_title
+        self.datasetDesc = dataset_desc
+        self.datasetFields = dataset_fields
+        self.datasetHash = dataset_hash
+        self.datasetSize = dataset_size
+        self.datasetRecordCount = dataset_record_count
+        self.datasetColumnCount = dataset_column_count
+        self.datasetStorageType = dataset_storage_type
+        self.datasetStoragePath = dataset_storage_path
+        self.dataSourceType = datasource_type
+        self.dataSourceMeta = datasource_meta
+        self.ownerAgencyName = agency
+        self.ownerUserName = user
+        self.status = status
+        self.statusDesc = status_desc
+        self.file_path = file_path
+        self.set_params(**params)
+        if self.datasetStoragePath is not None:
+            dataset_storage_path_dict = json.loads(self.datasetStoragePath)
+            if "filePath" in dataset_storage_path_dict.keys():
+                self.file_path = dataset_storage_path_dict.get("filePath")
+
+    def __repr__(self):
+        return f"dataset: {self.datasetId}, datasetTitle: {self.datasetTitle}, datasetFields: {self.datasetFields}, " \
+               f"datasetSize: {self.datasetSize}, datasetRecordCount: {self.datasetRecordCount}," \
+               f"datasetColumnCount: {self.datasetColumnCount}, datasetStorageType: {self.datasetStorageType}" \
+               f"ownerAgencyName: {self.ownerAgencyName}"
+
+
+class WeDPRDatasetClient(WeDPREntryPoint, BaseObject):
+    def __init__(self, http_config: HttpConfig, auth_config: AuthConfig, dataset_config: DatasetConfig):
+        if auth_config is None:
+            raise Exception("Must define the auth config!")
+        if http_config is None:
+            raise Exception("Must define the http config")
+        super().__init__(auth_config.access_key_id, auth_config.access_key_secret,
+                         auth_config.get_remote_entrypoints_list(), http_config, auth_config.nonce_len)
+        self.auth_config = auth_config
+        self.http_config = http_config
+        self.dataset_config = dataset_config
+
+    def query_dataset(self, dataset_id) -> DatasetMeta:
+        params = {}
+        params.update({"datasetId": dataset_id})
+        response_dict = self.send_request(
+            False, self.dataset_config.query_dataset_uri, params, None, None)
+        wedpr_response = WeDPRResponse(**response_dict)
+        # query the dataset failed
+        if wedpr_response.success() is False:
+            raise Exception(
+                f"Query dataset information failed for {wedpr_response.msg}, dataset_id: {dataset_id}")
+        # query success, deserialize to DatasetMeta
+        return DatasetMeta(**wedpr_response.data)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
index c04523d2..1281313e 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
@@ -86,7 +86,7 @@ def __repr__(self):
 
 class ModelInfo(BaseObject):
     def __init__(self, model, model_type, **params: Any):
-        
+
         self.type = model_type
         # self.setting = json.loads(model)
         self.setting = model
@@ -102,7 +102,7 @@ def __init__(self, model, model_type, **params: Any):
 
 
 class ModelResult:
-    def __init__(self, job_id: str, train_result = None, test_result = None, model = None, model_type = None):
+    def __init__(self, job_id: str, train_result=None, test_result=None, model=None, model_type=None):
         self.job_id = job_id
         self.train_result = train_result
         self.test_result = test_result
@@ -159,15 +159,14 @@ def get_queried_job(self):
 
 class JobDetailResponse(BaseObject):
     def __init__(self, job: JobInfo = None, **params: Any):
-        self.job = job
-        self.job_object = None
+        self.job = None
+        self.job_object = job
         self.modelResultDetail = None
         self.resultFileInfo = None
         self.model = None
         self.set_params(**params)
-        # deserialize the job_object
-        self.job_object = JobInfo(**self.job)
-        # TODO: deserialize the result
+        if self.job_object is None and self.job is not None:
+            self.job_object = JobInfo(**self.job)
 
     def __repr__(self):
         return f"job: {self.job_object}, modelResultDetail: {self.modelResultDetail}, resultFileInfo: {self.resultFileInfo}, model: {self.model}"
@@ -205,7 +204,7 @@ def query_job_detail(self, job_id, block_until_finish) -> JobDetailResponse:
         # failed case
         if job_result == None or job_result.job_status == None or (not job_result.job_status.run_success()):
             return JobDetailResponse(job=job_result, params=None)
-        # success case
+        # success case, query the job detail
         params = {}
         params["jobID"] = job_id
         response_dict = self.execute_with_retry(self.send_request,
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
index ac999867..e091c31f 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
@@ -5,6 +5,7 @@
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
 from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import WeDPRDatasetClient
 from wedpr_ml_toolkit.context.job_context import JobType
 from wedpr_ml_toolkit.context.job_context import PSIJobContext
 from wedpr_ml_toolkit.context.job_context import PreprocessingJobContext
@@ -14,6 +15,7 @@
 from wedpr_ml_toolkit.context.job_context import SecureLRPredictJobContext
 from wedpr_ml_toolkit.context.job_context import SecureLRTrainingJobContext
 from wedpr_ml_toolkit.context.data_context import DataContext
+from wedpr_ml_toolkit.context.model_setting import ModelSetting
 
 
 class WeDPRMlToolkit:
@@ -21,12 +23,18 @@ def __init__(self, config: WeDPRMlConfig):
         self.config = config
         self.remote_job_client = WeDPRRemoteJobClient(
             self.config.http_config, self.config.auth_config, self.config.job_config)
+        self.dataset_client = WeDPRDatasetClient(http_config=self.config.http_config,
+                                                 auth_config=self.config.auth_config,
+                                                 dataset_config=self.config.dataset_config)
         self.storage_entry_point = StorageEntryPoint(self.config.user_config,
                                                      self.config.storage_config)
 
     def get_config(self) -> WeDPRMlConfig:
         return self.config
 
+    def get_dataset_client(self) -> WeDPRDatasetClient:
+        return self.dataset_client
+
     def get_remote_job_client(self) -> WeDPRRemoteJobClient:
         return self.remote_job_client
 
@@ -42,26 +50,26 @@ def query_job_status(self, job_id, block_until_finish=False) -> JobInfo:
     def query_job_detail(self, job_id, block_until_finish=False) -> JobDetailResponse:
         return self.remote_job_client.query_job_detail(job_id, block_until_finish)
 
-    def build_job_context(self, job_type: JobType, project_id: str, dataset: DataContext, model_setting=None,
+    def build_job_context(self, job_type: JobType, project_id: str, dataset: DataContext, model_setting: ModelSetting = None,
                           id_fields='id', predict_algorithm=None):
         if job_type == JobType.PSI:
-            return PSIJobContext(self.remote_job_client, self.storage_entry_point, project_id, dataset, 
+            return PSIJobContext(self.remote_job_client, self.storage_entry_point, project_id, dataset,
                                  self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.PREPROCESSING:
-            return PreprocessingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+            return PreprocessingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
                                            model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.FEATURE_ENGINEERING:
-            return FeatureEngineeringJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+            return FeatureEngineeringJobContext(self.remote_job_client, self.storage_entry_point, project_id,
                                                 model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.XGB_TRAINING:
-            return SecureLGBMTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+            return SecureLGBMTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
                                                 model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.XGB_PREDICTING:
-            return SecureLGBMPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
+            return SecureLGBMPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id,
                                                model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.LR_TRAINING:
-            return SecureLRTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
-                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+            return SecureLRTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
+                                              model_setting, dataset, self.config.agency_config.agency_name, id_fields)
         if job_type == JobType.LR_PREDICTING:
-            return SecureLRPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id, 
-                                               model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)
+            return SecureLRPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id,
+                                             model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)

From 118a0365ade9d9916973af701d3bd62e6a4b6ab4 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 26 Nov 2024 22:31:07 +0800
Subject: [PATCH 104/120] update model file paths (#103)

* update model file paths

* fix sensitive information

* optimize log
---
 .../ppc_common/deps_services/file_object.py   |  85 ----
 .../ppc_common/deps_services/hdfs_storage.py  |  18 +-
 .../deps_services/sharding_file_object.py     | 429 ------------------
 .../ppc_common/deps_services/storage_api.py   |   2 +-
 .../tests/sharding_file_object_test.py        |  79 ----
 .../ppc_ml/feature/feature_importance.py      |   4 +-
 python/ppc_model/common/base_context.py       | 152 ++++---
 python/ppc_model/common/context.py            |   4 +-
 python/ppc_model/common/model_result.py       |  43 +-
 python/ppc_model/datasets/dataset.py          |   5 +-
 .../feature_engineering_context.py            |   9 +-
 .../feature_engineering_engine.py             |  15 +-
 .../vertical/active_party.py                  |   7 +-
 .../vertical/passive_party.py                 |   3 +-
 python/ppc_model/log/log_retriever.py         |  31 +-
 python/ppc_model/metrics/evaluation.py        |  16 +-
 python/ppc_model/metrics/model_plot.py        |   3 +-
 .../model_result/task_result_handler.py       |  77 ++--
 .../network/http/model_controller.py          |  14 +-
 .../local_processing_party.py                 |  17 +-
 .../local_processing/preprocessing.py         |   4 +-
 .../preprocessing/processing_context.py       |   9 +-
 python/ppc_model/secure_lgbm/monitor/core.py  |   5 +-
 .../secure_lgbm/monitor/evaluation_monitor.py |  17 +-
 .../feature/feature_evaluation_info.py        |   4 +-
 .../secure_lgbm/vertical/active_party.py      |   4 +-
 .../ppc_model/secure_lgbm/vertical/booster.py |  23 +-
 .../ppc_model/secure_lr/vertical/booster.py   |  20 +-
 .../secure_model_base/secure_model_booster.py |  10 +-
 .../secure_model_base/secure_model_context.py |   9 +-
 python/ppc_model/task/task_manager.py         |  29 +-
 python/wedpr_ml_toolkit/test/UserGuide.md     |   8 +-
 .../wedpr_ml_toolkit/test/config.properties   |   4 +-
 33 files changed, 323 insertions(+), 836 deletions(-)
 delete mode 100644 python/ppc_common/deps_services/file_object.py
 delete mode 100644 python/ppc_common/deps_services/sharding_file_object.py
 delete mode 100644 python/ppc_common/deps_services/tests/sharding_file_object_test.py

diff --git a/python/ppc_common/deps_services/file_object.py b/python/ppc_common/deps_services/file_object.py
deleted file mode 100644
index 1e69fef0..00000000
--- a/python/ppc_common/deps_services/file_object.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# -*- coding: utf-8 -*-
-from enum import Enum
-from abc import ABC, abstractmethod
-
-
-class SplitMode(Enum):
-    NONE = 0  # not split
-    SIZE = 1  # split by size
-    LINES = 2  # split by lines
-
-
-class FileObject(ABC):
-    @abstractmethod
-    def split(self, split_mode, granularity):
-        """
-        split large file into many small files with given granularity
-        """
-        pass
-
-    @abstractmethod
-    def download(self, enforce_flush=False):
-        """
-        download the files
-        """
-        pass
-
-    @abstractmethod
-    def upload(self, split_mode, granularity):
-        """
-        upload the files
-        """
-        pass
-
-    @abstractmethod
-    def delete(self):
-        """
-        delete the files
-        """
-        pass
-
-    @abstractmethod
-    def existed(self) -> bool:
-        """
-        check the file object exist or not
-        """
-        pass
-
-    @abstractmethod
-    def rename(self, storage_path: str):
-        """
-        rename the file object
-        """
-        pass
-
-    @abstractmethod
-    def hit_local_cache(self):
-        """hit the local cache or not
-        """
-        pass
-
-    @abstractmethod
-    def get_local_path(self):
-        """get the local path
-        """
-        pass
-
-    @abstractmethod
-    def get_remote_path(self):
-        """get the remote path
-        """
-        pass
-
-    @abstractmethod
-    def get_data(self):
-        """get data
-        """
-        pass
-
-    @abstractmethod
-    def save_data(self, data, split_mode, granularity):
-        pass
-
-    @abstractmethod
-    def update_data(self, updated_data, split_mode, granularity):
-        pass
diff --git a/python/ppc_common/deps_services/hdfs_storage.py b/python/ppc_common/deps_services/hdfs_storage.py
index 472c9ea4..67f2d81b 100644
--- a/python/ppc_common/deps_services/hdfs_storage.py
+++ b/python/ppc_common/deps_services/hdfs_storage.py
@@ -24,8 +24,6 @@ def __init__(self, endpoint, hdfs_user, hdfs_home=None):
                 HdfsStorage.DEFAULT_HDFS_USER_PATH, self._user)
 
         self.client = InsecureClient(endpoint, user=self._user)
-        # print(self.client.list('/'))
-        # print(self.client.list('/user/root/'))
 
     def get_home_path(self):
         return self._hdfs_storage_path
@@ -46,10 +44,22 @@ def download_file(self, hdfs_path, local_file_path, enable_cache=False):
                              hdfs_path), local_file_path)
         return
 
-    def upload_file(self, local_file_path, hdfs_path):
+    def upload_file(self, local_file_path, hdfs_path, owner=None, group=None):
         self.make_file_path(hdfs_path)
-        self.client.upload(os.path.join(self._hdfs_storage_path, hdfs_path),
+        hdfs_abs_path = os.path.join(self._hdfs_storage_path, hdfs_path)
+        self.client.upload(hdfs_abs_path,
                            local_file_path, overwrite=True)
+        if owner is None and group is None:
+            return
+        group_info = group
+        if group is None:
+            group_info = self._user
+        owner_info = group_info
+        if owner is not None:
+            owner_info = owner
+        # set the permission information
+        self.client.set_owner(hdfs_path=hdfs_abs_path,
+                              owner=owner_info, group=group_info)
         return
 
     def make_file_path(self, hdfs_path):
diff --git a/python/ppc_common/deps_services/sharding_file_object.py b/python/ppc_common/deps_services/sharding_file_object.py
deleted file mode 100644
index d89df5bd..00000000
--- a/python/ppc_common/deps_services/sharding_file_object.py
+++ /dev/null
@@ -1,429 +0,0 @@
-# -*- coding: utf-8 -*-
-from ppc_common.deps_services.file_object import FileObject, SplitMode
-from ppc_common.db_models.file_object_meta import FileObjectMeta
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
-import shutil
-# from memory_profiler import profile
-import os
-import time
-import uuid
-import copy
-
-
-class FileMeta:
-    SUB_FILE_OBJECT_DIR = "object"
-
-    def __init__(self, local_path, remote_path, file_count, logger):
-        self.logger = logger
-        self._local_path = local_path
-        self._remote_path = remote_path
-        if self._local_path is None:
-            self._local_path = os.path.basename(
-                self.remote_path) + "." + str(uuid.uuid4())
-            logger.info(f"generate random local path: {self._local_path}")
-        self._local_file_dir = os.path.dirname(self._local_path)
-        self._local_file_name = os.path.basename(self._local_path)
-        if self._remote_path is None:
-            self._remote_path = local_path
-        if file_count is not None:
-            self._file_count = file_count
-
-    @property
-    def local_path(self):
-        return self._local_path
-
-    @local_path.setter
-    def local_path(self, local_path):
-        self._local_path = local_path
-
-    @property
-    def remote_path(self):
-        return self._remote_path
-
-    @remote_path.setter
-    def remote_path(self, remote_path):
-        self._remote_path = remote_path
-
-    @property
-    def file_count(self):
-        return self._file_count
-
-    @file_count.setter
-    def file_count(self, file_count):
-        self._file_count = file_count
-
-    def get_local_sub_file_name(self, file_index):
-        return os.path.join(self.get_local_sub_files_dir(), str(file_index))
-
-    def get_local_sub_files_dir(self):
-        return os.path.join(
-            self._local_file_dir, FileMeta.SUB_FILE_OBJECT_DIR, self._local_file_name)
-
-    def mk_local_sub_files_dir(self):
-        local_sub_files_dir = self.get_local_sub_files_dir()
-        if os.path.exists(local_sub_files_dir):
-            return
-        os.makedirs(local_sub_files_dir)
-
-    def remove_local_cache(self):
-        local_sub_files_dir = self.get_local_sub_files_dir()
-        if os.path.exists(local_sub_files_dir) is False:
-            return
-        shutil.rmtree(local_sub_files_dir)
-
-    def get_remote_sub_file_name(self, file_index):
-        return os.path.join(self._remote_path, str(file_index))
-
-
-class ShardingFileObject(FileObject):
-    def __init__(self, local_path, remote_path, remote_storage_client, sql_storage, logger):
-        self.logger = logger
-        self._remote_storage_client = remote_storage_client
-        self._sql_storage = sql_storage
-        self._file_meta = FileMeta(local_path, remote_path, None, logger)
-        self.logger.info(
-            f"create ShardingFileObject, storage type: {self._remote_storage_client.storage_type()},local path: {self._file_meta.local_path}, remote path: {self._file_meta.remote_path}")
-
-    @property
-    def file_meta(self):
-        return self._file_meta
-
-    def split(self, split_mode, granularity):
-        """
-        split large file many small files
-        """
-        if split_mode is SplitMode.NONE or granularity is None:
-            return None
-        local_file_size = os.stat(self._file_meta.local_path).st_size
-        if local_file_size < granularity:
-            self.logger.info(
-                f"upload small files directly without split: {self._file_meta.local_path}")
-            return None
-        if split_mode is SplitMode.SIZE:
-            file_list = self._split_by_size(granularity)
-        if split_mode is SplitMode.LINES:
-            file_list = self._split_by_lines(granularity)
-        self._file_meta.file_count = len(file_list)
-        return file_list
-
-    # @profile
-    def _split_by_size(self, granularity):
-        file_index = 0
-        start_t = time.time()
-        file_list = []
-        with open(self._file_meta.local_path, "rb") as fp:
-            while True:
-                start = time.time()
-                fp.seek(file_index * granularity)
-                data = fp.read(granularity)
-                if not data:
-                    break
-                self._file_meta.mk_local_sub_files_dir()
-                sub_file_path = self._file_meta.get_local_sub_file_name(
-                    file_index)
-                with open(sub_file_path, "wb") as wfp:
-                    wfp.write(data)
-                    file_list.append(sub_file_path)
-                self.logger.info(
-                    f"split file by size, file: {self._file_meta.local_path}, sub_file: {sub_file_path}, time cost: {time.time() - start} seconds")
-                file_index += 1
-        self.logger.info(
-            f"split file by size, file: {self._file_meta.local_path}, split granularity: {granularity}, sub file count: {file_index}, time cost: {time.time() - start_t} seconds")
-        return (file_list)
-
-    # @profile
-    def _split_by_lines(self, granularity):
-        file_index = 0
-        self._file_meta.mk_local_sub_files_dir()
-        start = time.time()
-        file_list = []
-        with open(self._file_meta.local_path, "rb") as fp:
-            while True:
-                start_t = time.time()
-                lines = fp.readlines(granularity)
-                if not lines:
-                    break
-                file_name = self._file_meta.get_local_sub_file_name(file_index)
-                with open(file_name, "wb") as wfp:
-                    wfp.writelines(lines)
-                    file_list.append(file_name)
-                self.logger.debug(
-                    f"split file by lines, file: {self._file_meta.local_path}, sub file path: {file_name}, timecost: {time.time() - start_t} seconds")
-                file_index += 1
-        self.logger.info(
-            f"split file by lines, file: {self._file_meta.local_path}, split granularity: {granularity}, sub file count: {file_index}, timecost: {time.time() - start} seconds")
-        return (file_list)
-
-    def _local_cache_miss(self):
-        for file_index in range(0, self._file_meta.file_count):
-            sub_file_path = self._file_meta.get_local_sub_file_name(file_index)
-            if not os.path.exists(sub_file_path):
-                return True
-        return False
-
-    def _check_uploaded_files(self):
-        if self._local_cache_miss():
-            error_msg = f"check upload file failed, {self._file_meta.local_path} => {self._file_meta.remote_path}"
-            raise PpcException(
-                PpcErrorCode.FILE_OBJECT_UPLOAD_CHECK_FAILED.get_code(), error_msg)
-
-    def upload(self, split_mode, granularity):
-        """split and upload the file
-        """
-        if self.split(split_mode, granularity) is not None:
-            self._upload_chunks()
-            self.logger.info(
-                f"Upload success, remove local file cache: {self._file_meta.get_local_sub_files_dir()}")
-            self._file_meta.remove_local_cache()
-            return
-        # upload directly
-        start = time.time()
-        self.logger.info(
-            f"Upload: {self._file_meta.local_path}=>{self._file_meta.remote_path}")
-        self._remote_storage_client.upload_file(
-            self._file_meta.local_path, self._file_meta.remote_path)
-        self.logger.info(
-            f"Upload success: {self._file_meta.local_path}=>{self._file_meta.remote_path}, timecost: {time.time() - start}s")
-
-    def _upload_chunks(self):
-        """
-        upload the files
-        """
-        start = time.time()
-        self._check_uploaded_files()
-        for file_index in range(0, self._file_meta.file_count):
-            start_t = time.time()
-            local_file_path = self._file_meta.get_local_sub_file_name(
-                file_index)
-            remote_file_path = self._file_meta.get_remote_sub_file_name(
-                file_index)
-            self.logger.info(f"upload: {local_file_path}=>{remote_file_path}")
-            self._remote_storage_client.upload_file(
-                local_file_path, remote_file_path)
-            self.logger.info(
-                f"upload: {local_file_path}=>{remote_file_path} success, timecost: {time.time() - start_t} seconds")
-        self.logger.info(
-            f"upload: {self._file_meta.local_path}=>{self._file_meta.remote_path} success, timecost: {time.time() - start} seconds, begin to store the meta information")
-        start = time.time()
-        record = FileObjectMeta(
-            file_path=self._file_meta.remote_path, file_count=self._file_meta.file_count)
-        self._sql_storage.merge(record)
-        self.logger.info(
-            f"store meta for {self._file_meta.remote_path} success, timecost: {time.time() - start} seconds")
-
-    def _fetch_file_meta(self):
-        file_meta_info = self._sql_storage.query(
-            FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path)
-        # the file not exists
-        if file_meta_info is None or file_meta_info.count() == 0:
-            return False
-        self.logger.info(
-            f"fetch file meta information: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path}, file count: {file_meta_info.first().file_count}")
-        self._file_meta.file_count = file_meta_info.first().file_count
-        return True
-
-    def download(self, enforce_flush=False):
-        """
-        download the files
-        """
-        if enforce_flush is not True and self.hit_local_cache():
-            return
-        ret = self._fetch_file_meta()
-        start = time.time()
-        # the remote file not exists
-        if ret is False:
-            # no sharding case
-            if self._remote_storage_client.file_existed(self._file_meta.remote_path):
-                self.logger.info(
-                    f"Download file: {self._file_meta.remote_path}=>{self._file_meta.local_path}")
-                self._remote_storage_client.download_file(
-                    self._file_meta.remote_path, self._file_meta.local_path)
-                self.logger.info(
-                    f"Download file success: {self._file_meta.remote_path}=>{self._file_meta.local_path}, timecost: {time.time() - start}")
-                return
-            error_msg = f"Download file from {self._remote_storage_client.storage_type()}: {self._file_meta.remote_path} failed for the file not exists!"
-            self.logger.error(error_msg)
-            raise PpcException(
-                PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg)
-        # remove the local file
-        if os.path.exists(self._file_meta.local_path):
-            self.logger.info(
-                f"Download: remove the existed local file {self._file_meta.local_path}")
-            os.remove(self._file_meta.local_path)
-        # download from the remote storage client
-        start = time.time()
-        # merge the file
-        offset = 0
-        try:
-            with open(self._file_meta.local_path, "wb") as fp:
-                for file_index in range(0, self._file_meta.file_count):
-                    start_t = time.time()
-                    fp.seek(offset)
-                    remote_file_path = self._file_meta.get_remote_sub_file_name(
-                        file_index)
-                    local_file_path = self._file_meta.get_local_sub_file_name(
-                        file_index)
-                    self._remote_storage_client.download_file(
-                        remote_file_path, local_file_path)
-                    with open(local_file_path, "rb") as f:
-                        fp.write(f.read())
-                        offset += os.stat(local_file_path).st_size
-                    self.logger.info(
-                        f"Download: {self._remote_storage_client.storage_type()}:{remote_file_path}=>{local_file_path}, timecost: {time.time() - start_t} seconds")
-            # remove the local cache
-            self._file_meta.remove_local_cache()
-            self.logger.info(
-                f"Download: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds")
-        except Exception as e:
-            self.logger.warn(
-                f"Download: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{self._file_meta.local_path} failed, {e}")
-            self._remove_local_file()
-            raise e
-
-    def _remove_local_file(self):
-        if not os.path.exists(self._file_meta.local_path):
-            return
-        os.remove(self._file_meta.local_path)
-
-    def delete(self):
-        """
-        delete the files
-        """
-        if self._fetch_file_meta() is False:
-            if self._remote_storage_client.file_existed(self._file_meta.remote_path):
-                self._remote_storage_client.delete_file(
-                    self._file_meta.remote_path)
-                self.logger.info(
-                    f"Delete file {self._file_meta.remote_path} success")
-                return
-            self.logger.info(
-                f"Delete nothing for file {self._file_meta.remote_path} not exists")
-            return
-        start = time.time()
-        for file_index in range(0, self._file_meta.file_count):
-            start_t = time.time()
-            remote_file_path = self._file_meta.get_remote_sub_file_name(
-                file_index)
-            self._remote_storage_client.delete_file(remote_file_path)
-            self.logger.info(
-                f"Delete: {self._remote_storage_client.storage_type()}:{remote_file_path}, timecost: {time.time() - start_t} seconds")
-        self._delete_remote_dir()
-        # delete the record
-        self._sql_storage.delete(
-            FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path)
-        self.logger.info(
-            f"Delete: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds")
-
-    def _delete_remote_dir(self):
-        if self._remote_storage_client.file_existed(self._file_meta.remote_path):
-            self._remote_storage_client.delete_file(
-                self._file_meta.remote_path)
-
-    def existed(self) -> bool:
-        """
-        check the file object exist or not
-        """
-        if self._fetch_file_meta() is True:
-            return True
-        return self._remote_storage_client.file_existed(self._file_meta.remote_path)
-
-    def hit_local_cache(self):
-        """hit the local cache or not
-        """
-        if not self.existed():
-            error_msg = f"The file object: local:{self._file_meta.local_path}, remote: {self._file_meta.remote_path} not exists!"
-            self.logger.info(f"find local cache failed: {error_msg}")
-            raise PpcException(
-                PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg)
-        return os.path.exists(self._file_meta.local_path)
-
-    def rename(self, storage_path: str):
-        """
-        rename the file object
-        """
-        ret = self._fetch_file_meta()
-        start = time.time()
-        # the remote file not exists
-        if ret is False:
-            # no sharding case
-            if self._remote_storage_client.file_existed(self._file_meta.remote_path):
-                self.logger.info(
-                    f"Rename file: {self._file_meta.remote_path}=>{storage_path}")
-                self._remote_storage_client.file_rename(
-                    self._file_meta.remote_path, storage_path)
-                self.logger.info(
-                    f"Rename file success: {self._file_meta.remote_path}=>{storage_path}, timecost: {time.time() - start} seconds")
-                return
-            error_msg = f"Rename file {self._remote_storage_client.storage_type()} => {self._file_meta.remote_path} failed for the file not exists!"
-            self.logger.error(error_msg)
-            raise PpcException(
-                PpcErrorCode.FILE_OBJECT_NOT_EXISTS.get_code(), error_msg)
-        # rename from the remote storage client
-        start = time.time()
-        new_file_meta = copy.deepcopy(self._file_meta)
-        new_file_meta.remote_path = storage_path
-        for file_index in range(0, self._file_meta.file_count):
-            start_t = time.time()
-            remote_file_path = self._file_meta.get_remote_sub_file_name(
-                file_index)
-            new_file_path = new_file_meta.get_remote_sub_file_name(
-                file_index)
-            self._remote_storage_client.file_rename(
-                remote_file_path, new_file_path)
-            self.logger.info(
-                f"Rename: {self._remote_storage_client.storage_type()}:{remote_file_path}=>{new_file_path}, timecost: {time.time() - start_t} seconds")
-        # delete the old record
-        self._delete_remote_dir()
-        self._sql_storage.delete(
-            FileObjectMeta, FileObjectMeta.file_path == self._file_meta.remote_path)
-        self._file_meta.remote_path = storage_path
-        # update the meta
-        self.logger.info(
-            f"Rename: {self._remote_storage_client.storage_type()}:{self._file_meta.remote_path}=>{storage_path} success, file count: {self._file_meta.file_count}, timecost: {time.time() - start} seconds")
-        start = time.time()
-        record = FileObjectMeta(file_path=storage_path,
-                                file_count=self._file_meta.file_count)
-        self._sql_storage.merge(record)
-        self.logger.info(
-            f"Rename: store meta for {self._file_meta.remote_path} => {storage_path} success, timecost: {time.time() - start} seconds")
-
-    def get_data(self):
-        self.download()
-        with open(self._file_meta.local_path, 'r') as file:
-            file_content = file.read()
-            file_bytes = file_content.encode('utf-8')
-        os.remove(self._file_meta.local_path)
-        return file_bytes
-
-    def save_data(self, data, split_mode, granularity):
-        with open(self._file_meta.local_path, 'wb') as file:
-            file.write(data)
-        self.upload(split_mode, granularity)
-        os.remove(self._file_meta.local_path)
-        return
-
-    def update_data(self, updated_data, split_mode, granularity):
-        start = time.time()
-        # try to remove the remote file
-        if self.existed():
-            self.logger.info(
-                f"UpdateData: remove existed remote file: {self._file_meta.remote_path}")
-            self.delete()
-        # update the data
-        self.logger.info(
-            f"UpdateData: update the remote file: {self._file_meta.remote_path}")
-        self.save_data(data=updated_data if type(updated_data) is bytes else bytes(updated_data, "utf-8"),
-                       split_mode=split_mode, granularity=granularity)
-        self.logger.info(
-            f"UpdateData success: update the remote file: {self._file_meta.remote_path}, timecost: {time.time() - start}")
-        return len(updated_data)
-
-    def get_local_path(self):
-        """get the local path
-        """
-        return self._file_meta.local_path
-
-    def get_remote_path(self):
-        """get the remote path
-        """
-        return self._file_meta.remote_path
diff --git a/python/ppc_common/deps_services/storage_api.py b/python/ppc_common/deps_services/storage_api.py
index ffc622de..1f623fb1 100644
--- a/python/ppc_common/deps_services/storage_api.py
+++ b/python/ppc_common/deps_services/storage_api.py
@@ -13,7 +13,7 @@ def download_file(self, storage_path: str, local_file_path: str, enable_cache=Fa
         pass
 
     @abstractmethod
-    def upload_file(self, local_file_path: str, storage_path: str):
+    def upload_file(self, local_file_path: str, storage_path: str, owner=None, group=None):
         pass
 
     @abstractmethod
diff --git a/python/ppc_common/deps_services/tests/sharding_file_object_test.py b/python/ppc_common/deps_services/tests/sharding_file_object_test.py
deleted file mode 100644
index 60094585..00000000
--- a/python/ppc_common/deps_services/tests/sharding_file_object_test.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# -*- coding: utf-8 -*-
-import unittest
-from ppc_common.deps_services.sharding_file_object import ShardingFileObject
-from ppc_common.deps_services.file_object import SplitMode
-from ppc_common.deps_services.mysql_storage import MySQLStorage
-from ppc_common.deps_services.sql_storage_api import SQLStorageConfig
-from ppc_common.deps_services.hdfs_storage import HdfsStorage
-import sys
-import logging
-import pandas as pd
-
-
-class ShardingFileObjectTestWrapper:
-    def __init__(self, ut_obj, local_path, remote_path):
-        self.ut_obj = ut_obj
-        self.engine_url = "mysql://root:12345678@127.0.0.1:3306/ppc?autocommit=true&charset=utf8mb4"
-        self.sql_storage_config = SQLStorageConfig(url=self.engine_url)
-        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-        self.logger = logging.getLogger(__name__)
-        self.sql_storage = MySQLStorage(self.sql_storage_config)
-        remote_storage_url = "http://127.0.0.1:9870"
-        remote_storage_client = HdfsStorage(remote_storage_url, "chenyujie")
-        self.sharding_file_object = ShardingFileObject(
-            local_path, remote_path, remote_storage_client, self.sql_storage, self.logger)
-
-    def upload(self, split_mode, granularity, expected_lines):
-        file_list = self.sharding_file_object.upload(split_mode, granularity)
-        if split_mode is SplitMode.LINES:
-            self._check_lines(file_list, expected_lines)
-
-    def _check_lines(self, file_list, expected_lines):
-        lines = 0
-        # check the lines
-        i = 0
-        columns_info = None
-        df = None
-        for file in file_list:
-            # can be parsed by read_csv
-            if i == 0:
-                df = pd.read_csv(file, header=0)
-                columns_info = df.columns
-            else:
-                df = pd.read_csv(file, header=None)
-                df.columns = columns_info
-            lines += len(df)
-            i += 1
-        self.ut_obj.assertEqual(lines, expected_lines)
-
-
-class ShardingFileObjectTest(unittest.TestCase):
-
-    def test_split_by_lines(self):
-        local_path = "bak/train_test.csv"
-        remote_path = "train_test"
-        sharding_object_wrapper = ShardingFileObjectTestWrapper(
-            self, local_path, remote_path)
-        split_mode = SplitMode.LINES
-        # 100w
-        granularity = 20 * 1024 * 1024
-        sharding_object_wrapper.upload(split_mode, granularity, 500000)
-        sharding_object_wrapper.sharding_file_object.download()
-
-
-"""
-    def test_split_by_size(self):
-        local_path = "bak/train_test.csv"
-        remote_path = "train_test"
-        sharding_object_wrapper = ShardingFileObjectTestWrapper(
-            self, local_path, remote_path)
-        split_mode = SplitMode.SIZE
-        # 20M
-        granularity = 20 * 1024 * 1024
-        sharding_object_wrapper.sharding_file_object.split(
-            split_mode, granularity)
-"""
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/python/ppc_common/ppc_ml/feature/feature_importance.py b/python/ppc_common/ppc_ml/feature/feature_importance.py
index 2441a20e..2d612ff8 100644
--- a/python/ppc_common/ppc_ml/feature/feature_importance.py
+++ b/python/ppc_common/ppc_ml/feature/feature_importance.py
@@ -227,7 +227,7 @@ def update_feature_importance(self, fid, gain_list):
             self.feature_importance_dict[importance_type][fid_key].inc(
                 gain_list[importance_type])
 
-    def store(self, serialize_type: SerializeType, local_file_path, remote_file_path, storage_client):
+    def store(self, serialize_type: SerializeType, local_file_path, remote_file_path, storage_client, user=None):
         """store the feature importance into file, upload using storage_client
 
         Args:
@@ -241,6 +241,6 @@ def store(self, serialize_type: SerializeType, local_file_path, remote_file_path
         self.logger.info(
             f"Store feature_importance to {local_file_path}, file type: {serialize_type}")
         if storage_client is not None:
-            storage_client.upload_file(local_file_path, remote_file_path)
+            storage_client.upload_file(local_file_path, remote_file_path, user)
             self.logger.info(
                 f"Upload feature_importance to {local_file_path} success, file type: {serialize_type}")
diff --git a/python/ppc_model/common/base_context.py b/python/ppc_model/common/base_context.py
index c663bea8..ec905b06 100644
--- a/python/ppc_model/common/base_context.py
+++ b/python/ppc_model/common/base_context.py
@@ -25,8 +25,9 @@ class BaseContext:
     MODEL_FILE = "model.kpl"
     MODEL_ENC_FILE = "model_enc.kpl"
 
-    def __init__(self, job_id: str, job_temp_dir: str):
+    def __init__(self, job_id: str, job_temp_dir: str, user: str):
         self.job_id = job_id
+        self.user = user
         self.workspace = os.path.join(job_temp_dir, self.job_id)
         if not os.path.exists(self.workspace):
             os.makedirs(self.workspace)
@@ -43,25 +44,6 @@ def __init__(self, job_id: str, job_temp_dir: str):
             self.workspace, self.IV_SELECTED_FILE)
         self.selected_col_file = os.path.join(
             self.workspace, self.SELECTED_COL_FILE)
-        self.remote_selected_col_file = os.path.join(
-            self.job_id, self.SELECTED_COL_FILE)
-
-        self.summary_evaluation_file = os.path.join(
-            self.workspace, utils.MPC_XGB_EVALUATION_TABLE)
-        self.feature_importance_file = os.path.join(
-            self.workspace, utils.XGB_FEATURE_IMPORTANCE_TABLE)
-        self.feature_bin_file = os.path.join(
-            self.workspace, self.FEATURE_BIN_FILE)
-        self.model_data_file = os.path.join(
-            self.workspace, self.MODEL_DATA_FILE)
-        self.test_model_result_file = os.path.join(
-            self.workspace, self.TEST_MODEL_RESULT_FILE)
-        self.test_model_output_file = os.path.join(
-            self.workspace, self.TEST_MODEL_OUTPUT_FILE)
-        self.train_model_result_file = os.path.join(
-            self.workspace, self.TRAIN_MODEL_RESULT_FILE)
-        self.train_model_output_file = os.path.join(
-            self.workspace, self.TRAIN_MODEL_OUTPUT_FILE)
 
         self.train_metric_roc_file = os.path.join(
             self.workspace, utils.MPC_TRAIN_SET_METRIC_ROC_FILE)
@@ -92,63 +74,99 @@ def __init__(self, job_id: str, job_temp_dir: str):
             self.workspace, self.MODEL_FILE)
         self.model_enc_file = os.path.join(
             self.workspace, self.MODEL_ENC_FILE)
+        self.summary_evaluation_file = os.path.join(
+            self.workspace, utils.MPC_XGB_EVALUATION_TABLE)
+        self.feature_importance_file = os.path.join(
+            self.workspace, utils.XGB_FEATURE_IMPORTANCE_TABLE)
+        self.feature_bin_file = os.path.join(
+            self.workspace, self.FEATURE_BIN_FILE)
+        self.model_data_file = os.path.join(
+            self.workspace, self.MODEL_DATA_FILE)
+        self.test_model_result_file = os.path.join(
+            self.workspace, self.TEST_MODEL_RESULT_FILE)
+        self.test_model_output_file = os.path.join(
+            self.workspace, self.TEST_MODEL_OUTPUT_FILE)
+        self.train_model_result_file = os.path.join(
+            self.workspace, self.TRAIN_MODEL_RESULT_FILE)
+        self.train_model_output_file = os.path.join(
+            self.workspace, self.TRAIN_MODEL_OUTPUT_FILE)
 
-        self.remote_summary_evaluation_file = os.path.join(
-            self.job_id, utils.MPC_XGB_EVALUATION_TABLE)
-        self.remote_feature_importance_file = os.path.join(
-            self.job_id, utils.XGB_FEATURE_IMPORTANCE_TABLE)
-        self.remote_feature_bin_file = os.path.join(
-            self.job_id, self.FEATURE_BIN_FILE)
-        self.remote_model_data_file = os.path.join(
-            self.job_id, self.MODEL_DATA_FILE)
-        self.remote_test_model_output_file = os.path.join(
-            self.job_id, self.TEST_MODEL_OUTPUT_FILE)
-        self.remote_train_model_output_file = os.path.join(
-            self.job_id, self.TRAIN_MODEL_OUTPUT_FILE)
-
-        self.remote_train_metric_roc_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_SET_METRIC_ROC_FILE)
-        self.remote_train_metric_ks_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_FILE)
-        self.remote_train_metric_pr_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_SET_METRIC_PR_FILE)
-        self.remote_train_metric_acc_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE)
-        self.remote_test_metric_roc_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_METRIC_ROC_FILE)
-        self.remote_test_metric_ks_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_METRIC_KS_FILE)
-        self.remote_test_metric_pr_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_METRIC_PR_FILE)
-        self.remote_test_metric_acc_file = os.path.join(
-            self.job_id, utils.MPC_TRAIN_METRIC_ACCURACY_FILE)
-        self.remote_train_metric_ks_table = os.path.join(
-            self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_TABLE)
-        self.remote_test_metric_ks_table = os.path.join(
-            self.job_id, utils.MPC_TRAIN_METRIC_KS_TABLE)
-        self.remote_model_tree_prefix = os.path.join(
-            self.job_id, utils.XGB_TREE_PERFIX)
-        self.remote_metrics_iteration_file = os.path.join(
-            self.job_id, utils.METRICS_OVER_ITERATION_FILE)
-
-        self.remote_model_file = os.path.join(
-            self.job_id, self.MODEL_FILE)
-        self.remote_model_enc_file = os.path.join(
-            self.job_id, self.MODEL_ENC_FILE)
-
+        # the remote path
+        self.remote_base_dir = os.path.join(
+            self.user, "share", "jobs", "model")
+        self.remote_selected_col_file = os.path.join(self.remote_base_dir,
+                                                     self.job_id, self.SELECTED_COL_FILE)
+        self.remote_preprocessing_file = os.path.join(self.remote_base_dir,
+                                                      self.job_id, self.PREPROCESSING_RESULT_FILE)
+
+        self.remote_woe_iv_file = os.path.join(self.remote_base_dir,
+                                               self.job_id, self.WOE_IV_FILE)
+        self.remote_eval_column_file = os.path.join(self.remote_base_dir,
+                                                    self.job_id, self.EVAL_COLUMN_FILE)
+        self.remote_model_prepare_file = os.path.join(
+            self.remote_base_dir, self.job_id, self.MODEL_PREPARE_FILE)
+        self.remote_summary_evaluation_file = os.path.join(self.remote_base_dir,
+                                                           self.job_id, utils.MPC_XGB_EVALUATION_TABLE)
+        self.remote_feature_importance_file = os.path.join(self.remote_base_dir,
+                                                           self.job_id, utils.XGB_FEATURE_IMPORTANCE_TABLE)
+        self.remote_feature_bin_file = os.path.join(self.remote_base_dir,
+                                                    self.job_id, self.FEATURE_BIN_FILE)
+        self.remote_model_data_file = os.path.join(self.remote_base_dir,
+                                                   self.job_id, self.MODEL_DATA_FILE)
+        self.remote_test_model_output_file = os.path.join(self.remote_base_dir,
+                                                          self.job_id, self.TEST_MODEL_OUTPUT_FILE)
+        self.remote_train_model_output_file = os.path.join(self.remote_base_dir,
+                                                           self.job_id, self.TRAIN_MODEL_OUTPUT_FILE)
+
+        self.remote_train_metric_roc_file = os.path.join(self.remote_base_dir,
+                                                         self.job_id, utils.MPC_TRAIN_SET_METRIC_ROC_FILE)
+        self.remote_train_metric_ks_file = os.path.join(self.remote_base_dir,
+                                                        self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_FILE)
+        self.remote_train_metric_pr_file = os.path.join(self.remote_base_dir,
+                                                        self.job_id, utils.MPC_TRAIN_SET_METRIC_PR_FILE)
+        self.remote_train_metric_acc_file = os.path.join(self.remote_base_dir,
+                                                         self.job_id, utils.MPC_TRAIN_SET_METRIC_ACCURACY_FILE)
+        self.remote_test_metric_roc_file = os.path.join(self.remote_base_dir,
+                                                        self.job_id, utils.MPC_TRAIN_METRIC_ROC_FILE)
+        self.remote_test_metric_ks_file = os.path.join(self.remote_base_dir,
+                                                       self.job_id, utils.MPC_TRAIN_METRIC_KS_FILE)
+        self.remote_test_metric_pr_file = os.path.join(self.remote_base_dir,
+                                                       self.job_id, utils.MPC_TRAIN_METRIC_PR_FILE)
+        self.remote_test_metric_acc_file = os.path.join(self.remote_base_dir,
+                                                        self.job_id, utils.MPC_TRAIN_METRIC_ACCURACY_FILE)
+        self.remote_train_metric_ks_table = os.path.join(self.remote_base_dir,
+                                                         self.job_id, utils.MPC_TRAIN_SET_METRIC_KS_TABLE)
+        self.remote_test_metric_ks_table = os.path.join(self.remote_base_dir,
+                                                        self.job_id, utils.MPC_TRAIN_METRIC_KS_TABLE)
+        self.remote_model_tree_prefix = os.path.join(self.remote_base_dir,
+                                                     self.job_id, utils.XGB_TREE_PERFIX)
+        self.remote_metrics_iteration_file = os.path.join(self.remote_base_dir,
+                                                          self.job_id, utils.METRICS_OVER_ITERATION_FILE)
+
+        self.remote_model_file = os.path.join(self.remote_base_dir,
+                                              self.job_id, self.MODEL_FILE)
+        self.remote_model_enc_file = os.path.join(self.remote_base_dir,
+                                                  self.job_id, self.MODEL_ENC_FILE)
+
+        self.remote_log_path = os.path.join(self.remote_base_dir,
+                                            self.job_id, f"{job_id}.log")
+        self.remote_log_size_path = f"{self.remote_log_path}.size"
         # self.get_key_pair()
         self.load_key('aes_key.bin')
 
+    def get_remote_file_path(self, file_path):
+        return os.path.join(self.remote_base_dir,
+                            self.job_id, file_path)
+
+    def get_local_file_path(self, file_path):
+        return os.path.join(self.workspace, file_path)
+
     @staticmethod
     def load_file(storage_client, remote_path, local_path, logger):
         if not os.path.exists(local_path):
             logger.info(f"Download file from: {remote_path} to {local_path}")
             storage_client.download_file(remote_path, local_path)
 
-    @staticmethod
-    def feature_engineering_input_path(job_id: str, job_temp_dir: str):
-        return os.path.join(job_temp_dir, job_id, BaseContext.MODEL_PREPARE_FILE)
-
     def get_key_pair(self):
         with open('public_key.pem', 'rb') as f:
             self.public_pem = f.read()
diff --git a/python/ppc_model/common/context.py b/python/ppc_model/common/context.py
index 7a9ee774..b14fda4b 100644
--- a/python/ppc_model/common/context.py
+++ b/python/ppc_model/common/context.py
@@ -6,8 +6,8 @@
 
 class Context(BaseContext):
 
-    def __init__(self, job_id: str, task_id: str, components: Initializer, role: TaskRole = None):
-        super().__init__(job_id, components.config_data['JOB_TEMP_DIR'])
+    def __init__(self, job_id: str, task_id: str, user: str, components: Initializer, role: TaskRole = None):
+        super().__init__(job_id, components.config_data['JOB_TEMP_DIR'], user)
         self.my_agency_id = components.config_data['AGENCY_ID']
         self.task_id = task_id
         self.components = components
diff --git a/python/ppc_model/common/model_result.py b/python/ppc_model/common/model_result.py
index 0cf9e36b..bc68c308 100644
--- a/python/ppc_model/common/model_result.py
+++ b/python/ppc_model/common/model_result.py
@@ -9,6 +9,7 @@
 from ppc_model.common.context import Context
 from ppc_model.common.protocol import TaskRole
 from ppc_model.network.wedpr_model_transport import ModelRouter
+from ppc_model.common.base_context import BaseContext
 
 
 class ResultFileHandling:
@@ -45,7 +46,9 @@ def _process_fe_result(self):
                              sep=utils.CSV_SEP, header=True, index_label='id')
             # 存储column_info到hdfs给前端展示
             self._upload_file(self.ctx.components.storage_client,
-                              self.ctx.selected_col_file, self.ctx.remote_selected_col_file)
+                              self.ctx.selected_col_file,
+                              self.ctx.remote_selected_col_file,
+                              self.ctx.user)
 
     @staticmethod
     def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame):
@@ -70,9 +73,11 @@ def union_column_info(column_info1: pd.DataFrame, column_info2: pd.DataFrame):
         return column_info_conbine
 
     @staticmethod
-    def _upload_file(storage_client, local_file, remote_file):
+    def _upload_file(storage_client, local_file,
+                     remote_file, owner=None,
+                     group=None):
         if storage_client is not None:
-            storage_client.upload_file(local_file, remote_file)
+            storage_client.upload_file(local_file, remote_file, owner, group)
 
     @staticmethod
     def _download_file(storage_client, local_file, remote_file):
@@ -80,12 +85,12 @@ def _download_file(storage_client, local_file, remote_file):
             storage_client.download_file(remote_file, local_file)
 
     @staticmethod
-    def make_graph_data(components, job_id, graph_file_name):
+    def make_graph_data(components, ctx: BaseContext, graph_file_name):
         graph_format = 'svg+xml'
         # download with cache
-        remote_file_path = os.path.join(job_id, graph_file_name)
-        local_file_path = os.path.join(
-            components.job_cache_dir, remote_file_path)
+        remote_file_path = ctx.get_remote_file_path(graph_file_name)
+
+        local_file_path = ctx.get_local_file_path(graph_file_name)
         components.storage_client.download_file(
             remote_file_path, local_file_path, True)
         file_bytes = None
@@ -98,27 +103,24 @@ def make_graph_data(components, job_id, graph_file_name):
         time.sleep(0.1)
         return f"data:image/{graph_format};base64,{encoded_data}"
 
-    def get_remote_path(components, job_id, csv_file_name):
+    def get_remote_path(components, ctx: BaseContext, csv_file_name):
+        remote_file_path = ctx.get_remote_file_path(csv_file_name)
         if components.storage_client.get_home_path() is None:
-            return os.path.join(job_id, csv_file_name)
-        return os.path.join(components.storage_client.get_home_path(), job_id, csv_file_name)
+            return remote_file_path
+        return os.path.join(components.storage_client.get_home_path(), remote_file_path)
 
     @staticmethod
-    def make_csv_data(components, job_id, csv_file_name):
+    def make_csv_data(components, ctx: BaseContext, csv_file_name):
         import pandas as pd
         from io import StringIO
-        remote_file_path = os.path.join(job_id, csv_file_name)
-        local_file_path = os.path.join(
-            components.job_cache_dir, remote_file_path)
+        remote_file_path = ctx.get_remote_file_path(csv_file_name)
+        local_file_path = ctx.get_local_file_path(csv_file_name)
         components.storage_client.download_file(
             remote_file_path, local_file_path, True)
         file_bytes = None
         with open(local_file_path, 'r') as file:
             file_content = file.read()
             file_bytes = file_content.encode('utf-8')
-        # encoded_data = base64.b64encode(data).decode('ascii')
-        # csv_data = np.genfromtxt(data.decode(), delimiter=',')
-        # csv_data = np.genfromtxt(StringIO(data.decode()), delimiter=',')
         csv_data = ""
         if file_bytes is not None:
             csv_data = pd.read_csv(StringIO(file_bytes.decode())).astype('str')
@@ -136,10 +138,10 @@ def _remove_workspace(self):
     def _sync_result_files(self):
         for key, value in self.ctx.sync_file_list.items():
             self.sync_result_file(
-                self.ctx, self.ctx.model_router, value[0], value[1], key)
+                self.ctx, self.ctx.model_router, value[0], value[1], key, self.ctx.user)
 
     @staticmethod
-    def sync_result_file(ctx, model_router: ModelRouter, local_file, remote_file, key_file):
+    def sync_result_file(ctx, model_router: ModelRouter, local_file, remote_file, key_file, user):
         if ctx.role == TaskRole.ACTIVE_PARTY:
             with open(local_file, 'rb') as f:
                 byte_data = f.read()
@@ -154,7 +156,8 @@ def sync_result_file(ctx, model_router: ModelRouter, local_file, remote_file, ke
                 with open(local_file, 'wb') as f:
                     f.write(byte_data)
                 ResultFileHandling._upload_file(
-                    ctx.components.storage_client, local_file, remote_file)
+                    ctx.components.storage_client,
+                    local_file, remote_file, user)
 
 
 class CommonMessage(Enum):
diff --git a/python/ppc_model/datasets/dataset.py b/python/ppc_model/datasets/dataset.py
index ee90721d..4b486d95 100644
--- a/python/ppc_model/datasets/dataset.py
+++ b/python/ppc_model/datasets/dataset.py
@@ -37,8 +37,7 @@ def __init__(self, ctx: SecureLGBMContext, model_data=None, delimiter: str = ' '
         if model_data is None:
             # try to download the model_prepare_file
             BaseContext.load_file(ctx.components.storage_client,
-                                  os.path.join(
-                                      ctx.job_id, BaseContext.MODEL_PREPARE_FILE),
+                                  ctx.remote_model_prepare_file,
                                   ctx.model_prepare_file, ctx.components.logger())
             self.model_data = pd.read_csv(
                 ctx.model_prepare_file, header=0, delimiter=delimiter)
@@ -212,8 +211,6 @@ def _construct_dataset(self):
         if self.algorithm_type == AlgorithmType.Predict.name \
                 and not os.path.exists(self.selected_col_file):
             try:
-                self.ctx.remote_selected_col_file = os.path.join(
-                    self.ctx.model_params.training_job_id, self.ctx.SELECTED_COL_FILE)
                 ResultFileHandling._download_file(self.ctx.components.storage_client,
                                                   self.selected_col_file, self.ctx.remote_selected_col_file)
                 self._dataset_fe_selected(self.selected_col_file, 'id')
diff --git a/python/ppc_model/feature_engineering/feature_engineering_context.py b/python/ppc_model/feature_engineering/feature_engineering_context.py
index e8df2299..c4504942 100644
--- a/python/ppc_model/feature_engineering/feature_engineering_context.py
+++ b/python/ppc_model/feature_engineering/feature_engineering_context.py
@@ -26,10 +26,11 @@ def __init__(self,
                  feature: np.ndarray,
                  feature_name_list: list,
                  label: np.ndarray = None):
-        super().__init__(args['job_id'],
-                         task_id,
-                         components,
-                         role)
+        super().__init__(job_id=args['job_id'],
+                         task_id=task_id,
+                         components=components,
+                         role=role,
+                         user=args['user'])
         self.feature_name_list = feature_name_list
         self.participant_id_list = args['participant_id_list']
         self.result_receiver_id_list = args['result_receiver_id_list']
diff --git a/python/ppc_model/feature_engineering/feature_engineering_engine.py b/python/ppc_model/feature_engineering/feature_engineering_engine.py
index 3f0d1778..8b67fd68 100644
--- a/python/ppc_model/feature_engineering/feature_engineering_engine.py
+++ b/python/ppc_model/feature_engineering/feature_engineering_engine.py
@@ -6,7 +6,6 @@
 from ppc_model.feature_engineering.vertical.active_party import VerticalFeatureEngineeringActiveParty
 from ppc_model.feature_engineering.vertical.passive_party import VerticalFeatureEngineeringPassiveParty
 from ppc_model.interface.task_engine import TaskEngine
-import os
 
 
 class FeatureEngineeringEngine(TaskEngine):
@@ -14,16 +13,16 @@ class FeatureEngineeringEngine(TaskEngine):
 
     @staticmethod
     def run(task_id, args):
-        input_path = BaseContext.feature_engineering_input_path(
-            args['job_id'], components.config_data['JOB_TEMP_DIR'])
+        base_ctx = BaseContext(
+            job_id=args['job_id'], user=args["user"], job_temp_dir=components.config_data['JOB_TEMP_DIR'])
         # try to download the model_prepare_file
         BaseContext.load_file(components.storage_client,
-                              os.path.join(
-                                  args['job_id'], BaseContext.MODEL_PREPARE_FILE),
-                              input_path, components.logger())
+                              remote_path=base_ctx.remote_model_prepare_file,
+                              local_path=base_ctx.model_prepare_file, logger=components.logger())
+
         if args['is_label_holder']:
             field_list, label, feature = SecureDataset.read_dataset(
-                input_path, True)
+                base_ctx.model_prepare_file, True)
             context = FeatureEngineeringContext(
                 task_id=task_id,
                 args=args,
@@ -36,7 +35,7 @@ def run(task_id, args):
             vfe = VerticalFeatureEngineeringActiveParty(context)
         else:
             field_list, _, feature = SecureDataset.read_dataset(
-                input_path, False)
+                base_ctx.model_prepare_file, False)
             context = FeatureEngineeringContext(
                 task_id=task_id,
                 args=args,
diff --git a/python/ppc_model/feature_engineering/vertical/active_party.py b/python/ppc_model/feature_engineering/vertical/active_party.py
index cb40d404..de1ebed8 100644
--- a/python/ppc_model/feature_engineering/vertical/active_party.py
+++ b/python/ppc_model/feature_engineering/vertical/active_party.py
@@ -79,8 +79,10 @@ def _compute_self_woe_iv(self):
 
             self.iv_selected_df.loc[len(self.iv_selected_df)] = {'feature': field,
                                                                  'iv_selected': int(iv_total >= self.ctx.iv_thresh)}
+            log.info(
+                f"_compute_self_woe_iv, feature: {field}, iv_total: {iv_total}, iv_thres: {self.ctx.iv_thresh}")
         log.info(
-            f"Computing self woe/iv finished, task_id: {self.ctx.task_id}, time_costs: {time.time() - start_time}s")
+            f"Computing self woe/iv finished, task_id: {self.ctx.task_id}, time_costs: {time.time() - start_time}s, iv_thres: {self.ctx.iv_thresh}")
 
     def _compute_partner_woe_iv(self, enc_labels, partner_index):
         log = self.ctx.components.logger()
@@ -174,7 +176,8 @@ def _save_and_sync_fe_results(self):
         self.iv_selected_df.to_csv(
             self.ctx.iv_selected_file, sep=',', header=True, index=None)
         self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file,
-                                                       self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE)
+                                                       self.ctx.remote_woe_iv_file,
+                                                       self.ctx.user)
         log.info(f"Saving fe results finished, task_id: {task_id}")
         with open(self.ctx.woe_iv_file, 'rb') as f:
             woe_iv = f.read()
diff --git a/python/ppc_model/feature_engineering/vertical/passive_party.py b/python/ppc_model/feature_engineering/vertical/passive_party.py
index f3d1aaaa..bd9f659b 100644
--- a/python/ppc_model/feature_engineering/vertical/passive_party.py
+++ b/python/ppc_model/feature_engineering/vertical/passive_party.py
@@ -160,7 +160,8 @@ def _get_and_save_result(self):
             with open(self.ctx.woe_iv_file, 'wb') as f:
                 f.write(data)
             self.ctx.components.storage_client.upload_file(self.ctx.woe_iv_file,
-                                                           self.ctx.job_id + os.sep + self.ctx.WOE_IV_FILE)
+                                                           self.ctx.remote_woe_iv_file,
+                                                           self.ctx.user)
 
         # 保存来自标签方的iv筛选结果
         data = self.ctx.model_router.pop(
diff --git a/python/ppc_model/log/log_retriever.py b/python/ppc_model/log/log_retriever.py
index fd2a4f70..70e2b67c 100644
--- a/python/ppc_model/log/log_retriever.py
+++ b/python/ppc_model/log/log_retriever.py
@@ -3,6 +3,7 @@
 import logging
 from ppc_model.common.protocol import LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER
 from ppc_common.deps_services.storage_api import StorageApi
+from ppc_model.common.base_context import BaseContext
 import os
 
 
@@ -16,44 +17,48 @@ def __init__(self, logger, job_cache_dir, storage_client: StorageApi, log_path):
         self.storage_client = storage_client
         self.relative_log_path = log_path
 
-    def retrieve_log(self, job_id):
+    def retrieve_log(self, job_id, user):
         try:
+            ctx = BaseContext(
+                job_id=job_id, job_temp_dir=self.job_cache_dir, user=user)
             # get the log size
             log_size_byte = self.storage_client.get_data(
-                self._get_remote_log_size_path(job_id))
+                self._get_remote_log_size_path(ctx))
             log_size = int.from_bytes(log_size_byte, 'big')
             remote_log_path = os.path.join(
-                self.storage_client.get_home_path(), self._get_remote_log_path(job_id))
+                self.storage_client.get_home_path(), self._get_remote_log_path(ctx))
             if log_size > LogRetriever.max_retrieved_log_size:
                 return (log_size, remote_log_path, None)
             # get the log data
             log_data = self.storage_client.get_data(
-                self._get_remote_log_path(job_id)).decode(encoding='utf-8')
+                self._get_remote_log_path(ctx)).decode(encoding='utf-8')
             return (log_size, remote_log_path, log_data)
         except:
             return (None, None, None)
 
-    def _get_remote_log_path(self, job_id):
-        return os.path.join(job_id, f'{job_id}.log')
+    def _get_remote_log_path(self, ctx: BaseContext):
+        return ctx.remote_log_path
 
-    def _get_remote_log_size_path(self, job_id):
-        return f"{self._get_remote_log_path(job_id)}.size"
+    def _get_remote_log_size_path(self, ctx: BaseContext):
+        return ctx.remote_log_size_path
 
     def _get_local_job_log_path(self, job_id):
         return os.path.join(self.job_cache_dir, job_id, f'{job_id}.log')
 
-    def upload_log(self, job_id):
-        ret = self.record_model_job_log(job_id)
+    def upload_log(self, job_id, user):
+        ret = self._record_model_job_log(job_id)
         if ret is False:
             return
+        ctx = BaseContext(
+            job_id=job_id, job_temp_dir=self.job_cache_dir, user=user)
         # the log size
         log_size = os.path.getsize(self._get_local_job_log_path(job_id))
         # store the log size
         self.storage_client.save_data(log_size.to_bytes((log_size.bit_length() + 7) // 8, 'big'),
-                                      self._get_remote_log_size_path(job_id))
+                                      self._get_remote_log_size_path(ctx))
         # store the log path
         self.storage_client.upload_file(self._get_local_job_log_path(
-            job_id), self._get_remote_log_path(job_id))
+            job_id), self._get_remote_log_path(ctx), user)
 
     def _make_local_log_dir(self, job_id):
         local_log_path = self._get_local_job_log_path(job_id)
@@ -64,7 +69,7 @@ def _make_local_log_dir(self, job_id):
             return
         os.makedirs(parent_dir)
 
-    def record_model_job_log(self, job_id):
+    def _record_model_job_log(self, job_id):
         log_file = self._get_log_file_path()
         if log_file is None or log_file == "":
             current_working_dir = os.getcwd()
diff --git a/python/ppc_model/metrics/evaluation.py b/python/ppc_model/metrics/evaluation.py
index 33802c7c..011a1ae7 100644
--- a/python/ppc_model/metrics/evaluation.py
+++ b/python/ppc_model/metrics/evaluation.py
@@ -28,7 +28,7 @@ def __init__(self,
                  dataset: SecureDataset,
                  train_praba: np.ndarray = None,
                  test_praba: np.ndarray = None) -> None:
-
+        self.ctx = ctx
         self.job_id = ctx.job_id
         self.storage_client = ctx.components.storage_client
         self.summary_evaluation_file = ctx.summary_evaluation_file
@@ -93,7 +93,7 @@ def summary_evaluation(self, dataset, test_ks, test_auc, train_ks, train_auc):
         FeatureEvaluationResult.store_and_upload_summary(
             [train_evaluation, test_evaluation],
             self.summary_evaluation_file, self.remote_summary_evaluation_file,
-            self.storage_client)
+            self.storage_client, self.ctx.user)
 
     @staticmethod
     def calculate_ks_and_stats(predicted_proba, actual_label, num_buckets=10):
@@ -170,26 +170,26 @@ def evaluation_file(self, ctx, data_index: np.ndarray,
                     time.sleep(random.uniform(0.1, 3))
 
             ResultFileHandling._upload_file(
-                self.storage_client, self.metric_roc_file, self.remote_metric_roc_file)
+                self.storage_client, self.metric_roc_file, self.remote_metric_roc_file, ctx.user)
             ResultFileHandling._upload_file(
-                self.storage_client, self.metric_ks_file, self.remote_metric_ks_file)
+                self.storage_client, self.metric_ks_file, self.remote_metric_ks_file, ctx.user)
             ResultFileHandling._upload_file(
-                self.storage_client, self.metric_pr_file, self.remote_metric_pr_file)
+                self.storage_client, self.metric_pr_file, self.remote_metric_pr_file, ctx.user)
             ResultFileHandling._upload_file(
-                self.storage_client, self.metric_acc_file, self.remote_metric_acc_file)
+                self.storage_client, self.metric_acc_file, self.remote_metric_acc_file, ctx.user)
 
             # ks table
             ks_table = self.calculate_ks_and_stats(y_praba, y_true)
             ks_table.to_csv(self.metric_ks_table, header=True, index=None)
             ResultFileHandling._upload_file(
-                self.storage_client, self.metric_ks_table, self.remote_metric_ks_table)
+                self.storage_client, self.metric_ks_table, self.remote_metric_ks_table, ctx.user)
         else:
             ks_value = auc_value = None
 
         # predict result
         self._parse_model_result(data_index, y_true, y_praba)
         ResultFileHandling._upload_file(
-            self.storage_client, self.model_output_file, self.remote_model_output_file)
+            self.storage_client, self.model_output_file, self.remote_model_output_file, ctx.user)
 
         return ks_value, auc_value
 
diff --git a/python/ppc_model/metrics/model_plot.py b/python/ppc_model/metrics/model_plot.py
index 976d42ab..f3996275 100644
--- a/python/ppc_model/metrics/model_plot.py
+++ b/python/ppc_model/metrics/model_plot.py
@@ -63,7 +63,8 @@ def plot_tree(self):
                         time.sleep(random.uniform(0.1, 3))
 
                 ResultFileHandling._upload_file(
-                    self.storage_client, tree_file_path, remote_tree_file_path)
+                    self.storage_client, tree_file_path,
+                    remote_tree_file_path, self.ctx.user)
 
     def _graph_gtree(self, tree, leaf_id=0, depth=0, orient=None, split_info=None):
         self._leaf_id += 1
diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py
index 0a707504..d678bcde 100644
--- a/python/ppc_model/model_result/task_result_handler.py
+++ b/python/ppc_model/model_result/task_result_handler.py
@@ -6,16 +6,16 @@
 from ppc_common.ppc_ml.model.algorithm_info import ClassificationType
 from ppc_model.common.model_result import ResultFileHandling
 from ppc_common.ppc_ml.model.algorithm_info import EvaluationType
-from ppc_model.common.base_context import BaseContext
 from ppc_model.common.initializer import Initializer
 from enum import Enum
 
 
 class TaskResultRequest:
-    def __init__(self, job_id, task_type, only_fetch_log):
+    def __init__(self, job_id, task_type, only_fetch_log, user):
         self.job_id = job_id
         self.task_type = task_type
         self.only_fetch_log = only_fetch_log
+        self.user = user
 
 
 class DataType(Enum):
@@ -74,7 +74,9 @@ class JobEvaluationResult:
     }
 
     def __init__(self, property_name, classification_type,
+                 ctx: BaseContext,
                  job_id, evaluation_files, components):
+        self.ctx = ctx
         self.job_id = job_id
         self.classification_type = classification_type
         self.components = components
@@ -95,14 +97,14 @@ def _fetch_evaluation_result(self):
             f"fetch roc-evaluation from: {self.evaluation_files[EvaluationType.ROC]}")
         self.evaluation_results.append(DataItem("ROC", ResultFileHandling.make_graph_data(
             self.components,
-            self.job_id,
+            self.ctx,
             self.evaluation_files[EvaluationType.ROC]),
             DataType.IMAGE))
         self.logger.info(
             f"fetch pr-evaluation from: {self.evaluation_files[EvaluationType.PR]}")
         self.evaluation_results.append(DataItem("Precision Recall", ResultFileHandling.make_graph_data(
             self.components,
-            self.job_id,
+            self.ctx,
             self.evaluation_files[EvaluationType.PR]), DataType.IMAGE))
 
     def _fetch_two_classifcation_evaluation_result(self):
@@ -113,7 +115,7 @@ def _fetch_two_classifcation_evaluation_result(self):
             f"fetch ks-evaluation from: {self.evaluation_files[EvaluationType.KS]}")
         self.evaluation_results.append(DataItem("K-S", ResultFileHandling.make_graph_data(
             self.components,
-            self.job_id,
+            self.ctx,
             self.evaluation_files[EvaluationType.KS]),
             DataType.IMAGE))
 
@@ -122,7 +124,7 @@ def _fetch_two_classifcation_evaluation_result(self):
         self.evaluation_results.append(DataItem("Accuracy",
                                                 ResultFileHandling.make_graph_data(
                                                     self.components,
-                                                    self.job_id,
+                                                    self.ctx,
                                                     self.evaluation_files[EvaluationType.ACCURACY]),
                                                 DataType.IMAGE))
 
@@ -133,13 +135,15 @@ def _fetch_multi_classifcation_evaluation_result(self):
             f"fetch confusion-matrix-evaluation from: {self.evaluation_files[EvaluationType.CONFUSION_MATRIX]}")
         self.evaluation_results.append(DataItem("Confusion Matrix",
                                                 ResultFileHandling.make_graph_data(self.components,
-                                                                                   self.job_id,
+                                                                                   self.ctx,
                                                                                    self.evaluation_files[EvaluationType.CONFUSION_MATRIX]),
                                                 DataType.IMAGE))
 
     def load_ks_table(self, ks_table_file, ks_table_property):
-        ks_table_object = TableResult(components=self.components,
-                                      job_id=self.job_id, file_meta=ResultFileMeta(table_file_name=ks_table_file))
+        ks_table_object = TableResult(ctx=self.ctx,
+                                      components=self.components,
+                                      job_id=self.job_id,
+                                      file_meta=ResultFileMeta(table_file_name=ks_table_file))
         self.ks_table = ks_table_object.to_dict()
         self.ks_table_property = ks_table_property
 
@@ -154,14 +158,15 @@ def to_dict(self):
 
 
 class TableResult:
-    def __init__(self, components, job_id, file_meta):
+    def __init__(self, ctx: BaseContext, components, job_id, file_meta):
+        self.ctx = ctx
         self.components = components
         self.job_id = job_id
         self.file_meta = file_meta
 
     def to_dict(self):
         try:
-            df = ResultFileHandling.make_csv_data(self.components, self.job_id,
+            df = ResultFileHandling.make_csv_data(self.components, self.ctx,
                                                   self.file_meta.table_file_name)
             csv_columns = list(df.columns)
 
@@ -180,7 +185,8 @@ class FeatureProcessingResult:
         "PRPreview": ResultFileMeta("xgb_result_column_info_selected.csv"),
         "FEPreview": ResultFileMeta("woe_iv.csv", 5)}
 
-    def __init__(self, components, job_id, file_infos):
+    def __init__(self, ctx: BaseContext, components, job_id, file_infos):
+        self.ctx = ctx
         self.components = components
         self.job_id = job_id
         self.file_infos = file_infos
@@ -189,7 +195,7 @@ def __init__(self, components, job_id, file_infos):
 
     def _fetch_result(self):
         for property in self.file_infos.keys():
-            table_info = TableResult(self.components,
+            table_info = TableResult(self.ctx, self.components,
                                      self.job_id, self.file_infos[property]).to_dict()
             self.result.update({property: table_info})
 
@@ -205,10 +211,10 @@ class ModelJobResult:
     TEST_RESULT_PATH = "testResultPath"
     WOE_RESULT_PATH = "woeIVResultPath"
 
-    def __init__(self, xgb_job, job_id, components, property_name=DEFAULT_PROPERTY_NAME):
+    def __init__(self, ctx: BaseContext, xgb_job, job_id, components, property_name=DEFAULT_PROPERTY_NAME):
         self.job_id = job_id
+        self.ctx = ctx
         self.xgb_job = xgb_job
-        self.base_context = BaseContext(job_id, ".tmp")
         self.components = components
         self.logger = components.logger()
         self.property_name = property_name
@@ -231,7 +237,7 @@ def fetch_xgb_model_result(self):
         while i < 6:
             try:
                 tree_data = DataItem(data=ResultFileHandling.make_graph_data(self.components,
-                                                                             self.job_id,
+                                                                             self.ctx,
                                                                              utils.XGB_TREE_PERFIX + '_' + str(i) + '.svg'),
                                      name='tree-' + str(i), name_property="ModelPlotName", data_property="ModelPlotData",
                                      type=DataType.IMAGE)
@@ -242,33 +248,33 @@ def fetch_xgb_model_result(self):
 
     def load_result(self, result_path, result_property):
         self.result_property = result_property
-        job_result_object = TableResult(self.components,
+        job_result_object = TableResult(self.ctx, self.components,
                                         self.job_id, ResultFileMeta(result_path, 5))
         self.job_result = job_result_object.to_dict()
 
     def load_model_result_path(self, predict: bool):
         self.model_result_path_dict = dict()
         self.model_result_path = ResultFileHandling.get_remote_path(
-            self.components, self.job_id, BaseContext.MODEL_DATA_FILE)
+            self.components, self.ctx, BaseContext.MODEL_DATA_FILE)
         self.model_result_path_dict.update(
             {ModelJobResult.MODEL_RESULT_PATH: self.model_result_path})
 
         self.train_result_path = ResultFileHandling.get_remote_path(
-            self.components, self.job_id, BaseContext.TRAIN_MODEL_OUTPUT_FILE)
+            self.components, self.ctx, BaseContext.TRAIN_MODEL_OUTPUT_FILE)
         self.model_result_path_dict.update(
             {ModelJobResult.TRAIN_RESULT_PATH: self.train_result_path})
 
         self.model_result_path_dict.update(
             {ModelJobResult.TEST_RESULT_PATH: ResultFileHandling.get_remote_path(
-                self.components, self.job_id, BaseContext.TEST_MODEL_OUTPUT_FILE)})
+                self.components, self.ctx, BaseContext.TEST_MODEL_OUTPUT_FILE)})
 
         self.woe_iv_result_path = ResultFileHandling.get_remote_path(
-            self.components, self.job_id, BaseContext.WOE_IV_FILE)
+            self.components, self.ctx, BaseContext.WOE_IV_FILE)
         self.model_result_path_dict.update(
             {ModelJobResult.WOE_RESULT_PATH: self.woe_iv_result_path})
 
     def load_evaluation_table(self, evaluation_path, property):
-        evaluation_table_object = TableResult(self.components,
+        evaluation_table_object = TableResult(self.ctx, self.components,
                                               self.job_id, ResultFileMeta(evaluation_path))
         self.evaluation_table = {property: DataItem(name=property, data=evaluation_table_object.to_dict(),
                                                     type=DataType.TABLE).to_dict()}
@@ -276,14 +282,14 @@ def load_evaluation_table(self, evaluation_path, property):
     def load_feature_importance_table(self, feature_importance_path, property):
         if not self.xgb_job:
             return
-        feature_importance_table = TableResult(self.components,
+        feature_importance_table = TableResult(self.ctx, self.components,
                                                self.job_id, ResultFileMeta(feature_importance_path))
         self.feature_importance_table = {property: DataItem(name=property, data=feature_importance_table.to_dict(),
                                                             type=DataType.TABLE).to_dict()}
 
     def load_encrypted_model_data(self):
         try:
-            return self.components.storage_client.get_data(self.base_context.remote_model_enc_file).decode("utf-8")
+            return self.components.storage_client.get_data(self.ctx.remote_model_enc_file).decode("utf-8")
         except:
             pass
 
@@ -291,7 +297,7 @@ def load_iteration_metrics(self, iteration_path, property):
         if not self.xgb_job:
             return
         try:
-            iteration_metrics_data = DataItem(data=ResultFileHandling.make_graph_data(self.components, self.job_id, utils.METRICS_OVER_ITERATION_FILE),
+            iteration_metrics_data = DataItem(data=ResultFileHandling.make_graph_data(self.components, self.ctx, utils.METRICS_OVER_ITERATION_FILE),
                                               name='iteration_metrics', name_property="ModelPlotName", data_property="ModelPlotData",
                                               type=DataType.IMAGE)
             self.iteration_metrics = []
@@ -327,6 +333,8 @@ def __init__(self, task_result_request: TaskResultRequest, components: Initializ
         self.predict = False
         self.xgb_job = False
         self.model_data = None
+        self.ctx = BaseContext(job_id=task_result_request.job_id,
+                               job_temp_dir="tmp", user=task_result_request.user)
         if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.LR_PREDICTING.name:
             self.predict = True
         if self.task_result_request.task_type == ModelTask.XGB_PREDICTING.name or self.task_result_request.task_type == ModelTask.XGB_TRAINING.name:
@@ -349,7 +357,7 @@ def get_response(self):
                             "modelData": self.model_data}
         # record the log
         log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
-            self.task_result_request.job_id)
+            self.task_result_request.job_id, self.task_result_request.user)
         log_result = {}
         log_result.update({"logSize": log_size})
         log_result.update({"logPath": log_path})
@@ -361,6 +369,7 @@ def _get_evaluation_result(self):
         if not self.predict:
             # the train evaluation result
             self.train_evaluation_result = JobEvaluationResult(
+                ctx=self.ctx,
                 property_name="outputMetricsGraphs",
                 classification_type=ClassificationType.TWO,
                 job_id=self.task_result_request.job_id,
@@ -372,6 +381,7 @@ def _get_evaluation_result(self):
             self.result_list.append(self.train_evaluation_result)
 
             self.validation_evaluation_result = JobEvaluationResult(
+                ctx=self.ctx,
                 property_name="outputTrainMetricsGraphs",
                 classification_type=ClassificationType.TWO,
                 job_id=self.task_result_request.job_id,
@@ -382,8 +392,9 @@ def _get_evaluation_result(self):
                 utils.MPC_TRAIN_METRIC_KS_TABLE, "KSTable")
             self.result_list.append(self.validation_evaluation_result)
 
-            self.model = ModelJobResult(self.xgb_job,
-                                        self.task_result_request.job_id, self.components, ModelJobResult.DEFAULT_PROPERTY_NAME)
+            self.model = ModelJobResult(self.ctx, self.xgb_job,
+                                        self.task_result_request.job_id,
+                                        self.components, ModelJobResult.DEFAULT_PROPERTY_NAME)
             self.model.fetch_xgb_model_result()
             # the ks-auc table
             self.model.load_evaluation_table(
@@ -400,6 +411,7 @@ def _get_evaluation_result(self):
         if self.predict:
             # the train evaluation result
             self.predict_evaluation_result = JobEvaluationResult(
+                ctx=self.ctx,
                 property_name="outputMetricsGraphs",
                 classification_type=ClassificationType.TWO,
                 job_id=self.task_result_request.job_id,
@@ -411,14 +423,15 @@ def _get_evaluation_result(self):
             self.result_list.append(self.predict_evaluation_result)
 
         # load model_result
-        self.model_result = ModelJobResult(self.xgb_job,
-                                           self.task_result_request.job_id, self.components, ModelJobResult.DEFAULT_PROPERTY_NAME)
+        self.model_result = ModelJobResult(self.ctx, self.xgb_job,
+                                           self.task_result_request.job_id,
+                                           self.components, ModelJobResult.DEFAULT_PROPERTY_NAME)
         self.model_result.load_result(
             BaseContext.TRAIN_MODEL_OUTPUT_FILE, "outputTrainPreview")
         self.model_result.load_model_result_path(self.predict)
         self.result_list.append(self.model_result)
 
     def _get_feature_processing_result(self):
-        self.feature_processing_result = FeatureProcessingResult(
-            self.components, self.task_result_request.job_id, FeatureProcessingResult.DEFAULT_FEATURE_PROCESSING_FILES)
+        self.feature_processing_result = FeatureProcessingResult(self.ctx,
+                                                                 self.components, self.task_result_request.job_id, FeatureProcessingResult.DEFAULT_FEATURE_PROCESSING_FILES)
         self.result_list.append(self.feature_processing_result)
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index 4e1e235e..39e3301f 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -34,6 +34,11 @@ def post(self, model_id):
             components.logger().info(
                 f"run task request, task_id: {task_id}")
             task_type = args['task_type']
+            if 'user' not in args.keys():
+                raise Exception(
+                    f"Must pass the user that trigger the job, task_id: {task_id}")
+            if 'job_id' not in args.keys():
+                raise Exception(f"Must pass the job_id, task_id: {task_id}")
             components.task_manager.run_task(
                 task_id, ModelTask(task_type), (args,))
             return utils.BASE_RESPONSE
@@ -74,8 +79,9 @@ def delete(self, model_id):
             Kill a specific task by job_id.
             """
             job_id = model_id
-            components.logger().info(f"kill request, job_id: {job_id}")
-            components.task_manager.kill_task(job_id)
+            components.logger().info(
+                f"kill request, job_id: {job_id}")
+            components.task_manager.kill_task(job_id=job_id)
             return utils.BASE_RESPONSE
         except Exception as e:
             response = {}
@@ -104,7 +110,7 @@ def post(self, task_id):
             components.logger().info(
                 f"get_job_direct_result_response, job: {task_id}")
             task_result_request = TaskResultRequest(
-                task_id, task_type, only_fetch_log)
+                task_id, task_type, only_fetch_log, user_name)
             job_result_handler = TaskResultHandler(
                 task_result_request=task_result_request, components=components)
             response = job_result_handler.get_response()
@@ -115,5 +121,5 @@ def post(self, task_id):
             response = {}
             response.update({'errorCode': -1})
             response.update(
-                {'message': f'query task log for {model_id} failed for {e}'})
+                {'message': f'query task log for {task_id} failed for {e}'})
             return response
diff --git a/python/ppc_model/preprocessing/local_processing/local_processing_party.py b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
index 568f7cf6..4a71bfdf 100644
--- a/python/ppc_model/preprocessing/local_processing/local_processing_party.py
+++ b/python/ppc_model/preprocessing/local_processing/local_processing_party.py
@@ -28,22 +28,24 @@ def processing(self):
         job_algorithm_type = self.ctx.job_algorithm_type
         psi_result_path = self.ctx.psi_result_path
         model_prepare_file = self.ctx.model_prepare_file
+
         storage_client.download_file(dataset_path, dataset_file_path)
         if need_psi and (not utils.file_exists(psi_result_path)):
             storage_client.download_file(
                 self.ctx.remote_psi_result_path, psi_result_path)
-            log.info(
-                f"prepare_xgb_after_psi, make_dataset_to_xgb_data_plus_psi_data, dataset_file_path={dataset_file_path}, "
-                f"psi_result_path={psi_result_path}, model_prepare_file={model_prepare_file}, "
-                f"remote_psi_result_path: {self.ctx.remote_psi_result_path}")
+
+        log.info(
+            f"prepare_xgb_after_psi, make_dataset_to_xgb_data_plus_psi_data, local_dataset_file_path={dataset_file_path}, "
+            f"remote_dataset_path={dataset_path}, model_prepare_file={model_prepare_file}, "
+            f"remote_psi_result_path: {self.ctx.remote_psi_result_path}")
         self.make_dataset_to_xgb_data()
         storage_client.upload_file(
-            model_prepare_file, job_id + os.sep + BaseContext.MODEL_PREPARE_FILE)
+            model_prepare_file, self.ctx.remote_model_prepare_file, self.ctx.user)
         log.info(f"upload model_prepare_file to hdfs, job_id={job_id}")
         if job_algorithm_type == utils.AlgorithmType.Train.name:
             log.info(f"upload column_info to hdfs, job_id={job_id}")
             storage_client.upload_file(self.ctx.preprocessing_result_file,
-                                       job_id + os.sep + self.ctx.PREPROCESSING_RESULT_FILE)
+                                       self.ctx.remote_preprocessing_file, self.ctx.user)
         log.info(
             f"call prepare_xgb_after_psi success, job_id={job_id}, timecost: {time.time() - start}")
 
@@ -61,11 +63,12 @@ def make_dataset_to_xgb_data(self):
                 f"dataset_file_path not found: {dataset_file_path}")
         dataset_df = pd.read_csv(dataset_file_path)
         if need_run_psi:
-            log.info(f"psi_result_file_path:{psi_result_file_path}")
             psi_data = pd.read_csv(psi_result_file_path,
                                    delimiter=utils.CSV_SEP)
             dataset_df = pd.merge(dataset_df, psi_data, on=[
                                   'id']).sort_values(by='id', ascending=True)
+            log.info(
+                f"psi_result_file_path:{psi_result_file_path}, dataset_columns: {dataset_df.columns}")
 
         ppc_job_type = self.ctx.job_algorithm_type
         column_info = process_dataframe(
diff --git a/python/ppc_model/preprocessing/local_processing/preprocessing.py b/python/ppc_model/preprocessing/local_processing/preprocessing.py
index d46f41de..66346bb8 100644
--- a/python/ppc_model/preprocessing/local_processing/preprocessing.py
+++ b/python/ppc_model/preprocessing/local_processing/preprocessing.py
@@ -88,8 +88,8 @@ def process_dataframe(dataset_df: pd.DataFrame, model_setting: ModelSetting, xgb
                 eval_column = model_setting.eval_set_column
                 dataset_df[['id', eval_column]].to_csv(
                     ctx.eval_column_file, index=None)
-                ctx.components.storage_client.upload_file(
-                    ctx.eval_column_file, job_id + os.sep + ctx.EVAL_COLUMN_FILE)
+                ctx.components.storage_client.upload_file(ctx.eval_column_file,
+                                                          ctx.remote_eval_column_file, ctx.user)
                 if model_setting.eval_set_column != model_setting.psi_select_col:
                     dataset_df = dataset_df.drop(columns=[eval_column])
 
diff --git a/python/ppc_model/preprocessing/processing_context.py b/python/ppc_model/preprocessing/processing_context.py
index bc56bcd5..37b5dfae 100644
--- a/python/ppc_model/preprocessing/processing_context.py
+++ b/python/ppc_model/preprocessing/processing_context.py
@@ -11,10 +11,11 @@ def __init__(self,
                  task_id,
                  args,
                  components: Initializer):
-        super().__init__(args['job_id'],
-                         task_id,
-                         components,
-                         role=None)
+        super().__init__(job_id=args['job_id'],
+                         task_id=task_id,
+                         components=components,
+                         role=None,
+                         user=args["user"])
         self.dataset_path = args['dataset_path']
         self.dataset_file_path = os.path.join(
             self.workspace, args['dataset_id'])
diff --git a/python/ppc_model/secure_lgbm/monitor/core.py b/python/ppc_model/secure_lgbm/monitor/core.py
index 6e080a04..5c66ad8a 100644
--- a/python/ppc_model/secure_lgbm/monitor/core.py
+++ b/python/ppc_model/secure_lgbm/monitor/core.py
@@ -1,6 +1,6 @@
 import collections
 from typing import Dict, Any, Union, Tuple, List, Callable
-
+from ppc_model.common.base_context import BaseContext
 import numpy as np
 from sklearn import metrics
 
@@ -28,11 +28,13 @@ def __init__(
             self,
             y_true: np.ndarray,
             test_y_true: np.ndarray,
+            ctx: BaseContext = None,
             workspace: str = None,
             job_id: str = None,
             storage_client: str = None
     ) -> None:
         self.params: _BoosterParams = {}
+        self.ctx = ctx
         self.y_true = y_true
         self.test_y_true = test_y_true
         self.y_pred = None
@@ -41,6 +43,7 @@ def __init__(
         self.epoch = 0
         self.workspace = workspace
         self.job_id = job_id
+
         self.storage_client = storage_client
         self.history: _EvalsLog = collections.OrderedDict()
 
diff --git a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
index 29787b1e..1529381c 100644
--- a/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
+++ b/python/ppc_model/secure_lgbm/monitor/evaluation_monitor.py
@@ -38,19 +38,24 @@ def _draw_figure(model: _Model):
         plt.xticks(range(0, len(iterations), 5), fontsize=10, rotation=45)
     plt.yticks(fontsize=12)
 
-    file_path = os.path.join(path, METRICS_OVER_ITERATION_FILE)
-    plt.savefig(file_path, format='svg', dpi=300)
+    plt.savefig(model.ctx.get_local_file_path(
+        METRICS_OVER_ITERATION_FILE), format='svg', dpi=300)
     plt.close('all')
 
 
 def _upload_figure(model: _Model):
     storage_client = model.get_storage_client()
     if storage_client is not None:
-        path = model.get_workspace()
         job_id = model.get_job_id()
-        metrics_file_path = os.path.join(path, METRICS_OVER_ITERATION_FILE)
-        unique_file_path = os.path.join(job_id, METRICS_OVER_ITERATION_FILE)
-        storage_client.upload_file(metrics_file_path, unique_file_path)
+        local_metrics_file_path = model.ctx.get_local_file_path(
+            METRICS_OVER_ITERATION_FILE)
+        remote_unique_file_path = model.ctx.get_remote_file_path(
+            METRICS_OVER_ITERATION_FILE)
+        user = None
+        if model.ctx is not None:
+            user = model.ctx.user
+        storage_client.upload_file(
+            local_metrics_file_path, remote_unique_file_path, user)
 
 
 def _fmt_metric(
diff --git a/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py b/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py
index bb2530f2..b3c097af 100644
--- a/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py
+++ b/python/ppc_model/secure_lgbm/monitor/feature/feature_evaluation_info.py
@@ -83,8 +83,8 @@ def summary(evaluation_result_list):
         return pd.DataFrame(rows, columns=columns)
 
     @staticmethod
-    def store_and_upload_summary(evaluation_result_list, local_file_path, remote_file_path, storage_client):
+    def store_and_upload_summary(evaluation_result_list, local_file_path, remote_file_path, storage_client, user=None):
         df = FeatureEvaluationResult.summary(evaluation_result_list)
         df.to_csv(local_file_path, index=False)
         if storage_client is not None:
-            storage_client.upload_file(local_file_path, remote_file_path)
+            storage_client.upload_file(local_file_path, remote_file_path, user)
diff --git a/python/ppc_model/secure_lgbm/vertical/active_party.py b/python/ppc_model/secure_lgbm/vertical/active_party.py
index 22663562..9a54e7ec 100644
--- a/python/ppc_model/secure_lgbm/vertical/active_party.py
+++ b/python/ppc_model/secure_lgbm/vertical/active_party.py
@@ -431,6 +431,7 @@ def _init_early_stop(self):
 
         model = Booster(y_true=self.dataset.train_y, test_y_true=self.dataset.test_y,
                         workspace=self.ctx.workspace, job_id=self.ctx.job_id,
+                        ctx=self.ctx,
                         storage_client=self.storage_client)
 
         if callback_container:
@@ -475,7 +476,8 @@ def _end_active_data(self, is_train=True):
         if is_train:
             self.feature_importance_store.store(
                 serialize_type=SerializeType.CSV, local_file_path=self.ctx.feature_importance_file,
-                remote_file_path=self.ctx.remote_feature_importance_file, storage_client=self.storage_client)
+                remote_file_path=self.ctx.remote_feature_importance_file, storage_client=self.storage_client,
+                user=self.ctx.user)
 
             if self.callback_container:
                 with self.ctx.components.plot_lock:
diff --git a/python/ppc_model/secure_lgbm/vertical/booster.py b/python/ppc_model/secure_lgbm/vertical/booster.py
index 1fa7a5b6..8eb94f44 100644
--- a/python/ppc_model/secure_lgbm/vertical/booster.py
+++ b/python/ppc_model/secure_lgbm/vertical/booster.py
@@ -188,29 +188,30 @@ def _split_test_data(ctx, test_X, X_split):
         feat_bin = FeatureBinning(ctx)
         return feat_bin.data_binning(test_X, X_split)[0]
 
-    def save_model(self, file_path=None):
-        super().save_model(file_path, "lgbm_model")
+    def save_model(self):
+        super().save_model("lgbm_model")
 
-    def save_model_hook(self, file_path):
+    def save_model_hook(self):
         # save the feature_bin
-        if file_path is not None:
-            self.ctx.feature_bin_file = os.path.join(
-                file_path, self.ctx.FEATURE_BIN_FILE)
         if self._X_split is not None and not os.path.exists(self.ctx.feature_bin_file):
             X_split_dict = {k: v for k, v in zip(
                 self.dataset.feature_name, self._X_split)}
             with open(self.ctx.feature_bin_file, 'w') as f:
                 json.dump(X_split_dict, f)
             ResultFileHandling._upload_file(self.ctx.components.storage_client,
-                                            self.ctx.feature_bin_file, self.ctx.remote_feature_bin_file)
+                                            self.ctx.feature_bin_file,
+                                            self.ctx.remote_feature_bin_file,
+                                            self.ctx.user)
             self.logger.info(
-                f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished.")
+                f"task {self.ctx.task_id}: Saved x_split to {self.ctx.feature_bin_file} finished, split size: {len(X_split_dict.keys())}")
         if not os.path.exists(self.ctx.model_data_file):
             serial_trees = [self._serial_tree(tree) for tree in self._trees]
             with open(self.ctx.model_data_file, 'w') as f:
                 json.dump(serial_trees, f)
             ResultFileHandling._upload_file(self.ctx.components.storage_client,
-                                            self.ctx.model_data_file, self.ctx.remote_model_data_file)
+                                            self.ctx.model_data_file,
+                                            self.ctx.remote_model_data_file,
+                                            self.ctx.user)
             self.logger.info(
                 f"task {self.ctx.task_id}: Saved serial_trees to {self.ctx.model_data_file} finished.")
 
@@ -252,7 +253,9 @@ def merge_model_file(self, lgbm_model):
         with open(self.ctx.model_enc_file, 'w') as f:
             json.dump(lgbm_model, f)
         ResultFileHandling._upload_file(self.ctx.components.storage_client,
-                                        self.ctx.model_enc_file, self.ctx.remote_model_enc_file)
+                                        self.ctx.model_enc_file,
+                                        self.ctx.remote_model_enc_file,
+                                        self.ctx.user)
         self.logger.info(
             f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.")
 
diff --git a/python/ppc_model/secure_lr/vertical/booster.py b/python/ppc_model/secure_lr/vertical/booster.py
index 97400116..3d4b286e 100644
--- a/python/ppc_model/secure_lr/vertical/booster.py
+++ b/python/ppc_model/secure_lr/vertical/booster.py
@@ -145,12 +145,12 @@ def _calculate_deriv(self, x_, d, partner_index_list, d_other_list):
             deriv_x_i = np.frombuffer(self._receive_byte_data(
                 self.ctx, f'{LRMessage.D_MATMUL.value}_{self._iter_id}', partner_index), dtype=np.float)
             self.logger.info(
-                f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.')
+                f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv size: {deriv.size}.')
             self.logger.info(
-                f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv_x_i: {deriv_x_i}.')
+                f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv_x_i size: {deriv_x_i.size}.')
             deriv += deriv_x_i
         self.logger.info(
-            f'{self.ctx.components.config_data["AGENCY_ID"]}, deriv: {deriv}.')
+            f'{self.ctx.components.config_data["AGENCY_ID"]}, merged deriv size: {deriv.size}.')
         return deriv
 
     def _calculate_deriv1(self, x_, d, partner_index_list, d_other_list):
@@ -225,16 +225,18 @@ def _receive_byte_data(self, ctx, key_type, partner_index):
             f"data_size: {len(byte_data) / 1024}KB, time_costs: {time.time() - start_time}s")
         return byte_data
 
-    def save_model(self, file_path=None):
-        super().save_model(file_path, "lr_model")
+    def save_model(self):
+        super().save_model("lr_model")
 
-    def save_model_hook(self, model_file_path):
+    def save_model_hook(self):
         if not os.path.exists(self.ctx.model_data_file):
             serial_weight = list(self._train_weights)
             with open(self.ctx.model_data_file, 'w') as f:
                 json.dump(serial_weight, f)
             ResultFileHandling._upload_file(self.ctx.components.storage_client,
-                                            self.ctx.model_data_file, self.ctx.remote_model_data_file)
+                                            self.ctx.model_data_file,
+                                            self.ctx.remote_model_data_file,
+                                            self.ctx.user)
             self.logger.info(
                 f"task {self.ctx.task_id}: Saved serial_weight to {self.ctx.model_data_file} finished.")
 
@@ -267,7 +269,9 @@ def merge_model_file(self, lr_model):
         with open(self.ctx.model_enc_file, 'w') as f:
             json.dump(lr_model, f)
         ResultFileHandling._upload_file(self.ctx.components.storage_client,
-                                        self.ctx.model_enc_file, self.ctx.remote_model_enc_file)
+                                        self.ctx.model_enc_file,
+                                        self.ctx.remote_model_enc_file,
+                                        self.ctx.user)
         self.logger.info(
             f"task {self.ctx.task_id}: Saved enc model to {self.ctx.model_enc_file} finished.")
 
diff --git a/python/ppc_model/secure_model_base/secure_model_booster.py b/python/ppc_model/secure_model_base/secure_model_booster.py
index ac4f4f86..fa47d786 100644
--- a/python/ppc_model/secure_model_base/secure_model_booster.py
+++ b/python/ppc_model/secure_model_base/secure_model_booster.py
@@ -11,12 +11,8 @@ def __init__(self, ctx) -> None:
         super().__init__(ctx)
         self.logger = self.ctx.components.logger()
 
-    def save_model(self, file_path=None, model_type=None):
-        if file_path is not None:
-            self.ctx.model_data_file = os.path.join(
-                file_path, self.ctx.MODEL_DATA_FILE)
-
-        self.save_model_hook(file_path)
+    def save_model(self, model_type=None):
+        self.save_model_hook()
         model = {}
         model['model_type'] = model_type
         model['label_provider'] = self.ctx.participant_id_list[0]
@@ -36,5 +32,5 @@ def merge_model_file(self, lr_model):
         pass
 
     @abstractmethod
-    def save_model_hook(self, model_file_path):
+    def save_model_hook(self):
         pass
diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py
index f4282f40..0b68f773 100644
--- a/python/ppc_model/secure_model_base/secure_model_context.py
+++ b/python/ppc_model/secure_model_base/secure_model_context.py
@@ -108,10 +108,11 @@ def __init__(self,
         else:
             role = TaskRole.PASSIVE_PARTY
 
-        super().__init__(args['job_id'],
-                         task_id,
-                         components,
-                         role)
+        super().__init__(job_id=args['job_id'],
+                         task_id=task_id,
+                         components=components,
+                         role=role,
+                         user=args['user'])
         self.is_label_holder = args['is_label_holder']
         self.result_receiver_id_list = args['result_receiver_id_list']
         self.participant_id_list = args['participant_id_list']
diff --git a/python/ppc_model/task/task_manager.py b/python/ppc_model/task/task_manager.py
index bba6c80e..100a3199 100644
--- a/python/ppc_model/task/task_manager.py
+++ b/python/ppc_model/task/task_manager.py
@@ -1,7 +1,5 @@
 import datetime
 import json
-import logging
-import os
 import threading
 import time
 from typing import Callable, Union
@@ -10,7 +8,6 @@
 
 from ppc_common.ppc_async_executor.async_thread_executor import AsyncThreadExecutor
 from ppc_common.ppc_async_executor.thread_event_manager import ThreadEventManager
-from ppc_common.ppc_utils.exception import PpcException, PpcErrorCode
 from ppc_common.deps_services.sql_storage_api import SQLStorageAPI
 from ppc_model.common.protocol import ModelTask, TaskStatus, LOG_START_FLAG_FORMATTER, LOG_END_FLAG_FORMATTER
 from ppc_model.db.job_worker_record import JobWorkerRecord
@@ -18,11 +15,12 @@
 
 
 class TaskResult:
-    def __init__(self, job_id, task_id):
+    def __init__(self, job_id, task_id, user):
         self.task_status = TaskStatus.RUNNING.value
         self.start_time = datetime.datetime.now()
         self.job_id = job_id
         self.task_id = task_id
+        self.user = user
         self.diagnosis_msg = ""
         self.exec_result = None
 
@@ -34,6 +32,7 @@ def finalize(self):
         exec_result_dict = {}
         exec_result_dict.update({"timecost": self.get_timecost()})
         exec_result_dict.update({"diagnosis_msg": self.diagnosis_msg})
+        exec_result_dict.update({"user": self.user})
         self.exec_result = json.dumps(exec_result_dict)
 
     def __repr__(self):
@@ -143,7 +142,7 @@ def run_task(self, task_id: str, task_type: ModelTask, args=()):
                     f"Task already exists, task_id: {task_id}, status: {self._tasks[task_id].task_id}")
                 return
             self._tasks[task_id] = TaskResult(
-                job_id=args[0]['job_id'], task_id=task_id)
+                job_id=args[0]['job_id'], task_id=task_id, user=args[0]['user'])
         self.logger.info(LOG_START_FLAG_FORMATTER.format(job_id=job_id))
         self.logger.info(f"Run task, job_id: {job_id}, task_id: {task_id}")
         self._async_executor.execute(
@@ -154,29 +153,34 @@ def kill_task(self, job_id: str):
         终止任务
         """
         tasks = self.task_persistent.query_tasks(job_id)
+        user = None
         for task in tasks:
-            self.kill_one_task(task.worker_id)
+            user = self.kill_one_task(task.worker_id)
+        self.logger.info(f"kill_task, job {job_id} killed, upload the log")
+        if user is None:
+            return
         self.logger.info(LOG_END_FLAG_FORMATTER.format(
             job_id=job_id))
         # upload the log
-        self.logger.info(f"kill_task, job {job_id} killed, upload the log")
-        self.log_retriever.upload_log(job_id)
+        self.log_retriever.upload_log(job_id, user)
 
     def kill_one_task(self, task_id: str):
         task_result = None
         with self._rw_lock.gen_rlock():
             if task_id not in self._tasks.keys() or self._tasks[task_id].task_status != TaskStatus.RUNNING.value:
-                return
+                return None
             task_result = self._tasks[task_id]
         self.logger.info(f"Kill task, task_id: {task_id}")
         self._async_executor.kill(task_id)
-
+        user = task_result.user
         # persistent the status to killed
         with self._rw_lock.gen_wlock():
             task_result.task_status = TaskStatus.KILLED.value
             self.task_persistent.on_task_finished(task_result)
             self._tasks.pop(task_id)
-        self.logger.info(f"Kill task success, task_id: {task_id}")
+        self.logger.info(
+            f"Kill task success, task_id: {task_id}, user: {user}")
+        return user
 
     def task_finished(self, task_id: str) -> bool:
         (status, _, _) = self.status(task_id)
@@ -234,7 +238,8 @@ def _on_task_finish(self, task_id: str, is_succeeded: bool, error_msg: str = Non
             if self.task_persistent.job_finished(task_result.job_id):
                 self.logger.info(
                     f"_on_task_finish: all sub tasks finished, upload the log, task_info: {task_result}")
-                self.log_retriever.upload_log(task_result.job_id)
+                self.log_retriever.upload_log(
+                    task_result.job_id, task_result.user)
         finally:
             for handler in self._task_clear_handler:
                 handler(task_id)
diff --git a/python/wedpr_ml_toolkit/test/UserGuide.md b/python/wedpr_ml_toolkit/test/UserGuide.md
index 47a72092..d01f7391 100644
--- a/python/wedpr_ml_toolkit/test/UserGuide.md
+++ b/python/wedpr_ml_toolkit/test/UserGuide.md
@@ -8,12 +8,12 @@
 ```
 access_key_id=
 access_key_secret=
-remote_entrypoints=http://139.159.202.235:8005,http://139.159.202.235:8006
+remote_entrypoints=http://127.0.0.1:8005,http://127.0.0.1:8006
 
-agency_name=SGD
-workspace_path=/user/ppc/milestone2/sgd/
+agency_name=WeBank
+workspace_path=/user/ppc/webank/
 user=test_user
-storage_endpoint=http://192.168.0.18:50070
+storage_endpoint=http://127.0.0.1:50070
 ```
 
 3. 通过前端页面登录，例如：http://139.159.202.235:8005/
diff --git a/python/wedpr_ml_toolkit/test/config.properties b/python/wedpr_ml_toolkit/test/config.properties
index ac88fcad..8e8aaa5b 100644
--- a/python/wedpr_ml_toolkit/test/config.properties
+++ b/python/wedpr_ml_toolkit/test/config.properties
@@ -2,7 +2,7 @@ access_key_id=
 access_key_secret=
 remote_entrypoints=http://127.0.0.1:16000,http://127.0.0.1:16001
 
-agency_name=SGD
-workspace_path=/user/wedpr/milestone2/sgd/
+agency_name=WeBank
+workspace_path=/user/wedpr/webank/
 user=test_user
 storage_endpoint=http://127.0.0.1:50070

From 735f8bfacf3e796cb7fab48c87b71963f41cca37 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 27 Nov 2024 18:34:23 +0800
Subject: [PATCH 105/120] update the meta information when save_values (#104)

* refactor all path of result files to result_context

* update the meta information when save_values

* complement result handler
---
 python/wedpr_ml_toolkit/setup.py              |   2 +-
 .../wedpr_ml_toolkit/test/test_ml_toolkit.py  |  72 ++++--
 .../wedpr_ml_toolkit/common/utils/constant.py |  14 +-
 .../config/wedpr_ml_config.py                 |  12 +-
 .../wedpr_ml_toolkit/context/data_context.py  |  25 +-
 .../context/dataset_context.py                |  26 +-
 .../wedpr_ml_toolkit/context/job_context.py   | 238 +++++++-----------
 .../context/result/fe_result_context.py       |  34 ++-
 .../context/result/model_result_context.py    | 104 +++++---
 .../context/result/psi_result_context.py      |  24 +-
 .../context/result/result_context.py          |  57 ++++-
 .../transport/storage_entrypoint.py           |   3 +
 .../transport/wedpr_remote_dataset_client.py  |  22 +-
 .../transport/wedpr_remote_job_client.py      |  65 ++---
 .../wedpr_ml_toolkit/wedpr_ml_toolkit.py      |  39 ++-
 15 files changed, 417 insertions(+), 320 deletions(-)

diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
index ff256ff0..7e749442 100644
--- a/python/wedpr_ml_toolkit/setup.py
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr_ml_toolkit',
     packages=find_packages(),
-    version="1.0.0.dev-20241125",
+    version="1.0.0.dev-20241126",
     description="wedpr-ml-toolkit: The ML toolkit for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",
diff --git a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
index 6e87fa49..824d53c5 100644
--- a/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/test/test_ml_toolkit.py
@@ -2,13 +2,13 @@
 import unittest
 import numpy as np
 import pandas as pd
-from sklearn import metrics
 from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder
 from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit
 from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.context.job_context import JobType
 from wedpr_ml_toolkit.context.model_setting import PreprocessingSetting
+from wedpr_ml_toolkit.context.model_setting import ModelSetting
 
 
 class WeDPRMlToolkitTestWrapper:
@@ -25,6 +25,7 @@ def test_submit_job(self):
             # x1到x10列，随机数
             **{f'x{i}': np.random.rand(100) for i in range(1, 11)}
         })
+        # the dataset
         dataset1 = DatasetContext(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
                                   dataset_client=self.wedpr_ml_toolkit.get_dataset_client(),
                                   storage_workspace=self.wedpr_config.user_config.get_workspace_path(),
@@ -32,21 +33,11 @@ def test_submit_job(self):
                                   is_label_holder=True)
         dataset1.save_values(df, path='d-101')
 
-        # hdfs_path
+        # the dataset
         dataset2 = DatasetContext(storage_entrypoint=self.wedpr_ml_toolkit.get_storage_entry_point(),
                                   dataset_client=self.wedpr_ml_toolkit.get_dataset_client(),
                                   dataset_id="d-9743674298214405")
-
-        dataset2.storage_client = None
-        # dataset2.load_values()
-        if dataset2.storage_client is None:
-            # 支持更新dataset的values数据
-            df2 = pd.DataFrame({
-                'id': np.arange(0, 100),  # id列，顺序整数
-                # x1到x10列，随机数
-                **{f'z{i}': np.random.rand(100) for i in range(1, 11)}
-            })
-            dataset2.save_values(values=df2)
+        print(f"### dataset2 meta: {dataset2.dataset_meta}")
         if dataset1.storage_client is not None:
             # save values to dataset1
             dataset1.save_values(df)
@@ -70,6 +61,16 @@ def test_submit_job(self):
         psi_result = psi_job_context.fetch_job_result(psi_job_id, True)
         print(
             f"* fetch_job_result for psi job {psi_job_id} success, result: {psi_result}")
+        # build the psi result:
+        psi_result_ctx = self.wedpr_ml_toolkit.build_result_context(
+            psi_job_context, psi_result)
+        print(f"* psi_result_ctx: {psi_result_ctx}")
+        (psi_result_values, psi_result_columns,
+         psi_result_shape) = psi_result_ctx.result_dataset.load_values()
+        # obtain the intersection
+        print(
+            f"* psi result, psi_result_columns: {psi_result_columns}, "
+            f"psi_result_shape: {psi_result_shape}, psi_result_values: {psi_result_values}")
         # 初始化
         print(f"* build pre-processing data-context")
         preprocessing_data = DataContext(dataset1, dataset2)
@@ -77,13 +78,50 @@ def test_submit_job(self):
             JobType.PREPROCESSING, project_id, preprocessing_data, PreprocessingSetting())
         # 执行预处理任务
         print(f"* submit pre-processing job")
-        fe_job_id = preprocessing_job_context.submit()
-        print(f"* submit pre-processing job success, job_id: {fe_job_id}")
-        fe_result = preprocessing_job_context.fetch_job_result(fe_job_id, True)
+        preprocessing_job_id = preprocessing_job_context.submit()
+        print(
+            f"* submit pre-processing job success, job_id: {preprocessing_job_id}")
+        preprocessing_result = preprocessing_job_context.fetch_job_result(
+            preprocessing_job_id, True)
         print(
-            f"* fetch pre-processing job result success, job_id: {fe_job_id}, result: {fe_result}")
+            f"* fetch pre-processing job result success, job_id: {preprocessing_job_id}, result: {preprocessing_result}")
         print(preprocessing_job_context.participant_id_list,
               preprocessing_job_context.result_receiver_id_list)
+        # build the context
+        preprocessing_result_ctx = self.wedpr_ml_toolkit.build_result_context(preprocessing_job_context,
+                                                                              preprocessing_result)
+        print(
+            f"* preprocessing_result_ctx: {preprocessing_result_ctx.preprocessing_dataset}")
+        preprocessing_values, columns, shape = preprocessing_result_ctx.preprocessing_dataset.load_values()
+        print(
+            f"* preprocessing_result_dataset, columns: {columns}, shape: {shape}")
+        # test xgb job
+        xgb_data = DataContext(dataset1, dataset2)
+        model_setting = ModelSetting()
+        model_setting.use_psi = True
+        xgb_job_context = self.wedpr_ml_toolkit.build_job_context(
+            job_type=JobType.XGB_TRAINING, project_id=project_id,
+            dataset=xgb_data,
+            model_setting=model_setting, id_fields="id")
+        print(f"* construct xgb job context: participant_id_list: {xgb_job_context.participant_id_list}, "
+              f"result_receiver_id_list: {xgb_job_context.result_receiver_id_list}")
+        xgb_job_id = xgb_job_context.submit()
+        print(f"* submit xgb job success, {xgb_job_id}")
+        xgb_job_result = xgb_job_context.fetch_job_result(xgb_job_id, True)
+        print(f"* xgb job result: {xgb_job_result}")
+        xgb_job_context = self.wedpr_ml_toolkit.build_result_context(
+            job_context=xgb_job_context, job_result_detail=xgb_job_result)
+        print(f"* xgb job result: {xgb_job_context}")
+        # load the feature_importance information
+        (feature_importance_value, feature_importance_cols, feature_importance_shape) = \
+            xgb_job_context.feature_importance_dataset.load_values()
+        print(f"* xgb feature importance information: {feature_importance_cols}, "
+              f"{feature_importance_shape}, {feature_importance_value}")
+        # load the evaluation information
+        (evaluation_value, evaluation_cols, evaluation_shape) = \
+            xgb_job_context.evaluation_dataset.load_values()
+        print(f"* xgb evaluation information: {evaluation_cols}, "
+              f"{evaluation_shape}, {evaluation_value}")
 
     def test_query_job(self, job_id: str, block_until_finish):
         job_result = self.wedpr_ml_toolkit.query_job_status(
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
index b9e227d2..626b2c21 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/constant.py
@@ -7,11 +7,17 @@ class Constant:
     DEFAULT_SUBMIT_JOB_URI = f'{WEDPR_API_PREFIX}project/submitJob'
     DEFAULT_QUERY_JOB_STATUS_URL = f'{WEDPR_API_PREFIX}project/queryJobByCondition'
     DEFAULT_QUERY_JOB_DETAIL_URL = f'{WEDPR_API_PREFIX}scheduler/queryJobDetail'
+    # the dataset related url
     DEFAULT_QUERY_DATASET_URL = f'{WEDPR_API_PREFIX}dataset/queryDataset'
+    DEFAULT_UPDATED_DATASET_URL = f'{WEDPR_API_PREFIX}dataset/updateDatasetMeta'
     PSI_RESULT_FILE = "psi_result.csv"
 
     FEATURE_BIN_FILE = "feature_bin.json"
-    TEST_MODEL_OUTPUT_FILE = "test_output.csv"
-    TRAIN_MODEL_OUTPUT_FILE = "train_output.csv"
-
-    FE_RESULT_FILE = "fe_result.csv"
+    XGB_TREE_PREFIX = "xgb_tree"
+    MODEL_RESULT_FILE = XGB_TREE_PREFIX + '.json'
+    PREPROCESSING_RESULT_FILE = "preprocessing_result.csv"
+    EVALUATION_TABLE_FILE = "mpc_xgb_evaluation_table.csv"
+    FEATURE_IMPORTANCE_FILE = "xgb_result_feature_importance_table.csv"
+    FEATURE_SELECTION_FILE = "xgb_result_column_info_selected.csv"
+    MODEL_FILE = "model_enc.kpl"
+    WOE_IV_FILE = "woe_iv.csv"
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
index 10b77ed2..f6648aad 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
@@ -32,8 +32,11 @@ def __init__(self, polling_interval_s: int = 5, max_retries: int = 2, retry_dela
 
 
 class DatasetConfig(BaseObject):
-    def __init__(self, query_dataset_uri=Constant.DEFAULT_QUERY_DATASET_URL):
+    def __init__(self,
+                 query_dataset_uri=Constant.DEFAULT_QUERY_DATASET_URL,
+                 update_dataset_uri=Constant.DEFAULT_UPDATED_DATASET_URL):
         self.query_dataset_uri = query_dataset_uri
+        self.update_dataset_uri = update_dataset_uri
 
 
 class StorageConfig(BaseObject):
@@ -56,11 +59,6 @@ def __init__(self, timeout_seconds=3):
         self.timeout_seconds = timeout_seconds
 
 
-class AgencyConfig(BaseObject):
-    def __init__(self, agency_name=None):
-        self.agency_name = agency_name
-
-
 class WeDPRMlConfig:
     def __init__(self, config_dict):
         self.auth_config = AuthConfig()
@@ -73,8 +71,6 @@ def __init__(self, config_dict):
         self.user_config.set_params(**config_dict)
         self.http_config = HttpConfig()
         self.http_config.set_params(**config_dict)
-        self.agency_config = AgencyConfig()
-        self.agency_config.set_params(**config_dict)
         self.dataset_config = DatasetConfig()
 
 
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
index 3e4ee46b..b7d029b9 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/data_context.py
@@ -9,23 +9,6 @@ class DataContext:
     def __init__(self, *datasets):
         self.datasets = list(datasets)
 
-        self._check_datasets()
-
-    def _save_dataset(self, dataset: DatasetContext):
-        file_path = dataset.dataset_meta.file_path
-        if file_path is None:
-            dataset.dataset_id = utils.make_id(
-                utils.IdPrefixEnum.DATASET.value)
-            file_path = os.path.join(
-                dataset.storage_workspace, dataset.dataset_id)
-            if dataset.storage_client is not None:
-                dataset.storage_client.upload(
-                    dataset.values, file_path)
-
-    def _check_datasets(self):
-        for dataset in self.datasets:
-            self._save_dataset(dataset)
-
     def to_psi_format(self, merge_filed, result_receiver_id_list):
         dataset_psi = []
         for dataset in self.datasets:
@@ -40,14 +23,10 @@ def to_psi_format(self, merge_filed, result_receiver_id_list):
 
     def __generate_dataset_info__(self, id_field: str, receive_result: bool, label_provider: bool, dataset: DatasetContext):
         return {"idFields": [id_field],
-                "dataset": {"owner": dataset.dataset_meta.ownerUserName,
-                            "ownerAgency": dataset.dataset_meta.ownerAgencyName,
-                            "path": dataset.dataset_meta.file_path,
-                            "storageTypeStr": "HDFS",
+                "dataset": {"ownerAgency": dataset.dataset_meta.ownerAgencyName,
                             "datasetID": dataset.dataset_id},
                 "receiveResult": receive_result,
-                "labelProvider": label_provider
-                }
+                "labelProvider": label_provider}
 
     def to_model_formort(self, merge_filed, result_receiver_id_list):
         dataset_model = []
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
index 14243fa1..dfb9eb71 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
@@ -3,6 +3,7 @@
 from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
 from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import WeDPRDatasetClient
 from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+import io
 
 
 class DatasetContext:
@@ -27,6 +28,10 @@ def __init__(self,
         # the storage workspace
         self.storage_workspace = storage_workspace
 
+    def __repr__(self):
+        return f"dataset_id: {self.dataset_id}, " \
+               f"dataset_meta: {self.dataset_meta}"
+
     def load_values(self, header=None):
         # 加载hdfs的数据集
         if self.storage_client is not None:
@@ -37,6 +42,25 @@ def load_values(self, header=None):
             return values, values.columns, values.shape
 
     def save_values(self, values: pd.DataFrame = None, path=None):
+        # no values to save
+        if values is None:
+            return
+        csv_buffer = io.StringIO()
+        values.to_csv(csv_buffer, index=False)
+        value_bytes = csv_buffer.getvalue()
+        # update the meta firstly
+        if path is None and self.dataset_meta is not None and self.dataset_meta.datasetId is not None:
+            columns = values.columns.to_list()
+            dataset_meta = DatasetMeta(dataset_id=self.dataset_meta.datasetId,
+                                       dataset_fields=','.join(columns),
+                                       dataset_size=len(value_bytes),
+                                       dataset_record_count=len(values),
+                                       dataset_column_count=len(columns))
+            self.dataset_client.update_dataset(dataset_meta)
+            self.dataset_meta.datasetFields = ','.join(columns)
+            self.dataset_meta.dataset_record_count = len(values)
+            self.dataset_meta.columnCount = len(columns)
+        # update the content
         target_path = self.dataset_meta.file_path
         # 保存数据到hdfs目录
         if path is not None:
@@ -47,7 +71,7 @@ def save_values(self, values: pd.DataFrame = None, path=None):
             target_path = os.path.join(
                 self.storage_workspace, target_path)
         if self.storage_client is not None:
-            self.storage_client.upload(values, target_path)
+            self.storage_client.upload_bytes(value_bytes, target_path)
 
     def update_path(self, path: str = None):
         # 将数据集存入hdfs相同路径，替换旧数据集
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
index 81321ca5..0fadb81a 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/job_context.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import json
 
-from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobParam
@@ -9,21 +8,26 @@
 from abc import abstractmethod
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobType, ModelType
-from wedpr_ml_toolkit.transport.wedpr_remote_job_client import ModelResult
 from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
 from wedpr_ml_toolkit.context.model_setting import ModelSetting
+from wedpr_ml_toolkit.config.wedpr_ml_config import UserConfig
 
 
 class JobContext:
 
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint, project_id: str, dataset: DataContext = None, my_agency=None):
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 dataset: DataContext,
+                 user_config: UserConfig):
         if dataset is None:
             raise Exception("Must define the job related datasets!")
+        self.dataset = dataset
+        self.user_config = user_config
+
         self.remote_job_client = remote_job_client
         self.storage_entry_point = storage_entry_point
         self.project_id = project_id
-        self.dataset = dataset
-        self.create_agency = my_agency
         self.participant_id_list = []
         self.task_parties = []
         self.dataset_id_list = []
@@ -32,7 +36,8 @@ def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point:
         self.label_columns = None
         self.__init_participant__()
         self.__init_label_information__()
-        self.result_receiver_id_list = [my_agency]  # 仅限jupyter所在机构
+        self.result_receiver_id_list = [
+            self.user_config.agency_name]  # 仅限jupyter所在机构
         self.__check__()
 
     def __check__(self):
@@ -76,23 +81,7 @@ def get_job_type(self) -> JobType:
     def submit(self):
         return self.remote_job_client.submit_job(self.build())
 
-    @abstractmethod
-    def parse_result(self, job_id, block_until_success):
-        pass
-
     def fetch_job_result(self, job_id, block_until_success):
-        # job_result = self.query_job_status(job_id, block_until_success)
-        # # TODO: determine success or not here
-        # return self.parse_result(self.remote_job_client.query_job_detail(job_id, block_until_success))
-
-        # # query_job_status
-        # job_result = self.remote_job_client.poll_job_result(job_id, block_until_success)
-        # # failed case
-        # if job_result == None or job_result.job_status == None or (not job_result.job_status.run_success()):
-        #     raise Exception(f'job {job_id} running failed!')
-        # # success case
-        # ...
-
         # query_job_detail
         result_detail = self.remote_job_client.query_job_detail(
             job_id, block_until_success)
@@ -100,9 +89,15 @@ def fetch_job_result(self, job_id, block_until_success):
 
 
 class PSIJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 dataset: DataContext,
+                 user_config: UserConfig,
+                 merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.merge_field = merge_field
 
     def get_job_type(self) -> JobType:
@@ -111,30 +106,21 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_psi_format(
             self.merge_field, self.result_receiver_id_list)
-        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list}))
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps({'dataSetList': self.dataset_list}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-
-        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user, agency=self.create_agency,
-                                   file_path=result_detail.resultFileInfo['path'])
-        psi_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                    storage_workspace=None,
-                                    dataset_meta=dataset_meta)
-
-        return psi_result
-
 
 class PreprocessingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
+    def __init__(self, remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
                  project_id: str, model_setting: ModelSetting,
-                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+                 dataset: DataContext, user_config: UserConfig,
+                 merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
 
@@ -144,23 +130,23 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps({'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-
-        pre_result = result_detail
-        return pre_result
-
 
 class FeatureEngineeringJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, model_setting: ModelSetting,
-                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 model_setting: ModelSetting,
+                 dataset: DataContext,
+                 user_config: UserConfig, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
 
@@ -170,23 +156,24 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps({'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-
-        fe_result = result_detail
-        return fe_result
-
 
 class SecureLGBMTrainingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, model_setting: ModelSetting,
-                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 model_setting: ModelSetting,
+                 dataset: DataContext,
+                 user_config: UserConfig,
+                 merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
 
@@ -196,37 +183,24 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-            {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps({'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                   file_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
-        train_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                      storage_workspace=None,
-                                      dataset_meta=dataset_meta)
-
-        test_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                        file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
-        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                     storage_workspace=None,
-                                     dataset_meta=test_dataset_meta)
-
-        xgb_result = ModelResult(job_id, train_result, test_result,
-                                 result_detail.model, ModelType.XGB_MODEL_SETTING.name)
-        return xgb_result
-
 
 class SecureLGBMPredictJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, model_setting: ModelSetting, predict_algorithm,
-                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 model_setting: ModelSetting, predict_algorithm,
+                 dataset: DataContext,
+                 user_config: UserConfig,
+                 merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
         self.predict_algorithm = predict_algorithm
@@ -237,31 +211,24 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        # job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
-        #     {'dataSetList': self.dataset_list}).replace('"', '\\"'))
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id, param=json.dumps(
             {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict(),
              'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                   file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'],
-                                   agency=self.create_agency)
-        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                     dataset_meta=dataset_meta)
-
-        xgb_result = ModelResult(job_id, test_result=test_result)
-        return xgb_result
-
 
 class SecureLRTrainingJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, model_setting:  ModelSetting,
-                 dataset: DataContext = None, my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 model_setting:  ModelSetting,
+                 dataset: DataContext,
+                 user_config: UserConfig, merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
 
@@ -271,34 +238,25 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps(
             {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict()}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
 
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-        train_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                         file_path=result_detail.modelResultDetail['ModelResult']['trainResultPath'], agency=self.create_agency)
-        train_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                      dataset_meta=train_dataset_meta)
-
-        test_dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                        file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
-        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                     storage_workspace=None,
-                                     dataset_meta=test_dataset_meta)
-
-        lr_result = ModelResult(job_id, train_result, test_result,
-                                result_detail.model, ModelType.LR_MODEL_SETTING.name)
-        return lr_result
-
 
 class SecureLRPredictJobContext(JobContext):
-    def __init__(self, remote_job_client: WeDPRRemoteJobClient, storage_entry_point: StorageEntryPoint,
-                 project_id: str, model_setting: ModelSetting, predict_algorithm, dataset: DataContext = None,
-                 my_agency=None, merge_field: str = 'id'):
-        super().__init__(remote_job_client, storage_entry_point, project_id, dataset, my_agency)
+    def __init__(self,
+                 remote_job_client: WeDPRRemoteJobClient,
+                 storage_entry_point: StorageEntryPoint,
+                 project_id: str,
+                 model_setting: ModelSetting, predict_algorithm,
+                 dataset: DataContext,
+                 user_config: UserConfig,
+                 merge_field: str = 'id'):
+        super().__init__(remote_job_client, storage_entry_point,
+                         project_id, dataset, user_config)
         self.model_setting = model_setting
         self.merge_field = merge_field
         self.predict_algorithm = predict_algorithm
@@ -309,18 +267,10 @@ def get_job_type(self) -> JobType:
     def build(self) -> JobParam:
         self.dataset_list = self.dataset.to_model_formort(
             self.merge_field, self.result_receiver_id_list)
-        job_info = JobInfo(job_type=self.get_job_type(), project_id=self.project_id, param=json.dumps(
+        job_info = JobInfo(job_type=self.get_job_type(),
+                           project_id=self.project_id,
+                           param=json.dumps(
             {'dataSetList': self.dataset_list, 'modelSetting': self.model_setting.as_dict(),
              'modelPredictAlgorithm': json.dumps(self.predict_algorithm)}))
         job_param = JobParam(job_info, self.task_parties, self.dataset_id_list)
         return job_param
-
-    def parse_result(self, job_id, block_until_success):
-        result_detail = self.fetch_job_result(job_id, block_until_success)
-        dataset_meta = DatasetMeta(user=self.storage_entry_point.user_config.user,
-                                   file_path=result_detail.modelResultDetail['ModelResult']['testResultPath'], agency=self.create_agency)
-        test_result = DatasetContext(storage_entrypoint=self.storage_entry_point,
-                                     dataset_meta=dataset_meta)
-
-        lr_result = ModelResult(job_id, test_result=test_result)
-        return lr_result
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
index 81512241..f1fef48f 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
@@ -4,19 +4,33 @@
 from wedpr_ml_toolkit.common.utils.constant import Constant
 from wedpr_ml_toolkit.context.result.result_context import ResultContext
 from wedpr_ml_toolkit.context.job_context import JobContext
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
+
+
+class PreprocessingResultContext(ResultContext):
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.preprocessing_dataset: DatasetContext = None
+        super().__init__(job_context, job_result_detail)
+
+    def __repr__(self):
+        return f"preprocessing_dataset: {self.preprocessing_dataset}"
+
+    def parse_result(self):
+        self.preprocessing_dataset = self._generate_result_dataset_(
+            self.preprocessing_result_path)
 
 
 class FeResultContext(ResultContext):
 
-    def __init__(self, job_context: JobContext, job_id: str):
-        super().__init__(job_context, job_id)
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.fe_dataset: DatasetContext = None
+        super().__init__(job_context, job_result_detail)
+
+    def __repr__(self):
+        return f"fe_dataset: {self.fe_dataset}"
 
     def parse_result(self):
-        result_list = []
-        for dataset in self.job_context.dataset.datasets:
-            dataset.update_path(os.path.join(
-                self.job_id, Constant.FE_RESULT_FILE))
-            result_list.append(dataset)
-
-        fe_result = DataContext(*result_list)
-        return fe_result
+        self.fe_dataset = self._generate_result_dataset_(
+            self.fe_result_file_path)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
index c7b3bd48..e9d534e2 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
@@ -1,51 +1,73 @@
-import os
-import numpy as np
-
-from ppc_common.ppc_utils import utils
 from wedpr_ml_toolkit.context.result.result_context import ResultContext
-from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
-from wedpr_ml_toolkit.common.utils.constant import Constant
+from wedpr_ml_toolkit.context.result.fe_result_context import PreprocessingResultContext
+from wedpr_ml_toolkit.context.result.fe_result_context import FeResultContext
 from wedpr_ml_toolkit.context.job_context import JobContext
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 
 
-class ModelResultContext(ResultContext):
-    def __init__(self, job_context: JobContext, job_id: str, storage_entrypoint: StorageEntryPoint):
-        super().__init__(job_context, job_id)
-        self.storage_entrypoint = storage_entrypoint
+class ModelResultContext(PreprocessingResultContext, FeResultContext):
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        super().__init__(job_context, job_result_detail)
+        PreprocessingResultContext.__init__(
+            self, job_context, job_result_detail)
+        FeResultContext.__init__(self, job_context, job_result_detail)
 
 
-class SecureLGBMResultContext(ModelResultContext):
-    MODEL_DATA_FILE = utils.XGB_TREE_PERFIX + '.json'
+class TrainResultContext(ModelResultContext):
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.train_result_dataset: DatasetContext = None
+        self.test_result_dataset: DatasetContext = None
+        self.model_result_dataset: DatasetContext = None
+        self.model_dataset: DatasetContext = None
+        self.evaluation_dataset: DatasetContext = None
+        self.feature_selection_dataset: DatasetContext = None
+        self.feature_importance_dataset: DatasetContext = None
+        super().__init__(job_context, job_result_detail)
 
-    def __init__(self, job_context: JobContext, job_id: str, storage_entrypoint: StorageEntryPoint):
-        super().__init__(job_context, job_id, storage_entrypoint)
+    def __repr__(self):
+        return f"train_result_dataset: {self.train_result_dataset}," \
+               f"test_result_dataset: {self.test_result_dataset}," \
+               f"model_result_dataset: {self.model_result_dataset}," \
+               f"model_dataset: {self.model_dataset}," \
+               f"evaluation_dataset: {self.evaluation_dataset}," \
+               f"feature_selection_dataset: {self.feature_selection_dataset}," \
+               f"feature_importance_dataset: {self.feature_importance_dataset}," \
+               f"preprocessing_dataset: {self.preprocessing_dataset}," \
+               f"fe_dataset: {self.fe_dataset}," \
+               f"psi_dataset: {self.psi_result_file_path}"
 
     def parse_result(self):
+        # train_model_output
+        self.train_result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.modelResultDetail['ModelResult']['trainResultPath'])
+        #  test_model_output
+        self.test_result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.modelResultDetail['ModelResult']['testResultPath'])
+        # xgb_tree.json
+        self.model_result_dataset = self._generate_result_dataset_(
+            self.model_result_path)
+        # model_enc.kpl
+        self.model_dataset = self._generate_result_dataset_(self.model_path)
+        # mpc_xgb_evaluation_table.csv
+        self.evaluation_dataset = self._generate_result_dataset_(
+            self.evaluation_table_path)
+        # xgb_result_column_info_selected.csv
+        self.feature_selection_dataset = self._generate_result_dataset_(
+            self.feature_selection_file_path)
+        # xgb_result_feature_importance_table.csv
+        self.feature_importance_dataset = self._generate_result_dataset_(
+            self.feature_importance_result_path)
+
+
+class PredictResultContext(ResultContext):
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.model_result_dataset: DatasetContext = None
+        super().__init__(job_context, job_result_detail)
 
-        # train_praba, test_praba, train_y, test_y, feature_importance, split_xbin, trees, params
-        # 从hdfs读取结果文件信息，构造为属性
-        train_praba_path = os.path.join(
-            self.job_id, Constant.TRAIN_MODEL_OUTPUT_FILE)
-        test_praba_path = os.path.join(
-            self.job_id, Constant.TEST_MODEL_OUTPUT_FILE)
-        train_output = self.storage_entrypoint.download(train_praba_path)
-        test_output = self.storage_entrypoint.download(test_praba_path)
-        self.train_praba = train_output['class_pred'].values
-        self.test_praba = test_output['class_pred'].values
-        if 'class_label' in train_output.columns:
-            self.train_y = train_output['class_label'].values
-            self.test_y = test_output['class_label'].values
-        else:
-            self.train_y = None
-            self.test_y = None
-
-        feature_bin_path = os.path.join(self.job_id, Constant.FEATURE_BIN_FILE)
-        model_path = os.path.join(self.job_id, self.MODEL_DATA_FILE)
-        feature_bin_data = self.storage_entrypoint.download_data(
-            feature_bin_path)
-        model_data = self.storage_entrypoint.download_data(model_path)
-
-        self.feature_importance = ...
-        self.split_xbin = feature_bin_data
-        self.trees = model_data
-        self.params = ...
+    def __repr__(self):
+        return f"model_result_dataset: {self.model_result_dataset}"
+
+    def parse_result(self):
+        self.model_result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.modelResultDetail['ModelResult']['testResultPath'])
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
index f5a94113..67d5e3f1 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
@@ -1,24 +1,18 @@
 # -*- coding: utf-8 -*-
-import os
-
 from wedpr_ml_toolkit.context.job_context import JobContext
-from wedpr_ml_toolkit.context.data_context import DataContext
-from wedpr_ml_toolkit.common.utils.constant import Constant
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
 from wedpr_ml_toolkit.context.result.result_context import ResultContext
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
 
 
 class PSIResultContext(ResultContext):
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.result_dataset: DatasetContext = None
+        super().__init__(job_context, job_result_detail)
 
-    PSI_RESULT_FILE = "psi_result.csv"
-
-    def __init__(self, job_context: JobContext, job_id: str):
-        super().__init__(job_context, job_id)
+    def __repr__(self):
+        return f"result_dataset: {self.result_dataset}"
 
     def parse_result(self):
-        result_list = []
-        for dataset in self.job_context.dataset.datasets:
-            dataset.update_path(os.path.join(
-                self.job_id, Constant.PSI_RESULT_FILE))
-            result_list.append(dataset)
-
-        self.psi_result = DataContext(*result_list)
+        self.result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.resultFileInfo['path'])
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
index dcb0670a..bc313468 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
@@ -1,14 +1,67 @@
 # -*- coding: utf-8 -*-
 from wedpr_ml_toolkit.context.job_context import JobContext
+from wedpr_ml_toolkit.common.utils.constant import Constant
 from abc import abstractmethod
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
+import os
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+from wedpr_ml_toolkit.context.data_context import DatasetContext
 
 
 class ResultContext:
-    def __init__(self, job_context: JobContext, job_id: str):
-        self.job_id = job_id
+    def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        self.check_and_get_job_type(job_context, job_result_detail)
         self.job_context = job_context
+        self.job_result_detail = job_result_detail
+        self.job_id = self.job_result_detail.job_object.id
+        # init the result file path
+        self.model_base_dir = os.path.join(
+            "share", "jobs", "model", self.job_id)
+
+        self.feature_bin_file_path = os.path.join(
+            self.model_base_dir, Constant.FEATURE_BIN_FILE)
+        self.fe_result_file_path = os.path.join(
+            self.model_base_dir, Constant.WOE_IV_FILE)
+        self.model_result_path = os.path.join(
+            self.model_base_dir, Constant.MODEL_RESULT_FILE)
+        self.preprocessing_result_path = os.path.join(
+            self.model_base_dir, Constant.PREPROCESSING_RESULT_FILE)
+        self.feature_importance_result_path = os.path.join(
+            self.model_base_dir, Constant.FEATURE_IMPORTANCE_FILE)
+        self.evaluation_table_path = os.path.join(
+            self.model_base_dir, Constant.EVALUATION_TABLE_FILE)
+        self.feature_selection_file_path = os.path.join(
+            self.model_base_dir, Constant.FEATURE_SELECTION_FILE)
+        self.model_path = os.path.join(
+            self.model_base_dir, Constant.MODEL_FILE)
+        self.woe_iv_path = os.path.join(
+            self.model_base_dir, Constant.WOE_IV_FILE)
+
+        self.psi_result_file_path = os.path.join(
+            self.job_context.user_config.user, "share", "psi", f"psi-{self.job_id}")
+
         self.parse_result()
 
+    @staticmethod
+    def check_and_get_job_type(job_context: JobContext, job_result_detail: JobDetailResponse):
+        if job_context is None:
+            raise Exception(
+                "Invalid empty job_context!")
+        if job_result_detail is None:
+            raise Exception("Invalid empty job result!")
+        if job_result_detail.job_object is None:
+            raise Exception(
+                "Invalid job result: must contain the job information!")
+        return job_result_detail.job_object.jobType
+
+    def _generate_result_dataset_(self, dataset_file_path):
+        dataset_meta = DatasetMeta(user=self.job_context.storage_entry_point.user_config.user,
+                                   agency=self.job_context.user_config.agency_name,
+                                   file_path=dataset_file_path)
+        return DatasetContext(storage_entrypoint=self.job_context.storage_entry_point,
+                              storage_workspace=None,
+                              dataset_meta=dataset_meta)
+
     @abstractmethod
     def parse_result(self):
         pass
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
index 371ef823..092b1de0 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
@@ -13,6 +13,9 @@ def __init__(self, user_config: UserConfig, storage_config: StorageConfig):
         self.storage_client = HdfsStorageImpl(
             self.storage_config.storage_endpoint, self.user_config.user, self.user_config.get_workspace_path())
 
+    def upload_bytes(self, data, hdfs_path):
+        self.storage_client.save_data(data, hdfs_path)
+
     def upload(self, dataframe, hdfs_path):
         """
         上传Pandas DataFrame到HDFS
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
index 1e69833b..166529a6 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
@@ -36,8 +36,8 @@ def __init__(self, dataset_id: str = None,
         self.datasetFields = dataset_fields
         self.datasetHash = dataset_hash
         self.datasetSize = dataset_size
-        self.datasetRecordCount = dataset_record_count
-        self.datasetColumnCount = dataset_column_count
+        self.recordCount = dataset_record_count
+        self.columnCount = dataset_column_count
         self.datasetStorageType = dataset_storage_type
         self.datasetStoragePath = dataset_storage_path
         self.dataSourceType = datasource_type
@@ -55,8 +55,8 @@ def __init__(self, dataset_id: str = None,
 
     def __repr__(self):
         return f"dataset: {self.datasetId}, datasetTitle: {self.datasetTitle}, datasetFields: {self.datasetFields}, " \
-               f"datasetSize: {self.datasetSize}, datasetRecordCount: {self.datasetRecordCount}," \
-               f"datasetColumnCount: {self.datasetColumnCount}, datasetStorageType: {self.datasetStorageType}" \
+               f"datasetSize: {self.datasetSize}, recordCount: {self.recordCount}," \
+               f"columnCount: {self.columnCount}, datasetStorageType: {self.datasetStorageType}, " \
                f"ownerAgencyName: {self.ownerAgencyName}"
 
 
@@ -84,3 +84,17 @@ def query_dataset(self, dataset_id) -> DatasetMeta:
                 f"Query dataset information failed for {wedpr_response.msg}, dataset_id: {dataset_id}")
         # query success, deserialize to DatasetMeta
         return DatasetMeta(**wedpr_response.data)
+
+    def update_dataset(self, dataset_meta: DatasetMeta):
+        if dataset_meta.datasetId is None or len(dataset_meta.datasetId) == 0:
+            raise Exception(
+                f"Invalid dataset meta {dataset_meta}, must define the datasetId")
+        response_dict = self.send_request(is_post=True,
+                                          uri=self.dataset_config.update_dataset_uri,
+                                          headers=None,
+                                          params=None,
+                                          data=json.dumps(dataset_meta.as_dict()))
+        wedpr_response = WeDPRResponse(**response_dict)
+        if wedpr_response.success() is False:
+            raise Exception(
+                f"Update dataset meta {dataset_meta} failed, msg: {wedpr_response.msg}")
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
index 1281313e..4356c91d 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
@@ -60,16 +60,18 @@ def get_job_status(job_status_tr: str):
 
 
 class JobInfo(BaseObject):
-    def __init__(self, job_id: str = None, job_type: JobType = None, project_id: str = None, param: str = None, **params: Any):
+    def __init__(self, job_id: str = None,
+                 job_type: JobType = None,
+                 project_id: str = None,
+                 param: str = None, **params: Any):
         if job_id is not None:
             self.id = job_id
         self.name = None
         self.owner = None
         self.ownerAgency = None
+        self.jobType: str = None
         if job_type is not None:
             self.jobType = job_type.name
-        else:
-            self.jobType = None
         self.parties = None
         self.projectId = project_id
         self.param = param
@@ -78,35 +80,11 @@ def __init__(self, job_id: str = None, job_type: JobType = None, project_id: str
         self.createTime = None
         self.lastUpdateTime = None
         self.set_params(**params)
-        self.job_status = JobStatus.get_job_status(self.status)
+        self.job_status: JobStatus = JobStatus.get_job_status(self.status)
 
     def __repr__(self):
-        return f"job_id: {self.id}, owner: {self.owner}, ownerAgency: {self.ownerAgency}, jobType: {self.jobType}, status: {self.status}"
-
-
-class ModelInfo(BaseObject):
-    def __init__(self, model, model_type, **params: Any):
-
-        self.type = model_type
-        # self.setting = json.loads(model)
-        self.setting = model
-        self.startTime = None
-        self.endTime = None
-        self.step = None
-        self.id = None
-        self.name = None
-        self.agency = None
-        self.owner = None
-
-        self.set_params(**params)
-
-
-class ModelResult:
-    def __init__(self, job_id: str, train_result=None, test_result=None, model=None, model_type=None):
-        self.job_id = job_id
-        self.train_result = train_result
-        self.test_result = test_result
-        self.model = ModelInfo(model, model_type).__dict__
+        return f"job_id: {self.id}, owner: {self.owner}, ownerAgency: {self.ownerAgency}, " \
+               f"jobType: {self.jobType}, status: {self.status}"
 
 
 class JobParam:
@@ -128,7 +106,7 @@ def success(self):
 
 
 class QueryJobRequest(BaseObject):
-    def __init__(self, job_info):
+    def __init__(self, job_info: JobInfo):
         self.job = job_info
 
     def as_dict(self):
@@ -151,25 +129,26 @@ def __init__(self, **params: Any):
         for job_item in self.jobs:
             self.job_object_list.append(JobInfo(**job_item))
 
-    def get_queried_job(self):
+    def get_queried_job(self) -> JobInfo:
         if len(self.job_object_list) == 0:
             return None
         return self.job_object_list[0]
 
 
 class JobDetailResponse(BaseObject):
-    def __init__(self, job: JobInfo = None, **params: Any):
-        self.job = None
-        self.job_object = job
-        self.modelResultDetail = None
-        self.resultFileInfo = None
-        self.model = None
+    def __init__(self, job_info: JobInfo = None, **params: Any):
+        self.job: dict = None
+        self.job_object: JobInfo = job_info
+        self.modelResultDetail: dict = None
+        self.resultFileInfo: dict = None
+        self.model: dict = None
         self.set_params(**params)
         if self.job_object is None and self.job is not None:
             self.job_object = JobInfo(**self.job)
 
     def __repr__(self):
-        return f"job: {self.job_object}, modelResultDetail: {self.modelResultDetail}, resultFileInfo: {self.resultFileInfo}, model: {self.model}"
+        return f"job: {self.job_object}, modelResultDetail: {self.modelResultDetail}, " \
+               f"resultFileInfo: {self.resultFileInfo}, model: {self.model}"
 
 
 class WeDPRRemoteJobClient(WeDPREntryPoint, BaseObject):
@@ -202,8 +181,9 @@ def submit_job(self, job_params: JobParam) -> WeDPRResponse:
     def query_job_detail(self, job_id, block_until_finish) -> JobDetailResponse:
         job_result = self.poll_job_result(job_id, block_until_finish)
         # failed case
-        if job_result == None or job_result.job_status == None or (not job_result.job_status.run_success()):
-            return JobDetailResponse(job=job_result, params=None)
+        if job_result == None or job_result.job_status == None \
+                or (not job_result.job_status.run_success()):
+            return JobDetailResponse(job_info=job_result, params=None)
         # success case, query the job detail
         params = {}
         params["jobID"] = job_id
@@ -228,7 +208,8 @@ def poll_job_result(self, job_id, block_until_finish) -> JobInfo:
                                                     self.job_config.retry_delay_s,
                                                     True,
                                                     self.job_config.query_job_status_uri,
-                                                    None, None, json.dumps(QueryJobRequest(job_info=query_condition).as_dict()))
+                                                    None, None,
+                                                    json.dumps(QueryJobRequest(job_info=query_condition).as_dict()))
             wedpr_response = WeDPRResponse(**response_dict)
             if not wedpr_response.success():
                 raise Exception(
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
index e091c31f..7f9c01c2 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
@@ -3,7 +3,6 @@
 from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfig
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import WeDPRRemoteJobClient
 from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobInfo
-from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
 from wedpr_ml_toolkit.transport.storage_entrypoint import StorageEntryPoint
 from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import WeDPRDatasetClient
 from wedpr_ml_toolkit.context.job_context import JobType
@@ -14,8 +13,16 @@
 from wedpr_ml_toolkit.context.job_context import SecureLGBMTrainingJobContext
 from wedpr_ml_toolkit.context.job_context import SecureLRPredictJobContext
 from wedpr_ml_toolkit.context.job_context import SecureLRTrainingJobContext
+from wedpr_ml_toolkit.context.job_context import JobContext
 from wedpr_ml_toolkit.context.data_context import DataContext
 from wedpr_ml_toolkit.context.model_setting import ModelSetting
+from wedpr_ml_toolkit.context.result.result_context import ResultContext
+from wedpr_ml_toolkit.context.result.psi_result_context import PSIResultContext
+from wedpr_ml_toolkit.context.result.fe_result_context import FeResultContext
+from wedpr_ml_toolkit.context.result.fe_result_context import PreprocessingResultContext
+from wedpr_ml_toolkit.context.result.model_result_context import TrainResultContext
+from wedpr_ml_toolkit.context.result.model_result_context import PredictResultContext
+from wedpr_ml_toolkit.transport.wedpr_remote_job_client import JobDetailResponse
 
 
 class WeDPRMlToolkit:
@@ -50,26 +57,42 @@ def query_job_status(self, job_id, block_until_finish=False) -> JobInfo:
     def query_job_detail(self, job_id, block_until_finish=False) -> JobDetailResponse:
         return self.remote_job_client.query_job_detail(job_id, block_until_finish)
 
+    def build_result_context(self, job_context: JobContext, job_result_detail: JobDetailResponse):
+        job_type = ResultContext.check_and_get_job_type(
+            job_context, job_result_detail)
+        if job_type == JobType.PSI.name:
+            return PSIResultContext(job_context, job_result_detail)
+        if job_type == JobType.PREPROCESSING.name:
+            return PreprocessingResultContext(job_context, job_result_detail)
+        if job_type == JobType.FEATURE_ENGINEERING.name:
+            return FeResultContext(job_context, job_result_detail)
+        if job_type == JobType.XGB_TRAINING.name or job_type == JobType.LR_TRAINING.name:
+            return TrainResultContext(job_context, job_result_detail)
+        if job_type == JobType.XGB_PREDICTING.name or job_type == JobType.LR_PREDICTING.name:
+            return PredictResultContext(job_context, job_result_detail)
+        raise Exception(f"Unsupported job type: {job_type}")
+
     def build_job_context(self, job_type: JobType, project_id: str, dataset: DataContext, model_setting: ModelSetting = None,
                           id_fields='id', predict_algorithm=None):
         if job_type == JobType.PSI:
             return PSIJobContext(self.remote_job_client, self.storage_entry_point, project_id, dataset,
-                                 self.config.agency_config.agency_name, id_fields)
+                                 self.config.user_config, id_fields)
         if job_type == JobType.PREPROCESSING:
             return PreprocessingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                           model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+                                           model_setting, dataset, self.config.user_config, id_fields)
         if job_type == JobType.FEATURE_ENGINEERING:
             return FeatureEngineeringJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+                                                model_setting, dataset, self.config.user_config, id_fields)
         if job_type == JobType.XGB_TRAINING:
             return SecureLGBMTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                                model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+                                                model_setting, dataset, self.config.user_config, id_fields)
         if job_type == JobType.XGB_PREDICTING:
             return SecureLGBMPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                               model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)
+                                               model_setting, predict_algorithm, dataset, self.config.user_config, id_fields)
         if job_type == JobType.LR_TRAINING:
             return SecureLRTrainingJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                              model_setting, dataset, self.config.agency_config.agency_name, id_fields)
+                                              model_setting, dataset, self.config.user_config, id_fields)
         if job_type == JobType.LR_PREDICTING:
             return SecureLRPredictJobContext(self.remote_job_client, self.storage_entry_point, project_id,
-                                             model_setting, predict_algorithm, dataset, self.config.agency_config.agency_name, id_fields)
+                                             model_setting, predict_algorithm, dataset, self.config.user_config, id_fields)
+        raise Exception(f"Unsupported job type: {job_type}")

From 6e884d71cbefeca1291b483f9fd1fd5bb6095a21 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Thu, 28 Nov 2024 00:25:53 +0800
Subject: [PATCH 106/120] fix dataset bugs (#105)

* fix problems found

* fix dataset bugs
---
 .../secure_model_base/secure_model_context.py |   1 +
 .../jupyter-demo/test_dastaset.py             |  81 +++++
 .../wedpr_ml_toolkit/jupyter-demo/test_ml.py  | 288 ++++++++++++++++++
 python/wedpr_ml_toolkit/setup.py              |   2 +-
 .../context/dataset_context.py                |  10 +-
 .../context/result/fe_result_context.py       |  14 +-
 .../context/result/model_result_context.py    |  43 +--
 .../context/result/psi_result_context.py      |   7 +-
 .../context/result/result_context.py          |   9 +-
 .../transport/wedpr_remote_dataset_client.py  |  13 +-
 .../transport/wedpr_remote_job_client.py      |   4 +
 11 files changed, 417 insertions(+), 55 deletions(-)
 create mode 100644 python/wedpr_ml_toolkit/jupyter-demo/test_dastaset.py
 create mode 100644 python/wedpr_ml_toolkit/jupyter-demo/test_ml.py

diff --git a/python/ppc_model/secure_model_base/secure_model_context.py b/python/ppc_model/secure_model_base/secure_model_context.py
index 0b68f773..a4a8d5ef 100644
--- a/python/ppc_model/secure_model_base/secure_model_context.py
+++ b/python/ppc_model/secure_model_base/secure_model_context.py
@@ -119,6 +119,7 @@ def __init__(self,
 
         model_predict_algorithm_str = common_func.get_config_value(
             "model_predict_algorithm", None, args, False)
+        self.model_predict_algorithm = None
         if model_predict_algorithm_str is not None:
             self.model_predict_algorithm = json.loads(
                 model_predict_algorithm_str)
diff --git a/python/wedpr_ml_toolkit/jupyter-demo/test_dastaset.py b/python/wedpr_ml_toolkit/jupyter-demo/test_dastaset.py
new file mode 100644
index 00000000..d8e8b918
--- /dev/null
+++ b/python/wedpr_ml_toolkit/jupyter-demo/test_dastaset.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[4]:
+
+
+import sys
+import numpy as np
+import pandas as pd
+from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder
+from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
+from wedpr_ml_toolkit.transport.wedpr_remote_dataset_client import DatasetMeta
+
+
+# In[5]:
+
+
+# 读取配置文件
+wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')
+wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)
+
+
+# In[8]:
+
+
+# 注册 dataset，支持两种方式: pd.Dataframe, hdfs_path
+# 1. pd.Dataframe
+df = pd.DataFrame({
+    'id': np.arange(0, 100),  # id列，顺序整数
+    'y': np.random.randint(0, 2, size=100),
+    # x1到x10列，随机数
+    **{f't{i}': np.random.rand(100) for i in range(1, 11)}
+})
+
+dataset1_meta = DatasetMeta(file_path = "d-01", user_config = wedpr_config.user_config)
+
+dataset1 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),
+                          storage_workspace=wedpr_config.user_config.get_workspace_path(),
+                          dataset_meta = dataset1_meta)
+print(f"* dataset1: {dataset1.dataset_meta}")
+dataset1.save_values(df)
+print(f"* updated dataset1: {dataset1}")
+
+(values, cols, shape) = dataset1.load_values()
+print(f"* load values result: {cols}, {shape}, {values}")
+
+
+# In[11]:
+
+
+# 2. hdfs_path
+dataset2 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(), 
+                          dataset_client=wedpr_ml_toolkit.dataset_client,
+                          storage_workspace=wedpr_config.user_config.get_workspace_path(), 
+                          dataset_id = "d-9866227816474629")
+print(f"* dataset2 meta: {dataset2}")
+
+# load values
+(values, cols, shape) = dataset2.load_values(header=0)
+print(f"* dataset2 detail, cols: {cols}, shape: {shape}, values: {values}")
+
+
+# 支持更新dataset的values数据
+df2 = pd.DataFrame({
+        'id': np.arange(0, 100),  # id列，顺序整数
+        **{f'w{i}': np.random.rand(100) for i in range(1, 11)}  # x1到x10列，随机数
+    })
+dataset2.save_values(values=df2)
+
+print(f"*** updated dataset2 meta: {dataset2}")
+
+(values, cols, shape) = dataset2.load_values(header=0)
+print(f"*** updated dataset2 detail, cols: {cols}, shape: {shape}, values: {values}")
+
+
+# In[ ]:
+
+
+
+
diff --git a/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py b/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py
new file mode 100644
index 00000000..6b4b81d6
--- /dev/null
+++ b/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py
@@ -0,0 +1,288 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import sys
+import numpy as np
+import pandas as pd
+from wedpr_ml_toolkit.config.wedpr_ml_config import WeDPRMlConfigBuilder
+from wedpr_ml_toolkit.wedpr_ml_toolkit import WeDPRMlToolkit
+from wedpr_ml_toolkit.context.dataset_context import DatasetContext
+from wedpr_ml_toolkit.context.data_context import DataContext
+from wedpr_ml_toolkit.context.job_context import JobType
+from wedpr_ml_toolkit.context.model_setting import ModelSetting
+from wedpr_ml_toolkit.context.result.model_result_context import PredictResultContext
+from wedpr_ml_toolkit.context.result.model_result_context import TrainResultContext
+from wedpr_ml_toolkit.context.result.model_result_context import PreprocessingResultContext
+
+
+# In[2]:
+
+
+# 读取配置文件
+wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')
+wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)
+
+
+# In[3]:
+
+
+# dataset1
+dataset1 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),
+                          dataset_client=wedpr_ml_toolkit.dataset_client,
+                          dataset_id = 'd-9743660607744005',
+                          is_label_holder=True)
+print(f"* load dataset1: {dataset1}")
+(values, cols, shapes) = dataset1.load_values()
+print(f"* dataset1 detail: {cols}, {shapes}")
+print(f"* dataset1 value: {values}")
+
+
+# In[4]:
+
+
+# dataset2
+dataset2 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),
+                          dataset_client = wedpr_ml_toolkit.dataset_client,
+                          dataset_id = "d-9743674298214405")
+print(f"* dataset2: {dataset2}")
+
+# 构建 dataset context
+dataset = DataContext(dataset1, dataset2)
+print(dataset.datasets)
+
+# init the job context
+project_id = "9737304249804806"
+
+# 构造xgb任务配置
+model_setting = ModelSetting()
+model_setting.use_psi = True
+xgb_job_context = wedpr_ml_toolkit.build_job_context(
+    job_type = JobType.XGB_TRAINING, 
+    project_id = project_id, 
+    dataset = dataset, 
+    model_setting = model_setting, 
+    id_fields = "id")
+print(f"* build xgb job context: {xgb_job_context}")
+
+
+# In[5]:
+
+
+# 执行xgb任务
+xgb_job_id = xgb_job_context.submit()
+print(xgb_job_id)
+
+
+# In[7]:
+
+
+# 获取xgb任务结果
+print(xgb_job_id)
+#xgb_job_id = "9868279583877126"
+xgb_result_detail = xgb_job_context.fetch_job_result(xgb_job_id, True)
+# load the result context
+xgb_result_context = wedpr_ml_toolkit.build_result_context(job_context=xgb_job_context,
+                                                           job_result_detail=xgb_result_detail)
+print(f"* xgb job result ctx: {xgb_result_context}")
+
+xgb_test_dataset = xgb_result_context.test_result_dataset
+print(f"* xgb_test_dataset: {xgb_test_dataset}, file_path: {xgb_test_dataset.dataset_meta.file_path}")
+
+(data, cols, shapes) = xgb_test_dataset.load_values()
+print(f"* test dataset detail, columns: {cols}, shape: {shapes}, value: {data}")
+
+
+# In[8]:
+
+
+# evaluation result
+result_context: TrainResultContext = xgb_result_context
+evaluation_result_dataset = result_context.evaluation_dataset
+(eval_data, cols, shape) = evaluation_result_dataset.load_values(header=0)
+print(f"* evaluation detail, col: {cols}, shape: {shape}, eval_data: {eval_data}")
+
+
+# In[9]:
+
+
+# feature importance 
+feature_importance_dataset = result_context.feature_importance_dataset
+(feature_importance_data, cols, shape) = feature_importance_dataset.load_values()
+
+print(f"* feature_importance detail, col: {cols}, shape: {shape}, feature_importance_data: {feature_importance_data}")
+
+
+# In[10]:
+
+
+# 预处理结果
+preprocessing_dataset = result_context.preprocessing_dataset
+(preprocessing_data, cols, shape) = preprocessing_dataset.load_values()
+
+print(f"* preprocessing detail, col: {cols}, shape: {shape}, preprocessing_data: {preprocessing_data}")
+
+
+# In[11]:
+
+
+# 建模结果
+model_result_dataset = result_context.model_result_dataset
+(model_result, cols, shape) = model_result_dataset.load_values()
+
+print(f"* model_result detail, col: {cols}, shape: {shape}, model_result: {model_result}")
+
+
+# In[12]:
+
+
+# 明文处理预测结果
+from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score
+import matplotlib.pyplot as plt
+
+# 提取真实标签和预测概率
+y_true = data['class_label']
+y_pred_proba = data['class_pred']
+y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5
+
+# 计算评估指标
+accuracy = accuracy_score(y_true, y_pred)
+precision = precision_score(y_true, y_pred)
+recall = recall_score(y_true, y_pred)
+f1 = f1_score(y_true, y_pred)
+auc = roc_auc_score(y_true, y_pred_proba)
+
+print(f"Accuracy: {accuracy:.2f}")
+print(f"Precision: {precision:.2f}")
+print(f"Recall: {recall:.2f}")
+print(f"F1 Score: {f1:.2f}")
+print(f"AUC: {auc:.2f}")
+
+# ROC 曲线
+fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
+plt.figure(figsize=(12, 5))
+
+# ROC 曲线
+plt.subplot(1, 2, 1)
+plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')
+plt.plot([0, 1], [0, 1], 'k--')
+plt.xlabel('False Positive Rate')
+plt.ylabel('True Positive Rate')
+plt.title('ROC Curve')
+plt.legend()
+
+# 精确率-召回率曲线
+precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)
+plt.subplot(1, 2, 2)
+plt.plot(recall_vals, precision_vals)
+plt.xlabel('Recall')
+plt.ylabel('Precision')
+plt.title('Precision-Recall Curve')
+
+plt.tight_layout()
+plt.show()
+
+
+# In[13]:
+
+
+# 构造xgb预测任务配置
+predict_setting = ModelSetting()
+predict_setting.use_psi = True
+#model_predict_algorithm = {}
+#model_predict_algorithm.update({"setting": xgb_result_context.job_result_detail.model})
+predict_xgb_job_context = wedpr_ml_toolkit.build_job_context(
+    job_type=JobType.XGB_PREDICTING, 
+    project_id = project_id, 
+    dataset= dataset, 
+    model_setting= predict_setting, 
+    id_fields = "id", 
+    predict_algorithm = xgb_result_context.job_result_detail.model_predict_algorithm)
+print(f"* predict_xgb_job_context: {predict_xgb_job_context}")
+
+
+# In[14]:
+
+
+# 执行xgb预测任务
+# xgb_job_id = '9868428439267334'  # 测试时跳过创建新任务过程
+xgb_predict_job_id = predict_xgb_job_context.submit()
+print(xgb_predict_job_id)
+
+
+# In[15]:
+
+
+# query the job detail
+print(f"* xgb_predict_job_id: {xgb_predict_job_id}")
+
+predict_xgb_job_result = predict_xgb_job_context.fetch_job_result(xgb_predict_job_id, True)
+
+# generate the result context
+result_context = wedpr_ml_toolkit.build_result_context(job_context=predict_xgb_job_context, 
+                                                       job_result_detail=predict_xgb_job_result)
+
+xgb_predict_result_context : PredictResultContext = result_context 
+print(f"* result_context is {xgb_predict_result_context}")
+
+
+# In[16]:
+
+
+# 明文处理预测结果
+from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score
+import matplotlib.pyplot as plt
+
+
+(data, cols, shapes) = xgb_predict_result_context.model_result_dataset.load_values(header = 0)
+
+# 提取真实标签和预测概率
+y_true = data['class_label']
+y_pred_proba = data['class_pred']
+y_pred = np.where(y_pred_proba >= 0.5, 1, 0)  # 二分类阈值设为0.5
+
+# 计算评估指标
+accuracy = accuracy_score(y_true, y_pred)
+precision = precision_score(y_true, y_pred)
+recall = recall_score(y_true, y_pred)
+f1 = f1_score(y_true, y_pred)
+auc = roc_auc_score(y_true, y_pred_proba)
+
+print(f"Accuracy: {accuracy:.2f}")
+print(f"Precision: {precision:.2f}")
+print(f"Recall: {recall:.2f}")
+print(f"F1 Score: {f1:.2f}")
+print(f"AUC: {auc:.2f}")
+
+# ROC 曲线
+fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
+plt.figure(figsize=(12, 5))
+
+# ROC 曲线
+plt.subplot(1, 2, 1)
+plt.plot(fpr, tpr, label=f'AUC = {auc:.2f}')
+plt.plot([0, 1], [0, 1], 'k--')
+plt.xlabel('False Positive Rate')
+plt.ylabel('True Positive Rate')
+plt.title('ROC Curve')
+plt.legend()
+
+# 精确率-召回率曲线
+precision_vals, recall_vals, _ = precision_recall_curve(y_true, y_pred_proba)
+plt.subplot(1, 2, 2)
+plt.plot(recall_vals, precision_vals)
+plt.xlabel('Recall')
+plt.ylabel('Precision')
+plt.title('Precision-Recall Curve')
+
+plt.tight_layout()
+plt.show()
+
+
+# In[ ]:
+
+
+
+
diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
index 7e749442..e78e60b7 100644
--- a/python/wedpr_ml_toolkit/setup.py
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr_ml_toolkit',
     packages=find_packages(),
-    version="1.0.0.dev-20241126",
+    version="1.0.0.dev-20241129",
     description="wedpr-ml-toolkit: The ML toolkit for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
index dfb9eb71..f6d08d39 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/dataset_context.py
@@ -32,7 +32,7 @@ def __repr__(self):
         return f"dataset_id: {self.dataset_id}, " \
                f"dataset_meta: {self.dataset_meta}"
 
-    def load_values(self, header=None):
+    def load_values(self, header=0):
         # 加载hdfs的数据集
         if self.storage_client is not None:
             values = self.storage_client.download(
@@ -40,6 +40,7 @@ def load_values(self, header=None):
             if values is None:
                 return values, None, None
             return values, values.columns, values.shape
+        raise Exception("Must set the storage client to load data!")
 
     def save_values(self, values: pd.DataFrame = None, path=None):
         # no values to save
@@ -48,17 +49,20 @@ def save_values(self, values: pd.DataFrame = None, path=None):
         csv_buffer = io.StringIO()
         values.to_csv(csv_buffer, index=False)
         value_bytes = csv_buffer.getvalue()
+        columns = values.columns.to_list()
         # update the meta firstly
         if path is None and self.dataset_meta is not None and self.dataset_meta.datasetId is not None:
-            columns = values.columns.to_list()
             dataset_meta = DatasetMeta(dataset_id=self.dataset_meta.datasetId,
                                        dataset_fields=','.join(columns),
                                        dataset_size=len(value_bytes),
                                        dataset_record_count=len(values),
                                        dataset_column_count=len(columns))
             self.dataset_client.update_dataset(dataset_meta)
+        # update the dataset meta
+        if self.dataset_meta is not None:
+            self.dataset_meta.datasetSize = len(value_bytes)
             self.dataset_meta.datasetFields = ','.join(columns)
-            self.dataset_meta.dataset_record_count = len(values)
+            self.dataset_meta.recordCount = len(values)
             self.dataset_meta.columnCount = len(columns)
         # update the content
         target_path = self.dataset_meta.file_path
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
index f1fef48f..a7ad3b16 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/fe_result_context.py
@@ -11,26 +11,20 @@
 
 class PreprocessingResultContext(ResultContext):
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        self.preprocessing_dataset: DatasetContext = None
         super().__init__(job_context, job_result_detail)
+        self.preprocessing_dataset = self._generate_result_dataset_(
+            self.preprocessing_result_path)
 
     def __repr__(self):
         return f"preprocessing_dataset: {self.preprocessing_dataset}"
 
-    def parse_result(self):
-        self.preprocessing_dataset = self._generate_result_dataset_(
-            self.preprocessing_result_path)
-
 
 class FeResultContext(ResultContext):
 
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        self.fe_dataset: DatasetContext = None
         super().__init__(job_context, job_result_detail)
+        self.fe_dataset = self._generate_result_dataset_(
+            self.fe_result_file_path)
 
     def __repr__(self):
         return f"fe_dataset: {self.fe_dataset}"
-
-    def parse_result(self):
-        self.fe_dataset = self._generate_result_dataset_(
-            self.fe_result_file_path)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
index e9d534e2..c93d9606 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/model_result_context.py
@@ -8,7 +8,6 @@
 
 class ModelResultContext(PreprocessingResultContext, FeResultContext):
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        super().__init__(job_context, job_result_detail)
         PreprocessingResultContext.__init__(
             self, job_context, job_result_detail)
         FeResultContext.__init__(self, job_context, job_result_detail)
@@ -16,28 +15,7 @@ def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse
 
 class TrainResultContext(ModelResultContext):
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        self.train_result_dataset: DatasetContext = None
-        self.test_result_dataset: DatasetContext = None
-        self.model_result_dataset: DatasetContext = None
-        self.model_dataset: DatasetContext = None
-        self.evaluation_dataset: DatasetContext = None
-        self.feature_selection_dataset: DatasetContext = None
-        self.feature_importance_dataset: DatasetContext = None
-        super().__init__(job_context, job_result_detail)
-
-    def __repr__(self):
-        return f"train_result_dataset: {self.train_result_dataset}," \
-               f"test_result_dataset: {self.test_result_dataset}," \
-               f"model_result_dataset: {self.model_result_dataset}," \
-               f"model_dataset: {self.model_dataset}," \
-               f"evaluation_dataset: {self.evaluation_dataset}," \
-               f"feature_selection_dataset: {self.feature_selection_dataset}," \
-               f"feature_importance_dataset: {self.feature_importance_dataset}," \
-               f"preprocessing_dataset: {self.preprocessing_dataset}," \
-               f"fe_dataset: {self.fe_dataset}," \
-               f"psi_dataset: {self.psi_result_file_path}"
-
-    def parse_result(self):
+        ModelResultContext.__init__(self, job_context, job_result_detail)
         # train_model_output
         self.train_result_dataset = self._generate_result_dataset_(
             self.job_result_detail.modelResultDetail['ModelResult']['trainResultPath'])
@@ -59,15 +37,24 @@ def parse_result(self):
         self.feature_importance_dataset = self._generate_result_dataset_(
             self.feature_importance_result_path)
 
+    def __repr__(self):
+        return f"train_result_dataset: {self.train_result_dataset}," \
+               f"test_result_dataset: {self.test_result_dataset}," \
+               f"model_result_dataset: {self.model_result_dataset}," \
+               f"model_dataset: {self.model_dataset}," \
+               f"evaluation_dataset: {self.evaluation_dataset}," \
+               f"feature_selection_dataset: {self.feature_selection_dataset}," \
+               f"feature_importance_dataset: {self.feature_importance_dataset}," \
+               f"preprocessing_dataset: {self.preprocessing_dataset}," \
+               f"fe_dataset: {self.fe_dataset}," \
+               f"psi_dataset: {self.psi_result_file_path}"
+
 
 class PredictResultContext(ResultContext):
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        self.model_result_dataset: DatasetContext = None
         super().__init__(job_context, job_result_detail)
+        self.model_result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.modelResultDetail['ModelResult']['testResultPath'])
 
     def __repr__(self):
         return f"model_result_dataset: {self.model_result_dataset}"
-
-    def parse_result(self):
-        self.model_result_dataset = self._generate_result_dataset_(
-            self.job_result_detail.modelResultDetail['ModelResult']['testResultPath'])
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
index 67d5e3f1..fbc24725 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/psi_result_context.py
@@ -7,12 +7,9 @@
 
 class PSIResultContext(ResultContext):
     def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse):
-        self.result_dataset: DatasetContext = None
         super().__init__(job_context, job_result_detail)
+        self.result_dataset = self._generate_result_dataset_(
+            self.job_result_detail.resultFileInfo['path'])
 
     def __repr__(self):
         return f"result_dataset: {self.result_dataset}"
-
-    def parse_result(self):
-        self.result_dataset = self._generate_result_dataset_(
-            self.job_result_detail.resultFileInfo['path'])
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
index bc313468..fba85a01 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
@@ -40,8 +40,6 @@ def __init__(self, job_context: JobContext, job_result_detail: JobDetailResponse
         self.psi_result_file_path = os.path.join(
             self.job_context.user_config.user, "share", "psi", f"psi-{self.job_id}")
 
-        self.parse_result()
-
     @staticmethod
     def check_and_get_job_type(job_context: JobContext, job_result_detail: JobDetailResponse):
         if job_context is None:
@@ -52,6 +50,9 @@ def check_and_get_job_type(job_context: JobContext, job_result_detail: JobDetail
         if job_result_detail.job_object is None:
             raise Exception(
                 "Invalid job result: must contain the job information!")
+        if job_result_detail.job_object.job_status.run_failed():
+            raise Exception(
+                f"RunFailed job can't construct the result context!")
         return job_result_detail.job_object.jobType
 
     def _generate_result_dataset_(self, dataset_file_path):
@@ -61,7 +62,3 @@ def _generate_result_dataset_(self, dataset_file_path):
         return DatasetContext(storage_entrypoint=self.job_context.storage_entry_point,
                               storage_workspace=None,
                               dataset_meta=dataset_meta)
-
-    @abstractmethod
-    def parse_result(self):
-        pass
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
index 166529a6..9c70e3e1 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_dataset_client.py
@@ -5,6 +5,7 @@
 from wedpr_ml_toolkit.config.wedpr_ml_config import DatasetConfig
 from wedpr_ml_toolkit.config.wedpr_ml_config import HttpConfig
 from wedpr_ml_toolkit.config.wedpr_ml_config import AuthConfig
+from wedpr_ml_toolkit.config.wedpr_ml_config import UserConfig
 from typing import Any
 import json
 
@@ -28,6 +29,7 @@ def __init__(self, dataset_id: str = None,
                  status: str = None,
                  status_desc: str = None,
                  file_path: str = None,
+                 user_config: UserConfig = None,
                  **params: Any):
         self.datasetId = dataset_id
         self.datasetLabel = dataset_label
@@ -44,6 +46,10 @@ def __init__(self, dataset_id: str = None,
         self.dataSourceMeta = datasource_meta
         self.ownerAgencyName = agency
         self.ownerUserName = user
+        self.user_config = user_config
+        if self.user_config is not None:
+            self.ownerAgencyName = self.user_config.agency_name
+            self.ownerUserName = self.user_config.user
         self.status = status
         self.statusDesc = status_desc
         self.file_path = file_path
@@ -54,10 +60,13 @@ def __init__(self, dataset_id: str = None,
                 self.file_path = dataset_storage_path_dict.get("filePath")
 
     def __repr__(self):
-        return f"dataset: {self.datasetId}, datasetTitle: {self.datasetTitle}, datasetFields: {self.datasetFields}, " \
+        return f"id: {self.datasetId}, " \
+               f"file_path, {self.file_path}, " \
+               f"datasetTitle: {self.datasetTitle}, datasetFields: {self.datasetFields}, " \
                f"datasetSize: {self.datasetSize}, recordCount: {self.recordCount}," \
                f"columnCount: {self.columnCount}, datasetStorageType: {self.datasetStorageType}, " \
-               f"ownerAgencyName: {self.ownerAgencyName}"
+               f"user: {self.ownerUserName}, " \
+               f"agency: {self.ownerAgencyName}"
 
 
 class WeDPRDatasetClient(WeDPREntryPoint, BaseObject):
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
index 4356c91d..2d5545a4 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
@@ -142,9 +142,13 @@ def __init__(self, job_info: JobInfo = None, **params: Any):
         self.modelResultDetail: dict = None
         self.resultFileInfo: dict = None
         self.model: dict = None
+        self.model_predict_algorithm = {}
         self.set_params(**params)
         if self.job_object is None and self.job is not None:
             self.job_object = JobInfo(**self.job)
+        # generate the model_predict algorithm
+        if self.model is not None:
+            self.model_predict_algorithm.update({"setting": self.model})
 
     def __repr__(self):
         return f"job: {self.job_object}, modelResultDetail: {self.modelResultDetail}, " \

From 0c714ddeb4a634bea2222e623bf242573829dbbd Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Thu, 28 Nov 2024 00:35:06 +0800
Subject: [PATCH 107/120] add config.properties in conf dir (#106)

---
 python/wedpr_ml_toolkit/{test => conf}/config.properties | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename python/wedpr_ml_toolkit/{test => conf}/config.properties (100%)

diff --git a/python/wedpr_ml_toolkit/test/config.properties b/python/wedpr_ml_toolkit/conf/config.properties
similarity index 100%
rename from python/wedpr_ml_toolkit/test/config.properties
rename to python/wedpr_ml_toolkit/conf/config.properties

From 3def5ddd4e9d630e5aaf93f0b6f749bf91abdbc8 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 3 Dec 2024 11:55:44 +0800
Subject: [PATCH 108/120] hdfs support keberos auth (#107)

* hdfs support krb5

* update libhdfs3 to support krb5

* hdfs support keberos auth

* update ppc_builder

* fix centos ci

* add more logs

* fix hdfs auth bug

* support specify the krb5.conf

* fix centos ci
---
 .github/workflows/cpp_full_node_workflow.yml  | 32 ++++++--
 .github/workflows/cpp_sdk_workflow.yml        | 25 ++++--
 .github/workflows/cpp_toolkit_workflow.yml    | 25 ++++--
 cpp/cmake/CompilerSettings.cmake              |  2 +-
 cpp/cmake/TargetSettings.cmake                |  2 +-
 cpp/ppc-framework/protocol/Krb5AuthConfig.h   | 68 +++++++++++++++
 cpp/ppc-framework/protocol/Protocol.h         |  9 +-
 cpp/tools/build_ppc.sh                        | 38 ++++++++-
 cpp/tools/build_wedpr_cem.sh                  | 12 ++-
 cpp/tools/build_wedpr_mpc.sh                  | 38 ++++++++-
 .../ppc-builder/conf/config-example.toml      | 25 +++++-
 cpp/tools/ppc-builder/src/common/utilities.py |  2 +
 .../src/config/ppc_deploy_config.py           | 22 +++++
 .../src/config/ppc_node_config_generator.py   | 26 +++++-
 cpp/tools/ppc-builder/src/tpl/config.ini.node | 13 ++-
 cpp/tools/ppc-builder/src/tpl/krb5.conf       | 17 ++++
 cpp/vcpkg-configuration.json                  |  6 +-
 cpp/vcpkg.json                                |  8 +-
 .../ppc-tools/src/config/PPCConfig.cpp        | 32 +++++++-
 .../ppc-tools/src/config/PPCConfig.h          |  2 +-
 .../ppc-tools/src/config/StorageConfig.h      |  1 +
 .../ppc-storage/src/hdfs/Common.h             |  2 +-
 .../ppc-storage/src/hdfs/HDFSStorage.cpp      | 20 ++++-
 .../src/hdfs/auth/Krb5CredLoader.cpp          | 76 +++++++++++++++++
 .../src/hdfs/auth/Krb5CredLoader.h            | 82 +++++++++++++++++++
 cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp |  2 +-
 .../ppc_common/deps_services/hdfs_storage.py  | 21 ++---
 .../deps_services/krb5_hdfs_storage.py        | 24 ++++++
 .../ppc_common/deps_services/storage_api.py   | 51 ++++++++++++
 .../deps_services/storage_loader.py           | 19 +++--
 .../deps_services/tests/hdfs_storage_test.py  | 44 ++++++++++
 python/ppc_model/conf/application-sample.yml  | 15 ++--
 python/requirements.txt                       |  2 +
 33 files changed, 689 insertions(+), 74 deletions(-)
 create mode 100644 cpp/ppc-framework/protocol/Krb5AuthConfig.h
 create mode 100644 cpp/tools/ppc-builder/src/tpl/krb5.conf
 create mode 100644 cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp
 create mode 100644 cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h
 create mode 100644 python/ppc_common/deps_services/krb5_hdfs_storage.py
 create mode 100644 python/ppc_common/deps_services/tests/hdfs_storage_test.py

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index fee8d67a..4512a8e2 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build:
@@ -99,23 +101,37 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test ARGS="--output-on-failure"
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-x64-linux-dbg-err.log
+          path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-linux-dbg-err.log
+        
 
   build_centos:
     name: build_centos full node
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     continue-on-error: true
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+          container:
+            - centos-7.9
     container:
       image: docker.io/centos:7
       volumes:
         - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+        - /node20217:/node20217:rw,rshared
+        - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 5
+      - name: install nodejs20glibc2.17
+        if: ${{ matrix.container == 'centos-7.9' }}
+        run: |
+          curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+          tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+      - name: Check out the repo
+        uses: actions/checkout@v4
       - uses: actions/cache@v3
         id: deps_cache
         with:
@@ -148,6 +164,10 @@ jobs:
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Prepare vcpkg
+        if: runner.os != 'Windows'
+        uses: friendlyanon/setup-vcpkg@v1
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
       - name: Check disk space
         run: df . -h
       - name: Free disk space
@@ -169,7 +189,7 @@ jobs:
           rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
-          cmake3 -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+          cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
       - name: FreeDiskSpace
         run: |
           df -lh
diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml
index e75cc273..4c361914 100644
--- a/.github/workflows/cpp_sdk_workflow.yml
+++ b/.github/workflows/cpp_sdk_workflow.yml
@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build_sdk:
@@ -105,20 +107,27 @@ jobs:
 
   build_centos_for_sdk:
       name: build_centos_for_sdk
-      runs-on: ${{ matrix.os }}
+      runs-on: ubuntu-latest
       continue-on-error: true
       strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest]
+          container:
+            - centos-7.9
       container:
         image: docker.io/centos:7
         volumes:
           - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+          - /node20217:/node20217:rw,rshared
+          - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
       steps:
-        - uses: actions/checkout@v3
-          with:
-            fetch-depth: 5
+        - name: install nodejs20glibc2.17
+          if: ${{ matrix.container == 'centos-7.9' }}
+          run: |
+            curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+            tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        - name: Check out the repo
+          uses: actions/checkout@v4
         - uses: actions/cache@v3
           id: deps_cache
           with:
@@ -149,11 +158,15 @@ jobs:
             yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
             yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
             yum install -y git
+        - name: Prepare vcpkg
+          if: runner.os != 'Windows'
+          uses: friendlyanon/setup-vcpkg@v1
+          with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
         - name: Build
           run: |
             alias cmake='cmake3'
             . /opt/rh/devtoolset-11/enable
             mkdir -p cpp/build
             cd cpp/build
-            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
             cmake3 --build . --parallel 3
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index 267d8c51..04635c77 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build_wedpr_toolkit:
@@ -111,20 +113,27 @@ jobs:
 
   build_centos_for_toolkit:
       name: build_centos_for_toolkit
-      runs-on: ${{ matrix.os }}
+      runs-on: ubuntu-latest
       continue-on-error: true
       strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest]
+            container:
+              - centos-7.9
       container:
         image: docker.io/centos:7
         volumes:
           - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+          - /node20217:/node20217:rw,rshared
+          - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
       steps:
-        - uses: actions/checkout@v3
-          with:
-            fetch-depth: 5
+        - name: install nodejs20glibc2.17
+          if: ${{ matrix.container == 'centos-7.9' }}
+          run: |
+            curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+            tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        - name: Check out the repo
+          uses: actions/checkout@v4
         - uses: actions/cache@v3
           id: deps_cache
           with:
@@ -155,13 +164,17 @@ jobs:
             yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
             yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
             yum install -y git
+        - name: Prepare vcpkg
+          if: runner.os != 'Windows'
+          uses: friendlyanon/setup-vcpkg@v1
+          with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
         - name: Build
           run: |
             alias cmake='cmake3'
             . /opt/rh/devtoolset-11/enable
             mkdir -p cpp/build
             cd cpp/build
-            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
             cmake3 --build . --parallel 3
         - uses: actions/upload-artifact@v3
           with:
diff --git a/cpp/cmake/CompilerSettings.cmake b/cpp/cmake/CompilerSettings.cmake
index 1d2efdc8..e6a91d9b 100644
--- a/cpp/cmake/CompilerSettings.cmake
+++ b/cpp/cmake/CompilerSettings.cmake
@@ -197,7 +197,7 @@ endif()
 
 # rust static library linking requirements for macos
 if(APPLE)
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Security")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Security -framework Kerberos")
 else()
    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ldl")
 endif()
diff --git a/cpp/cmake/TargetSettings.cmake b/cpp/cmake/TargetSettings.cmake
index 1eabf08e..e66f019b 100644
--- a/cpp/cmake/TargetSettings.cmake
+++ b/cpp/cmake/TargetSettings.cmake
@@ -87,7 +87,7 @@ set(AIR_BINARY_NAME ppc-air-node)
 set(CEM_BINARY_NAME wedpr-cem)
 set(MPC_BINARY_NAME wedpr-mpc)
 
-set(HDFS_LIB libhdfs3-static)
+set(HDFS_LIB libhdfs3)
 
 # set cpu-info
 set(CPU_FEATURES_LIB "")
diff --git a/cpp/ppc-framework/protocol/Krb5AuthConfig.h b/cpp/ppc-framework/protocol/Krb5AuthConfig.h
new file mode 100644
index 00000000..a7ed09b3
--- /dev/null
+++ b/cpp/ppc-framework/protocol/Krb5AuthConfig.h
@@ -0,0 +1,68 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Krb5AuthConfig.h
+ * @author: yujiechen
+ * @date 2024-12-2
+ */
+#pragma once
+#include "ppc-framework/Common.h"
+#include 
+#include 
+#include 
+
+namespace ppc::protocol
+{
+struct Krb5AuthConfig
+{
+    using Ptr = std::shared_ptr;
+    std::string principal;
+    std::string password;
+    std::string ccachePath;
+    std::string authConfigFilePath = "./conf/krb5.conf";
+    void check() const
+    {
+        if (principal.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the principal!"));
+        }
+        if (password.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the password!"));
+        }
+        if (ccachePath.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the ccachePath!"));
+        }
+        if (authConfigFilePath.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(
+                WeDPRException() << bcos::errinfo_comment(
+                    "Invalid krb5 auth config: Must set the authConfigFilePath!"));
+        }
+    }
+
+    inline std::string desc() const
+    {
+        std::stringstream oss;
+        oss << LOG_KV("principal", principal) << LOG_KV("ccachePath", ccachePath)
+            << LOG_KV("authConfigFilePath", authConfigFilePath);
+        return oss.str();
+    }
+};
+}  // namespace ppc::protocol
\ No newline at end of file
diff --git a/cpp/ppc-framework/protocol/Protocol.h b/cpp/ppc-framework/protocol/Protocol.h
index f16ec21a..ee893d81 100644
--- a/cpp/ppc-framework/protocol/Protocol.h
+++ b/cpp/ppc-framework/protocol/Protocol.h
@@ -23,6 +23,7 @@
 #define WIN32_LEAN_AND_MEAN
 #include 
 #endif
+#include "Krb5AuthConfig.h"
 #include "ppc-framework/Common.h"
 #include 
 #include 
@@ -452,6 +453,7 @@ struct FileStorageConnectionOption
     bool replaceDataNodeOnFailure = false;
     // the default connection-timeout for the hdfs is 1000ms
     uint16_t connectionTimeout = 1000;
+    Krb5AuthConfig::Ptr authConfig;
 
     void check() const
     {
@@ -470,6 +472,10 @@ struct FileStorageConnectionOption
             BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
                                       "Invalid HDFS Option, Must set valid namenodeport!"));
         }
+        if (authConfig)
+        {
+            authConfig->check();
+        }
     }
     inline std::string desc() const
     {
@@ -477,7 +483,8 @@ struct FileStorageConnectionOption
         oss << LOG_KV("nameNode", nameNode) << LOG_KV("nameNodePort", nameNodePort)
             << LOG_KV("user", userName) << LOG_KV("token", token)
             << LOG_KV("replace-datanode-on-failure", replaceDataNodeOnFailure)
-            << LOG_KV("connectionTimeout", connectionTimeout);
+            << LOG_KV("connectionTimeout", connectionTimeout)
+            << LOG_KV("authInfo", authConfig ? authConfig->desc() : "null");
         return oss.str();
     }
 };
diff --git a/cpp/tools/build_ppc.sh b/cpp/tools/build_ppc.sh
index 401169ad..621f5c99 100644
--- a/cpp/tools/build_ppc.sh
+++ b/cpp/tools/build_ppc.sh
@@ -476,7 +476,7 @@ generate_node_config_ini() {
 
 [hdfs_storage]
     ; the hdfs configuration
-    user = app
+    user = root
     name_node = 127.0.0.1
     name_node_port = 9000
     token =
@@ -484,6 +484,16 @@ generate_node_config_ini() {
     replace-datanode-on-failure = false
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 1000
+    ; enable auth or not, default is false
+    ; enable_krb5_auth = false
+    ; the hdfs kerberos auth principal, used when enable_krb5_auth
+    ; auth_principal = root@NODE.DC1.CONSUL
+    ; the hdfs kerberos auth password, used when enable_krb5_auth
+    ; auth_password =
+    ; the ccache path, used when enable_krb5_auth
+    ; ccache_path = /tmp/krb5cc_ppc_node
+    ; the krb5.conf path
+    ; krb5_conf_path  = conf/krb5.conf
 
 
 [ra2018psi]
@@ -541,6 +551,31 @@ generate_node_config_ini() {
 EOF
 }
 
+generate_krb5_file_template()
+{
+  local filepath=$1
+  mkdir -p $(dirname $filepath)   
+   cat << EOF > "${filepath}"
+[libdefaults]
+ default_realm = NODE.DC1.CONSUL
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ NODE.DC1.CONSUL = {
+  kdc = 
+  admin_server =
+ }
+
+[domain_realm]
+ .node.dc1.consul = NODE.DC1.CONSUL
+ node.dc1.consul = NODE.DC1.CONSUL
+EOF
+}
+
 generate_script_template()
 {
     local filepath=$1
@@ -915,6 +950,7 @@ deploy_nodes()
             private_key=$(generate_private_key "${node_dir}/conf")
             node_id=$(cat "${node_dir}/conf/node.nodeid")
             generate_node_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" "${node_id}"
+            generate_krb5_file_template "${node_dir}/conf/krb5.conf"
             generate_p2p_connected_conf "${node_dir}/${p2p_connected_conf_name}" "${connected_nodes}" "false"
             set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1))
             ((++count))
diff --git a/cpp/tools/build_wedpr_cem.sh b/cpp/tools/build_wedpr_cem.sh
index c972e559..8299c61e 100644
--- a/cpp/tools/build_wedpr_cem.sh
+++ b/cpp/tools/build_wedpr_cem.sh
@@ -354,7 +354,7 @@ generate_config_ini() {
 
 [hdfs_storage]
     ; the hdfs configuration
-    user = ppc
+    user = root
     name_node = 127.0.0.1
     name_node_port = 9900
     token =
@@ -362,6 +362,16 @@ generate_config_ini() {
     replace-datanode-on-failure = false
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 2000
+    ; enable auth or not, default is false
+    ; enable_krb5_auth = false
+    ; the hdfs kerberos auth principal, used when enable_krb5_auth
+    ; auth_principal = root@NODE.DC1.CONSUL
+    ; the hdfs kerberos auth password, used when enable_krb5_auth
+    ; auth_password =
+    ; the ccache path, used when enable_krb5_auth
+    ; ccache_path = /tmp/krb5cc_ppc_node
+    ; the krb5.conf path
+    ; krb5_conf_path  = conf/krb5.conf
 
 [cert]
     ; directory the certificates located in
diff --git a/cpp/tools/build_wedpr_mpc.sh b/cpp/tools/build_wedpr_mpc.sh
index 938b9184..c3d109dd 100644
--- a/cpp/tools/build_wedpr_mpc.sh
+++ b/cpp/tools/build_wedpr_mpc.sh
@@ -358,7 +358,7 @@ generate_config_ini() {
 
 [hdfs_storage]
     ; the hdfs configuration
-    user = ppc
+    user = root
     name_node = 127.0.0.1
     name_node_port = 9900
     token =
@@ -366,6 +366,16 @@ generate_config_ini() {
     replace-datanode-on-failure = false
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 2000
+    ; enable auth or not, default is false
+    ; enable_krb5_auth = false
+    ; the hdfs kerberos auth principal, used when enable_krb5_auth
+    ; auth_principal = root@NODE.DC1.CONSUL
+    ; the hdfs kerberos auth password, used when enable_krb5_auth
+    ; auth_password =
+    ; the ccache path, used when enable_krb5_auth
+    ; ccache_path = /tmp/krb5cc_ppc_node
+    ; the krb5.conf path
+    ; krb5_conf_path  = conf/krb5.conf
 
 [transport]
    ; the endpoint information
@@ -396,6 +406,31 @@ generate_config_ini() {
 EOF
 }
 
+generate_krb5_file_template()
+{
+  local filepath=$1
+  mkdir -p $(dirname $filepath)   
+   cat << EOF > "${filepath}"
+[libdefaults]
+ default_realm = NODE.DC1.CONSUL
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ NODE.DC1.CONSUL = {
+  kdc = 
+  admin_server =
+ }
+
+[domain_realm]
+ .node.dc1.consul = NODE.DC1.CONSUL
+ node.dc1.consul = NODE.DC1.CONSUL
+EOF
+}
+
 generate_script_template()
 {
     local filepath=$1
@@ -723,6 +758,7 @@ deploy_nodes()
     private_key=$(generate_private_key "${output_dir}/conf")
     node_id=$(cat "${output_dir}/conf/node.nodeid")
     generate_config_ini "${output_dir}/config.ini"  "${listen_ip}" "${rpc_port}" "${agency_info}" ${agency_id} "${listen_ip}" "${grpc_port}" "${node_id}"
+    generate_krb5_file_template "{output_dir}/conf/krb5.conf"
     print_result
 }
 
diff --git a/cpp/tools/ppc-builder/conf/config-example.toml b/cpp/tools/ppc-builder/conf/config-example.toml
index a27685dd..867f113a 100644
--- a/cpp/tools/ppc-builder/conf/config-example.toml
+++ b/cpp/tools/ppc-builder/conf/config-example.toml
@@ -90,10 +90,20 @@ holding_msg_minutes = 30
             database = ""
         # the hdfs storage config
         [agency.node.hdfs_storage]
-            user = ""
+            user = "root"
             name_node = "127.0.0.1"
             name_node_port = 9000
             token = ""
+            # enable auth or not, default is false
+            enable_krb5_auth = false
+            # the hdfs kerberos auth principal, used when enable_krb5_auth
+            auth_principal = "root@NODE.DC1.CONSUL"
+            # the hdfs kerberos auth password, used when enable_krb5_auth
+            auth_password = ""
+            # the ccache path, used when enable_krb5_auth
+            ccache_path = "/tmp/krb5cc_ppc_node"
+            # the krb5 conf path
+            krb5_conf_path = "conf/krb5.conf"
         # the gateway config
         [agency.node.gateway]
             gateway_grpc_target = ["127.0.0.1:40600", "127.0.0.1:40601"]
@@ -175,10 +185,21 @@ holding_msg_minutes = 30
             database = ""
         # the hdfs storage config
         [agency.node.hdfs_storage]
-            user = ""
+            user = "root"
             name_node = "127.0.0.1"
             name_node_port = 9000
             token = ""
+            # enable auth or not, default is false
+            enable_krb5_auth = false
+            # the hdfs kerberos auth principal, used when enable_krb5_auth
+            auth_principal = "root@NODE.DC1.CONSUL"
+            # the hdfs kerberos auth password, used when enable_krb5_auth
+            auth_password = ""
+            # the ccache path, used when enable_krb5_auth
+            ccache_path = "/tmp/krb5cc_ppc_node"
+            # the krb5 conf path
+            krb5_conf_path = "conf/krb5.conf"
+
         # the gateway config
         [agency.node.gateway]
             gateway_grpc_target = ["127.0.0.1:40620", "127.0.0.1:40621"]
diff --git a/cpp/tools/ppc-builder/src/common/utilities.py b/cpp/tools/ppc-builder/src/common/utilities.py
index 5e4b3c32..5462a9db 100644
--- a/cpp/tools/ppc-builder/src/common/utilities.py
+++ b/cpp/tools/ppc-builder/src/common/utilities.py
@@ -29,6 +29,8 @@ class ConfigInfo:
         pwd_path, tpl_abs_path, "config.ini.node")
     gateway_config_tpl_path = os.path.join(
         pwd_path, tpl_abs_path, "config.ini.gateway")
+    krb5_config_tpl_path = os.path.join(
+        pwd_path, tpl_abs_path, "krb5.conf")
 
     ppc_gateway_binary_name = "ppc-gateway-service"
     ppc_node_binary_name = "ppc-pro-node"
diff --git a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py
index 88bae36f..d8c642e4 100644
--- a/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py
+++ b/cpp/tools/ppc-builder/src/config/ppc_deploy_config.py
@@ -116,6 +116,28 @@ def __init__(self, config, config_section, must_exist):
             self.config, "name_node_port", None, must_exist, config_section)
         self.token = utilities.get_item_value(
             self.config, "token", "", False, config_section)
+        # enable auth or not
+        enable_krb5_auth = utilities.get_item_value(
+            self.config, "enable_krb5_auth", "",
+            False, config_section)
+        self.enable_krb5_auth_str = utilities.convert_bool_to_str(
+            enable_krb5_auth)
+        # auth principal
+        self.auth_principal = utilities.get_item_value(
+            self.config, "auth_principal",
+            "", enable_krb5_auth, config_section)
+        # auth password
+        self.auth_password = utilities.get_item_value(
+            self.config, "auth_password",
+            "", enable_krb5_auth, config_section)
+        # cacche path
+        self.ccache_path = utilities.get_item_value(
+            self.config, "ccache_path",
+            "", enable_krb5_auth, config_section)
+        # the krb5.conf
+        self.krb5_conf_path = utilities.get_item_value(
+            self.config, "krb5_conf_path",
+            "conf/krb5.conf", enable_krb5_auth, config_section)
 
 
 class RA2018PSIConfig:
diff --git a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py
index 615d1d70..8a835ae1 100644
--- a/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py
+++ b/cpp/tools/ppc-builder/src/config/ppc_node_config_generator.py
@@ -109,8 +109,8 @@ def __generate_single_node_inner_config__(self, tpl_config_path, node_path, priv
         self.__generate_storage_config__(
             config_content, node_config.storage_config)
         # load the hdfs_storage_config
-        self.__generate_hdfs_storage_config__(
-            config_content, node_config.hdfs_storage_config)
+        self.__generate_hdfs_storage_config__(node_path, utilities.ConfigInfo.krb5_config_tpl_path,
+                                              config_content, node_config.hdfs_storage_config)
         # load the ra2018psi config
         self.__generate_ra2018psi_config__(
             config_content, node_config.ra2018psi_config)
@@ -182,7 +182,7 @@ def __generate_storage_config__(self, config_content, storage_config):
         config_content[section_name]["password"] = storage_config.password
         config_content[section_name]["database"] = storage_config.database
 
-    def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config):
+    def __generate_hdfs_storage_config__(self, node_path, krb5_tpl_file_path, config_content, hdfs_storage_config):
         if hdfs_storage_config is None:
             return
         section_name = "hdfs_storage"
@@ -191,8 +191,26 @@ def __generate_hdfs_storage_config__(self, config_content, hdfs_storage_config):
         config_content[section_name]["name_node_port"] = str(
             hdfs_storage_config.name_node_port)
         config_content[section_name]["token"] = hdfs_storage_config.token
+        config_content[section_name]["enable_krb5_auth"] = hdfs_storage_config.enable_krb5_auth_str
+        config_content[section_name]["auth_principal"] = hdfs_storage_config.auth_principal
+        config_content[section_name]["auth_password"] = hdfs_storage_config.auth_password
+        config_content[section_name]["ccache_path"] = hdfs_storage_config.ccache_path
+        config_content[section_name]["krb5_conf_path"] = hdfs_storage_config.krb5_conf_path
+        # copy krb5.conf to krb5_conf_path specified path
+        dst_path = os.path.join(node_path, hdfs_storage_config.krb5_conf_path)
+        if hdfs_storage_config.krb5_conf_path.startswith("/"):
+            dst_path = hdfs_storage_config.krb5_conf_path
+        command = "cp %s %s" % (krb5_tpl_file_path, dst_path)
+        (ret, output) = utilities.execute_command_and_getoutput(command)
+        if ret is False:
+            utilities.log_error("copy krb5 configuration from %s to %s failed, error: %s") % (
+                krb5_tpl_file_path, dst_path, output)
+            return False
+        return True
 
-    def __generate_transport_config__(self, config_content, node_config, node_id, deploy_ip, node_index):
+    def __generate_transport_config__(self, config_content,
+                                      node_config, node_id,
+                                      deploy_ip, node_index):
         """_summary_
 
         Args:
diff --git a/cpp/tools/ppc-builder/src/tpl/config.ini.node b/cpp/tools/ppc-builder/src/tpl/config.ini.node
index 2f0b7ab6..5bbb29b3 100644
--- a/cpp/tools/ppc-builder/src/tpl/config.ini.node
+++ b/cpp/tools/ppc-builder/src/tpl/config.ini.node
@@ -53,7 +53,7 @@
 
 [hdfs_storage]
     ; the hdfs configuration
-    user = app
+    user = root
     name_node = 127.0.0.1
     name_node_port = 9000
     token =
@@ -61,7 +61,16 @@
     replace-datanode-on-failure = false
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 1000
-
+    ; enable auth or not, default is false
+    ; enable_krb5_auth = false
+    ; the hdfs kerberos auth principal, used when enable_krb5_auth
+    ; auth_principal = root@NODE.DC1.CONSUL
+    ; the hdfs kerberos auth password, used when enable_krb5_auth
+    ; auth_password =
+    ; the ccache path, used when enable_krb5_auth
+    ; ccache_path = /tmp/krb5cc_ppc_node
+    ; the krb5.conf path
+    ; krb5_conf_path  = conf/krb5.conf
 
 [ra2018psi]
     ; The database used to store cuckoo-filter
diff --git a/cpp/tools/ppc-builder/src/tpl/krb5.conf b/cpp/tools/ppc-builder/src/tpl/krb5.conf
new file mode 100644
index 00000000..6e612c20
--- /dev/null
+++ b/cpp/tools/ppc-builder/src/tpl/krb5.conf
@@ -0,0 +1,17 @@
+[libdefaults]
+ default_realm = NODE.DC1.CONSUL
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ NODE.DC1.CONSUL = {
+  kdc = 
+  admin_server =
+ }
+
+[domain_realm]
+ .node.dc1.consul = NODE.DC1.CONSUL
+ node.dc1.consul = NODE.DC1.CONSUL
diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json
index 0a2e904f..3c205926 100644
--- a/cpp/vcpkg-configuration.json
+++ b/cpp/vcpkg-configuration.json
@@ -3,14 +3,16 @@
     {
       "kind": "git",
       "repository": "https://github.com/FISCO-BCOS/registry",
-      "baseline": "489334d1f374134163a3d9719c3345fdd1d268d3",
+      "baseline": "535c4fc74badd509b94e8a736452fa8a76bff944",
       "packages": [
         "openssl",
         "bcos-utilities",
         "bcos-boostssl",
         "tarscpp",
         "libhdfs3",
-        "tbb"
+        "tbb",
+        "gsasl",
+        "krb5"
       ]
     }
   ]
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index 6b2f1f75..a3a838c8 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -1,8 +1,8 @@
 {
-  "name": "fiscobcos",
-  "version-string": "3.7.3",
+  "name": "wedpr-component",
+  "version-string": "3.0.0",
   "homepage": "https://github.com/WeBankBlockchain/WeDPR-Component",
-  "description": "FISCO BCOS",
+  "description": "wedpr component",
   "dependencies": [
     {
       "name": "vcpkg-cmake",
@@ -30,7 +30,7 @@
     },
     { 
       "name": "libhdfs3", 
-      "version": "2024-04-27#1"
+      "version": "2024-11-29#0"
     },
     {
       "name": "tbb",
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index 27234303..59b34c2a 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -580,8 +580,37 @@ void PPCConfig::loadHDFSConfig(boost::property_tree::ptree const& _pt)
     // connection-timeout
     option->connectionTimeout = _pt.get("hdfs_storage.connection-timeout", 1000);
     m_storageConfig.fileStorageConnectionOpt = option;
+    auto enableAuth = _pt.get("hdfs_storage.enable_krb5_auth", false);
+    if (enableAuth)
+    {
+        loadKrb5AuthConfig(_pt);
+    }
     PPCConfig_LOG(INFO) << LOG_DESC("loadStorageConfig: load hdfs connection option success")
-                        << option->desc();
+                        << LOG_KV("enableAuth", enableAuth) << option->desc();
+}
+
+void PPCConfig::loadKrb5AuthConfig(boost::property_tree::ptree const& pt)
+{
+    auto authConfig = std::make_shared();
+    // the principal
+    authConfig->principal = pt.get("hdfs_storage.auth_principal", "");
+    // the password
+    authConfig->password = pt.get("hdfs_storage.auth_password", "");
+    // the ccachePath
+    authConfig->ccachePath =
+        pt.get("hdfs_storage.ccache_path", "/tmp/krb5cc_ppc_node");
+    // the krb5.conf path
+    authConfig->authConfigFilePath =
+        pt.get("hdfs_storage.krb5_conf_path", "./conf/krb5.conf");
+    // relative path case
+    if (!authConfig->authConfigFilePath.starts_with("/"))
+    {
+        auto joinedPath =
+            boost::filesystem::absolute(boost::filesystem::path(authConfig->authConfigFilePath));
+        authConfig->authConfigFilePath = joinedPath.string();
+    }
+    m_storageConfig.fileStorageConnectionOpt->authConfig = authConfig;
+    PPCConfig_LOG(INFO) << LOG_DESC("loadKrb5AuthConfig") << LOG_KV("config", authConfig->desc());
 }
 
 void PPCConfig::loadSQLConfig(boost::property_tree::ptree const& _pt)
@@ -633,6 +662,5 @@ void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt)
     {
         m_mpcConfig.threadPoolSize = std::thread::hardware_concurrency();
     }
-
     loadHDFSConfig(_pt);
 }
\ No newline at end of file
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index 6ac74457..aa620731 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -264,7 +264,7 @@ class PPCConfig
         std::string const& sectionName, boost::property_tree::ptree const& pt);
 
     virtual void loadHDFSConfig(boost::property_tree::ptree const& _pt);
-
+    virtual void loadKrb5AuthConfig(boost::property_tree::ptree const& pt);
     virtual void loadSQLConfig(boost::property_tree::ptree const& _pt);
 
 
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h
index 8e3d7033..e9bd4dce 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/StorageConfig.h
@@ -18,6 +18,7 @@
  * @date 2022-11-4
  */
 #pragma once
+#include "ppc-framework/protocol/Krb5AuthConfig.h"
 #include 
 #include 
 #include 
diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h b/cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h
index ba02d6e1..17f8a2a4 100644
--- a/cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h
+++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/Common.h
@@ -20,7 +20,7 @@
 #pragma once
 #include "../Common.h"
 #define HDFS_STORAGE_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("HDFS_STORAGE")
-
+#define HDFS_AUTH_LOG(LEVEL) BCOS_LOG(LEVEL) << LOG_BADGE("HDFS_STORAGE_AUTH")
 namespace ppc::storage
 {
 DERIVE_PPC_EXCEPTION(ConnectToHDFSFailed);
diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp
index 8eaa5455..8d467235 100644
--- a/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp
+++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/HDFSStorage.cpp
@@ -18,6 +18,7 @@
  * @date 2022-11-30
  */
 #include "HDFSStorage.h"
+#include "auth/Krb5CredLoader.h"
 #include 
 #include 
 #include 
@@ -41,6 +42,8 @@ HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option)
     // the default 'rpc.client.connect.retry' is 10
     auto connectTimeout =
         std::to_string(_option->connectionTimeout);  // set 1s as the connectTimeout
+    HDFS_STORAGE_LOG(INFO) << LOG_DESC("create HDFSStorage") << _option->desc()
+                           << LOG_KV("connectTimeout", connectTimeout);
     hdfsBuilderConfSetStr(m_builder.get(), "rpc.client.connect.timeout", connectTimeout.c_str());
 
     // default disable output.replace-datanode-on-failure, set to false to resolve append data error
@@ -53,8 +56,6 @@ HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option)
         hdfsBuilderConfSetStr(m_builder.get(), "output.replace-datanode-on-failure", "false");
     }
 
-    HDFS_STORAGE_LOG(INFO) << LOG_DESC("create HDFSStorage") << _option->desc()
-                           << LOG_KV("connectTimeout", connectTimeout);
     // set name node
     hdfsBuilderSetNameNode(m_builder.get(), _option->nameNode.c_str());
     hdfsBuilderSetNameNodePort(m_builder.get(), _option->nameNodePort);
@@ -66,6 +67,19 @@ HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option)
     {
         hdfsBuilderSetToken(m_builder.get(), _option->token.c_str());
     }
+    // init the auth information
+    if (_option->authConfig)
+    {
+        // set auth type to Kerberos
+        hdfsBuilderConfSetStr(m_builder.get(), "hadoop.security.authentication", "kerberos");
+        // init and store the auth information into the cache
+        auto ctx = std::make_shared(_option->authConfig);
+        ctx->init();
+        HDFS_STORAGE_LOG(INFO) << LOG_DESC("SetKerbTicketCachePath")
+                               << LOG_KV("ccachePath", _option->authConfig->ccachePath);
+        // set the ccache file path
+        hdfsBuilderSetKerbTicketCachePath(m_builder.get(), _option->authConfig->ccachePath.c_str());
+    }
     // connect to the hdfs, Note: the m_fs is a pointer
     m_fs = std::shared_ptr(
         hdfsBuilderConnect(m_builder.get()), HDFSFSDeleter());
@@ -75,6 +89,8 @@ HDFSStorage::HDFSStorage(FileStorageConnectionOption::Ptr const& _option)
             ConnectToHDFSFailed() << errinfo_comment(
                 "Connect to hdfs failed! error: " + std::string(hdfsGetLastError())));
     }
+    HDFS_STORAGE_LOG(INFO) << LOG_DESC("create HDFSStorage success") << _option->desc()
+                           << LOG_KV("connectTimeout", connectTimeout);
 }
 
 
diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp
new file mode 100644
index 00000000..498aaa7e
--- /dev/null
+++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.cpp
@@ -0,0 +1,76 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Krb5CredLoader.cpp
+ * @author: yujiechen
+ * @date 2024-12-1
+ */
+#include "Krb5CredLoader.h"
+#include "../Common.h"
+
+using namespace ppc::storage;
+using namespace ppc;
+using namespace ppc::protocol;
+using namespace bcos;
+
+void Krb5Context::init()
+{
+    HDFS_AUTH_LOG(INFO) << LOG_DESC("init Krb5Context") << m_config->desc();
+
+    // init the profile
+    auto ret = profile_init_path(m_config->authConfigFilePath.c_str(), &m_profile);
+    if (ret)
+    {
+        BOOST_THROW_EXCEPTION(WeDPRException() << errinfo_comment(
+                                  "load Krb5Context failed for profile_init_path failed!"));
+    }
+    m_profilePtr = &m_profile;
+    // load krb5 ctx
+    auto error = krb5_init_context_profile(m_profile, 1, &m_ctx);
+    checkResult(error, "krb5_init_context_profile");
+
+    // init the principal
+    error = krb5_parse_name(m_ctx, m_config->principal.c_str(), &m_principal);
+    checkResult(error, "krb5_parse_name");
+    // init credential
+    error = krb5_get_init_creds_password(
+        m_ctx, &m_credsObj, m_principal, m_config->password.c_str(), NULL, NULL, 0, NULL, NULL);
+    checkResult(error, "krb5_get_init_creds_password");
+    m_creds = &m_credsObj;
+    // init the ccache
+    error = krb5_cc_resolve(m_ctx, m_config->ccachePath.c_str(), &m_ccache);
+    checkResult(error, "krb5_cc_resolve");
+
+    error = krb5_cc_initialize(m_ctx, m_ccache, m_principal);
+    checkResult(error, "krb5_cc_initialize");
+    // store the credential
+    error = krb5_cc_store_cred(m_ctx, m_ccache, m_creds);
+    HDFS_AUTH_LOG(INFO) << LOG_DESC("init Krb5Context success") << m_config->desc();
+}
+
+void Krb5Context::checkResult(krb5_error_code const& error, std::string const& method)
+{
+    if (!error)
+    {
+        HDFS_AUTH_LOG(INFO) << LOG_DESC("init Krb5Context: ") << method << " success";
+        return;
+    }
+    auto msg = krb5_get_error_message(m_ctx, error);
+    HDFS_AUTH_LOG(ERROR) << LOG_DESC("init Krb5Context failed") << LOG_KV("method", method)
+                         << LOG_KV("reason", msg);
+    BOOST_THROW_EXCEPTION(
+        WeDPRException() << errinfo_comment(
+            "load Krb5Context failed, method: " + method + ", reason: " + std::string(msg)));
+}
\ No newline at end of file
diff --git a/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h
new file mode 100644
index 00000000..e2caeb30
--- /dev/null
+++ b/cpp/wedpr-storage/ppc-storage/src/hdfs/auth/Krb5CredLoader.h
@@ -0,0 +1,82 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Krb5CredLoader.h
+ * @author: yujiechen
+ * @date 2024-12-1
+ */
+#pragma once
+#include "ppc-framework/protocol/Krb5AuthConfig.h"
+#include 
+#include 
+#include 
+
+namespace ppc::storage
+{
+class Krb5Context
+{
+public:
+    using Ptr = std::shared_ptr;
+    Krb5Context(ppc::protocol::Krb5AuthConfig::Ptr const& config) : m_config(config) {}
+
+    virtual ~Krb5Context()
+    {
+        if (m_principal)
+        {
+            krb5_free_principal(m_ctx, m_principal);
+        }
+        if (m_creds)
+        {
+            krb5_free_cred_contents(m_ctx, m_creds);
+        }
+        if (m_ctx)
+        {
+            krb5_free_context(m_ctx);
+        }
+        if (m_profilePtr)
+        {
+            profile_release(m_profile);
+        }
+    }
+
+    virtual void init();
+
+private:
+    void checkResult(krb5_error_code const& error, std::string const& method);
+
+protected:
+    ppc::protocol::Krb5AuthConfig::Ptr m_config;
+    krb5_context m_ctx = NULL;
+    profile_t m_profile;
+    profile_t* m_profilePtr = NULL;
+    krb5_principal m_principal = NULL;
+    krb5_creds m_credsObj;
+    krb5_creds* m_creds = NULL;
+    krb5_ccache m_ccache = NULL;
+};
+
+class Krb5CredLoader
+{
+public:
+    using Ptr = std::shared_ptr;
+    Krb5CredLoader() = default;
+    virtual ~Krb5CredLoader() = default;
+
+    virtual Krb5Context::Ptr load(ppc::protocol::Krb5AuthConfig::Ptr const& config) const
+    {
+        return std::make_shared(config);
+    }
+};
+}  // namespace ppc::storage
\ No newline at end of file
diff --git a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
index 31f923d1..0cd35e2a 100644
--- a/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
+++ b/cpp/wedpr-transport/ppc-rpc/src/RpcMemory.cpp
@@ -86,7 +86,7 @@ bcos::Error::Ptr RpcMemory::insertTask(protocol::Task::Ptr _task)
             RPC_STATUS_LOG(INFO) << LOG_DESC("find the existed not running-task")
                                  << LOG_KV("task", _task->id())
                                  << LOG_KV("status", taskResult->status());
-            if (taskResult->status() != toString(TaskState::COMPLETED))
+            if (taskResult->status() != toString(TaskStatus::COMPLETED))
             {
                 // erase the task_id
                 m_front->eraseTaskInfo(_task->id());
diff --git a/python/ppc_common/deps_services/hdfs_storage.py b/python/ppc_common/deps_services/hdfs_storage.py
index 67f2d81b..aae0d851 100644
--- a/python/ppc_common/deps_services/hdfs_storage.py
+++ b/python/ppc_common/deps_services/hdfs_storage.py
@@ -3,27 +3,20 @@
 
 from hdfs.client import InsecureClient
 from ppc_common.ppc_utils import common_func
-from ppc_common.deps_services.storage_api import StorageApi, StorageType
+from ppc_common.deps_services.storage_api import StorageApi, StorageType, HDFSStorageConfig
 
 from ppc_common.ppc_utils import utils
 
 
 class HdfsStorage(StorageApi):
 
-    DEFAULT_HDFS_USER = "ppc"
-    DEFAULT_HDFS_USER_PATH = "/user/"
-
     # endpoint: http://127.0.0.1:50070
-    def __init__(self, endpoint, hdfs_user, hdfs_home=None):
-        self.endpoint = endpoint
-        self._user = common_func.get_config_value(
-            "HDFS_USER", HdfsStorage.DEFAULT_HDFS_USER, hdfs_user, False)
-        self._hdfs_storage_path = hdfs_home
-        if hdfs_home is None:
-            self._hdfs_storage_path = os.path.join(
-                HdfsStorage.DEFAULT_HDFS_USER_PATH, self._user)
-
-        self.client = InsecureClient(endpoint, user=self._user)
+    def __init__(self, hdfs_config: HDFSStorageConfig, init_client: bool = True):
+        self.endpoint = hdfs_config.hdfs_url
+        self._user = hdfs_config.hdfs_user
+        self._hdfs_storage_path = hdfs_config.hdfs_home
+        if init_client:
+            self.client = InsecureClient(self.endpoint, user=self._user)
 
     def get_home_path(self):
         return self._hdfs_storage_path
diff --git a/python/ppc_common/deps_services/krb5_hdfs_storage.py b/python/ppc_common/deps_services/krb5_hdfs_storage.py
new file mode 100644
index 00000000..fbc643c8
--- /dev/null
+++ b/python/ppc_common/deps_services/krb5_hdfs_storage.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+import os
+from krbcontext.context import krbContext
+from hdfs.ext.kerberos import KerberosClient
+from ppc_common.deps_services.hdfs_storage import HdfsStorage
+from ppc_common.deps_services.storage_api import HDFSStorageConfig
+
+
+class Krb5HdfsStorage(HdfsStorage):
+    def __init__(self, hdfs_config: HDFSStorageConfig):
+        super().__init__(hdfs_config, False)
+        self.hdfs_config = hdfs_config
+        self.krb5_ctx = krbContext(
+            using_keytab=True,
+            principal=self.hdfs_config.hdfs_auth_principal,
+            keytab_file=self.hdfs_config.hdfs_auth_secret_file_path)
+
+        self.client = KerberosClient(self.hdfs_config.hdfs_url)
+        self.client = KerberosClient(
+            krb_principal=self.hdfs_config.hdfs_auth_principal,
+            krb_keytab=self.hdfs_config.hdfs_auth_secret_file_path,
+            krb_ccache_path="/tmp/hdfs",
+            hdfs_namenode_address=self.hdfs_config.hdfs_url,
+            timeout=10000)
diff --git a/python/ppc_common/deps_services/storage_api.py b/python/ppc_common/deps_services/storage_api.py
index 1f623fb1..a3e85b3f 100644
--- a/python/ppc_common/deps_services/storage_api.py
+++ b/python/ppc_common/deps_services/storage_api.py
@@ -1,12 +1,63 @@
 from abc import ABC, abstractmethod
 from enum import Enum
 from typing import AnyStr
+from ppc_common.ppc_utils import common_func
+import os
 
 
 class StorageType(Enum):
     HDFS = 'HDFS'
 
 
+class HDFSStorageConfig:
+    DEFAULT_HDFS_USER = "ppc"
+    DEFAULT_HDFS_USER_PATH = "/user/"
+
+    def __init__(self, hdfs_url: str = None,
+                 hdfs_user: str = None,
+                 hdfs_home: str = None,
+                 enable_krb5_auth: bool = False,
+                 hdfs_auth_principal: str = None,
+                 hdfs_auth_secret_file_path: str = None):
+        self.hdfs_url = hdfs_url
+        self.hdfs_user = hdfs_user
+        self.hdfs_home = hdfs_home
+        self.enable_krb5_auth = enable_krb5_auth
+        self.hdfs_auth_principal = hdfs_auth_principal
+        self.hdfs_auth_secret_file_path = hdfs_auth_secret_file_path
+
+    def __repr__(self):
+        return f"hdfs_user: {self.hdfs_user}, hdfs_home: {self.hdfs_home}, hdfs_url: {self.hdfs_url}, " \
+               f"enable_krb5_auth: {self.enable_krb5_auth}, hdfs_auth_principal: {self.hdfs_auth_principal}, " \
+               f"hdfs_auth_secret_file_path: {self.hdfs_auth_secret_file_path}"
+
+    def load_config(self, config: dict, logger):
+        self.hdfs_url = common_func.get_config_value(
+            'HDFS_URL', None, config, True)
+        self.hdfs_user = common_func.get_config_value(
+            'HDFS_USER', self.DEFAULT_HDFS_USER, config, False)
+        self.hdfs_home = common_func.get_config_value(
+            "HDFS_HOME", os.path.join(self.DEFAULT_HDFS_USER_PATH, self.hdfs_user), config, False)
+        # the auth information
+        self.enable_krb5_auth = common_func.get_config_value(
+            "HDFS_ENABLE_AUTH", False, config, False)
+        require_auth_info = False
+        if self.enable_krb5_auth is True:
+            require_auth_info = True
+        # the principal
+        self.hdfs_auth_principal = common_func.get_config_value(
+            "HDFS_AUTH_PRINCIPAL", None, config, require_auth_info
+        )
+        # the keytab file path
+        self.hdfs_auth_secret_file_path = common_func.get_config_value(
+            "HDFS_AUTH_KEYTAB_PATH", None, config, require_auth_info
+        )
+        if logger is not None:
+            logger.info(f"*** load hdfs storage config : {self}")
+        else:
+            print(f"*** load hdfs storage config : {self}")
+
+
 class StorageApi(ABC):
     @abstractmethod
     def download_file(self, storage_path: str, local_file_path: str, enable_cache=False):
diff --git a/python/ppc_common/deps_services/storage_loader.py b/python/ppc_common/deps_services/storage_loader.py
index 0688648a..991ce37e 100644
--- a/python/ppc_common/deps_services/storage_loader.py
+++ b/python/ppc_common/deps_services/storage_loader.py
@@ -1,14 +1,21 @@
 from ppc_common.deps_services.storage_api import StorageType
+from ppc_common.deps_services.storage_api import HDFSStorageConfig
 from ppc_common.deps_services.hdfs_storage import HdfsStorage
-from ppc_common.ppc_utils import common_func
+from ppc_common.deps_services.krb5_hdfs_storage import Krb5HdfsStorage
+
+
+class HDFSStorageLoader:
+    @staticmethod
+    def load(hdfs_config: HDFSStorageConfig):
+        if hdfs_config.enable_krb5_auth is False:
+            return HdfsStorage(hdfs_config)
+        return Krb5HdfsStorage(hdfs_config)
 
 
 def load(config: dict, logger):
     if config['STORAGE_TYPE'] == StorageType.HDFS.value:
-        hdfs_user = common_func.get_config_value(
-            'HDFS_USER', None, config, False)
-        hdfs_home = common_func.get_config_value(
-            "HDFS_HOME", None, config, False)
-        return HdfsStorage(config['HDFS_URL'],  hdfs_user, hdfs_home)
+        hdfs_config = HDFSStorageConfig()
+        hdfs_config.load_config(config, logger)
+        return HDFSStorageLoader.load(hdfs_config)
     else:
         raise Exception('unsupported storage type')
diff --git a/python/ppc_common/deps_services/tests/hdfs_storage_test.py b/python/ppc_common/deps_services/tests/hdfs_storage_test.py
new file mode 100644
index 00000000..755322be
--- /dev/null
+++ b/python/ppc_common/deps_services/tests/hdfs_storage_test.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+import unittest
+from ppc_common.deps_services.storage_api import HDFSStorageConfig
+from ppc_common.deps_services.storage_loader import HDFSStorageLoader
+
+
+class HDFSStorageWrapper:
+    def __init__(self):
+        # use the default config
+        hdfs_url = "hdfs://127.0.0.1:9900"
+        hdfs_user = "wedpr"
+        hdfs_home = "/user/ppc"
+        enable_krb5_auth = False
+        hdfs_auth_principal = ""
+        hdfs_auth_secret_file_path = ""
+        self.hdfs_config = HDFSStorageConfig(
+            hdfs_url=hdfs_url, hdfs_user=hdfs_user,
+            hdfs_home=hdfs_home, enable_krb5_auth=enable_krb5_auth,
+            hdfs_auth_principal=hdfs_auth_principal,
+            hdfs_auth_secret_file_path=hdfs_auth_secret_file_path)
+        self.hdfs_storage = HDFSStorageLoader.load(self.hdfs_config)
+
+    def test_file_op(self, file_path):
+        hdfs_file_path = f"test/{file_path}"
+        print(f"*** upload file test ***")
+        self.hdfs_storage.upload_file(
+            local_file_path=file_path, hdfs_path=hdfs_file_path)
+        print(f"*** upload file test success***")
+        print(f"*** download file test ***")
+        local_file_path = f"{file_path}.download"
+        self.hdfs_storage.download_file(
+            hdfs_path=hdfs_file_path, local_file_path=local_file_path)
+        print(f"*** download file test success ***")
+
+
+class TestHDFSStorage(unittest.TestCase):
+    def test_file_op(self):
+        file_path = "test.csv"
+        wrapper = HDFSStorageWrapper()
+        wrapper.test_file_op(file_path)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index 1bc97e73..a472260b 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -17,17 +17,14 @@ HDFS_URL: "http://127.0.0.1:50070"
 # HDFS,
 STORAGE_TYPE: "HDFS"
 HDFS_URL: "http://127.0.0.1:9870"
-HDFS_USER: "ppc"
+HDFS_USER: "root"
 HDFS_HOME: "/user/ppc/model/webank"
+HDFS_ENABLE_AUTH: False
+# the hdfs auth principal
+HDFS_AUTH_PRINCIPAL: "root@NODE.DC1.CONSUL"
+# the auth key-tab path
+HDFS_AUTH_KEYTAB_PATH: "./hdfs-wedpr.keytab"
 
-# ECDSA or GM
-CRYPTO_TYPE: "ECDSA"
-private_key: ""
-public_key: ""
-gm_private_key: ""
-gm_public_key: ""
-
-UPLOAD_FOLDER: "./upload_data_folder"
 JOB_TEMP_DIR: ".cache/job"
 
 # the transport config
diff --git a/python/requirements.txt b/python/requirements.txt
index 4476487c..92919807 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -56,3 +56,5 @@ MarkupSafe>=2.1.1
 urllib3==1.26.18
 phe
 chardet
+krbcontext
+requests_kerberos

From 4cc2618ad3f0869f64c5a0240b565ea52b136ba8 Mon Sep 17 00:00:00 2001
From: octopus <912554887@qq.com>
Date: Tue, 3 Dec 2024 16:43:54 +0800
Subject: [PATCH 109/120] upload spdz result file when task failed

---
 .../ppc-mpc/src/MPCService.cpp                | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
index 1bce96ee..0a064e0c 100644
--- a/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
+++ b/cpp/wedpr-computing/ppc-mpc/src/MPCService.cpp
@@ -231,6 +231,17 @@ void MPCService::doRun(const JobInfo& jobInfo)
     std::string outResult;
     execCommand(mpcCmd, outExitStatus, outResult);
 
+    // 4 upload result file
+    std::string resultFileHdfsPath =  jobInfo.outputFilePath;
+    std::string resultFileLocalPath = localPathPrefix + MPC_RESULT_FILE;
+    writeStringToFile(outResult, resultFileLocalPath);
+
+    auto resultFileReader =
+        initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE);
+    auto resultFileWriter =
+        initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS);
+    readAndSaveFile(resultFileLocalPath, resultFileHdfsPath, resultFileReader, resultFileWriter);
+
     if (outExitStatus != MPC_SUCCESS)
     {
         MPC_LOG(ERROR) << LOG_DESC("[MPCService][doRun]") 
@@ -244,17 +255,6 @@ void MPCService::doRun(const JobInfo& jobInfo)
     MPC_LOG(INFO) << LOG_DESC("[MPCService][doRun]") << LOG_KV("jobId", jobId) 
         << LOG_DESC("run mpc job successfully");
 
-    // 4 upload result file
-    std::string resultFileHdfsPath =  jobInfo.outputFilePath;
-    std::string resultFileLocalPath = localPathPrefix + MPC_RESULT_FILE;
-    writeStringToFile(outResult, resultFileLocalPath);
-
-    auto resultFileReader =
-        initialize_lineReader(jobInfo, resultFileLocalPath, DataResourceType::FILE);
-    auto resultFileWriter =
-        initialize_lineWriter(jobInfo, resultFileHdfsPath, DataResourceType::HDFS);
-    readAndSaveFile(resultFileLocalPath, resultFileHdfsPath, resultFileReader, resultFileWriter);
-
     MPC_LOG(INFO) << LOG_DESC("do run mpc") << LOG_KV("jodId", jobInfo.jobId)<< LOG_KV("timecost(ms)", utcSteadyTime() - startT);
 }
 

From ae83bb09e7bbee6784426f95f034bacf06db10ff Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 3 Dec 2024 16:49:30 +0800
Subject: [PATCH 110/120] fix krb5_hdfs_storage (#108)

* ppc_model support krb5 auth

* fix krb5_hdfs_storage

* fix wedpr_ml_toolkit hdfs storage
---
 .../deps_services/krb5_hdfs_storage.py        | 18 +++-------
 .../ppc_common/deps_services/storage_api.py   | 26 +++++++++-----
 .../deps_services/storage_loader.py           |  6 ++--
 .../deps_services/tests/hdfs_storage_test.py  | 19 ++++++----
 python/ppc_common/ppc_utils/common_func.py    |  5 +++
 python/ppc_model/conf/application-sample.yml  |  7 ++--
 python/ppc_model/model_crypto/crypto_aes.py   | 15 ++++----
 python/requirements.txt                       |  3 +-
 .../wedpr_ml_toolkit/conf/config.properties   |  5 +++
 python/wedpr_ml_toolkit/requirements.txt      |  2 +-
 .../common/utils/hdfs_storage_impl.py         | 32 +++++++++--------
 .../wedpr_ml_toolkit/common/utils/utils.py    |  6 ++++
 .../config/wedpr_ml_config.py                 | 35 +++++++++++++++----
 .../context/result/result_context.py          |  2 +-
 .../transport/storage_entrypoint.py           |  6 ++--
 .../wedpr_ml_toolkit/wedpr_ml_toolkit.py      |  4 +--
 16 files changed, 118 insertions(+), 73 deletions(-)

diff --git a/python/ppc_common/deps_services/krb5_hdfs_storage.py b/python/ppc_common/deps_services/krb5_hdfs_storage.py
index fbc643c8..fe5de3c1 100644
--- a/python/ppc_common/deps_services/krb5_hdfs_storage.py
+++ b/python/ppc_common/deps_services/krb5_hdfs_storage.py
@@ -1,24 +1,16 @@
 # -*- coding: utf-8 -*-
-import os
-from krbcontext.context import krbContext
 from hdfs.ext.kerberos import KerberosClient
 from ppc_common.deps_services.hdfs_storage import HdfsStorage
 from ppc_common.deps_services.storage_api import HDFSStorageConfig
 
 
 class Krb5HdfsStorage(HdfsStorage):
-    def __init__(self, hdfs_config: HDFSStorageConfig):
+    def __init__(self, hdfs_config: HDFSStorageConfig, logger):
         super().__init__(hdfs_config, False)
         self.hdfs_config = hdfs_config
-        self.krb5_ctx = krbContext(
-            using_keytab=True,
-            principal=self.hdfs_config.hdfs_auth_principal,
-            keytab_file=self.hdfs_config.hdfs_auth_secret_file_path)
-
-        self.client = KerberosClient(self.hdfs_config.hdfs_url)
         self.client = KerberosClient(
-            krb_principal=self.hdfs_config.hdfs_auth_principal,
-            krb_keytab=self.hdfs_config.hdfs_auth_secret_file_path,
-            krb_ccache_path="/tmp/hdfs",
-            hdfs_namenode_address=self.hdfs_config.hdfs_url,
+            url=self.hdfs_config.hdfs_url,
+            principal=self.hdfs_config.hdfs_auth_principal,
+            hostname_override=self.hdfs_config.hdfs_hostname_override,
+            password=self.hdfs_config.hdfs_auth_password,
             timeout=10000)
diff --git a/python/ppc_common/deps_services/storage_api.py b/python/ppc_common/deps_services/storage_api.py
index a3e85b3f..c07ad210 100644
--- a/python/ppc_common/deps_services/storage_api.py
+++ b/python/ppc_common/deps_services/storage_api.py
@@ -18,18 +18,19 @@ def __init__(self, hdfs_url: str = None,
                  hdfs_home: str = None,
                  enable_krb5_auth: bool = False,
                  hdfs_auth_principal: str = None,
-                 hdfs_auth_secret_file_path: str = None):
+                 hdfs_auth_password: str = None,
+                 hdfs_hostname_override: str = None):
         self.hdfs_url = hdfs_url
         self.hdfs_user = hdfs_user
         self.hdfs_home = hdfs_home
         self.enable_krb5_auth = enable_krb5_auth
         self.hdfs_auth_principal = hdfs_auth_principal
-        self.hdfs_auth_secret_file_path = hdfs_auth_secret_file_path
+        self.hdfs_auth_password = hdfs_auth_password
+        self.hdfs_hostname_override = hdfs_hostname_override
 
     def __repr__(self):
         return f"hdfs_user: {self.hdfs_user}, hdfs_home: {self.hdfs_home}, hdfs_url: {self.hdfs_url}, " \
-               f"enable_krb5_auth: {self.enable_krb5_auth}, hdfs_auth_principal: {self.hdfs_auth_principal}, " \
-               f"hdfs_auth_secret_file_path: {self.hdfs_auth_secret_file_path}"
+               f"enable_krb5_auth: {self.enable_krb5_auth}, hdfs_auth_principal: {self.hdfs_auth_principal}"
 
     def load_config(self, config: dict, logger):
         self.hdfs_url = common_func.get_config_value(
@@ -38,6 +39,7 @@ def load_config(self, config: dict, logger):
             'HDFS_USER', self.DEFAULT_HDFS_USER, config, False)
         self.hdfs_home = common_func.get_config_value(
             "HDFS_HOME", os.path.join(self.DEFAULT_HDFS_USER_PATH, self.hdfs_user), config, False)
+
         # the auth information
         self.enable_krb5_auth = common_func.get_config_value(
             "HDFS_ENABLE_AUTH", False, config, False)
@@ -48,14 +50,22 @@ def load_config(self, config: dict, logger):
         self.hdfs_auth_principal = common_func.get_config_value(
             "HDFS_AUTH_PRINCIPAL", None, config, require_auth_info
         )
-        # the keytab file path
-        self.hdfs_auth_secret_file_path = common_func.get_config_value(
-            "HDFS_AUTH_KEYTAB_PATH", None, config, require_auth_info
-        )
+        # the password
+        self.hdfs_auth_password = common_func.get_config_value(
+            "HDFS_AUTH_PASSWORD", None, config, require_auth_info)
+        #  the hostname override
+        self.hdfs_hostname_override = common_func.get_config_value(
+            "HDFS_HOSTNAME_OVERRIDE", None, config, require_auth_info)
         if logger is not None:
             logger.info(f"*** load hdfs storage config : {self}")
         else:
             print(f"*** load hdfs storage config : {self}")
+        self._check()
+
+    def _check(self):
+        common_func.require_non_empty("HDFS_URL", self.hdfs_url)
+        common_func.require_non_empty("HDFS_USER", self.hdfs_user)
+        common_func.require_non_empty("HDFS_HOME", self.hdfs_home)
 
 
 class StorageApi(ABC):
diff --git a/python/ppc_common/deps_services/storage_loader.py b/python/ppc_common/deps_services/storage_loader.py
index 991ce37e..959d6e78 100644
--- a/python/ppc_common/deps_services/storage_loader.py
+++ b/python/ppc_common/deps_services/storage_loader.py
@@ -6,16 +6,16 @@
 
 class HDFSStorageLoader:
     @staticmethod
-    def load(hdfs_config: HDFSStorageConfig):
+    def load(hdfs_config: HDFSStorageConfig, logger):
         if hdfs_config.enable_krb5_auth is False:
             return HdfsStorage(hdfs_config)
-        return Krb5HdfsStorage(hdfs_config)
+        return Krb5HdfsStorage(hdfs_config, logger)
 
 
 def load(config: dict, logger):
     if config['STORAGE_TYPE'] == StorageType.HDFS.value:
         hdfs_config = HDFSStorageConfig()
         hdfs_config.load_config(config, logger)
-        return HDFSStorageLoader.load(hdfs_config)
+        return HDFSStorageLoader.load(hdfs_config, logger)
     else:
         raise Exception('unsupported storage type')
diff --git a/python/ppc_common/deps_services/tests/hdfs_storage_test.py b/python/ppc_common/deps_services/tests/hdfs_storage_test.py
index 755322be..0618b04b 100644
--- a/python/ppc_common/deps_services/tests/hdfs_storage_test.py
+++ b/python/ppc_common/deps_services/tests/hdfs_storage_test.py
@@ -2,23 +2,28 @@
 import unittest
 from ppc_common.deps_services.storage_api import HDFSStorageConfig
 from ppc_common.deps_services.storage_loader import HDFSStorageLoader
+import logging
 
 
 class HDFSStorageWrapper:
     def __init__(self):
+        self.logger = logging.getLogger("HDFSStorageWrapper")
         # use the default config
-        hdfs_url = "hdfs://127.0.0.1:9900"
-        hdfs_user = "wedpr"
+        hdfs_url = "http://127.0.0.1:50070"
+        hdfs_user = "root"
         hdfs_home = "/user/ppc"
-        enable_krb5_auth = False
-        hdfs_auth_principal = ""
-        hdfs_auth_secret_file_path = ""
+        enable_krb5_auth = True
+        hdfs_auth_principal = "root@NODE.DC1.CONSUL"
+        hdfs_auth_password = "root"
+        hdfs_hostname_override = "wedpr-0001"
         self.hdfs_config = HDFSStorageConfig(
             hdfs_url=hdfs_url, hdfs_user=hdfs_user,
             hdfs_home=hdfs_home, enable_krb5_auth=enable_krb5_auth,
             hdfs_auth_principal=hdfs_auth_principal,
-            hdfs_auth_secret_file_path=hdfs_auth_secret_file_path)
-        self.hdfs_storage = HDFSStorageLoader.load(self.hdfs_config)
+            hdfs_auth_password=hdfs_auth_password,
+            hdfs_hostname_override=hdfs_hostname_override)
+        self.hdfs_storage = HDFSStorageLoader.load(
+            self.hdfs_config, self.logger)
 
     def test_file_op(self, file_path):
         hdfs_file_path = f"test/{file_path}"
diff --git a/python/ppc_common/ppc_utils/common_func.py b/python/ppc_common/ppc_utils/common_func.py
index e907d6bd..a39a22c1 100644
--- a/python/ppc_common/ppc_utils/common_func.py
+++ b/python/ppc_common/ppc_utils/common_func.py
@@ -26,3 +26,8 @@ def get_file_encoding(file_path):
             raise Exception(f"Unknown File Encoding, file: {file_path}")
         encoding = file_chardet["encoding"]
     return encoding
+
+
+def require_non_empty(value_property, value):
+    if value is None or len(value) == 0:
+        raise Exception(f"the ${value_property} must non-empty!")
diff --git a/python/ppc_model/conf/application-sample.yml b/python/ppc_model/conf/application-sample.yml
index a472260b..3c69234c 100644
--- a/python/ppc_model/conf/application-sample.yml
+++ b/python/ppc_model/conf/application-sample.yml
@@ -16,14 +16,15 @@ SQLALCHEMY_DATABASE_URI: "mysql://[*user_ppcsmodeladm]:[*pass_ppcsmodeladm]@[@43
 HDFS_URL: "http://127.0.0.1:50070"
 # HDFS,
 STORAGE_TYPE: "HDFS"
-HDFS_URL: "http://127.0.0.1:9870"
 HDFS_USER: "root"
 HDFS_HOME: "/user/ppc/model/webank"
 HDFS_ENABLE_AUTH: False
 # the hdfs auth principal
 HDFS_AUTH_PRINCIPAL: "root@NODE.DC1.CONSUL"
-# the auth key-tab path
-HDFS_AUTH_KEYTAB_PATH: "./hdfs-wedpr.keytab"
+# the auth password
+HDFS_AUTH_PASSWORD: ""
+# the host name override
+HDFS_HOSTNAME_OVERRIDE: "wedpr-0001"
 
 JOB_TEMP_DIR: ".cache/job"
 
diff --git a/python/ppc_model/model_crypto/crypto_aes.py b/python/ppc_model/model_crypto/crypto_aes.py
index e63ee2ea..df0e6f40 100644
--- a/python/ppc_model/model_crypto/crypto_aes.py
+++ b/python/ppc_model/model_crypto/crypto_aes.py
@@ -23,17 +23,16 @@ def load_key_from_file(filename):
         key = file.read()
     return key
 
-
-# key = load_key_from_file('aes_key.bin')
+# AES加密函数
 
 
-# AES加密函数
 def encrypt_data(key, plaintext):
     # 使用随机生成的初始向量 (IV)
     iv = os.urandom(16)  # AES块大小为128位（16字节）
-    
+
     # 创建AES加密器
-    cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+    cipher = Cipher(algorithms.AES(key), modes.CBC(iv),
+                    backend=default_backend())
     encryptor = cipher.encryptor()
 
     # 对数据进行填充（AES要求输入的块大小为128位）
@@ -54,11 +53,13 @@ def decrypt_data(key, ciphertext):
     actual_ciphertext = ciphertext[16:]
 
     # 创建AES解密器
-    cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+    cipher = Cipher(algorithms.AES(key), modes.CBC(iv),
+                    backend=default_backend())
     decryptor = cipher.decryptor()
 
     # 解密数据
-    decrypted_padded_data = decryptor.update(actual_ciphertext) + decryptor.finalize()
+    decrypted_padded_data = decryptor.update(
+        actual_ciphertext) + decryptor.finalize()
 
     # 去除填充
     unpadder = padding.PKCS7(128).unpadder()
diff --git a/python/requirements.txt b/python/requirements.txt
index 92919807..7b753c1b 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -56,5 +56,4 @@ MarkupSafe>=2.1.1
 urllib3==1.26.18
 phe
 chardet
-krbcontext
-requests_kerberos
+requests_kerberos>=0.15.0
diff --git a/python/wedpr_ml_toolkit/conf/config.properties b/python/wedpr_ml_toolkit/conf/config.properties
index 8e8aaa5b..3fdec9b3 100644
--- a/python/wedpr_ml_toolkit/conf/config.properties
+++ b/python/wedpr_ml_toolkit/conf/config.properties
@@ -5,4 +5,9 @@ remote_entrypoints=http://127.0.0.1:16000,http://127.0.0.1:16001
 agency_name=WeBank
 workspace_path=/user/wedpr/webank/
 user=test_user
+
 storage_endpoint=http://127.0.0.1:50070
+enable_krb5_auth=False
+hdfs_auth_principal=root@NODE.DC1.CONSUL
+hdfs_auth_password=root
+hdfs_hostname_override=wedpr-0001
\ No newline at end of file
diff --git a/python/wedpr_ml_toolkit/requirements.txt b/python/wedpr_ml_toolkit/requirements.txt
index 39045603..fd33a62e 100644
--- a/python/wedpr_ml_toolkit/requirements.txt
+++ b/python/wedpr_ml_toolkit/requirements.txt
@@ -2,4 +2,4 @@ setuptools>=70.0.0
 hdfs>=2.7.2
 requests~=2.31.0
 requests_toolbelt==0.9.1
-
+requests_kerberos>=0.15.0
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py
index 2e525d94..761d152f 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/hdfs_storage_impl.py
@@ -3,25 +3,27 @@
 
 from hdfs.client import InsecureClient
 from wedpr_ml_toolkit.common.utils import utils
+from wedpr_ml_toolkit.config.wedpr_ml_config import StorageConfig
+from hdfs.ext.kerberos import KerberosClient
 
 
 class HdfsStorageImpl:
-
-    DEFAULT_HDFS_USER = "ppc"
-    DEFAULT_HDFS_USER_PATH = "/user/"
-
     # endpoint: http://127.0.0.1:50070
-    def __init__(self, endpoint, hdfs_user, hdfs_home=None):
-        self.endpoint = endpoint
-        self._user = hdfs_user
-        self._hdfs_storage_path = hdfs_home
-        if hdfs_home is None:
-            self._hdfs_storage_path = os.path.join(
-                HdfsStorage.DEFAULT_HDFS_USER_PATH, self._user)
-
-        self.client = InsecureClient(endpoint, user=self._user)
-        # print(self.client.list('/'))
-        # print(self.client.list('/user/root/'))
+    def __init__(self, storage_config: StorageConfig):
+        self.hdfs_config = storage_config
+        self._hdfs_storage_path = self.hdfs_config.user_config.get_workspace_path()
+        self.client = None
+        if self.hdfs_config.get_enable_krb5_auth() is True:
+            self.client = KerberosClient(
+                url=self.hdfs_config.storage_endpoint,
+                principal=self.hdfs_config.hdfs_auth_principal,
+                hostname_override=self.hdfs_config.hdfs_hostname_override,
+                password=self.hdfs_config.hdfs_auth_password,
+                timeout=10000)
+        else:
+            self.client = InsecureClient(
+                self.hdfs_config.storage_endpoint,
+                user=self.hdfs_config.user_config.user)
 
     def get_home_path(self):
         return self._hdfs_storage_path
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
index 0468971e..9bc61962 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/common/utils/utils.py
@@ -50,3 +50,9 @@ def get_config_value(key, default_value, config_value, required):
     if value is None:
         return default_value
     return value
+
+
+def str_to_bool(str_value):
+    if str_value.lower() == "true":
+        return True
+    return False
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
index f6648aad..c30d6ec9 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
@@ -3,6 +3,7 @@
 from wedpr_ml_toolkit.common.utils.base_object import BaseObject
 from wedpr_ml_toolkit.common.utils.constant import Constant
 from wedpr_ml_toolkit.common.utils.properies_parser import Properties
+from wedpr_ml_toolkit.common.utils import utils
 
 
 class AuthConfig(BaseObject):
@@ -39,11 +40,6 @@ def __init__(self,
         self.update_dataset_uri = update_dataset_uri
 
 
-class StorageConfig(BaseObject):
-    def __init__(self, storage_endpoint: str = None):
-        self.storage_endpoint = storage_endpoint
-
-
 class UserConfig(BaseObject):
     def __init__(self, agency_name: str = None, workspace_path: str = None, user_name: str = None):
         self.agency_name = agency_name
@@ -54,6 +50,31 @@ def get_workspace_path(self):
         return os.path.join(self.workspace_path, self.user)
 
 
+class StorageConfig(BaseObject):
+    def __init__(self,
+                 user_config: UserConfig,
+                 storage_endpoint: str = None,
+                 enable_krb5_auth: bool = False,
+                 hdfs_auth_principal: str = None,
+                 hdfs_auth_password: str = None,
+                 hdfs_hostname_override: str = None):
+        self.user_config = user_config
+        self.storage_endpoint = storage_endpoint
+        self.enable_krb5_auth = enable_krb5_auth
+        self.hdfs_auth_principal = hdfs_auth_principal
+        self.hdfs_auth_password = hdfs_auth_password
+        self.hdfs_hostname_override = hdfs_hostname_override
+
+    def get_enable_krb5_auth(self) -> bool:
+        if self.enable_krb5_auth is None or len(self.enable_krb5_auth) == 0:
+            return False
+        return utils.str_to_bool(self.enable_krb5_auth)
+
+    def __repr__(self):
+        return f"hdfs_user: {self.hdfs_user}, hdfs_home: {self.hdfs_home}, hdfs_url: {self.hdfs_url}, " \
+            f"enable_krb5_auth: {self.enable_krb5_auth}, hdfs_auth_principal: {self.hdfs_auth_principal}"
+
+
 class HttpConfig(BaseObject):
     def __init__(self, timeout_seconds=3):
         self.timeout_seconds = timeout_seconds
@@ -65,13 +86,13 @@ def __init__(self, config_dict):
         self.auth_config.set_params(**config_dict)
         self.job_config = JobConfig()
         self.job_config.set_params(**config_dict)
-        self.storage_config = StorageConfig()
-        self.storage_config.set_params(**config_dict)
         self.user_config = UserConfig()
         self.user_config.set_params(**config_dict)
         self.http_config = HttpConfig()
         self.http_config.set_params(**config_dict)
         self.dataset_config = DatasetConfig()
+        self.storage_config = StorageConfig(self.user_config)
+        self.storage_config.set_params(**config_dict)
 
 
 class WeDPRMlConfigBuilder:
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
index fba85a01..e580d018 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/context/result/result_context.py
@@ -56,7 +56,7 @@ def check_and_get_job_type(job_context: JobContext, job_result_detail: JobDetail
         return job_result_detail.job_object.jobType
 
     def _generate_result_dataset_(self, dataset_file_path):
-        dataset_meta = DatasetMeta(user=self.job_context.storage_entry_point.user_config.user,
+        dataset_meta = DatasetMeta(user=self.job_context.user_config.user,
                                    agency=self.job_context.user_config.agency_name,
                                    file_path=dataset_file_path)
         return DatasetContext(storage_entrypoint=self.job_context.storage_entry_point,
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
index 092b1de0..dd750102 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/storage_entrypoint.py
@@ -7,11 +7,9 @@
 
 
 class StorageEntryPoint:
-    def __init__(self, user_config: UserConfig, storage_config: StorageConfig):
+    def __init__(self, storage_config: StorageConfig):
         self.storage_config = storage_config
-        self.user_config = user_config
-        self.storage_client = HdfsStorageImpl(
-            self.storage_config.storage_endpoint, self.user_config.user, self.user_config.get_workspace_path())
+        self.storage_client = HdfsStorageImpl(self.storage_config)
 
     def upload_bytes(self, data, hdfs_path):
         self.storage_client.save_data(data, hdfs_path)
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
index 7f9c01c2..ab2383c3 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/wedpr_ml_toolkit.py
@@ -33,8 +33,8 @@ def __init__(self, config: WeDPRMlConfig):
         self.dataset_client = WeDPRDatasetClient(http_config=self.config.http_config,
                                                  auth_config=self.config.auth_config,
                                                  dataset_config=self.config.dataset_config)
-        self.storage_entry_point = StorageEntryPoint(self.config.user_config,
-                                                     self.config.storage_config)
+        self.storage_entry_point = StorageEntryPoint(
+            self.config.storage_config)
 
     def get_config(self) -> WeDPRMlConfig:
         return self.config

From 96876f6610832d0d8c81819eb001d57952d840b9 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 4 Dec 2024 20:32:00 +0800
Subject: [PATCH 111/120] update toolkit version (#110)

* fix ci

* update toolkit version

* add Dockerfile

* add agencyID to psi/mpc serviceName

* support fetch log and fetch job result seperately
---
 .github/workflows/cpp_full_node_workflow.yml  |  3 +-
 .github/workflows/cpp_sdk_workflow.yml        |  2 +-
 .github/workflows/cpp_toolkit_workflow.yml    |  4 +-
 cpp/tools/install_depends.sh                  | 22 +++++-----
 cpp/wedpr-helper/ppc-utilities/Utilities.h    |  5 +++
 cpp/wedpr-initializer/Initializer.cpp         |  6 ++-
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp    | 11 ++---
 docker-files/cpp/Dockerfile                   | 44 +++++++++++++++++++
 .../model_result/task_result_handler.py       | 24 +++++-----
 .../network/http/model_controller.py          |  8 ++--
 python/wedpr_ml_toolkit/setup.py              |  2 +-
 .../config/wedpr_ml_config.py                 |  2 +-
 12 files changed, 95 insertions(+), 38 deletions(-)
 create mode 100644 docker-files/cpp/Dockerfile

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 4512a8e2..d8f263e0 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, ubuntu-20.04]
+        os: [macos-13, ubuntu-20.04]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -74,6 +74,7 @@ jobs:
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
+          rm -rf /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl
           mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Publish Error
diff --git a/.github/workflows/cpp_sdk_workflow.yml b/.github/workflows/cpp_sdk_workflow.yml
index 4c361914..3ba1ec06 100644
--- a/.github/workflows/cpp_sdk_workflow.yml
+++ b/.github/workflows/cpp_sdk_workflow.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-2019, macos-12]
+        os: [ubuntu-22.04, windows-2019, macos-13]
     steps:
       - uses: actions/checkout@v2
         with:
diff --git a/.github/workflows/cpp_toolkit_workflow.yml b/.github/workflows/cpp_toolkit_workflow.yml
index 04635c77..025f276c 100644
--- a/.github/workflows/cpp_toolkit_workflow.yml
+++ b/.github/workflows/cpp_toolkit_workflow.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-2019, macos-12]
+        os: [ubuntu-22.04, windows-2019, macos-13]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -72,7 +72,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=OFF -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index 5852e25f..1e2a1e6c 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -75,12 +75,12 @@ install_centos_depends()
 {
     LOG_INFO "install depends for centos ..."
     # install the basic package
-    sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf
+    sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel autoconf
     LOG_INFO "install basic-package-depends for centos success..."
     
-    LOG_INFO "install gsasl ... "
-    install_gsasl_depend "centos"
-    LOG_INFO "install gsasl success!"
+    #LOG_INFO "install gsasl ... "
+    #install_gsasl_depend "centos"
+    #LOG_INFO "install gsasl success!"
 
     LOG_INFO "install nasm ... "
     install_nasm_depend
@@ -93,12 +93,12 @@ install_ubuntu_depends()
 {
     LOG_INFO "install depends for ubuntu ..."
     # install the basic package
-    sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf
+    sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget autoconf
     LOG_INFO "install basic-package-depends for ubuntu success..."
    
-    LOG_INFO "install gsasl ... "
-    install_gsasl_depend "ubuntu"
-    LOG_INFO "install gsasl success!"
+    #LOG_INFO "install gsasl ... "
+    #install_gsasl_depend "ubuntu"
+    #LOG_INFO "install gsasl success!"
 
     LOG_INFO "install nasm ... "
     install_nasm_depend
@@ -125,9 +125,9 @@ install_iconv_depend()
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
-    brew install autoconf nasm lcov iconv
-    install_gsasl_depend "macos"
-    install_iconv_depend
+    brew install autoconf nasm lcov
+    #install_gsasl_depend "macos"
+    #install_iconv_depend
     LOG_INFO "install depends for macos success ..."
 }
 
diff --git a/cpp/wedpr-helper/ppc-utilities/Utilities.h b/cpp/wedpr-helper/ppc-utilities/Utilities.h
index c5e52009..9832b65d 100644
--- a/cpp/wedpr-helper/ppc-utilities/Utilities.h
+++ b/cpp/wedpr-helper/ppc-utilities/Utilities.h
@@ -64,6 +64,11 @@ inline std::string generateUUID()
     static thread_local auto uuid_gen = boost::uuids::basic_random_generator();
     return boost::uuids::to_string(uuid_gen());
 }
+
+inline std::string getServiceName(std::string const& agencyID, std::string const& serviceID)
+{
+    return agencyID + "_" + serviceID;
+}
 template 
 inline std::string printVector(T const& list)
 {
diff --git a/cpp/wedpr-initializer/Initializer.cpp b/cpp/wedpr-initializer/Initializer.cpp
index 384fd44a..a0ecaea0 100644
--- a/cpp/wedpr-initializer/Initializer.cpp
+++ b/cpp/wedpr-initializer/Initializer.cpp
@@ -40,6 +40,7 @@
 #include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h"
 #include "ppc-psi/src/ra2018-psi/RA2018PSIFactory.h"
 #include "ppc-tools/src/config/PPCConfig.h"
+#include "wedpr-helper/ppc-utilities/Utilities.h"
 #include 
 #include 
 
@@ -86,8 +87,9 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     TransportBuilder transportBuilder;
     // register the serviceInfo
     auto serviceConfig = m_serviceConfigBuilder.buildServiceConfig();
-    auto entryPoint =
-        m_serviceConfigBuilder.buildEntryPoint(PSI_SERVICE_TYPE, m_config->accessEntrypoint());
+    auto entryPoint = m_serviceConfigBuilder.buildEntryPoint(
+        getServiceName(m_config->agencyID(), PSI_SERVICE_TYPE), m_config->accessEntrypoint());
+
     serviceConfig.addEntryPoint(entryPoint);
     auto serviceMeta = serviceConfig.encode();
     m_config->frontConfig()->setMeta(serviceMeta);
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index afe8dd20..36187f03 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -22,6 +22,7 @@
 #include "ppc-framework/protocol/Constant.h"
 #include "ppc-mpc/src/MPCService.h"
 #include "ppc-tools/src/config/PPCConfig.h"
+#include "wedpr-helper/ppc-utilities/Utilities.h"
 #include "wedpr-protocol/protocol/src/ServiceConfig.h"
 #include "wedpr-transport/sdk/src/TransportBuilder.h"
 
@@ -61,7 +62,7 @@ void MPCInitializer::init(std::string const& _configPath)
     auto threadPool = std::make_shared("mpc-pool", threadPoolSize);
 
     INIT_LOG(INFO) << LOG_DESC("init the mpc threadpool")
-        << LOG_KV("threadPoolSize", threadPoolSize);
+                   << LOG_KV("threadPoolSize", threadPoolSize);
 
     auto mpcService = std::make_shared();
     mpcService->setMPCConfig(mpcConfig);
@@ -71,11 +72,11 @@ void MPCInitializer::init(std::string const& _configPath)
     m_rpc->registerHandler("run", std::bind(&MPCService::runMpcRpc, mpcService,
                                       std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("asyncRun", std::bind(&MPCService::asyncRunMpcRpc, mpcService,
-                                      std::placeholders::_1, std::placeholders::_2));
+                                           std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService,
                                        std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("query", std::bind(&MPCService::queryMpcRpc, mpcService,
-                                       std::placeholders::_1, std::placeholders::_2));
+                                        std::placeholders::_1, std::placeholders::_2));
     INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success");
     // init the transport
     initTransport(pt);
@@ -89,8 +90,8 @@ void MPCInitializer::initTransport(boost::property_tree::ptree const& property)
 
     // add the service meta
     ServiceConfigBuilder serviceConfigBuilder;
-    auto entryPoint =
-        serviceConfigBuilder.buildEntryPoint(MPC_SERVICE_TYPE, m_config->accessEntrypoint());
+    auto entryPoint = serviceConfigBuilder.buildEntryPoint(
+        getServiceName(m_config->agencyID(), MPC_SERVICE_TYPE), m_config->accessEntrypoint());
     auto serviceConfig = serviceConfigBuilder.buildServiceConfig();
     serviceConfig.addEntryPoint(entryPoint);
     auto serviceMeta = serviceConfig.encode();
diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
new file mode 100644
index 00000000..6be13fc1
--- /dev/null
+++ b/docker-files/cpp/Dockerfile
@@ -0,0 +1,44 @@
+FROM ubuntu:18.04 as builder
+
+LABEL maintainer service@webank.com
+
+WORKDIR /
+
+ARG SOURCE_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive \
+    SOURCE=${SOURCE_BRANCH:-master}
+
+RUN apt-get -q update && apt-get install -qy --no-install-recommends \
+    vim curl git make build-essential cmake  \
+    libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config curl zip tar ccache uuid-runtime automake autoconf \
+    m4 tcpdump net-tools libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev gcc g++ \
+    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && apt-get install -qy --no-install-recommends tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+# install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES=1
+
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} \
+    && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \
+    && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log
+
+
+FROM ubuntu:18.04
+LABEL maintainer service@webank.com
+    
+RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev \
+        && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+        && apt-get install -qy --no-install-recommends tzdata \
+        && dpkg-reconfigure --frontend noninteractive tzdata \
+        && rm -rf /var/lib/apt/lists/*
+    
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /usr/local/bin/ppc-gateway-service
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /usr/local/bin/ppc-pro-node
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-air-node /usr/local/bin/ppc-air-node
+COPY --from=builder /WeDPR-Component/cpp/build/bin/wedpr-mpc /usr/local/bin/wedpr-mpc
\ No newline at end of file
diff --git a/python/ppc_model/model_result/task_result_handler.py b/python/ppc_model/model_result/task_result_handler.py
index d678bcde..70f99a6e 100644
--- a/python/ppc_model/model_result/task_result_handler.py
+++ b/python/ppc_model/model_result/task_result_handler.py
@@ -11,10 +11,13 @@
 
 
 class TaskResultRequest:
-    def __init__(self, job_id, task_type, only_fetch_log, user):
+    def __init__(self, job_id, task_type,
+                 fetch_log: bool,
+                 fetch_result: bool, user):
         self.job_id = job_id
         self.task_type = task_type
-        self.only_fetch_log = only_fetch_log
+        self.fetch_log = fetch_log
+        self.fetch_result = fetch_result
         self.user = user
 
 
@@ -346,7 +349,7 @@ def __init__(self, task_result_request: TaskResultRequest, components: Initializ
 
     def get_response(self):
         response = dict()
-        if not self.task_result_request.only_fetch_log:
+        if self.task_result_request.fetch_result is True:
             merged_result = dict()
             for result in self.result_list:
                 merged_result.update(result.to_dict())
@@ -356,13 +359,14 @@ def get_response(self):
                 response = {"jobPlanetResult":  merged_result,
                             "modelData": self.model_data}
         # record the log
-        log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
-            self.task_result_request.job_id, self.task_result_request.user)
-        log_result = {}
-        log_result.update({"logSize": log_size})
-        log_result.update({"logPath": log_path})
-        log_result.update({"logContent": log_content})
-        response.update({"logDetail": log_result})
+        if self.task_result_request.fetch_log is True:
+            log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
+                self.task_result_request.job_id, self.task_result_request.user)
+            log_result = {}
+            log_result.update({"logSize": log_size})
+            log_result.update({"logPath": log_path})
+            log_result.update({"logContent": log_content})
+            response.update({"logDetail": log_result})
         return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response)
 
     def _get_evaluation_result(self):
diff --git a/python/ppc_model/network/http/model_controller.py b/python/ppc_model/network/http/model_controller.py
index 39e3301f..d99efc2a 100644
--- a/python/ppc_model/network/http/model_controller.py
+++ b/python/ppc_model/network/http/model_controller.py
@@ -105,12 +105,12 @@ def post(self, task_id):
                 f"get task result, task_id: {task_id}, args: {args}")
             user_name = args['user']
             task_type = args['jobType']
-            only_fetch_log = {'True': True, 'False': False}.get(
-                args['onlyFetchLog'])
+            fetch_log = args['fetchLog']
+            fetch_job_result = args['fetchJobResult']
             components.logger().info(
-                f"get_job_direct_result_response, job: {task_id}")
+                f"get_job_direct_result_response, job: {task_id}, fetch_log: {fetch_log}, fetch_job_result: {fetch_job_result}")
             task_result_request = TaskResultRequest(
-                task_id, task_type, only_fetch_log, user_name)
+                task_id, task_type, fetch_log, fetch_job_result, user_name)
             job_result_handler = TaskResultHandler(
                 task_result_request=task_result_request, components=components)
             response = job_result_handler.get_response()
diff --git a/python/wedpr_ml_toolkit/setup.py b/python/wedpr_ml_toolkit/setup.py
index e78e60b7..e1bba009 100644
--- a/python/wedpr_ml_toolkit/setup.py
+++ b/python/wedpr_ml_toolkit/setup.py
@@ -20,7 +20,7 @@ def run(self):
 setup_args = dict(
     name='wedpr_ml_toolkit',
     packages=find_packages(),
-    version="1.0.0.dev-20241129",
+    version="1.0.0.dev-20241201",
     description="wedpr-ml-toolkit: The ML toolkit for WeDPR",
     long_description_content_type="text/markdown",
     author="WeDPR Development Team",
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
index c30d6ec9..2b5f6e9c 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/config/wedpr_ml_config.py
@@ -54,7 +54,7 @@ class StorageConfig(BaseObject):
     def __init__(self,
                  user_config: UserConfig,
                  storage_endpoint: str = None,
-                 enable_krb5_auth: bool = False,
+                 enable_krb5_auth: str = "False",
                  hdfs_auth_principal: str = None,
                  hdfs_auth_password: str = None,
                  hdfs_hostname_override: str = None):

From 187ecc1e21092ab5d4357bc2349defbd5179ff72 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Thu, 5 Dec 2024 14:36:02 +0800
Subject: [PATCH 112/120] add meta to peers info (#111)

---
 cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp
index 2ecde528..131a7eda 100644
--- a/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp
+++ b/cpp/wedpr-protocol/protobuf/src/NodeInfoImpl.cpp
@@ -62,4 +62,5 @@ void NodeInfoImpl::toJson(Json::Value& jsonObject) const
         componentsInfo.append(it);
     }
     jsonObject["components"] = componentsInfo;
+    jsonObject["meta"] = meta();
 }
\ No newline at end of file

From 9609ec7dce9005b7810a585208c9f12dfb15e1bf Mon Sep 17 00:00:00 2001
From: ywy2090 <912554887@qq.com>
Date: Fri, 6 Dec 2024 09:42:07 +0800
Subject: [PATCH 113/120] register spdz connected endpoint info 2 gateway
 (#112)

* register spdz connected endpoint info 2 gateway

* fix agency id not load bug
---
 cpp/ppc-framework/protocol/Constant.h              |  1 +
 cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h  |  3 +++
 .../ppc-tools/src/config/PPCConfig.cpp             | 14 ++++++++++++++
 cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h  |  5 +++++
 cpp/wedpr-main/mpc-node/MPCInitializer.cpp         | 12 +++++++++---
 5 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/cpp/ppc-framework/protocol/Constant.h b/cpp/ppc-framework/protocol/Constant.h
index fd9b8306..0b441f4b 100644
--- a/cpp/ppc-framework/protocol/Constant.h
+++ b/cpp/ppc-framework/protocol/Constant.h
@@ -26,5 +26,6 @@ namespace ppc::protocol
 {
 const static std::string PSI_SERVICE_TYPE = "PSI";
 const static std::string MPC_SERVICE_TYPE = "MPC";
+const static std::string SPDZ_SERVICE_TYPE = "SPDZ";
 const static size_t LARGE_MSG_THRESHOLD = 30 * 1024 * 1024;
 }  // namespace ppc::protocol
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
index 955c3924..c4e9418b 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/MPCConfig.h
@@ -30,5 +30,8 @@ struct MPCConfig
     std::string mpcRootPathNoGateway;
     uint64_t readPerBatchLines;
     int threadPoolSize;
+
+    std::string spdzConnectedIP;
+    int spdzConnectedPort;
 };
 }  // namespace ppc::tools
\ No newline at end of file
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
index 59b34c2a..51aef29f 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.cpp
@@ -653,10 +653,24 @@ void PPCConfig::loadCEMConfig(boost::property_tree::ptree const& _pt)
 
 void PPCConfig::loadMPCConfig(boost::property_tree::ptree const& _pt)
 {
+    // the agencyID
+    m_agencyID = _pt.get("agency.id", "");
+    if (m_agencyID.empty())
+    {
+        BOOST_THROW_EXCEPTION(InvalidConfig() << errinfo_comment("Must set agency.id"));
+    }
+
+    PPCConfig_LOG(INFO) << LOG_DESC("load agency id") << LOG_KV("agencyID", m_agencyID);
+
     m_mpcConfig.jobPath = _pt.get("mpc.job_path", "./");
     m_mpcConfig.mpcRootPath = _pt.get("mpc.mpc_root_path", "./");
     m_mpcConfig.mpcRootPathNoGateway = _pt.get("mpc.mpc_root_path_no_gateway", "./");
     m_mpcConfig.readPerBatchLines = _pt.get("mpc.read_per_batch_lines", 100000);
+    m_mpcConfig.spdzConnectedPort = _pt.get("spdz.connected_port", 5894);
+    m_mpcConfig.spdzConnectedIP = _pt.get("transport.host_ip", "");
+    
+    PPCConfig_LOG(INFO) << LOG_DESC("spdzConnectedPort") << LOG_KV("spdzConnectedIP", m_mpcConfig.spdzConnectedIP) << LOG_KV("spdzConnectedPort", m_mpcConfig.spdzConnectedPort);
+
     m_mpcConfig.threadPoolSize = _pt.get("mpc.async_thread_pool_size", -1);
     if (m_mpcConfig.threadPoolSize < 0)
     {
diff --git a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
index aa620731..fe863f4e 100644
--- a/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
+++ b/cpp/wedpr-helper/ppc-tools/src/config/PPCConfig.h
@@ -249,6 +249,11 @@ class PPCConfig
         return m_frontConfig->selfEndPoint().host() + ":" + std::to_string(m_rpcConfig.listenPort);
     }
 
+    std::string spdzConnectedEndPoint() const // for mpc
+    {
+        return m_mpcConfig.spdzConnectedIP + ":" + std::to_string(m_mpcConfig.spdzConnectedPort);
+    }
+
     // load the front config
     virtual void loadFrontConfig(bool requireTransport,
         ppc::front::FrontConfigBuilder::Ptr const& frontConfigBuilder,
diff --git a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
index 36187f03..eb3deb63 100644
--- a/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
+++ b/cpp/wedpr-main/mpc-node/MPCInitializer.cpp
@@ -90,10 +90,16 @@ void MPCInitializer::initTransport(boost::property_tree::ptree const& property)
 
     // add the service meta
     ServiceConfigBuilder serviceConfigBuilder;
-    auto entryPoint = serviceConfigBuilder.buildEntryPoint(
-        getServiceName(m_config->agencyID(), MPC_SERVICE_TYPE), m_config->accessEntrypoint());
     auto serviceConfig = serviceConfigBuilder.buildServiceConfig();
-    serviceConfig.addEntryPoint(entryPoint);
+    
+    auto mpcEntryPoint =
+    serviceConfigBuilder.buildEntryPoint(getServiceName(m_config->agencyID(), MPC_SERVICE_TYPE), m_config->accessEntrypoint());
+    serviceConfig.addEntryPoint(mpcEntryPoint);
+
+    auto spdzEntryPoint =
+    serviceConfigBuilder.buildEntryPoint(getServiceName(m_config->agencyID(), SPDZ_SERVICE_TYPE), m_config->spdzConnectedEndPoint());
+    serviceConfig.addEntryPoint(spdzEntryPoint);
+
     auto serviceMeta = serviceConfig.encode();
     m_config->frontConfig()->setMeta(serviceMeta);
     INIT_LOG(INFO) << LOG_DESC("initTransport: register serviceMeta")

From af06ae8e8fd346d9a93b3c5bc7d9ca49d97db438 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Fri, 6 Dec 2024 16:32:26 +0800
Subject: [PATCH 114/120] add Docker file for model service (#113)

---
 docker-files/cpp/Dockerfile                 | 48 ++++++++++++++---
 docker-files/model/Dockerfile               | 33 ++++++++++++
 docker-files/model/depends/requirements.txt | 59 +++++++++++++++++++++
 3 files changed, 132 insertions(+), 8 deletions(-)
 create mode 100644 docker-files/model/Dockerfile
 create mode 100644 docker-files/model/depends/requirements.txt

diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
index 6be13fc1..d14397eb 100644
--- a/docker-files/cpp/Dockerfile
+++ b/docker-files/cpp/Dockerfile
@@ -29,16 +29,48 @@ RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursiv
     && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log
 
 
-FROM ubuntu:18.04
+FROM ubuntu:18.04 as wedpr-gateway-service
 LABEL maintainer service@webank.com
     
-RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev \
+RUN apt-get -q update && apt-get install -qy --no-install-recommends vim \
         && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
         && apt-get install -qy --no-install-recommends tzdata \
         && dpkg-reconfigure --frontend noninteractive tzdata \
-        && rm -rf /var/lib/apt/lists/*
-    
-COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /usr/local/bin/ppc-gateway-service
-COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /usr/local/bin/ppc-pro-node
-COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-air-node /usr/local/bin/ppc-air-node
-COPY --from=builder /WeDPR-Component/cpp/build/bin/wedpr-mpc /usr/local/bin/wedpr-mpc
\ No newline at end of file
+        && rm -rf /var/lib/apt/lists/* \
+
+RUN mkdir -p /data/home/wedpr/wedpr-gateway-service
+
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /data/home/wedpr/wedpr-gateway-service/ppc-gateway-service
+
+ENTRYPOINT ["/data/home/wedpr/wedpr-gateway-service/ppc-gateway-service", "-c", "config.ini"]
+
+
+FROM ubuntu:18.04 as wedpr-pro-node-service
+LABEL maintainer service@webank.com
+
+RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \
+        && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+        && apt-get install -qy --no-install-recommends tzdata \
+        && dpkg-reconfigure --frontend noninteractive tzdata \
+        && rm -rf /var/lib/apt/lists/* \
+
+RUN mkdir -p /data/home/wedpr/wedpr-pro-node-service
+
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /data/home/wedpr/wedpr-pro-node-service/ppc-pro-node
+
+ENTRYPOINT ["/data/home/wedpr/wedpr-pro-node-service/ppc-pro-node", "-c", "config.ini"]
+
+FROM ubuntu:18.04 as wedpr-mpc-service
+LABEL maintainer service@webank.com
+
+RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \
+        && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+        && apt-get install -qy --no-install-recommends tzdata \
+        && dpkg-reconfigure --frontend noninteractive tzdata \
+        && rm -rf /var/lib/apt/lists/* \
+
+RUN mkdir -p /data/home/wedpr/wedpr-mpc-service
+
+COPY --from=builder /WeDPR-Component/cpp/build/bin/wedpr-mpc /data/home/wedpr/wedpr-mpc-service/wedpr-mpc
+
+ENTRYPOINT ["/data/home/wedpr/wedpr-mpc-service/wedpr-mpc", "-c", "config.ini"]
diff --git a/docker-files/model/Dockerfile b/docker-files/model/Dockerfile
new file mode 100644
index 00000000..7c57b93b
--- /dev/null
+++ b/docker-files/model/Dockerfile
@@ -0,0 +1,33 @@
+FROM wedpr-image:latest as wedpr-model-service
+
+LABEL maintainer service@webank.com
+
+ENV LANG zh_CN.UTF-8
+ENV LANGUAGE zh_CN.UTF-8
+ENV LC_ALL zh_CN.UTF-8
+
+WORKDIR /data/home/wedpr
+ARG SOURCE_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive \
+    SOURCE=${SOURCE_BRANCH:-master}
+
+RUN mkdir -p /data/home/wedpr
+# copy requirements
+COPY depends/requirements.txt /data/home/wedpr/requirements.txt
+
+# install the requirements
+RUN pip install --no-cache-dir -i https://pypi.mirrors.ustc.edu.cn/simple/ -r /data/home/wedpr/requirements.txt
+
+# obtain the source
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE}
+# move the files to the /data/app path
+RUN mkdir -p /data/home/wedpr/wedpr-model/ \
+    && mv /data/home/wedpr/WeDPR-Component/python/ppc_common /data/home/wedpr/wedpr-model/ppc-common \
+    && mv /data/home/wedpr/WeDPR-Component/python/ppc_model /data/home/wedpr/wedpr-model/ppc-model \
+    && mv /data/home/wedpr/WeDPR-Component/python/aes_key.bin /data/home/wedpr/wedpr-model/ppc-model \
+    && cp /data/home/wedpr/model/ppc-model/tools/*.sh /data/home/wedpr/wedpr-model/ppc-model
+
+# clear the WeDPR-Component
+RUN rm -rf /data/home/wedpr/WeDPR-Component
+
+ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-model/start.sh", "true"]
diff --git a/docker-files/model/depends/requirements.txt b/docker-files/model/depends/requirements.txt
new file mode 100644
index 00000000..7b753c1b
--- /dev/null
+++ b/docker-files/model/depends/requirements.txt
@@ -0,0 +1,59 @@
+cx-Oracle==8.3.0
+click>=8.0
+pytest
+cheroot==8.5.2
+flask_restx==1.3.0
+configobj~=5.0.6
+Flask_SQLAlchemy==3.1.0
+cryptography~=41.0.5
+pandas
+hypothesis~=5.48.0
+parsimonious~=0.8.1
+SQLAlchemy==2.0.16
+argcomplete~=1.12.2
+cytoolz~=0.10.1
+six~=1.15.0
+attrdict~=2.0.1
+Flask~=2.2.5
+pymitter~=0.3.0
+requests~=2.31.0
+requests_toolbelt==0.9.1
+lru_dict==1.1.6
+promise~=2.3
+#protobuf==3.19.0
+# protobuf>=4.21.6,<5.0dev
+protobuf>=5.27.1
+pycryptodome==3.9.9
+pyjwt
+pyyaml
+mysqlclient==2.1.0
+waitress==3.0.0
+sqlparse~=0.4.1
+toolz~=0.11.1
+tenacity==7.0.0
+coincurve~=13.0.0
+google~=3.0.0
+paste~=3.5.0
+func_timeout==4.3.0
+cheroot==8.5.2
+gmssl~=3.2.1
+readerwriterlock~=1.0.4
+jsoncomment~=0.2.3
+matplotlib~=3.2.2
+seaborn~=0.10.1
+sqlvalidator==0.0.17
+requests-toolbelt==0.9.1
+hdfs
+scikit-learn~=0.24.2
+gmpy2
+networkx
+pydot
+snowland-smx
+numpy==1.23.1
+graphviz
+xlrd~=1.0.0
+MarkupSafe>=2.1.1
+urllib3==1.26.18
+phe
+chardet
+requests_kerberos>=0.15.0

From 435d12453013275b2b282a8c756f857c26c08f09 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Mon, 9 Dec 2024 12:02:35 +0800
Subject: [PATCH 115/120] fix docker file (#114)

* fix docker file

* fix ci

* add docker ci

* update change log
---
 .github/workflows/cpp_full_node_workflow.yml  | 34 ++++++-
 .github/workflows/docker.yml                  | 98 +++++++++++++++++++
 .github/workflows/model_docker_image_base.yml | 40 ++++++++
 .github/workflows/vcpkg_docker_image.yml      | 46 +++++++++
 ChangeLog.md                                  | 44 ++++++---
 cpp/cmake/Version.cmake                       |  4 +-
 cpp/tools/install_depends.sh                  |  2 +-
 cpp/vcpkg-configuration.json                  |  2 +-
 cpp/vcpkg.json                                | 17 +++-
 docker-files/cpp/Dockerfile                   | 51 ++++------
 docker-files/cpp/build.sh                     | 16 +++
 docker-files/cpp/vcpkg/Dockerfile             | 34 +++++++
 docker-files/cpp/vcpkg/build.sh               |  4 +
 docker-files/model/Dockerfile                 | 33 -------
 docker-files/model/base/Dockerfile            | 11 +++
 docker-files/model/base/build.sh              |  4 +
 .../model/{ => base}/depends/requirements.txt |  0
 docker-files/model/model/Dockerfile           | 29 ++++++
 docker-files/model/model/build.sh             |  4 +
 19 files changed, 382 insertions(+), 91 deletions(-)
 create mode 100644 .github/workflows/docker.yml
 create mode 100644 .github/workflows/model_docker_image_base.yml
 create mode 100644 .github/workflows/vcpkg_docker_image.yml
 create mode 100644 docker-files/cpp/build.sh
 create mode 100644 docker-files/cpp/vcpkg/Dockerfile
 create mode 100644 docker-files/cpp/vcpkg/build.sh
 delete mode 100644 docker-files/model/Dockerfile
 create mode 100644 docker-files/model/base/Dockerfile
 create mode 100644 docker-files/model/base/build.sh
 rename docker-files/model/{ => base}/depends/requirements.txt (100%)
 create mode 100644 docker-files/model/model/Dockerfile
 create mode 100644 docker-files/model/model/build.sh

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index d8f263e0..4371131d 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -69,13 +69,25 @@ jobs:
           export CXX='g++-10'
           sudo bash -x cpp/tools/install_depends.sh -o ubuntu
           mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          echo "Disk usage before delete files: "`df -lh`
+          du -sch ${{ env.VCPKG_ROOT }}/buildtrees/*
+          du -sch ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src/*
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/*-dbg
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/krb5/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/gsasl/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/protobuf/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/wedprcrypto/src
+          rm -rf ${{ env.VCPKG_ROOT }}buildtrees/openssl/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/*/*-dbg
+          echo "Disk usage after delete files: "`df -lh`
           make -j3
       - name: Build for macOS
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
-          rm -rf /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl
-          mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          mkdir -p cpp/build && cd cpp/build 
+          export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Publish Error
         if: always()
@@ -108,7 +120,13 @@ jobs:
         with:
           name: build-x64-linux-dbg-err.log
           path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-linux-dbg-err.log
-        
+      - name: Publish Error macos
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-x64-osx-dbg-err.log
+          path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log
+              
 
   build_centos:
     name: build_centos full node
@@ -160,7 +178,7 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y wget java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
+          yum install -y wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
@@ -190,7 +208,7 @@ jobs:
           rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
-          cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+          cmake3  -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON ../
       - name: FreeDiskSpace
         run: |
           df -lh
@@ -223,3 +241,9 @@ jobs:
         with:
           name: ppc-gateway-service-x64
           path: ./cpp/build/bin/ppc-gateway-service
+      - name: Publish Error macos
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vcpkg-manifest-install.log
+          path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
\ No newline at end of file
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 00000000..b68cde3c
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,98 @@
+name: WeDPR-Component  docker ci
+on:
+  push:
+    paths-ignore:
+      - "docs/**"
+      - "Changelog.md"
+      - "README.md"
+    tags:
+      - 'v3.*.*'
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "python/**"
+      - "Changelog.md"
+      - "README.md"
+  release:
+    types: [prereleased]
+    
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUST_BACKTRACE: 1
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+
+jobs:
+  build-docker:
+    if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get branch name
+        uses: nelonoel/branch-name@v1.0.1
+      - name: Fetch tag
+        run: |
+          git fetch --tags --force
+      - name: Login to Docker Hub
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKER_FISCOPR_USER }}
+          password: ${{ secrets.DOCKER_FISCOPR_TOKEN }}
+      - name: Get git tag
+        uses: little-core-labs/get-git-tag@v3.0.1
+        id: tag_data
+        with:
+          tagRegex: (.*)  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+          tagRegexGroup: 1 # Optional. Default is 1.
+      # the model image
+      - name: Build and Push model images
+        run: |
+          [[ ${{github.ref}} == */tags/* ]] && MODEL_DOCKER_TAG="${GIT_TAG_NAME}" || MODEL_DOCKER_TAG="${BRANCH_NAME}"
+          MODEL_DOCKER_TAG="fiscoorg/wedpr-model-service:${MODEL_DOCKER_TAG}"
+          echo "* Begin to build model docker: ${MODEL_DOCKER_TAG}"
+          cd docker-files/model/model 
+          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${MODEL_DOCKER_TAG} .
+          echo "* Build model docker: ${MODEL_DOCKER_TAG} success"
+          docker push ${MODEL_DOCKER_TAG}
+          echo "* Push model docker: ${MODEL_DOCKER_TAG} success"
+      # the wedpr-gateway-service image
+      - name: Build and Push wedpr-gateway-service images
+        run: |
+          [[ ${{github.ref}} == */tags/* ]] && GW_DOCKER_TAG="${GIT_TAG_NAME}" || GW_DOCKER_TAG="${BRANCH_NAME}"
+          GW_DOCKER_TAG="fiscoorg/wedpr-gateway-service:${GW_DOCKER_TAG}"
+          echo "* Begin to build wedpr-gateway-service docker: ${GW_DOCKER_TAG}"
+          cd docker-files/cpp/
+          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} .
+          echo "* Build wedpr-gateway-service docker: ${GW_DOCKER_TAG} success"
+          docker push ${GW_DOCKER_TAG}
+          echo "* Push wedpr-gateway-service docker: ${GW_DOCKER_TAG} success"
+      # the wedpr-pro-node-service image
+      - name: Build and Push wedpr-pro-node-service images
+        run: |
+          [[ ${{github.ref}} == */tags/* ]] && NODE_DOCKER_TAG="${GIT_TAG_NAME}" || NODE_DOCKER_TAG="${BRANCH_NAME}"
+          NODE_DOCKER_TAG="fiscoorg/wedpr-pro-node-service:${NODE_DOCKER_TAG}"
+          echo "* Begin to build wedpr-pro-node-service docker: ${NODE_DOCKER_TAG}"
+          cd docker-files/cpp/
+          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-pro-node-service -t ${NODE_DOCKER_TAG} .
+          echo "* Build wedpr-pro-node-service docker: ${NODE_DOCKER_TAG} success"
+          docker push ${NODE_DOCKER_TAG}
+          echo "* Push wedpr-pro-node-service docker: ${NODE_DOCKER_TAG} success"
+      # the wedpr-mpc-service image
+      - name: Build and Push wedpr-mpc-service images
+        run: |
+          [[ ${{github.ref}} == */tags/* ]] && MPC_DOCKER_TAG="${GIT_TAG_NAME}" || MPC_DOCKER_TAG="${BRANCH_NAME}"
+          MPC_DOCKER_TAG="fiscoorg/wedpr-mpc-service:${MPC_DOCKER_TAG}"
+          echo "* Begin to build wedpr-mpc-service docker: ${MPC_DOCKER_TAG}"
+          cd docker-files/cpp/
+          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-mpc-service -t ${MPC_DOCKER_TAG} .
+          echo "* Build wedpr-mpc-service docker: ${MPC_DOCKER_TAG} success"
+          docker push ${MPC_DOCKER_TAG}
+          echo "* Push wedpr-mpc-service docker: ${MPC_DOCKER_TAG} success"
\ No newline at end of file
diff --git a/.github/workflows/model_docker_image_base.yml b/.github/workflows/model_docker_image_base.yml
new file mode 100644
index 00000000..0af4ab5a
--- /dev/null
+++ b/.github/workflows/model_docker_image_base.yml
@@ -0,0 +1,40 @@
+name: Manual trigger wedpr-model-service-base-image docker image build
+
+on:
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'the input tag'
+
+jobs:
+  manual-build-model-base-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get branch name
+        uses: nelonoel/branch-name@v1.0.1
+      - name: Fetch tag
+        run: |
+          git fetch --tags --force
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_FISCOPR_USER }}
+          password: ${{ secrets.DOCKER_FISCOPR_TOKEN }}
+
+      # the model image
+      - name: Build and Push wedpr-model-service-base-image images
+        run: |
+          echo "* Manual build wedpr-model-service-base-image image, docker version: ${{ github.event.inputs.tags }}"
+          DOCKER_TAG="fiscoorg/wedpr-model-service-base-image:${{ github.event.inputs.tags }}"
+          echo "* Begin to build ${DOCKER_TAG}"
+          cd docker-files/model/base
+          docker build -t ${DOCKER_TAG} .
+          echo "* Build ${DOCKER_TAG} success"
+          docker push ${DOCKER_TAG}
+          echo "* Push ${DOCKER_TAG} success"
\ No newline at end of file
diff --git a/.github/workflows/vcpkg_docker_image.yml b/.github/workflows/vcpkg_docker_image.yml
new file mode 100644
index 00000000..33c03299
--- /dev/null
+++ b/.github/workflows/vcpkg_docker_image.yml
@@ -0,0 +1,46 @@
+name: Manual trigger vcpkg-cache docker image build
+
+on:
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'the input tag'
+
+jobs:
+  manual-build-vcpkg-cache-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get branch name
+        uses: nelonoel/branch-name@v1.0.1
+      - name: Fetch tag
+        run: |
+          git fetch --tags --force
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_FISCOPR_USER }}
+          password: ${{ secrets.DOCKER_FISCOPR_TOKEN }}
+
+      # the model image
+      - name: Build and Push wedpr-component-vcpkg-cache images
+        run: |
+          echo "* Manual build wedpr-component-vcpkg-cache image, docker version: ${{ github.event.inputs.tags }}"
+          DOCKER_TAG="fiscoorg/wedpr-component-vcpkg-cache:${{ github.event.inputs.tags }}"
+          echo "* Begin to build ${DOCKER_TAG}"
+          cd docker-files/cpp/vcpkg
+          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} -t ${DOCKER_TAG} .
+          echo "* Build ${DOCKER_TAG} success"
+          docker push ${DOCKER_TAG}
+          echo "* Push ${DOCKER_TAG} success"
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: autoconf-x64-linux-err.log
+          path: /WeDPR-Component/cpp/vcpkg/buildtrees/gsasl/autoconf-x64-linux-err.log
\ No newline at end of file
diff --git a/ChangeLog.md b/ChangeLog.md
index 8b7b54d4..67c02c6f 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,21 +1,37 @@
-### 1.0.0-rc1
-(2024-08-21)
+# 3.0.0
+(2024-12-9)
 
-**新增**
+### 新增
 
-#### 隐私计算组件
+**隐私计算组件**
 
-- **PSI**: 实现多种类型隐私求交集算法，包括CM2020, RA2018, ECDH-PSI, ECDH-Multi-PSI等
-- **MPC**: 安全多方计算相关组件
-- **MPCSQL**: 基于安全多方计算协议，支持联合分析查询任务
-- **PIR**: 匿踪查询组件
+- PSI: 实现多种类型隐私求交集算法，包括CM2020, RA2018, ECDH-PSI, ECDH-Multi-PSI等
+- MPC: 安全多方计算
+- PIR: 匿踪查询核心组件
 
-#### 隐私计算互联互通
+**联合建模组件**
 
-- ECDH PSI算法与隐语互联互通
+- 2+方的多方联合XGB/LR建模组件（训练 + 离线预测）
+- 2+方多方特征工程组件（特征分箱，WOE/IV计算等）
+- 多方数据预处理组件
 
-#### 隐私建模组件
+**统一网关**
+
+- 支持基于最短路径的消息路由转发
+- 支持按节点ID、服务名、机构名进行路由寻址
+- 支持服务注册和服务发现
+
+**统一网关SDK**
+
+- 提供Java/Python网关SDK，支持接入网关与其他节点、服务或者机构进行通信
+- 可向网关注册服务
+- 可从网关拉取服务信息
+
+**专家模式SDK**
+
+- `wedpr-ml-toolkit`: 可通过python sdk向WeDPR隐私计算平台发起各类任务，并获取对应的结果，扩展了WeDPR隐私计算系统功能的可扩展性，可在不侵入式修改系统的前提下，实现数据开发人员的各类数据分析需求
+
+**密码学组件**
+
+- 实现了各种同态算法，包括`Paillier`, `IHC`等
 
-- 2+方的多方联合XGB组件（训练 + 离线预测）
-- 2+方多方特征工程组件（特征分箱，WOE/IV计算等）
-- 预处理组件
\ No newline at end of file
diff --git a/cpp/cmake/Version.cmake b/cpp/cmake/Version.cmake
index 206620b2..8259d91c 100644
--- a/cpp/cmake/Version.cmake
+++ b/cpp/cmake/Version.cmake
@@ -1,2 +1,2 @@
-set(VERSION "1.0.0")
-set(PYTHON_TOOLKIT_VERSION "1.0.0")
+set(VERSION "3.0.0")
+set(PYTHON_TOOLKIT_VERSION "3.0.0")
diff --git a/cpp/tools/install_depends.sh b/cpp/tools/install_depends.sh
index 1e2a1e6c..28f3f682 100644
--- a/cpp/tools/install_depends.sh
+++ b/cpp/tools/install_depends.sh
@@ -125,7 +125,7 @@ install_iconv_depend()
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
-    brew install autoconf nasm lcov
+    brew install autoconf nasm lcov automake
     #install_gsasl_depend "macos"
     #install_iconv_depend
     LOG_INFO "install depends for macos success ..."
diff --git a/cpp/vcpkg-configuration.json b/cpp/vcpkg-configuration.json
index 3c205926..e62c646e 100644
--- a/cpp/vcpkg-configuration.json
+++ b/cpp/vcpkg-configuration.json
@@ -3,7 +3,7 @@
     {
       "kind": "git",
       "repository": "https://github.com/FISCO-BCOS/registry",
-      "baseline": "535c4fc74badd509b94e8a736452fa8a76bff944",
+      "baseline": "d45a1d99c6b01713252bcb734c03f2c1ac0e44b8",
       "packages": [
         "openssl",
         "bcos-utilities",
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index a3a838c8..684f225f 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -51,6 +51,14 @@
     {
       "name": "jsoncpp",
       "version": "1.9.5"
+    },
+    {
+      "name":"gsasl",
+      "version":  "2.2.1-no-gssapi#0"
+    },
+    {
+      "name": "libxml2",
+      "version": "2.11.5#1"
     }
   ],
   "features": {
@@ -116,10 +124,15 @@
           "name": "bcos-boostssl",
           "version>=": "3.2.3"
         },
+        {
+          "name": "libxml2",
+          "version>=": "2.11.5#1",
+          "features": ["lzma", "zlib"]
+        },
         "tarscpp",
         "tbb",
-        "libxml2",
-        "libiconv"
+        "libiconv",
+        "gsasl"
       ]
     },
     "sdk": {
diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
index d14397eb..88b161d2 100644
--- a/docker-files/cpp/Dockerfile
+++ b/docker-files/cpp/Dockerfile
@@ -1,42 +1,28 @@
-FROM ubuntu:18.04 as builder
+FROM fiscoorg/wedpr-component-vcpkg-cache:v3.0.0 as builder
 
-LABEL maintainer service@webank.com
+LABEL maintainer=service@webank.com
 
 WORKDIR /
 
 ARG SOURCE_BRANCH
 ENV DEBIAN_FRONTEND=noninteractive \
-    SOURCE=${SOURCE_BRANCH:-master}
+    SOURCE=${SOURCE_BRANCH:-main}
 
-RUN apt-get -q update && apt-get install -qy --no-install-recommends \
-    vim curl git make build-essential cmake  \
-    libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config curl zip tar ccache uuid-runtime automake autoconf \
-    m4 tcpdump net-tools libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev gcc g++ \
-    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
-    && apt-get install -qy --no-install-recommends tzdata \
-    && dpkg-reconfigure --frontend noninteractive tzdata \
-    && rm -rf /var/lib/apt/lists/*
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE}  \
+# ln the cached vcpkg and compile
+RUN cd WeDPR-Component/cpp && rm -rf vcpkg && ln -s /vcpkg  \
+    && mkdir -p build && cd build  \
+    && cmake .. -DBUILD_STATIC=ON  \
+    && make -j8
 
-# install rust
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
-
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-ENV VCPKG_FORCE_SYSTEM_BINARIES=1
-
-RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} \
-    && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \
-    && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log
-
-
-FROM ubuntu:18.04 as wedpr-gateway-service
-LABEL maintainer service@webank.com
+FROM ubuntu:22.04 as wedpr-gateway-service
+LABEL maintainer=service@webank.com
     
 RUN apt-get -q update && apt-get install -qy --no-install-recommends vim \
         && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
         && apt-get install -qy --no-install-recommends tzdata \
         && dpkg-reconfigure --frontend noninteractive tzdata \
-        && rm -rf /var/lib/apt/lists/* \
+        && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /data/home/wedpr/wedpr-gateway-service
 
@@ -44,15 +30,14 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /data/hom
 
 ENTRYPOINT ["/data/home/wedpr/wedpr-gateway-service/ppc-gateway-service", "-c", "config.ini"]
 
-
-FROM ubuntu:18.04 as wedpr-pro-node-service
-LABEL maintainer service@webank.com
+FROM ubuntu:22.04 as wedpr-pro-node-service
+LABEL maintainer=service@webank.com
 
 RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \
         && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
         && apt-get install -qy --no-install-recommends tzdata \
         && dpkg-reconfigure --frontend noninteractive tzdata \
-        && rm -rf /var/lib/apt/lists/* \
+        && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /data/home/wedpr/wedpr-pro-node-service
 
@@ -60,14 +45,14 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /data/home/wedpr
 
 ENTRYPOINT ["/data/home/wedpr/wedpr-pro-node-service/ppc-pro-node", "-c", "config.ini"]
 
-FROM ubuntu:18.04 as wedpr-mpc-service
-LABEL maintainer service@webank.com
+FROM ubuntu:22.04 as wedpr-mpc-service
+LABEL maintainer=service@webank.com
 
 RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \
         && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
         && apt-get install -qy --no-install-recommends tzdata \
         && dpkg-reconfigure --frontend noninteractive tzdata \
-        && rm -rf /var/lib/apt/lists/* \
+        && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /data/home/wedpr/wedpr-mpc-service
 
diff --git a/docker-files/cpp/build.sh b/docker-files/cpp/build.sh
new file mode 100644
index 00000000..0e13dd39
--- /dev/null
+++ b/docker-files/cpp/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+echo "========= BUILD IMAGES for WEDPR-COMPONENT, BRANCH: ${1} ========="
+
+echo "* build wedpr-gateway-service image"
+docker build --target wedpr-gateway-service --build-arg SOURCE_BRANCH=${1}  -t wedpr-gateway-service .
+echo "* build wedpr-gateway-service image success"
+
+echo "* build wedpr-pro-node-service image"
+docker build --target wedpr-pro-node-service --build-arg SOURCE_BRANCH=${1}  -t wedpr-pro-node-service .
+echo "* build wedpr-pro-node-service image success"
+
+echo "* build wedpr-mpc-service image"
+docker build --target wedpr-mpc-service --build-arg SOURCE_BRANCH=${1}  -t wedpr-mpc-service .
+echo "* build wedpr-mpc-service image success"
+
+echo "========= BUILD IMAGES for WEDPR-COMPONENT, BRANCH: ${1} ========="
\ No newline at end of file
diff --git a/docker-files/cpp/vcpkg/Dockerfile b/docker-files/cpp/vcpkg/Dockerfile
new file mode 100644
index 00000000..8d6a86dc
--- /dev/null
+++ b/docker-files/cpp/vcpkg/Dockerfile
@@ -0,0 +1,34 @@
+FROM ubuntu:22.04 as wedpr-component-vcpkg-cache
+
+LABEL maintainer=service@webank.com
+
+WORKDIR /
+
+ARG SOURCE_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive \
+    SOURCE=${SOURCE_BRANCH:-main}
+# Note: depends on python3; default jdk for jni
+RUN apt-get -q update && apt-get install -qy --no-install-recommends \
+    vim curl lcov git make nasm build-essential cmake wget libtool ca-certificates python3.11 python3-dev \
+    libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config zip tar ccache uuid-runtime automake autoconf \
+    m4 tcpdump net-tools gcc g++ default-jdk \
+    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && apt-get install -qy --no-install-recommends tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+# install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES=1
+
+# first download and compile the vcpkg dependecies
+RUN git clone https://github.com/cyjseagull/WeDPR-Component --recursive --depth=1 -b ${SOURCE} \
+    && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \
+    && cmake .. -DBUILD_STATIC=ON
+
+# move the vcpkg dependecies to /vcpkg
+RUN mv /WeDPR-Component/cpp/vcpkg /
+
diff --git a/docker-files/cpp/vcpkg/build.sh b/docker-files/cpp/vcpkg/build.sh
new file mode 100644
index 00000000..56ed1efa
--- /dev/null
+++ b/docker-files/cpp/vcpkg/build.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "* build image: wedpr-component-vcpkg-cache, branch: ${1}"
+docker build --build-arg SOURCE_BRANCH=${1} -t wedpr-component-vcpkg-cache .
+echo "* build image: wedpr-component-vcpkg-cache success, branch: ${1}"
diff --git a/docker-files/model/Dockerfile b/docker-files/model/Dockerfile
deleted file mode 100644
index 7c57b93b..00000000
--- a/docker-files/model/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-FROM wedpr-image:latest as wedpr-model-service
-
-LABEL maintainer service@webank.com
-
-ENV LANG zh_CN.UTF-8
-ENV LANGUAGE zh_CN.UTF-8
-ENV LC_ALL zh_CN.UTF-8
-
-WORKDIR /data/home/wedpr
-ARG SOURCE_BRANCH
-ENV DEBIAN_FRONTEND=noninteractive \
-    SOURCE=${SOURCE_BRANCH:-master}
-
-RUN mkdir -p /data/home/wedpr
-# copy requirements
-COPY depends/requirements.txt /data/home/wedpr/requirements.txt
-
-# install the requirements
-RUN pip install --no-cache-dir -i https://pypi.mirrors.ustc.edu.cn/simple/ -r /data/home/wedpr/requirements.txt
-
-# obtain the source
-RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE}
-# move the files to the /data/app path
-RUN mkdir -p /data/home/wedpr/wedpr-model/ \
-    && mv /data/home/wedpr/WeDPR-Component/python/ppc_common /data/home/wedpr/wedpr-model/ppc-common \
-    && mv /data/home/wedpr/WeDPR-Component/python/ppc_model /data/home/wedpr/wedpr-model/ppc-model \
-    && mv /data/home/wedpr/WeDPR-Component/python/aes_key.bin /data/home/wedpr/wedpr-model/ppc-model \
-    && cp /data/home/wedpr/model/ppc-model/tools/*.sh /data/home/wedpr/wedpr-model/ppc-model
-
-# clear the WeDPR-Component
-RUN rm -rf /data/home/wedpr/WeDPR-Component
-
-ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-model/start.sh", "true"]
diff --git a/docker-files/model/base/Dockerfile b/docker-files/model/base/Dockerfile
new file mode 100644
index 00000000..05c5a433
--- /dev/null
+++ b/docker-files/model/base/Dockerfile
@@ -0,0 +1,11 @@
+FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-model-service-base-image
+
+LABEL maintainer=service@webank.com
+
+RUN apt-get install -qy --no-install-recommends pkg-config default-libmysqlclient-dev
+RUN mkdir -p /data/home/wedpr
+# copy requirements
+COPY depends/requirements.txt /data/home/wedpr/requirements.txt
+
+# install the requirements
+RUN pip install -i https://pypi.mirrors.ustc.edu.cn/simple/ -r /data/home/wedpr/requirements.txt
diff --git a/docker-files/model/base/build.sh b/docker-files/model/base/build.sh
new file mode 100644
index 00000000..55788924
--- /dev/null
+++ b/docker-files/model/base/build.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "* build image: wedpr-model-service-base-image"
+docker build -t wedpr-model-service-base-image .
+echo "* build image: wedpr-model-service-base-image success"
\ No newline at end of file
diff --git a/docker-files/model/depends/requirements.txt b/docker-files/model/base/depends/requirements.txt
similarity index 100%
rename from docker-files/model/depends/requirements.txt
rename to docker-files/model/base/depends/requirements.txt
diff --git a/docker-files/model/model/Dockerfile b/docker-files/model/model/Dockerfile
new file mode 100644
index 00000000..ea69413a
--- /dev/null
+++ b/docker-files/model/model/Dockerfile
@@ -0,0 +1,29 @@
+FROM wedpr-model-service-base-image:v3.0.0 as wedpr-model-service
+LABEL maintainer=service@webank.com
+
+ENV LANG zh_CN.UTF-8
+ENV LANGUAGE zh_CN.UTF-8
+ENV LC_ALL zh_CN.UTF-8
+
+WORKDIR /data/home/wedpr
+ARG SOURCE_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive \
+    SOURCE=${SOURCE_BRANCH:-main}
+
+# install the gateway sdk
+# TODO: replace with pypip
+RUN pip install -i https://test.pypi.org/simple/ wedpr-python-gateway-sdk
+
+# obtain the source, no need to fetch the submodule(vcpkg) here
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git -b ${SOURCE}
+# move the files to the /data/app path
+RUN mkdir -p /data/home/wedpr/wedpr-model/ \
+    && mv /data/home/wedpr/WeDPR-Component/python/ppc_common /data/home/wedpr/wedpr-model/ppc_common \
+    && mv /data/home/wedpr/WeDPR-Component/python/ppc_model /data/home/wedpr/wedpr-model/ppc_model \
+    && mv /data/home/wedpr/WeDPR-Component/python/aes_key.bin /data/home/wedpr/wedpr-model/ppc_model \
+    && cp /data/home/wedpr/wedpr-model/ppc_model/tools/*.sh /data/home/wedpr/wedpr-model/ppc_model
+
+# clear the WeDPR-Component
+RUN rm -rf /data/home/wedpr/WeDPR-Component
+
+ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-model/start.sh", "true"]
diff --git a/docker-files/model/model/build.sh b/docker-files/model/model/build.sh
new file mode 100644
index 00000000..51ffda7a
--- /dev/null
+++ b/docker-files/model/model/build.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+echo "* build image: wedpr-model-service, branch: ${1}"
+docker build --build-arg SOURCE_BRANCH=${1} -t wedpr-model-service .
+echo "* build image: wedpr-model-service success, branch: ${1}"
\ No newline at end of file

From b1d307f8582192e90fc4c531c4aa990ab26e127d Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 10 Dec 2024 18:02:10 +0800
Subject: [PATCH 116/120] fix ecdh psi coredump (#115)

* test docker.yml

* update vcpkg docker

* disable default features

* update vcpkg-cache dockerfile

* fix ecdh psi coredump
---
 .github/workflows/cpp_full_node_workflow.yml  | 47 ++++++++++++++-----
 .github/workflows/docker.yml                  |  9 ++--
 cpp/vcpkg.json                                |  1 -
 .../src/ecdh-multi-psi/EcdhMultiCache.cpp     | 16 +++++--
 docker-files/cpp/Dockerfile                   | 28 +++++++++--
 docker-files/cpp/vcpkg/Dockerfile             |  3 ++
 docker-files/model/model/Dockerfile           |  2 +-
 7 files changed, 78 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 4371131d..9800dd95 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -86,8 +86,13 @@ jobs:
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
+          brew install libiconv m4 automake autoconf libtool
+          echo "## m4:"`which m4`
+          export DYLD_LIBRARY_PATH=/usr/local/opt/libiconv/lib:${DYLD_LIBRARY_PATH}
+          export LDFLAGS="-L/usr/local/opt/libiconv/lib"
+          export CPPFLAGS="-I/usr/local/opt/libiconv/include"
           mkdir -p cpp/build && cd cpp/build 
-          export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
+          cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Publish Error
         if: always()
@@ -125,9 +130,14 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: build-x64-osx-dbg-err.log
-          path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log
+          path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-osx-dbg-err.log     
+      - name: Publish Error macos
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: autoconf-x64-osx-err.log
+          path: /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/autoconf-x64-osx-err.log
               
-
   build_centos:
     name: build_centos full node
     runs-on: ubuntu-latest
@@ -178,7 +188,7 @@ jobs:
           yum update -y
           yum install -y epel-release centos-release-scl centos-release-scl-rh
           yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-          yum install -y wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
+          yum install -y libtool wget java-11-openjdk-devel git autoconf automake make gcc gcc-c++ glibc-static glibc-devel cmake3 ccache devtoolset-11 libzstd-devel zlib-devel flex bison python-devel python3-devel
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly-2022-07-28
@@ -200,6 +210,13 @@ jobs:
           df /tmp
           du /tmp | sort -n
           ls -lh /tmp
+      - name: upgradeDepends
+        run: |  
+          wget http://ftp.gnu.org/gnu/automake/automake-1.14.1.tar.gz
+          tar -xf automake-1.14.1.tar.gz
+          cd automake-1.14.1
+          ./configure && make -j4 && make install
+          cd ..
       - name: DependenciesBuild
         run: |
           bash -x cpp/tools/install_depends.sh -o centos
@@ -212,13 +229,13 @@ jobs:
       - name: FreeDiskSpace
         run: |
           df -lh
-          du -sch /usr/local/share/vcpkg/buildtrees/*
-          du -sch /usr/local/share/vcpkg/buildtrees/grpc/src/*
-          rm -rf /usr/local/share/vcpkg/buildtrees/grpc/src
-          rm -rf /usr/local/share/vcpkg/buildtrees/grpc/*-dbg
-          rm -rf /usr/local/share/vcpkg/buildtrees/openssl/src
-          rm -rf /usr/local/share/vcpkg/buildtrees/openssl/*-dbg
-          rm -rf /usr/local/share/vcpkg/buildtrees/*/*-dbg
+          du -sch /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/*
+          du -sch /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/src/*
+          rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/src
+          rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/grpc/*-dbg
+          rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/openssl/src
+          rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/openssl/*-dbg
+          rm -rf /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/*/*-dbg
           df -lh
       - name: Compile
         run: |   
@@ -246,4 +263,10 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: vcpkg-manifest-install.log
-          path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
\ No newline at end of file
+          path: /__w/WeDPR-Component/WeDPR-Component/cpp/build/vcpkg-manifest-install.log
+      - name: Publish Error macos
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: autoconf-x64-linux-err.log
+          path: /__w/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/autoconf-x64-linux-err.log
\ No newline at end of file
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b68cde3c..b15a5aa3 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -7,6 +7,8 @@ on:
       - "README.md"
     tags:
       - 'v3.*.*'
+    branches:
+      - feature-milestone2-gateway
   pull_request:
     paths-ignore:
       - "docs/**"
@@ -25,12 +27,10 @@ env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
   ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
   ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
 
 jobs:
   build-docker:
-    if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true
+    if: github.event.pull_request.merged == true || github.event_name == 'push'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -43,7 +43,6 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
         with:
-          registry: ${{ env.REGISTRY }}
           username: ${{ secrets.DOCKER_FISCOPR_USER }}
           password: ${{ secrets.DOCKER_FISCOPR_TOKEN }}
       - name: Get git tag
@@ -70,7 +69,7 @@ jobs:
           GW_DOCKER_TAG="fiscoorg/wedpr-gateway-service:${GW_DOCKER_TAG}"
           echo "* Begin to build wedpr-gateway-service docker: ${GW_DOCKER_TAG}"
           cd docker-files/cpp/
-          docker build --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} .
+          docker build --no-cache --build-arg SOURCE_BRANCH=${BRANCH_NAME} --target wedpr-gateway-service -t ${GW_DOCKER_TAG} .
           echo "* Build wedpr-gateway-service docker: ${GW_DOCKER_TAG} success"
           docker push ${GW_DOCKER_TAG}
           echo "* Push wedpr-gateway-service docker: ${GW_DOCKER_TAG} success"
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index 684f225f..a1f68255 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -131,7 +131,6 @@
         },
         "tarscpp",
         "tbb",
-        "libiconv",
         "gsasl"
       ]
     },
diff --git a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
index 5e7a7168..52c4e083 100644
--- a/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
+++ b/cpp/wedpr-computing/ppc-psi/src/ecdh-multi-psi/EcdhMultiCache.cpp
@@ -328,15 +328,23 @@ bool CalculatorCache::tryToFinalize()
         {
             continue;
         }
+        auto data = getPlainDataByIndex(it.second.plainDataIndex);
         if (it.second.plainDataIndex > 0)
         {
-            m_intersectionResult.emplace_back(getPlainDataByIndex(it.second.plainDataIndex));
+            m_intersectionResult.emplace_back(data);
         }
-        // means the header field, swap with the first element
         if (it.second.plainDataIndex == 0)
         {
-            m_intersectionResult.emplace_back(m_intersectionResult[0]);
-            m_intersectionResult[0] = getPlainDataByIndex(it.second.plainDataIndex);
+            // means the header field, swap with the first element
+            if (m_intersectionResult.size() > 0)
+            {
+                m_intersectionResult.emplace_back(m_intersectionResult[0]);
+                m_intersectionResult[0] = data;
+            }
+            else
+            {
+                m_intersectionResult.emplace_back(data);
+            }
         }
     }
     m_cacheState = CacheState::Finalized;
diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
index 88b161d2..192445cc 100644
--- a/docker-files/cpp/Dockerfile
+++ b/docker-files/cpp/Dockerfile
@@ -1,4 +1,4 @@
-FROM fiscoorg/wedpr-component-vcpkg-cache:v3.0.0 as builder
+FROM ubuntu:22.04 as builder
 
 LABEL maintainer=service@webank.com
 
@@ -8,13 +8,31 @@ ARG SOURCE_BRANCH
 ENV DEBIAN_FRONTEND=noninteractive \
     SOURCE=${SOURCE_BRANCH:-main}
 
-RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE}  \
+# Note: depends on python3; default jdk for jni
+RUN apt-get -q update && apt-get install -qy --no-install-recommends \
+    vim curl lcov git make nasm build-essential cmake wget libtool ca-certificates python3.11 python3-dev \
+    libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config zip tar ccache uuid-runtime automake autoconf \
+    m4 tcpdump net-tools gcc g++ default-jdk \
+    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && apt-get install -qy --no-install-recommends tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+# install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES=1
+
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component --recursive --depth=1 -b ${SOURCE} 
+
 # ln the cached vcpkg and compile
-RUN cd WeDPR-Component/cpp && rm -rf vcpkg && ln -s /vcpkg  \
+RUN cd WeDPR-Component/cpp  \
     && mkdir -p build && cd build  \
-    && cmake .. -DBUILD_STATIC=ON  \
+    && cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC=ON  \
     && make -j8
-
+    
 FROM ubuntu:22.04 as wedpr-gateway-service
 LABEL maintainer=service@webank.com
     
diff --git a/docker-files/cpp/vcpkg/Dockerfile b/docker-files/cpp/vcpkg/Dockerfile
index 8d6a86dc..9e1d75f4 100644
--- a/docker-files/cpp/vcpkg/Dockerfile
+++ b/docker-files/cpp/vcpkg/Dockerfile
@@ -32,3 +32,6 @@ RUN git clone https://github.com/cyjseagull/WeDPR-Component --recursive --depth=
 # move the vcpkg dependecies to /vcpkg
 RUN mv /WeDPR-Component/cpp/vcpkg /
 
+# remove the WeDPR-Component
+RUN rm -rf /WeDPR-Component/ && rm -rf /vcpkg/buildtrees/*/src && rm -rf /vcpkg/buildtrees/*/*-dbg
+
diff --git a/docker-files/model/model/Dockerfile b/docker-files/model/model/Dockerfile
index ea69413a..9d6a044b 100644
--- a/docker-files/model/model/Dockerfile
+++ b/docker-files/model/model/Dockerfile
@@ -1,4 +1,4 @@
-FROM wedpr-model-service-base-image:v3.0.0 as wedpr-model-service
+FROM fiscoorg/wedpr-model-service-base-image:v3.0.0 as wedpr-model-service
 LABEL maintainer=service@webank.com
 
 ENV LANG zh_CN.UTF-8

From 9d4a51e9fb350cc085a359f75b636bd81bcaa863 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 10 Dec 2024 18:15:06 +0800
Subject: [PATCH 117/120] update readme (#116)

---
 README.md                    |  30 ++++++++++++++++++++++++------
 static/images/wedpr_logo.png | Bin 0 -> 24151 bytes
 2 files changed, 24 insertions(+), 6 deletions(-)
 create mode 100644 static/images/wedpr_logo.png

diff --git a/README.md b/README.md
index 8bdd4c71..fd18f059 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-# WeDPR
+# WeDPR-Component
 
-![](https://wedpr-lab.readthedocs.io/zh_CN/latest/_static/images/wedpr_logo.png)
+![](./static/images/wedpr_logo.png)
 
 
 [![CodeFactor](https://www.codefactor.io/repository/github/webankblockchain/wedpr-component/badge?s=a4c3fb6ffd39e7618378fe13b6bd06c5846cc103)](https://www.codefactor.io/repository/github/webankblockchain/wedpr-component)
@@ -8,15 +8,33 @@
 [![GitHub activity](https://img.shields.io/github/commit-activity/m/WeBankBlockchain/WeDPR-Component)](https://github.com/WeBankBlockchain/WeDPR-Component/pulse)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
 
-微众银行多方大数据隐私计算平台[WeDPR](https://github.com/WeBankBlockchain/WeDPR)核心组件库，包括：
+微众银行多方大数据隐私计算平台[WeDPR-Component](https://github.com/WeBankBlockchain/WeDPR-Component)核心组件库，包括：
 
-- 丰富的隐私计算算法组件，支持隐私求交集、匿踪查询、多方联合分析(隐私SQL)、数据预处理、隐私建模和预测等能力，以满足多样化的业务场景
-- 统一网关：稳定的跨机构网络通信组件，支持基于Rip协议的路由转发协议
+
+**丰富的隐私计算任务支持**
+
+- **隐私求交集**: 包括两方隐私求交集任务和多方隐私求交集任务，并从性能、网络带宽、使用场景等多方面考虑，实现了多种隐私求交集算法，包括CM2020(性能高), RA2018(非平衡PSI算法，适用于CS模式), ECDH-PSI;
+- **匿踪查询**: 基于OT算法构建匿踪查询，可将数据集发布为匿踪查询服务开放给相关用户使用;
+- **联合建模**: 基于SecureLGB和SecureLR算法支持多方数据联合建模，并可将建模结果发布为模型用于预测，满足了80%多方数据联合建模需求;
+- **联合分析**: 基于安全多方计算算法，提供了类SQL/Python的隐私数据联合分析语法，可在不引入额外学习成本的前提下，满足数据开发人员基于多方数据进行联合分析的需求;
+
+
+**统一网关**
+
+- 支持基于最短路径的消息路由转发
+- 支持按节点ID、服务名、机构名进行路由寻址
+- 支持服务注册和服务发现
+
+**统一网关SDK**
+
+- 提供Java/Python网关SDK，支持接入网关与其他节点、服务或者机构进行通信
+- 可向网关注册服务
+- 可从网关拉取服务信息
 
 
 ## 技术文档
 
-- [文档](https://wedpr-lab.readthedocs.io/zh-cn/latest/)
+- [文档](https://wedpr-document.readthedocs.io/zh-cn/latest/)
 - [代码](https://github.com/WeBankBlockchain/WeDPR-Component)
 
 
diff --git a/static/images/wedpr_logo.png b/static/images/wedpr_logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..8f10965f45e3a9205214b29f73c3248b60194a1c
GIT binary patch
literal 24151
zcmcdTg;!KhxJyV$2_i@>NFy!Xf{1i?cX!9qDH77%-QC^Yv4Diq-5~ug{@(i+9_Mhl
zdw1rWnXhXIm6aAnL%~M@fk0?t-@eF$K=8dF5KIIT0`SIlWK9M5gKYav%>e{rAba_R
zam^EO27yRGVqZQhx~3kj2-q(VzB{oSEfZ5afBpV5Hcjj*%BY$zm(n1fY7OknN2WHv
zEEQc^!T#c(%1qNJp}m@?QN_8VM#A{{!~fwN5>L@+wxm*jfGg84JGIn-g-E
zaI)560<3*X#W-9;H?leicBK;t{u^(!SoYwuR(!h~ANd9Z(iMPZ1_p(lXS04@SX7^^
zyJLQSnw{~0G`x#n8PTyRx!onmU;{?@;(5AP_SNCL9PP7lUbemE(puh?!JeWyS^`llKO2
zFQ|$HkVepv9G+*Q`j9B295;Uf8lVe63h;Y@_r+J72?wbk7=rn4U}Y}zQ_};c-=3ad
zl^q*cEC1;$u-CMI-^re5d)B=3OFQ2yawhA7TJ77wgFuskFJRz(+jT$Re~_T*$on^e
z=w*VBx;z?tJPF-S49e29X9QgiFdQE<1O^0h0+9L*>AHQZYQ;#czF53j>;7Kq`PXdK
zLusOyvCZbmn!U1>6Ga!^9Du3JPcQfq3~!~L$oo(#ZBv;Spnk*X)o!+@Mj`dQRvKso
zcWbfVHc|Kvz(U#w02d^f@@#C{B|^zc7gKbX6O4?*21N)2Hj6aX(VYkGIG(_3C&Xa$
zxBw#)c_a33wZ(@*$;S%x0Qa{3n+|f5ldhq=ul)Uu5nvkL-U(UQtg|qqpDyxOSC`nr
zr*pmP-`Jxv-rnV}h}Gfo$iad@HZKR3m8R`^TC|Jx4pO9{tT6e2BgY|s>qHNO7bj$q
z>ptG9BYb<1kJ5}smnTp6_q!-SaD*3v$Bu-ZyHD72V^b1ZV8onfYX)*}b5WTwh=GH=sVAD
zt$6R8NLy87&fmOqAvvKgP_IWmLMx?H1(OBLw%rnFtpR~{V{y0K{D(8h_C^BXlY%a*
zoBZ~;t_$GBNqN{>Ld7S6)pUt)0Nhgp0|~H^2xX5h2$b{(>rei?PtH^PUIhm%S@|6W
zID#WSxbZSv>+aOB%>0mJrc}E-Z!+QU)-DS}ia{oNNsXD{F16GC|m+cBPHX#l!dW(ffY69Gh4ukV?T
za4KWYv-e!gl456xypq`9@-0AlWm!@LkPm}`gs5;Fo-<1d0IF{8X}%E{rh$EzpRm!~(rTY@M)(XMTY
z3~`|Yf|6Cl4+0VW9H#&D(|hMPl4RaQZgOP#ZR=F*;l68G>+dH32Vd#{Y8L*Jz=_)VPOJxaAAJ=6Y8-luq
zp(h@BqFP33&i!t{8LvqJ0qXRB5jNUUrHV>nI-{ZOSP+cV;O&FbUWzjXj}gG`mLe~h
z5C$+o?l|2%N*7a{jQp4|84#eakO%1lu=^(b8V1y<{35xWBvON80S{J!Cl$#3<7Taq
z&9NNI&mEW80LGVJ)Pe5LLqaaOUj{|!&beQ#z6)3Jm{m{M>jHXw*xOYCU=T7!aqNIc
z##1I>oqBJ{3rFO?K4rNm79@OotXy%n_#n_a&I^4j1#ibatmyKFgYrzEVs?4Z6Ga|a
zkWWG(aHLEMGX`P*)5`pLwak}^wMrF*jES&?G=E@uzNvdt>T#InxP4wR2&m~BBlX})U>bv;LV2S
z)OgIvMK&QdC%`t7Svu|B&*S$PARh{rsdPiNYxeb?t(DNn<8dHRTbi^V=3nR
z9{`3Y(ff}+G2YlewqV*i^8HNI0fq8vzz+%FPNDf5aqUHLH321H?Jy-MVL-N>iYkEa
zw2L{bxC)SakwJs%k`-02h}gC@+-lx)?XD^k7&p0e
zBeiQCii%5_cmz056y-l&911&Ejh8c5WpOru<;LI6dF>{be9}er%EpT#E)_U6^UL;J
zDZ8MhIsNx8SOHoWO*s<47^Z8*u3*79fcKGK^mQg#HQCmz0(Iy3Pg#-iXYaR!*h;`e
z@{oUW<_2|}Pz*!^KVjbV^RM(x1H6-t(B_=Z=WmcqlALko%vq7mT-1ijOoD+j?Ca%
z7vnFOQ6I;GPiy5PY>Jc<R@I{@iIp1_D9-0i1%`
zBuSLnIJt|}F<+yl2vu5QfqX(<7&|joF?3Fg-XQV0LhBEyFPYA
zL>?Kifz3?Mfg{uD4*^mQQjFdo2#bgeHv*L6mlqxZfs(bHJuM5|TKYZY5*A^=obLD
zARkP}HcNZV{$+R&X#by!CWzFm&3?23VP=;nNXb(G!#n@m&&%g&Q9w`l{I`WKioi~M{QvvFIp;a+6bQBRu^|Aa{7U2Pov=cN
zhMvWy?kmRw2;lX<)44x#e4j{=9Dlza*Gf$_JfWpyt^L5YZG3msw8Jk$9BYd(j-Z$q~XdM08;^}u;09?n%e8@o^b0Pq+EW_t_poI2c_Wg1tT}#KGcWhfTR$z=?)#pRq4WMB2kqFBjyAQ|
z_{Up&_-l1Ni(W#B*_vL;+G#!ZPYheFkGC;`lM;B_t9Kr)2X}5Y?-I4JV*|x#O+W{D
z5_zPI!w!ggs`GM{JA}Sahq!$IE}3LF=3-=+du8l)my$KQm^O2IQ{8HnH9KlVd|xsy
zZ||UY7M}yTOe&J1I7>CWBj)!gs8bvzZ#{YyV`XSZaxYL4EUasHfjTyo>
zK=PSt3@0%46EtubxkXut`y}y@O^9T^O-c5QhH*GAwIX4gYEw3WU-UE6!syMp0e=k-
z*RwI50%CNv;P0w|6YGV^y%EZQHwp6>e`y7Dkd%y?G@k{V8H>eL=iv48=Y1%#m`f
zi27mLIu|ov5a77Y%Bp{Rb}1`LJNCmHf(@&2r6{B&;PspN0uAijrN?9hcW4)yD!1gV
z?#28TX(Zg0bce|0qk7$rFOtQbpn6X;T^U7gy3+RLquC2Po-nqBN)+QN*t$C+T!6=$
zxZWa0@-B!Bnd&JcwZzlLR7psT@)np~0(2dH@VE%&-BCqIU-G2|Pa;+;rrs|L{oh&}{e(+XsH)VoZ-hId_Th
zf?Uq&sUfyVnwKX6=-5--v$Jd01=IF+8~RnlCS
zGbNCJ+GC0Pz5|3ToP$Jn{2$DbR|HbEskUGOM~Fzq{G&PupS-|cYbsmY9z*25c;*_H
zcd3j>@AKC6?lHwqC}FEOlRlfzE@ujtDIoW`?!pBGqE!7IO|+@q;lZ3~?IFiShOlDP
zbLv#$UC
z;G@G~
z=@Vk(jHR0qF5LcF@Cm!gNB6g`!7O3g85gfe
zbL>BO2_&Glz19&2gGlG0Nl)}As)WZs;4)x$BDT!<9uI%tU{l4tlP
z&Qpbs#0(&&$L{u^#~{6r!!~2`?G#}7HWhNg#FsB$FjxJP+cY&&ajK4KvCEXgU6
z<~L=49y4RrF7IsKdf#&CRAM|TOxKy$R%{XKY8=-#p;8oL_Ar#gsga-%b9|nfWz^W<
zR8xActV`Dm&^ww8dFK|%d!Un?~
zgA!c>Z?i30J+qdX9LY`K<4$PEan+O@si+uPDg{;9IUuQam=C}LY~W^gwc5t5lfszj
zzHlP$!dTZ*-Sj{;O-BEZq+~HxLb0?2uE?BwA4;nUvt@%D{|X3@8G=epGwJ|Hg%QtN
z$2(2lhM~vNhl*N|7(+mGTk8jFp(=2o_v;yDkEn#K3Uk|v?wn4GrWr;%9)R<|?+I~U
z9j;#R_4LY8Sqvvi3&x+QAfdcvEKgya;B#0-+oHNW$larG{^h(VtB)ia1$(g56zH&Y
z{j28eY|wS_2ho}af4q}~$OaP_gT$gu`DG)>A2`g?SWlRSxcnhdIhSxnF|7)jhg>0j
zwljh+>LS9BIs~;VnLlWpT9QBx9&D+xTzT>PKZ|~*f{DaQ(bPuu`oSzXaC4rlCEGcAK$Pn{1qNarQ!f56W1?&99HCKI
zE|r1vwkNFVsxUHSic&o)EdfbRJmYnuL(-kq~~@Wx=ueB`1voOA7M3K
zd9g^>+(s3cpJR$UQ+G&FpkjX-z?eTaJX8+5_{XE#PhW>BrPc3EZQC=^%P>H*=^gps
z#VNg>sYa?J-=IMwH}ltC#o_2{T`O7i5H_HeKiT-~6wC=q+Fj|zVsKH!5)J#qS;Z0G
zr_wGNlEd?_epKepw5SIc*AlHc%M%$FH;jq<(TzMg6QL(96jGo9*7VS|yuTkp9gK1acn2S&xv?$`loX(o!gMYGk
z-Hn`qKUw2H-6Hh@gcs|JhHlQ=UmCF
znl*xVJVZm~1@*LrirPDL?fiA4O;l}tLMVL*@cMPPf33F;brVvd;N1oYq+WQa?B;`8
zA@kLxpww9T>S9LeVj>4Wc7$b3)?H$W@Kk|xp0p_*0~UY4+&V2AZH+#+^ZIY@5wv^zSItswO*vg;
z>Y(Pd6P|HA>Qgz*KdLqmT%lT{4W
z5a=L3nF0hys#1vASUn60^pTLS(&AJ7>6F0e)s>AB&bG%zep@c=Ny&ylz8&^sf;@UJ
zHCV|Q(9p-9x^De|1b@MTOM8yf(p&YT3=6VM*f_cVU}2-PKd52aWcAL?_K_2~Ew{=qL}HFp*<+_x1Nt
zM+>d7B;T|uD&y*yMvdkU0u{v?wz)B9xNXIYZ_Nc~5%huBh%TvDdN`rGkMK+4(7NcW
zVEKigmkPFG8@;r<;Y8-T%$kNG`IHf|*8_RF|Ac^6i#yeoV{N)$$X)HnMPV45Ro%^wZb(nppZk$h6y9<5H=9
zx`Ql0>v>7`y`?Ec{P^cJld{DXD<;o(!TK7m6V3V~gSrEkidTBr;Ui=!9!5fm}se2=Q^Q|vOvPJWfst_Yu`biwN-bub-{iyXX3z|l`G0Sur5K*%>T-5FiuWvVr7a|99CcLj?54x=3Q|6~8k4}fMCtNO~_R{V}5`4jo
zzeVdoU|7X~Wi|!4VbZJ&QzE}xj`p_@)Q;Yv&QjjaU|Vs#2YQ057_ygl_bY~S1R4>J
zxDNaI)|eeE&X8v$q+x83E|9DCL>E9pcYm|wb)uKFnN)JKse;>~Xxes0FGCFWHA|NwsmE19f*I!DFK$h&{+Rj5^L4$3z?W2B_daVl
z%Qs_8&{79ad52BKKAdnzm~Rk4KNRDsh|^!s13%(=r*;i$}`
zJLs)2PLXM~8{s`{6}PwY_%<0&ll$OfEou0O;Hc4}Cc!Tx$LTx6AFJer5(jLIO=8hz
zOli1zi%#clem#pu>m-;8L%)wD@K@kvvs{tNZp?_)NfG#AcO(|$-lJ@a+z$1~^?E^g
z$xmw*tFwpfhj@@D%JbSpe{2Ly@{avXg$F{pj4d0}?_0)5@qdOx>3%!7VtpeqPU*Lx
zX<6&@MPXjSs|Xky5N7K(aO{@@cL|T1WsFdKGK=zVT79OS7eAM6D#j$k)F6Y&n&0A?
z17efxS-@+nnov|yt6NaJt`wlu8vET*jMXieD7VZ0khJb_w^Y(2DE;ZJ+
zCZJk^CcZ|)nQ9d=3YxONitV5D6H_GwSWL3q%^agLIZ`zX=5K)+D)jbmZP<`jsLm3a6CwV2KRHA=hOiPKzGZ-2s`Sr7UfqwJ~1TDj|204H%CDo
z&*a|5l)j5)tu70pmik%sLUdJm=8RHti26zJ)9eCd;aRV6cpZ(OmV>r#MO4zDjvlU`)3kMb
zg@zLW=RA9sH@@3i8LrPrigJxy
zV8+!ZMtCm_XIh$pUE*8H2SF=c0M9^z2=KR4LOa*6n$6LAw1e|NHP+eujF$gn8wreONcC@z{|Hjjv;5ic3HA!((>u*bdt=bShjDTs>X
zU-1E6XwrMpbHYYF!UCz+@$c<>Ak6?cM0=j=ylw!0VS|N|+co-|)l+&3}uvw|t4hds&l=>l+q05#yh(B)63)|QZeL8BOOWKTXqUG&bv
zmww*6a689i3(^a$Vt|$ZW#qtjeycH6M{iv_AUA2avX#1muVMDL$BqBkcV2=ELs(`a
zFNFS%C0@d$M)R~m9(sbfV&XDLKr&wP#FPPEE*s{BvN4^KTg|s|@UmnbGzs}NgwBmW
zYW_QBd@-L(MPoxuS>@-AM~*rex6WBMLEP8%49!Qci3mdtbsY(T
zI%(Oq*!L-EWLsv!`#5?ni3MgdJaXy0{H8V$<;WbrcFJ*N@H7ON$HQ3WNhn#>KdRfj
zCG@vq{eccYH%v?TQD>SchJ%HQKPEU|>&IE$m9!J{*COn{i-$zUMmRtX<#wMGH?Wnu
zqs5B|h||Y#)v4eLknV!4Sgy~KV^64eo3m}5R{uhsxe!FBZJ#HAxZMhfr+1iTHZ#H?
z=cQ~RV)VTcPBa1iOi6;0)EL~UH>V=>-e>F-BKO8Z<69zbmQ~=IvVA7j_48FArV$t@
zsP_mlBSK;TRH!Ces`2-dRCK3(no@Cl(Oi)VZ_17grqZU0X$!RbvoB9vb1^2bN04{L
zvi#>KNc(l81rK$SKYqDQb_)B_}8#f;iO{Y$Uc{?}q^%gANJ2YYwejbFYrI4Ye5q-AdDBn#sZkAu9V
zaN5`APr_T=bJfbxTXMKdKb0#B@E+7y0?8$A=31{$@w6F2~RmO3x5lv
z+Us6&lIh|7OtZ3Qg~P6(gV%yAJdzCVkMAD|O7EAPz})fNf^WioReyy4atU=J{1TX`
zU!4`SeMXI8;%p%N7RowT+p>wjY!F+v@p@!VPX$@InQFi|P*wafGoRi0)oY&`6hdU2
zI`s9@bVjV8#b&0BCW$RQPm@Mpo3{=QCbYfH1?(h|RYEMj#S_h}DiSKet%`CSO`=c$L(fnkSyfBsBCp1
z0MdR~7rl}OI^DBEfL`*ik8{V@Q2|l{wUdM)XS&7$@si!~vFf3ggcC!GQ|4Y@O`F`7y~*FSZrU;HDVz>J{nDs(C>~KpA-G
zQclT7Y&i1bICU$!D`%L9-BY=prSsaY{ixm8fPLgKYut@=eHNCZzi-2|}aG8$s6%&nLP$
zy?S;0txII(uDQS0i+J6}B>8m(@o_0<*$XmxHc(22c^L~dEOnLn!Q
z`8;$(`5A)s)~rQEFOz7?_cwa)hOD*+{{gx$iGN=Zkq(DCq}4Ca`9
z8^?h;zHu*-`73D~4`|^vbH$1O0i4okU!d~AbhPqL7^~~E`HAghryQaLRS~8?0xgwd
zk>w!?oN*aWmr{zX&>8l5uKzo(okH|5|7}_9=2vfmhNK(otx7!k8=wX(1w~%?mP#RJ
zUd>bN>#k%H6bjDdN;1lv>V9S;T{~f^A5lRqSE~+!6;8-(J9_GcOuO%-#vUcw&Zz_5
z5f@$kdgaO70)BcgYn0eP`*Cq@e*6A>NG>y&kFeup0(MS^{!xDHF>?tWLE(^~eyFLh
zD|~xMBjqB{K_s{+a+uyaO7_bJ8(UsD5v#ywYaWDY_p~s#0chHJb}DswwGMM!D6{pp
zE)GHVI>=aTJ#lMr8T>tQ{}NC}s+ucob&FJr?80JpNfrv@eu~J_-qkoEX7oiJ78hU%
z1IHtObe+}YEgv9SVn^VdVgqYm-WiPJut`gaYyhU=pkT)Q9woP05lw)(Xg>til8w>L
zT;q57Id_6N^uvw@5}$YxdP%Sa$9^-@T8ZF*%lr<64Yn)v^19Z}+sk4JQPuNgBJtBc
z)_wF4`^eh3>5r%HWd-a1n{(eN7`HEf*xHV=B>}f70R}8`d%MtzR+|-pr3n6A@;&ED
znJlN^>cS#CDy$l=Z^RY{5ei&u0a;pJtxFAB5;wKg8Q@`nY$A@r9D&6^`E_L
zK<&Azbv~*a=-NV=-D>LQS`6our@ZN6j9>LD(AIzFCG#!tEwj54#Ek5vv|u>nvq2Z-
zlOtgzj4+1wB?Y~j3Wkp8!E*HiSl({tM3{(DHOA*i^^VZ=;OLM24TrPN>yW(9KR#Nq
z`(z1?c9?6qPlylJTkP=qX;`ua;$y*
z@gz5T;#JuK(BRcQv&~cNgxg6i9M(#0GFkPW~&QEXNSkV#~z2}V$Z3N=MJfWICmuH|02#
zF<+2r{DJkS?Jy4)RZ#V%>d1JC@G>9wTNPNEc}*8V
z*P_N@E1K|Ow&A?$a`pk9r#htrPdRlWXFH%~o==0S@zcD{*U$Jqe0&bSc6Bmav)EWn
zlE9t!tW{VAyNb?C||Lv;iqJT+7QUtHwXHF0x6}Ch3-z^I
zu59(=w+M~|X}tWS+{Y?*v^+yg*-n_E&y`24hy=l2)`}r&FAbd9vBsH?GMRkomWbGg
z&Eb%7wS1d|CUm#I$7X3NoWeK;Cbi)SR%!Ra${}Z&MH&QQKbEcm&rbIjS5s7I)pknP
z7$8h{ozB^fNhgh;sdKJA$ToN?EBvjUo&RJBc93>y-E`DF^NK9$AYxc17V6_-7gs7yvmj?{
z=_Ii7{kmd{pI&mOd{w{W#opia$t-ko-p9^`a7i)b)ncaW)PYPahF`-gqEC4P5Wo$a
zFcSNvEFV|E7EoHFDX(zViCdGZ+-1i~`TKZQZoeZkE~WZ>7@4-0JuJ)nH>J7@%q)`}
zzxMsFNSt6OyClZ1BXe9>W=n^HLl`|tO&uOaLd}mtf~HeM;TXr3V_B+CyAdVKgZ&3O
zDT0y?0)^pJ&5CtS%x@|VF6s?9d}H8|@rCp3!YKoj8S$MMeyAgOcX3q4<;Cm}Ny=Ki
z8lE~*P6Scjc$u|N^ly*L<)N>=w>+OM_ifj^9xCr*Sk9qi3kT=v-S+6kql4#}CI=(}^sJk-hh2+Gxkq;(Tz$oy^sEd?wO1Z^
z{vsdx1ud6FZ1P=)dP~~%G$eVrn%%Eg2H8~J9=FJuoB607X(Z#LxDi>oQxOgKlgk`@
zsS3OK^p^Cd6}~=+)oOPcXpNgM>t-sBB8u8)TJTjJ=38r=a2B;_A^bQ(xz+OeUNihN
zJO89vjiJ0nnFoa36FBC
zsyi8H$txZ-jr!Q|jKRhG7t`0Cq$2M~GLj_Rz{4_^@?N2mp*ol!l5+ml(T*kR7#;T<
z4vWrvc)sg^@y@Uccd}}G`V_`DbZRrWOKY<9w+XCBTEBb6hS8e=kuwa9-7ff`aUW!r
zRVTD-Ix&q_W|Zf|P^u0)NL}1B?|9}8#NuR_%jHMcOAO3IUld%3i^cR;13*_Ra__g}
zEoXDgc0|97FWf9n`SWNPXlZl=qcGoVZc8eE?DaFHrEkkH{yxHXVt$F{vAucr+kWam
zMI5nYLrO_9Oj`g-xmn$ki(uU3cY7E|C|!xviBN&@o)gGQ;Y_Nr#?%=$(_zpo7Hw4ftr5%$+e};XUfegc*evukcQxWkm*|T@&
z<(}%4X`-w)wcY(qg?09Rf|j4=V(!iPvuBB&Zq3c5N#fZZ(!)z`sq2d!oeT>^5_lwF
z8R_A?MJ*$o>fR-iOO%I}D;rxK)p?pKK+0tDPn5mM;`ownSa#Twn{0BL5@z!iRU9aU
zseSPdO`SFrR(WH9CC1i%v%@KrOPQT5&ZY{|Y2{b!zt>CTsJaRY^x{tt&;S>xuF}NW
z)qG$Xuy;FIbUy~;hZL>q$;a=SwI_KqOMo#RBlGMY16PErw;zKjqF?;h@-bAyp|b5&
z;nTfcDvU-7Fm!UbwqbV&LMq>>~?4
zqCOUusn$U|KAkr034w8rTmE+EiBVkQd>7lCo~gK1AYM#jnLZkG(zm=W7rgj4AryV7
zIvHao(n^T}EVnbQzj6O;Ih2}ujC!hUJZ}(4U1pX6eTNk{bt2ljQ&!t+Gs|Nh
z)DhP|M0MibiDsfZ%IAhfxh45ipnV4j8gD_ePIy{bF*d$mN8#8`kQC4pE|;+J-rb~sXBb{@&ePC?@H>JbB3rz8wgf3-
zQtk;YdR+~Lpe3MpZqPjXSpxtmPY`#R@%%V;HHT1Wytr5YEahfIuw-`eB?P6avhazx
zVfVF2pla$U){w)*V|Th-dkjNzQ#4{sf}+2Zjn#jk`3J)4!R65Qh$dn(dDSdoAiKu1
zOiqLqa`v0upv(sPP9e`%l~PGR>snKP%q3U-5@Q7L=rBX9^0_#h
zOnkfkek%?eUY`+2h?P3d{ETDwr!JQ{YmC;`Q*ZA9YA_A42Lo*nSI%oQHjWZ^L1qxw
z@LE@7TSHW!Sa9&G-;H5z-dxeiS=L&e^ef5yk1&9Wl5y+{0`i*p(j-CotQpvJ!EW@!
zL3e(>%73j5g9WVMJdYvVW_&u8!IpmKxF^`Gr;VQ6SED{2Md&+D5!q%WU)Q4Vjy>7N
zdEGn{>D?h^V^?)c9Pf>~Llb7xv9+!V<>|HsfA}gl)*=%8x_8Xh67$Io*ciDR~wcW%TOZ7Cp`5Z5MXpXWJ=0shosbMKgXj)d?uUoRUN^M2?qae
zisIK>>ahSK!xnRXduL57^@oMIlLphTb0VCELr}H_3+g{LE=5-s_5Sw-;V0Ymoq@p;
zE6p+5+jT%*BiwGJekj_DYkZGfq^EA_h3OJp
z`*!saRq8J3?5vh7$L9DURXoF9xW>piCjzsq#{d&@D6-GlH|#w3iqL6W;j38iG46t|
zldrd;t>Q=CKo6X$;&>Cj<8N_f*`@A~U1of%cylzzn*m(GF-&UJ#tPE%xMP5z+KI|x
z==I86u7m|0&0)U1?Hx*`^3UI>SbAxJq9TV@$hoD%DZo~#<9*Bj*uR3um7!BI_oYVI
z3SswGm{ZZ_MJswK)*((`i$0gP%-9Yk4~+S*%-uX1K8VvfVUQNkl+W6GGh}e8>2~LE
zntW9C{iv!uZe#gPBUau{^qW8
zvMzJH-}>;b0-Sg!gTBc0
zBLXWR5(PQ=Sa`Ttsw}DB)8!@4^qUuJtqBurH`naXnIB~V7a^7v*xp2K-{r&pPV?)L
z8)_9T>Od@#m+v#f8H;(`^`#-u+{`lRpK%}OBP;&B%
z7IhnmLklw9Tp;3BQ{F4>EIx(~FGr;a13WuHehbiqQCW@%HCh?`fb(GZ^Frflo|93LN
zZsSI?lIPX=6QBwHQGaj32l2V+LX^%O!l0dEwnuP13i$qPJSCQWz4Tg#Oz)W&S0wJw
zvy|ub~{;2;fWT-DHhXXGp805G2e8!z#Xdu)q@xbzk-fn$O>%Tm0_dzk<`!Y7#t+
zdjhA|1O^f?Td9|IF?K)n5-0BQ49Hsz51QL^pG|r95%hnVW|Y7}k5wI17bvD59Q!z#
z{&+qmT+@aQcjpELD}0kO!A8Gr_d?!$G#=fFQVdK&9P{?@Jk@u9n<1k5zCV@4i$Q2&
zehscjvqoug$iFjW_K38<@1QBGyIpT0aRcc7z$KPw7Jk#+sV^3ofo(;JivN*{B)7`$
zaP0C^nJ=DHTTg*2)k=9rXYjP{^Xw&G$u?O(y7egU%{E(#LD~#KIU6y)bC0{V?p}-T
z#ob^d)dzU}F$k(@K{Jgn^i$k5i9(|94QrHvNJAs(Go}y*5djXNVhAZ)8iKwr8#Dtj
z3PUW!RG#Dh{3;i6qmbXN^hxBhz27bU^c;$6!h=&~^-rRK>e+KpnXoqtq{q+gj}9X?9jlYAcM%4Z7!B+nFiT`0fTS`Qv3!lw
zd88V{=ikXH8aG_bMNtf)!TIK-YFuS5ykS8R8}8qKpb6yF(-V<_j?etAC&X95H4{M2
z!k`<6XVhcwbRng#Eib*8xT`x~dvW%_)hgY5yVpK)01cV%mvOnjqcPXIEKBPkkcL
zoF3?3#{f(Dpm}+BjA7b=?3;+f`*zX{>)TdI*-7QR2QyGznQV+HFBYV$M7Rg%cJZ&t
zK-iT6+w?dr4kEBTW$JjToz%w|Fup7dY{T8|_y*hk&X>#_bIWhWO;Qw9p%+jf5FrNA
z4`dhXddv1xZBkd*9fHA29EV9jI`4kAL=4?@uu!k)Yy0k%zrgc{^@~tr>>+8#O=Mjp
zi3Dpp>2C_&oPn2xDb(<@x&jipe+Cg#F!~rq>pXj~#MD25mfK3{Sk{KT=e_uW%UtPF8AW$NO2qo<*@YfUz?5|QP+E4FhS+*`V`
z#wY02o-9)0>qTbJ*4FAPToxeX&T`XPCpqDEt3`pQlZ(|u5FC9P8F>FAAZZUMy-}fR
z{ObISym+znBCH&YZw<8!BN8kjQEA3P(E7Hex|}f>d2d1Bu&BL;Lw&eFJJJBv9ftF8
zDf{1^!w&;JaITqptNR8@BDf2+Tk*kFWz|?BJTx{&b_;*%ACH!_g)vGnB{(r#LNkHV
z2~ENTN1K>wv!GPr{9#ZIl@_P0c5IJeytpxXF%O$IoV?C=@awV#uCX}Q7$0Jm{C1(t
zGU0YGCDJ{RI*G85rc`4=w!Aslp6qiqRd7xktP7`qY0?zYnHpP?Rbr6uKpbmZOgpV)_fyRyZstubGoz
z=g1Z}tZj>k2~78}-FlTG>it+=PYDXXov(p=bp!h;fB8D9X6#_)s=1b)ua0@gb(v+i
z#*Wz1Q|IQDJ`H-O!9L(mur74SJAPYb)q0{mi3{q#`)H%ay$Iw=-bNAZ8j-Y!8%MZ5
zbFI|v2~O4Gd`s;=NWI%HnZ!TjDdx0{OOOO^xt_iJUw}>w7kYmtegbGFDdvS8!+pDn
zm1iaihWYXji8;muO@jl{hb|nac>DvKhYMhF(o3_dy$hf*=1>WjhQEs2v9tI{^RLC!
zfr9c$!u)6R87&22j@P?hjwUTEf@fdiwVwWR_d~R(FzQRHuHsln1T5TE
zWUSg>&DIrhcJ1&Tj6~qPxAJcSWEQugN(y9hkp9b=c=?($St*3x{c)C-OmeMq*JT5^
z649ure7o6`g$oM*y=&yRHpVceQDEREH*UytB-@xsV!?+|^wI`(@iD|gQy1-e?c=H(
zE4jq&s5j6POFDL*6%zEFYX0D_ktPQd}D%Oi}8}^W*t_Qj{I8rd7+yiyW+vIP)e8aoe?#h~V+Ay{M5{Rcc
z-T0O(*FsYk?OKgnZ7u9(|7*NRJ9CVoMoZ6`TW36`GhJ=(0O4-=SJxTJU&wZO!QuDk
z^ZnOsb$kOCQr=JHf^>OLdQ;bE%OC2hZ~Z9??WIi*o4!)lcV#aK06y`y(CyG4Yzo?5
zZY*8Ac&md?dO*vBt90kCH`P;K`}EUR
zS<=6iaKG)zUhpT&%i|*iK%w&2pOyUCyC6O?B|93{1skWXpXS_UO8Os-Z^ULS1@8FY0(B-+ZHsuL
z=aK5eo_dsWpdW#N1Apwq#1Tio8gk|2qo=xUTeSJT-{I$eC8uc_Kk6|)uB!iI18|qX
zQ63SN`Ae6a0I$xCA7izKB@jAy|H^_g-v^f_X5;za=sd;&VLpfUp$1}u`Jk-{(Pu+UVaA2
z@>V5eRSpT9&h(0j!atPR{0#4NYSu{A(wNaoc(R1PA}M=$pT0t#Q3V(UX-=aSrlPCX
zu0PXf^#)7zJ9j1hcOBNZ0{yERBu#+lt0@l6sgsFDE9+>@UoxHgpI*-Uk?QXa;8&C_
zu8dGFS#h(YO$$i6lS;d|ab|HJn;?m6%G
zIp=wv^PKZO=XqYMKP6F2PE%2OZn7&}z!%y&Q(}4?cgOx%s_F*q+k-blPuiicdxjU4|B#*BYHkzt
zONlNE?EA0nx(at0Z?@rfe{ae8v`8AR-;N|2$()a&dz?`>wrYAwE=Vu9|C~wshTVjW
zltw!@xS1Rrm%P~qPJ*QUUR!rJfWWX(@0=zv#cGRXye%C?MYlxYmZP@pA#Ra-V@swZvJ>FG_8hg8QiZQ#~LBr
zf4unIw$$^YwL$jPKEeFeh+ibObLZ%ny@tF=hZk
z9rEy?aFFI;Ir&uDKo1ZXc$e7c-!BU!GR-H*#s@BnHfQ1P8qazWv(|>c!PbOj+Z`M|
zo{}-i^(yLTcE3EECql!u2m=mF2!SmoW!Yfek$o2bX+Zj
zH;^?}awp9BdQpSYZ_f?8I@VsQA?R4dI*PsZ8CF%zO^-=d3
zgDyYPOfECZ1fB$|+Me$gz@qNUBXe%0%16tyM%6%!-K}Zs
z>C}_1)#;zcjrSe)M{CWeYSkq&`n4qVL%R}Zz?qx$!DRbxVzi}O#rZ73=6~kbSSy3c#NY
z
zrY+7XtmSiBdE9^wE}_-i(R{L2=|m2H4QJ`9rJ4$oCFajgAM`@dfbD?D-YI!;+X=7%
z`f;g6P8sAtx4&Na$KTS9QnhO<=i1qjS1qcMA(=+RB0_ROvB71hskon?jfiSB^U;Yx5iB564g~W0%G;Koj!6&ge~J;D
zRd%|=Ca+nOQ5>>aV?6xFN=?1DrHoQ&;5_Txc0Y916EPp4^s}5e(Y9Wx?5F$qg&V0_
zc#Cmd#4f#VhZPyt?Sn@=Q*O}-@n^!hJ*qhL6lg1BTt9uh6(LBx8}#OHqMJXB?IwI9KhZT_9eV?-e@k
z>|9`tM9
z$@e)k%FgxQZy0H~;@lq5HTs(HR>8w=uRRWFDmLD{^8Foc2qk$(HIjZ<^39UBIVnmm
z7)8oueF`7s_`Dr?X?ez9Q^J_=u3)}ux6XK2iE(7_xOvZRayxRe+hf<}>%J&0rKewY
zY;N?fl(iGfA?1=aj^L+O@nC{N&gLXJFJjc&^}#@DQ=)yR+`NZS6gPK_UozXG=fmyH
z%Watfm&I`kFIXL(*BdvDO*G#3>aCm9=#
z9oP$UC2u!ZsypmHg}+McB~=*^fIgFYRvUys+8f_$t-x>oiU+**@UY-&F3t<$9nSon
z!|3?e?7Y)UyC7{IBnUXWZVMMt=Cxr=%MwJyqPiPd}<0-iD2#Z1Xa%C+Vix{jDie6EUpI*E)_%v1&5S`%$FMrn%}$f8^2m
zMZhkXYbtO1IWf0ZOq$+J(E8Q#E+-MUcCWC7ocbSniz8v}H8SnTMJ`(3XTKiVu(>o>
z?M`}6xW6}gCCj=Ut$G-8n+I)fy}IQf{^+B8@>)thRSRjeq5p+E+bzQHE%fZBOJ1_j
z-Hd)udTr*s_z%2klBr|-_}^bkV623e>~o}{7cPFPt#W%}H9hg8mO@0pKd~-f<~&F_
zI1SGgt85;W=*t*PUM;$!-MgOlO!T($P4FezZ;x``{ySrj1L+9lYzxbcg$voLe!bq{
zM08j29WjGhSPl8VhGgyirn7h0dA&Z9Z@DMloJ<=jDnv#4eIVTB*kQLl%k
z5enas>*Z^+>BIBVzVM#RQ~DX@uygLH*#7ic;J68*7s%mgEZF9PX@70^qzT)S?
zA|zVC?4)(HU|(h(yHTa*2I3^*1m_IjVCk}g6t(1rK(auxwq$RmoT*fMDKeMj@3I)3
zW46E4Wt8##YPJ_38-6Be_B+w-k0+Mc(vzZj(?62YYN1&x+4ep59qiwTi2ElR7bdS(
zxr!}*)Q@sSqetac_KBthz3*+Fi1~>sk4O}2PH+%JGCFv>Hl9`6l|BtJe+GY7f~gLP
zmDo-)CasQ*OXv9h_@Y)BJw@A?jf29A;kzcQC?V1S+f&X6qkVt);ZX{yv6g}2NJj``psQdq;{Ur`;u
z@Ap1r+Pi{;YsZTI>(2KgxpzGuT9F5vu@@Z_#BlPhLHi}$&fSkjyN7xEBT~7q6V14B
zwj+U$a&y7%l)=61#OvSe5yrs0ys0_lc*^9w#dMI2)&}tLyXA6+om&_^`_o&%z1`-mT^>8S#?Dp
zRnT=yb}@~jV^UP9b|YA;cdu3NT=`LH?I2BTH;(yY{)~o3L+2zXeb$a&{I+k;+wEvO
z8CsH*UkZG2#d4zKdJ{h0$a@uyKl6I5u)or`?8X5aPK}MXx~MmWMXF!_F;;M6{N&hc
zbVg>>WX#(8j+$?#WaC4Rv};sTj?*uWFh4IuNe7wZxCXCuM+@~RWWOO0q>NXul{_C`
za&R=MR(a^MaSG8~C};qm22hZ=<2*SC?HAc>t>!rLmaihvFNx-8n9f0-8ghRrTRRoD*
zMBhlc%d7!T5JJ=dkXh0hMhwu?{-e`?0N(81afwg30SFB$KEthH0je&Dr9jR-woz}}
zq(Tpnj|xCUIc#uUQs@hL2E072@(Oo2M}r!8OFuNq$?xEDN)`JQ)`&pkYobA+5ZH44
z2f?X@NM6=21wKo}yt(JMm>1Sw4sMHC1R$uH-x+@ri>oYB4Cl5FrVK
zTt2h*e(t0R0uNfyv2>yFN@DDE;^0wh2$e`S)SMfY0Ir0$aGYG=X{mN$U@xp-TFN@-
z9Ije@eqQFD&UHzF_SS2(VLbIEh+pZa4+u@$ys1EmtLa@eB?0L2PI(d
zvgNkB@LmH3L5-EHsBK~U*7woCYz-h#lzeSj0*Xhj13)DQ?>`45KTBpbFyBx{NVzC6
zuQw&MVR8)fXgmNTx)#nrWTjLHcopPmyz`g5D>K0Uk8{~Asv}gt^3AhpG;;}h(z>#E
zRlR>)mM`RiDzp2Il)Kh05Brsm#EZc5q356)7wB!ynUH2N--%+H*Y;#~FFkvABV=?W
zTJDj>4X&csE(JP4;#zisL;$;Trc?IwFa!nobtUB^wMp-jcuQjPZ!lgk*%;ns#5OlP
z0!iNgsm=saY^Z7sf5kgBkx%hV^BT{YtxIxlf0@sMIWN9m6{rt44N_rkdCoDL{HOB;@K?P4_5(%txsj;mzMnJjJ
zI~t=ViJyEB4VA&dAglgxP2VJj*d&3x>m>dPk^g;*y3(;UuA9&}{Cq1c;4s>fM!x>a
znwO-_&0xZCGIy8^3)zp&8t?cE1D9e-4T;DIt~dR$0ACknbreK~S2+ysza%j*AQ=Jt
zn+ar>iZ=j34jf=sdxYmzE~Hu=d8o#ZJzx-u>ML+10l+i}iAmek$Ol+i$D^}ug4~xH
ze%t%&T}?(1nzkhZ&l;kH>{k9sVI7l3QTZ3#Kw&zDO{^~_!;n~8D~hvIp9lX4FNj<9y;dRd&
za=cqs5lqeGT?aD%b}a59zzch!VXDTMJRGmw9pb5qb?AE#TMrU)BC+MtnH^TVz~%eJ
zpNYbSIqWhM{9Q${CY|Bc+tfK)_%Ax~D)36+sjDG)o!^mmAIaqy;2*b5UaH@HRwccy
zVEP)vi)KDv1`Q!0+n1H3>3XMz?Gp4HIF250XG3L_GhnGekCBkZO~YSnBV)c3@gjY8
zt^u-FAq8b2XhoTy64W3-HD=ChWufj1FBr_op}5t#oEX?Bfn;5w{^48A`Qqu`P0;@x
z?ujcx_I@?w1P)?`2ACS`u?J0;D0z3@F0cQitKcnirv(1t26jX;HfmWo77oUEiF9TOQjM0uKtt
zvlDWKDlOtS>0z+eGbFR^7l4fv_I2CEBvEkEe|ciCGvpJNk#q~keqWoqwlWqg9Nz;(
z=K-yLvc&K&8dB@JZrpL^2^EhRfc#@=?U_BC$`h7Zd?1-#KJXzz|a|Fq#%~fvqND=@}JYynK=9J+zCg5KbiqBQK?F*fJ2W)1k
zx5djok0o>%7`>lL09#^){DF0|scR)Vu_og4SN^o&qf
zf%a(0pzRtP*j%FID?T>13^hb!h4&h~fZS%l6$L2z^7e&2Hl%>@?QpnWkl|D$=s64n
zEm1Ro13gs8MV74v&Am)WHk(|)rfll9YEX$}&=5JHeUG1VC4H`;{DJ?z3^NHD{2B%M
z|4d@ALIF;5FW(*~8E_V%klP2Tb(XPtX2?Ys?T%Z$89lfHShy6gVGqgoCbo4}?Vm4=eeGdi%iZbm+?&)p;axUyjCGzO>073cF$w6=!
P{0~!AdZ1V;ZyNAF7+vVe

literal 0
HcmV?d00001


From 7e61a06199ee24ed0407b9325a60d916a2994339 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Tue, 10 Dec 2024 21:38:37 +0800
Subject: [PATCH 118/120] fix queryJobDetail  (#117)

* try to fix macos ci

* fix queryJobDetail
---
 .github/workflows/cpp_full_node_workflow.yml  |  1 +
 .../wedpr_ml_toolkit/jupyter-demo/test_ml.py  | 74 +++++++++----------
 .../transport/wedpr_remote_job_client.py      | 23 ++++--
 3 files changed, 55 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 9800dd95..8acd0d1f 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -91,6 +91,7 @@ jobs:
           export DYLD_LIBRARY_PATH=/usr/local/opt/libiconv/lib:${DYLD_LIBRARY_PATH}
           export LDFLAGS="-L/usr/local/opt/libiconv/lib"
           export CPPFLAGS="-I/usr/local/opt/libiconv/include"
+          export PATH="/usr/bin:${PATH}"
           mkdir -p cpp/build && cd cpp/build 
           cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
diff --git a/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py b/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py
index 6b4b81d6..e60dc733 100644
--- a/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py
+++ b/python/wedpr_ml_toolkit/jupyter-demo/test_ml.py
@@ -4,6 +4,8 @@
 # In[1]:
 
 
+import matplotlib.pyplot as plt
+from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score
 import sys
 import numpy as np
 import pandas as pd
@@ -22,7 +24,8 @@
 
 
 # 读取配置文件
-wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file('config.properties')
+wedpr_config = WeDPRMlConfigBuilder.build_from_properties_file(
+    'config.properties')
 wedpr_ml_toolkit = WeDPRMlToolkit(wedpr_config)
 
 
@@ -32,7 +35,7 @@
 # dataset1
 dataset1 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),
                           dataset_client=wedpr_ml_toolkit.dataset_client,
-                          dataset_id = 'd-9743660607744005',
+                          dataset_id='d-9743660607744005',
                           is_label_holder=True)
 print(f"* load dataset1: {dataset1}")
 (values, cols, shapes) = dataset1.load_values()
@@ -45,8 +48,8 @@
 
 # dataset2
 dataset2 = DatasetContext(storage_entrypoint=wedpr_ml_toolkit.get_storage_entry_point(),
-                          dataset_client = wedpr_ml_toolkit.dataset_client,
-                          dataset_id = "d-9743674298214405")
+                          dataset_client=wedpr_ml_toolkit.dataset_client,
+                          dataset_id="d-9743674298214405")
 print(f"* dataset2: {dataset2}")
 
 # 构建 dataset context
@@ -60,11 +63,10 @@
 model_setting = ModelSetting()
 model_setting.use_psi = True
 xgb_job_context = wedpr_ml_toolkit.build_job_context(
-    job_type = JobType.XGB_TRAINING, 
-    project_id = project_id, 
-    dataset = dataset, 
-    model_setting = model_setting, 
-    id_fields = "id")
+    job_type=JobType.XGB_TRAINING,
+    project_id=project_id,
+    dataset=dataset,
+    model_setting=model_setting)
 print(f"* build xgb job context: {xgb_job_context}")
 
 
@@ -81,7 +83,7 @@
 
 # 获取xgb任务结果
 print(xgb_job_id)
-#xgb_job_id = "9868279583877126"
+# xgb_job_id = "9868279583877126"
 xgb_result_detail = xgb_job_context.fetch_job_result(xgb_job_id, True)
 # load the result context
 xgb_result_context = wedpr_ml_toolkit.build_result_context(job_context=xgb_job_context,
@@ -89,10 +91,12 @@
 print(f"* xgb job result ctx: {xgb_result_context}")
 
 xgb_test_dataset = xgb_result_context.test_result_dataset
-print(f"* xgb_test_dataset: {xgb_test_dataset}, file_path: {xgb_test_dataset.dataset_meta.file_path}")
+print(
+    f"* xgb_test_dataset: {xgb_test_dataset}, file_path: {xgb_test_dataset.dataset_meta.file_path}")
 
 (data, cols, shapes) = xgb_test_dataset.load_values()
-print(f"* test dataset detail, columns: {cols}, shape: {shapes}, value: {data}")
+print(
+    f"* test dataset detail, columns: {cols}, shape: {shapes}, value: {data}")
 
 
 # In[8]:
@@ -102,17 +106,19 @@
 result_context: TrainResultContext = xgb_result_context
 evaluation_result_dataset = result_context.evaluation_dataset
 (eval_data, cols, shape) = evaluation_result_dataset.load_values(header=0)
-print(f"* evaluation detail, col: {cols}, shape: {shape}, eval_data: {eval_data}")
+print(
+    f"* evaluation detail, col: {cols}, shape: {shape}, eval_data: {eval_data}")
 
 
 # In[9]:
 
 
-# feature importance 
+# feature importance
 feature_importance_dataset = result_context.feature_importance_dataset
 (feature_importance_data, cols, shape) = feature_importance_dataset.load_values()
 
-print(f"* feature_importance detail, col: {cols}, shape: {shape}, feature_importance_data: {feature_importance_data}")
+print(
+    f"* feature_importance detail, col: {cols}, shape: {shape}, feature_importance_data: {feature_importance_data}")
 
 
 # In[10]:
@@ -122,7 +128,8 @@
 preprocessing_dataset = result_context.preprocessing_dataset
 (preprocessing_data, cols, shape) = preprocessing_dataset.load_values()
 
-print(f"* preprocessing detail, col: {cols}, shape: {shape}, preprocessing_data: {preprocessing_data}")
+print(
+    f"* preprocessing detail, col: {cols}, shape: {shape}, preprocessing_data: {preprocessing_data}")
 
 
 # In[11]:
@@ -132,15 +139,14 @@
 model_result_dataset = result_context.model_result_dataset
 (model_result, cols, shape) = model_result_dataset.load_values()
 
-print(f"* model_result detail, col: {cols}, shape: {shape}, model_result: {model_result}")
+print(
+    f"* model_result detail, col: {cols}, shape: {shape}, model_result: {model_result}")
 
 
 # In[12]:
 
 
 # 明文处理预测结果
-from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score
-import matplotlib.pyplot as plt
 
 # 提取真实标签和预测概率
 y_true = data['class_label']
@@ -191,15 +197,14 @@
 # 构造xgb预测任务配置
 predict_setting = ModelSetting()
 predict_setting.use_psi = True
-#model_predict_algorithm = {}
-#model_predict_algorithm.update({"setting": xgb_result_context.job_result_detail.model})
+# model_predict_algorithm = {}
+# model_predict_algorithm.update({"setting": xgb_result_context.job_result_detail.model})
 predict_xgb_job_context = wedpr_ml_toolkit.build_job_context(
-    job_type=JobType.XGB_PREDICTING, 
-    project_id = project_id, 
-    dataset= dataset, 
-    model_setting= predict_setting, 
-    id_fields = "id", 
-    predict_algorithm = xgb_result_context.job_result_detail.model_predict_algorithm)
+    job_type=JobType.XGB_PREDICTING,
+    project_id=project_id,
+    dataset=dataset,
+    model_setting=predict_setting,
+    predict_algorithm=xgb_result_context.job_result_detail.model_predict_algorithm)
 print(f"* predict_xgb_job_context: {predict_xgb_job_context}")
 
 
@@ -218,13 +223,14 @@
 # query the job detail
 print(f"* xgb_predict_job_id: {xgb_predict_job_id}")
 
-predict_xgb_job_result = predict_xgb_job_context.fetch_job_result(xgb_predict_job_id, True)
+predict_xgb_job_result = predict_xgb_job_context.fetch_job_result(
+    xgb_predict_job_id, True)
 
 # generate the result context
-result_context = wedpr_ml_toolkit.build_result_context(job_context=predict_xgb_job_context, 
+result_context = wedpr_ml_toolkit.build_result_context(job_context=predict_xgb_job_context,
                                                        job_result_detail=predict_xgb_job_result)
 
-xgb_predict_result_context : PredictResultContext = result_context 
+xgb_predict_result_context: PredictResultContext = result_context
 print(f"* result_context is {xgb_predict_result_context}")
 
 
@@ -232,11 +238,9 @@
 
 
 # 明文处理预测结果
-from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, accuracy_score, f1_score, precision_score, recall_score
-import matplotlib.pyplot as plt
 
 
-(data, cols, shapes) = xgb_predict_result_context.model_result_dataset.load_values(header = 0)
+(data, cols, shapes) = xgb_predict_result_context.model_result_dataset.load_values(header=0)
 
 # 提取真实标签和预测概率
 y_true = data['class_label']
@@ -282,7 +286,3 @@
 
 
 # In[ ]:
-
-
-
-
diff --git a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
index 2d5545a4..32e59440 100644
--- a/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
+++ b/python/wedpr_ml_toolkit/wedpr_ml_toolkit/transport/wedpr_remote_job_client.py
@@ -155,6 +155,17 @@ def __repr__(self):
                f"resultFileInfo: {self.resultFileInfo}, model: {self.model}"
 
 
+class JobDetailRequest(BaseObject):
+    def __init__(self, job_id=None,
+                 fetch_job_detail=True,
+                 fetch_job_result=True,
+                 fetch_log=False):
+        self.jobID = job_id
+        self.fetchJobDetail = fetch_job_detail
+        self.fetchJobResult = fetch_job_result
+        self.fetchLog = fetch_log
+
+
 class WeDPRRemoteJobClient(WeDPREntryPoint, BaseObject):
     def __init__(self, http_config: HttpConfig, auth_config: AuthConfig, job_config: JobConfig):
         if auth_config is None:
@@ -189,19 +200,19 @@ def query_job_detail(self, job_id, block_until_finish) -> JobDetailResponse:
                 or (not job_result.job_status.run_success()):
             return JobDetailResponse(job_info=job_result, params=None)
         # success case, query the job detail
-        params = {}
-        params["jobID"] = job_id
+        job_detail_requests = JobDetailRequest(job_id)
         response_dict = self.execute_with_retry(self.send_request,
                                                 self.job_config.max_retries,
                                                 self.job_config.retry_delay_s,
-                                                False,
+                                                True,
                                                 self.job_config.query_job_detail_uri,
-                                                params,
-                                                None, None)
+                                                None,
+                                                None, json.dumps(job_detail_requests.as_dict()))
         wedpr_response = WeDPRResponse(**response_dict)
         if not wedpr_response.success():
             raise Exception(
-                f"query_job_detail exception, job: {job_id}, code: {wedpr_response.code}, msg: {wedpr_response.msg}")
+                f"query_job_detail exception, job: {job_id}, "
+                f"code: {wedpr_response.code}, msg: {wedpr_response.msg}")
         return JobDetailResponse(**(wedpr_response.data))
 
     def poll_job_result(self, job_id, block_until_finish) -> JobInfo:

From 3e653164bd38806433b8b362a3491f0658d6c550 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 11 Dec 2024 14:06:31 +0800
Subject: [PATCH 119/120] update wedpr-mpc-service base image (#118)

* try to fix ci

* update wedpr-mpc-service base image
---
 .github/workflows/cpp_full_node_workflow.yml | 12 ++++--------
 docker-files/cpp/Dockerfile                  |  2 +-
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cpp_full_node_workflow.yml b/.github/workflows/cpp_full_node_workflow.yml
index 8acd0d1f..a2c6386c 100644
--- a/.github/workflows/cpp_full_node_workflow.yml
+++ b/.github/workflows/cpp_full_node_workflow.yml
@@ -72,21 +72,17 @@ jobs:
           echo "Disk usage before delete files: "`df -lh`
           du -sch ${{ env.VCPKG_ROOT }}/buildtrees/*
           du -sch ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src/*
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/src
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/grpc/*-dbg
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/krb5/src
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/gsasl/src
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/protobuf/src
-          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/wedprcrypto/src
-          rm -rf ${{ env.VCPKG_ROOT }}buildtrees/openssl/src
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/*/src
           rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/*/*-dbg
+          rm -rf ${{ env.VCPKG_ROOT }}/buildtrees/*/*-rel
           echo "Disk usage after delete files: "`df -lh`
           make -j3
       - name: Build for macOS
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
-          brew install libiconv m4 automake autoconf libtool
+          brew install libiconv  
+          brew reinstall m4 autoconf libtool automake
           echo "## m4:"`which m4`
           export DYLD_LIBRARY_PATH=/usr/local/opt/libiconv/lib:${DYLD_LIBRARY_PATH}
           export LDFLAGS="-L/usr/local/opt/libiconv/lib"
diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
index 192445cc..03f5ca68 100644
--- a/docker-files/cpp/Dockerfile
+++ b/docker-files/cpp/Dockerfile
@@ -63,7 +63,7 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /data/home/wedpr
 
 ENTRYPOINT ["/data/home/wedpr/wedpr-pro-node-service/ppc-pro-node", "-c", "config.ini"]
 
-FROM ubuntu:22.04 as wedpr-mpc-service
+FROM fiscoorg:wedpr-mpc-base-ubuntu:v3.0.0 as wedpr-mpc-service
 LABEL maintainer=service@webank.com
 
 RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \

From 26ae0b0970907ea43c8f5544bac80d7ddd7758b9 Mon Sep 17 00:00:00 2001
From: cyjseagull 
Date: Wed, 11 Dec 2024 14:20:07 +0800
Subject: [PATCH 120/120] fix docker file

---
 docker-files/cpp/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-files/cpp/Dockerfile b/docker-files/cpp/Dockerfile
index 03f5ca68..6a6144c2 100644
--- a/docker-files/cpp/Dockerfile
+++ b/docker-files/cpp/Dockerfile
@@ -63,7 +63,7 @@ COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /data/home/wedpr
 
 ENTRYPOINT ["/data/home/wedpr/wedpr-pro-node-service/ppc-pro-node", "-c", "config.ini"]
 
-FROM fiscoorg:wedpr-mpc-base-ubuntu:v3.0.0 as wedpr-mpc-service
+FROM fiscoorg/wedpr-mpc-base-ubuntu:v3.0.0 as wedpr-mpc-service
 LABEL maintainer=service@webank.com
 
 RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user \